diff --git a/adnl/CMakeLists.txt b/adnl/CMakeLists.txt index 909f259a9e..c5536c8eda 100644 --- a/adnl/CMakeLists.txt +++ b/adnl/CMakeLists.txt @@ -81,7 +81,7 @@ if (NOT TON_ONLY_TONLIB) add_library(adnl STATIC ${ADNL_SOURCE}) target_include_directories(adnl PUBLIC $) -target_link_libraries(adnl PUBLIC tdactor ton_crypto tl_api tdnet tddb keys keyring ) +target_link_libraries(adnl PUBLIC tdactor ton_crypto tl_api tdnet tddb keys keyring metrics) add_executable(adnl-proxy ${ADNL_PROXY_SOURCE}) target_include_directories(adnl-proxy PUBLIC $) diff --git a/adnl/adnl-network-manager.cpp b/adnl/adnl-network-manager.cpp index 2d5b8883ad..ff12f78745 100644 --- a/adnl/adnl-network-manager.cpp +++ b/adnl/adnl-network-manager.cpp @@ -30,6 +30,26 @@ td::actor::ActorOwn AdnlNetworkManager::create(td::uint16 po return td::actor::create_actor("NetworkManager", port); } +void AdnlNetworkManagerImpl::start_up() { + alarm_timestamp() = td::Timestamp::in(60.0); + add_collector(metrics_collector_.get()); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, udp_ingress_bytes_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, udp_ingress_packets_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, udp_ingress_drops_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, + udp_ingress_proxy_control_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, + udp_proxy_ingress_bytes_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, udp_egress_bytes_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, udp_egress_packets_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, udp_egress_drops_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, + udp_proxy_egress_bytes_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, + udp_proxy_egress_packets_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, listening_sockets_); +} + AdnlNetworkManagerImpl::OutDesc *AdnlNetworkManagerImpl::choose_out_iface(td::uint8 cat, td::uint32 priority) { auto it = out_desc_.upper_bound(priority); while (true) { @@ -108,19 +128,25 @@ void AdnlNetworkManagerImpl::add_proxy_addr(td::IPAddress addr, td::uint16 local } void AdnlNetworkManagerImpl::receive_udp_message(td::UdpMessage message, size_t idx) { + udp_ingress_packets_->add(1); + udp_ingress_bytes_->add(message.data.size()); if (!callback_) { + udp_ingress_drop_no_callback_->add(1); LOG(ERROR) << this << ": dropping IN message [?->?]: peer table unitialized"; return; } if (message.error.is_error()) { + udp_ingress_drop_error_->add(1); VLOG(ADNL_WARNING) << this << ": dropping ERROR message: " << message.error; return; } if (message.data.size() < 32) { + udp_ingress_drop_too_short_->add(1); VLOG(ADNL_WARNING) << this << ": received too small proxy packet of size " << message.data.size(); return; } if (message.data.size() >= get_mtu() + 128) { + udp_ingress_drop_too_huge_->add(1); VLOG(ADNL_NOTICE) << this << ": received huge packet of size " << message.data.size(); } CHECK(idx < udp_sockets_.size()); @@ -138,10 +164,12 @@ void AdnlNetworkManagerImpl::receive_udp_message(td::UdpMessage message, size_t CHECK(proxy_iface.is_proxy()); auto R = in_desc_[it->second].proxy->decrypt(std::move(message.data)); if (R.is_error()) { + udp_ingress_drop_bad_proxy_decrypt_->add(1); VLOG(ADNL_WARNING) << this << ": failed to decrypt proxy mesage: " << R.move_as_error(); return; } auto packet = R.move_as_ok(); + udp_proxy_ingress_bytes_->add(packet.data.size()); if (packet.flags & 1) { message.address.init_host_port(td::IPAddress::ipv4_to_str(packet.ip), packet.port).ensure(); } else { @@ -149,27 +177,33 @@ void AdnlNetworkManagerImpl::receive_udp_message(td::UdpMessage message, size_t } if ((packet.flags & 6) == 6) { if (packet.seqno <= 0 || packet.adnl_start_time < 0) { + udp_ingress_drop_bad_proxy_seqno_->add(1); VLOG(ADNL_WARNING) << this << ": dropping proxy packet: invalid start_time/seqno"; return; } if (proxy_iface.received.packet_is_delivered(packet.adnl_start_time, packet.seqno)) { + udp_ingress_drop_bad_proxy_seqno_->add(1); VLOG(ADNL_WARNING) << this << ": dropping duplicate proxy packet"; return; } } if (packet.flags & 8) { if (packet.date < td::Clocks::system() - 60 || packet.date > td::Clocks::system() + 60) { + udp_ingress_drop_bad_proxy_time_->add(1); VLOG(ADNL_WARNING) << this << ": dropping proxy packet: bad time " << packet.date; return; } } if (!(packet.flags & (1 << 16))) { + udp_ingress_drop_bad_proxy_outflag_->add(1); VLOG(ADNL_WARNING) << this << ": dropping proxy packet: packet has outbound flag"; return; } if (packet.flags & (1 << 17)) { + udp_ingress_proxy_control_->add(1); auto F = fetch_tl_object(std::move(packet.data), true); if (F.is_error()) { + udp_ingress_drop_bad_control_->add(1); VLOG(ADNL_WARNING) << this << ": dropping proxy packet: bad control packet"; return; } @@ -205,6 +239,7 @@ void AdnlNetworkManagerImpl::receive_udp_message(td::UdpMessage message, size_t } if (!from_proxy) { if (socket.in_desc == std::numeric_limits::max()) { + udp_ingress_drop_no_in_desc_->add(1); VLOG(ADNL_WARNING) << this << ": received bad packet to proxy-only listenung port"; return; } @@ -226,12 +261,14 @@ void AdnlNetworkManagerImpl::send_udp_packet(AdnlNodeIdShort src_id, AdnlNodeIdS td::uint32 priority, td::BufferSlice data) { auto it = adnl_id_2_cat_.find(src_id); if (it == adnl_id_2_cat_.end()) { + udp_egress_drop_unknown_src_->add(1); VLOG(ADNL_WARNING) << this << ": dropping OUT message [" << src_id << "->" << dst_id << "]: unknown src"; return; } auto out = choose_out_iface(it->second, priority); if (!out) { + udp_egress_drop_no_route_->add(1); VLOG(ADNL_WARNING) << this << ": dropping OUT message [" << src_id << "->" << dst_id << "]: no out rules"; return; } @@ -246,12 +283,15 @@ void AdnlNetworkManagerImpl::send_udp_packet(AdnlNodeIdShort src_id, AdnlNodeIdS CHECK(M.data.size() <= get_mtu()); + udp_egress_packets_->add(1); + udp_egress_bytes_->add(M.data.size()); td::actor::send_closure(socket.server, &td::UdpServer::send, std::move(M)); } else { AdnlProxy::Packet p; p.flags = 7; p.ip = dst_addr.get_ipv4(); p.port = static_cast(dst_addr.get_port()); + auto inner_size = data.size(); p.data = std::move(data); p.adnl_start_time = Adnl::adnl_start_time(); p.seqno = ++v.out_seqno; @@ -262,6 +302,10 @@ void AdnlNetworkManagerImpl::send_udp_packet(AdnlNodeIdShort src_id, AdnlNodeIdS M.address = v.proxy_addr; M.data = std::move(enc); + udp_egress_packets_->add(1); + udp_egress_bytes_->add(M.data.size()); + udp_proxy_egress_packets_->add(1); + udp_proxy_egress_bytes_->add(inner_size); td::actor::send_closure(socket.server, &td::UdpServer::send, std::move(M)); } } @@ -282,6 +326,8 @@ void AdnlNetworkManagerImpl::proxy_register(OutDesc &desc) { M.address = desc.proxy_addr; M.data = std::move(enc); + udp_egress_packets_->add(1); + udp_egress_bytes_->add(M.data.size()); auto &socket = udp_sockets_[desc.socket_idx]; td::actor::send_closure(socket.server, &td::UdpServer::send, std::move(M)); } diff --git a/adnl/adnl-network-manager.h b/adnl/adnl-network-manager.h index 642ce1081e..fd2a7497e8 100644 --- a/adnl/adnl-network-manager.h +++ b/adnl/adnl-network-manager.h @@ -27,6 +27,8 @@ #include "adnl-node-id.hpp" #include "adnl-proxy-types.h" +#include "metrics/metrics-collectors.h" + namespace td { class UdpServer; } @@ -54,7 +56,7 @@ class AdnlNetworkConnection : public td::actor::Actor { virtual ~AdnlNetworkConnection() = default; }; -class AdnlNetworkManager : public td::actor::Actor { +class AdnlNetworkManager : public td::actor::Actor, public virtual metrics::CollectorWrapper { public: //using ConnHandle = td::uint64; class Callback { diff --git a/adnl/adnl-network-manager.hpp b/adnl/adnl-network-manager.hpp index f457da65c3..30e2d4cdef 100644 --- a/adnl/adnl-network-manager.hpp +++ b/adnl/adnl-network-manager.hpp @@ -19,7 +19,9 @@ #pragma once #include +#include +#include "metrics/metrics-collectors.h" #include "td/actor/PromiseFuture.h" #include "td/actor/actor.h" #include "td/net/TcpListener.h" @@ -106,9 +108,7 @@ class AdnlNetworkManagerImpl : public AdnlNetworkManager { } void alarm() override; - void start_up() override { - alarm_timestamp() = td::Timestamp::in(60.0); - } + void start_up() override; void add_in_addr(InDesc desc, size_t socket_idx) { for (size_t idx = 0; idx < in_desc_.size(); idx++) { @@ -155,6 +155,51 @@ class AdnlNetworkManagerImpl : public AdnlNetworkManager { td::uint64 received_messages_ = 0; td::uint64 sent_messages_ = 0; + using CounterPtr = std::shared_ptr>; + metrics::MultiCollector::Own metrics_collector_ = metrics::MultiCollector::create("adnl_net"); + metrics::Labeled>::Ptr udp_ingress_drops_ = + metrics::Labeled>::make( + "reason", "udp_ingress_drops_total", "UDP ingress packets dropped, by reason."); + metrics::Labeled>::Ptr udp_egress_drops_ = + metrics::Labeled>::make( + "reason", "udp_egress_drops_total", "ADNL outbound packets dropped, by reason."); + metrics::AtomicCounter::Ptr udp_ingress_bytes_ = metrics::AtomicCounter::make( + "udp_ingress_bytes_total", "Total UDP bytes received on ADNL sockets (wire-level)."); + metrics::AtomicCounter::Ptr udp_ingress_packets_ = metrics::AtomicCounter::make( + "udp_ingress_packets_total", "Total UDP packets received on ADNL sockets."); + metrics::AtomicCounter::Ptr udp_ingress_proxy_control_ = + metrics::AtomicCounter::make("udp_ingress_proxy_control_total", "ADNL proxy control packets received."); + metrics::AtomicCounter::Ptr udp_proxy_ingress_bytes_ = metrics::AtomicCounter::make( + "udp_proxy_ingress_bytes_total", "Total payload bytes received via ADNL proxy (after decrypt)."); + metrics::AtomicCounter::Ptr udp_egress_bytes_ = metrics::AtomicCounter::make( + "udp_egress_bytes_total", "Total UDP bytes sent on ADNL sockets (wire-level)."); + metrics::AtomicCounter::Ptr udp_egress_packets_ = metrics::AtomicCounter::make( + "udp_egress_packets_total", "Total UDP packets sent on ADNL sockets."); + metrics::AtomicCounter::Ptr udp_proxy_egress_bytes_ = metrics::AtomicCounter::make( + "udp_proxy_egress_bytes_total", "Total inner payload bytes wrapped through ADNL proxy."); + metrics::AtomicCounter::Ptr udp_proxy_egress_packets_ = metrics::AtomicCounter::make( + "udp_proxy_egress_packets_total", "Total packets wrapped through ADNL proxy."); + metrics::LambdaGauge::Ptr listening_sockets_ = metrics::LambdaGauge::make( + "listening_sockets", + [this] { + return std::vector{ + metrics::Sample{.label_set = {}, .value = static_cast(udp_sockets_.size())}}; + }, + "Number of UDP sockets owned by the ADNL network manager."); + + CounterPtr udp_ingress_drop_no_callback_ = udp_ingress_drops_->label("no_callback"); + CounterPtr udp_ingress_drop_error_ = udp_ingress_drops_->label("error"); + CounterPtr udp_ingress_drop_too_short_ = udp_ingress_drops_->label("too_short"); + CounterPtr udp_ingress_drop_too_huge_ = udp_ingress_drops_->label("too_huge"); + CounterPtr udp_ingress_drop_bad_proxy_decrypt_ = udp_ingress_drops_->label("bad_proxy_decrypt"); + CounterPtr udp_ingress_drop_bad_proxy_seqno_ = udp_ingress_drops_->label("bad_proxy_seqno"); + CounterPtr udp_ingress_drop_bad_proxy_time_ = udp_ingress_drops_->label("bad_proxy_time"); + CounterPtr udp_ingress_drop_bad_proxy_outflag_ = udp_ingress_drops_->label("bad_proxy_outflag"); + CounterPtr udp_ingress_drop_bad_control_ = udp_ingress_drops_->label("bad_control"); + CounterPtr udp_ingress_drop_no_in_desc_ = udp_ingress_drops_->label("no_in_desc"); + CounterPtr udp_egress_drop_unknown_src_ = udp_egress_drops_->label("unknown_src"); + CounterPtr udp_egress_drop_no_route_ = udp_egress_drops_->label("no_route"); + std::vector udp_sockets_; std::map port_2_socket_; diff --git a/adnl/adnl-peer-table.cpp b/adnl/adnl-peer-table.cpp index 54f3b533a4..89c143e269 100644 --- a/adnl/adnl-peer-table.cpp +++ b/adnl/adnl-peer-table.cpp @@ -16,7 +16,6 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "absl/functional/internal/any_invocable.h" #include "td/db/RocksDb.h" #include "td/utils/Random.h" #include "td/utils/crypto.h" @@ -50,7 +49,9 @@ td::actor::ActorOwn Adnl::create(std::string db, td::actor::ActorIdadd(1); if (data.size() < 32) { + inbound_drop_too_short_->add(1); VLOG(ADNL_WARNING) << this << ": dropping IN message [?->?]: message too short: len=" << data.size(); return; } @@ -61,6 +62,7 @@ void AdnlPeerTableImpl::receive_packet(td::IPAddress addr, AdnlCategoryMask cat_ auto it = local_ids_.find(dst); if (it != local_ids_.end()) { if (!cat_mask.test(it->second.cat)) { + inbound_drop_cat_mismatch_->add(1); VLOG(ADNL_WARNING) << this << ": dropping IN message [?->" << dst << "]: category mismatch"; return; } @@ -72,6 +74,7 @@ void AdnlPeerTableImpl::receive_packet(td::IPAddress addr, AdnlCategoryMask cat_ auto it2 = channels_.find(dst_chan_id); if (it2 != channels_.end()) { if (!cat_mask.test(it2->second.second)) { + inbound_drop_cat_mismatch_->add(1); VLOG(ADNL_WARNING) << this << ": dropping IN message to channel [?->" << dst << "]: category mismatch"; return; } @@ -79,6 +82,7 @@ void AdnlPeerTableImpl::receive_packet(td::IPAddress addr, AdnlCategoryMask cat_ return; } + inbound_drop_unknown_dst_->add(1); VLOG(ADNL_DEBUG) << this << ": dropping IN message [?->" << dst << "]: unknown dst " << dst << " (len=" << (data.size() + 32) << ")"; } @@ -121,6 +125,8 @@ AdnlPeerTableImpl::PeerPair *AdnlPeerTableImpl::get_peer_pair_if_exists(AdnlNode } void AdnlPeerTableImpl::receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket packet, td::uint64 serialized_size) { + decrypt_packets_->add(1); + decrypt_bytes_->add(serialized_size); packet.run_basic_checks().ensure(); if (!packet.inited_from_short()) { @@ -187,6 +193,7 @@ void AdnlPeerTableImpl::send_message_in(AdnlNodeIdShort src, AdnlNodeIdShort dst td::uint32 flags) { auto it2 = local_ids_.find(src); if (it2 == local_ids_.end()) { + app_send_drop_unknown_src_->add(1); LOG(ERROR) << this << ": dropping OUT message [" << src << "->" << dst << "]: unknown src"; return; } @@ -201,16 +208,19 @@ void AdnlPeerTableImpl::send_message_in(AdnlNodeIdShort src, AdnlNodeIdShort dst void AdnlPeerTableImpl::answer_query(AdnlNodeIdShort src, AdnlNodeIdShort dst, AdnlQueryId query_id, td::BufferSlice data) { if (data.size() > get_mtu()) { + app_send_drop_too_big_->add(1); LOG(ERROR) << this << ": dropping OUT message [" << src << "->" << dst << "]: message too big: size=" << data.size(); return; } + app_metrics_->record_send("answer", data.as_slice()); send_message_in(src, dst, adnlmessage::AdnlMessageAnswer{query_id, std::move(data)}, 0); } void AdnlPeerTableImpl::send_query(AdnlNodeIdShort src, AdnlNodeIdShort dst, std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice data) { if (data.size() > huge_packet_max_size()) { + app_send_drop_too_big_->add(1); VLOG(ADNL_WARNING) << "dropping too big packet [" << src << "->" << dst << "]: size=" << data.size(); VLOG(ADNL_WARNING) << "DUMP: " << td::buffer_to_hex(data.as_slice().truncate(128)); return; @@ -218,9 +228,11 @@ void AdnlPeerTableImpl::send_query(AdnlNodeIdShort src, AdnlNodeIdShort dst, std auto it2 = local_ids_.find(src); if (it2 == local_ids_.end()) { + app_send_drop_unknown_src_->add(1); LOG(ERROR) << this << ": dropping OUT message [" << src << "->" << dst << "]: unknown src"; return; } + app_metrics_->record_send("query", data.as_slice()); auto &peer_info = peers_[dst]; td::actor::send_closure(get_peer_pair(dst, peer_info, src, it2->second), &AdnlPeerPair::send_query, name, @@ -354,6 +366,18 @@ void AdnlPeerTableImpl::unregister_channel(AdnlChannelIdShort id) { } void AdnlPeerTableImpl::start_up() { + add_collector(metrics_collector_.get()); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, app_metrics_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, app_send_dropped_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, inbound_packets_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, inbound_dropped_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, decrypt_packets_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, decrypt_bytes_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, local_ids_gauge_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, peers_gauge_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, peer_pairs_gauge_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, channels_gauge_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, static_nodes_gauge_); } void AdnlPeerTableImpl::write_new_addr_list_to_db(AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id, AdnlDbItem node, @@ -384,6 +408,7 @@ AdnlPeerTableImpl::AdnlPeerTableImpl(std::string db_root, td::actor::ActorIdrecord_deliver("message", data.as_slice()); td::actor::send_closure(it->second.local_id, &AdnlLocalId::deliver, src, std::move(data)); } } @@ -391,6 +416,7 @@ void AdnlPeerTableImpl::deliver_query(AdnlNodeIdShort src, AdnlNodeIdShort dst, td::Promise promise) { auto it = local_ids_.find(dst); if (it != local_ids_.end()) { + app_metrics_->record_deliver("query", data.as_slice()); td::actor::send_closure(it->second.local_id, &AdnlLocalId::deliver_query, src, std::move(data), std::move(promise)); } else { LOG(WARNING) << "deliver query: unknown dst " << dst; diff --git a/adnl/adnl-peer-table.hpp b/adnl/adnl-peer-table.hpp index dbe3d7dc84..a04c0bdc32 100644 --- a/adnl/adnl-peer-table.hpp +++ b/adnl/adnl-peer-table.hpp @@ -19,9 +19,13 @@ #pragma once #include +#include #include +#include #include "keys/encryptor.h" +#include "metrics/app-traffic-metrics.h" +#include "metrics/metrics-collectors.h" #include "adnl-peer-table.h" #include "adnl-peer.h" @@ -51,10 +55,12 @@ class AdnlPeerTableImpl : public AdnlPeerTable { } void send_message_ex(AdnlNodeIdShort src, AdnlNodeIdShort dst, td::BufferSlice data, td::uint32 flags) override { if (data.size() > huge_packet_max_size()) { + app_send_drop_too_big_->add(1); VLOG(ADNL_WARNING) << "dropping too big packet [" << src << "->" << dst << "]: size=" << data.size(); VLOG(ADNL_WARNING) << "DUMP: " << td::buffer_to_hex(data.as_slice().truncate(128)); return; } + app_metrics_->record_send("custom", data.as_slice()); send_message_in(src, dst, AdnlMessage{adnlmessage::AdnlMessageCustom{std::move(data)}}, flags); } void answer_query(AdnlNodeIdShort src, AdnlNodeIdShort dst, AdnlQueryId query_id, td::BufferSlice data) override; @@ -156,6 +162,66 @@ class AdnlPeerTableImpl : public AdnlPeerTable { AdnlNodeIdShort proxy_addr_; + using CounterPtr = std::shared_ptr>; + metrics::MultiCollector::Own metrics_collector_ = metrics::MultiCollector::create("adnl"); + metrics::AppTrafficMetrics::Ptr app_metrics_ = metrics::AppTrafficMetrics::make(); + metrics::Labeled>::Ptr app_send_dropped_ = + metrics::Labeled>::make( + "reason", "app_send_dropped_total", "Outbound application messages ADNL dropped before forwarding."); + metrics::AtomicCounter::Ptr inbound_packets_ = metrics::AtomicCounter::make( + "inbound_packets_total", "ADNL packets entering the peer table from the network manager."); + metrics::Labeled>::Ptr inbound_dropped_ = + metrics::Labeled>::make( + "reason", "inbound_dropped_total", "ADNL inbound packets dropped before decryption."); + metrics::AtomicCounter::Ptr decrypt_packets_ = metrics::AtomicCounter::make( + "decrypt_packets_total", "ADNL packets that completed decryption and reached the peer table."); + metrics::AtomicCounter::Ptr decrypt_bytes_ = metrics::AtomicCounter::make( + "decrypt_bytes_total", "Bytes accepted from the network after ADNL packet decryption."); + metrics::LambdaGauge::Ptr local_ids_gauge_ = metrics::LambdaGauge::make( + "local_ids", + [this] { + return std::vector{ + metrics::Sample{.label_set = {}, .value = static_cast(local_ids_.size())}}; + }, + "Number of ADNL local ids currently registered."); + metrics::LambdaGauge::Ptr peers_gauge_ = metrics::LambdaGauge::make( + "peers", + [this] { + return std::vector{metrics::Sample{.label_set = {}, .value = static_cast(peers_.size())}}; + }, + "Number of distinct remote peer ids tracked by ADNL."); + metrics::LambdaGauge::Ptr peer_pairs_gauge_ = metrics::LambdaGauge::make( + "peer_pairs", + [this] { + td::uint64 peer_pair_count = 0; + for (auto &[_, peer_info] : peers_) { + peer_pair_count += peer_info.peers.size(); + } + return std::vector{ + metrics::Sample{.label_set = {}, .value = static_cast(peer_pair_count)}}; + }, + "Number of (local_id, peer_id) ADNL peer pairs."); + metrics::LambdaGauge::Ptr channels_gauge_ = metrics::LambdaGauge::make( + "channels", + [this] { + return std::vector{ + metrics::Sample{.label_set = {}, .value = static_cast(channels_.size())}}; + }, + "Number of registered ADNL channels."); + metrics::LambdaGauge::Ptr static_nodes_gauge_ = metrics::LambdaGauge::make( + "static_nodes", + [this] { + return std::vector{ + metrics::Sample{.label_set = {}, .value = static_cast(static_nodes_.size())}}; + }, + "Number of static ADNL nodes loaded from config."); + + CounterPtr app_send_drop_too_big_ = app_send_dropped_->label("too_big"); + CounterPtr app_send_drop_unknown_src_ = app_send_dropped_->label("unknown_src"); + CounterPtr inbound_drop_too_short_ = inbound_dropped_->label("too_short"); + CounterPtr inbound_drop_cat_mismatch_ = inbound_dropped_->label("cat_mismatch"); + CounterPtr inbound_drop_unknown_dst_ = inbound_dropped_->label("unknown_dst"); + void set_peer_pair_idle(AdnlNodeIdShort l_id, AdnlNodeIdShort p_id, PeerPair &peer_pair, bool value); void gc_peer_pairs(AdnlNodeIdShort local_id, LocalIdInfo &local_id_info); diff --git a/adnl/adnl.h b/adnl/adnl.h index 7582fcc204..fe7ffa5a32 100644 --- a/adnl/adnl.h +++ b/adnl/adnl.h @@ -21,6 +21,7 @@ #include "auto/tl/ton_api.h" #include "common/errorcode.h" #include "keyring/keyring.h" +#include "metrics/metrics-collectors.h" #include "td/actor/actor.h" #include "td/utils/port/IPAddress.h" @@ -62,7 +63,7 @@ class AdnlSenderInterface : public td::actor::Actor { class AdnlTunnel : public td::actor::Actor {}; -class Adnl : public AdnlSenderInterface { +class Adnl : public AdnlSenderInterface, public virtual metrics::CollectorWrapper { public: class Callback { public: diff --git a/metrics/CMakeLists.txt b/metrics/CMakeLists.txt index 452503eda5..d9a975bd87 100644 --- a/metrics/CMakeLists.txt +++ b/metrics/CMakeLists.txt @@ -1,6 +1,6 @@ -add_library(metrics STATIC prometheus-exporter.cpp metrics-types.cpp metrics-collectors.cpp) +add_library(metrics STATIC prometheus-exporter.cpp metrics-types.cpp metrics-collectors.cpp tl-traffic-bucket.cpp app-traffic-metrics.cpp) target_include_directories(metrics PUBLIC $/..) -target_link_libraries(metrics PUBLIC tdnet tonlib tonhttp tdutils tdactor) +target_link_libraries(metrics PUBLIC tdnet tonlib tonhttp tdutils tdactor tl_api) add_executable(metrics-example metrics-example.cpp) target_include_directories(metrics-example PUBLIC $) diff --git a/metrics/app-traffic-metrics.cpp b/metrics/app-traffic-metrics.cpp new file mode 100644 index 0000000000..78782f9136 --- /dev/null +++ b/metrics/app-traffic-metrics.cpp @@ -0,0 +1,94 @@ +#include "app-traffic-metrics.h" + +namespace ton::metrics { + +AppTrafficMetrics::Ptr AppTrafficMetrics::make(std::string send_tl_base, std::string deliver_tl_base) { + return std::shared_ptr(new AppTrafficMetrics(std::move(send_tl_base), std::move(deliver_tl_base))); +} + +AppTrafficMetrics::AppTrafficMetrics(std::string send_tl_base, std::string deliver_tl_base) + : send_bytes_(Labeled>::make( + "kind", "app_send_bytes_total", "Bytes the application asked to send (raw payload, by kind).")) + , send_messages_( + Labeled>::make("kind", "app_send_messages_total", + "Messages the application asked to send (by kind).")) + , deliver_bytes_(Labeled>::make( + "kind", "app_deliver_bytes_total", "Bytes delivered to the application (raw payload, by kind).")) + , deliver_messages_(Labeled>::make( + "kind", "app_deliver_messages_total", "Messages delivered to the application (by kind).")) + , send_by_tl_(Labeled::make( + "kind", std::move(send_tl_base), "Bytes sent by inner TL constructor.", "Messages sent by inner TL constructor.")) + , deliver_by_tl_( + Labeled::make("kind", std::move(deliver_tl_base), + "Bytes delivered by inner TL constructor.", + "Messages delivered by inner TL constructor.")) { +} + +AppTrafficMetrics::CounterPtr AppTrafficMetrics::send_bytes(std::string kind) { + return send_bytes_->label(std::move(kind)); +} + +AppTrafficMetrics::CounterPtr AppTrafficMetrics::send_messages(std::string kind) { + return send_messages_->label(std::move(kind)); +} + +AppTrafficMetrics::CounterPtr AppTrafficMetrics::deliver_bytes(std::string kind) { + return deliver_bytes_->label(std::move(kind)); +} + +AppTrafficMetrics::CounterPtr AppTrafficMetrics::deliver_messages(std::string kind) { + return deliver_messages_->label(std::move(kind)); +} + +void AppTrafficMetrics::account_send_tl(std::string kind, td::Slice payload) { + send_by_tl_->label(std::move(kind))->account(payload); +} + +void AppTrafficMetrics::account_send_tl_with_magic(std::string kind, td::int32 magic, td::uint64 size) { + send_by_tl_->label(std::move(kind))->account_with_magic(magic, size); +} + +void AppTrafficMetrics::account_deliver_tl(std::string kind, td::Slice payload) { + deliver_by_tl_->label(std::move(kind))->account(payload); +} + +void AppTrafficMetrics::account_deliver_tl_with_magic(std::string kind, td::int32 magic, td::uint64 size) { + deliver_by_tl_->label(std::move(kind))->account_with_magic(magic, size); +} + +void AppTrafficMetrics::record_send(std::string kind, td::Slice payload) { + send_bytes(kind)->add(payload.size()); + send_messages(kind)->add(1); + account_send_tl(std::move(kind), payload); +} + +void AppTrafficMetrics::record_send_with_magic(std::string kind, td::int32 magic, td::uint64 size) { + send_bytes(kind)->add(size); + send_messages(kind)->add(1); + account_send_tl_with_magic(std::move(kind), magic, size); +} + +void AppTrafficMetrics::record_deliver(std::string kind, td::Slice payload) { + deliver_bytes(kind)->add(payload.size()); + deliver_messages(kind)->add(1); + account_deliver_tl(std::move(kind), payload); +} + +void AppTrafficMetrics::record_deliver_with_magic(std::string kind, td::int32 magic, td::uint64 size) { + deliver_bytes(kind)->add(size); + deliver_messages(kind)->add(1); + account_deliver_tl_with_magic(std::move(kind), magic, size); +} + +MetricSet AppTrafficMetrics::collect() { + MetricSet set = {}; + set = std::move(set).join(send_bytes_->collect()); + set = std::move(set).join(send_messages_->collect()); + set = std::move(set).join(deliver_bytes_->collect()); + set = std::move(set).join(deliver_messages_->collect()); + set = std::move(set).join(send_by_tl_->collect()); + set = std::move(set).join(deliver_by_tl_->collect()); + return set; +} + +} // namespace ton::metrics diff --git a/metrics/app-traffic-metrics.h b/metrics/app-traffic-metrics.h new file mode 100644 index 0000000000..0a770d92b5 --- /dev/null +++ b/metrics/app-traffic-metrics.h @@ -0,0 +1,45 @@ +#pragma once + +#include + +#include "metrics-collectors.h" +#include "tl-traffic-bucket.h" + +namespace ton::metrics { + +class AppTrafficMetrics final : public Collector { + public: + using Ptr = std::shared_ptr; + using CounterPtr = std::shared_ptr>; + + static Ptr make(std::string send_tl_base = "app_send", std::string deliver_tl_base = "app_deliver"); + + CounterPtr send_bytes(std::string kind); + CounterPtr send_messages(std::string kind); + CounterPtr deliver_bytes(std::string kind); + CounterPtr deliver_messages(std::string kind); + + void account_send_tl(std::string kind, td::Slice payload); + void account_send_tl_with_magic(std::string kind, td::int32 magic, td::uint64 size); + void account_deliver_tl(std::string kind, td::Slice payload); + void account_deliver_tl_with_magic(std::string kind, td::int32 magic, td::uint64 size); + + void record_send(std::string kind, td::Slice payload); + void record_send_with_magic(std::string kind, td::int32 magic, td::uint64 size); + void record_deliver(std::string kind, td::Slice payload); + void record_deliver_with_magic(std::string kind, td::int32 magic, td::uint64 size); + + MetricSet collect() override; + + private: + explicit AppTrafficMetrics(std::string send_tl_base, std::string deliver_tl_base); + + Labeled>::Ptr send_bytes_; + Labeled>::Ptr send_messages_; + Labeled>::Ptr deliver_bytes_; + Labeled>::Ptr deliver_messages_; + Labeled::Ptr send_by_tl_; + Labeled::Ptr deliver_by_tl_; +}; + +} // namespace ton::metrics diff --git a/metrics/metrics-collectors.h b/metrics/metrics-collectors.h index 8d7dab3f25..f4de70c627 100644 --- a/metrics/metrics-collectors.h +++ b/metrics/metrics-collectors.h @@ -156,6 +156,16 @@ class Labeled : public Instrument> { std::mutex mutex_; }; +template +class MultiLabeled : public Labeled> { + using Labeled>::Labeled; +}; + +template +class MultiLabeled : public Labeled { + using Labeled::Labeled; +}; + template void CollectorWrapper::add_collector(td::actor::ActorId collector) { CHECK(!collector.empty()); diff --git a/metrics/tl-traffic-bucket.cpp b/metrics/tl-traffic-bucket.cpp new file mode 100644 index 0000000000..ff01df569e --- /dev/null +++ b/metrics/tl-traffic-bucket.cpp @@ -0,0 +1,93 @@ +#include "auto/tl/ton_api.h" +#include "td/utils/as.h" + +#include "tl-traffic-bucket.h" + +namespace ton::metrics { + +TlTrafficBucket::TlTrafficBucket(std::string metric_name_base, std::optional bytes_help, + std::optional messages_help) + : metric_name_base_(std::move(metric_name_base)) + , bytes_help_(std::move(bytes_help)) + , messages_help_(std::move(messages_help)) { +} + +void TlTrafficBucket::account(td::Slice data) { + if (data.size() < sizeof(td::int32)) { + unknown_bytes += data.size(); + unknown_messages++; + return; + } + account_with_magic(td::as(data.data()), data.size()); +} + +static std::optional ton_api_id_name(int magic) { + auto result = ton::ton_api::Object::nameof(magic); + if (result == std::nullopt) { + result = ton::ton_api::Object::nameof(magic); + } + return result; +} + +void TlTrafficBucket::account_with_magic(td::int32 magic, td::uint64 size) { + if (ton_api_id_name(magic) == std::nullopt) { + unknown_bytes += size; + unknown_messages++; + return; + } + bytes[magic] += size; + messages[magic]++; +} + +MetricSet TlTrafficBucket::collect() { + if (metric_name_base_.empty()) { + return {}; + } + MetricSet set; + auto build = [&](std::string name_suffix, std::optional help, const std::map &counts, + td::uint64 unknown_value) { + MetricFamily fam{.name = metric_name_base_ + name_suffix, .type = "counter", .help = std::move(help), .metrics = {}}; + auto push = [&](std::string tl_name, td::uint64 value) { + fam.metrics.push_back(Metric{ + .suffix = "", + .label_set = LabelSet{.labels = {{"tl", std::move(tl_name)}}}, + .samples = {Sample{.label_set = {}, .value = static_cast(value)}}, + }); + }; + for (auto &[magic, value] : counts) { + auto name = ton_api_id_name(magic); + push(name.value(), value); + } + push("", unknown_value); + set.families.push_back(std::move(fam)); + }; + build("_bytes_by_tl_total", bytes_help_, bytes, unknown_bytes); + build("_messages_by_tl_total", messages_help_, messages, unknown_messages); + return set; +} + +void render_tl_bucket(MetricSet &set, const std::string &base, const std::string &bucket_label_value, + const TlTrafficBucket &bucket, std::optional bytes_help, + std::optional messages_help, std::string bucket_label_key) { + auto build = [&](std::string name_suffix, std::optional help, + const std::map &counts, td::uint64 unknown_value) { + MetricFamily fam{.name = base + name_suffix, .type = "counter", .help = std::move(help), .metrics = {}}; + auto push = [&](std::string tl_name, td::uint64 value) { + fam.metrics.push_back(Metric{ + .suffix = "", + .label_set = LabelSet{.labels = {{bucket_label_key, bucket_label_value}, {"tl", std::move(tl_name)}}}, + .samples = {Sample{.label_set = {}, .value = static_cast(value)}}, + }); + }; + for (auto &[magic, value] : counts) { + auto name = ton_api_id_name(magic); + push(name.value(), value); + } + push("", unknown_value); + set.families.push_back(std::move(fam)); + }; + build("_bytes_by_tl_total", std::move(bytes_help), bucket.bytes, bucket.unknown_bytes); + build("_messages_by_tl_total", std::move(messages_help), bucket.messages, bucket.unknown_messages); +} + +} // namespace ton::metrics diff --git a/metrics/tl-traffic-bucket.h b/metrics/tl-traffic-bucket.h new file mode 100644 index 0000000000..12cdc61b16 --- /dev/null +++ b/metrics/tl-traffic-bucket.h @@ -0,0 +1,57 @@ +#pragma once +#include +#include +#include +#include + +#include "metrics-collectors.h" +#include "td/utils/Slice.h" +#include "td/utils/int_types.h" + +namespace ton::metrics { + +// Per-TL-constructor traffic counter. Use one instance per (layer, kind) pair (e.g. rldp send/query/...). +// +// Magics that don't match a known TL constructor — including wire garbage and short payloads — +// are funnelled into a single `unknown` bucket so cardinality stays bounded by the schema size. +struct TlTrafficBucket : public Instrument { + TlTrafficBucket() = default; + explicit TlTrafficBucket(std::string metric_name_base, std::optional bytes_help = std::nullopt, + std::optional messages_help = std::nullopt); + + // Indexed by TL constructor id. Only valid (schema-known) ids are inserted as keys here. + std::map bytes; + std::map messages; + + // Single "unknown" bucket for wire payloads whose first 4 bytes don't match a known constructor. + td::uint64 unknown_bytes = 0; + td::uint64 unknown_messages = 0; + + // Increment counters for `data` (a serialized TL object). `data` may be empty / shorter than 4. + void account(td::Slice data); + + // Same accounting as account() but with the magic already extracted. Use this when the + // accounting happens in a different actor from the one that owns the BufferSlice — pass + // (magic, size) through send_closure rather than copying the whole payload. Pass magic == 0 + // (or any other unknown value) to force the unknown bucket. + void account_with_magic(td::int32 magic, td::uint64 size); + + MetricSet collect() override; + + private: + std::string metric_name_base_; + std::optional bytes_help_; + std::optional messages_help_; +}; + +// Append two MetricFamily entries (`_bytes_by_tl_total`, `_messages_by_tl_total`) +// to `set`, labeled {=, tl=}. +// +// `base` is something like "app_send"; the "_bytes_by_tl_total" / "_messages_by_tl_total" +// suffixes are appended. Use `bucket_label_key` = "kind" for per-layer kinds, or "type" for +// overlay-type partitioning, etc. +void render_tl_bucket(MetricSet &set, const std::string &base, const std::string &bucket_label_value, + const TlTrafficBucket &bucket, std::optional bytes_help = std::nullopt, + std::optional messages_help = std::nullopt, std::string bucket_label_key = "kind"); + +} // namespace ton::metrics diff --git a/overlay/CMakeLists.txt b/overlay/CMakeLists.txt index 58dab3c1a0..c5fc15dac4 100644 --- a/overlay/CMakeLists.txt +++ b/overlay/CMakeLists.txt @@ -22,5 +22,5 @@ target_include_directories(overlay PUBLIC $/.. ${OPENSSL_INCLUDE_DIR} ) -target_link_libraries(overlay PRIVATE tdutils tdactor adnl tl_api dht fec) +target_link_libraries(overlay PRIVATE tdutils tdactor adnl tl_api dht fec metrics) diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index cbeb47fa77..fb3c9d5a7c 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -21,11 +21,14 @@ #include "adnl/utils.hpp" #include "auto/tl/ton_api.h" #include "auto/tl/ton_api.hpp" +#include "metrics/metrics-types.h" +#include "metrics/prometheus-exporter.h" #include "td/actor/actor.h" #include "td/actor/common.h" #include "td/db/RocksDb.h" #include "td/utils/Random.h" #include "td/utils/Status.h" +#include "td/utils/as.h" #include "td/utils/buffer.h" #include "td/utils/overloaded.h" #include "td/utils/port/Poll.h" @@ -410,6 +413,8 @@ void OverlayManager::send_message_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeI if (it2 != it->second.end()) { td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_out_ctr, dst, object.size(), false, false); + td::int32 magic = object.size() >= sizeof(td::int32) ? td::as(object.data()) : 0; + td::actor::send_closure(it2->second.overlay, &Overlay::note_outbound_message_tl, magic, object.size()); if (!it2->second.member_certificate.empty()) { // do not send certificate here, we hope that all our neighbours already know of out certificate // we send it every second to some random nodes. Here we don't want to increase the size of the message @@ -547,7 +552,22 @@ OverlayManager::OverlayManager(std::string db_root, td::actor::ActorId manager) : manager_(std::move(manager)) { + } + + void collect(metrics::MetricsPromise P) { + td::actor::send_closure(manager_, &OverlayManager::collect_metrics, std::move(P)); + } + + private: + td::actor::ActorId manager_; +}; + void OverlayManager::start_up() { + metrics_collector_ = td::actor::create_actor("overlaymetricscollector", actor_id(this)); + add_collector(metrics_collector_.get()); if (!db_root_.empty()) { with_db_ = true; std::shared_ptr kv = @@ -785,6 +805,207 @@ td::Status OverlayMemberCertificate::check_signature(const adnl::AdnlNodeIdShort return td::Status::OK(); } +namespace { + +// scope is `{ "type": "shard", ... }`; pull the "type" out without dragging in a JSON parser. +std::string extract_type(const std::string &scope) { + static const std::string key = "\"type\""; + auto k = scope.find(key); + if (k == std::string::npos) { + return "unknown"; + } + auto colon = scope.find(':', k + key.size()); + if (colon == std::string::npos) { + return "unknown"; + } + auto start = scope.find('"', colon + 1); + if (start == std::string::npos) { + return "unknown"; + } + auto end = scope.find('"', start + 1); + if (end == std::string::npos) { + return "unknown"; + } + return scope.substr(start + 1, end - start - 1); +} + +struct OverlayTypeAggregate { + td::uint64 traffic_out_bytes = 0; + td::uint64 traffic_in_bytes = 0; + td::uint64 traffic_out_packets = 0; + td::uint64 traffic_in_packets = 0; + td::uint64 responses_out_bytes = 0; + td::uint64 responses_in_bytes = 0; + td::uint64 responses_out_packets = 0; + td::uint64 responses_in_packets = 0; + td::uint64 broadcast_errors = 0; + td::uint64 fec_broadcast_errors = 0; + td::uint64 overlay_count = 0; + td::uint64 peers = 0; + td::uint64 alive_peers = 0; + td::uint64 neighbours = 0; + metrics::TlTrafficBucket messages_sent_by_tl; + metrics::TlTrafficBucket messages_received_by_tl; + metrics::TlTrafficBucket broadcasts_sent_by_tl; + metrics::TlTrafficBucket broadcasts_received_by_tl; +}; + +void merge_into(metrics::TlTrafficBucket &dst, const metrics::TlTrafficBucket &src) { + for (auto &[magic, value] : src.bytes) { + dst.bytes[magic] += value; + } + for (auto &[magic, value] : src.messages) { + dst.messages[magic] += value; + } + dst.unknown_bytes += src.unknown_bytes; + dst.unknown_messages += src.unknown_messages; +} + +} // namespace + +void OverlayManager::collect_metrics(metrics::MetricsPromise P) { + using metrics::MetricFamily; + using metrics::MetricSet; + // Async fan-out: ask every overlay for its accumulated stats, aggregate by overlay type, then + // emit a labeled MetricSet to the promise. One round-trip per overlay per scrape. + class Collector : public td::actor::Actor { + public: + explicit Collector(metrics::MetricsPromise promise) : promise_(std::move(promise)) { + } + + void inc_pending() { + pending_++; + } + void on_snapshot(td::Result r) { + if (r.is_ok()) { + auto snap = r.move_as_ok(); + auto type = extract_type(snap.scope); + auto &agg = by_type_[type]; + agg.overlay_count++; + agg.traffic_out_bytes += snap.traffic_out_bytes; + agg.traffic_in_bytes += snap.traffic_in_bytes; + agg.traffic_out_packets += snap.traffic_out_packets; + agg.traffic_in_packets += snap.traffic_in_packets; + agg.responses_out_bytes += snap.responses_out_bytes; + agg.responses_in_bytes += snap.responses_in_bytes; + agg.responses_out_packets += snap.responses_out_packets; + agg.responses_in_packets += snap.responses_in_packets; + agg.broadcast_errors += snap.broadcast_errors; + agg.fec_broadcast_errors += snap.fec_broadcast_errors; + agg.peers += snap.peers; + agg.alive_peers += snap.alive_peers; + agg.neighbours += snap.neighbours; + merge_into(agg.messages_sent_by_tl, snap.messages_sent_by_tl); + merge_into(agg.messages_received_by_tl, snap.messages_received_by_tl); + merge_into(agg.broadcasts_sent_by_tl, snap.broadcasts_sent_by_tl); + merge_into(agg.broadcasts_received_by_tl, snap.broadcasts_received_by_tl); + } + dec_pending(); + } + void dec_pending() { + if (--pending_ != 0) { + return; + } + MetricSet set = build_set(); + promise_.set_value(std::move(set).wrap("overlay")); + stop(); + } + + private: + metrics::MetricsPromise promise_; + size_t pending_{1}; + std::map by_type_; + + MetricSet build_set() { + MetricSet set; + auto labeled = [&](std::string name, std::string type, std::optional help, auto getter) { + MetricFamily fam{.name = std::move(name), .type = std::move(type), .help = std::move(help), .metrics = {}}; + for (auto &[overlay_type, agg] : by_type_) { + fam.metrics.push_back(metrics::Metric{ + .suffix = "", + .label_set = metrics::LabelSet{.labels = {{"type", overlay_type}}}, + .samples = {metrics::Sample{.label_set = {}, .value = static_cast(getter(agg))}}, + }); + } + set.families.push_back(std::move(fam)); + }; + auto labeled_dir = [&](std::string name, std::string type, std::optional help, auto out_getter, + auto in_getter) { + MetricFamily fam{.name = std::move(name), .type = std::move(type), .help = std::move(help), .metrics = {}}; + for (auto &[overlay_type, agg] : by_type_) { + for (auto [direction, value] : + {std::pair{std::string("out"), out_getter(agg)}, std::pair{std::string("in"), in_getter(agg)}}) { + fam.metrics.push_back(metrics::Metric{ + .suffix = "", + .label_set = metrics::LabelSet{.labels = {{"type", overlay_type}, {"direction", std::move(direction)}}}, + .samples = {metrics::Sample{.label_set = {}, .value = static_cast(value)}}, + }); + } + } + set.families.push_back(std::move(fam)); + }; + labeled_dir( + "traffic_bytes_total", "counter", "Bytes flowing through overlays (broadcasts + messages).", + [](auto &a) { return a.traffic_out_bytes; }, [](auto &a) { return a.traffic_in_bytes; }); + labeled_dir( + "traffic_packets_total", "counter", "Packets flowing through overlays.", + [](auto &a) { return a.traffic_out_packets; }, [](auto &a) { return a.traffic_in_packets; }); + labeled_dir( + "responses_bytes_total", "counter", "Bytes flowing through overlays attributable to query responses.", + [](auto &a) { return a.responses_out_bytes; }, [](auto &a) { return a.responses_in_bytes; }); + labeled_dir( + "responses_packets_total", "counter", "Packets flowing through overlays attributable to responses.", + [](auto &a) { return a.responses_out_packets; }, [](auto &a) { return a.responses_in_packets; }); + labeled("broadcast_errors_total", "counter", "Broadcast validation errors observed in overlays.", + [](auto &a) { return a.broadcast_errors; }); + labeled("fec_broadcast_errors_total", "counter", "FEC broadcast validation errors observed in overlays.", + [](auto &a) { return a.fec_broadcast_errors; }); + labeled("count", "gauge", "Number of registered overlays.", [](auto &a) { return a.overlay_count; }); + labeled("peers", "gauge", "Sum of overlay peer counts.", [](auto &a) { return a.peers; }); + labeled("alive_peers", "gauge", "Sum of alive peer counts.", [](auto &a) { return a.alive_peers; }); + labeled("neighbours", "gauge", "Sum of selected neighbour counts.", [](auto &a) { return a.neighbours; }); + // Per-(type, tl) breakdown for messages and broadcasts crossing the overlay boundary. + // Each call expands into bytes_by_tl_total + messages_by_tl_total families with labels + // {kind=type, tl=schema_name}. The label key "kind" here carries the overlay type. + auto append_tl = [&](std::string base, auto getter, std::optional bytes_help, + std::optional messages_help) { + for (auto &[overlay_type, agg] : by_type_) { + metrics::render_tl_bucket(set, base, overlay_type, getter(agg), bytes_help, messages_help, "type"); + } + }; + append_tl( + "messages_sent", [](auto &a) -> auto & { return a.messages_sent_by_tl; }, + "Bytes sent via overlay direct messages, by overlay type and inner TL.", + "Messages sent via overlay direct messages, by overlay type and inner TL."); + append_tl( + "messages_received", [](auto &a) -> auto & { return a.messages_received_by_tl; }, + "Bytes received via overlay direct messages, by overlay type and inner TL.", + "Messages received via overlay direct messages, by overlay type and inner TL."); + append_tl( + "broadcasts_sent", [](auto &a) -> auto & { return a.broadcasts_sent_by_tl; }, + "Bytes sent via overlay broadcasts, by overlay type and inner TL.", + "Broadcasts sent, by overlay type and inner TL."); + append_tl( + "broadcasts_received", [](auto &a) -> auto & { return a.broadcasts_received_by_tl; }, + "Bytes received via overlay broadcasts, by overlay type and inner TL.", + "Broadcasts received, by overlay type and inner TL."); + return set; + } + }; + + auto collector = td::actor::create_actor("overlaymetrics", std::move(P)).release(); + for (auto &[local_id, by_oid] : overlays_) { + for (auto &[overlay_id, descr] : by_oid) { + td::actor::send_closure(collector, &Collector::inc_pending); + td::actor::send_closure(descr.overlay, &Overlay::get_acc_stats, + [collector](td::Result r) { + td::actor::send_closure(collector, &Collector::on_snapshot, std::move(r)); + }); + } + } + td::actor::send_closure(collector, &Collector::dec_pending); +} + } // namespace overlay } // namespace ton diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h index 9f069547cb..dd537aa873 100644 --- a/overlay/overlay-manager.h +++ b/overlay/overlay-manager.h @@ -23,6 +23,7 @@ #include "adnl/adnl.h" #include "dht/dht.h" +#include "metrics/metrics-collectors.h" #include "td/actor/actor.h" #include "td/db/KeyValueAsync.h" @@ -45,6 +46,8 @@ class OverlayManager : public Overlays { public: OverlayManager(std::string db_root, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId dht, OverlayManagerBufferLimits buffer_limits = {}); + + void collect_metrics(metrics::MetricsPromise P); void start_up() override; void save_to_db(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, std::vector nodes); @@ -165,6 +168,8 @@ class OverlayManager : public Overlays { using DbType = td::KeyValueAsync; bool with_db_ = false; DbType db_; + class MetricsCollector; + td::actor::ActorOwn metrics_collector_; class AdnlCallback : public adnl::Adnl::Callback { public: diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index a74ccf95df..b5322e7d26 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -675,6 +675,11 @@ void OverlayImpl::update_peer_err_ctr(adnl::AdnlNodeIdShort peer_id, bool is_fec src_peer->broadcast_errors++; } } + if (is_fec) { + fec_broadcast_errors_acc_++; + } else { + broadcast_errors_acc_++; + } } void OverlayImpl::update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint64 msg_size, bool is_query, @@ -690,8 +695,10 @@ void OverlayImpl::update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::u } } total_traffic_ctr.add_packet(msg_size, false); + total_traffic_acc.add_packet(msg_size, false); if (is_response) { total_traffic_responses_ctr.add_packet(msg_size, false); + total_traffic_responses_acc.add_packet(msg_size, false); } } @@ -708,8 +715,10 @@ void OverlayImpl::update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::ui } } total_traffic_ctr.add_packet(msg_size, true); + total_traffic_acc.add_packet(msg_size, true); if (is_response) { total_traffic_responses_ctr.add_packet(msg_size, true); + total_traffic_responses_acc.add_packet(msg_size, true); } } diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index 36642cff65..385d53d0e5 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -289,6 +289,7 @@ void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, tl_object_ptr(data.clone(), true); if (X.is_error()) { VLOG(OVERLAY_DEBUG) << this << ": received custom message"; + messages_received_by_tl_.account(data.as_slice()); callback_->receive_message(src, overlay_id_, std::move(data)); return; } @@ -527,6 +528,7 @@ void OverlayImpl::send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::Bu VLOG(OVERLAY_WARNING) << "broadcast source certificate is invalid"; return; } + broadcasts_sent_by_tl_.account(data.as_slice()); flags &= ~Overlays::BroadcastFlagNoTwostep(); broadcasts_simple_.send(this, send_as, std::move(data), flags); } @@ -542,6 +544,7 @@ void OverlayImpl::send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td VLOG(OVERLAY_WARNING) << "broadcast source certificate is invalid"; return; } + broadcasts_sent_by_tl_.account(data.as_slice()); bool no_twostep = flags & Overlays::BroadcastFlagNoTwostep(); flags &= ~Overlays::BroadcastFlagNoTwostep(); if (opts_.send_twostep_broadcast_ && !no_twostep) { @@ -660,6 +663,7 @@ void OverlayImpl::broadcast_twostep_signed_fec(BroadcastTwostepDataFec &&data, } void OverlayImpl::deliver_broadcast(PublicKeyHash source, td::BufferSlice data, td::BufferSlice extra) { + broadcasts_received_by_tl_.account(data.as_slice()); callback_->receive_broadcast_with_extra(source, overlay_id_, std::move(data), std::move(extra)); } @@ -788,6 +792,33 @@ void OverlayImpl::get_stats(td::Promise promise) { + AccStatsSnapshot snap; + snap.scope = scope_; + snap.traffic_out_bytes = total_traffic_acc.out_bytes; + snap.traffic_in_bytes = total_traffic_acc.in_bytes; + snap.traffic_out_packets = total_traffic_acc.out_packets; + snap.traffic_in_packets = total_traffic_acc.in_packets; + snap.responses_out_bytes = total_traffic_responses_acc.out_bytes; + snap.responses_in_bytes = total_traffic_responses_acc.in_bytes; + snap.responses_out_packets = total_traffic_responses_acc.out_packets; + snap.responses_in_packets = total_traffic_responses_acc.in_packets; + snap.broadcast_errors = broadcast_errors_acc_; + snap.fec_broadcast_errors = fec_broadcast_errors_acc_; + iterate_all_peers([&](const adnl::AdnlNodeIdShort &, const OverlayPeer &peer) { + snap.peers++; + if (peer.is_alive()) { + snap.alive_peers++; + } + }); + snap.neighbours = neighbours_cnt(); + snap.messages_sent_by_tl = messages_sent_by_tl_; + snap.messages_received_by_tl = messages_received_by_tl_; + snap.broadcasts_sent_by_tl = broadcasts_sent_by_tl_; + snap.broadcasts_received_by_tl = broadcasts_received_by_tl_; + promise.set_value(std::move(snap)); +} + bool OverlayImpl::has_valid_broadcast_certificate(const PublicKeyHash &source, size_t size, bool is_fec) { if (size > std::numeric_limits::max()) { return false; diff --git a/overlay/overlay.h b/overlay/overlay.h index dad88f247d..0ed677a3ca 100644 --- a/overlay/overlay.h +++ b/overlay/overlay.h @@ -20,6 +20,7 @@ #include "adnl/adnl.h" #include "auto/tl/ton_api.h" +#include "metrics/tl-traffic-bucket.h" #include "td/actor/actor.h" #include "td/utils/buffer.h" #include "td/utils/int_types.h" @@ -71,6 +72,34 @@ class Overlay : public td::actor::Actor { virtual void receive_nodes_from_db(tl_object_ptr nodes) = 0; virtual void receive_nodes_from_db_v2(tl_object_ptr nodes) = 0; virtual void get_stats(td::Promise> promise) = 0; + + // Snapshot for the metrics layer: monotonic counters + per-overlay scalars + scope JSON. + // Returned synchronously into the promise; one round-trip per overlay per scrape. + struct AccStatsSnapshot { + std::string scope; // raw JSON; the metrics layer extracts the "type" field. + td::uint64 traffic_out_bytes = 0; + td::uint64 traffic_in_bytes = 0; + td::uint64 traffic_out_packets = 0; + td::uint64 traffic_in_packets = 0; + td::uint64 responses_out_bytes = 0; + td::uint64 responses_in_bytes = 0; + td::uint64 responses_out_packets = 0; + td::uint64 responses_in_packets = 0; + td::uint64 broadcast_errors = 0; + td::uint64 fec_broadcast_errors = 0; + td::uint64 peers = 0; + td::uint64 alive_peers = 0; + td::uint64 neighbours = 0; + metrics::TlTrafficBucket messages_sent_by_tl; + metrics::TlTrafficBucket messages_received_by_tl; + metrics::TlTrafficBucket broadcasts_sent_by_tl; + metrics::TlTrafficBucket broadcasts_received_by_tl; + }; + virtual void get_acc_stats(td::Promise promise) = 0; + + // Cross-actor accounting hooks: the OverlayManager extracts the magic in its own actor and + // ships just the (magic, size) tuple to avoid copying the payload via send_closure. + virtual void note_outbound_message_tl(td::int32 magic, td::uint64 size) = 0; virtual void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint64 msg_size, bool is_query, bool is_response) = 0; virtual void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint64 msg_size, bool is_query, diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 8cb2fc3b27..de041854f5 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -72,6 +72,25 @@ struct TrafficStats { tl_object_ptr tl() const; }; +// Same shape as TrafficStats but never reset; used as the source of monotonic Prometheus counters. +// (TrafficStats above is repurposed by alarm() to hold a 50s rate, which is unsuitable for `rate()` queries.) +struct AccTrafficStats { + td::uint64 out_bytes = 0; + td::uint64 in_bytes = 0; + td::uint64 out_packets = 0; + td::uint64 in_packets = 0; + + void add_packet(td::uint64 size, bool in) { + if (in) { + in_bytes += size; + in_packets++; + } else { + out_bytes += size; + out_packets++; + } + } +}; + class OverlayPeer { public: adnl::AdnlNodeIdShort get_id() const { @@ -308,6 +327,10 @@ class OverlayImpl : public Overlay { td::Result get_encryptor(PublicKey source); void get_stats(td::Promise> promise) override; + void get_acc_stats(td::Promise promise) override; + void note_outbound_message_tl(td::int32 magic, td::uint64 size) override { + messages_sent_by_tl_.account_with_magic(magic, size); + } void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint64 msg_size, bool is_query, bool is_response) override; @@ -519,6 +542,14 @@ class OverlayImpl : public Overlay { } peer_list_; TrafficStats total_traffic, total_traffic_ctr; TrafficStats total_traffic_responses, total_traffic_responses_ctr; + AccTrafficStats total_traffic_acc; + AccTrafficStats total_traffic_responses_acc; + td::uint64 broadcast_errors_acc_ = 0; + td::uint64 fec_broadcast_errors_acc_ = 0; + metrics::TlTrafficBucket messages_sent_by_tl_; + metrics::TlTrafficBucket messages_received_by_tl_; + metrics::TlTrafficBucket broadcasts_sent_by_tl_; + metrics::TlTrafficBucket broadcasts_received_by_tl_; OverlayOptions opts_; adnl::PeersMtuGuard peers_mtu_guard_; diff --git a/overlay/overlays.h b/overlay/overlays.h index 342262f876..ff7d535658 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -24,6 +24,7 @@ #include "adnl/adnl-sender-ex.h" #include "auto/tl/ton_api.h" #include "dht/dht.h" +#include "metrics/metrics-collectors.h" #include "td/actor/PromiseFuture.h" #include "td/actor/actor.h" #include "td/utils/RateLimiterWindow.h" @@ -310,7 +311,7 @@ struct OverlayManagerBufferLimits { td::uint64 max_data_size = 0; }; -class Overlays : public td::actor::Actor { +class Overlays : public td::actor::Actor, public virtual metrics::CollectorWrapper { public: class Callback { public: diff --git a/quic/quic-common.h b/quic/quic-common.h index 418ad4d337..0ae48e92c9 100644 --- a/quic/quic-common.h +++ b/quic/quic-common.h @@ -19,7 +19,24 @@ struct QuicConnectionStats { int64_t bytes_unacked = 0, bytes_unsent = 0; int64_t total_sids = 0, open_sids = 0; double mean_rtt = 0; - + // Extended (per-conn snapshot, summed only by absolute counts): + int64_t pkt_sent = 0, pkt_recv = 0, pkt_lost = 0; + int64_t bytes_in_flight = 0; + int64_t cwnd = 0; + // RTT samples in seconds (per-conn snapshots; aggregated as a sum, not a useful average): + double min_rtt_s = 0; + double latest_rtt_s = 0; + double rttvar_s = 0; + // ngtcp2 stream-data boundary: bytes the application asked ngtcp2 to send on streams (cumulative + // input to buffer_stream) vs bytes ngtcp2 yielded back via stream-data callbacks. The diff + // between these and the bytes_tx/bytes_rx packet-level counters is QUIC framing overhead. + int64_t stream_bytes_buffered = 0; + int64_t stream_bytes_received = 0; + + // Summing QuicConnectionStats is only meaningful for the cumulative counters and the + // snapshot "current state" gauges (bytes_in_flight, cwnd, open_sids). RTT fields are + // per-connection observations and the outer Stats::Entry aggregator reweights them — here + // they are left at zero so nobody can accidentally surface a sum-of-RTTs as a metric. QuicConnectionStats operator+(const QuicConnectionStats& other) const { return { .bytes_rx = bytes_rx + other.bytes_rx, @@ -29,9 +46,22 @@ struct QuicConnectionStats { .bytes_unsent = bytes_unsent + other.bytes_unsent, .total_sids = total_sids + other.total_sids, .open_sids = open_sids + other.open_sids, + .mean_rtt = 0, + .pkt_sent = pkt_sent + other.pkt_sent, + .pkt_recv = pkt_recv + other.pkt_recv, + .pkt_lost = pkt_lost + other.pkt_lost, + .bytes_in_flight = bytes_in_flight + other.bytes_in_flight, + .cwnd = cwnd + other.cwnd, + .min_rtt_s = 0, + .latest_rtt_s = 0, + .rttvar_s = 0, + .stream_bytes_buffered = stream_bytes_buffered + other.stream_bytes_buffered, + .stream_bytes_received = stream_bytes_received + other.stream_bytes_received, }; } + // Subtraction is used for two-snapshot deltas over counters; RTT fields are gauges for + // which subtraction has no meaning, so they keep the later value. QuicConnectionStats operator-(const QuicConnectionStats& other) const { return { .bytes_rx = bytes_rx - other.bytes_rx, @@ -41,10 +71,46 @@ struct QuicConnectionStats { .bytes_unsent = bytes_unsent - other.bytes_unsent, .total_sids = total_sids - other.total_sids, .open_sids = open_sids - other.open_sids, + .mean_rtt = mean_rtt, + .pkt_sent = pkt_sent - other.pkt_sent, + .pkt_recv = pkt_recv - other.pkt_recv, + .pkt_lost = pkt_lost - other.pkt_lost, + .bytes_in_flight = bytes_in_flight - other.bytes_in_flight, + .cwnd = cwnd - other.cwnd, + .min_rtt_s = min_rtt_s, + .latest_rtt_s = latest_rtt_s, + .rttvar_s = rttvar_s, + .stream_bytes_buffered = stream_bytes_buffered - other.stream_bytes_buffered, + .stream_bytes_received = stream_bytes_received - other.stream_bytes_received, }; } }; +// Cumulative wire-level UDP counters for a single side (ingress or egress). +struct UdpSideCounters { + td::uint64 bytes = 0; + td::uint64 packets = 0; + td::uint64 syscalls = 0; + + UdpSideCounters& operator+=(const UdpSideCounters& o) { + bytes += o.bytes; + packets += o.packets; + syscalls += o.syscalls; + return *this; + } +}; + +struct UdpCounters { + UdpSideCounters ingress; + UdpSideCounters egress; + + UdpCounters& operator+=(const UdpCounters& o) { + ingress += o.ingress; + egress += o.egress; + return *this; + } +}; + enum class CongestionControlAlgo { Cubic, // default Reno, diff --git a/quic/quic-pimpl.cpp b/quic/quic-pimpl.cpp index be6bb8c515..4392c22a83 100644 --- a/quic/quic-pimpl.cpp +++ b/quic/quic-pimpl.cpp @@ -566,6 +566,7 @@ td::Status QuicConnectionPImpl::buffer_stream(QuicStreamID sid, td::BufferSlice if (st.fin_pending || st.fin_submitted) { return td::Status::Error("stream already closed"); } + stream_bytes_buffered_ += data.size(); st.writer_.append(std::move(data)); st.reader_.sync_with_writer(); st.pin_.sync_with_writer(); @@ -584,6 +585,7 @@ QuicConnectionStats QuicConnectionPImpl::get_stats() { bytes_unacked += stream.pin_.size(); bytes_unsent += stream.reader_.size(); } + constexpr double ns_to_s = 1e-9; return { .bytes_rx = static_cast(info.bytes_recv), .bytes_tx = static_cast(info.bytes_sent), @@ -593,6 +595,16 @@ QuicConnectionStats QuicConnectionPImpl::get_stats() { .total_sids = static_cast(sids_encountered), .open_sids = static_cast(streams_.size()), .mean_rtt = static_cast(info.smoothed_rtt), + .pkt_sent = static_cast(info.pkt_sent), + .pkt_recv = static_cast(info.pkt_recv), + .pkt_lost = static_cast(info.pkt_lost), + .bytes_in_flight = static_cast(info.bytes_in_flight), + .cwnd = static_cast(info.cwnd), + .min_rtt_s = static_cast(info.min_rtt) * ns_to_s, + .latest_rtt_s = static_cast(info.latest_rtt) * ns_to_s, + .rttvar_s = static_cast(info.rttvar) * ns_to_s, + .stream_bytes_buffered = static_cast(stream_bytes_buffered_), + .stream_bytes_received = static_cast(stream_bytes_received_), }; } @@ -648,6 +660,7 @@ int QuicConnectionPImpl::on_handshake_completed() { } int QuicConnectionPImpl::on_recv_stream_data(uint32_t flags, int64_t stream_id, td::Slice data) { + stream_bytes_received_ += data.size(); Callback::StreamDataEvent event{ .sid = stream_id, .data = td::BufferSlice{data}, .fin = (flags & NGTCP2_STREAM_DATA_FLAG_FIN) != 0}; diff --git a/quic/quic-pimpl.h b/quic/quic-pimpl.h index c9f4481450..9916263252 100644 --- a/quic/quic-pimpl.h +++ b/quic/quic-pimpl.h @@ -221,6 +221,9 @@ struct QuicConnectionPImpl { bool local_cid_callbacks_enabled_{false}; size_t sids_encountered = 0; + // Cumulative ngtcp2 stream-data boundary counters. + td::uint64 stream_bytes_buffered_ = 0; + td::uint64 stream_bytes_received_ = 0; std::unordered_map streams_; std::deque ready_streams_; QuicStreamID write_sid_ = -1; diff --git a/quic/quic-sender.cpp b/quic/quic-sender.cpp index 3ace992c1f..f9dd1bde51 100644 --- a/quic/quic-sender.cpp +++ b/quic/quic-sender.cpp @@ -298,6 +298,20 @@ std::vector QuicSender::Stats::Entry::dump() const { metrics::MetricFamily::make_scalar("unsent_bytes", "gauge", server_stats.impl_stats.bytes_unsent), metrics::MetricFamily::make_scalar("open_sids", "gauge", server_stats.impl_stats.open_sids), metrics::MetricFamily::make_scalar("mean_rtt", "gauge", server_stats.impl_stats.mean_rtt), + metrics::MetricFamily::make_scalar("pkt_sent_total", "counter", server_stats.impl_stats.pkt_sent), + metrics::MetricFamily::make_scalar("pkt_recv_total", "counter", server_stats.impl_stats.pkt_recv), + metrics::MetricFamily::make_scalar("pkt_lost_total", "counter", server_stats.impl_stats.pkt_lost), + metrics::MetricFamily::make_scalar("bytes_in_flight", "gauge", server_stats.impl_stats.bytes_in_flight), + metrics::MetricFamily::make_scalar("cwnd_bytes", "gauge", server_stats.impl_stats.cwnd), + metrics::MetricFamily::make_scalar("min_rtt_seconds", "gauge", server_stats.impl_stats.min_rtt_s), + metrics::MetricFamily::make_scalar("latest_rtt_seconds", "gauge", server_stats.impl_stats.latest_rtt_s), + metrics::MetricFamily::make_scalar("rttvar_seconds", "gauge", server_stats.impl_stats.rttvar_s), + metrics::MetricFamily::make_scalar( + "stream_bytes_buffered_total", "counter", server_stats.impl_stats.stream_bytes_buffered, + "Cumulative bytes the sender handed to ngtcp2 stream buffers (pre-QUIC-framing)."), + metrics::MetricFamily::make_scalar( + "stream_bytes_received_total", "counter", server_stats.impl_stats.stream_bytes_received, + "Cumulative bytes ngtcp2 yielded to the sender via stream-data callbacks (post-QUIC-framing)."), }; } @@ -310,7 +324,29 @@ std::vector QuicSender::Stats::dump() const { auto label_set = metrics::LabelSet{.labels = {{"src", src_v}, {"dst", dst_v}}}; whole_per_path_set = std::move(whole_per_path_set).join(std::move(path_set).label(label_set)); } - return std::move(summary_set).wrap("summary").join(std::move(whole_per_path_set).wrap("per_path")).families; + metrics::MetricSet udp_set{ + .families = { + metrics::MetricFamily::make_scalar("ingress_bytes_total", "counter", static_cast(udp.ingress.bytes), + "Total UDP bytes received on QUIC sockets (wire-level)."), + metrics::MetricFamily::make_scalar("ingress_packets_total", "counter", + static_cast(udp.ingress.packets), + "Total UDP datagrams received on QUIC sockets."), + metrics::MetricFamily::make_scalar("ingress_syscalls_total", "counter", + static_cast(udp.ingress.syscalls), + "recvmmsg/recvmsg syscalls on QUIC sockets."), + metrics::MetricFamily::make_scalar("egress_bytes_total", "counter", static_cast(udp.egress.bytes), + "Total UDP bytes sent on QUIC sockets (wire-level)."), + metrics::MetricFamily::make_scalar("egress_packets_total", "counter", static_cast(udp.egress.packets), + "Total UDP datagrams sent on QUIC sockets (post-GSO expansion)."), + metrics::MetricFamily::make_scalar("egress_syscalls_total", "counter", + static_cast(udp.egress.syscalls), + "sendmmsg/sendmsg syscalls on QUIC sockets."), + }}; + return std::move(summary_set) + .wrap("summary") + .join(std::move(udp_set).wrap("udp")) + .join(std::move(whole_per_path_set).wrap("per_path")) + .families; } td::actor::Task QuicSender::collect_stats() { @@ -318,6 +354,7 @@ td::actor::Task QuicSender::collect_stats() { for (auto &[_, server] : servers_) { auto serv_stats = co_await td::actor::ask(server, &QuicServer::collect_stats); stats.summary = stats.summary + Stats::Entry{.server_stats = serv_stats.summary}; + stats.udp += serv_stats.udp; for (auto &[id, conn_stats] : serv_stats.per_conn) { if (!by_cid_.contains(id)) continue; @@ -329,10 +366,14 @@ td::actor::Task QuicSender::collect_stats() { // TODO(avevad): remove obsolete Stats and collect metrics directly void QuicSender::collect(td::Promise P) { - td::actor::send_closure(actor_id(this), &QuicSender::collect_stats, - td::make_promise([P = std::move(P)](td::Result R) mutable { - P.set_value(metrics::MetricSet{.families = R.move_as_ok().dump()}.wrap("quic")); - })); + auto app_set = app_metrics_->collect(); + + td::actor::send_closure( + actor_id(this), &QuicSender::collect_stats, + td::make_promise([P = std::move(P), app_set = std::move(app_set)](td::Result R) mutable { + auto stats_set = metrics::MetricSet{.families = R.move_as_ok().dump()}; + P.set_value(std::move(stats_set).join(std::move(app_set)).wrap("quic")); + })); } void QuicSender::on_mtu_updated(td::optional local_id, @@ -378,6 +419,7 @@ td::actor::Task QuicSender::send_message_coro(adnl::AdnlNodeIdShort sr td::actor::Task QuicSender::send_message_coro_inner(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) { + app_metrics_->record_send("message", data.as_slice()); auto conn = co_await find_or_create_connection({src, dst}); td::BufferSlice wire_data = create_serialize_tl_object(std::move(data)); co_await td::actor::ask(conn->server, &QuicServer::send_stream, conn->cid, StreamOptions{get_peer_mtu(src, dst)}, @@ -388,6 +430,7 @@ td::actor::Task QuicSender::send_message_coro_inner(adnl::AdnlNodeIdSh td::actor::Task QuicSender::send_query_coro(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, std::string name, td::Timestamp timeout, td::BufferSlice data, std::optional limit) { + app_metrics_->record_send("query", data.as_slice()); auto conn = co_await find_or_create_connection({src, dst}); auto query_size = data.size(); auto query_magic = get_magic(data); @@ -657,11 +700,13 @@ void QuicSender::on_closed(QuicConnectionId cid) { void QuicSender::on_request(std::shared_ptr connection, QuicStreamID stream_id, ton_api::quic_query &query) { + app_metrics_->record_deliver("query", query.data_.as_slice()); on_inbound_query(connection, stream_id, std::move(query.data_)).start_immediate().detach(); } void QuicSender::on_request(std::shared_ptr connection, QuicStreamID stream_id, ton_api::quic_message &message) { + app_metrics_->record_deliver("message", message.data_.as_slice()); td::actor::send_closure(adnl_, &adnl::AdnlPeerTable::deliver, connection->path.second, connection->path.first, std::move(message.data_)); // TODO: use unidirectional stream, so there will be no need to process result @@ -685,6 +730,7 @@ void QuicSender::on_answer(Connection &connection, QuicStreamID stream_id, ton_a LOG(ERROR) << "Answer from unknown stream_id"; return; } + app_metrics_->record_deliver("answer", answer.data_.as_slice()); it->second.set_result(std::move(answer.data_)); connection.responses.erase(it); } diff --git a/quic/quic-sender.h b/quic/quic-sender.h index 00671e1dea..ebb1e00f34 100644 --- a/quic/quic-sender.h +++ b/quic/quic-sender.h @@ -5,6 +5,7 @@ #include "adnl/adnl-peer-table.h" #include "adnl/adnl-sender-ex.h" #include "keyring/keyring.h" +#include "metrics/app-traffic-metrics.h" #include "metrics/metrics-collectors.h" #include "td/actor/coro_task.h" @@ -42,11 +43,12 @@ class QuicSender : public adnl::AdnlSenderEx, public virtual metrics::AsyncColle } [[nodiscard]] std::vector dump() const; - }; - - Entry summary = {.server_stats = {.total_conns = 0}}; + } summary = {}; std::map per_path; + // Server-level UDP wire counters aggregated across all QuicServers. + UdpCounters udp = {}; + [[nodiscard]] std::vector dump() const; }; @@ -87,6 +89,8 @@ class QuicSender : public adnl::AdnlSenderEx, public virtual metrics::AsyncColle std::map> servers_; std::map local_keys_; + metrics::AppTrafficMetrics::Ptr app_metrics_ = metrics::AppTrafficMetrics::make(); + void start_up() override; td::actor::Task send_message_coro(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, diff --git a/quic/quic-server.cpp b/quic/quic-server.cpp index b3b512ac4f..dad6ff288a 100644 --- a/quic/quic-server.cpp +++ b/quic/quic-server.cpp @@ -432,6 +432,10 @@ void QuicServer::collect_stats(td::Promise P) { stats.summary = stats.summary + entry; stats.per_conn[id] = entry; } + stats.udp.ingress = { + .bytes = ingress_stats_.bytes, .packets = ingress_stats_.packets, .syscalls = ingress_stats_.syscalls}; + stats.udp.egress = { + .bytes = egress_stats_.bytes, .packets = egress_stats_.packets, .syscalls = egress_stats_.syscalls}; return P.set_value(std::move(stats)); } diff --git a/quic/quic-server.h b/quic/quic-server.h index af1174edec..a17da8ea4e 100644 --- a/quic/quic-server.h +++ b/quic/quic-server.h @@ -110,28 +110,36 @@ class QuicServer : public td::actor::Actor, public td::ObserverBase { struct Stats { struct Entry { - size_t total_conns = 1; + // Leaf (per-conn) entries are constructed with `.total_conns = 1`; the aggregate summary + // and sender-side per-path snapshots default to 0 and absorb leaves via operator+. + size_t total_conns = 0; QuicConnectionStats impl_stats = {}; Entry operator+(const Entry &other) const { Entry res = {.total_conns = total_conns + other.total_conns, .impl_stats = impl_stats + other.impl_stats}; - auto tc = total_conns + other.total_conns; - if (tc > 0) - res.impl_stats.mean_rtt = (static_cast(total_conns) * impl_stats.mean_rtt + - static_cast(other.total_conns) * other.impl_stats.mean_rtt) / - static_cast(tc); + if (res.total_conns == 0) { + return res; + } + auto w1 = static_cast(total_conns), w2 = static_cast(other.total_conns); + auto w = static_cast(res.total_conns); + auto mix = [&](double a, double b) { return (w1 * a + w2 * b) / w; }; + res.impl_stats.mean_rtt = mix(impl_stats.mean_rtt, other.impl_stats.mean_rtt); + res.impl_stats.min_rtt_s = mix(impl_stats.min_rtt_s, other.impl_stats.min_rtt_s); + res.impl_stats.latest_rtt_s = mix(impl_stats.latest_rtt_s, other.impl_stats.latest_rtt_s); + res.impl_stats.rttvar_s = mix(impl_stats.rttvar_s, other.impl_stats.rttvar_s); return res; } Entry operator-(const Entry &other) const { - Entry res = {.total_conns = total_conns - other.total_conns, .impl_stats = impl_stats - other.impl_stats}; - res.impl_stats.mean_rtt = impl_stats.mean_rtt; - return res; + return {.total_conns = total_conns - other.total_conns, .impl_stats = impl_stats - other.impl_stats}; } }; Entry summary = {.total_conns = 0}; std::unordered_map per_conn = {}; + + // Server-level UDP wire stats (single instance per QuicServer). + UdpCounters udp = {}; }; void collect_stats(td::Promise P); diff --git a/rldp/CMakeLists.txt b/rldp/CMakeLists.txt index 361b350948..77c6ed0fc0 100644 --- a/rldp/CMakeLists.txt +++ b/rldp/CMakeLists.txt @@ -15,5 +15,5 @@ target_include_directories(rldp PUBLIC $/.. ${OPENSSL_INCLUDE_DIR} ) -target_link_libraries(rldp PRIVATE tdutils tdactor fec adnl tl_api) +target_link_libraries(rldp PRIVATE tdutils tdactor fec adnl tl_api metrics) diff --git a/rldp/rldp-in.hpp b/rldp/rldp-in.hpp index ab002dd8f5..11636d4a1e 100644 --- a/rldp/rldp-in.hpp +++ b/rldp/rldp-in.hpp @@ -19,10 +19,14 @@ #pragma once #include +#include #include +#include #include "adnl/adnl-peer-table.h" #include "adnl/adnl-query.h" +#include "metrics/app-traffic-metrics.h" +#include "metrics/metrics-collectors.h" #include "td/utils/List.h" #include "tl-utils/tl-utils.hpp" @@ -105,6 +109,7 @@ class RldpIn : public RldpImpl { } protected: + void start_up() override; void on_mtu_updated(td::optional local_id, td::optional peer_id) override { } @@ -125,6 +130,71 @@ class RldpIn : public RldpImpl { std::map max_size_; std::set local_ids_; + + using CounterPtr = std::shared_ptr>; + metrics::MultiCollector::Own metrics_collector_ = metrics::MultiCollector::create("rldp"); + metrics::AppTrafficMetrics::Ptr app_metrics_ = metrics::AppTrafficMetrics::make(); + metrics::Labeled>::Ptr transfers_completed_ = + metrics::Labeled>::make( + "direction", "transfers_completed_total", + "RLDP transfers that finished (out includes both success and timeout-completion)."); + metrics::Labeled>::Ptr transfers_failed_ = + metrics::Labeled>::make( + "direction", "transfers_failed_total", "RLDP transfers that failed."); + metrics::Labeled>::Ptr bytes_sent_to_adnl_ = + metrics::Labeled>::make( + "kind", "bytes_sent_to_adnl_total", "Serialized RLDP message bytes handed to ADNL (post FEC encoding)."); + metrics::Labeled>::Ptr parts_sent_to_adnl_ = + metrics::Labeled>::make( + "kind", "parts_sent_to_adnl_total", "RLDP messages handed to ADNL."); + metrics::Labeled>::Ptr bytes_received_ = + metrics::Labeled>::make( + "kind", "bytes_received_total", "Serialized RLDP message bytes received from ADNL (pre FEC decoding)."); + metrics::Labeled>::Ptr parts_received_ = + metrics::Labeled>::make("kind", "parts_received_total", + "RLDP messages received from ADNL."); + metrics::Labeled>::Ptr parse_errors_ = + metrics::Labeled>::make("where", "parse_errors_total", + "RLDP TL parse failures."); + metrics::AtomicCounter::Ptr transfers_started_ = + metrics::AtomicCounter::make("transfers_started_total", "RLDP outbound transfers initiated."); + metrics::LambdaGauge::Ptr outbound_transfers_ = metrics::LambdaGauge::make( + "outbound_transfers", + [this] { + return std::vector{ + metrics::Sample{.label_set = {}, .value = static_cast(senders_.size())}}; + }, + "Active outbound RLDP transfers."); + metrics::LambdaGauge::Ptr inbound_transfers_ = metrics::LambdaGauge::make( + "inbound_transfers", + [this] { + return std::vector{ + metrics::Sample{.label_set = {}, .value = static_cast(receivers_.size())}}; + }, + "Active inbound RLDP transfers."); + metrics::LambdaGauge::Ptr lru_size_gauge_ = metrics::LambdaGauge::make( + "lru_size", + [this] { + return std::vector{metrics::Sample{.label_set = {}, .value = static_cast(lru_size_)}}; + }, + "Recent completed transfers in dedup LRU."); + + std::shared_ptr metrics_ = std::make_shared(RldpMetrics{ + .sent_to_adnl_part = {.bytes = bytes_sent_to_adnl_->label("part"), .msgs = parts_sent_to_adnl_->label("part")}, + .sent_to_adnl_confirm = + {.bytes = bytes_sent_to_adnl_->label("confirm"), .msgs = parts_sent_to_adnl_->label("confirm")}, + .sent_to_adnl_complete = + {.bytes = bytes_sent_to_adnl_->label("complete"), .msgs = parts_sent_to_adnl_->label("complete")}, + .received_part = {.bytes = bytes_received_->label("part"), .msgs = parts_received_->label("part")}, + .received_confirm = {.bytes = bytes_received_->label("confirm"), .msgs = parts_received_->label("confirm")}, + .received_complete = {.bytes = bytes_received_->label("complete"), .msgs = parts_received_->label("complete")}, + .transfers_started = transfers_started_, + .transfers_completed_out = transfers_completed_->label("out"), + .transfers_completed_in = transfers_completed_->label("in"), + .transfers_failed_in = transfers_failed_->label("in"), + .parse_errors_part = parse_errors_->label("part"), + .parse_errors_message = parse_errors_->label("message"), + }); }; } // namespace rldp diff --git a/rldp/rldp-peer.cpp b/rldp/rldp-peer.cpp index 994625f2d2..55df9ce87d 100644 --- a/rldp/rldp-peer.cpp +++ b/rldp/rldp-peer.cpp @@ -84,7 +84,9 @@ void RldpTransferSenderImpl::send_one_part(td::uint32 seqno) { auto symbol = encoder_->gen_symbol(seqno); auto obj = create_tl_object(transfer_id_, fec_type_.tl(), part_, data_.size(), seqno, std::move(symbol.data)); - td::actor::send_closure(adnl_, &adnl::Adnl::send_message, local_id_, peer_id_, serialize_tl_object(obj, true)); + auto serialized = serialize_tl_object(obj, true); + metrics_->sent_to_adnl_part.record(serialized.size()); + td::actor::send_closure(adnl_, &adnl::Adnl::send_message, local_id_, peer_id_, std::move(serialized)); } void RldpTransferSenderImpl::confirm(td::uint32 part, td::uint32 seqno) { @@ -104,16 +106,19 @@ void RldpTransferSenderImpl::complete(td::uint32 part) { td::actor::ActorOwn RldpTransferSender::create( TransferId transfer_id, adnl::AdnlNodeIdShort local_id, adnl::AdnlNodeIdShort peer_id, td::BufferSlice data, - td::Timestamp timeout, td::actor::ActorId rldp, td::actor::ActorId adnl) { + td::Timestamp timeout, td::actor::ActorId rldp, td::actor::ActorId adnl, + std::shared_ptr metrics) { return td::actor::create_actor("sender", transfer_id, local_id, peer_id, std::move(data), - timeout, rldp, adnl); + timeout, rldp, adnl, std::move(metrics)); } void RldpTransferReceiverImpl::receive_part(fec::FecType fec_type, td::uint32 part, td::uint64 total_size, td::uint32 seqno, td::BufferSlice data) { if (part < part_) { auto obj = create_tl_object(transfer_id_, part); - td::actor::send_closure(adnl_, &adnl::Adnl::send_message, local_id_, peer_id_, serialize_tl_object(obj, true)); + auto serialized = serialize_tl_object(obj, true); + metrics_->sent_to_adnl_complete.record(serialized.size()); + td::actor::send_closure(adnl_, &adnl::Adnl::send_message, local_id_, peer_id_, std::move(serialized)); return; } if (part > part_) { @@ -151,7 +156,9 @@ void RldpTransferReceiverImpl::receive_part(fec::FecType fec_type, td::uint32 pa offset_ += data.data.size(); data_parts_.push_back(std::move(data.data)); auto obj = create_tl_object(transfer_id_, part_); - td::actor::send_closure(adnl_, &adnl::Adnl::send_message, local_id_, peer_id_, serialize_tl_object(obj, true)); + auto serialized = serialize_tl_object(obj, true); + metrics_->sent_to_adnl_complete.record(serialized.size()); + td::actor::send_closure(adnl_, &adnl::Adnl::send_message, local_id_, peer_id_, std::move(serialized)); part_++; cnt_ = 0; max_seqno_ = 0; @@ -165,7 +172,9 @@ void RldpTransferReceiverImpl::receive_part(fec::FecType fec_type, td::uint32 pa if (cnt_ >= 10) { auto obj = create_tl_object(transfer_id_, part_, max_seqno_); - td::actor::send_closure(adnl_, &adnl::Adnl::send_message, local_id_, peer_id_, serialize_tl_object(obj, true)); + auto serialized = serialize_tl_object(obj, true); + metrics_->sent_to_adnl_confirm.record(serialized.size()); + td::actor::send_closure(adnl_, &adnl::Adnl::send_message, local_id_, peer_id_, std::move(serialized)); cnt_ = 0; } } @@ -199,9 +208,9 @@ void RldpTransferReceiverImpl::alarm() { td::actor::ActorOwn RldpTransferReceiver::create( TransferId transfer_id, adnl::AdnlNodeIdShort local_id, adnl::AdnlNodeIdShort peer_id, td::uint64 total_size, td::Timestamp timeout, td::actor::ActorId rldp, td::actor::ActorId adnl, - td::Promise promise) { + td::Promise promise, std::shared_ptr metrics) { return td::actor::create_actor("receiver", transfer_id, local_id, peer_id, total_size, - timeout, rldp, adnl, std::move(promise)); + timeout, rldp, adnl, std::move(promise), std::move(metrics)); } } // namespace rldp diff --git a/rldp/rldp-peer.h b/rldp/rldp-peer.h index d5457eaf86..bf8c12aa5a 100644 --- a/rldp/rldp-peer.h +++ b/rldp/rldp-peer.h @@ -18,6 +18,8 @@ */ #pragma once +#include + #include "adnl/adnl.h" #include "fec/fec.h" @@ -26,6 +28,7 @@ namespace ton { namespace rldp { class RldpImpl; +struct RldpMetrics; using TransferId = td::Bits256; @@ -39,7 +42,8 @@ class RldpTransferSender : public td::actor::Actor { static td::actor::ActorOwn create(TransferId transfer_id, adnl::AdnlNodeIdShort local_id, adnl::AdnlNodeIdShort peer_id, td::BufferSlice data, td::Timestamp timeout, td::actor::ActorId rldp, - td::actor::ActorId adnl); + td::actor::ActorId adnl, + std::shared_ptr metrics); }; class RldpTransferReceiver : public td::actor::Actor { @@ -53,7 +57,8 @@ class RldpTransferReceiver : public td::actor::Actor { adnl::AdnlNodeIdShort peer_id, td::uint64 total_size, td::Timestamp timeout, td::actor::ActorId rldp, td::actor::ActorId adnl, - td::Promise promise); + td::Promise promise, + std::shared_ptr metrics); }; } // namespace rldp diff --git a/rldp/rldp-peer.hpp b/rldp/rldp-peer.hpp index 08b97e164b..8eaecc9f85 100644 --- a/rldp/rldp-peer.hpp +++ b/rldp/rldp-peer.hpp @@ -19,6 +19,7 @@ #pragma once #include +#include #include "fec/fec.h" @@ -51,14 +52,15 @@ class RldpTransferSenderImpl : public RldpTransferSender { RldpTransferSenderImpl(TransferId transfer_id, adnl::AdnlNodeIdShort local_id, adnl::AdnlNodeIdShort peer_id, td::BufferSlice data, td::Timestamp timeout, td::actor::ActorId rldp, - td::actor::ActorId adnl) + td::actor::ActorId adnl, std::shared_ptr metrics) : transfer_id_(transfer_id) , local_id_(local_id) , peer_id_(peer_id) , data_(std::move(data)) , timeout_(timeout) , rldp_(rldp) - , adnl_(adnl) { + , adnl_(adnl) + , metrics_(std::move(metrics)) { } private: @@ -80,6 +82,7 @@ class RldpTransferSenderImpl : public RldpTransferSender { td::Timestamp timeout_; td::actor::ActorId rldp_; td::actor::ActorId adnl_; + std::shared_ptr metrics_; }; class RldpTransferReceiverImpl : public RldpTransferReceiver { @@ -93,7 +96,8 @@ class RldpTransferReceiverImpl : public RldpTransferReceiver { RldpTransferReceiverImpl(TransferId transfer_id, adnl::AdnlNodeIdShort local_id, adnl::AdnlNodeIdShort peer_id, td::uint64 total_size, td::Timestamp timeout, td::actor::ActorId rldp, - td::actor::ActorId adnl, td::Promise promise) + td::actor::ActorId adnl, td::Promise promise, + std::shared_ptr metrics) : transfer_id_(transfer_id) , local_id_(local_id) , peer_id_(peer_id) @@ -101,7 +105,8 @@ class RldpTransferReceiverImpl : public RldpTransferReceiver { , timeout_(timeout) , rldp_(rldp) , adnl_(adnl) - , promise_(std::move(promise)) { + , promise_(std::move(promise)) + , metrics_(std::move(metrics)) { } private: @@ -127,6 +132,7 @@ class RldpTransferReceiverImpl : public RldpTransferReceiver { td::actor::ActorId adnl_; td::Promise promise_; + std::shared_ptr metrics_; }; } // namespace rldp diff --git a/rldp/rldp.cpp b/rldp/rldp.cpp index 48c60b29aa..bc4a320fa0 100644 --- a/rldp/rldp.cpp +++ b/rldp/rldp.cpp @@ -36,18 +36,14 @@ TransferId RldpIn::transfer(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst td::Random::secure_bytes(transfer_id.as_slice()); } - senders_.emplace(transfer_id, - RldpTransferSender::create(transfer_id, src, dst, std::move(data), timeout, actor_id(this), adnl_)); + metrics_->transfers_started->add(1); + senders_.emplace(transfer_id, RldpTransferSender::create(transfer_id, src, dst, std::move(data), timeout, + actor_id(this), adnl_, metrics_)); return transfer_id; } void RldpIn::send_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) { - td::Bits256 id; - td::Random::secure_bytes(id.as_slice()); - - auto B = serialize_tl_object(create_tl_object(id, std::move(data)), true); - - transfer(src, dst, td::Timestamp::in(10.0), std::move(B)); + send_message_ex(src, dst, td::Timestamp::in(10.0), std::move(data)); } void RldpIn::send_message_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, @@ -55,6 +51,7 @@ void RldpIn::send_message_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort ds td::Bits256 id; td::Random::secure_bytes(id.as_slice()); + app_metrics_->record_send("message", data.as_slice()); auto B = serialize_tl_object(create_tl_object(id, std::move(data)), true); transfer(src, dst, timeout, std::move(B)); @@ -66,6 +63,7 @@ void RldpIn::send_query_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, auto query_id = adnl::AdnlQuery::random_query_id(); auto date = static_cast(timeout.at_unix()) + 1; + app_metrics_->record_send("query", data.as_slice()); auto B = serialize_tl_object(create_tl_object(query_id, max_answer_size, date, std::move(data)), true); @@ -89,6 +87,7 @@ void RldpIn::send_query_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, void RldpIn::answer_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, adnl::AdnlQueryId query_id, TransferId transfer_id, td::BufferSlice data) { + app_metrics_->record_send("answer", data.as_slice()); auto B = serialize_tl_object(create_tl_object(query_id, std::move(data)), true); transfer(src, dst, timeout, std::move(B), transfer_id); @@ -100,13 +99,29 @@ void RldpIn::alarm_query(adnl::AdnlQueryId query_id, TransferId transfer_id) { } void RldpIn::receive_message_part(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, td::BufferSlice data) { + auto data_size = data.size(); auto F = fetch_tl_object(std::move(data), true); if (F.is_error()) { + metrics_->parse_errors_part->add(1); VLOG(RLDP_INFO) << "failed to parse rldp packet [" << source << "->" << local_id << "]: " << F.move_as_error(); return; } + auto obj = F.move_as_ok(); + switch (obj->get_id()) { + case ton_api::rldp_messagePart::ID: + metrics_->received_part.record(data_size); + break; + case ton_api::rldp_confirm::ID: + metrics_->received_confirm.record(data_size); + break; + case ton_api::rldp_complete::ID: + metrics_->received_complete.record(data_size); + break; + default: + break; + } - ton_api::downcast_call(*F.move_as_ok().get(), [&](auto &obj) { this->process_message_part(source, local_id, obj); }); + ton_api::downcast_call(*obj.get(), [&](auto &o) { this->process_message_part(source, local_id, o); }); } void RldpIn::process_message_part(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, @@ -154,9 +169,9 @@ void RldpIn::process_message_part(adnl::AdnlNodeIdShort source, adnl::AdnlNodeId td::actor::send_closure(SelfId, &RldpIn::receive_message, source, local_id, transfer_id, R.move_as_ok()); }); - receivers_.emplace(part.transfer_id_, - RldpTransferReceiver::create(part.transfer_id_, local_id, source, part.total_size_, - td::Timestamp::in(60.0), actor_id(this), adnl_, std::move(P))); + receivers_.emplace(part.transfer_id_, RldpTransferReceiver::create(part.transfer_id_, local_id, source, + part.total_size_, td::Timestamp::in(60.0), + actor_id(this), adnl_, std::move(P), metrics_)); it = receivers_.find(part.transfer_id_); } td::actor::send_closure(it->second, &RldpTransferReceiver::receive_part, F.move_as_ok(), part.part_, part.total_size_, @@ -183,6 +198,7 @@ void RldpIn::receive_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort td::BufferSlice data) { auto F = fetch_tl_object(std::move(data), true); if (F.is_error()) { + metrics_->parse_errors_message->add(1); VLOG(RLDP_INFO) << "failed to parse rldp packet [" << source << "->" << local_id << "]: " << F.move_as_error(); return; } @@ -193,11 +209,13 @@ void RldpIn::receive_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, ton_api::rldp_message &message) { + app_metrics_->record_deliver("message", message.data_.as_slice()); td::actor::send_closure(adnl_, &adnl::AdnlPeerTable::deliver, source, local_id, std::move(message.data_)); } void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, ton_api::rldp_query &message) { + app_metrics_->record_deliver("query", message.data_.as_slice()); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), source, local_id, timeout = td::Timestamp::at_unix(message.timeout_), query_id = message.query_id_, max_answer_size = static_cast(message.max_answer_size_), @@ -226,6 +244,7 @@ void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort ton_api::rldp_answer &message) { auto it = queries_.find(message.query_id_); if (it != queries_.end()) { + app_metrics_->record_deliver("answer", message.data_.as_slice()); td::actor::send_closure(it->second, &adnl::AdnlQuery::result, std::move(message.data_)); queries_.erase(it); max_size_.erase(transfer_id); @@ -235,11 +254,17 @@ void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort } void RldpIn::transfer_completed(TransferId transfer_id) { + metrics_->transfers_completed_out->add(1); senders_.erase(transfer_id); VLOG(RLDP_DEBUG) << "rldp: completed transfer " << transfer_id << "; " << senders_.size() << " out transfer pending "; } void RldpIn::in_transfer_completed(TransferId transfer_id, bool success) { + if (success) { + metrics_->transfers_completed_in->add(1); + } else { + metrics_->transfers_failed_in->add(1); + } receivers_.erase(transfer_id); if (!success || lru_set_.count(transfer_id) == 1) { return; @@ -271,6 +296,22 @@ void RldpIn::add_id(adnl::AdnlNodeIdShort local_id) { local_ids_.insert(local_id); } +void RldpIn::start_up() { + add_collector(metrics_collector_.get()); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, app_metrics_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, transfers_started_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, transfers_completed_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, transfers_failed_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, bytes_sent_to_adnl_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, parts_sent_to_adnl_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, bytes_received_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, parts_received_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, parse_errors_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, outbound_transfers_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, inbound_transfers_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, lru_size_gauge_); +} + void RldpIn::get_conn_ip_str(adnl::AdnlNodeIdShort l_id, adnl::AdnlNodeIdShort p_id, td::Promise promise) { td::actor::send_closure(adnl_, &adnl::AdnlPeerTable::get_conn_ip_str, l_id, p_id, std::move(promise)); } diff --git a/rldp/rldp.h b/rldp/rldp.h index e796d2bdd4..487ced87f3 100644 --- a/rldp/rldp.h +++ b/rldp/rldp.h @@ -24,7 +24,7 @@ namespace ton { namespace rldp { -class Rldp : public adnl::AdnlSenderEx { +class Rldp : public adnl::AdnlSenderEx, public virtual metrics::CollectorWrapper { public: Rldp() : AdnlSenderEx(adnl::Adnl::get_mtu()) { } diff --git a/rldp/rldp.hpp b/rldp/rldp.hpp index f75d0ca336..eb99dd3ab0 100644 --- a/rldp/rldp.hpp +++ b/rldp/rldp.hpp @@ -19,8 +19,10 @@ #pragma once #include +#include #include "adnl/adnl-query.h" +#include "metrics/metrics-collectors.h" #include "tl-utils/tl-utils.hpp" #include "rldp-peer.h" @@ -38,6 +40,27 @@ constexpr int VERBOSITY_NAME(RLDP_EXTRA_DEBUG) = verbosity_DEBUG + 1; using TransferId = td::Bits256; +struct RldpMetrics { + struct AtomicKindCounter { + std::shared_ptr> bytes; + std::shared_ptr> msgs; + + void record(td::uint64 size) const { + bytes->add(size); + msgs->add(1); + } + }; + using CounterPtr = std::shared_ptr>; + AtomicKindCounter sent_to_adnl_part, sent_to_adnl_confirm, sent_to_adnl_complete; + AtomicKindCounter received_part, received_confirm, received_complete; + CounterPtr transfers_started; + CounterPtr transfers_completed_out; + CounterPtr transfers_completed_in; + CounterPtr transfers_failed_in; + CounterPtr parse_errors_part; + CounterPtr parse_errors_message; +}; + class RldpImpl : public Rldp { public: virtual void transfer_completed(TransferId transfer_id) = 0; diff --git a/rldp2/CMakeLists.txt b/rldp2/CMakeLists.txt index 47860b7f01..6d3dcf42ee 100644 --- a/rldp2/CMakeLists.txt +++ b/rldp2/CMakeLists.txt @@ -51,5 +51,5 @@ if (GSL_FOUND AND BLAS_FOUND) target_link_libraries(rldp2 PRIVATE gsl blas) target_compile_definitions(rldp2 PRIVATE -DTON_HAVE_GSL=1) endif() -target_link_libraries(rldp2 PUBLIC tdutils tdactor fec adnl tl_api) +target_link_libraries(rldp2 PUBLIC tdutils tdactor fec adnl tl_api metrics) diff --git a/rldp2/rldp-in.hpp b/rldp2/rldp-in.hpp index c04fa93412..1d34b6fc67 100644 --- a/rldp2/rldp-in.hpp +++ b/rldp2/rldp-in.hpp @@ -19,10 +19,14 @@ #pragma once #include +#include #include +#include #include "adnl/adnl-peer-table.h" #include "adnl/adnl-query.h" +#include "metrics/app-traffic-metrics.h" +#include "metrics/metrics-collectors.h" #include "td/utils/List.h" #include "tl-utils/tl-utils.hpp" @@ -96,6 +100,7 @@ class RldpIn : public RldpImpl { } protected: + void start_up() override; void on_mtu_updated(td::optional local_id, td::optional peer_id) override; @@ -114,6 +119,51 @@ class RldpIn : public RldpImpl { std::set local_ids_; + metrics::MultiCollector::Own metrics_collector_ = metrics::MultiCollector::create("rldp2"); + metrics::AppTrafficMetrics::Ptr app_metrics_ = metrics::AppTrafficMetrics::make(); + metrics::AtomicCounter::Ptr bytes_sent_to_adnl_ = + metrics::AtomicCounter::make("bytes_sent_to_adnl_total", "RLDP2 serialized bytes handed to ADNL."); + metrics::AtomicCounter::Ptr parts_sent_to_adnl_ = + metrics::AtomicCounter::make("parts_sent_to_adnl_total", "RLDP2 messages handed to ADNL."); + metrics::AtomicCounter::Ptr bytes_received_from_adnl_ = metrics::AtomicCounter::make( + "bytes_received_from_adnl_total", "RLDP2 serialized bytes received from ADNL."); + metrics::AtomicCounter::Ptr parts_received_from_adnl_ = + metrics::AtomicCounter::make("parts_received_from_adnl_total", "RLDP2 messages received from ADNL."); + metrics::AtomicCounter::Ptr parse_errors_ = + metrics::AtomicCounter::make("parse_errors_total", "RLDP2 message TL parse failures."); + metrics::Labeled>::Ptr transfers_received_ = + metrics::Labeled>::make( + "result", "transfers_received_total", "Inbound RLDP2 transfers concluded (success or error)."); + metrics::Labeled>::Ptr transfers_sent_ = + metrics::Labeled>::make( + "result", "transfers_sent_total", "Outbound RLDP2 transfers concluded (success or error)."); + metrics::LambdaGauge::Ptr connections_active_ = metrics::LambdaGauge::make( + "connections_active", + [this] { + return std::vector{ + metrics::Sample{.label_set = {}, .value = static_cast(connections_.size())}}; + }, + "Active RLDP2 connections."); + metrics::LambdaGauge::Ptr queries_pending_ = metrics::LambdaGauge::make( + "queries_pending", + [this] { + return std::vector{ + metrics::Sample{.label_set = {}, .value = static_cast(queries_.size())}}; + }, + "Pending RLDP2 queries awaiting answers."); + + std::shared_ptr metrics_ = std::make_shared(Rldp2Metrics{ + .bytes_sent_to_adnl = bytes_sent_to_adnl_, + .parts_sent_to_adnl = parts_sent_to_adnl_, + .bytes_received_from_adnl = bytes_received_from_adnl_, + .parts_received_from_adnl = parts_received_from_adnl_, + .parse_errors = parse_errors_, + .transfers_received_ok = transfers_received_->label("ok"), + .transfers_received_err = transfers_received_->label("error"), + .transfers_sent_ok = transfers_sent_->label("ok"), + .transfers_sent_err = transfers_sent_->label("error"), + }); + td::actor::ActorId get_or_create_connection(adnl::AdnlNodeIdShort local_id, adnl::AdnlNodeIdShort peer_id, bool incoming, td::Timestamp timeout = {}); diff --git a/rldp2/rldp.cpp b/rldp2/rldp.cpp index 0ea414ab05..2e521d5651 100644 --- a/rldp2/rldp.cpp +++ b/rldp2/rldp.cpp @@ -36,8 +36,8 @@ struct RldpIn::Connection { class RldpConnectionActor : public td::actor::Actor, private ConnectionCallback { public: RldpConnectionActor(td::actor::ActorId rldp, adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, - td::actor::ActorId adnl) - : rldp_(std::move(rldp)), src_(src), dst_(dst), adnl_(std::move(adnl)) {}; + td::actor::ActorId adnl, std::shared_ptr metrics) + : rldp_(std::move(rldp)), src_(src), dst_(dst), adnl_(std::move(adnl)), metrics_(std::move(metrics)) {}; void send(TransferId transfer_id, td::BufferSlice query, td::Timestamp timeout = td::Timestamp::never()) { connection_.send(transfer_id, std::move(query), timeout); @@ -47,6 +47,10 @@ class RldpConnectionActor : public td::actor::Actor, private ConnectionCallback connection_.set_receive_limits(transfer_id, timeout, max_size); } void receive_raw(td::BufferSlice data) { + if (metrics_) { + metrics_->parts_received_from_adnl->add(1); + metrics_->bytes_received_from_adnl->add(data.size()); + } connection_.receive_raw(std::move(data)); yield(); } @@ -59,6 +63,7 @@ class RldpConnectionActor : public td::actor::Actor, private ConnectionCallback adnl::AdnlNodeIdShort src_; adnl::AdnlNodeIdShort dst_; td::actor::ActorId adnl_; + std::shared_ptr metrics_; RldpConnection connection_; void loop() override { @@ -66,6 +71,10 @@ class RldpConnectionActor : public td::actor::Actor, private ConnectionCallback } void send_raw(td::BufferSlice data) override { + if (metrics_) { + metrics_->parts_sent_to_adnl->add(1); + metrics_->bytes_sent_to_adnl->add(data.size()); + } send_closure(adnl_, &adnl::Adnl::send_message, src_, dst_, std::move(data)); } void receive(TransferId transfer_id, td::Result data) override { @@ -96,6 +105,7 @@ void RldpIn::send_message_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort ds td::Bits256 id; td::Random::secure_bytes(id.as_slice()); + app_metrics_->record_send("message", data.as_slice()); auto B = serialize_tl_object(create_tl_object(id, std::move(data)), true); auto transfer_id = get_random_transfer_id(); @@ -109,6 +119,7 @@ void RldpIn::send_query_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, auto query_id = adnl::AdnlQuery::random_query_id(); auto date = static_cast(timeout.at_unix()) + 1; + app_metrics_->record_send("query", data.as_slice()); auto B = serialize_tl_object(create_tl_object(query_id, max_answer_size, date, std::move(data)), true); @@ -123,6 +134,7 @@ void RldpIn::send_query_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, void RldpIn::answer_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, adnl::AdnlQueryId query_id, TransferId transfer_id, td::BufferSlice data) { + app_metrics_->record_send("answer", data.as_slice()); auto B = serialize_tl_object(create_tl_object(query_id, std::move(data)), true); send_closure(get_or_create_connection(src, dst, false, timeout), &RldpConnectionActor::send, transfer_id, @@ -157,8 +169,8 @@ td::actor::ActorId RldpIn::get_or_create_connection(adnl::A VLOG(RLDP_INFO) << "dropping incoming packet " << local_id << " <- " << peer_id << " : peer not allowed"; return {}; } - auto connection = - td::actor::create_actor("RldpConnection", actor_id(this), local_id, peer_id, adnl_); + auto connection = td::actor::create_actor("RldpConnection", actor_id(this), local_id, peer_id, + adnl_, metrics_); td::actor::send_closure(connection, &RldpConnectionActor::set_default_mtu, mtu); auto res = connection.get(); connections_[std::make_pair(local_id, peer_id)] = {std::move(connection), timeout}; @@ -172,6 +184,7 @@ td::actor::ActorId RldpIn::get_or_create_connection(adnl::A void RldpIn::receive_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, td::Result r_data) { if (r_data.is_error()) { + metrics_->transfers_received_err->add(1); auto it = queries_.find(transfer_id); if (it != queries_.end()) { it->second.set_error(r_data.move_as_error()); @@ -181,11 +194,13 @@ void RldpIn::receive_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort } return; } + metrics_->transfers_received_ok->add(1); auto data = r_data.move_as_ok(); //LOG(ERROR) << "RECEIVE MESSAGE " << data.size(); auto F = fetch_tl_object(std::move(data), true); if (F.is_error()) { + metrics_->parse_errors->add(1); VLOG(RLDP_INFO) << "failed to parse rldp packet [" << source << "->" << local_id << "]: " << F.move_as_error(); return; } @@ -196,11 +211,13 @@ void RldpIn::receive_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, ton_api::rldp_message &message) { + app_metrics_->record_deliver("message", message.data_.as_slice()); td::actor::send_closure(adnl_, &adnl::AdnlPeerTable::deliver, source, local_id, std::move(message.data_)); } void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, ton_api::rldp_query &message) { + app_metrics_->record_deliver("query", message.data_.as_slice()); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), source, local_id, timeout = td::Timestamp::at_unix(message.timeout_), query_id = message.query_id_, max_answer_size = static_cast(message.max_answer_size_), @@ -229,6 +246,7 @@ void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort ton_api::rldp_answer &message) { auto it = queries_.find(transfer_id); if (it != queries_.end()) { + app_metrics_->record_deliver("answer", message.data_.as_slice()); it->second.set_value(std::move(message.data_)); queries_.erase(it); } else { @@ -237,7 +255,11 @@ void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort } void RldpIn::on_sent(TransferId transfer_id, td::Result state) { - //TODO: completed transfer + if (state.is_ok()) { + metrics_->transfers_sent_ok->add(1); + } else { + metrics_->transfers_sent_err->add(1); + } } void RldpIn::add_id(adnl::AdnlNodeIdShort local_id) { @@ -255,6 +277,22 @@ void RldpIn::add_id(adnl::AdnlNodeIdShort local_id) { local_ids_.insert(local_id); } +void RldpIn::start_up() { + add_collector(metrics_collector_.get()); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, app_metrics_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, bytes_sent_to_adnl_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, parts_sent_to_adnl_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, + bytes_received_from_adnl_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, + parts_received_from_adnl_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, parse_errors_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, transfers_received_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, transfers_sent_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, connections_active_); + td::actor::send_closure(metrics_collector_.get(), &metrics::MultiCollector::add_sync_collector, queries_pending_); +} + void RldpIn::get_conn_ip_str(adnl::AdnlNodeIdShort l_id, adnl::AdnlNodeIdShort p_id, td::Promise promise) { td::actor::send_closure(adnl_, &adnl::AdnlPeerTable::get_conn_ip_str, l_id, p_id, std::move(promise)); } diff --git a/rldp2/rldp.h b/rldp2/rldp.h index 67bfedbbde..a3e02c6b55 100644 --- a/rldp2/rldp.h +++ b/rldp2/rldp.h @@ -24,7 +24,7 @@ namespace ton { namespace rldp2 { -class Rldp : public adnl::AdnlSenderEx { +class Rldp : public adnl::AdnlSenderEx, public virtual metrics::CollectorWrapper { public: Rldp() : AdnlSenderEx(default_mtu()) { } diff --git a/rldp2/rldp.hpp b/rldp2/rldp.hpp index 79c2ef9da4..6cf15c0aea 100644 --- a/rldp2/rldp.hpp +++ b/rldp2/rldp.hpp @@ -19,8 +19,10 @@ #pragma once #include +#include #include "adnl/adnl-query.h" +#include "metrics/metrics-collectors.h" #include "tl-utils/tl-utils.hpp" #include "rldp.h" @@ -37,6 +39,19 @@ constexpr int VERBOSITY_NAME(RLDP_EXTRA_DEBUG) = verbosity_DEBUG + 1; using TransferId = td::Bits256; +struct Rldp2Metrics { + using CounterPtr = std::shared_ptr>; + CounterPtr bytes_sent_to_adnl; + CounterPtr parts_sent_to_adnl; + CounterPtr bytes_received_from_adnl; + CounterPtr parts_received_from_adnl; + CounterPtr parse_errors; + CounterPtr transfers_received_ok; + CounterPtr transfers_received_err; + CounterPtr transfers_sent_ok; + CounterPtr transfers_sent_err; +}; + class RldpImpl : public Rldp { public: //virtual void transfer_completed(TransferId transfer_id) = 0; diff --git a/tdtl/td/tl/tl_generate.cpp b/tdtl/td/tl/tl_generate.cpp index 4e1397103b..3b44fc8e0a 100644 --- a/tdtl/td/tl/tl_generate.cpp +++ b/tdtl/td/tl/tl_generate.cpp @@ -386,6 +386,14 @@ void write_class(tl_outputer &out, const tl_type *t, const std::set out.append(w.gen_fetch_switch_end()); out.append(w.gen_fetch_function_end(false, -1, empty_vars, -1)); + + out.append(w.gen_nameof_function_begin(class_name)); + for (std::size_t j = 0; j < t->constructors_num; j++) { + if (w.is_combinator_supported(t->constructors[j])) { + out.append(w.gen_nameof_case(t->constructors[j])); + } + } + out.append(w.gen_nameof_function_end(class_name)); } std::vector storers = w.get_storers(); @@ -680,6 +688,26 @@ void write_tl(const tl_config &config, tl_outputer &out, const TL_writer &w) { } out.append(w.gen_fetch_switch_end()); out.append(w.gen_fetch_function_end(false, -1, empty_vars, -1)); + + out.append(w.gen_nameof_function_begin(w.gen_base_type_class_name(i))); + for (std::size_t type = 0; type < types_n; type++) { + tl_type *t = config.get_type_by_num(type); + if (t->constructors_num == 0 || w.is_built_in_simple_type(t->name) || w.is_built_in_complex_type(t->name) || + (t->flags & FLAG_COMPLEX)) { // built-in or complex types + continue; + } + if (t->arity != i) { // additional condition + continue; + } + + for (std::size_t k = 0; k < t->constructors_num; k++) { + if (w.is_combinator_supported(t->constructors[k]) && + is_reachable_for_parser(-1, t->constructors[k]->name, request_types, result_types, w)) { + out.append(w.gen_nameof_case(t->constructors[k])); + } + } + } + out.append(w.gen_nameof_function_end(w.gen_base_type_class_name(i))); } std::vector additional_functions = w.get_additional_functions(); @@ -747,6 +775,22 @@ void write_tl(const tl_config &config, tl_outputer &out, const TL_writer &w) { out.append(w.gen_fetch_function_end(false, -1, empty_vars, -1)); } + for (std::size_t j = 0; j < parsers.size(); j++) { + if (w.get_parser_mode(-1) == TL_writer::Client) { + continue; + } + + out.append(w.gen_nameof_function_begin(w.gen_base_function_class_name())); + for (std::size_t function = 0; function < functions_n; function++) { + tl_combinator *t = config.get_function_by_num(function); + + if (w.is_combinator_supported(t)) { + out.append(w.gen_nameof_case(t)); + } + } + out.append(w.gen_nameof_function_end(w.gen_base_function_class_name())); + } + std::vector storers = w.get_storers(); for (std::size_t j = 0; j < storers.size(); j++) { if (w.get_storer_mode(-1) == TL_writer::Server) { diff --git a/tdtl/td/tl/tl_writer.h b/tdtl/td/tl/tl_writer.h index 53d2091b87..335484a1b4 100644 --- a/tdtl/td/tl/tl_writer.h +++ b/tdtl/td/tl/tl_writer.h @@ -136,6 +136,9 @@ class TL_writer { virtual std::string gen_fetch_function_end(bool has_parent, int field_num, const std::vector &vars, int parser_type) const = 0; + virtual std::string gen_nameof_function_begin(const std::string & class_name) const { return ""; } + virtual std::string gen_nameof_function_end(const std::string & class_name) const { return ""; } + virtual std::string gen_fetch_function_result_begin(const std::string &parser_name, const std::string &class_name, const tl_tree *result) const = 0; virtual std::string gen_fetch_function_result_end() const = 0; @@ -151,6 +154,8 @@ class TL_writer { virtual std::string gen_fetch_switch_case(const tl_combinator *t, int arity) const = 0; virtual std::string gen_fetch_switch_end() const = 0; + virtual std::string gen_nameof_case(const tl::tl_combinator *t) const { return ""; } + virtual std::string gen_constructor_begin(int fields_num, const std::string &class_name, bool is_default) const = 0; virtual std::string gen_constructor_parameter(int field_num, const std::string &class_name, const arg &a, bool is_default) const = 0; diff --git a/tl/generate/generate_common.cpp b/tl/generate/generate_common.cpp index 0b9b3d1d4b..fbbd99d00f 100644 --- a/tl/generate/generate_common.cpp +++ b/tl/generate/generate_common.cpp @@ -25,10 +25,14 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include +#include #include #include #include "td/tl/tl_config.h" +#include "td/tl/tl_core.h" +#include "td/tl/tl_file_outputer.h" #include "td/tl/tl_generate.h" #include "td/tl/tl_writer.h" @@ -59,14 +63,14 @@ static void generate_cpp(const std::string &directory, const std::string &tl_nam int main() { generate_cpp("auto/tl", "ton_api", "std::string", "td::BufferSlice", "std::string", "td::BufferSlice", - {"\"tl/tl_object_parse.h\"", "\"tl/tl_object_store.h\"", "\"td/utils/int_types.h\"", + {"", "\"tl/tl_object_parse.h\"", "\"tl/tl_object_store.h\"", "\"td/utils/int_types.h\"", "\"crypto/common/bitstring.h\""}, - {"", "\"td/utils/buffer.h\"", "\"crypto/common/bitstring.h\""}); + {"", "", "\"td/utils/buffer.h\"", "\"crypto/common/bitstring.h\""}); generate_cpp("auto/tl", "lite_api", "std::string", "td::BufferSlice", "std::string", "td::BufferSlice", - {"\"tl/tl_object_parse.h\"", "\"tl/tl_object_store.h\"", "\"td/utils/int_types.h\"", + {"", "\"tl/tl_object_parse.h\"", "\"tl/tl_object_store.h\"", "\"td/utils/int_types.h\"", "\"crypto/common/bitstring.h\""}, - {"", "\"td/utils/buffer.h\"", "\"crypto/common/bitstring.h\""}); + {"", "", "\"td/utils/buffer.h\"", "\"crypto/common/bitstring.h\""}); td::gen_json_converter(td::tl::read_tl_config_from_file("scheme/ton_api.tlo"), "auto/tl/ton_api_json", "ton_api", td::tl::TL_writer::Mode::All); diff --git a/tl/generate/tl_writer_cpp.cpp b/tl/generate/tl_writer_cpp.cpp index 056f3cf71e..65047d45bd 100644 --- a/tl/generate/tl_writer_cpp.cpp +++ b/tl/generate/tl_writer_cpp.cpp @@ -582,6 +582,19 @@ std::string TD_TL_writer_cpp::gen_fetch_function_end(bool has_parent, int field_ "}\n"; } +std::string TD_TL_writer_cpp::gen_nameof_function_begin(const std::string &class_name) const { + std::string mapped_type = "int "; + std::string returned_type = "std::optional "; + return "\n" + returned_type + class_name + "::nameof(" + mapped_type + "constructor) {\n" + + " switch (constructor) {\n"; +} + +std::string TD_TL_writer_cpp::gen_nameof_function_end(const std::string &class_name) const { + return " }\n" + " return std::nullopt;\n" + "}\n"; +} + std::string TD_TL_writer_cpp::gen_fetch_function_result_begin(const std::string &parser_name, const std::string &class_name, const tl::tl_tree *result) const { @@ -663,6 +676,13 @@ std::string TD_TL_writer_cpp::gen_fetch_switch_end() const { " }\n"; } +std::string TD_TL_writer_cpp::gen_nameof_case(const tl::tl_combinator *t) const { + return " case " + gen_class_name(t->name) + + "::ID:\n" + " return \"" + + t->name + "\";\n"; +} + std::string TD_TL_writer_cpp::gen_constructor_begin(int fields_num, const std::string &class_name, bool is_default) const { return "\n" + class_name + "::" + class_name + "("; diff --git a/tl/generate/tl_writer_cpp.h b/tl/generate/tl_writer_cpp.h index 230a9093cf..5e989e8b1b 100644 --- a/tl/generate/tl_writer_cpp.h +++ b/tl/generate/tl_writer_cpp.h @@ -96,6 +96,9 @@ class TD_TL_writer_cpp : public TD_TL_writer { std::string gen_fetch_function_end(bool has_parent, int field_num, const std::vector &vars, int parser_type) const override; + std::string gen_nameof_function_begin(const std::string& class_name) const override; + std::string gen_nameof_function_end(const std::string& class_name) const override; + std::string gen_fetch_function_result_begin(const std::string &parser_name, const std::string &class_name, const tl::tl_tree *result) const override; std::string gen_fetch_function_result_end() const override; @@ -111,6 +114,8 @@ class TD_TL_writer_cpp : public TD_TL_writer { std::string gen_fetch_switch_case(const tl::tl_combinator *t, int arity) const override; std::string gen_fetch_switch_end() const override; + std::string gen_nameof_case(const tl::tl_combinator *t) const override; + std::string gen_constructor_begin(int fields_num, const std::string &class_name, bool is_default) const override; std::string gen_constructor_field_init(int field_num, const std::string &class_name, const tl::arg &a, bool is_default) const override; diff --git a/tl/generate/tl_writer_h.cpp b/tl/generate/tl_writer_h.cpp index e14cfebe4a..77ada7611c 100644 --- a/tl/generate/tl_writer_h.cpp +++ b/tl/generate/tl_writer_h.cpp @@ -307,6 +307,16 @@ std::string TD_TL_writer_h::gen_fetch_function_result_any_end(bool is_proxy) con return ""; } +std::string TD_TL_writer_h::gen_nameof_function_begin(const std::string &class_name) const { + std::string mapped_type = "int "; + std::string returned_type = "std::optional "; + return "\n static " + returned_type + "nameof(" + mapped_type + "constructor);\n"; +} + +std::string TD_TL_writer_h::gen_nameof_function_end(const std::string &class_name) const { + return ""; +} + std::string TD_TL_writer_h::gen_store_function_begin(const std::string &storer_name, const std::string &class_name, int arity, std::vector &vars, int storer_type) const { @@ -336,6 +346,10 @@ std::string TD_TL_writer_h::gen_fetch_switch_end() const { return ""; } +std::string TD_TL_writer_h::gen_nameof_case(const tl::tl_combinator *t) const { + return ""; +} + std::string TD_TL_writer_h::gen_constructor_begin(int fields_num, const std::string &class_name, bool is_default) const { return "\n" diff --git a/tl/generate/tl_writer_h.h b/tl/generate/tl_writer_h.h index 4ab306ec3f..9da40149e7 100644 --- a/tl/generate/tl_writer_h.h +++ b/tl/generate/tl_writer_h.h @@ -89,6 +89,9 @@ class TD_TL_writer_h : public TD_TL_writer { bool is_proxy) const override; std::string gen_fetch_function_result_any_end(bool is_proxy) const override; + std::string gen_nameof_function_begin(const std::string& class_name) const override; + std::string gen_nameof_function_end(const std::string& class_name) const override; + std::string gen_store_function_begin(const std::string &storer_name, const std::string &class_name, int arity, std::vector &vars, int storer_type) const override; std::string gen_store_function_end(const std::vector &vars, int storer_type) const override; @@ -97,6 +100,8 @@ class TD_TL_writer_h : public TD_TL_writer { std::string gen_fetch_switch_case(const tl::tl_combinator *t, int arity) const override; std::string gen_fetch_switch_end() const override; + std::string gen_nameof_case(const tl::tl_combinator* t) const override; + std::string gen_constructor_begin(int fields_num, const std::string &class_name, bool is_default) const override; std::string gen_constructor_field_init(int field_num, const std::string &class_name, const tl::arg &a, bool is_default) const override; diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 94bad47528..1188ae0945 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -84,6 +84,7 @@ #include "block-auto.h" #include "block-parse.h" #include "git.h" +#include "manager-disk.hpp" #if TON_USE_JEMALLOC #include @@ -2218,6 +2219,8 @@ void ValidatorEngine::start_adnl() { adnl_network_manager_ = ton::adnl::AdnlNetworkManager::create(config_.out_port); adnl_ = ton::adnl::Adnl::create(db_root_, keyring_.get()); td::actor::send_closure(adnl_, &ton::adnl::Adnl::register_network_manager, adnl_network_manager_.get()); + td::actor::send_closure(exporter_, &ton::PrometheusExporter::register_collector, adnl_.get()); + td::actor::send_closure(exporter_, &ton::PrometheusExporter::register_collector, adnl_network_manager_.get()); reload_adnl_addrs(); td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_static_nodes_from_config, std::move(adnl_static_nodes_)); started_adnl(); @@ -2308,6 +2311,8 @@ void ValidatorEngine::started_dht() { void ValidatorEngine::start_rldp() { rldp_ = ton::rldp::Rldp::create(adnl_.get()); rldp2_ = ton::rldp2::Rldp::create(adnl_.get()); + td::actor::send_closure(exporter_, &ton::PrometheusExporter::register_collector, rldp_.get()); + td::actor::send_closure(exporter_, &ton::PrometheusExporter::register_collector, rldp2_.get()); auto peer_table = td::actor::actor_dynamic_cast(adnl_.get()); CHECK(!peer_table.empty()); CHECK(!keyring_.empty()); @@ -2332,6 +2337,8 @@ void ValidatorEngine::start_overlays() { }; overlay_manager_ = ton::overlay::Overlays::create(db_root_, keyring_.get(), adnl_.get(), dht_nodes_[default_dht_node_].get(), buffer_limits); + td::actor::send_closure(exporter_, &ton::PrometheusExporter::register_collector, + overlay_manager_.get()); } started_overlays(); } @@ -2346,8 +2353,10 @@ void ValidatorEngine::start_validator() { !state_serializer_disabled_flag_); load_collator_options(); - validator_manager_ = ton::validator::ValidatorManagerFactory::create( - validator_options_, db_root_, keyring_.get(), adnl_.get(), rldp2_.get(), quic_.get(), overlay_manager_.get()); + validator_manager_ = ton::validator::ValidatorManagerFactory::create(validator_options_, db_root_, keyring_.get(), + adnl_.get(), rldp2_.get(), quic_.get(), + overlay_manager_.get()); + td::actor::send_closure(exporter_.get(), &ton::PrometheusExporter::register_collector, validator_manager_.get()); for (auto &v : config_.validators) { td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_permanent_key, v.first, diff --git a/validator/full-node-custom-overlays.cpp b/validator/full-node-custom-overlays.cpp index 8db3afeefc..fbc386f200 100644 --- a/validator/full-node-custom-overlays.cpp +++ b/validator/full-node-custom-overlays.cpp @@ -85,7 +85,8 @@ void FullNodeCustomOverlay::process_block_broadcast(PublicKeyHash src, ton_api:: } VLOG(FULL_NODE_DEBUG) << "Received block broadcast " << (B.ok().sig_set->is_final() ? "" : "(approve signatures) ") << "in custom overlay \"" << name_ << "\" from " << src << ": " << B.ok().block_id.to_str(); - td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok(), false); + td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok(), false, + BroadcastSource::custom_overlay); } void FullNodeCustomOverlay::obtain_state_for_decompression(PublicKeyHash src, @@ -122,7 +123,8 @@ void FullNodeCustomOverlay::process_block_broadcast_with_state(PublicKeyHash src } VLOG(FULL_NODE_DEBUG) << "Received block broadcast in custom overlay \"" << name_ << "\" from " << src << ": " << B.ok().block_id.to_str(); - td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok(), true); + td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok(), true, + BroadcastSource::custom_overlay); } void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query) { @@ -180,7 +182,7 @@ void FullNodeCustomOverlay::process_block_candidate_broadcast(PublicKeyHash src, VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in custom overlay \"" << name_ << "\" from " << src << ": " << block_id.to_str(); td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, cc_seqno, - validator_set_hash, std::move(data)); + validator_set_hash, std::move(data), BroadcastSource::custom_overlay); } void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query) { diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index dae83955a6..3f192bdb5e 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -76,7 +76,8 @@ void FullNodeFastSyncOverlay::process_block_broadcast(PublicKeyHash src, ton_api } VLOG(FULL_NODE_DEBUG) << "Received block broadcast " << (B.ok().sig_set->is_final() ? "" : "(approve signatures) ") << "in fast sync overlay from " << src << ": " << B.ok().block_id.to_str(); - td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok(), false); + td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok(), false, + BroadcastSource::fast_sync_overlay); } void FullNodeFastSyncOverlay::obtain_state_for_decompression(PublicKeyHash src, @@ -113,7 +114,8 @@ void FullNodeFastSyncOverlay::process_block_broadcast_with_state(PublicKeyHash s VLOG(FULL_NODE_DEBUG) << "Received V2 block broadcast in fast sync overlay from " << src << ": " << B.ok().block_id.to_str(); - td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok(), true); + td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok(), true, + BroadcastSource::fast_sync_overlay); } void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_outMsgQueueProofBroadcast &query) { @@ -191,7 +193,7 @@ void FullNodeFastSyncOverlay::process_block_candidate_broadcast(PublicKeyHash sr } VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in fast sync overlay from " << src << ": " << block_id.to_str(); td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, cc_seqno, - validator_set_hash, std::move(data)); + validator_set_hash, std::move(data), BroadcastSource::fast_sync_overlay); } void FullNodeFastSyncOverlay::process_telemetry_broadcast( diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index bd73a88c48..4bcf7d7226 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -248,7 +248,7 @@ void FullNodeShardImpl::get_next_block() { auto P = td::PromiseCreator::lambda([SelfId](td::Result R) { td::actor::send_closure(SelfId, &FullNodeShardImpl::got_next_block, std::move(R)); }); - td::actor::send_closure(validator_manager, &ValidatorManagerInterface::validate_block, R.move_as_ok(), + td::actor::send_closure(validator_manager, &ValidatorManagerInterface::got_next_masterchain_block, R.move_as_ok(), std::move(P)); } else { auto S = R.move_as_error(); @@ -819,7 +819,7 @@ void FullNodeShardImpl::process_block_candidate_broadcast(PublicKeyHash src, ton } VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate from " << src << ": " << block_id.to_str(); td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, cc_seqno, - validator_set_hash, std::move(data)); + validator_set_hash, std::move(data), BroadcastSource::public_overlay); } void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { @@ -869,7 +869,8 @@ void FullNodeShardImpl::process_block_broadcast(PublicKeyHash src, ton_api::tonN //} VLOG(FULL_NODE_DEBUG) << "Received block broadcast " << (B.ok().sig_set->is_final() ? "" : "(approve signatures) ") << "from " << src << ": " << B.ok().block_id.to_str(); - td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok(), false); + td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok(), false, + BroadcastSource::public_overlay); } void FullNodeShardImpl::obtain_state_for_decompression(PublicKeyHash src, @@ -905,7 +906,8 @@ void FullNodeShardImpl::process_block_broadcast_with_state(PublicKeyHash src, return; } VLOG(FULL_NODE_DEBUG) << "Received block broadcast from " << src << ": " << B.ok().block_id.to_str(); - td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok(), true); + td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok(), true, + BroadcastSource::public_overlay); } void FullNodeShardImpl::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { diff --git a/validator/full-node.cpp b/validator/full-node.cpp index ddbc1a0d2f..fc0ad4e7ee 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -606,10 +606,10 @@ void FullNodeImpl::new_key_block(BlockHandle handle) { } } -void FullNodeImpl::process_block_broadcast(BlockBroadcast broadcast, bool signatures_checked) { +void FullNodeImpl::process_block_broadcast(BlockBroadcast broadcast, bool signatures_checked, BroadcastSource source) { send_block_broadcast_to_custom_overlays(broadcast); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_block_broadcast, std::move(broadcast), - signatures_checked, [](td::Result R) { + signatures_checked, source, td::PromiseCreator::lambda([](td::Result R) { if (R.is_error()) { if (R.error().code() == ErrorCode::notready) { LOG(DEBUG) << "dropped broadcast: " << R.move_as_error(); @@ -617,14 +617,15 @@ void FullNodeImpl::process_block_broadcast(BlockBroadcast broadcast, bool signat LOG(INFO) << "dropped broadcast: " << R.move_as_error(); } } - }); + })); } void FullNodeImpl::process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, - td::uint32 validator_set_hash, td::BufferSlice data) { + td::uint32 validator_set_hash, td::BufferSlice data, + BroadcastSource source) { send_block_candidate_broadcast_to_custom_overlays(block_id, cc_seqno, validator_set_hash, data); td::actor::ask(validator_manager_, &ValidatorManagerInterface::new_block_candidate_broadcast, block_id, cc_seqno, - std::move(data)) + std::move(data), source) .detach(); } diff --git a/validator/full-node.h b/validator/full-node.h index be7d817457..4a84cb31f8 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -102,9 +102,10 @@ class FullNode : public td::actor::Actor { virtual void add_custom_overlay(CustomOverlayParams params, td::Promise promise) = 0; virtual void del_custom_overlay(std::string name, td::Promise promise) = 0; - virtual void process_block_broadcast(BlockBroadcast broadcast, bool signatures_checked = false) = 0; + virtual void process_block_broadcast(BlockBroadcast broadcast, bool signatures_checked, BroadcastSource source) = 0; virtual void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, - td::uint32 validator_set_hash, td::BufferSlice data) = 0; + td::uint32 validator_set_hash, td::BufferSlice data, + BroadcastSource source) = 0; virtual void process_shard_block_info_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; virtual void get_out_msg_queue_query_token(td::Promise> promise) = 0; diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 5c1ec59011..74ca57251c 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -94,9 +94,9 @@ class FullNodeImpl : public FullNode { void got_key_block_config(td::Ref config); void new_key_block(BlockHandle handle); - void process_block_broadcast(BlockBroadcast broadcast, bool signatures_checked = false) override; + void process_block_broadcast(BlockBroadcast broadcast, bool signatures_checked, BroadcastSource source) override; void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) override; + td::BufferSlice data, BroadcastSource source) override; void process_shard_block_info_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; void get_out_msg_queue_query_token(td::Promise> promise) override; diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index f7b95cc533..ba7547b7ed 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -936,6 +936,7 @@ void AcceptBlockQuery::written_block_next() { void AcceptBlockQuery::written_block_info_2() { VLOG(VALIDATOR_DEBUG) << "written_block_info_2()"; + td::actor::send_closure(manager_, &ValidatorManager::on_block_accepted, id_); if (handle_->id().is_masterchain()) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { check_send_error(SelfId, R) || td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::applied); diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 0ac883cd82..4a0ba6eef4 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -347,8 +347,12 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void set_next_block(BlockIdExt prev, BlockIdExt next, td::Promise promise) = 0; virtual void new_block(BlockHandle handle, td::Ref state, td::Promise promise) = 0; + virtual void on_block_accepted(BlockIdExt block_id) { + } - virtual void send_get_block_request(BlockIdExt id, td::uint32 priority, td::Promise promise) = 0; + virtual td::actor::Task send_get_block_request(BlockIdExt id, td::uint32 priority) { + co_return td::Status::Error("not implemented"); + } virtual void send_get_zero_state_request(BlockIdExt id, td::uint32 priority, td::Promise promise) = 0; virtual void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 12326d7149..5cf6983940 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -60,12 +60,7 @@ void ValidatorManagerImpl::validate_block_proof_link(BlockIdExt block_id, td::Bu UNREACHABLE(); } -void ValidatorManagerImpl::validate_block(ReceivedBlock block, td::Promise promise) { - UNREACHABLE(); -} - -void ValidatorManagerImpl::new_block_broadcast(BlockBroadcast broadcast, bool signatures_checked, - td::Promise promise) { +void ValidatorManagerImpl::got_next_masterchain_block(ReceivedBlock block, td::Promise promise) { UNREACHABLE(); } @@ -962,11 +957,6 @@ void ValidatorManagerImpl::get_last_liteserver_state_block( return get_top_masterchain_state_block(std::move(promise)); } -void ValidatorManagerImpl::send_get_block_request(BlockIdExt id, td::uint32 priority, - td::Promise promise) { - UNREACHABLE(); -} - void ValidatorManagerImpl::send_get_zero_state_request(BlockIdExt id, td::uint32 priority, td::Promise promise) { UNREACHABLE(); diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index faf536b574..b5b573c9ec 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -96,8 +96,7 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise promise) override { UNREACHABLE(); } - void validate_block(ReceivedBlock block, td::Promise promise) override; - void new_block_broadcast(BlockBroadcast broadcast, bool signatures_checked, td::Promise promise) override; + void got_next_masterchain_block(ReceivedBlock block, td::Promise promise) override; void validate_block_broadcast_signatures(BlockBroadcast broadcast, td::Promise promise) override; void wait_state_by_prev_blocks(BlockIdExt block_id, std::vector prev_blocks, td::Promise> promise) override; @@ -248,7 +247,6 @@ class ValidatorManagerImpl : public ValidatorManager { void get_top_masterchain_state_block(td::Promise, BlockIdExt>> promise) override; void get_last_liteserver_state_block(td::Promise, BlockIdExt>> promise) override; - void send_get_block_request(BlockIdExt id, td::uint32 priority, td::Promise promise) override; void send_get_zero_state_request(BlockIdExt id, td::uint32 priority, td::Promise promise) override; void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, td::uint32 priority, td::Promise promise) override; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 376801bbb0..d864f0d425 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -105,10 +105,7 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise promise) override { UNREACHABLE(); } - void validate_block(ReceivedBlock block, td::Promise promise) override { - UNREACHABLE(); - } - void new_block_broadcast(BlockBroadcast broadcast, bool signatures_checked, td::Promise promise) override { + void got_next_masterchain_block(ReceivedBlock block, td::Promise promise) override { UNREACHABLE(); } void wait_state_by_prev_blocks(BlockIdExt block_id, std::vector prev_blocks, @@ -319,9 +316,6 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } - void send_get_block_request(BlockIdExt id, td::uint32 priority, td::Promise promise) override { - UNREACHABLE(); - } void send_get_zero_state_request(BlockIdExt id, td::uint32 priority, td::Promise promise) override { UNREACHABLE(); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 9f07081fa8..257527d271 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -195,7 +195,7 @@ void ValidatorManagerImpl::validate_block_proof_rel(BlockIdExt block_id, BlockId } } -void ValidatorManagerImpl::validate_block(ReceivedBlock block, td::Promise promise) { +void ValidatorManagerImpl::got_next_masterchain_block(ReceivedBlock block, td::Promise promise) { auto blkid = block.id; auto pp = create_block(std::move(block)); if (pp.is_error()) { @@ -203,6 +203,7 @@ void ValidatorManagerImpl::validate_block(ReceivedBlock block, td::Promise R) mutable { @@ -215,30 +216,39 @@ void ValidatorManagerImpl::validate_block(ReceivedBlock block, td::Promise promise) { +td::actor::Task<> ValidatorManagerImpl::new_block_broadcast(BlockBroadcast broadcast, bool signatures_checked, + BroadcastSource source) { if (!started_) { - promise.set_error(td::Status::Error(ErrorCode::notready, "node not started")); - return; + co_return td::Status::Error(ErrorCode::notready, "node not started"); } if (!need_monitor(broadcast.block_id.shard_full())) { - promise.set_error(td::Status::Error("not monitoring shard")); - return; + co_return td::Status::Error("not monitoring shard"); } - promise = [SelfId = actor_id(this), promise = std::move(promise), block_id = broadcast.block_id, - cc_seqno = broadcast.sig_set->get_catchain_seqno(), - is_final = broadcast.sig_set->is_final()](td::Result R) mutable { - if (R.is_ok() && is_final) { - td::actor::ask(SelfId, &ValidatorManagerImpl::validated_accepted_block_broadcast, block_id, cc_seqno).detach(); - } - promise.set_result(std::move(R)); - }; BlockIdExt block_id = broadcast.block_id; + switch (source) { + case BroadcastSource::public_overlay: + update_block_receive_stats(block_id, BlockReceiveStats::block_broadcast_public); + break; + case BroadcastSource::fast_sync_overlay: + update_block_receive_stats(block_id, BlockReceiveStats::block_broadcast_fast_sync); + break; + case BroadcastSource::custom_overlay: + update_block_receive_stats(block_id, BlockReceiveStats::block_broadcast_custom); + break; + } + bool is_final = broadcast.sig_set->is_final(); + CatchainSeqno cc_seqno = broadcast.sig_set->get_catchain_seqno(); + auto [task, promise] = td::actor::StartedTask<>::make_bridge(); td::actor::create_actor(PSTRING() << "broadcast" << block_id.id.to_str(), std::move(broadcast), last_masterchain_block_handle_, last_masterchain_state_, last_known_key_block_handle_, actor_id(this), td::Timestamp::in(20.0), std::move(promise), false, signatures_checked) .release(); + co_await std::move(task); + if (is_final) { + validated_accepted_block_broadcast(block_id, cc_seqno).start().detach(); + } + co_return {}; } void ValidatorManagerImpl::validate_block_broadcast_signatures(BlockBroadcast broadcast, @@ -505,7 +515,7 @@ void ValidatorManagerImpl::new_shard_block_description_broadcast(BlockIdExt bloc } td::actor::Task<> ValidatorManagerImpl::new_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, - td::BufferSlice data) { + td::BufferSlice data, BroadcastSource source) { if (!last_masterchain_block_handle_ || !started_) { VLOG(VALIDATOR_DEBUG) << "dropping top shard block broadcast: not inited"; co_return td::Unit{}; @@ -515,6 +525,17 @@ td::actor::Task<> ValidatorManagerImpl::new_block_candidate_broadcast(BlockIdExt co_return td::Unit{}; } add_cached_block_data(block_id, data.clone()); + switch (source) { + case BroadcastSource::public_overlay: + update_block_receive_stats(block_id, BlockReceiveStats::candidate_broadcast_public); + break; + case BroadcastSource::fast_sync_overlay: + update_block_receive_stats(block_id, BlockReceiveStats::candidate_broadcast_fast_sync); + break; + case BroadcastSource::custom_overlay: + update_block_receive_stats(block_id, BlockReceiveStats::candidate_broadcast_custom); + break; + } if (opts_->nonfinal_ls_queries_enabled()) { auto handle = co_await td::actor::ask(actor_id(this), &ValidatorManagerImpl::get_block_handle, block_id, true); @@ -1468,6 +1489,7 @@ void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate can td::Promise promise) { if (!id.is_masterchain()) { add_cached_block_data(id, candidate.data.clone()); + update_block_receive_stats(id, BlockReceiveStats::candidate_stored); } if (cache_only) { promise.set_value(td::Unit{}); @@ -1602,6 +1624,25 @@ void ValidatorManagerImpl::new_block_cont(BlockHandle handle, td::Ref(create_tl_block_id(handle->id()))) .detach(); } + + auto &receive_stats = block_receive_stats_.get(handle->id()); + if (!receive_stats.applied && started_) { + receive_stats.applied = true; + auto wc_label = handle->id().is_masterchain() ? "-1" : "0"; + new_total_->label(wc_label)->add(1); + first_received_from_total_->label(wc_label) + ->label(BlockReceiveStats::TYPE_NAMES[(int)receive_stats.get_earliest_type()]) + ->add(1); + for (size_t i = 0; i < BlockReceiveStats::N_TYPES; ++i) { + if (receive_stats.received_at[i]) { + received_from_total_->label(wc_label)->label(BlockReceiveStats::TYPE_NAMES[i])->add(1); + } + } + } +} + +void ValidatorManagerImpl::on_block_accepted(BlockIdExt block_id) { + update_block_receive_stats(block_id, BlockReceiveStats::block_accepted); } void ValidatorManagerImpl::new_block(BlockHandle handle, td::Ref state, td::Promise promise) { @@ -1744,13 +1785,16 @@ void ValidatorManagerImpl::get_last_liteserver_state_block( } } -void ValidatorManagerImpl::send_get_block_request(BlockIdExt id, td::uint32 priority, - td::Promise promise) { +td::actor::Task ValidatorManagerImpl::send_get_block_request(BlockIdExt id, td::uint32 priority) { if (auto cached = cached_block_data_.get_if_exists(id, false)) { LOG(DEBUG) << "send_get_block_request: got result from block data cache for " << id.to_str(); - return promise.set_value(ReceivedBlock{id, cached->clone()}); + co_return ReceivedBlock{id, cached->clone()}; } + auto [task, promise] = td::actor::StartedTask::make_bridge(); callback_->download_block(id, priority, td::Timestamp::in(10.0), std::move(promise)); + auto result = co_await std::move(task); + update_block_receive_stats(id, BlockReceiveStats::block_download); + co_return result; } void ValidatorManagerImpl::send_get_zero_state_request(BlockIdExt id, td::uint32 priority, @@ -1891,7 +1935,6 @@ void ValidatorManagerImpl::get_block_proof_link_from_import(BlockIdExt block_id, } namespace { - class StatsCallback final : public ton::stats::Callback { public: StatsCallback(td::actor::ActorId manager) : manager_(manager) { @@ -1923,6 +1966,9 @@ class StatsCallback final : public ton::stats::Callback { void ValidatorManagerImpl::start_up() { stats::install_callback(std::make_unique(actor_id(this))); + td::actor::send_closure(collector_, &metrics::MultiCollector::add_sync_collector, received_from_total_); + td::actor::send_closure(collector_, &metrics::MultiCollector::add_sync_collector, first_received_from_total_); + td::actor::send_closure(collector_, &metrics::MultiCollector::add_sync_collector, new_total_); db_ = create_db_actor(actor_id(this), db_root_, opts_); actor_stats_ = td::actor::create_actor("actor_stats"); @@ -3794,6 +3840,18 @@ void ValidatorManagerImpl::cleanup_nonfinal_groups() { } } +void ValidatorManagerImpl::update_block_receive_stats(BlockIdExt block_id, BlockReceiveStats::Type type) { + auto &stats = block_receive_stats_.get(block_id); + auto idx = (int)type; + if (stats.received_at[idx]) { + return; + } + stats.received_at[idx] = td::Timestamp::now(); + if (stats.applied) { + new_total_->label(block_id.is_masterchain() ? "-1" : "0")->add(1); + } +} + } // namespace validator } // namespace ton diff --git a/validator/manager.h b/validator/manager.h index cccb32d08e..0793841688 100644 --- a/validator/manager.h +++ b/validator/manager.h @@ -18,6 +18,7 @@ */ #pragma once #include "adnl/adnl.h" +#include "metrics/prometheus-exporter.h" #include "quic/quic-sender.h" #include "rldp2/rldp.h" #include "validator/validator.h" diff --git a/validator/manager.hpp b/validator/manager.hpp index 48e0fcb52b..8a4d9fe2fe 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -74,7 +74,7 @@ class BlockHandleLru : public td::ListNode { BlockHandle handle_; }; -class ValidatorManagerImpl : public ValidatorManager { +class ValidatorManagerImpl : public ValidatorManager, public virtual metrics::CollectorWrapper { private: // WAITERS // @@ -315,8 +315,9 @@ class ValidatorManagerImpl : public ValidatorManager { void validate_block_proof_link(BlockIdExt block_id, td::BufferSlice proof, td::Promise promise) override; void validate_block_proof_rel(BlockIdExt block_id, BlockIdExt rel_block_id, td::BufferSlice proof, td::Promise promise) override; - void validate_block(ReceivedBlock block, td::Promise promise) override; - void new_block_broadcast(BlockBroadcast broadcast, bool signatures_checked, td::Promise promise) override; + void got_next_masterchain_block(ReceivedBlock block, td::Promise promise) override; + td::actor::Task<> new_block_broadcast(BlockBroadcast broadcast, bool signatures_checked, + BroadcastSource source) override; void validate_block_broadcast_signatures(BlockBroadcast broadcast, td::Promise promise) override; td::actor::Task<> validated_accepted_block_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno); @@ -351,8 +352,8 @@ class ValidatorManagerImpl : public ValidatorManager { void new_ihr_message(td::BufferSlice data) override; void new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; - td::actor::Task<> new_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, - td::BufferSlice data) override; + td::actor::Task<> new_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data, + BroadcastSource source) override; void add_ext_server_id(adnl::AdnlNodeIdShort id) override; void add_ext_server_port(td::uint16 port) override; @@ -457,12 +458,13 @@ class ValidatorManagerImpl : public ValidatorManager { void new_block(BlockHandle handle, td::Ref state, td::Promise promise) override; void new_block_cont(BlockHandle handle, td::Ref state, td::Promise promise); + void on_block_accepted(BlockIdExt block_id) override; void get_top_masterchain_state(td::Promise> promise) override; void get_top_masterchain_block(td::Promise promise) override; void get_top_masterchain_state_block(td::Promise, BlockIdExt>> promise) override; void get_last_liteserver_state_block(td::Promise, BlockIdExt>> promise) override; - void send_get_block_request(BlockIdExt id, td::uint32 priority, td::Promise promise) override; + td::actor::Task send_get_block_request(BlockIdExt id, td::uint32 priority) override; void send_get_zero_state_request(BlockIdExt id, td::uint32 priority, td::Promise promise) override; void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, td::uint32 priority, td::Promise promise) override; @@ -542,6 +544,7 @@ class ValidatorManagerImpl : public ValidatorManager { , rldp2_(rldp2) , quic_(quic) , overlays_(overlays) { + add_collector(collector_.get()); } public: @@ -766,6 +769,59 @@ class ValidatorManagerImpl : public ValidatorManager { bool is_valid_nonfinal_group(ShardIdFull shard, CatchainSeqno cc_seqno); void process_accepted_nonfinal_block(BlockIdExt block_id, CatchainSeqno cc_seqno); void cleanup_nonfinal_groups(); + + struct BlockReceiveStats { + enum Type { + block_broadcast_public = 0, + block_broadcast_fast_sync = 1, + block_broadcast_custom = 2, + block_download = 3, + candidate_broadcast_public = 4, + candidate_broadcast_fast_sync = 5, + candidate_broadcast_custom = 6, + candidate_stored = 7, + block_accepted = 8, + unknown = 9 + }; + static constexpr size_t N_TYPES = 10; + static constexpr const char *TYPE_NAMES[] = {"block_broadcast_public", + "block_broadcast_fast_sync", + "block_broadcast_custom", + "block_download", + "candidate_broadcast_public", + "candidate_broadcast_fast_sync", + "candidate_broadcast_custom", + "candidate_stored", + "block_accepted", + "unknown"}; + td::Timestamp received_at[N_TYPES]; + bool applied = false; + + Type get_earliest_type() const { + Type result = unknown; + td::Timestamp result_ts; + for (size_t i = 0; i < N_TYPES; i++) { + if (received_at[i] && (!result_ts || received_at[i] < result_ts)) { + result = (Type)i; + result_ts = received_at[i]; + } + } + return result; + } + }; + td::LRUCache block_receive_stats_{1000}; + + td::actor::ActorOwn collector_ = metrics::MultiCollector::create("blocks"); + metrics::MultiLabeled, std::string, std::string>::Ptr first_received_from_total_ = + metrics::MultiLabeled, std::string, std::string>::make( + "wc", "type", "first_received_from_total"); + metrics::MultiLabeled, std::string, std::string>::Ptr received_from_total_ = + metrics::MultiLabeled, std::string, std::string>::make("wc", "type", + "received_from_total"); + metrics::MultiLabeled, std::string>::Ptr new_total_ = + metrics::MultiLabeled, std::string>::make("wc", "new_total"); + + void update_block_receive_stats(BlockIdExt block_id, BlockReceiveStats::Type type); }; } // namespace validator diff --git a/validator/types.h b/validator/types.h index 3b096748a4..cbe96e39b5 100644 --- a/validator/types.h +++ b/validator/types.h @@ -40,4 +40,6 @@ struct BlockBroadcast { } }; +enum class BroadcastSource { public_overlay, fast_sync_overlay, custom_overlay }; + } // namespace ton::validator diff --git a/validator/validator.h b/validator/validator.h index 049d3857c2..6d5cb90bad 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -38,6 +38,8 @@ #include "td/actor/coro_utils.h" #include "ton/ton-types.h" +#include "metrics/metrics-collectors.h" + #include "types.h" namespace ton { @@ -237,7 +239,7 @@ struct ValidatorManagerOptions : public td::CntObject { size_t max_mempool_num = 999999, bool initial_sync_disabled = false); }; -class ValidatorManagerInterface : public td::actor::Actor { +class ValidatorManagerInterface : public td::actor::Actor, public virtual metrics::CollectorWrapper { public: class Callback { public: @@ -305,9 +307,11 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void validate_block_proof_link(BlockIdExt block_id, td::BufferSlice proof, td::Promise promise) = 0; virtual void validate_block_proof_rel(BlockIdExt block_id, BlockIdExt rel_block_id, td::BufferSlice proof, td::Promise promise) = 0; - virtual void validate_block(ReceivedBlock block, td::Promise promise) = 0; - virtual void new_block_broadcast(BlockBroadcast broadcast, bool signatures_checked, - td::Promise promise) = 0; + virtual void got_next_masterchain_block(ReceivedBlock block, td::Promise promise) = 0; + virtual td::actor::Task<> new_block_broadcast(BlockBroadcast broadcast, bool signatures_checked, + BroadcastSource source) { + co_return td::Status::Error("not implemented"); + } virtual void validate_block_broadcast_signatures(BlockBroadcast broadcast, td::Promise promise) = 0; //virtual void create_validate_block(BlockId block, td::BufferSlice data, td::Promise promise) = 0; @@ -350,7 +354,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; virtual td::actor::Task<> new_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, - td::BufferSlice data) { + td::BufferSlice data, BroadcastSource source) { co_return td::Unit{}; }