diff --git a/tdutils/td/utils/buffer.cpp b/tdutils/td/utils/buffer.cpp index a8b9b5924b..6215d17672 100644 --- a/tdutils/td/utils/buffer.cpp +++ b/tdutils/td/utils/buffer.cpp @@ -22,6 +22,11 @@ #include "td/utils/buffer.h" #include "td/utils/port/thread_local.h" +#if TD_PORT_POSIX +#include +#include +#endif + // fixes https://bugs.llvm.org/show_bug.cgi?id=33723 for clang >= 3.6 + c++11 + libc++ #if TD_CLANG && _LIBCPP_VERSION #define TD_OFFSETOF __builtin_offsetof @@ -95,13 +100,46 @@ BufferAllocator::ReaderPtr BufferAllocator::create_reader(const ReaderPtr &raw) void BufferAllocator::dec_ref_cnt(BufferRaw *ptr) { int left = ptr->ref_cnt_.fetch_sub(1, std::memory_order_acq_rel); if (left == 1) { - auto buf_size = max(sizeof(BufferRaw), TD_OFFSETOF(BufferRaw, data_) + ptr->data_size_); - buffer_mem -= buf_size; + if (ptr->external_data_) { +#if TD_PORT_POSIX + munmap(ptr->external_data_, ptr->data_size_); +#endif + // Header was allocated with sizeof(BufferRaw); trailing data_[1] unused. + buffer_mem -= sizeof(BufferRaw); + } else { + auto buf_size = max(sizeof(BufferRaw), TD_OFFSETOF(BufferRaw, data_) + ptr->data_size_); + buffer_mem -= buf_size; + } ptr->~BufferRaw(); delete[] ptr; } } +BufferAllocator::ReaderPtr BufferAllocator::create_reader_mmap(int fd, size_t size) { +#if TD_PORT_POSIX + void *addr = mmap(nullptr, size, PROT_READ, MAP_PRIVATE, fd, 0); + if (addr == MAP_FAILED) { + return ReaderPtr(); + } + // Advise kernel: we will read sequentially and do not need pages after that. + // This helps the kernel decide when to drop pages under memory pressure. + ::madvise(addr, size, MADV_SEQUENTIAL); + + buffer_mem += sizeof(BufferRaw); + auto *buffer_raw = reinterpret_cast(new char[sizeof(BufferRaw)]); + new (buffer_raw) BufferRaw(size, static_cast(addr)); + // Reader-only: mark fully-written, no writer. + buffer_raw->end_.store(size, std::memory_order_relaxed); + buffer_raw->was_reader_ = true; + buffer_raw->has_writer_.store(false, std::memory_order_release); + return ReaderPtr(buffer_raw); +#else + (void)fd; + (void)size; + return ReaderPtr(); +#endif +} + BufferRaw *BufferAllocator::create_buffer_raw(size_t size) { size = (size + 7) & -8; diff --git a/tdutils/td/utils/buffer.h b/tdutils/td/utils/buffer.h index 7bd69c0bb7..25c83a6bdc 100644 --- a/tdutils/td/utils/buffer.h +++ b/tdutils/td/utils/buffer.h @@ -31,6 +31,11 @@ namespace td { struct BufferRaw { explicit BufferRaw(size_t size) : data_size_(size) { } + // mmap-backed variant: data lives at `external`, not in the flexible array. + // Lifetime of the external region is owned by this BufferRaw via dec_ref_cnt. + BufferRaw(size_t size, unsigned char *external) : data_size_(size), external_data_(external) { + } + size_t data_size_; // Constant after first reader is created. @@ -46,7 +51,19 @@ struct BufferRaw { std::atomic has_writer_{true}; bool was_reader_{false}; + // If non-null, this buffer is mmap-backed and data_ is unused. + // dec_ref_cnt will munmap(external_data_, data_size_) on last reference. + unsigned char *external_data_ = nullptr; + alignas(4) unsigned char data_[1]; + + // Effective data pointer: external region if mmap-backed, else inline flexible array. + unsigned char *data() { + return external_data_ ? external_data_ : data_; + } + const unsigned char *data() const { + return external_data_ ? external_data_ : data_; + } }; class BufferAllocator { @@ -74,6 +91,11 @@ class BufferAllocator { static ReaderPtr create_reader(size_t size); + // Create a read-only BufferSlice backed by mmap() of the given file descriptor. + // The fd may be closed after this call; mmap retains its own reference to the file. + // Returns null ReaderPtr on mmap failure. + static ReaderPtr create_reader_mmap(int fd, size_t size); + static ReaderPtr create_reader(const WriterPtr &raw); static ReaderPtr create_reader(const ReaderPtr &raw); @@ -153,7 +175,7 @@ class BufferSlice { if (is_null()) { return Slice(); } - return Slice(buffer_->data_ + begin_, size()); + return Slice(buffer_->data() + begin_, size()); } operator Slice() const { @@ -164,7 +186,7 @@ class BufferSlice { if (is_null()) { return MutableSlice(); } - return MutableSlice(buffer_->data_ + begin_, size()); + return MutableSlice(buffer_->data() + begin_, size()); } Slice prepare_read() const { @@ -191,8 +213,8 @@ class BufferSlice { BufferSlice from_slice(Slice slice) const { auto res = BufferSlice(BufferAllocator::create_reader(buffer_)); - res.begin_ = static_cast(slice.ubegin() - buffer_->data_); - res.end_ = static_cast(slice.uend() - buffer_->data_); + res.begin_ = static_cast(slice.ubegin() - buffer_->data()); + res.end_ = static_cast(slice.uend() - buffer_->data()); CHECK(buffer_->begin_ <= res.begin_); CHECK(res.begin_ <= res.end_); CHECK(res.end_ <= buffer_->end_.load(std::memory_order_relaxed)); @@ -301,14 +323,14 @@ class BufferWriter { return MutableSlice(); } auto end = buffer_->end_.load(std::memory_order_relaxed); - return MutableSlice(buffer_->data_ + buffer_->begin_, buffer_->data_ + end); + return MutableSlice(buffer_->data() + buffer_->begin_, buffer_->data() + end); } Slice as_slice() const { if (is_null()) { return Slice(); } auto end = buffer_->end_.load(std::memory_order_relaxed); - return Slice(buffer_->data_ + buffer_->begin_, buffer_->data_ + end); + return Slice(buffer_->data() + buffer_->begin_, buffer_->data() + end); } MutableSlice prepare_prepend() { @@ -316,14 +338,14 @@ class BufferWriter { return MutableSlice(); } CHECK(!buffer_->was_reader_); - return MutableSlice(buffer_->data_, buffer_->begin_); + return MutableSlice(buffer_->data(), buffer_->begin_); } MutableSlice prepare_append() { if (is_null()) { return MutableSlice(); } auto end = buffer_->end_.load(std::memory_order_relaxed); - return MutableSlice(buffer_->data_ + end, buffer_->data_size_ - end); + return MutableSlice(buffer_->data() + end, buffer_->data_size_ - end); } void confirm_append(size_t size) { if (is_null()) { diff --git a/tdutils/td/utils/filesystem.cpp b/tdutils/td/utils/filesystem.cpp index 975a4c1055..88a92e8aca 100644 --- a/tdutils/td/utils/filesystem.cpp +++ b/tdutils/td/utils/filesystem.cpp @@ -94,6 +94,27 @@ Result read_file_secure(CSlice path, int64 size, int64 offset) { return read_file_impl(path, size, offset); } +Result read_file_mmap(CSlice path) { +#if TD_PORT_POSIX + TRY_RESULT(from_file, FileFd::open(path, FileFd::Read)); + TRY_RESULT(file_size, from_file.get_size()); + if (file_size <= 0) { + // Empty file or error — fall back to read_file which returns an empty BufferSlice cleanly. + return read_file(path); + } + auto native_fd = from_file.get_native_fd().fd(); + auto reader = BufferAllocator::create_reader_mmap(native_fd, static_cast(file_size)); + // fd may be closed after mmap; mmap retains its own reference. + from_file.close(); + if (!reader) { + return Status::Error("mmap failed"); + } + return BufferSlice(std::move(reader)); +#else + return read_file(path); +#endif +} + // Very straightforward function. Don't expect much of it. Status copy_file(CSlice from, CSlice to, int64 size) { TRY_RESULT(content, read_file(from, size)); diff --git a/tdutils/td/utils/filesystem.h b/tdutils/td/utils/filesystem.h index d43fd49594..a65f765d5f 100644 --- a/tdutils/td/utils/filesystem.h +++ b/tdutils/td/utils/filesystem.h @@ -29,6 +29,11 @@ Result read_file(CSlice path, int64 size = -1, int64 offset = 0); Result read_file_str(CSlice path, int64 size = -1, int64 offset = 0); Result read_file_secure(CSlice path, int64 size = -1, int64 offset = 0); +// mmap-backed read of the whole file. Returns a BufferSlice whose underlying storage +// is a read-only mmap(MAP_PRIVATE) of the file. Freeing the BufferSlice munmaps. +// Only available on POSIX; falls back to read_file on other platforms. +Result read_file_mmap(CSlice path); + Status copy_file(CSlice from, CSlice to, int64 size = -1) TD_WARN_UNUSED_RESULT; struct WriteFileOptions { diff --git a/validator/db/files-async.hpp b/validator/db/files-async.hpp index 177f685763..5c740c1b6b 100644 --- a/validator/db/files-async.hpp +++ b/validator/db/files-async.hpp @@ -101,7 +101,27 @@ class ReadFile : public td::actor::Actor { public: enum Flags : td::uint32 { f_disable_log = 1 }; void start_up() override { - auto S = td::read_file(file_name_, max_length_, offset_); + // Use mmap-backed read for whole-file reads when the file is large. + // This cuts peak RSS by the file size during state deserialization: the kernel + // pages content in on demand instead of the process holding all of it in heap. + // We only use mmap for whole-file reads (offset 0, no size cap) since mmap maps + // the entire file at once. + constexpr td::int64 kMmapThreshold = 64 << 20; // 64 MB + td::Result S; + if (offset_ == 0 && max_length_ == -1) { + auto stat = td::stat(file_name_); + if (stat.is_ok() && stat.ok().size_ >= kMmapThreshold) { + S = td::read_file_mmap(file_name_); + if (S.is_error()) { + // mmap may legitimately fail (e.g., /proc files, special FS); fall back. + S = td::read_file(file_name_, max_length_, offset_); + } + } else { + S = td::read_file(file_name_, max_length_, offset_); + } + } else { + S = td::read_file(file_name_, max_length_, offset_); + } if (S.is_ok()) { promise_.set_result(S.move_as_ok()); } else {