From 849d995346713b57fe9de59f86003eb3f42a21e8 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 6 Jul 2022 15:10:54 +0300 Subject: [PATCH 001/388] Add getvalidatorsessions query to validator-engine-console --- test/test-validator-session-state.cpp | 4 ++ tl/generate/scheme/ton_api.tl | 4 ++ tl/generate/scheme/ton_api.tlo | Bin 68244 -> 68852 bytes .../validator-engine-console-query.cpp | 23 ++++++++++++ .../validator-engine-console-query.h | 20 ++++++++++ .../validator-engine-console.cpp | 1 + validator-engine/validator-engine.cpp | 27 ++++++++++++++ validator-engine/validator-engine.hpp | 2 + .../validator-session-description.cpp | 5 +++ .../validator-session-description.h | 1 + .../validator-session-description.hpp | 1 + validator-session/validator-session.cpp | 13 +++++++ validator-session/validator-session.h | 2 + validator-session/validator-session.hpp | 2 + validator/manager-disk.hpp | 4 ++ validator/manager-hardfork.hpp | 4 ++ validator/manager.cpp | 35 +++++++++++++++++- validator/manager.hpp | 4 +- validator/validator-group.cpp | 28 +++++++++++++- validator/validator-group.hpp | 3 ++ validator/validator.h | 2 + 21 files changed, 181 insertions(+), 4 deletions(-) diff --git a/test/test-validator-session-state.cpp b/test/test-validator-session-state.cpp index 819c1cc2d..02ec4866a 100644 --- a/test/test-validator-session-state.cpp +++ b/test/test-validator-session-state.cpp @@ -103,6 +103,10 @@ class Description : public ton::validatorsession::ValidatorSessionDescription { td::uint32 get_max_priority() const override { return opts_.round_candidates - 1; } + td::uint32 get_node_by_priority(td::uint32 round, td::uint32 priority) const override { + CHECK(priority <= get_max_priority()); + return (round + priority) % get_total_nodes(); + } td::uint32 get_unixtime(td::uint64 ts) const override { return static_cast(ts >> 32); } diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 7dc6d8ca4..1909c71a0 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -630,6 +630,8 @@ engine.validator.overlayStatsNode adnl_id:int256 ip_addr:string bdcst_errors:int engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 scope:string nodes:(vector engine.validator.overlayStatsNode) stats:(vector engine.validator.oneStat) = engine.validator.OverlayStats; engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.OverlaysStats; +engine.validator.validatorSessionInfo current_block:tonNode.blockId self:int256 current_round:int next_producers:(vector int256) = engine.validator.ValidatorSessionInfo; +engine.validator.validatorSessionsInfo sessions:(vector engine.validator.validatorSessionInfo) = engine.validator.ValidatorSessionsInfo; ---functions--- @@ -680,6 +682,8 @@ engine.validator.importCertificate overlay_id:int256 local_id:adnl.id.short sign engine.validator.signShardOverlayCertificate workchain:int shard:long signed_key:engine.validator.KeyHash expire_at:int max_size:int = overlay.Certificate; engine.validator.importShardOverlayCertificate workchain:int shard:long signed_key:engine.validator.KeyHash cert:overlay.Certificate = engine.validator.Success; +engine.validator.getValidatorSessionsInfo = engine.validator.ValidatorSessionsInfo; + ---types--- storage.pong = storage.Pong; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index c49612788983ca6776a9977c3b0bbb6bc2701e05..9af19e27884de50bfd6dae0e45cdbc98835e7dab 100644 GIT binary patch delta 415 zcmbO-mF3G!7T!m*^{p77;Pgh`Y;i3M!LQd;Q}fa@^HTN75_2+B5=-)n^ul1Q;MC&c z%=|phytMp@>!h@=E?s&{jiBmckn+v(;;wv*j+>XN{?G8nZVTK#BwH95Kx%Iu6Y^mJ zx>NeueNEov(xRf&yps5&oc!c$hRN?QNKV$dsK5yqpWJv|o|i8#wW1`xpeR43G&yy$ zVw3#j6MiBNAhTfpy(-5QgWdh;?f|==0i #include @@ -1005,3 +1008,23 @@ td::Status ImportShardOverlayCertificateQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << "successfully sent certificate to overlay manager\n"; return td::Status::OK(); } + +td::Status GetValidatorSessionsInfoQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status GetValidatorSessionsInfoQuery::send() { + auto b = ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GetValidatorSessionsInfoQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX( + f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + std::string s = td::json_encode(td::ToJson(*f), true); + td::TerminalIO::out() << "---------\n" << s << "--------\n"; + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 15b77314a..dea2273db 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1075,3 +1075,23 @@ class ImportShardOverlayCertificateQuery : public Query { std::string in_file_; }; +class GetValidatorSessionsInfoQuery : public Query { + public: + GetValidatorSessionsInfoQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "getvalidatorsessions"; + } + static std::string get_help() { + return "getvalidatorsessions\tprint info about validator session"; + } + std::string name() const override { + return get_name(); + } + + private: +}; diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 1b3f33760..2c04971b5 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -140,6 +140,7 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index f380b003c..7783dc38e 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -3280,6 +3280,33 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getOverla }); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getValidatorSessionsInfo &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + + if (validator_manager_.empty()) { + promise.set_value( + create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "validator manager not started"))); + return; + } + + auto P = td::PromiseCreator::lambda( + [promise = std::move(promise)]( + td::Result> R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value(ton::serialize_tl_object(R.move_as_ok(), true)); + } + }); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::get_validator_sessions_info, + std::move(P)); +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 3f4fea2c6..e0364baba 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -407,6 +407,8 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getOverlaysStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_getValidatorSessionsInfo &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator-session/validator-session-description.cpp b/validator-session/validator-session-description.cpp index 42059053a..001f04058 100644 --- a/validator-session/validator-session-description.cpp +++ b/validator-session/validator-session-description.cpp @@ -79,6 +79,11 @@ td::uint32 ValidatorSessionDescriptionImpl::get_max_priority() const { return opts_.round_candidates - 1; } +td::uint32 ValidatorSessionDescriptionImpl::get_node_by_priority(td::uint32 round, td::uint32 priority) const { + CHECK(priority <= get_max_priority()); + return (round + priority) % get_total_nodes(); +} + ValidatorSessionCandidateId ValidatorSessionDescriptionImpl::candidate_id( td::uint32 src_idx, ValidatorSessionRootHash root_hash, ValidatorSessionFileHash file_hash, ValidatorSessionCollatedDataFileHash collated_data_file_hash) const { diff --git a/validator-session/validator-session-description.h b/validator-session/validator-session-description.h index 60e006d08..ac5043125 100644 --- a/validator-session/validator-session-description.h +++ b/validator-session/validator-session-description.h @@ -85,6 +85,7 @@ class ValidatorSessionDescription { virtual ValidatorWeight get_total_weight() const = 0; virtual td::int32 get_node_priority(td::uint32 src_idx, td::uint32 round) const = 0; virtual td::uint32 get_max_priority() const = 0; + virtual td::uint32 get_node_by_priority(td::uint32 round, td::uint32 priority) const = 0; virtual td::uint32 get_unixtime(td::uint64 t) const = 0; virtual td::uint32 get_attempt_seqno(td::uint64 t) const = 0; virtual td::uint32 get_self_idx() const = 0; diff --git a/validator-session/validator-session-description.hpp b/validator-session/validator-session-description.hpp index d65369ace..d7c12b101 100644 --- a/validator-session/validator-session-description.hpp +++ b/validator-session/validator-session-description.hpp @@ -101,6 +101,7 @@ class ValidatorSessionDescriptionImpl : public ValidatorSessionDescription { } td::int32 get_node_priority(td::uint32 src_idx, td::uint32 round) const override; td::uint32 get_max_priority() const override; + td::uint32 get_node_by_priority(td::uint32 round, td::uint32 priority) const override; td::uint32 get_unixtime(td::uint64 ts) const override { return static_cast(ts >> 32); } diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 3f56e3a31..5d4d4fd03 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -19,6 +19,7 @@ #include "validator-session.hpp" #include "td/utils/Random.h" #include "td/utils/crypto.h" +#include "ton/ton-tl.hpp" namespace ton { @@ -937,6 +938,18 @@ void ValidatorSessionImpl::stats_set_candidate_status(td::uint32 round, PublicKe it->block_status = status; } +void ValidatorSessionImpl::get_session_info( + td::Promise> promise) { + std::vector next_producers; + for (td::uint32 round = cur_round_; round < cur_round_ + 20; ++round) { + td::uint32 node = description().get_node_by_priority(round, 0); + next_producers.push_back(description().get_source_id(node).bits256_value()); + } + promise.set_result(create_tl_object( + create_tl_block_id_simple(BlockId{}), description().get_source_id(local_idx()).bits256_value(), + cur_round_, std::move(next_producers))); +} + td::actor::ActorOwn ValidatorSession::create( catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, diff --git a/validator-session/validator-session.h b/validator-session/validator-session.h index 376cac45e..fed75020e 100644 --- a/validator-session/validator-session.h +++ b/validator-session/validator-session.h @@ -92,6 +92,8 @@ class ValidatorSession : public td::actor::Actor { virtual void start() = 0; virtual void destroy() = 0; + virtual void get_session_info(td::Promise> promise) = 0; + static td::actor::ActorOwn create( catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 54bba33d1..a4e6bf03f 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -159,6 +159,8 @@ class ValidatorSessionImpl : public ValidatorSession { void stats_add_round(); void stats_set_candidate_status(td::uint32 round, PublicKeyHash src, int status); + void get_session_info(td::Promise> promise) override; + public: ValidatorSessionImpl(catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index b56f8b3ea..3f98273e0 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -367,6 +367,10 @@ class ValidatorManagerImpl : public ValidatorManager { void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override { UNREACHABLE(); } + void get_validator_sessions_info( + td::Promise> promise) override { + UNREACHABLE(); + } private: PublicKeyHash local_id_; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 11470cfd3..17042dbf1 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -426,6 +426,10 @@ class ValidatorManagerImpl : public ValidatorManager { void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override { UNREACHABLE(); } + void get_validator_sessions_info( + td::Promise> promise) override { + UNREACHABLE(); + } private: td::Ref opts_; diff --git a/validator/manager.cpp b/validator/manager.cpp index d6a9b4dda..81cd2e0ea 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2539,7 +2539,7 @@ void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id, stats.creator.bits256_value(), stats.total_validators, stats.total_weight, stats.signatures, stats.signatures_weight, stats.approve_signatures, stats.approve_signatures_weight, stats.first_round, std::move(rounds)); - std::string s = td::json_encode(td::ToJson(*obj.get()), false); + std::string s = td::json_encode(td::ToJson(*obj), false); s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); std::ofstream file; @@ -2550,6 +2550,39 @@ void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id, LOG(INFO) << "Writing validator session stats for " << block_id.id; } +void ValidatorManagerImpl::get_validator_sessions_info( + td::Promise> promise) { + std::vector> groups; + for (const auto& g : validator_groups_) { + groups.push_back(g.second.get()); + } + struct IntermediateData { + std::vector> groups; + std::vector> result; + td::Promise> promise; + + static void step(IntermediateData data) { + if (data.groups.empty()) { + data.promise.set_result( + create_tl_object(std::move(data.result))); + return; + } + auto group = std::move(data.groups.back()); + data.groups.pop_back(); + auto P = td::PromiseCreator::lambda( + [data = + std::move(data)](td::Result> R) mutable { + if (R.is_ok()) { + data.result.push_back(R.move_as_ok()); + } + step(std::move(data)); + }); + td::actor::send_closure(group, &ValidatorGroup::get_session_info, std::move(P)); + } + }; + IntermediateData::step({std::move(groups), {}, std::move(promise)}); +} + td::actor::ActorOwn ValidatorManagerFactory::create( td::Ref opts, std::string db_root, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, diff --git a/validator/manager.hpp b/validator/manager.hpp index d7f2c8a33..c855e1cc7 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -531,8 +531,10 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) override; void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override; + void get_validator_sessions_info( + td::Promise> promise) override; - private: + private: td::Timestamp resend_shard_blocks_at_; td::Timestamp check_waiters_at_; td::Timestamp check_shard_clients_; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 22ee35a52..333259b4c 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -21,6 +21,7 @@ #include "ton/ton-io.hpp" #include "td/utils/overloaded.h" #include "common/delay.h" +#include "ton/ton-tl.hpp" namespace ton { @@ -159,14 +160,18 @@ void ValidatorGroup::get_approved_candidate(PublicKey source, RootHash root_hash std::move(promise)); } -BlockIdExt ValidatorGroup::create_next_block_id(RootHash root_hash, FileHash file_hash) const { +BlockId ValidatorGroup::create_next_block_id_simple() const { BlockSeqno seqno = 0; for (auto &p : prev_block_ids_) { if (seqno < p.id.seqno) { seqno = p.id.seqno; } } - return BlockIdExt{shard_.workchain, shard_.shard, seqno + 1, root_hash, file_hash}; + return BlockId{shard_.workchain, shard_.shard, seqno + 1}; +} + +BlockIdExt ValidatorGroup::create_next_block_id(RootHash root_hash, FileHash file_hash) const { + return BlockIdExt{create_next_block_id_simple(), root_hash, file_hash}; } std::unique_ptr ValidatorGroup::make_validator_session_callback() { @@ -307,6 +312,25 @@ void ValidatorGroup::destroy() { stop(); } +void ValidatorGroup::get_session_info( + td::Promise> promise) { + if (session_.empty() || !started_) { + promise.set_error(td::Status::Error(ErrorCode::notready, "session not started")); + } + auto P = td::PromiseCreator::lambda( + [promise = std::move(promise), block_id = create_next_block_id_simple()]( + td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + return; + } + auto info = R.move_as_ok(); + info->current_block_ = create_tl_block_id_simple(block_id); + promise.set_result(std::move(info)); + }); + td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_session_info, std::move(P)); +} + } // namespace validator } // namespace ton diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 15256984b..3216f5fe6 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -44,6 +44,7 @@ class ValidatorGroup : public td::actor::Actor { td::Promise promise); void get_approved_candidate(PublicKey source, RootHash root_hash, FileHash file_hash, FileHash collated_data_file_hash, td::Promise promise); + BlockId create_next_block_id_simple() const; BlockIdExt create_next_block_id(RootHash root_hash, FileHash file_hash) const; void start(std::vector prev, BlockIdExt min_masterchain_block_id, UnixTime min_ts); @@ -56,6 +57,8 @@ class ValidatorGroup : public td::actor::Actor { } } + void get_session_info(td::Promise> promise); + ValidatorGroup(ShardIdFull shard, PublicKeyHash local_id, ValidatorSessionId session_id, td::Ref validator_set, validatorsession::ValidatorSessionOptions config, td::actor::ActorId keyring, td::actor::ActorId adnl, diff --git a/validator/validator.h b/validator/validator.h index fce119a49..b3d32e307 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -214,6 +214,8 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void run_ext_query(td::BufferSlice data, td::Promise promise) = 0; virtual void prepare_stats(td::Promise>> promise) = 0; + virtual void get_validator_sessions_info( + td::Promise> promise) = 0; }; } // namespace validator From 625516c5682f2ed12a27cd50b338b27dfda60a46 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 12 Jul 2022 11:04:31 +0300 Subject: [PATCH 002/388] Lite-mode for validate-query, more collated data in collator --- validator/fabric.h | 4 +- validator/impl/collator-impl.h | 7 +- validator/impl/collator.cpp | 89 +++++++++++++++++-- validator/impl/fabric.cpp | 5 +- validator/impl/validate-query.cpp | 138 +++++++++++++++++++++++++----- validator/impl/validate-query.hpp | 6 +- validator/manager-disk.cpp | 2 +- 7 files changed, 214 insertions(+), 37 deletions(-) diff --git a/validator/fabric.h b/validator/fabric.h index 67b6ae9ee..3515c0da8 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -25,6 +25,8 @@ namespace ton { namespace validator { +enum ValidateMode { fake = 1, lite = 2 }; + td::actor::ActorOwn create_db_actor(td::actor::ActorId manager, std::string db_root_); td::actor::ActorOwn create_liteserver_cache_actor(td::actor::ActorId manager, std::string db_root); @@ -73,7 +75,7 @@ void run_check_proof_link_query(BlockIdExt id, td::Ref proof, td::act void run_validate_query(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id, std::vector prev, BlockCandidate candidate, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, bool is_fake = false); + td::Promise promise, unsigned mode = 0); void run_collate_query(ShardIdFull shard, td::uint32 min_ts, const BlockIdExt& min_masterchain_block_id, std::vector prev, Ed25519_PublicKey local_id, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 2dced7495..997fea366 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -205,7 +205,11 @@ class Collator final : public td::actor::Actor { unsigned block_create_total_{0}; std::vector bad_ext_msgs_, delay_ext_msgs_; Ref shard_account_blocks_; // ShardAccountBlocks + + std::map> blocks_with_state_proofs_; + std::vector neighbor_proof_builders_; std::vector> collated_roots_; + std::unique_ptr block_candidate; td::PerfWarningTimer perf_timer_{"collate", 0.1}; @@ -235,7 +239,8 @@ class Collator final : public td::actor::Actor { void after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res); bool fix_one_processed_upto(block::MsgProcessedUpto& proc, const ton::ShardIdFull& owner); bool fix_processed_upto(block::MsgProcessedUptoCollection& upto); - void got_neighbor_out_queue(int i, td::Result> res); + void got_neighbor_block_data(td::Result> res); + void got_neighbor_block_state(int i, td::Result> res); bool adjust_shard_config(); bool store_shard_fees(ShardIdFull shard, const block::CurrencyCollection& fees, const block::CurrencyCollection& created); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 6aedcc21a..4d26f4b5d 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -502,6 +502,7 @@ void Collator::after_get_block_data(int idx, td::Result> res) { prev_mc_block = prev_block_data[0]; mc_block_root = prev_mc_block->root_cell(); } + blocks_with_state_proofs_[prev_block_data[idx]->root_cell()->get_hash().bits()] = prev_block_data[idx]; } check_pending(); } @@ -614,26 +615,59 @@ bool Collator::request_neighbor_msg_queues() { neighbors_.emplace_back(*shard_ptr); } int i = 0; + neighbor_proof_builders_.resize(neighbors_.size()); for (block::McShardDescr& descr : neighbors_) { LOG(DEBUG) << "neighbor #" << i << " : " << descr.blk_.to_str(); + if (descr.blk_.seqno() != 0) { + ++pending; + send_closure_later(manager, &ValidatorManager::wait_block_data_short, descr.blk_, priority(), timeout, + [self = get_self(), i](td::Result> res) { + LOG(DEBUG) << "got answer to wait_block_data for neighbor #" << i; + send_closure_later(std::move(self), &Collator::got_neighbor_block_data, std::move(res)); + }); + } ++pending; - send_closure_later(manager, &ValidatorManager::wait_block_message_queue_short, descr.blk_, priority(), timeout, - [self = get_self(), i](td::Result> res) { - td::actor::send_closure(std::move(self), &Collator::got_neighbor_out_queue, i, std::move(res)); + send_closure_later(manager, &ValidatorManager::wait_block_state_short, descr.blk_, priority(), timeout, + [self = get_self(), i](td::Result> res) { + LOG(DEBUG) << "got answer to wait_block_state for neighbor #" << i; + send_closure_later(std::move(self), &Collator::got_neighbor_block_state, i, std::move(res)); }); ++i; } return true; } -void Collator::got_neighbor_out_queue(int i, td::Result> res) { - LOG(DEBUG) << "obtained outbound queue for neighbor #" << i; +void Collator::got_neighbor_block_data(td::Result> res) { + --pending; + if (res.is_error()) { + fatal_error(res.move_as_error()); + return; + } + auto block_data = res.move_as_ok(); + blocks_with_state_proofs_[block_data->root_cell()->get_hash().bits()] = block_data; + check_pending(); +} + +void Collator::got_neighbor_block_state(int i, td::Result> res) { --pending; if (res.is_error()) { fatal_error(res.move_as_error()); return; } - Ref outq_descr = res.move_as_ok(); + Ref state = res.move_as_ok(); + neighbor_proof_builders_.at(i) = vm::MerkleProofBuilder{state->root_cell()}; + auto new_state = ShardStateQ::fetch(state->get_block_id(), {}, neighbor_proof_builders_.at(i).root()); + if (new_state.is_error()) { + fatal_error(new_state.move_as_error()); + return; + } + auto outq_descr_res = new_state.move_as_ok()->message_queue(); + if (outq_descr_res.is_error()) { + fatal_error(outq_descr_res.move_as_error()); + return; + } + LOG(DEBUG) << "obtained outbound queue for neighbor #" << i; + Ref outq_descr = outq_descr_res.move_as_ok(); block::McShardDescr& descr = neighbors_.at(i); if (outq_descr->get_block_id() != descr.blk_) { LOG(DEBUG) << "outq_descr->id = " << outq_descr->get_block_id().to_str() << " ; descr.id = " << descr.blk_.to_str(); @@ -3949,7 +3983,6 @@ Ref Collator::collate_shard_block_descr_set() { } bool Collator::create_collated_data() { - // TODO: store something into collated_roots_ // 1. store the set of used shard block descriptions if (!used_shard_block_descr_.empty()) { auto cell = collate_shard_block_descr_set(); @@ -3959,7 +3992,47 @@ bool Collator::create_collated_data() { } collated_roots_.push_back(std::move(cell)); } - // 2. ... + // 2. Proofs for hashes of states: previous states + neighbors + for (const auto& p : blocks_with_state_proofs_) { + vm::MerkleProofBuilder mpb{p.second->root_cell()}; + block::gen::Block::Record block; + if (!tlb::unpack_cell(mpb.root(), block) || block.state_update->load_cell().is_error()) { + return fatal_error("cannot generate Merkle proof for previous block"); + } + Ref proof = mpb.extract_proof(); + if (proof.is_null()) { + return fatal_error("cannot generate Merkle proof for previous block"); + } + collated_roots_.push_back(std::move(proof)); + } + // 3. Previous state proof (only shadchains) + std::map> proofs; + if (!is_masterchain()) { + state_usage_tree_->set_use_mark_for_is_loaded(false); + Ref state_proof = vm::MerkleProof::generate(prev_state_root_, state_usage_tree_.get()); + if (state_proof.is_null()) { + return fatal_error("cannot generate Merkle proof for previous state"); + } + proofs[prev_state_root_->get_hash().bits()] = std::move(state_proof); + } + // 4. Proofs for message queues + for (vm::MerkleProofBuilder &mpb : neighbor_proof_builders_) { + Ref proof = mpb.extract_proof(); + if (proof.is_null()) { + return fatal_error("cannot generate Merkle proof for neighbor"); + } + auto it = proofs.emplace(mpb.root()->get_hash().bits(), proof); + if (!it.second) { + it.first->second = vm::MerkleProof::combine(it.first->second, std::move(proof)); + if (it.first->second.is_null()) { + return fatal_error("cannot combine merkle proofs"); + } + } + } + + for (auto& p : proofs) { + collated_roots_.push_back(std::move(p.second)); + } return true; } diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index 196595aed..c3f4338d4 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -191,17 +191,18 @@ void run_check_proof_link_query(BlockIdExt id, td::Ref proof, td::act void run_validate_query(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id, std::vector prev, BlockCandidate candidate, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, bool is_fake) { + td::Promise promise, unsigned mode) { BlockSeqno seqno = 0; for (auto& p : prev) { if (p.seqno() > seqno) { seqno = p.seqno(); } } + bool is_fake = mode & ValidateMode::fake; td::actor::create_actor( PSTRING() << (is_fake ? "fakevalidate" : "validateblock") << shard.to_str() << ":" << (seqno + 1), shard, min_ts, min_masterchain_block_id, std::move(prev), std::move(candidate), std::move(validator_set), std::move(manager), - timeout, std::move(promise), is_fake) + timeout, std::move(promise), mode) .release(); } diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 97a4bab96..b9347fcf7 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -31,6 +31,7 @@ #include "vm/cells/MerkleProof.h" #include "vm/cells/MerkleUpdate.h" #include "common/errorlog.h" +#include "fabric.h" #include namespace ton { @@ -51,7 +52,7 @@ std::string ErrorCtx::as_string() const { ValidateQuery::ValidateQuery(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id, std::vector prev, BlockCandidate candidate, Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, bool is_fake) + td::Promise promise, unsigned mode) : shard_(shard) , id_(candidate.id) , min_ts(min_ts) @@ -62,7 +63,8 @@ ValidateQuery::ValidateQuery(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_ , manager(std::move(manager)) , timeout(timeout) , main_promise(std::move(promise)) - , is_fake_(is_fake) + , is_fake_(mode & ValidateMode::fake) + , is_lite_(mode & ValidateMode::lite) , shard_pfx_(shard_.shard) , shard_pfx_len_(ton::shard_prefix_length(shard_)) { proc_hash_.zero(); @@ -257,18 +259,20 @@ void ValidateQuery::start_up() { td::actor::send_closure_later( std::move(self), &ValidateQuery::after_get_latest_mc_state, std::move(res)); }); - // 3. load state(s) corresponding to previous block(s) + // 3. load state(s) corresponding to previous block(s) (non-lite mode or masterchain) prev_states.resize(prev_blocks.size()); - for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { - // 3.1. load state - LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; - ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), - timeout, [self = get_self(), i](td::Result> res) -> void { - LOG(DEBUG) << "got answer to wait_block_state_short query #" << i; - td::actor::send_closure_later( - std::move(self), &ValidateQuery::after_get_shard_state, i, std::move(res)); - }); + if (is_masterchain() || !is_lite_) { + for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { + // 3.1. load state + LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; + ++pending; + td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), + timeout, [self = get_self(), i](td::Result> res) -> void { + LOG(DEBUG) << "got answer to wait_block_state_short query #" << i; + td::actor::send_closure_later( + std::move(self), &ValidateQuery::after_get_shard_state, i, std::move(res)); + }); + } } // 4. unpack block candidate (while necessary data is being loaded) if (!unpack_block_candidate()) { @@ -988,6 +992,9 @@ bool ValidateQuery::check_this_shard_mc_info() { */ bool ValidateQuery::compute_prev_state() { + if (!is_masterchain() && is_lite_) { + return compute_prev_state_lite_mode(); + } CHECK(prev_states.size() == 1u + after_merge_); prev_state_root_ = prev_states[0]->root_cell(); CHECK(prev_state_root_.not_null()); @@ -1006,6 +1013,46 @@ bool ValidateQuery::compute_prev_state() { return true; } +bool ValidateQuery::compute_prev_state_lite_mode() { + td::Bits256 state_hash; + if (id_.seqno() == 1) { + if (prev_blocks.size() != 1) { + return reject_query("seqno is 1, but number of previous blocks is not 1"); + } + state_hash = prev_blocks[0].root_hash; + } else { + std::vector> prev_state_roots(prev_blocks.size()); + for (size_t i = 0; i < prev_blocks.size(); ++i) { + prev_state_roots[i] = get_virt_state_root(prev_blocks[i].root_hash); + if (prev_state_roots[i].is_null()) { + return reject_query(PSTRING() << "cannot get hash of previous state root: " << prev_blocks[i]); + } + } + + if (prev_state_roots.size() == 1) { + state_hash = prev_state_roots[0]->get_hash().bits(); + } else { + CHECK(prev_state_roots.size() == 2); + Ref merged; + if (!block::gen::t_ShardState.cell_pack_split_state(merged, prev_state_roots[0], prev_state_roots[1])) { + return fatal_error(-667, "cannot construct mechanically merged previously state"); + } + state_hash = merged->get_hash().bits(); + } + } + if (state_hash != prev_state_hash_) { + return reject_query("previous state hash mismatch for block "s + id_.to_str() + " : block header declares " + + prev_state_hash_.to_hex() + " , actual " + state_hash.to_hex()); + } + auto it = virt_roots_.find(state_hash); + if (it == virt_roots_.end()) { + return reject_query(PSTRING() << "no state root for previous block in collated data (hash = " + << state_hash.to_hex() << ")"); + } + prev_state_root_ = it->second; + return true; +} + bool ValidateQuery::compute_next_state() { LOG(DEBUG) << "computing next state"; auto res = vm::MerkleUpdate::validate(state_update_); @@ -1209,15 +1256,42 @@ bool ValidateQuery::request_neighbor_queues() { neighbors_.emplace_back(*shard_ptr); } int i = 0; - for (block::McShardDescr& descr : neighbors_) { - LOG(DEBUG) << "requesting outbound queue of neighbor #" << i << " : " << descr.blk_.to_str(); - ++pending; - send_closure_later(manager, &ValidatorManager::wait_block_message_queue_short, descr.blk_, priority(), timeout, - [self = get_self(), i](td::Result> res) { - td::actor::send_closure(std::move(self), &ValidateQuery::got_neighbor_out_queue, i, - std::move(res)); - }); - ++i; + if (is_lite_) { + for (block::McShardDescr& descr : neighbors_) { + LOG(DEBUG) << "getting outbound queue of neighbor #" << i << " from collated data : " << descr.blk_.to_str(); + td::Bits256 state_root_hash; + if (descr.blk_.seqno() == 0) { + state_root_hash = descr.blk_.root_hash; + } else { + Ref state_root = get_virt_state_root(descr.blk_.root_hash); + if (state_root.is_null()) { + return reject_query(PSTRING() << "cannot get hash of state root: " << descr.blk_); + } + state_root_hash = state_root->get_hash().bits(); + } + auto it = virt_roots_.find(state_root_hash); + if (it == virt_roots_.end()) { + return reject_query(PSTRING() << "cannot get state root form collated data: " << descr.blk_); + } + auto state = ShardStateQ::fetch(descr.blk_, {}, it->second); + if (state.is_error()) { + return reject_query("cannot fetch shard state from collated data", state.move_as_error()); + } + ++pending; + send_closure_later(get_self(), &ValidateQuery::got_neighbor_out_queue, i, state.move_as_ok()->message_queue()); + ++i; + } + } else { + for (block::McShardDescr& descr : neighbors_) { + LOG(DEBUG) << "requesting outbound queue of neighbor #" << i << " : " << descr.blk_.to_str(); + ++pending; + send_closure_later(manager, &ValidatorManager::wait_block_message_queue_short, descr.blk_, priority(), timeout, + [self = get_self(), i](td::Result> res) { + td::actor::send_closure(std::move(self), &ValidateQuery::got_neighbor_out_queue, i, + std::move(res)); + }); + ++i; + } } return true; } @@ -5430,6 +5504,24 @@ bool ValidateQuery::check_mc_block_extra() { return true; } +Ref ValidateQuery::get_virt_state_root(td::Bits256 block_root_hash) { + auto it = virt_roots_.find(block_root_hash); + if (it == virt_roots_.end()) { + return {}; + } + Ref root = it->second; + block::gen::Block::Record block; + if (!tlb::unpack_cell(root, block)) { + return {}; + } + vm::CellSlice upd_cs{vm::NoVmSpec(), block.state_update}; + if (!(upd_cs.is_special() && upd_cs.prefetch_long(8) == 4 // merkle update + && upd_cs.size_ext() == 0x20228)) { + return {}; + } + return vm::MerkleProof::virtualize_raw(upd_cs.prefetch_ref(1), {0, 1}); +} + /* * * MAIN VALIDATOR FUNCTION @@ -5533,7 +5625,7 @@ bool ValidateQuery::try_validate() { } catch (vm::VmError& err) { return fatal_error(-666, err.get_msg()); } catch (vm::VmVirtError& err) { - return fatal_error(-666, err.get_msg()); + return reject_query(err.get_msg()); } return save_candidate(); } diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 2aef04afa..3a400e755 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -118,7 +118,7 @@ class ValidateQuery : public td::actor::Actor { ValidateQuery(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id, std::vector prev, BlockCandidate candidate, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, bool is_fake = false); + td::Promise promise, unsigned mode = 0); private: int verbosity{3 * 1}; @@ -142,6 +142,7 @@ class ValidateQuery : public td::actor::Actor { bool is_key_block_{false}; bool update_shard_cc_{false}; bool is_fake_{false}; + bool is_lite_{false}; bool prev_key_block_exists_{false}; bool debug_checks_{false}; bool outq_cleanup_partial_{false}; @@ -286,6 +287,7 @@ class ValidateQuery : public td::actor::Actor { bool extract_collated_data(); bool try_validate(); bool compute_prev_state(); + bool compute_prev_state_lite_mode(); bool compute_next_state(); bool unpack_merge_prev_state(); bool unpack_prev_state(); @@ -368,6 +370,8 @@ class ValidateQuery : public td::actor::Actor { bool check_one_shard_fee(ShardIdFull shard, const block::CurrencyCollection& fees, const block::CurrencyCollection& create); bool check_mc_block_extra(); + + Ref get_virt_state_root(td::Bits256 block_root_hash); }; } // namespace validator diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 46ad50faa..7563718d5 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -153,7 +153,7 @@ void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vector prev, BlockIdExt last, From 415ace3da94ce3c73ab0dee3a50935cac4e4ab60 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 12 Jul 2022 21:15:04 +0300 Subject: [PATCH 003/388] Exporting and importing candidates using validator-engine-console; lite mode for validator group --- tl/generate/scheme/ton_api.tl | 6 + tl/generate/scheme/ton_api.tlo | Bin 68852 -> 69376 bytes .../validator-engine-console-query.cpp | 63 ++++++++++ .../validator-engine-console-query.h | 68 +++++++++++ .../validator-engine-console.cpp | 3 + validator-engine/validator-engine.cpp | 88 ++++++++++++++ validator-engine/validator-engine.hpp | 6 + validator/impl/validate-query.cpp | 1 + validator/interfaces/validator-manager.h | 2 + validator/manager-disk.hpp | 12 ++ validator/manager-hardfork.hpp | 13 ++ validator/manager.cpp | 115 +++++++++++++++++- validator/manager.hpp | 10 +- validator/validator-group.cpp | 18 ++- validator/validator-group.hpp | 6 +- validator/validator.h | 4 + 16 files changed, 410 insertions(+), 5 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 1909c71a0..eca152354 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -633,6 +633,8 @@ engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.validatorSessionInfo current_block:tonNode.blockId self:int256 current_round:int next_producers:(vector int256) = engine.validator.ValidatorSessionInfo; engine.validator.validatorSessionsInfo sessions:(vector engine.validator.validatorSessionInfo) = engine.validator.ValidatorSessionsInfo; +engine.validator.requiredBlockCandidates block_ids:(vector tonNode.blockId) = engine.validator.RequiredBlockCandidates; + ---functions--- engine.validator.getTime = engine.validator.Time; @@ -684,6 +686,10 @@ engine.validator.importShardOverlayCertificate workchain:int shard:long signed_k engine.validator.getValidatorSessionsInfo = engine.validator.ValidatorSessionsInfo; +engine.validator.generateBlockCandidate block_id:tonNode.BlockId = db.Candidate; +engine.validator.getRequiredBlockCandidates = engine.validator.RequiredBlockCandidates; +engine.validator.importBlockCandidate block:db.candidate = engine.validator.Success; + ---types--- storage.pong = storage.Pong; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 9af19e27884de50bfd6dae0e45cdbc98835e7dab..252c6d0564df80fb0a38fe6fab78645f0e29d6ad 100644 GIT binary patch delta 372 zcmew|lcixE3-6=Z`c@23aAqU#a*oLvj3S#)aNOY4-CSOrsF9kNo|%`bSC*KQnUYwN zU!)h5T3DJ{l$zp{lb@XJoS2sa5>G9joX9D?d4X6DKcmxTUA6jjPwcvju()); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); + TRY_RESULT_ASSIGN(seqno_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(file_, tokenizer_.get_token()); + return td::Status::OK(); +} + +td::Status GenerateBlockCandidateQuery::send() { + auto b = ton::create_serialize_tl_object( + ton::create_tl_block_id_simple(ton::BlockId(wc_, shard_, seqno_))); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GenerateBlockCandidateQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + TRY_STATUS_PREFIX(td::write_file(file_, data.as_slice()), "failed to write block to file"); + td::TerminalIO::out() << "successfully written candidate to file\n"; + return td::Status::OK(); +} + +td::Status GetRequiredBlockCandidatesQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status GetRequiredBlockCandidatesQuery::send() { + auto b = ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GetRequiredBlockCandidatesQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX( + f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << td::json_encode(td::ToJson(*f), true); + return td::Status::OK(); +} + +td::Status ImportBlockCandidateQuery::run() { + TRY_RESULT_ASSIGN(file_, tokenizer_.get_token()); + return td::Status::OK(); +} + +td::Status ImportBlockCandidateQuery::send() { + TRY_RESULT(data, td::read_file(file_)); + TRY_RESULT_PREFIX(candidate, ton::fetch_tl_object(data.as_slice(), true), + "invalid file: "); + auto b = ton::create_serialize_tl_object(std::move(candidate)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ImportBlockCandidateQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "successfully imported a block candidate\n"; + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index dea2273db..e4236f6dc 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1092,6 +1092,74 @@ class GetValidatorSessionsInfoQuery : public Query { std::string name() const override { return get_name(); } +}; + +class GenerateBlockCandidateQuery : public Query { + public: + GenerateBlockCandidateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "genblock"; + } + static std::string get_help() { + return "genblock \t" + "generate a block candidate for a given shard (seqno mush match the next seqno for the shard), " + "candidate is saved to "; + } + std::string name() const override { + return get_name(); + } + + private: + td::int32 wc_; + td::int64 shard_; + td::int32 seqno_; + std::string file_; +}; + +class GetRequiredBlockCandidatesQuery : public Query { + public: + GetRequiredBlockCandidatesQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "getrequiredblockcandidates"; + } + static std::string get_help() { + return "getrequiredblockcandidates\t" + "get a list of block candidates that the validator is currently waiting for"; + } + std::string name() const override { + return get_name(); + } +}; + +class ImportBlockCandidateQuery : public Query { + public: + ImportBlockCandidateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "importblockcandidate"; + } + static std::string get_help() { + return "importblockcandidate \t" + "load a block candidate from a given file"; + } + std::string name() const override { + return get_name(); + } private: + std::string file_; }; diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 2c04971b5..b4dcad22b 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -141,6 +141,9 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 7783dc38e..2e158e079 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -3307,6 +3307,94 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getValida std::move(P)); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_generateBlockCandidate &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (validator_manager_.empty()) { + promise.set_value( + create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "validator manager not started"))); + return; + } + ton::BlockId block_id = ton::create_block_id_simple(query.block_id_); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::generate_block_candidate, + block_id, [promise = std::move(promise)](td::Result R) mutable { + if (R.is_ok()) { + auto block = R.move_as_ok(); + auto result = ton::create_serialize_tl_object( + ton::PublicKey{ton::pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), + ton::create_tl_block_id(block.id), std::move(block.data), + std::move(block.collated_data)); + promise.set_result(std::move(result)); + } else { + promise.set_value(create_control_query_error(R.move_as_error())); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getRequiredBlockCandidates &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (validator_manager_.empty()) { + promise.set_value( + create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "validator manager not started"))); + return; + } + td::actor::send_closure( + validator_manager_, &ton::validator::ValidatorManagerInterface::get_required_block_candidates, + [promise = std::move(promise)](td::Result> R) mutable { + if (R.is_ok()) { + std::vector> block_ids; + for (const ton::BlockId &block_id : R.move_as_ok()) { + block_ids.push_back(ton::create_tl_block_id_simple(block_id)); + } + auto result = ton::create_serialize_tl_object( + std::move(block_ids)); + promise.set_result(std::move(result)); + } else { + promise.set_value(create_control_query_error(R.move_as_error())); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importBlockCandidate &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (validator_manager_.empty()) { + promise.set_value( + create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "validator manager not started"))); + return; + } + + auto collated_data_hash = td::sha256_bits256(query.block_->collated_data_); + auto key = ton::PublicKey{query.block_->source_}; + auto e_key = ton::Ed25519_PublicKey{key.ed25519_value().raw()}; + ton::BlockCandidate candidate{e_key, ton::create_block_id(query.block_->id_), collated_data_hash, + std::move(query.block_->data_), std::move(query.block_->collated_data_)}; + + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::import_block_candidate, + std::move(candidate), + [promise = std::move(promise)](td::Result R) mutable { + if (R.is_ok()) { + promise.set_result(ton::serialize_tl_object( + ton::create_tl_object(), true)); + } else { + promise.set_value(create_control_query_error(R.move_as_error())); + } + }); +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index e0364baba..beb24237d 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -409,6 +409,12 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getValidatorSessionsInfo &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_generateBlockCandidate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_getRequiredBlockCandidates &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_importBlockCandidate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index b9347fcf7..bd0d3b2a4 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -449,6 +449,7 @@ bool ValidateQuery::init_parse() { return reject_query("after_merge value mismatch in block header"); } rand_seed_ = extra.rand_seed; + created_by_ = extra.created_by; if (created_by_ != extra.created_by) { return reject_query("block candidate "s + id_.to_str() + " has creator " + created_by_.to_hex() + " but the block header contains different value " + extra.created_by.to_hex()); diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 4ee276d38..c72ec307c 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -169,6 +169,8 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) = 0; + virtual void wait_block_candidate(BlockId block_id, td::Timestamp timeout, td::Promise promise) = 0; + static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) { return ts / (1 << 17) != prev_ts / (1 << 17); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 3f98273e0..6bad11715 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -371,6 +371,18 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { UNREACHABLE(); } + void generate_block_candidate(BlockId block_id, td::Promise promise) override { + UNREACHABLE(); + } + void get_required_block_candidates(td::Promise> promise) override { + UNREACHABLE(); + } + void import_block_candidate(BlockCandidate candidate, td::Promise promise) override { + UNREACHABLE(); + } + void wait_block_candidate(BlockId block_id, td::Timestamp timeout, td::Promise promise) override { + UNREACHABLE(); + } private: PublicKeyHash local_id_; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 17042dbf1..a8aaecfc3 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -430,6 +430,19 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { UNREACHABLE(); } + void generate_block_candidate(BlockId block_id, td::Promise promise) override { + UNREACHABLE(); + } + void get_required_block_candidates(td::Promise> promise) override { + UNREACHABLE(); + } + void import_block_candidate(BlockCandidate candidate, td::Promise promise) override { + UNREACHABLE(); + } + void wait_block_candidate(BlockId block_id, td::Timestamp timeout, td::Promise promise) override { + UNREACHABLE(); + } + private: td::Ref opts_; diff --git a/validator/manager.cpp b/validator/manager.cpp index 81cd2e0ea..0d8929ace 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2031,7 +2031,7 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group auto G = td::actor::create_actor( "validatorgroup", shard, validator_id, session_id, validator_set, opts, keyring_, adnl_, rldp_, overlays_, db_root_, actor_id(this), init_session, - opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno())); + opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), true); return G; } } @@ -2583,6 +2583,119 @@ void ValidatorManagerImpl::get_validator_sessions_info( IntermediateData::step({std::move(groups), {}, std::move(promise)}); } +void ValidatorManagerImpl::generate_block_candidate(BlockId block_id, td::Promise promise) { + if (!block_id.is_valid_full()) { + promise.set_error(td::Status::Error("invalid block id")); + return; + } + if (last_masterchain_state_.is_null()) { + promise.set_error(td::Status::Error("not started")); + return; + } + ShardIdFull shard_id = block_id.shard_full(); + std::vector prev; + auto shard = last_masterchain_state_->get_shard_from_config(shard_id); + if (shard.not_null()) { + if (shard->before_split()) { + promise.set_error(td::Status::Error("shard is before_split")); + return; + } + if (shard->before_merge()) { + promise.set_error(td::Status::Error("shard is before_merge")); + return; + } + prev.push_back(shard->top_block_id()); + } else { + auto parent = shard_id.pfx_len() == 0 ? td::Ref() + : last_masterchain_state_->get_shard_from_config(shard_parent(shard_id)); + if (parent.not_null() && parent->before_split()) { + prev.push_back(parent->top_block_id()); + } else { + auto child_l = last_masterchain_state_->get_shard_from_config(shard_child(shard_id, true)); + auto child_r = last_masterchain_state_->get_shard_from_config(shard_child(shard_id, false)); + if (child_l.not_null() && child_r.not_null() && child_l->before_merge() && child_r->before_merge()) { + prev.push_back(child_l->top_block_id()); + prev.push_back(child_r->top_block_id()); + } + } + if (prev.empty()) { + promise.set_error(td::Status::Error("no such shard")); + return; + } + } + + BlockSeqno next_seqno = 0; + for (const BlockIdExt& prev_id : prev) { + next_seqno = std::max(next_seqno, prev_id.seqno() + 1); + } + if (next_seqno != block_id.seqno) { + promise.set_error(td::Status::Error(PSTRING() << "seqno mismatch: asked for seqno " << block_id.seqno + << ", but actual next seqno is " << next_seqno)); + return; + } + + Ed25519_PublicKey local_id{Bits256::zero()}; + td::Ref validator_set = last_masterchain_state_->get_validator_set(shard_id); + if (validator_set.is_null()) { + promise.set_error(td::Status::Error("cannot get validator set")); + return; + } + run_collate_query(shard_id, last_masterchain_state_->get_unix_time(), last_masterchain_block_id_, std::move(prev), + local_id, std::move(validator_set), actor_id(this), td::Timestamp::in(10.0), std::move(promise)); +} + +void ValidatorManagerImpl::get_required_block_candidates(td::Promise> promise) { + std::vector block_ids; + for (const auto& p : pending_block_candidates_) { + block_ids.push_back(p.first); + } + promise.set_result(std::move(block_ids)); +} + +void ValidatorManagerImpl::import_block_candidate(BlockCandidate candidate, td::Promise promise) { + auto it = pending_block_candidates_.find(candidate.id.id); + if (it != pending_block_candidates_.end()) { + while (!it->second.empty()) { + auto promise = std::move(it->second.back().first); + it->second.pop_back(); + if (it->second.empty()) { + promise.set_result(std::move(candidate)); + } else { + promise.set_result(candidate.clone()); + } + } + pending_block_candidates_.erase(it); + } + promise.set_result(td::Unit()); +} + +void ValidatorManagerImpl::wait_block_candidate(BlockId block_id, td::Timestamp timeout, + td::Promise promise) { + pending_block_candidates_[block_id].emplace_back(std::move(promise), timeout); + delay_action([SelfId = actor_id(this), block_id, timeout]() { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::cleanup_old_pending_candidates, block_id, timeout); + }, timeout); +} + +void ValidatorManagerImpl::cleanup_old_pending_candidates(BlockId block_id, td::Timestamp now) { + auto it = pending_block_candidates_.find(block_id); + if (it == pending_block_candidates_.end()) { + return; + } + it->second.erase(std::remove_if(it->second.begin(), it->second.end(), + [&](std::pair, td::Timestamp> &p) { + if (p.second.is_in_past(now)) { + p.first.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); + return true; + } + return false; + }), + it->second.end()); + if (it->second.empty()) { + pending_block_candidates_.erase(it); + } +} + td::actor::ActorOwn ValidatorManagerFactory::create( td::Ref opts, std::string db_root, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, diff --git a/validator/manager.hpp b/validator/manager.hpp index c855e1cc7..1fbf60a9d 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -534,7 +534,12 @@ class ValidatorManagerImpl : public ValidatorManager { void get_validator_sessions_info( td::Promise> promise) override; - private: + void generate_block_candidate(BlockId block_id, td::Promise promise) override; + void get_required_block_candidates(td::Promise> promise) override; + void import_block_candidate(BlockCandidate candidate, td::Promise promise) override; + void wait_block_candidate(BlockId block_id, td::Timestamp timeout, td::Promise promise) override; + + private: td::Timestamp resend_shard_blocks_at_; td::Timestamp check_waiters_at_; td::Timestamp check_shard_clients_; @@ -598,6 +603,9 @@ class ValidatorManagerImpl : public ValidatorManager { private: std::map> shard_client_waiters_; + + std::map, td::Timestamp>>> pending_block_candidates_; + void cleanup_old_pending_candidates(BlockId block_id, td::Timestamp now); }; } // namespace validator diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 333259b4c..f5c357982 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -35,6 +35,22 @@ void ValidatorGroup::generate_block_candidate(td::uint32 round_id, td::Promise R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + } else { + BlockCandidate candidate = R.move_as_ok(); + candidate.pubkey = pubkey; + promise.set_result(std::move(candidate)); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::wait_block_candidate, create_next_block_id_simple(), + td::Timestamp::in(15.0), std::move(P)); + return; + } run_collate_query(shard_, min_ts_, min_masterchain_block_id_, prev_block_ids_, Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, validator_set_, manager_, td::Timestamp::in(10.0), std::move(promise)); @@ -79,7 +95,7 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat VLOG(VALIDATOR_DEBUG) << "validating block candidate " << next_block_id; block.id = next_block_id; run_validate_query(shard_, min_ts_, min_masterchain_block_id_, prev_block_ids_, std::move(block), validator_set_, - manager_, td::Timestamp::in(10.0), std::move(P)); + manager_, td::Timestamp::in(10.0), std::move(P), lite_mode_ ? ValidateMode::lite : 0); } void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash src, td::BufferSlice block_data, diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 3216f5fe6..5d760899a 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -64,7 +64,7 @@ class ValidatorGroup : public td::actor::Actor { td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, td::actor::ActorId validator_manager, bool create_session, - bool allow_unsafe_self_blocks_resync) + bool allow_unsafe_self_blocks_resync, bool lite_mode = false) : shard_(shard) , local_id_(std::move(local_id)) , session_id_(session_id) @@ -77,7 +77,8 @@ class ValidatorGroup : public td::actor::Actor { , db_root_(std::move(db_root)) , manager_(validator_manager) , init_(create_session) - , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) { + , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) + , lite_mode_(lite_mode) { } private: @@ -118,6 +119,7 @@ class ValidatorGroup : public td::actor::Actor { bool init_ = false; bool started_ = false; bool allow_unsafe_self_blocks_resync_; + bool lite_mode_ = false; td::uint32 last_known_round_id_ = 0; }; diff --git a/validator/validator.h b/validator/validator.h index b3d32e307..09081f1e3 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -216,6 +216,10 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void prepare_stats(td::Promise>> promise) = 0; virtual void get_validator_sessions_info( td::Promise> promise) = 0; + + virtual void generate_block_candidate(BlockId block_id, td::Promise promise) = 0; + virtual void get_required_block_candidates(td::Promise> promise) = 0; + virtual void import_block_candidate(BlockCandidate candidate, td::Promise promise) = 0; }; } // namespace validator From bdfca7afef41b90705900b9166334b567bec916b Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 13 Jul 2022 12:13:51 +0300 Subject: [PATCH 004/388] Remove unused code --- validator/fabric.h | 8 ++-- validator/impl/CMakeLists.txt | 1 - validator/impl/collate-query-impl.h | 63 ----------------------------- validator/impl/collator-impl.h | 3 +- validator/impl/collator.cpp | 3 +- validator/impl/collator.h | 19 --------- validator/impl/fabric.cpp | 14 +++---- validator/impl/validate-query.cpp | 3 +- validator/impl/validate-query.hpp | 3 +- validator/manager-disk.cpp | 4 +- validator/manager.cpp | 16 +++----- validator/validator-group.cpp | 9 ++--- validator/validator-group.hpp | 3 +- 13 files changed, 28 insertions(+), 121 deletions(-) delete mode 100644 validator/impl/collate-query-impl.h diff --git a/validator/fabric.h b/validator/fabric.h index 3515c0da8..158ea7e00 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -72,12 +72,12 @@ void run_check_proof_query(BlockIdExt id, td::Ref proof, td::actor::Actor td::Ref rel_key_block_proof, bool skip_check_signatures = false); void run_check_proof_link_query(BlockIdExt id, td::Ref proof, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); -void run_validate_query(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id, - std::vector prev, BlockCandidate candidate, td::Ref validator_set, +void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, + BlockCandidate candidate, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, unsigned mode = 0); -void run_collate_query(ShardIdFull shard, td::uint32 min_ts, const BlockIdExt& min_masterchain_block_id, - std::vector prev, Ed25519_PublicKey local_id, td::Ref validator_set, +void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, + Ed25519_PublicKey local_id, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, diff --git a/validator/impl/CMakeLists.txt b/validator/impl/CMakeLists.txt index 459e77244..224599f86 100644 --- a/validator/impl/CMakeLists.txt +++ b/validator/impl/CMakeLists.txt @@ -25,7 +25,6 @@ set(TON_VALIDATOR_SOURCE accept-block.hpp block.hpp check-proof.hpp - collate-query-impl.h collator-impl.h collator.h config.hpp diff --git a/validator/impl/collate-query-impl.h b/validator/impl/collate-query-impl.h deleted file mode 100644 index 838326468..000000000 --- a/validator/impl/collate-query-impl.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once - -#include "validator/interfaces/validator-manager.h" - -namespace ton { - -namespace validator { - -class CollateQuery : public td::actor::Actor { - public: - CollateQuery(ShardIdFull shard, td::uint32 min_ts, BlockIdExt min_masterchain_block_id, std::vector prev, - td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise); - CollateQuery(ShardIdFull shard, td::uint32 min_ts, BlockIdExt min_masterchain_block_id, ZeroStateIdExt zero_state_id, - td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise); - - void alarm() override; - - void abort_query(td::Status reason); - void finish_query(); - - void start_up() override; - void got_prev_state(td::Ref state); - void written_block_data(); - void written_block_collated_data(); - - private: - ShardIdFull shard_; - UnixTime min_ts_; - BlockIdExt min_masterchain_block_id_; - std::vector prev_; - ZeroStateIdExt zero_state_id_; - td::Ref validator_set_; - td::actor::ActorId manager_; - td::Timestamp timeout_; - td::Promise promise_; - - BlockCandidate candidate_; - UnixTime ts_; -}; - -} // namespace validator - -} // namespace ton diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 997fea366..8a17e9531 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -64,7 +64,6 @@ class Collator final : public td::actor::Actor { bool libraries_changed_{false}; bool prev_key_block_exists_{false}; bool is_hardfork_{false}; - UnixTime min_ts; BlockIdExt min_mc_block_id; std::vector prev_blocks; std::vector> prev_states; @@ -86,7 +85,7 @@ class Collator final : public td::actor::Actor { static constexpr bool shard_splitting_enabled = true; public: - Collator(ShardIdFull shard, bool is_hardfork, td::uint32 min_ts, BlockIdExt min_masterchain_block_id, + Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector prev, Ref validator_set, Ed25519_PublicKey collator_id, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); ~Collator() override = default; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 4d26f4b5d..49c945961 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -54,13 +54,12 @@ static inline bool dbg(int c) { return true; } -Collator::Collator(ShardIdFull shard, bool is_hardfork, UnixTime min_ts, BlockIdExt min_masterchain_block_id, +Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector prev, td::Ref validator_set, Ed25519_PublicKey collator_id, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise) : shard_(shard) , is_hardfork_(is_hardfork) - , min_ts(min_ts) , min_mc_block_id{min_masterchain_block_id} , prev_blocks(std::move(prev)) , created_by_(collator_id) diff --git a/validator/impl/collator.h b/validator/impl/collator.h index c92fa80e3..5acc20e8d 100644 --- a/validator/impl/collator.h +++ b/validator/impl/collator.h @@ -24,26 +24,7 @@ #include "vm/cells.h" namespace ton { -using td::Ref; extern int collator_settings; // +1 = force want_split, +2 = force want_merge -class Collator : public td::actor::Actor { - protected: - Collator() = default; - - public: - virtual ~Collator() = default; - static td::actor::ActorOwn create_collator( - td::actor::ActorId block_db, - ShardIdFull shard /* , td::actor::ActorId validator_manager */); - virtual void generate_block_candidate(ShardIdFull shard, td::Promise promise) = 0; - virtual td::Result register_external_message_cell(Ref ext_msg) = 0; - virtual td::Result register_external_message(td::Slice ext_msg_boc) = 0; - virtual td::Result register_ihr_message_cell(Ref ihr_msg) = 0; - virtual td::Result register_ihr_message(td::Slice ihr_msg_boc) = 0; - virtual td::Result register_shard_signatures_cell(Ref shard_blk_signatures) = 0; - virtual td::Result register_shard_signatures(td::Slice shard_blk_signatures_boc) = 0; -}; - } // namespace ton diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index c3f4338d4..6958ad890 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -188,7 +188,7 @@ void run_check_proof_link_query(BlockIdExt id, td::Ref proof, td::act .release(); } -void run_validate_query(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id, +void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, BlockCandidate candidate, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, unsigned mode) { @@ -200,14 +200,14 @@ void run_validate_query(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_maste } bool is_fake = mode & ValidateMode::fake; td::actor::create_actor( - PSTRING() << (is_fake ? "fakevalidate" : "validateblock") << shard.to_str() << ":" << (seqno + 1), shard, min_ts, + PSTRING() << (is_fake ? "fakevalidate" : "validateblock") << shard.to_str() << ":" << (seqno + 1), shard, min_masterchain_block_id, std::move(prev), std::move(candidate), std::move(validator_set), std::move(manager), timeout, std::move(promise), mode) .release(); } -void run_collate_query(ShardIdFull shard, td::uint32 min_ts, const BlockIdExt& min_masterchain_block_id, - std::vector prev, Ed25519_PublicKey collator_id, td::Ref validator_set, +void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, + Ed25519_PublicKey collator_id, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise) { BlockSeqno seqno = 0; @@ -217,8 +217,8 @@ void run_collate_query(ShardIdFull shard, td::uint32 min_ts, const BlockIdExt& m } } td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, false, - min_ts, min_masterchain_block_id, std::move(prev), std::move(validator_set), - collator_id, std::move(manager), timeout, std::move(promise)) + min_masterchain_block_id, std::move(prev), std::move(validator_set), collator_id, + std::move(manager), timeout, std::move(promise)) .release(); } @@ -231,7 +231,7 @@ void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_b seqno = p.seqno(); } } - td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, 0, + td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, min_masterchain_block_id, std::move(prev), td::Ref{}, Ed25519_PublicKey{Bits256::zero()}, std::move(manager), timeout, std::move(promise)) .release(); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index bd0d3b2a4..aac85c8f1 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -49,13 +49,12 @@ std::string ErrorCtx::as_string() const { return a; } -ValidateQuery::ValidateQuery(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id, +ValidateQuery::ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, BlockCandidate candidate, Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, unsigned mode) : shard_(shard) , id_(candidate.id) - , min_ts(min_ts) , min_mc_block_id(min_masterchain_block_id) , prev_blocks(std::move(prev)) , block_candidate(std::move(candidate)) diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 3a400e755..6fe8010f5 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -115,7 +115,7 @@ class ValidateQuery : public td::actor::Actor { } public: - ValidateQuery(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id, std::vector prev, + ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, BlockCandidate candidate, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, unsigned mode = 0); @@ -125,7 +125,6 @@ class ValidateQuery : public td::actor::Actor { int pending{0}; const ShardIdFull shard_; const BlockIdExt id_; - UnixTime min_ts; BlockIdExt min_mc_block_id; std::vector prev_blocks; std::vector> prev_states; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 7563718d5..26da9e165 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -128,7 +128,7 @@ void ValidatorManagerImpl::sync_complete(td::Promise promise) { } Ed25519_PublicKey created_by{td::Bits256::zero()}; td::as(created_by.as_bits256().data() + 32 - 4) = ((unsigned)std::time(nullptr) >> 8); - run_collate_query(shard_id, 0, last_masterchain_block_id_, prev, created_by, val_set, actor_id(this), + run_collate_query(shard_id, last_masterchain_block_id_, prev, created_by, val_set, actor_id(this), td::Timestamp::in(10.0), std::move(P)); } @@ -152,7 +152,7 @@ void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vectorsecond.empty()) { - td::actor::send_closure(it2->second, &ValidatorGroup::start, prev, last_masterchain_block_id_, - last_masterchain_state_->get_unix_time()); + td::actor::send_closure(it2->second, &ValidatorGroup::start, prev, last_masterchain_block_id_); } new_validator_groups_.emplace(val_group_id, std::move(it2->second)); } else { auto G = create_validator_group(val_group_id, shard, val_set, opts, started_); if (!G.empty()) { - td::actor::send_closure(G, &ValidatorGroup::start, prev, last_masterchain_block_id_, - last_masterchain_state_->get_unix_time()); + td::actor::send_closure(G, &ValidatorGroup::start, prev, last_masterchain_block_id_); } new_validator_groups_.emplace(val_group_id, std::move(G)); } @@ -1887,15 +1885,13 @@ void ValidatorManagerImpl::update_shards() { auto it2 = next_validator_groups_.find(val_group_id); if (it2 != next_validator_groups_.end()) { if (!it2->second.empty()) { - td::actor::send_closure(it2->second, &ValidatorGroup::start, prev, last_masterchain_block_id_, - last_masterchain_state_->get_unix_time()); + td::actor::send_closure(it2->second, &ValidatorGroup::start, prev, last_masterchain_block_id_); } new_validator_groups_.emplace(val_group_id, std::move(it2->second)); } else { auto G = create_validator_group(val_group_id, shard, val_set, opts, started_); if (!G.empty()) { - td::actor::send_closure(G, &ValidatorGroup::start, prev, last_masterchain_block_id_, - last_masterchain_state_->get_unix_time()); + td::actor::send_closure(G, &ValidatorGroup::start, prev, last_masterchain_block_id_); } new_validator_groups_.emplace(val_group_id, std::move(G)); } @@ -2640,8 +2636,8 @@ void ValidatorManagerImpl::generate_block_candidate(BlockId block_id, td::Promis promise.set_error(td::Status::Error("cannot get validator set")); return; } - run_collate_query(shard_id, last_masterchain_state_->get_unix_time(), last_masterchain_block_id_, std::move(prev), - local_id, std::move(validator_set), actor_id(this), td::Timestamp::in(10.0), std::move(promise)); + run_collate_query(shard_id, last_masterchain_block_id_, std::move(prev), local_id, std::move(validator_set), + actor_id(this), td::Timestamp::in(10.0), std::move(promise)); } void ValidatorManagerImpl::get_required_block_candidates(td::Promise> promise) { diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index f5c357982..ab7f20627 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -51,7 +51,7 @@ void ValidatorGroup::generate_block_candidate(td::uint32 round_id, td::Promise prev, BlockIdExt min_masterchain_block_id, UnixTime min_ts) { +void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterchain_block_id) { prev_block_ids_ = prev; min_masterchain_block_id_ = min_masterchain_block_id; - min_ts_ = min_ts; started_ = true; if (init_) { diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 5d760899a..91d801cd0 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -47,7 +47,7 @@ class ValidatorGroup : public td::actor::Actor { BlockId create_next_block_id_simple() const; BlockIdExt create_next_block_id(RootHash root_hash, FileHash file_hash) const; - void start(std::vector prev, BlockIdExt min_masterchain_block_id, UnixTime min_ts); + void start(std::vector prev, BlockIdExt min_masterchain_block_id); void create_session(); void destroy(); void start_up() override { @@ -103,7 +103,6 @@ class ValidatorGroup : public td::actor::Actor { std::vector prev_block_ids_; BlockIdExt min_masterchain_block_id_; - UnixTime min_ts_; td::Ref validator_set_; validatorsession::ValidatorSessionOptions config_; From 996c23e506bdb930bc8517f6c6bb91809a89b051 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 18 Jul 2022 18:35:06 +0300 Subject: [PATCH 005/388] Shardchain validation without monitoring shardchains --- create-hardfork/create-hardfork.cpp | 4 +- overlay/overlay-manager.cpp | 9 +++ overlay/overlay-manager.h | 2 + overlay/overlay.cpp | 16 ++++- overlay/overlay.hpp | 7 ++ overlay/overlays.h | 2 + test/test-ton-collator.cpp | 4 +- validator-engine/validator-engine.cpp | 7 +- validator-engine/validator-engine.hpp | 4 ++ validator/fabric.h | 4 ++ validator/full-node-shard.cpp | 92 +++++++++++++++++++-------- validator/full-node-shard.h | 4 +- validator/full-node-shard.hpp | 6 +- validator/full-node.cpp | 24 +++---- validator/full-node.hpp | 2 +- validator/impl/accept-block.cpp | 31 +++++++++ validator/impl/accept-block.hpp | 6 ++ validator/impl/fabric.cpp | 11 ++++ validator/manager.cpp | 4 +- validator/validator-group.cpp | 49 ++++++-------- validator/validator-group.hpp | 6 +- validator/validator.h | 4 +- 22 files changed, 210 insertions(+), 88 deletions(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index 15533768d..849a31b12 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -236,9 +236,7 @@ class HardforkCreator : public td::actor::Actor { td::actor::send_closure(id_, &ton::validator::ValidatorManager::sync_complete, td::PromiseCreator::lambda([](td::Unit) {})); } - void add_shard(ton::ShardIdFull) override { - } - void del_shard(ton::ShardIdFull) override { + void subscribe_to_shard(ton::ShardIdFull) override { } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index 41750b4fa..2eac5a0c2 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -98,6 +98,15 @@ void OverlayManager::create_public_overlay(adnl::AdnlNodeIdShort local_id, Overl std::move(callback), std::move(rules), scope)); } +void OverlayManager::create_public_overlay_no_subscribe(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + OverlayPrivacyRules rules, td::string scope) { + CHECK(!dht_node_.empty()); + auto id = overlay_id.compute_short_id(); + register_overlay(local_id, id, + Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), nullptr, + std::move(rules), scope)); +} + void OverlayManager::create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules) { diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h index d9739bf38..b03261e80 100644 --- a/overlay/overlay-manager.h +++ b/overlay/overlay-manager.h @@ -52,6 +52,8 @@ class OverlayManager : public Overlays { void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) override; + void create_public_overlay_no_subscribe(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + OverlayPrivacyRules rules, td::string scope) override; void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules) override; diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index 3c707e601..a6d0bc89f 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -149,6 +149,9 @@ void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, if (R.is_error()) { // allow custom query to be here + if (!subscribed()) { + return; + } callback_->receive_query(src, overlay_id_, std::move(data), std::move(promise)); return; } @@ -225,6 +228,9 @@ void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice dat } auto X = fetch_tl_object(data.clone(), true); if (X.is_error()) { + if (!subscribed()) { + return; + } VLOG(OVERLAY_DEBUG) << this << ": received custom message"; callback_->receive_message(src, overlay_id_, std::move(data)); return; @@ -268,7 +274,7 @@ void OverlayImpl::alarm() { } if (public_) { - if (peers_.size() > 0) { + if (peers_.size() > 0 && subscribed()) { auto P = get_random_peer(); if (P) { send_random_peers(P->get_id(), {}); @@ -336,7 +342,7 @@ void OverlayImpl::receive_dht_nodes(td::Result res, bool dummy) { } void OverlayImpl::update_dht_nodes(OverlayNode node) { - if (!public_) { + if (!public_ || !subscribed()) { return; } @@ -497,6 +503,9 @@ void OverlayImpl::send_new_fec_broadcast_part(PublicKeyHash local_id, Overlay::B } void OverlayImpl::deliver_broadcast(PublicKeyHash source, td::BufferSlice data) { + if (!subscribed()) { + return; + } callback_->receive_broadcast(source, overlay_id_, std::move(data)); } @@ -569,6 +578,9 @@ void OverlayImpl::set_privacy_rules(OverlayPrivacyRules rules) { } void OverlayImpl::check_broadcast(PublicKeyHash src, td::BufferSlice data, td::Promise promise) { + if (!subscribed()) { + return; + } callback_->check_broadcast(src, overlay_id_, std::move(data), std::move(promise)); } diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 1d5f98611..ddcb02671 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -251,8 +251,15 @@ class OverlayImpl : public Overlay { } private: + bool subscribed() const { + return (bool)callback_; + } + template void process_query(adnl::AdnlNodeIdShort src, T &query, td::Promise promise) { + if (!subscribed()) { + return; + } callback_->receive_query(src, overlay_id_, serialize_tl_object(&query, true), std::move(promise)); } diff --git a/overlay/overlays.h b/overlay/overlays.h index 45316254e..6781b6c32 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -194,6 +194,8 @@ class Overlays : public td::actor::Actor { virtual void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) = 0; + virtual void create_public_overlay_no_subscribe(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + OverlayPrivacyRules rules, td::string scope) = 0; virtual void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules) = 0; diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index 9ed5c7814..1e46bc7df 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -323,9 +323,7 @@ class TestNode : public td::actor::Actor { td::actor::send_closure(id_, &ton::validator::ValidatorManager::sync_complete, td::PromiseCreator::lambda([](td::Unit) {})); } - void add_shard(ton::ShardIdFull) override { - } - void del_shard(ton::ShardIdFull) override { + void subscribe_to_shard(ton::ShardIdFull) override { } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 2e158e079..113455249 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1300,10 +1300,10 @@ td::Status ValidatorEngine::load_global_config() { validator_options_ = ton::validator::ValidatorManagerOptions::create(zero_state, init_block); validator_options_.write().set_shard_check_function( - [](ton::ShardIdFull shard, ton::CatchainSeqno cc_seqno, + [masterchain_only = masterchain_only_](ton::ShardIdFull shard, ton::CatchainSeqno cc_seqno, ton::validator::ValidatorManagerOptions::ShardCheckMode mode) -> bool { if (mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_monitor) { - return true; + return shard.is_masterchain() || !masterchain_only; } CHECK(mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_validate); return true; @@ -3648,6 +3648,9 @@ int main(int argc, char *argv[]) { }); return td::Status::OK(); }); + p.add_option('M', "masterchain-only", "don't track shardchains", [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_masterchain_only); }); + }); td::uint32 threads = 7; p.add_checked_option( 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice fname) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index beb24237d..0a7f8e264 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -202,6 +202,7 @@ class ValidatorEngine : public td::actor::Actor { bool started_ = false; ton::BlockSeqno truncate_seqno_{0}; std::string session_logs_file_; + bool masterchain_only_ = false; std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; @@ -253,6 +254,9 @@ class ValidatorEngine : public td::actor::Actor { void add_key_to_set(ton::PublicKey key) { keys_[key.compute_short_id()] = key; } + void set_masterchain_only() { + masterchain_only_ = true; + } void start_up() override; ValidatorEngine() { } diff --git a/validator/fabric.h b/validator/fabric.h index 158ea7e00..6f06dcc9d 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -59,6 +59,10 @@ void run_fake_accept_block_query(BlockIdExt id, td::Ref data, std::ve td::Promise promise); void run_hardfork_accept_block_query(BlockIdExt id, td::Ref data, td::actor::ActorId manager, td::Promise promise); +void run_broadcast_only_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, + td::Ref validator_set, td::Ref signatures, + td::Ref approve_signatures, + td::actor::ActorId manager, td::Promise promise); void run_apply_block_query(BlockIdExt id, td::Ref block, BlockIdExt masterchain_block_id, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 1094c8968..8e579f3c8 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -69,31 +69,41 @@ void Neighbour::update_roundtrip(double t) { } void FullNodeShardImpl::create_overlay() { - class Callback : public overlay::Overlays::Callback { - public: - void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { - // just ignore - } - void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, - td::Promise promise) override { - td::actor::send_closure(node_, &FullNodeShardImpl::receive_query, src, std::move(data), std::move(promise)); - } - void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { - td::actor::send_closure(node_, &FullNodeShardImpl::receive_broadcast, src, std::move(data)); - } - void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, - td::Promise promise) override { - td::actor::send_closure(node_, &FullNodeShardImpl::check_broadcast, src, std::move(data), std::move(promise)); - } - Callback(td::actor::ActorId node) : node_(node) { - } + if (subscribed_) { + class Callback : public overlay::Overlays::Callback { + public: + void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, + td::BufferSlice data) override { + // just ignore + } + void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + td::actor::send_closure(node_, &FullNodeShardImpl::receive_query, src, std::move(data), std::move(promise)); + } + void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { + td::actor::send_closure(node_, &FullNodeShardImpl::receive_broadcast, src, std::move(data)); + } + void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + td::actor::send_closure(node_, &FullNodeShardImpl::check_broadcast, src, std::move(data), std::move(promise)); + } + Callback(td::actor::ActorId node) : node_(node) { + } - private: - td::actor::ActorId node_; - }; + private: + td::actor::ActorId node_; + }; - td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay, adnl_id_, overlay_id_full_.clone(), - std::make_unique(actor_id(this)), rules_, PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() << ", \"workchain_id\": " << get_workchain() << " }"); + td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay, adnl_id_, overlay_id_full_.clone(), + std::make_unique(actor_id(this)), rules_, + PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() + << ", \"workchain_id\": " << get_workchain() << " }"); + } else { + td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay_no_subscribe, adnl_id_, + overlay_id_full_.clone(), rules_, + PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() + << ", \"workchain_id\": " << get_workchain() << " }"); + } td::actor::send_closure(rldp_, &rldp::Rldp::add_id, adnl_id_); if (cert_) { @@ -119,6 +129,15 @@ void FullNodeShardImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promis create_overlay(); } +void FullNodeShardImpl::subscribe_to_shard() { + if (subscribed_ || !client_.empty()) { + return; + } + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); + subscribed_ = true; + create_overlay(); +} + void FullNodeShardImpl::try_get_next_block(td::Timestamp timeout, td::Promise promise) { if (timeout.is_in_past()) { promise.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); @@ -576,8 +595,14 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ex } void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query) { + auto block_id = create_block_id(query.block_->block_); + if (block_id.shard_full() != shard_) { + LOG(WARNING) << "dropping newShardBlockBroadcast: expected shard " << shard_.to_str() << ", got shard " + << block_id.shard_full().to_str(); + return; + } td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block, - create_block_id(query.block_->block_), query.block_->cc_seqno_, + block_id, query.block_->cc_seqno_, std::move(query.block_->data_)); } @@ -588,6 +613,11 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_bl } BlockIdExt block_id = create_block_id(query.id_); + if (block_id.shard_full() != shard_) { + LOG(WARNING) << "dropping blockBroadcast: expected shard " << shard_.to_str() << ", got shard " + << block_id.shard_full().to_str(); + return; + } BlockBroadcast B{block_id, std::move(signatures), static_cast(query.catchain_seqno_), @@ -817,6 +847,10 @@ void FullNodeShardImpl::start_up() { } } +void FullNodeShardImpl::tear_down() { + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); +} + void FullNodeShardImpl::sign_new_certificate(PublicKeyHash sign_by) { if (sign_by.is_zero()) { return; @@ -1053,7 +1087,7 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client) + td::actor::ActorId client, bool subscribe) : shard_(shard) , local_id_(local_id) , adnl_id_(adnl_id) @@ -1063,16 +1097,18 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, , rldp_(rldp) , overlays_(overlays) , validator_manager_(validator_manager) - , client_(client) { + , client_(client) + , subscribed_(subscribe) { } td::actor::ActorOwn FullNodeShard::create( ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, - td::actor::ActorId validator_manager, td::actor::ActorId client) { + td::actor::ActorId validator_manager, td::actor::ActorId client, + bool subscribe) { return td::actor::create_actor("tonnode", shard, local_id, adnl_id, zero_state_file_hash, keyring, - adnl, rldp, overlays, validator_manager, client); + adnl, rldp, overlays, validator_manager, client, subscribe); } } // namespace fullnode diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index c1712baf2..e45e2325c 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -36,6 +36,7 @@ class FullNodeShard : public td::actor::Actor { virtual ShardIdFull get_shard_full() const = 0; virtual void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) = 0; + virtual void subscribe_to_shard() = 0; virtual void send_ihr_message(td::BufferSlice data) = 0; virtual void send_external_message(td::BufferSlice data) = 0; @@ -70,7 +71,8 @@ class FullNodeShard : public td::actor::Actor { ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, - td::actor::ActorId validator_manager, td::actor::ActorId client); + td::actor::ActorId validator_manager, td::actor::ActorId client, + bool subscribe); }; } // namespace fullnode diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index a2dd5cc46..33705bc79 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -80,6 +80,7 @@ class FullNodeShardImpl : public FullNodeShard { void create_overlay(); void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; + virtual void subscribe_to_shard() override; //td::Result fetch_block(td::BufferSlice data); void prevalidate_block(BlockIdExt block_id, td::BufferSlice data, td::BufferSlice proof, @@ -167,6 +168,7 @@ class FullNodeShardImpl : public FullNodeShard { void set_handle(BlockHandle handle, td::Promise promise) override; void start_up() override; + void tear_down() override; void alarm() override; void update_validators(std::vector public_key_hashes, PublicKeyHash local_hash) override; @@ -202,7 +204,7 @@ class FullNodeShardImpl : public FullNodeShard { td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client); + td::actor::ActorId client, bool subscribe); private: bool use_new_download() const { @@ -239,6 +241,8 @@ class FullNodeShardImpl : public FullNodeShard { td::Timestamp reload_neighbours_at_; td::Timestamp ping_neighbours_at_; adnl::AdnlNodeIdShort last_pinged_neighbour_ = adnl::AdnlNodeIdShort::zero(); + + bool subscribed_ = false; }; } // namespace fullnode diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 6606f2152..5712d3cf7 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -120,14 +120,17 @@ void FullNodeImpl::initial_read_complete(BlockHandle top_handle) { td::actor::send_closure(it->second, &FullNodeShard::set_handle, top_handle, std::move(P)); } -void FullNodeImpl::add_shard(ShardIdFull shard) { +void FullNodeImpl::add_shard(ShardIdFull shard, bool subscribe) { while (true) { - if (shards_.count(shard) == 0) { + auto it = shards_.find(shard); + if (it == shards_.end()) { shards_.emplace(shard, FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, keyring_, adnl_, - rldp_, overlays_, validator_manager_, client_)); + rldp_, overlays_, validator_manager_, client_, subscribe)); if (all_validators_.size() > 0) { td::actor::send_closure(shards_[shard], &FullNodeShard::update_validators, all_validators_, sign_cert_by_); } + } else if (subscribe) { + td::actor::send_closure(it->second, &FullNodeShard::subscribe_to_shard); } else { break; } @@ -148,7 +151,7 @@ void FullNodeImpl::sync_completed() { } void FullNodeImpl::send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) { - auto shard = get_shard(ShardIdFull{masterchainId}); + auto shard = get_shard(dst); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping OUT ihr message to unknown shard"; return; @@ -166,7 +169,7 @@ void FullNodeImpl::send_ext_message(AccountIdPrefixFull dst, td::BufferSlice dat } void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { - auto shard = get_shard(ShardIdFull{masterchainId, shardIdAll}); + auto shard = get_shard(block_id.shard_full()); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; return; @@ -175,7 +178,7 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s } void FullNodeImpl::send_broadcast(BlockBroadcast broadcast) { - auto shard = get_shard(ShardIdFull{masterchainId}); + auto shard = get_shard(broadcast.block_id.shard_full()); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; return; @@ -381,11 +384,8 @@ void FullNodeImpl::start_up() { void initial_read_complete(BlockHandle handle) override { td::actor::send_closure(id_, &FullNodeImpl::initial_read_complete, handle); } - void add_shard(ShardIdFull shard) override { - td::actor::send_closure(id_, &FullNodeImpl::add_shard, shard); - } - void del_shard(ShardIdFull shard) override { - td::actor::send_closure(id_, &FullNodeImpl::del_shard, shard); + void subscribe_to_shard(ShardIdFull shard) override { + td::actor::send_closure(id_, &FullNodeImpl::add_shard, shard, true); } void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) override { td::actor::send_closure(id_, &FullNodeImpl::send_ihr_message, dst, std::move(data)); @@ -465,7 +465,7 @@ FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id , validator_manager_(validator_manager) , client_(client) , db_root_(db_root) { - add_shard(ShardIdFull{masterchainId}); + add_shard(ShardIdFull{masterchainId}, true); } td::actor::ActorOwn FullNode::create(ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 6d57f4a82..f3ce9d050 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -52,7 +52,7 @@ class FullNodeImpl : public FullNode { void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; - void add_shard(ShardIdFull shard); + void add_shard(ShardIdFull shard, bool subscribe = false); void del_shard(ShardIdFull shard); void sync_completed(); diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index a5c4d9a12..74adcd696 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -92,6 +92,28 @@ AcceptBlockQuery::AcceptBlockQuery(ForceFork ffork, BlockIdExt id, td::Ref data, std::vector prev, + td::Ref validator_set, td::Ref signatures, + td::Ref approve_signatures, + td::actor::ActorId manager, td::Promise promise) + : id_(id) + , data_(std::move(data)) + , prev_(std::move(prev)) + , validator_set_(std::move(validator_set)) + , signatures_(std::move(signatures)) + , approve_signatures_(std::move(approve_signatures)) + , is_fake_(false) + , is_fork_(false) + , send_broadcast_(true) + , broadcast_only_(false) + , manager_(manager) + , promise_(std::move(promise)) { + state_keep_old_hash_.clear(); + state_old_hash_.clear(); + state_hash_.clear(); + CHECK(prev_.size() > 0); +} + bool AcceptBlockQuery::precheck_header() { VLOG(VALIDATOR_DEBUG) << "precheck_header()"; // 0. sanity check @@ -384,6 +406,15 @@ void AcceptBlockQuery::start_up() { return; } + if (broadcast_only_) { + if (!create_new_proof()) { + fatal_error("cannot generate proof for block "s + id_.to_str()); + return; + } + applied(); + return; + } + td::actor::send_closure( manager_, &ValidatorManager::get_block_handle, id_, true, [SelfId = actor_id(this)](td::Result R) { check_send_error(SelfId, R) || diff --git a/validator/impl/accept-block.hpp b/validator/impl/accept-block.hpp index 0b8cf2c32..b45095b4f 100644 --- a/validator/impl/accept-block.hpp +++ b/validator/impl/accept-block.hpp @@ -48,6 +48,7 @@ class AcceptBlockQuery : public td::actor::Actor { public: struct IsFake {}; struct ForceFork {}; + struct BroadcastOnly{}; AcceptBlockQuery(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, td::Ref approve_signatures, bool send_broadcast, @@ -57,6 +58,10 @@ class AcceptBlockQuery : public td::actor::Actor { td::Promise promise); AcceptBlockQuery(ForceFork ffork, BlockIdExt id, td::Ref data, td::actor::ActorId manager, td::Promise promise); + AcceptBlockQuery(BroadcastOnly, BlockIdExt id, td::Ref data, std::vector prev, + td::Ref validator_set, td::Ref signatures, + td::Ref approve_signatures, td::actor::ActorId manager, + td::Promise promise); private: static constexpr td::uint32 priority() { @@ -98,6 +103,7 @@ class AcceptBlockQuery : public td::actor::Actor { bool is_fake_; bool is_fork_; bool send_broadcast_; + bool broadcast_only_{false}; bool ancestors_split_{false}, is_key_block_{false}; td::Timestamp timeout_ = td::Timestamp::in(600.0); td::actor::ActorId manager_; diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index 6958ad890..e86b9b90d 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -151,6 +151,17 @@ void run_hardfork_accept_block_query(BlockIdExt id, td::Ref data, .release(); } +void run_broadcast_only_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, + td::Ref validator_set, td::Ref signatures, + td::Ref approve_signatures, + td::actor::ActorId manager, + td::Promise promise) { + td::actor::create_actor("broadcastaccept", AcceptBlockQuery::BroadcastOnly(), id, std::move(data), + prev, std::move(validator_set), std::move(signatures), + std::move(approve_signatures), manager, std::move(promise)) + .release(); +} + void run_apply_block_query(BlockIdExt id, td::Ref block, BlockIdExt masterchain_block_id, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise) { diff --git a/validator/manager.cpp b/validator/manager.cpp index 182ba9c3b..1d639291e 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2367,7 +2367,7 @@ void ValidatorManagerImpl::get_shard_client_state(bool from_db, td::Promiseadd_shard(shard); + callback_->subscribe_to_shard(shard); } void ValidatorManagerImpl::update_async_serializer_state(AsyncSerializerState state, td::Promise promise) { @@ -2396,7 +2396,7 @@ void ValidatorManagerImpl::get_archive_slice(td::uint64 archive_id, td::uint64 o } bool ValidatorManagerImpl::is_validator() { - return temp_keys_.size() > 0 || permanent_keys_.size() > 0; + return true; // temp_keys_.size() > 0 || permanent_keys_.size() > 0; } PublicKeyHash ValidatorManagerImpl::get_validator(ShardIdFull shard, td::Ref val_set) { diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index ab7f20627..9efda4131 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -121,9 +121,15 @@ void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash s td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, next_block_id, std::move(stats)); auto block = block_data.size() > 0 ? create_block(next_block_id, std::move(block_data)).move_as_ok() : td::Ref{}; + accept_block_query(next_block_id, std::move(block), std::move(prev_block_ids_), std::move(sig_set), + std::move(approve_sig_set), src == local_id_, std::move(promise)); + prev_block_ids_ = std::vector{next_block_id}; +} - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id = next_block_id, block, prev = prev_block_ids_, - sig_set, approve_sig_set, +void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref block, std::vector prev, + td::Ref sig_set, td::Ref approve_sig_set, + bool send_broadcast, td::Promise promise) { + auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { if (R.error().code() == ErrorCode::cancelled) { @@ -131,35 +137,22 @@ void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash s return; } LOG_CHECK(R.error().code() == ErrorCode::timeout || R.error().code() == ErrorCode::notready) << R.move_as_error(); - td::actor::send_closure(SelfId, &ValidatorGroup::retry_accept_block_query, block_id, std::move(block), - std::move(prev), std::move(sig_set), std::move(approve_sig_set), std::move(promise)); - } else { - promise.set_value(R.move_as_ok()); - } - }); - - run_accept_block_query(next_block_id, std::move(block), prev_block_ids_, validator_set_, std::move(sig_set), - std::move(approve_sig_set), src == local_id_, manager_, std::move(P)); - prev_block_ids_ = std::vector{next_block_id}; -} - -void ValidatorGroup::retry_accept_block_query(BlockIdExt block_id, td::Ref block, - std::vector prev, td::Ref sig_set, - td::Ref approve_sig_set, - td::Promise promise) { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id, block, prev, sig_set, approve_sig_set, - promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - LOG_CHECK(R.error().code() == ErrorCode::timeout) << R.move_as_error(); - td::actor::send_closure(SelfId, &ValidatorGroup::retry_accept_block_query, block_id, std::move(block), - std::move(prev), std::move(sig_set), std::move(approve_sig_set), std::move(promise)); + td::actor::send_closure(SelfId, &ValidatorGroup::accept_block_query, block_id, std::move(block), std::move(prev), + std::move(sig_set), std::move(approve_sig_set), false, std::move(promise)); } else { promise.set_value(R.move_as_ok()); } }); - run_accept_block_query(block_id, std::move(block), prev, validator_set_, std::move(sig_set), - std::move(approve_sig_set), false, manager_, std::move(P)); + if (shard_.is_masterchain() || lite_mode_) { + run_accept_block_query(block_id, std::move(block), std::move(prev), validator_set_, std::move(sig_set), + std::move(approve_sig_set), send_broadcast, manager_, std::move(P)); + } else if (send_broadcast) { + run_broadcast_only_accept_block_query(block_id, std::move(block), std::move(prev), validator_set_, + std::move(sig_set), std::move(approve_sig_set), manager_, std::move(P)); + } else { + promise.set_value(td::Unit()); + } } void ValidatorGroup::skip_round(td::uint32 round_id) { @@ -311,8 +304,8 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch auto block = p.block.size() > 0 ? create_block(next_block_id, std::move(p.block)).move_as_ok() : td::Ref{}; - retry_accept_block_query(next_block_id, std::move(block), prev_block_ids_, std::move(p.sigs), - std::move(p.approve_sigs), std::move(p.promise)); + accept_block_query(next_block_id, std::move(block), std::move(prev_block_ids_), std::move(p.sigs), + std::move(p.approve_sigs), false, std::move(p.promise)); prev_block_ids_ = std::vector{next_block_id}; } postponed_accept_.clear(); diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 91d801cd0..bd5e10430 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -39,9 +39,9 @@ class ValidatorGroup : public td::actor::Actor { std::vector approve_signatures, validatorsession::ValidatorSessionStats stats, td::Promise promise); void skip_round(td::uint32 round); - void retry_accept_block_query(BlockIdExt block_id, td::Ref block, std::vector prev, - td::Ref sigs, td::Ref approve_sigs, - td::Promise promise); + void accept_block_query(BlockIdExt block_id, td::Ref block, std::vector prev, + td::Ref sigs, td::Ref approve_sigs, + bool send_broadcast, td::Promise promise); void get_approved_candidate(PublicKey source, RootHash root_hash, FileHash file_hash, FileHash collated_data_file_hash, td::Promise promise); BlockId create_next_block_id_simple() const; diff --git a/validator/validator.h b/validator/validator.h index 09081f1e3..132ff06e6 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -111,8 +111,8 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual ~Callback() = default; virtual void initial_read_complete(BlockHandle top_masterchain_blocks) = 0; - virtual void add_shard(ShardIdFull shard) = 0; - virtual void del_shard(ShardIdFull shard) = 0; + virtual void subscribe_to_shard(ShardIdFull shard) = 0; + //virtual void del_shard(ShardIdFull shard) = 0; virtual void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; From 53270a00e660a7ae492aef93ca90c382a20101d6 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 20 Jul 2022 15:39:50 +0300 Subject: [PATCH 006/388] Add CollatorNode and make validators request blocks from it --- crypto/block/block.tlb | 4 + keys/keys.hpp | 4 + tl/generate/scheme/ton_api.tl | 12 +- tl/generate/scheme/ton_api.tlo | Bin 69376 -> 70276 bytes ton/ton-types.h | 8 + .../validator-engine-console-query.cpp | 21 +++ .../validator-engine-console-query.h | 24 +++ .../validator-engine-console.cpp | 1 + validator-engine/validator-engine.cpp | 99 ++++++++++-- validator-engine/validator-engine.hpp | 14 ++ validator/CMakeLists.txt | 2 + validator/collator-node.cpp | 145 ++++++++++++++++++ validator/collator-node.hpp | 51 ++++++ validator/impl/shard.cpp | 24 ++- validator/impl/shard.hpp | 4 + validator/impl/validate-query.cpp | 1 - validator/interfaces/shard.h | 2 + validator/manager-disk.hpp | 4 + validator/manager-hardfork.hpp | 3 + validator/manager.cpp | 12 +- validator/manager.hpp | 5 + validator/shard-client.cpp | 16 ++ validator/validator-group.cpp | 134 +++++++++++++--- validator/validator-group.hpp | 18 ++- validator/validator.h | 2 + 25 files changed, 577 insertions(+), 33 deletions(-) create mode 100644 validator/collator-node.cpp create mode 100644 validator/collator-node.hpp diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 20bacaa49..c9a95535f 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -739,6 +739,10 @@ _ OracleBridgeParams = ConfigParam 71; // Ethereum bridge _ OracleBridgeParams = ConfigParam 72; // Binance Smart Chain bridge _ OracleBridgeParams = ConfigParam 73; // Polygon bridge +// Set of collators: each collator is (workchain:int32 shard:int64 adnl_id:int256) +colator_set#a0 collators:(HashmapE 352 Unit) = CollatorSet; +_ CollatorSet = ConfigParam 81; + // // PROOFS // diff --git a/keys/keys.hpp b/keys/keys.hpp index cf883bbe6..6ed51f08f 100644 --- a/keys/keys.hpp +++ b/keys/keys.hpp @@ -253,6 +253,10 @@ class PublicKey { td::BufferSlice export_as_slice() const; static td::Result import(td::Slice s); + bool is_ed25519() const { + return pub_key_.get_offset() == pub_key_.offset(); + } + pubkeys::Ed25519 ed25519_value() const { CHECK(pub_key_.get_offset() == pub_key_.offset()); return pub_key_.get(); diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index eca152354..3a81ab585 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -572,6 +572,7 @@ engine.dht id:int256 = engine.Dht; engine.validatorTempKey key:int256 expire_at:int = engine.ValidatorTempKey; engine.validatorAdnlAddress id:int256 expire_at:int = engine.ValidatorAdnlAddress; engine.validator id:int256 temp_keys:(vector engine.validatorTempKey) adnl_addrs:(vector engine.validatorAdnlAddress) election_date:int expire_at:int = engine.Validator; +engine.collator adnl_id:int256 workchain:int shard:long = engine.Validator; engine.liteServer id:int256 port:int = engine.LiteServer; engine.controlProcess id:int256 permissions:int = engine.ControlProcess; engine.controlInterface id:int256 port:int allowed:(vector engine.controlProcess) = engine.ControlInterface; @@ -582,7 +583,7 @@ engine.validator.fullNodeMaster port:int adnl:int256 = engine.validator.FullNode engine.validator.fullNodeSlave ip:int port:int adnl:PublicKey = engine.validator.FullNodeSlave; engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) dht:(vector engine.dht) - validators:(vector engine.validator) fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave) + validators:(vector engine.Validator) fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave) fullnodemasters:(vector engine.validator.fullNodeMaster) liteservers:(vector engine.liteServer) control:(vector engine.controlInterface) gc:engine.gc = engine.validator.Config; @@ -690,6 +691,8 @@ engine.validator.generateBlockCandidate block_id:tonNode.BlockId = db.Candidate; engine.validator.getRequiredBlockCandidates = engine.validator.RequiredBlockCandidates; engine.validator.importBlockCandidate block:db.candidate = engine.validator.Success; +engine.validator.addCollator adnl_id:int256 workchain:int shard:long = engine.validator.Success; + ---types--- storage.pong = storage.Pong; @@ -741,3 +744,10 @@ validatorSession.statsRound timestamp:long producers:(vector validatorSession.st validatorSession.stats id:tonNode.blockId timestamp:long self:int256 creator:int256 total_validators:int total_weight:long signatures:int signatures_weight:long approve_signatures:int approve_signatures_weight:long first_round:int rounds:(vector validatorSession.statsRound) = validatorSession.Stats; + +collatorNode.generateBlockSuccess candidate:db.Candidate = collatorNode.GenerateBlockResult; +collatorNode.generateBlockError code:int message:string = collatorNode.GenerateBlockResult; + +---functions--- +collatorNode.generateBlock workchain:int shard:long min_mc_id:tonNode.blockIdExt prev_blocks:(vector tonNode.blockIdExt) + creator:int256 = collatorNode.GenerateBlockResult; \ No newline at end of file diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 252c6d0564df80fb0a38fe6fab78645f0e29d6ad..479455feb679ece457175e95b6b44e8c7807cfa9 100644 GIT binary patch delta 520 zcmZpe$I`Nth4;~HeJchiIJ=RTnM3M`LEd$R(RgNoW z*3-qm<#C(rT2z!@1ky9Pr6qs`#Mo>Q@vH6`y?Zl?Z!UW;%#7S)u6zE7;`CqRqz8 z@bGC_6Umso`R84KCQZ!?kJd@Sj0XoRN_fD-fpxOrQEy((+|0cA+~oMo6rf8dJ4T4} za2FJ%mc=K5!mfB~1Ea=dfpC_|I#*;SA28*B_|oRV9wyeyU(dA2OrH2yOb8NzK<(Yj To@~bO%j5?y1vdY9#GwNKPASF; delta 170 zcmZo!%F-~8h4;~HeJchiIJ1$LnPYPR#~n@<%~$aqlMRGKCV%0R+#D;!%*e7t^3b`> zDZ&B5j82=ksvXzeToQUmlM|%w<}o23*2#j6DwAvL_k(qAzSHo7i4~+ne6nGi2v1sO zI*0>O$UV8YT6^*jE0xI}Epu2w+8H;$Y%zwKuh6z8k}+}f+57%Xn-v~!VA@>sj6(+i DhTB6J diff --git a/ton/ton-types.h b/ton/ton-types.h index 8cb73de6d..754fb86a1 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -397,6 +397,9 @@ struct Ed25519_PublicKey { bool operator==(const Ed25519_PublicKey& other) const { return _pubkey == other._pubkey; } + bool operator!=(const Ed25519_PublicKey& other) const { + return _pubkey != other._pubkey; + } bool clear() { _pubkey.set_zero(); return true; @@ -479,4 +482,9 @@ struct ValidatorSessionConfig { static const td::uint32 BLOCK_HASH_COVERS_DATA_FROM_VERSION = 2; }; +struct CollatorNodeDescr { + ShardIdFull shard; + NodeIdShort adnl_id; +}; + } // namespace ton diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 028c5b1ae..9c5b6572d 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1091,3 +1091,24 @@ td::Status ImportBlockCandidateQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << "successfully imported a block candidate\n"; return td::Status::OK(); } + +td::Status AddCollatorQuery::run() { + TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); + return td::Status::OK(); +} + +td::Status AddCollatorQuery::send() { + auto b = ton::create_serialize_tl_object( + adnl_id_.tl(), wc_, shard_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status AddCollatorQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "successfully added collator\n"; + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index e4236f6dc..34b2ede3e 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1163,3 +1163,27 @@ class ImportBlockCandidateQuery : public Query { private: std::string file_; }; + +class AddCollatorQuery : public Query { + public: + AddCollatorQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "addcollator"; + } + static std::string get_help() { + return "addcollator \tadd collator with given adnl_id and shard"; + } + std::string name() const override { + return get_name(); + } + + private: + ton::PublicKeyHash adnl_id_; + td::int32 wc_; + td::int64 shard_; +}; diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index b4dcad22b..d70e2cf2e 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -144,6 +144,7 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 113455249..09526279f 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -125,15 +125,24 @@ Config::Config(ton::ton_api::engine_validator_config &config) { for (auto &dht : config.dht_) { config_add_dht_node(ton::PublicKeyHash{dht->id_}).ensure(); } - for (auto &val : config.validators_) { - auto key = ton::PublicKeyHash{val->id_}; - config_add_validator_permanent_key(key, val->election_date_, val->expire_at_).ensure(); - for (auto &temp : val->temp_keys_) { - config_add_validator_temp_key(key, ton::PublicKeyHash{temp->key_}, temp->expire_at_).ensure(); - } - for (auto &adnl : val->adnl_addrs_) { - config_add_validator_adnl_id(key, ton::PublicKeyHash{adnl->id_}, adnl->expire_at_).ensure(); - } + for (auto &v : config.validators_) { + ton::ton_api::downcast_call( + *v, td::overloaded( + [&](ton::ton_api::engine_validator &val) { + auto key = ton::PublicKeyHash{val.id_}; + config_add_validator_permanent_key(key, val.election_date_, val.expire_at_).ensure(); + for (auto &temp : val.temp_keys_) { + config_add_validator_temp_key(key, ton::PublicKeyHash{temp->key_}, temp->expire_at_).ensure(); + } + for (auto &adnl : val.adnl_addrs_) { + config_add_validator_adnl_id(key, ton::PublicKeyHash{adnl->id_}, adnl->expire_at_).ensure(); + } + }, + [&](ton::ton_api::engine_collator &col) { + auto key = ton::PublicKeyHash{col.adnl_id_}; + ton::ShardIdFull shard(col.workchain_, col.shard_); + config_add_collator(key, shard).ensure(); + })); } config_add_full_node_adnl_id(ton::PublicKeyHash{config.fullnode_}).ensure(); @@ -192,7 +201,7 @@ ton::tl_object_ptr Config::tl() const { dht_vec.push_back(ton::create_tl_object(x.tl())); } - std::vector> val_vec; + std::vector> val_vec; for (auto &val : validators) { std::vector> temp_vec; for (auto &t : val.second.temp_keys) { @@ -205,6 +214,10 @@ ton::tl_object_ptr Config::tl() const { val_vec.push_back(ton::create_tl_object( val.first.tl(), std::move(temp_vec), std::move(adnl_val_vec), val.second.election_date, val.second.expire_at)); } + for (auto &col : collators) { + val_vec.push_back(ton::create_tl_object( + col.adnl_id.tl(), col.shard.workchain, col.shard.shard)); + } std::vector> full_node_slaves_vec; for (auto &x : full_node_slaves) { @@ -383,6 +396,18 @@ td::Result Config::config_add_validator_adnl_id(ton::PublicKeyHash perm_ke } } +td::Result Config::config_add_collator(ton::PublicKeyHash addr, ton::ShardIdFull shard) { + if (!shard.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard: " << shard.to_str()); + } + Collator c{addr, shard}; + if (std::find(collators.begin(), collators.end(), c) != collators.end()) { + return false; + } + collators.push_back(c); + return true; +} + td::Result Config::config_add_full_node_adnl_id(ton::PublicKeyHash id) { if (full_node == id) { return false; @@ -1836,6 +1861,19 @@ void ValidatorEngine::start_lite_server() { } void ValidatorEngine::started_lite_server() { + start_collator(); +} + +void ValidatorEngine::start_collator() { + for (auto &c : config_.collators) { + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_collator, + ton::adnl::AdnlNodeIdShort(c.adnl_id), c.shard); + } + + started_collator(); +} + +void ValidatorEngine::started_collator() { start_control_interface(); } @@ -3395,6 +3433,47 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importBlo }); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollator &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + auto id = ton::PublicKeyHash{query.adnl_id_}; + auto shard = ton::ShardIdFull(query.workchain_, query.shard_); + if (!shard.is_valid_ext()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "invalid shard"))); + return; + } + + auto R = config_.config_add_collator(id, shard); + if (R.is_error()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!R.move_as_ok()) { + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + return; + } + if (!validator_manager_.empty()) { + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_collator, + ton::adnl::AdnlNodeIdShort(id), shard); + } + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 0a7f8e264..037288cee 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -66,6 +66,14 @@ struct Config { ton::UnixTime election_date; ton::UnixTime expire_at; }; + struct Collator { + ton::PublicKeyHash adnl_id; + ton::ShardIdFull shard; + + bool operator==(const Collator& b) const { + return adnl_id == b.adnl_id && shard == b.shard; + } + }; struct Control { ton::PublicKeyHash key; std::map clients; @@ -81,6 +89,7 @@ struct Config { std::map adnl_ids; std::set dht_ids; std::map validators; + std::vector collators; ton::PublicKeyHash full_node = ton::PublicKeyHash::zero(); std::vector full_node_slaves; std::map full_node_masters; @@ -104,6 +113,7 @@ struct Config { ton::UnixTime expire_at); td::Result config_add_validator_adnl_id(ton::PublicKeyHash perm_key, ton::PublicKeyHash adnl_id, ton::UnixTime expire_at); + td::Result config_add_collator(ton::PublicKeyHash addr, ton::ShardIdFull shard); td::Result config_add_full_node_adnl_id(ton::PublicKeyHash id); td::Result config_add_full_node_slave(td::IPAddress addr, ton::PublicKey id); td::Result config_add_full_node_master(td::int32 port, ton::PublicKeyHash id); @@ -293,6 +303,8 @@ class ValidatorEngine : public td::actor::Actor { void add_lite_server(ton::PublicKeyHash id, td::uint16 port); void start_lite_server(); void started_lite_server(); + void start_collator(); + void started_collator(); void add_control_interface(ton::PublicKeyHash id, td::uint16 port); void add_control_process(ton::PublicKeyHash id, td::uint16 port, ton::PublicKeyHash pub, td::int32 permissions); @@ -419,6 +431,8 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_importBlockCandidate &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_addCollator &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index 65d7b3485..443ae23ea 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -53,6 +53,7 @@ set(VALIDATOR_HEADERS import-db-slice.hpp + collator-node.hpp manager-disk.h manager-disk.hpp manager-init.h @@ -68,6 +69,7 @@ set(VALIDATOR_HEADERS set(VALIDATOR_SOURCE apply-block.cpp block-handle.cpp + collator-node.cpp get-next-key-blocks.cpp import-db-slice.cpp shard-client.cpp diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp new file mode 100644 index 000000000..958b5f136 --- /dev/null +++ b/validator/collator-node.cpp @@ -0,0 +1,145 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "collator-node.hpp" +#include "ton/ton-tl.hpp" +#include "fabric.h" + +namespace ton { + +namespace validator { + +CollatorNode::CollatorNode(adnl::AdnlNodeIdShort local_id, td::actor::ActorId manager, + td::actor::ActorId adnl, td::actor::ActorId rldp) + : local_id_(local_id), manager_(std::move(manager)), adnl_(std::move(adnl)), rldp_(std::move(rldp)) { +} + +void CollatorNode::start_up() { + class Cb : public adnl::Adnl::Callback { + public: + Cb(td::actor::ActorId id) : id_(std::move(id)) { + } + void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { + } + void receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + td::actor::send_closure(id_, &CollatorNode::receive_query, src, std::move(data), std::move(promise)); + } + + private: + td::actor::ActorId id_; + }; + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlock::ID), + std::make_unique(actor_id(this))); + td::actor::send_closure(rldp_, &rldp::Rldp::add_id, adnl::AdnlNodeIdShort(local_id_)); +} + +void CollatorNode::tear_down() { + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlock::ID)); +} + +void CollatorNode::add_shard(ShardIdFull shard) { + LOG(INFO) << "Collator node: local_id=" << local_id_ << " , shard=" << shard.to_str(); + shards_.push_back(shard); +} + +static td::BufferSlice serialize_error(td::Status error) { + return create_serialize_tl_object(error.code(), error.message().c_str()); +} + +void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, + td::Promise promise) { + auto status = [&]() -> td::Status { + TRY_RESULT(f, fetch_tl_object(std::move(data), true)); + ShardIdFull shard(f->workchain_, f->shard_); + if (!shard.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard " << shard.to_str()); + } + bool found = false; + for (ShardIdFull our_shard : shards_) { + if (shard_is_ancestor(shard, our_shard)) { + found = true; + break; + } + } + if (!found) { + return td::Status::Error(PSTRING() << "this node doesn't collate shard " << shard.to_str()); + } + if (f->prev_blocks_.size() != 1 && f->prev_blocks_.size() != 2) { + return td::Status::Error(PSTRING() << "invalid size of prev_blocks: " << f->prev_blocks_.size()); + } + std::vector prev_blocks; + for (const auto& b : f->prev_blocks_) { + prev_blocks.push_back(create_block_id(b)); + } + BlockIdExt min_mc_id = create_block_id(f->min_mc_id_); + Ed25519_PublicKey creator(f->creator_); + + LOG(INFO) << "Query from " << src << ": shard=" << shard.to_str() << ", min_mc_id=" << min_mc_id.to_str(); + + auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this), prev_blocks = std::move(prev_blocks), + promise = std::move(promise)](td::Result> R) mutable { + if (R.is_error()) { + LOG(WARNING) << "Query from " << src << ", error: " << R.error(); + promise.set_result(serialize_error(R.move_as_error_prefix("failed to get masterchain state: "))); + } else { + td::Ref state(R.move_as_ok()); + if (state.is_null()) { + LOG(WARNING) << "Query from " << src << ", error: failed to get masterchain state"; + promise.set_result(serialize_error(R.move_as_error_prefix("failed to get masterchain state: "))); + return; + } + td::actor::send_closure(SelfId, &CollatorNode::receive_query_cont, src, shard, std::move(state), + std::move(prev_blocks), creator, std::move(promise)); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, min_mc_id, 1, td::Timestamp::in(5.0), + std::move(P)); + return td::Status::OK(); + }(); + if (status.is_error()) { + LOG(WARNING) << "Query from " << src << ", error: " << status; + promise.set_result(serialize_error(std::move(status))); + } +} + +void CollatorNode::receive_query_cont(adnl::AdnlNodeIdShort src, ShardIdFull shard, + td::Ref min_mc_state, std::vector prev_blocks, + Ed25519_PublicKey creator, td::Promise promise) { + auto P = td::PromiseCreator::lambda([promise = std::move(promise), src](td::Result R) mutable { + if (R.is_error()) { + LOG(WARNING) << "Query from " << src << ", error: " << R.error(); + promise.set_result(serialize_error(R.move_as_error())); + } else { + LOG(INFO) << "Query from " << src << ", success"; + auto block = R.move_as_ok(); + auto result = create_serialize_tl_object( + create_tl_object(PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), + create_tl_block_id(block.id), std::move(block.data), + std::move(block.collated_data))); + promise.set_result(std::move(result)); + } + }); + + run_collate_query(shard, min_mc_state->get_block_id(), std::move(prev_blocks), creator, + min_mc_state->get_validator_set(shard), manager_, td::Timestamp::in(10.0), std::move(P)); +} + +} // namespace validator + +} // namespace ton diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp new file mode 100644 index 000000000..854e98ce2 --- /dev/null +++ b/validator/collator-node.hpp @@ -0,0 +1,51 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "interfaces/validator-manager.h" +#include "rldp/rldp.h" + +namespace ton { + +namespace validator { + +class ValidatorManager; + +class CollatorNode : public td::actor::Actor { + public: + CollatorNode(adnl::AdnlNodeIdShort local_id, td::actor::ActorId manager, + td::actor::ActorId adnl, td::actor::ActorId rldp); + void start_up() override; + void tear_down() override; + void add_shard(ShardIdFull shard); + + private: + void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); + void receive_query_cont(adnl::AdnlNodeIdShort src, ShardIdFull shard, td::Ref min_mc_state, + std::vector prev_blocks, Ed25519_PublicKey creator, + td::Promise promise); + + adnl::AdnlNodeIdShort local_id_; + td::actor::ActorId manager_; + td::actor::ActorId adnl_; + td::actor::ActorId rldp_; + std::vector shards_; +}; + +} // namespace validator + +} // namespace ton diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index a96f1a819..dbf244002 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -364,7 +364,8 @@ td::Status MasterchainStateQ::mc_init() { td::Status MasterchainStateQ::mc_reinit() { auto res = block::ConfigInfo::extract_config( root_cell(), block::ConfigInfo::needStateRoot | block::ConfigInfo::needValidatorSet | - block::ConfigInfo::needShardHashes | block::ConfigInfo::needPrevBlocks); + block::ConfigInfo::needShardHashes | block::ConfigInfo::needPrevBlocks | + block::ConfigInfo::needWorkchainInfo); cur_validators_.reset(); next_validators_.reset(); if (res.is_error()) { @@ -510,6 +511,27 @@ bool MasterchainStateQ::check_old_mc_block_id(const ton::BlockIdExt& blkid, bool return config_ && config_->check_old_mc_block_id(blkid, strict); } +std::vector MasterchainStateQ::get_collator_set() const { + block::gen::CollatorSet::Record rec; + auto cell = config_->get_config_param(81); + if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { + return {}; + } + vm::Dictionary dict{rec.collators->prefetch_ref(), 32 + 64 + 256}; + std::vector collators; + dict.check_for_each([&](Ref, td::ConstBitPtr key, int n) { + CHECK(n == 32 + 64 + 256); + auto workchain = (td::int32)key.get_int(32); + key.advance(32); + td::uint64 shard = key.get_uint(64); + key.advance(64); + td::Bits256 adnl_id(key); + collators.push_back({ShardIdFull(workchain, shard), adnl_id}); + return true; + }); + return collators; +} + td::uint32 MasterchainStateQ::min_split_depth(WorkchainId workchain_id) const { if (!config_) { return 0; diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index 0056df119..2b411d944 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -147,6 +147,10 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { return td::make_ref(config_); } } + block::WorkchainSet get_workchain_list() const override { + return config_ ? config_->get_workchain_list() : block::WorkchainSet(); + } + std::vector get_collator_set() const override; private: ZeroStateIdExt zerostate_id_; diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index aac85c8f1..369f1f2a8 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -448,7 +448,6 @@ bool ValidateQuery::init_parse() { return reject_query("after_merge value mismatch in block header"); } rand_seed_ = extra.rand_seed; - created_by_ = extra.created_by; if (created_by_ != extra.created_by) { return reject_query("block candidate "s + id_.to_str() + " has creator " + created_by_.to_hex() + " but the block header contains different value " + extra.created_by.to_hex()); diff --git a/validator/interfaces/shard.h b/validator/interfaces/shard.h index 96ca8da26..ff458d604 100644 --- a/validator/interfaces/shard.h +++ b/validator/interfaces/shard.h @@ -81,6 +81,8 @@ class MasterchainState : virtual public ShardState { ton::LogicalTime* end_lt = nullptr) const = 0; virtual bool check_old_mc_block_id(const ton::BlockIdExt& blkid, bool strict = false) const = 0; virtual td::Result> get_config_holder() const = 0; + virtual block::WorkchainSet get_workchain_list() const = 0; + virtual std::vector get_collator_set() const = 0; virtual td::Status prepare() { return td::Status::OK(); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 6bad11715..91235ebf6 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -384,6 +384,10 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } + void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { + UNREACHABLE(); + } + private: PublicKeyHash local_id_; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index a8aaecfc3..80f6df93c 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -443,6 +443,9 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } + void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { + UNREACHABLE(); + } private: td::Ref opts_; diff --git a/validator/manager.cpp b/validator/manager.cpp index 1d639291e..9af007726 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2025,7 +2025,8 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group auto validator_id = get_validator(shard, validator_set); CHECK(!validator_id.is_zero()); auto G = td::actor::create_actor( - "validatorgroup", shard, validator_id, session_id, validator_set, opts, keyring_, adnl_, rldp_, overlays_, + "validatorgroup", shard, validator_id, session_id, validator_set, last_masterchain_state_->get_collator_set(), + opts, keyring_, adnl_, rldp_, overlays_, db_root_, actor_id(this), init_session, opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), true); return G; @@ -2692,6 +2693,15 @@ void ValidatorManagerImpl::cleanup_old_pending_candidates(BlockId block_id, td:: } } +void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) { + auto it = collator_nodes_.find(id); + if (it == collator_nodes_.end()) { + auto actor = td::actor::create_actor("collatornode", id, actor_id(this), adnl_, rldp_); + it = collator_nodes_.emplace(id, std::move(actor)).first; + } + td::actor::send_closure(it->second, &CollatorNode::add_shard, shard); +} + td::actor::ActorOwn ValidatorManagerFactory::create( td::Ref opts, std::string db_root, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, diff --git a/validator/manager.hpp b/validator/manager.hpp index 1fbf60a9d..94657d49d 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -28,6 +28,7 @@ #include "state-serializer.hpp" #include "rldp/rldp.h" #include "token-manager.h" +#include "collator-node.hpp" #include #include @@ -539,6 +540,8 @@ class ValidatorManagerImpl : public ValidatorManager { void import_block_candidate(BlockCandidate candidate, td::Promise promise) override; void wait_block_candidate(BlockId block_id, td::Timestamp timeout, td::Promise promise) override; + void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; + private: td::Timestamp resend_shard_blocks_at_; td::Timestamp check_waiters_at_; @@ -606,6 +609,8 @@ class ValidatorManagerImpl : public ValidatorManager { std::map, td::Timestamp>>> pending_block_candidates_; void cleanup_old_pending_candidates(BlockId block_id, td::Timestamp now); + + std::map> collator_nodes_; }; } // namespace validator diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index c7d87fb42..af44f16e8 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -248,9 +248,11 @@ void ShardClient::get_processed_masterchain_block_id(td::Promise pro void ShardClient::build_shard_overlays() { auto v = masterchain_state_->get_shards(); + std::set workchains; for (auto &x : v) { auto shard = x->shard(); + workchains.insert(shard.workchain); if (opts_->need_monitor(shard)) { auto d = masterchain_state_->soft_min_split_depth(shard.workchain); auto l = shard_prefix_length(shard.shard); @@ -264,6 +266,20 @@ void ShardClient::build_shard_overlays() { } } } + + for (const auto &wpair : masterchain_state_->get_workchain_list()) { + ton::WorkchainId wc = wpair.first; + const block::WorkchainInfo *winfo = wpair.second.get(); + if (workchains.count(wc) == 0 && winfo->active && winfo->enabled_since <= masterchain_state_->get_unix_time()) { + auto shard = ShardIdFull(wc); + if (opts_->need_monitor(shard) && created_overlays_.count(shard) == 0) { + td::actor::send_closure(manager_, &ValidatorManager::subscribe_to_shard, shard); + BlockIdExt block_id(shard.workchain, shard.shard, 0, winfo->zerostate_root_hash, winfo->zerostate_file_hash); + td::actor::send_closure_later(manager_, &ValidatorManager::wait_block_state_short, block_id, 0, + td::Timestamp::in(5.0), [](td::Result>) {}); + } + } + } } void ShardClient::force_update_shard_client(BlockHandle handle, td::Promise promise) { diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 9efda4131..bb493f912 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -22,6 +22,7 @@ #include "td/utils/overloaded.h" #include "common/delay.h" #include "ton/ton-tl.hpp" +#include "td/utils/Random.h" namespace ton { @@ -36,19 +37,7 @@ void ValidatorGroup::generate_block_candidate(td::uint32 round_id, td::Promise R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - } else { - BlockCandidate candidate = R.move_as_ok(); - candidate.pubkey = pubkey; - promise.set_result(std::move(candidate)); - } - }); - td::actor::send_closure(manager_, &ValidatorManager::wait_block_candidate, create_next_block_id_simple(), - td::Timestamp::in(15.0), std::move(P)); + send_collate_query(round_id, td::Timestamp::in(10.0), std::move(promise)); return; } run_collate_query(shard_, min_masterchain_block_id_, prev_block_ids_, @@ -65,6 +54,17 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat promise.set_error(td::Status::Error(ErrorCode::notready, "too old")); return; } + + if (approved_candidates_cache_round_ != round_id) { + approved_candidates_cache_round_ = round_id; + approved_candidates_cache_.clear(); + } + CacheKey cache_key = block_to_cache_key(block); + auto it = approved_candidates_cache_.find(cache_key); + if (it != approved_candidates_cache_.end()) { + promise.set_result(it->second); + } + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), round_id, block = block.clone(), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { @@ -80,11 +80,16 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat td::Timestamp::in(0.1)); } else { auto v = R.move_as_ok(); - v.visit(td::overloaded([&](UnixTime ts) { promise.set_result(ts); }, - [&](CandidateReject reject) { - promise.set_error(td::Status::Error(ErrorCode::protoviolation, - PSTRING() << "bad candidate: " << reject.reason)); - })); + v.visit(td::overloaded( + [&](UnixTime ts) { + td::actor::send_closure(SelfId, &ValidatorGroup::update_approve_cache, round_id, block_to_cache_key(block), + ts); + promise.set_result(ts); + }, + [&](CandidateReject reject) { + promise.set_error( + td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad candidate: " << reject.reason)); + })); } }); if (!started_) { @@ -98,6 +103,14 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat td::Timestamp::in(10.0), std::move(P), lite_mode_ ? ValidateMode::lite : 0); } +void ValidatorGroup::update_approve_cache(td::uint32 round_id, CacheKey key, UnixTime value) { + if (approved_candidates_cache_round_ != round_id) { + approved_candidates_cache_round_ = round_id; + approved_candidates_cache_.clear(); + } + approved_candidates_cache_[key] = value; +} + void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash src, td::BufferSlice block_data, RootHash root_hash, FileHash file_hash, std::vector signatures, @@ -339,6 +352,91 @@ void ValidatorGroup::get_session_info( td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_session_info, std::move(P)); } +void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeout, + td::Promise promise) { + adnl::AdnlNodeIdShort collator = adnl::AdnlNodeIdShort::zero(); + // TODO: some other way for storing and choosing collators for real network + int cnt = 0; + for (const CollatorNodeDescr& c : collator_set_) { + if (shard_is_ancestor(shard_, c.shard) && td::Random::fast(0, cnt) == 0) { + collator = adnl::AdnlNodeIdShort(c.adnl_id); + ++cnt; + } + } + + if (collator.is_zero()) { + promise.set_error(td::Status::Error(PSTRING() << "no collator for shard " << shard_.to_str())); + return; + } + + std::vector> prev_blocks; + for (const BlockIdExt &p : prev_block_ids_) { + prev_blocks.push_back(create_tl_block_id(p)); + } + td::BufferSlice query = create_serialize_tl_object( + shard_.workchain, shard_.shard, create_tl_block_id(min_masterchain_block_id_), std::move(prev_blocks), + local_id_full_.ed25519_value().raw()); + + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), round_id, promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix("rldp query failed: ")); + return; + } + td::actor::send_closure(SelfId, &ValidatorGroup::receive_collate_query_response, round_id, R.move_as_ok(), + std::move(promise)); + }); + LOG(INFO) << "collate query for " << shard_.to_str() << ": send query to " << collator; + size_t max_answer_size = config_.max_block_size + config_.max_collated_data_size + 256; + td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, adnl::AdnlNodeIdShort(local_id_), collator, "collatequery", + std::move(P), timeout, std::move(query), max_answer_size); +} + +void ValidatorGroup::receive_collate_query_response(td::uint32 round_id, td::BufferSlice data, + td::Promise promise) { + if (round_id < last_known_round_id_) { + promise.set_error(td::Status::Error("too old")); + return; + } + TRY_RESULT_PROMISE(promise, f, fetch_tl_object(data, true)); + tl_object_ptr b; + ton_api::downcast_call(*f, td::overloaded( + [&](ton_api::collatorNode_generateBlockError &r) { + td::Status error = td::Status::Error(r.code_, r.message_); + promise.set_error(error.move_as_error_prefix("collate query: ")); + }, + [&](ton_api::collatorNode_generateBlockSuccess &r) { b = std::move(r.candidate_); })); + if (!b) { + return; + } + auto key = PublicKey{b->source_}; + if (!key.is_ed25519()) { + promise.set_error(td::Status::Error("collate query: block candidate source mismatch")); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + if (e_key != Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}) { + promise.set_error(td::Status::Error("collate query: block candidate source mismatch")); + return; + } + auto block_id = ton::create_block_id(b->id_); + if (block_id.shard_full() != shard_) { + promise.set_error(td::Status::Error("collate query: shard mismatch")); + return; + } + auto collated_data_hash = td::sha256_bits256(b->collated_data_); + BlockCandidate candidate(e_key, block_id, collated_data_hash, std::move(b->data_), std::move(b->collated_data_)); + + auto P = td::PromiseCreator::lambda( + [candidate = candidate.clone(), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix("validate received block error: ")); + return; + } + promise.set_result(std::move(candidate)); + }); + validate_block_candidate(round_id, std::move(candidate), std::move(P)); +} + } // namespace validator } // namespace ton diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index bd5e10430..d0a96d659 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -55,12 +55,14 @@ class ValidatorGroup : public td::actor::Actor { init_ = false; create_session(); } + td::actor::send_closure(rldp_, &rldp::Rldp::add_id, adnl::AdnlNodeIdShort(local_id_)); } void get_session_info(td::Promise> promise); ValidatorGroup(ShardIdFull shard, PublicKeyHash local_id, ValidatorSessionId session_id, - td::Ref validator_set, validatorsession::ValidatorSessionOptions config, + td::Ref validator_set, std::vector collator_set, + validatorsession::ValidatorSessionOptions config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, td::actor::ActorId validator_manager, bool create_session, @@ -69,6 +71,7 @@ class ValidatorGroup : public td::actor::Actor { , local_id_(std::move(local_id)) , session_id_(session_id) , validator_set_(std::move(validator_set)) + , collator_set_(std::move(collator_set)) , config_(std::move(config)) , keyring_(keyring) , adnl_(adnl) @@ -83,6 +86,8 @@ class ValidatorGroup : public td::actor::Actor { private: std::unique_ptr make_validator_session_callback(); + void send_collate_query(td::uint32 round_id, td::Timestamp timeout, td::Promise promise); + void receive_collate_query_response(td::uint32 round_id, td::BufferSlice data, td::Promise promise); struct PostponedAccept { RootHash root_hash; @@ -105,6 +110,7 @@ class ValidatorGroup : public td::actor::Actor { BlockIdExt min_masterchain_block_id_; td::Ref validator_set_; + std::vector collator_set_; validatorsession::ValidatorSessionOptions config_; td::actor::ActorId keyring_; @@ -120,6 +126,16 @@ class ValidatorGroup : public td::actor::Actor { bool allow_unsafe_self_blocks_resync_; bool lite_mode_ = false; td::uint32 last_known_round_id_ = 0; + + typedef std::tuple CacheKey; + std::map approved_candidates_cache_; + td::uint32 approved_candidates_cache_round_ = 0; + + void update_approve_cache(td::uint32 round_id, CacheKey key, UnixTime value); + + static CacheKey block_to_cache_key(const BlockCandidate& block) { + return std::make_tuple(block.pubkey.as_bits256(), block.id, sha256_bits256(block.data), block.collated_file_hash); + } }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 132ff06e6..66007910b 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -220,6 +220,8 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void generate_block_candidate(BlockId block_id, td::Promise promise) = 0; virtual void get_required_block_candidates(td::Promise> promise) = 0; virtual void import_block_candidate(BlockCandidate candidate, td::Promise promise) = 0; + + virtual void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; }; } // namespace validator From 7ac60bea7d5858f91fc218ca13a59c4388647f0e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 29 Jul 2022 10:39:02 +0300 Subject: [PATCH 007/388] New shard overlays --- create-hardfork/create-hardfork.cpp | 2 +- overlay/overlay-manager.cpp | 19 +- overlay/overlay-manager.h | 4 +- overlay/overlay-peers.cpp | 4 +- overlay/overlay.cpp | 78 ++++++-- overlay/overlay.hpp | 8 +- overlay/overlays.h | 4 +- test/test-ton-collator.cpp | 2 +- ton/ton-types.h | 2 + validator-engine/validator-engine.cpp | 67 +++++-- validator-engine/validator-engine.hpp | 1 + validator/fabric.h | 4 +- validator/full-node-shard.cpp | 44 +++-- validator/full-node-shard.h | 4 +- validator/full-node-shard.hpp | 6 +- validator/full-node.cpp | 239 +++++++++++++++++------ validator/full-node.h | 5 +- validator/full-node.hpp | 27 ++- validator/impl/accept-block.cpp | 44 +---- validator/impl/accept-block.hpp | 9 +- validator/impl/fabric.cpp | 15 +- validator/interfaces/validator-manager.h | 6 +- validator/manager-disk.hpp | 2 +- validator/manager-hardfork.hpp | 3 +- validator/manager.cpp | 16 +- validator/manager.hpp | 3 +- validator/net/download-block-new.cpp | 4 +- validator/shard-client.cpp | 34 +--- validator/validator-group.cpp | 18 +- validator/validator.h | 8 +- 30 files changed, 411 insertions(+), 271 deletions(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index 849a31b12..27260ee02 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -236,7 +236,7 @@ class HardforkCreator : public td::actor::Actor { td::actor::send_closure(id_, &ton::validator::ValidatorManager::sync_complete, td::PromiseCreator::lambda([](td::Unit) {})); } - void subscribe_to_shard(ton::ShardIdFull) override { + void update_shard_configuration(td::Ref state) override { } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index 2eac5a0c2..62d98eb00 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -92,14 +92,15 @@ void OverlayManager::delete_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdSho void OverlayManager::create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) { CHECK(!dht_node_.empty()); + CHECK(callback != nullptr); auto id = overlay_id.compute_short_id(); register_overlay(local_id, id, Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), std::move(callback), std::move(rules), scope)); } -void OverlayManager::create_public_overlay_no_subscribe(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, - OverlayPrivacyRules rules, td::string scope) { +void OverlayManager::create_public_overlay_external(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + OverlayPrivacyRules rules, td::string scope) { CHECK(!dht_node_.empty()); auto id = overlay_id.compute_short_id(); register_overlay(local_id, id, @@ -266,12 +267,16 @@ void OverlayManager::get_overlay_random_peers(adnl::AdnlNodeIdShort local_id, Ov td::uint32 max_peers, td::Promise> promise) { auto it = overlays_.find(local_id); - if (it != overlays_.end()) { - auto it2 = it->second.find(overlay_id); - if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::get_overlay_random_peers, max_peers, std::move(promise)); - } + if (it == overlays_.end()) { + promise.set_error(td::Status::Error(PSTRING() << "no such local id " << local_id)); + return; + } + auto it2 = it->second.find(overlay_id); + if (it2 == it->second.end()) { + promise.set_error(td::Status::Error(PSTRING() << "no such overlay " << overlay_id)); + return; } + td::actor::send_closure(it2->second, &Overlay::get_overlay_random_peers, max_peers, std::move(promise)); } td::actor::ActorOwn Overlays::create(std::string db_root, td::actor::ActorId keyring, diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h index b03261e80..a3ed4a54a 100644 --- a/overlay/overlay-manager.h +++ b/overlay/overlay-manager.h @@ -52,8 +52,8 @@ class OverlayManager : public Overlays { void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) override; - void create_public_overlay_no_subscribe(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, - OverlayPrivacyRules rules, td::string scope) override; + void create_public_overlay_external(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + OverlayPrivacyRules rules, td::string scope) override; void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules) override; diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index 0539383f9..dbb6f32a8 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -142,7 +142,9 @@ void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::BufferSlic void OverlayImpl::send_random_peers_cont(adnl::AdnlNodeIdShort src, OverlayNode node, td::Promise promise) { std::vector> vec; - vec.emplace_back(node.tl()); + if (!is_external()) { + vec.emplace_back(node.tl()); + } for (td::uint32 i = 0; i < nodes_to_send(); i++) { auto P = get_random_peer(); diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index a6d0bc89f..8a655c6e5 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -18,6 +18,7 @@ */ #include "auto/tl/ton_api.h" #include "td/utils/Random.h" +#include "common/delay.h" #include "adnl/utils.hpp" #include "dht/dht.h" @@ -73,7 +74,12 @@ OverlayImpl::OverlayImpl(td::actor::ActorId keyring, td::actor , scope_(scope) { overlay_id_ = id_full_.compute_short_id(); - VLOG(OVERLAY_INFO) << this << ": creating " << (public_ ? "public" : "private"); + if (is_external()) { + CHECK(public_); + VLOG(OVERLAY_INFO) << this << ": creating public external"; + } else { + VLOG(OVERLAY_INFO) << this << ": creating " << (public_ ? "public" : "private"); + } for (auto &node : nodes) { CHECK(!public_); @@ -86,6 +92,7 @@ OverlayImpl::OverlayImpl(td::actor::ActorId keyring, td::actor void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeers &query, td::Promise promise) { + CHECK(!is_external()); if (public_) { VLOG(OVERLAY_DEBUG) << this << ": received " << query.peers_->nodes_.size() << " nodes from " << src << " in getRandomPeers query"; @@ -106,6 +113,7 @@ void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getR void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcast &query, td::Promise promise) { + CHECK(!is_external()); auto it = broadcasts_.find(query.hash_); if (it == broadcasts_.end()) { VLOG(OVERLAY_NOTICE) << this << ": received getBroadcastQuery(" << query.hash_ << ") from " << src @@ -127,16 +135,17 @@ void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getB void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcastList &query, td::Promise promise) { + CHECK(!is_external()); VLOG(OVERLAY_WARNING) << this << ": DROPPING getBroadcastList query"; promise.set_error(td::Status::Error(ErrorCode::protoviolation, "dropping get broadcast list query")); } -/*void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, adnl::AdnlQueryId query_id, ton_api::overlay_customQuery &query) { - callback_->receive_query(src, query_id, id_, std::move(query.data_)); -} -*/ - void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise) { + if (is_external()) { + LOG(OVERLAY_WARNING) << "dropping query in external overlay " << overlay_id_; + promise.set_error(td::Status::Error("overlay is external")); + return; + } if (!public_) { auto P = peers_.get(src); if (P == nullptr) { @@ -149,9 +158,6 @@ void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, if (R.is_error()) { // allow custom query to be here - if (!subscribed()) { - return; - } callback_->receive_query(src, overlay_id_, std::move(data), std::move(promise)); return; } @@ -165,27 +171,32 @@ void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr bcast) { + CHECK(!is_external()); return BroadcastSimple::create(this, message_from, std::move(bcast)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr b) { + CHECK(!is_external()); return OverlayFecBroadcastPart::create(this, message_from, std::move(b)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr b) { + CHECK(!is_external()); return OverlayFecBroadcastPart::create(this, message_from, std::move(b)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr bcast) { + CHECK(!is_external()); return td::Status::Error(ErrorCode::protoviolation, PSTRING() << "received strange message broadcastNotFound from " << message_from); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg) { + CHECK(!is_external()); auto it = fec_broadcasts_.find(msg->hash_); if (it != fec_broadcasts_.end()) { VLOG(OVERLAY_DEBUG) << this << ": received fec opt-out message from " << message_from << " for broadcast " @@ -200,6 +211,7 @@ td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg) { + CHECK(!is_external()); auto it = fec_broadcasts_.find(msg->hash_); if (it != fec_broadcasts_.end()) { VLOG(OVERLAY_DEBUG) << this << ": received fec completed message from " << message_from << " for broadcast " @@ -214,12 +226,17 @@ td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg) { + CHECK(!is_external()); VLOG(OVERLAY_DEBUG) << this << ": received unicast from " << message_from; callback_->receive_message(message_from, overlay_id_, std::move(msg->data_)); return td::Status::OK(); } void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) { + if (is_external()) { + LOG(OVERLAY_WARNING) << "dropping message in external overlay " << overlay_id_; + return; + } if (!public_) { if (peers_.get(src) == nullptr) { VLOG(OVERLAY_WARNING) << this << ": received query in private overlay from unknown source " << src; @@ -228,9 +245,6 @@ void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice dat } auto X = fetch_tl_object(data.clone(), true); if (X.is_error()) { - if (!subscribed()) { - return; - } VLOG(OVERLAY_DEBUG) << this << ": received custom message"; callback_->receive_message(src, overlay_id_, std::move(data)); return; @@ -274,7 +288,7 @@ void OverlayImpl::alarm() { } if (public_) { - if (peers_.size() > 0 && subscribed()) { + if (peers_.size() > 0) { auto P = get_random_peer(); if (P) { send_random_peers(P->get_id(), {}); @@ -330,6 +344,10 @@ void OverlayImpl::receive_dht_nodes(td::Result res, bool dummy) { VLOG(OVERLAY_NOTICE) << this << ": can not get value from DHT: " << res.move_as_error(); } + if (is_external()) { + return; + } + VLOG(OVERLAY_INFO) << this << ": adding self node to DHT overlay's nodes"; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), oid = print_id()](td::Result R) { if (R.is_error()) { @@ -342,7 +360,7 @@ void OverlayImpl::receive_dht_nodes(td::Result res, bool dummy) { } void OverlayImpl::update_dht_nodes(OverlayNode node) { - if (!public_ || !subscribed()) { + if (!public_) { return; } @@ -399,12 +417,30 @@ void OverlayImpl::bcast_gc() { } void OverlayImpl::send_message_to_neighbours(td::BufferSlice data) { + if (neighbours_.empty()) { + // TODO: limit retries + delay_action( + [SelfId = actor_id(this), data = std::move(data)]() mutable { + td::actor::send_closure(SelfId, &OverlayImpl::send_message_to_neighbours, std::move(data)); + }, + td::Timestamp::in(0.5)); + return; + } for (auto &n : neighbours_) { td::actor::send_closure(manager_, &OverlayManager::send_message, n, local_id_, overlay_id_, data.clone()); } } void OverlayImpl::send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) { + if (neighbours_.empty()) { + // TODO: limit retries + delay_action( + [SelfId = actor_id(this), send_as, flags, data = std::move(data)]() mutable { + td::actor::send_closure(SelfId, &OverlayImpl::send_broadcast, send_as, flags, std::move(data)); + }, + td::Timestamp::in(0.5)); + return; + } auto S = BroadcastSimple::create_new(actor_id(this), keyring_, send_as, std::move(data), flags); if (S.is_error()) { LOG(WARNING) << "failed to send broadcast: " << S; @@ -412,6 +448,15 @@ void OverlayImpl::send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::Bu } void OverlayImpl::send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) { + if (neighbours_.empty()) { + // TODO: limit retries + delay_action( + [SelfId = actor_id(this), send_as, flags, data = std::move(data)]() mutable { + td::actor::send_closure(SelfId, &OverlayImpl::send_broadcast_fec, send_as, flags, std::move(data)); + }, + td::Timestamp::in(0.5)); + return; + } OverlayOutboundFecBroadcast::create(std::move(data), flags, actor_id(this), send_as); } @@ -503,7 +548,7 @@ void OverlayImpl::send_new_fec_broadcast_part(PublicKeyHash local_id, Overlay::B } void OverlayImpl::deliver_broadcast(PublicKeyHash source, td::BufferSlice data) { - if (!subscribed()) { + if (is_external()) { return; } callback_->receive_broadcast(source, overlay_id_, std::move(data)); @@ -578,7 +623,8 @@ void OverlayImpl::set_privacy_rules(OverlayPrivacyRules rules) { } void OverlayImpl::check_broadcast(PublicKeyHash src, td::BufferSlice data, td::Promise promise) { - if (!subscribed()) { + if (is_external()) { + promise.set_result(td::Unit()); return; } callback_->check_broadcast(src, overlay_id_, std::move(data), std::move(promise)); diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index ddcb02671..59b3ec06a 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -251,15 +251,13 @@ class OverlayImpl : public Overlay { } private: - bool subscribed() const { - return (bool)callback_; + bool is_external() const { + return callback_ == nullptr; } template void process_query(adnl::AdnlNodeIdShort src, T &query, td::Promise promise) { - if (!subscribed()) { - return; - } + CHECK(!is_external()); callback_->receive_query(src, overlay_id_, serialize_tl_object(&query, true), std::move(promise)); } diff --git a/overlay/overlays.h b/overlay/overlays.h index 6781b6c32..82617fab6 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -194,8 +194,8 @@ class Overlays : public td::actor::Actor { virtual void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) = 0; - virtual void create_public_overlay_no_subscribe(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, - OverlayPrivacyRules rules, td::string scope) = 0; + virtual void create_public_overlay_external(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + OverlayPrivacyRules rules, td::string scope) = 0; virtual void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules) = 0; diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index 1e46bc7df..b3d3f9db4 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -323,7 +323,7 @@ class TestNode : public td::actor::Actor { td::actor::send_closure(id_, &ton::validator::ValidatorManager::sync_complete, td::PromiseCreator::lambda([](td::Unit) {})); } - void subscribe_to_shard(ton::ShardIdFull) override { + void update_shard_configuration(td::Ref state) override { } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } diff --git a/ton/ton-types.h b/ton/ton-types.h index 754fb86a1..7d0101d10 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -57,6 +57,8 @@ constexpr unsigned min_split_merge_interval = 30; // split/merge interval must constexpr unsigned max_split_merge_delay = 1000; // end of split/merge interval must be at most 1000 seconds in the future +constexpr int max_shard_pfx_len = 60; + enum GlobalCapabilities { capIhrEnabled = 1, capCreateStatsEnabled = 2, diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 09526279f..1bb820a42 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1324,18 +1324,6 @@ td::Status ValidatorEngine::load_global_config() { } validator_options_ = ton::validator::ValidatorManagerOptions::create(zero_state, init_block); - validator_options_.write().set_shard_check_function( - [masterchain_only = masterchain_only_](ton::ShardIdFull shard, ton::CatchainSeqno cc_seqno, - ton::validator::ValidatorManagerOptions::ShardCheckMode mode) -> bool { - if (mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_monitor) { - return shard.is_masterchain() || !masterchain_only; - } - CHECK(mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_validate); - return true; - /*ton::ShardIdFull p{ton::basechainId, ((cc_seqno * 1ull % 4) << 62) + 1}; - auto s = ton::shard_prefix(p, 2); - return shard.is_masterchain() || ton::shard_intersects(shard, s);*/ - }); if (state_ttl_ != 0) { validator_options_.write().set_state_ttl(state_ttl_); } @@ -1389,6 +1377,39 @@ td::Status ValidatorEngine::load_global_config() { return td::Status::OK(); } +void ValidatorEngine::init_validator_options() { + if (!masterchain_only_) { + validator_options_.write().set_shard_check_function( + [](ton::ShardIdFull shard, ton::CatchainSeqno cc_seqno, + ton::validator::ValidatorManagerOptions::ShardCheckMode mode) -> bool { + if (mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_monitor) { + return true; + } + CHECK(mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_validate); + return true; + }); + } else { + std::vector shards = {ton::ShardIdFull(ton::masterchainId)}; + for (const auto& c : config_.collators) { + shards.push_back(c.shard); + } + validator_options_.write().set_shard_check_function( + [shards = std::move(shards)](ton::ShardIdFull shard, ton::CatchainSeqno cc_seqno, + ton::validator::ValidatorManagerOptions::ShardCheckMode mode) -> bool { + if (mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_monitor) { + for (auto s : shards) { + if (shard_is_ancestor(shard, s)) { + return true; + } + } + return false; + } + CHECK(mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_validate); + return true; + }); + } +} + void ValidatorEngine::load_empty_local_config(td::Promise promise) { auto ret_promise = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { @@ -1671,6 +1692,7 @@ void ValidatorEngine::got_key(ton::PublicKey key) { } void ValidatorEngine::start() { + init_validator_options(); read_config_ = true; start_adnl(); } @@ -1825,19 +1847,22 @@ void ValidatorEngine::start_full_node() { }; full_node_client_ = ton::adnl::AdnlExtMultiClient::create(std::move(vec), std::make_unique()); } + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &ValidatorEngine::started_full_node); + }); full_node_ = ton::validator::fullnode::FullNode::create( short_id, ton::adnl::AdnlNodeIdShort{config_.full_node}, validator_options_->zero_block_id().file_hash, - keyring_.get(), adnl_.get(), rldp_.get(), + validator_options_, keyring_.get(), adnl_.get(), rldp_.get(), default_dht_node_.is_zero() ? td::actor::ActorId{} : dht_nodes_[default_dht_node_].get(), - overlay_manager_.get(), validator_manager_.get(), full_node_client_.get(), db_root_); - } - - for (auto &v : config_.validators) { - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_permanent_key, v.first, - [](td::Unit) {}); + overlay_manager_.get(), validator_manager_.get(), full_node_client_.get(), db_root_, std::move(P)); + for (auto &v : config_.validators) { + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_permanent_key, v.first, + [](td::Unit) {}); + } + } else { + started_full_node(); } - - started_full_node(); } void ValidatorEngine::started_full_node() { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 037288cee..bf47e2d79 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -276,6 +276,7 @@ class ValidatorEngine : public td::actor::Actor { void load_empty_local_config(td::Promise promise); void load_local_config(td::Promise promise); void load_config(td::Promise promise); + void init_validator_options(); void start(); diff --git a/validator/fabric.h b/validator/fabric.h index 6f06dcc9d..12452ec55 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -52,7 +52,7 @@ void run_check_external_message(td::BufferSlice data, td::actor::ActorId data, std::vector prev, td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, bool send_broadcast, + td::Ref approve_signatures, bool send_broadcast, bool apply, td::actor::ActorId manager, td::Promise promise); void run_fake_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::actor::ActorId manager, @@ -61,7 +61,7 @@ void run_hardfork_accept_block_query(BlockIdExt id, td::Ref data, td::actor::ActorId manager, td::Promise promise); void run_broadcast_only_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, + td::Ref approve_signatures, bool send_block_broadcast, td::actor::ActorId manager, td::Promise promise); void run_apply_block_query(BlockIdExt id, td::Ref block, BlockIdExt masterchain_block_id, td::actor::ActorId manager, td::Timestamp timeout, diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 8e579f3c8..7eb00db8f 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -69,7 +69,7 @@ void Neighbour::update_roundtrip(double t) { } void FullNodeShardImpl::create_overlay() { - if (subscribed_) { + if (active_) { class Callback : public overlay::Overlays::Callback { public: void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, @@ -99,7 +99,7 @@ void FullNodeShardImpl::create_overlay() { PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() << ", \"workchain_id\": " << get_workchain() << " }"); } else { - td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay_no_subscribe, adnl_id_, + td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay_external, adnl_id_, overlay_id_full_.clone(), rules_, PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() << ", \"workchain_id\": " << get_workchain() << " }"); @@ -129,12 +129,12 @@ void FullNodeShardImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promis create_overlay(); } -void FullNodeShardImpl::subscribe_to_shard() { - if (subscribed_ || !client_.empty()) { +void FullNodeShardImpl::set_active(bool active) { + if (active_ == active || shard_.is_masterchain()) { return; } td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); - subscribed_ = true; + active_ = active; create_overlay(); } @@ -576,6 +576,10 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise) { + if (!active_) { + promise.set_error(td::Status::Error("shard is inactive")); + return; + } auto B = fetch_tl_object(std::move(query), true); if (B.is_error()) { promise.set_error(td::Status::Error(ErrorCode::protoviolation, "cannot parse tonnode query")); @@ -596,28 +600,23 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ex void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query) { auto block_id = create_block_id(query.block_->block_); - if (block_id.shard_full() != shard_) { - LOG(WARNING) << "dropping newShardBlockBroadcast: expected shard " << shard_.to_str() << ", got shard " - << block_id.shard_full().to_str(); - return; - } td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block, block_id, query.block_->cc_seqno_, std::move(query.block_->data_)); } void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { - std::vector signatures; - for (auto &sig : query.signatures_) { - signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)}); - } - BlockIdExt block_id = create_block_id(query.id_); if (block_id.shard_full() != shard_) { - LOG(WARNING) << "dropping blockBroadcast: expected shard " << shard_.to_str() << ", got shard " - << block_id.shard_full().to_str(); + LOG(FULL_NODE_WARNING) << "dropping block broadcast: shard mismatch. overlay=" << shard_.to_str() + << " block=" << block_id.to_str(); return; } + + std::vector signatures; + for (auto &sig : query.signatures_) { + signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)}); + } BlockBroadcast B{block_id, std::move(signatures), static_cast(query.catchain_seqno_), @@ -639,6 +638,9 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_bl } void FullNodeShardImpl::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { + if (!active_) { + return; + } auto B = fetch_tl_object(std::move(broadcast), true); if (B.is_error()) { return; @@ -1087,7 +1089,7 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, bool subscribe) + td::actor::ActorId client, bool active) : shard_(shard) , local_id_(local_id) , adnl_id_(adnl_id) @@ -1098,7 +1100,7 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, , overlays_(overlays) , validator_manager_(validator_manager) , client_(client) - , subscribed_(subscribe) { + , active_(shard.is_masterchain() || active) { } td::actor::ActorOwn FullNodeShard::create( @@ -1106,9 +1108,9 @@ td::actor::ActorOwn FullNodeShard::create( td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, - bool subscribe) { + bool active) { return td::actor::create_actor("tonnode", shard, local_id, adnl_id, zero_state_file_hash, keyring, - adnl, rldp, overlays, validator_manager, client, subscribe); + adnl, rldp, overlays, validator_manager, client, active); } } // namespace fullnode diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index e45e2325c..2b2098a69 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -36,7 +36,7 @@ class FullNodeShard : public td::actor::Actor { virtual ShardIdFull get_shard_full() const = 0; virtual void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) = 0; - virtual void subscribe_to_shard() = 0; + virtual void set_active(bool active) = 0; virtual void send_ihr_message(td::BufferSlice data) = 0; virtual void send_external_message(td::BufferSlice data) = 0; @@ -72,7 +72,7 @@ class FullNodeShard : public td::actor::Actor { td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, - bool subscribe); + bool active = true); }; } // namespace fullnode diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 33705bc79..9a022cbf0 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -80,7 +80,7 @@ class FullNodeShardImpl : public FullNodeShard { void create_overlay(); void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; - virtual void subscribe_to_shard() override; + void set_active(bool active) override; //td::Result fetch_block(td::BufferSlice data); void prevalidate_block(BlockIdExt block_id, td::BufferSlice data, td::BufferSlice proof, @@ -204,7 +204,7 @@ class FullNodeShardImpl : public FullNodeShard { td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, bool subscribe); + td::actor::ActorId client, bool active = true); private: bool use_new_download() const { @@ -242,7 +242,7 @@ class FullNodeShardImpl : public FullNodeShard { td::Timestamp ping_neighbours_at_; adnl::AdnlNodeIdShort last_pinged_neighbour_ = adnl::AdnlNodeIdShort::zero(); - bool subscribed_ = false; + bool active_ = false; }; } // namespace fullnode diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 5712d3cf7..9c26cf3c4 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -20,6 +20,7 @@ #include "ton/ton-shard.h" #include "ton/ton-io.hpp" #include "td/actor/MultiPromise.h" +#include "ton/ton-types.h" namespace ton { @@ -27,6 +28,8 @@ namespace validator { namespace fullnode { +static const double INACTIVE_SHARD_TTL = 120.0; + void FullNodeImpl::add_permanent_key(PublicKeyHash key, td::Promise promise) { if (local_keys_.count(key)) { promise.set_value(td::Unit()); @@ -47,7 +50,9 @@ void FullNodeImpl::add_permanent_key(PublicKeyHash key, td::Promise pr } for (auto &shard : shards_) { - td::actor::send_closure(shard.second, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + if (!shard.second.actor.empty()) { + td::actor::send_closure(shard.second.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + } } promise.set_value(td::Unit()); } @@ -71,7 +76,9 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise pr } for (auto &shard : shards_) { - td::actor::send_closure(shard.second, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + if (!shard.second.actor.empty()) { + td::actor::send_closure(shard.second.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + } } promise.set_value(td::Unit()); } @@ -79,22 +86,25 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise pr void FullNodeImpl::sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) { - auto it = shards_.find(shard_id); - if(it == shards_.end()) { - promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); - return; - } - td::actor::send_closure(it->second, &FullNodeShard::sign_overlay_certificate, signed_key, expiry_at, max_size, std::move(promise)); + auto it = shards_.find(shard_id); + if(it == shards_.end() || it->second.actor.empty()) { + promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); + return; + } + td::actor::send_closure(it->second.actor, &FullNodeShard::sign_overlay_certificate, signed_key, expiry_at, max_size, + std::move(promise)); } void FullNodeImpl::import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) { - auto it = shards_.find(shard_id); - if(it == shards_.end()) { - promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); - } - td::actor::send_closure(it->second, &FullNodeShard::import_overlay_certificate, signed_key, cert, std::move(promise)); + auto it = shards_.find(shard_id); + if(it == shards_.end() || it->second.actor.empty()) { + promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); + return; + } + td::actor::send_closure(it->second.actor, &FullNodeShard::import_overlay_certificate, signed_key, cert, + std::move(promise)); } void FullNodeImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) { @@ -105,7 +115,9 @@ void FullNodeImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promisesecond, &FullNodeShard::set_handle, top_handle, std::move(P)); + CHECK(it != shards_.end() && !it->second.actor.empty()); + td::actor::send_closure(it->second.actor, &FullNodeShard::set_handle, top_handle, std::move(P)); } -void FullNodeImpl::add_shard(ShardIdFull shard, bool subscribe) { - while (true) { - auto it = shards_.find(shard); - if (it == shards_.end()) { - shards_.emplace(shard, FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, keyring_, adnl_, - rldp_, overlays_, validator_manager_, client_, subscribe)); - if (all_validators_.size() > 0) { - td::actor::send_closure(shards_[shard], &FullNodeShard::update_validators, all_validators_, sign_cert_by_); +void FullNodeImpl::update_shard_configuration(td::Ref state) { + std::map new_shards; + std::set new_active; + new_shards[ShardIdFull(masterchainId)] = state->get_block_id(); + new_active.insert(ShardIdFull(masterchainId)); + std::set workchains; + auto cur_time = state->get_unix_time(); + + auto set_active = [&](ShardIdFull shard) { + while (new_active.insert(shard).second && shard.pfx_len() > 0) { + shard = shard_parent(shard); + } + }; + + for (auto &info : state->get_shards()) { + auto shard = info->shard(); + workchains.insert(shard.workchain); + new_shards[shard] = info->top_block_id(); + bool will_split = shard.pfx_len() < max_shard_pfx_len && ((info->fsm_state() == McShardHash::FsmState::fsm_split && + info->fsm_utime() < cur_time + 60) || info->before_split()); + bool will_merge = shard.pfx_len() > 0 && ((info->fsm_state() == McShardHash::FsmState::fsm_merge && + info->fsm_utime() < cur_time + 60) || info->before_merge()); + if (opts_->need_monitor(shard)) { + set_active(shard); + } + if (will_merge && opts_->need_monitor(shard_parent(shard))) { + set_active(shard); + set_active(shard_sibling(shard)); + } + for (int id = 0; id < 2; ++id) { + if (will_split && opts_->need_monitor(shard_child(shard, id))) { + set_active(shard_child(shard, id)); } - } else if (subscribe) { - td::actor::send_closure(it->second, &FullNodeShard::subscribe_to_shard); - } else { - break; } - if (shard.shard == shardIdAll) { - break; + } + for (const auto &wpair : state->get_workchain_list()) { + ton::WorkchainId wc = wpair.first; + const block::WorkchainInfo *winfo = wpair.second.get(); + if (workchains.count(wc) == 0 && winfo->active && winfo->enabled_since <= cur_time) { + auto shard = ShardIdFull(wc); + new_shards[shard] = BlockIdExt(wc, shard.shard, 0, winfo->zerostate_root_hash, winfo->zerostate_file_hash); + if (opts_->need_monitor(shard)) { + set_active(shard); + } + } + } + + auto info_set_active = [&](ShardIdFull shard, ShardInfo& info, bool active) { + if (info.active == active) { + return; + } + if (info.actor.empty()) { + add_shard_actor(shard, active); + return; + } + info.active = active; + td::actor::send_closure(info.actor, &FullNodeShard::set_active, active); + info.delete_at = active ? td::Timestamp::never() : td::Timestamp::in(INACTIVE_SHARD_TTL); + }; + + for (auto shard : new_shards) { + auto &info = shards_[shard.first]; + info.exists = true; + if (!info.active && new_active.count(shard.first)) { + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::wait_block_state_short, shard.second, 0, + td::Timestamp::in(60.0), [](td::Result>){}); + } + } + + for (auto& p : shards_) { + ShardIdFull shard = p.first; + ShardInfo &info = p.second; + info.exists = new_shards.count(shard); + info_set_active(shard, info, new_active.count(shard)); + } + + for (ShardIdFull shard : new_active) { + info_set_active(shard, shards_[shard], true); + } + + auto it = shards_.begin(); + while (it != shards_.end()) { + if (!it->second.active && it->second.delete_at && it->second.delete_at.is_in_past()) { + it->second.actor.reset(); + it->second.delete_at = td::Timestamp::never(); + } + if (!it->second.exists && it->second.actor.empty()) { + it = shards_.erase(it); + } else { + ++it; } - shard = shard_parent(shard); } } -void FullNodeImpl::del_shard(ShardIdFull shard) { - LOG(FATAL) << "deleting shards not implemented: shard=" << shard; - shards_.erase(shard); +void FullNodeImpl::add_shard_actor(ShardIdFull shard, bool active) { + ShardInfo &info = shards_[shard]; + if (!info.actor.empty()) { + return; + } + info.actor = FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, keyring_, adnl_, rldp_, + overlays_, validator_manager_, client_, active); + info.active = active; + info.delete_at = active ? td::Timestamp::never() : td::Timestamp::in(INACTIVE_SHARD_TTL); + if (all_validators_.size() > 0) { + td::actor::send_closure(info.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + } } void FullNodeImpl::sync_completed() { @@ -169,7 +263,7 @@ void FullNodeImpl::send_ext_message(AccountIdPrefixFull dst, td::BufferSlice dat } void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { - auto shard = get_shard(block_id.shard_full()); + auto shard = get_shard(ShardIdFull{masterchainId}); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; return; @@ -178,7 +272,7 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s } void FullNodeImpl::send_broadcast(BlockBroadcast broadcast) { - auto shard = get_shard(broadcast.block_id.shard_full()); + auto shard = get_shard(broadcast.block_id.shard_full(), true); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; return; @@ -262,19 +356,38 @@ void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, std::string tm std::move(promise)); } -td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { - add_shard(ShardIdFull{shard.workchain, shardIdAll}); - while (shards_.count(shard) == 0) { - if (shard.shard == shardIdAll) { - return td::actor::ActorId{}; +td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard, bool exact) { + if (!exact) { + ShardIdFull s = shard; + while (true) { + auto it = shards_.find(s); + if (it != shards_.end() && it->second.exists) { + if (it->second.actor.empty()) { + add_shard_actor(s, false); + } + if (!it->second.active) { + it->second.delete_at = td::Timestamp::in(INACTIVE_SHARD_TTL); + } + return it->second.actor.get(); + } + if (s.pfx_len() == 0) { + break; + } + s = shard_parent(s); } - shard = shard_parent(shard); } - return shards_[shard].get(); + auto &info = shards_[shard]; + if (info.actor.empty()) { + add_shard_actor(shard, false); + } + if (!info.active) { + info.delete_at = td::Timestamp::in(INACTIVE_SHARD_TTL); + } + return info.actor.get(); } td::actor::ActorId FullNodeImpl::get_shard(AccountIdPrefixFull dst) { - return get_shard(shard_prefix(dst, 60)); + return get_shard(shard_prefix(dst, max_shard_pfx_len)); } void FullNodeImpl::got_key_block_proof(td::Ref proof) { @@ -307,7 +420,7 @@ void FullNodeImpl::got_key_block_proof(td::Ref proof) { CHECK(all_validators_.size() > 0); for (auto &shard : shards_) { - td::actor::send_closure(shard.second, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + td::actor::send_closure(shard.second.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); } } @@ -339,7 +452,9 @@ void FullNodeImpl::got_zero_block_state(td::Ref state) { CHECK(all_validators_.size() > 0); for (auto &shard : shards_) { - td::actor::send_closure(shard.second, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + if (!shard.second.actor.empty()) { + td::actor::send_closure(shard.second.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + } } } @@ -384,8 +499,8 @@ void FullNodeImpl::start_up() { void initial_read_complete(BlockHandle handle) override { td::actor::send_closure(id_, &FullNodeImpl::initial_read_complete, handle); } - void subscribe_to_shard(ShardIdFull shard) override { - td::actor::send_closure(id_, &FullNodeImpl::add_shard, shard, true); + void update_shard_configuration(td::Ref state) override { + td::actor::send_closure(id_, &FullNodeImpl::update_shard_configuration, std::move(state)); } void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) override { td::actor::send_closure(id_, &FullNodeImpl::send_ihr_message, dst, std::move(data)); @@ -443,20 +558,21 @@ void FullNodeImpl::start_up() { td::actor::ActorId id_; }; - auto P = td::PromiseCreator::lambda([](td::Unit R) {}); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::install_callback, - std::make_unique(actor_id(this)), std::move(P)); + std::make_unique(actor_id(this)), std::move(started_promise_)); } FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId dht, - td::actor::ActorId overlays, + td::Ref opts, td::actor::ActorId keyring, + td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId dht, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, std::string db_root) + td::actor::ActorId client, std::string db_root, + td::Promise started_promise) : local_id_(local_id) , adnl_id_(adnl_id) , zero_state_file_hash_(zero_state_file_hash) + , opts_(opts) , keyring_(keyring) , adnl_(adnl) , rldp_(rldp) @@ -464,20 +580,23 @@ FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id , overlays_(overlays) , validator_manager_(validator_manager) , client_(client) - , db_root_(db_root) { - add_shard(ShardIdFull{masterchainId}, true); + , db_root_(db_root) + , started_promise_(std::move(started_promise)) { + add_shard_actor(ShardIdFull{masterchainId}, true); } td::actor::ActorOwn FullNode::create(ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, - FileHash zero_state_file_hash, + FileHash zero_state_file_hash, td::Ref opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId dht, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, std::string db_root) { - return td::actor::create_actor("fullnode", local_id, adnl_id, zero_state_file_hash, keyring, adnl, rldp, - dht, overlays, validator_manager, client, db_root); + td::actor::ActorId client, std::string db_root, + td::Promise started_promise) { + return td::actor::create_actor("fullnode", local_id, adnl_id, zero_state_file_hash, opts, keyring, adnl, + rldp, dht, overlays, validator_manager, client, db_root, + std::move(started_promise)); } } // namespace fullnode diff --git a/validator/full-node.h b/validator/full-node.h index cdf39d6fa..7342a8229 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -73,13 +73,14 @@ class FullNode : public td::actor::Actor { } static td::actor::ActorOwn create(ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, - FileHash zero_state_file_hash, + FileHash zero_state_file_hash, td::Ref opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId dht, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, std::string db_root); + td::actor::ActorId client, std::string db_root, + td::Promise started_promise); }; } // namespace fullnode diff --git a/validator/full-node.hpp b/validator/full-node.hpp index f3ce9d050..47cca0f57 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -52,8 +52,7 @@ class FullNodeImpl : public FullNode { void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; - void add_shard(ShardIdFull shard, bool subscribe = false); - void del_shard(ShardIdFull shard); + void update_shard_configuration(td::Ref state); void sync_completed(); @@ -82,21 +81,31 @@ class FullNodeImpl : public FullNode { void start_up() override; FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId dht, - td::actor::ActorId overlays, + td::Ref opts, td::actor::ActorId keyring, + td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId dht, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, std::string db_root); + td::actor::ActorId client, std::string db_root, + td::Promise started_promise); private: + void add_shard_actor(ShardIdFull shard, bool active); + PublicKeyHash local_id_; adnl::AdnlNodeIdShort adnl_id_; FileHash zero_state_file_hash_; + td::Ref opts_; td::actor::ActorId get_shard(AccountIdPrefixFull dst); - td::actor::ActorId get_shard(ShardIdFull dst); + td::actor::ActorId get_shard(ShardIdFull shard, bool exact = false); - std::map> shards_; + struct ShardInfo { + bool exists = false; + td::actor::ActorOwn actor; + bool active = false; + td::Timestamp delete_at = td::Timestamp::never(); + }; + std::map shards_; td::actor::ActorId keyring_; td::actor::ActorId adnl_; @@ -112,6 +121,8 @@ class FullNodeImpl : public FullNode { std::vector all_validators_; std::set local_keys_; + + td::Promise started_promise_; }; } // namespace fullnode diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index 74adcd696..6491a7116 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -41,7 +41,7 @@ using namespace std::literals::string_literals; AcceptBlockQuery::AcceptBlockQuery(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, bool send_broadcast, + td::Ref approve_signatures, bool send_broadcast, bool apply, td::actor::ActorId manager, td::Promise promise) : id_(id) , data_(std::move(data)) @@ -52,6 +52,7 @@ AcceptBlockQuery::AcceptBlockQuery(BlockIdExt id, td::Ref data, std:: , is_fake_(false) , is_fork_(false) , send_broadcast_(send_broadcast) + , apply_(apply) , manager_(manager) , promise_(std::move(promise)) { state_keep_old_hash_.clear(); @@ -92,28 +93,6 @@ AcceptBlockQuery::AcceptBlockQuery(ForceFork ffork, BlockIdExt id, td::Ref data, std::vector prev, - td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, - td::actor::ActorId manager, td::Promise promise) - : id_(id) - , data_(std::move(data)) - , prev_(std::move(prev)) - , validator_set_(std::move(validator_set)) - , signatures_(std::move(signatures)) - , approve_signatures_(std::move(approve_signatures)) - , is_fake_(false) - , is_fork_(false) - , send_broadcast_(true) - , broadcast_only_(false) - , manager_(manager) - , promise_(std::move(promise)) { - state_keep_old_hash_.clear(); - state_old_hash_.clear(); - state_hash_.clear(); - CHECK(prev_.size() > 0); -} - bool AcceptBlockQuery::precheck_header() { VLOG(VALIDATOR_DEBUG) << "precheck_header()"; // 0. sanity check @@ -357,7 +336,9 @@ bool AcceptBlockQuery::check_send_error(td::actor::ActorId Sel } void AcceptBlockQuery::finish_query() { - ValidatorInvariants::check_post_accept(handle_); + if (apply_) { + ValidatorInvariants::check_post_accept(handle_); + } if (is_masterchain()) { CHECK(handle_->inited_proof()); } else { @@ -406,15 +387,6 @@ void AcceptBlockQuery::start_up() { return; } - if (broadcast_only_) { - if (!create_new_proof()) { - fatal_error("cannot generate proof for block "s + id_.to_str()); - return; - } - applied(); - return; - } - td::actor::send_closure( manager_, &ValidatorManager::get_block_handle, id_, true, [SelfId = actor_id(this)](td::Result R) { check_send_error(SelfId, R) || @@ -479,6 +451,10 @@ void AcceptBlockQuery::written_block_signatures() { void AcceptBlockQuery::written_block_info() { VLOG(VALIDATOR_DEBUG) << "written block info"; if (data_.not_null()) { + if (!apply_) { + written_state({}); + return; + } auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { check_send_error(SelfId, R) || td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::got_prev_state, R.move_as_ok()); @@ -555,7 +531,7 @@ void AcceptBlockQuery::written_state(td::Ref upd_state) { return; } - if (state_keep_old_hash_ != state_old_hash_) { + if (apply_ && state_keep_old_hash_ != state_old_hash_) { fatal_error(PSTRING() << "invalid previous state hash in newly-created proof: expected " << state_->root_hash().to_hex() << ", found in update " << state_old_hash_.to_hex()); return; diff --git a/validator/impl/accept-block.hpp b/validator/impl/accept-block.hpp index b45095b4f..1a1f2a543 100644 --- a/validator/impl/accept-block.hpp +++ b/validator/impl/accept-block.hpp @@ -48,20 +48,15 @@ class AcceptBlockQuery : public td::actor::Actor { public: struct IsFake {}; struct ForceFork {}; - struct BroadcastOnly{}; AcceptBlockQuery(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, bool send_broadcast, + td::Ref approve_signatures, bool send_broadcast, bool apply, td::actor::ActorId manager, td::Promise promise); AcceptBlockQuery(IsFake fake, BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::actor::ActorId manager, td::Promise promise); AcceptBlockQuery(ForceFork ffork, BlockIdExt id, td::Ref data, td::actor::ActorId manager, td::Promise promise); - AcceptBlockQuery(BroadcastOnly, BlockIdExt id, td::Ref data, std::vector prev, - td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, td::actor::ActorId manager, - td::Promise promise); private: static constexpr td::uint32 priority() { @@ -103,7 +98,7 @@ class AcceptBlockQuery : public td::actor::Actor { bool is_fake_; bool is_fork_; bool send_broadcast_; - bool broadcast_only_{false}; + bool apply_ = true; bool ancestors_split_{false}, is_key_block_{false}; td::Timestamp timeout_ = td::Timestamp::in(600.0); td::actor::ActorId manager_; diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index e86b9b90d..47979af8b 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -127,10 +127,10 @@ td::Result> create_ihr_message(td::BufferSlice data) { void run_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, bool send_broadcast, + td::Ref approve_signatures, bool send_broadcast, bool apply, td::actor::ActorId manager, td::Promise promise) { td::actor::create_actor("accept", id, std::move(data), prev, std::move(validator_set), - std::move(signatures), std::move(approve_signatures), send_broadcast, + std::move(signatures), std::move(approve_signatures), send_broadcast, apply, manager, std::move(promise)) .release(); } @@ -151,17 +151,6 @@ void run_hardfork_accept_block_query(BlockIdExt id, td::Ref data, .release(); } -void run_broadcast_only_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, - td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, - td::actor::ActorId manager, - td::Promise promise) { - td::actor::create_actor("broadcastaccept", AcceptBlockQuery::BroadcastOnly(), id, std::move(data), - prev, std::move(validator_set), std::move(signatures), - std::move(approve_signatures), manager, std::move(promise)) - .release(); -} - void run_apply_block_query(BlockIdExt id, td::Ref block, BlockIdExt masterchain_block_id, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise) { diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index c72ec307c..209f871f2 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -64,10 +64,6 @@ class ValidatorManager : public ValidatorManagerInterface { std::function write_data, td::Promise promise) = 0; virtual void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) = 0; - virtual void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) = 0; - virtual void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) = 0; virtual void set_block_data(BlockHandle handle, td::Ref data, td::Promise promise) = 0; virtual void wait_block_data(BlockHandle handle, td::uint32 priority, td::Timestamp, @@ -135,7 +131,7 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_shard_client_state(bool from_db, td::Promise promise) = 0; - virtual void subscribe_to_shard(ShardIdFull shard) = 0; + virtual void update_shard_configuration(td::Ref state) = 0; virtual void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) = 0; virtual void get_async_serializer_state(td::Promise promise) = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 91235ebf6..501ee078c 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -253,7 +253,7 @@ class ValidatorManagerImpl : public ValidatorManager { void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; - void subscribe_to_shard(ShardIdFull shard) override { + void update_shard_configuration(td::Ref state) override { } void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) override { diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 80f6df93c..677827714 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -321,7 +321,8 @@ class ValidatorManagerImpl : public ValidatorManager { void get_shard_client_state(bool from_db, td::Promise promise) override { UNREACHABLE(); } - void subscribe_to_shard(ShardIdFull shard) override { + void update_shard_configuration(td::Ref state) override { + UNREACHABLE(); } void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) override { diff --git a/validator/manager.cpp b/validator/manager.cpp index 9af007726..7c5e7d3f1 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -377,7 +377,7 @@ void ValidatorManagerImpl::new_external_message(td::BufferSlice data) { } auto R = create_ext_message(std::move(data)); if (R.is_error()) { - VLOG(VALIDATOR_NOTICE) << "dropping bad ihr message: " << R.move_as_error(); + VLOG(VALIDATOR_NOTICE) << "dropping bad external message: " << R.move_as_error(); return; } add_external_message(R.move_as_ok()); @@ -453,9 +453,9 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refblock_id().shard_full(), desc->catchain_seqno()}] = desc; VLOG(VALIDATOR_DEBUG) << "new shard block descr for " << desc->block_id(); - if (last_masterchain_block_handle_ && last_masterchain_seqno_ > 0 && - desc->generated_at() < last_masterchain_block_handle_->unix_time() + 60) { - delay_action( + if (opts_->need_monitor(desc->block_id().shard_full()) && last_masterchain_block_handle_ && + last_masterchain_seqno_ > 0 && desc->generated_at() < last_masterchain_block_handle_->unix_time() + 60) { + delay_action( [SelfId = actor_id(this), desc]() { auto P = td::PromiseCreator::lambda([](td::Result> R) { if (R.is_error()) { @@ -2367,8 +2367,8 @@ void ValidatorManagerImpl::get_shard_client_state(bool from_db, td::Promisesubscribe_to_shard(shard); +void ValidatorManagerImpl::update_shard_configuration(td::Ref state) { + callback_->update_shard_configuration(state); } void ValidatorManagerImpl::update_async_serializer_state(AsyncSerializerState state, td::Promise promise) { @@ -2397,6 +2397,7 @@ void ValidatorManagerImpl::get_archive_slice(td::uint64 archive_id, td::uint64 o } bool ValidatorManagerImpl::is_validator() { + // TODO: change is_vaidator to other condition in some cases return true; // temp_keys_.size() > 0 || permanent_keys_.size() > 0; } @@ -2700,6 +2701,9 @@ void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh it = collator_nodes_.emplace(id, std::move(actor)).first; } td::actor::send_closure(it->second, &CollatorNode::add_shard, shard); + if (shard.is_masterchain()) { + collating_masterchain_ = true; + } } td::actor::ActorOwn ValidatorManagerFactory::create( diff --git a/validator/manager.hpp b/validator/manager.hpp index 94657d49d..1105b2b62 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -447,7 +447,7 @@ class ValidatorManagerImpl : public ValidatorManager { void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; - void subscribe_to_shard(ShardIdFull shard) override; + void update_shard_configuration(td::Ref state) override; void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) override; void get_async_serializer_state(td::Promise promise) override; @@ -611,6 +611,7 @@ class ValidatorManagerImpl : public ValidatorManager { void cleanup_old_pending_candidates(BlockId block_id, td::Timestamp now); std::map> collator_nodes_; + bool collating_masterchain_ = false; }; } // namespace validator diff --git a/validator/net/download-block-new.cpp b/validator/net/download-block-new.cpp index ef5ed7e54..1063d8a35 100644 --- a/validator/net/download-block-new.cpp +++ b/validator/net/download-block-new.cpp @@ -201,10 +201,10 @@ void DownloadBlockNew::got_node_to_download(adnl::AdnlNodeIdShort node) { } if (client_.empty()) { td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, download_from_, local_id_, overlay_id_, - "get_proof", std::move(P), td::Timestamp::in(3.0), std::move(q), + "get_block_full", std::move(P), td::Timestamp::in(3.0), std::move(q), FullNode::max_proof_size() + FullNode::max_block_size() + 128, rldp_); } else { - td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "get_prepare", + td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "get_block_full", create_serialize_tl_object_suffix(std::move(q)), td::Timestamp::in(1.0), std::move(P)); } diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index af44f16e8..50c6075d8 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -247,39 +247,7 @@ void ShardClient::get_processed_masterchain_block_id(td::Promise pro } void ShardClient::build_shard_overlays() { - auto v = masterchain_state_->get_shards(); - std::set workchains; - - for (auto &x : v) { - auto shard = x->shard(); - workchains.insert(shard.workchain); - if (opts_->need_monitor(shard)) { - auto d = masterchain_state_->soft_min_split_depth(shard.workchain); - auto l = shard_prefix_length(shard.shard); - if (l > d) { - shard = shard_prefix(shard, d); - } - - if (created_overlays_.count(shard) == 0) { - created_overlays_.insert(shard); - td::actor::send_closure(manager_, &ValidatorManager::subscribe_to_shard, shard); - } - } - } - - for (const auto &wpair : masterchain_state_->get_workchain_list()) { - ton::WorkchainId wc = wpair.first; - const block::WorkchainInfo *winfo = wpair.second.get(); - if (workchains.count(wc) == 0 && winfo->active && winfo->enabled_since <= masterchain_state_->get_unix_time()) { - auto shard = ShardIdFull(wc); - if (opts_->need_monitor(shard) && created_overlays_.count(shard) == 0) { - td::actor::send_closure(manager_, &ValidatorManager::subscribe_to_shard, shard); - BlockIdExt block_id(shard.workchain, shard.shard, 0, winfo->zerostate_root_hash, winfo->zerostate_file_hash); - td::actor::send_closure_later(manager_, &ValidatorManager::wait_block_state_short, block_id, 0, - td::Timestamp::in(5.0), [](td::Result>) {}); - } - } - } + td::actor::send_closure(manager_, &ValidatorManager::update_shard_configuration, masterchain_state_); } void ShardClient::force_update_shard_client(BlockHandle handle, td::Promise promise) { diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index bb493f912..8eae61332 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -105,8 +105,7 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat void ValidatorGroup::update_approve_cache(td::uint32 round_id, CacheKey key, UnixTime value) { if (approved_candidates_cache_round_ != round_id) { - approved_candidates_cache_round_ = round_id; - approved_candidates_cache_.clear(); + return; } approved_candidates_cache_[key] = value; } @@ -157,15 +156,9 @@ void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref } }); - if (shard_.is_masterchain() || lite_mode_) { - run_accept_block_query(block_id, std::move(block), std::move(prev), validator_set_, std::move(sig_set), - std::move(approve_sig_set), send_broadcast, manager_, std::move(P)); - } else if (send_broadcast) { - run_broadcast_only_accept_block_query(block_id, std::move(block), std::move(prev), validator_set_, - std::move(sig_set), std::move(approve_sig_set), manager_, std::move(P)); - } else { - promise.set_value(td::Unit()); - } + run_accept_block_query(block_id, std::move(block), std::move(prev), validator_set_, std::move(sig_set), + std::move(approve_sig_set), send_broadcast, shard_.is_masterchain() || !lite_mode_, manager_, + std::move(P)); } void ValidatorGroup::skip_round(td::uint32 round_id) { @@ -386,7 +379,7 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo td::actor::send_closure(SelfId, &ValidatorGroup::receive_collate_query_response, round_id, R.move_as_ok(), std::move(promise)); }); - LOG(INFO) << "collate query for " << shard_.to_str() << ": send query to " << collator; + LOG(INFO) << "collate query for " << create_next_block_id_simple().to_str() << ": send query to " << collator; size_t max_answer_size = config_.max_block_size + config_.max_collated_data_size + 256; td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, adnl::AdnlNodeIdShort(local_id_), collator, "collatequery", std::move(P), timeout, std::move(query), max_answer_size); @@ -412,6 +405,7 @@ void ValidatorGroup::receive_collate_query_response(td::uint32 round_id, td::Buf auto key = PublicKey{b->source_}; if (!key.is_ed25519()) { promise.set_error(td::Status::Error("collate query: block candidate source mismatch")); + return; } auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; if (e_key != Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}) { diff --git a/validator/validator.h b/validator/validator.h index 66007910b..ab013cb08 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -111,8 +111,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual ~Callback() = default; virtual void initial_read_complete(BlockHandle top_masterchain_blocks) = 0; - virtual void subscribe_to_shard(ShardIdFull shard) = 0; - //virtual void del_shard(ShardIdFull shard) = 0; + virtual void update_shard_configuration(td::Ref state) = 0; virtual void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; @@ -208,6 +207,11 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_block_by_seqno_from_db(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise promise) = 0; + virtual void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) = 0; + virtual void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) = 0; + virtual void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) = 0; virtual void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise) = 0; From 1869a2506287dbc66e04ac52ab7b620e9960371e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 29 Jul 2022 12:08:17 +0300 Subject: [PATCH 008/388] Process adnl query errors --- adnl/adnl-message.cpp | 3 +++ adnl/adnl-message.h | 21 ++++++++++++++++++++- adnl/adnl-peer.cpp | 16 ++++++++++++++++ adnl/adnl-peer.hpp | 1 + adnl/adnl-query.cpp | 6 +++++- adnl/adnl-query.h | 1 + rldp/rldp-in.hpp | 4 ++++ rldp/rldp.cpp | 22 ++++++++++++++++++++++ rldp2/rldp-in.hpp | 4 ++++ rldp2/rldp.cpp | 22 ++++++++++++++++++++++ tl/generate/scheme/ton_api.tl | 2 ++ tl/generate/scheme/ton_api.tlo | Bin 70276 -> 70468 bytes 12 files changed, 100 insertions(+), 2 deletions(-) diff --git a/adnl/adnl-message.cpp b/adnl/adnl-message.cpp index 0d7129783..1d3ba5f8a 100644 --- a/adnl/adnl-message.cpp +++ b/adnl/adnl-message.cpp @@ -46,6 +46,9 @@ AdnlMessage::AdnlMessage(tl_object_ptr message) { [&](ton_api::adnl_message_part &msg) { message_ = adnlmessage::AdnlMessagePart{msg.hash_, static_cast(msg.total_size_), static_cast(msg.offset_), std::move(msg.data_)}; + }, + [&](ton_api::adnl_message_queryError &msg) { + message_ = adnlmessage::AdnlMessageQueryError{msg.query_id_}; })); } diff --git a/adnl/adnl-message.h b/adnl/adnl-message.h index 43849e982..d557c3195 100644 --- a/adnl/adnl-message.h +++ b/adnl/adnl-message.h @@ -170,6 +170,24 @@ class AdnlMessageAnswer { td::BufferSlice data_; }; +class AdnlMessageQueryError { + public: + explicit AdnlMessageQueryError(AdnlQueryId query_id) : query_id_(query_id) { + } + const auto &query_id() const { + return query_id_; + } + td::uint32 size() const { + return 36; + } + tl_object_ptr tl() const { + return create_tl_object(query_id_); + } + + private: + AdnlQueryId query_id_; +}; + class AdnlMessagePart { public: AdnlMessagePart(td::Bits256 hash, td::uint32 total_size, td::uint32 offset, td::BufferSlice data) @@ -220,7 +238,8 @@ class AdnlMessage { private: td::Variant + adnlmessage::AdnlMessageQuery, adnlmessage::AdnlMessageAnswer, adnlmessage::AdnlMessagePart, + adnlmessage::AdnlMessageQueryError> message_{Empty{}}; public: diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp index 35ba2a115..1d73b8f6c 100644 --- a/adnl/adnl-peer.cpp +++ b/adnl/adnl-peer.cpp @@ -524,10 +524,14 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageQuery &mess flags = static_cast(0)](td::Result R) { if (R.is_error()) { LOG(WARNING) << "failed to answer query: " << R.move_as_error(); + td::actor::send_closure(SelfId, &AdnlPeerPairImpl::send_message, + OutboundAdnlMessage{adnlmessage::AdnlMessageQueryError{query_id}, flags}); } else { auto data = R.move_as_ok(); if (data.size() > Adnl::huge_packet_max_size()) { LOG(WARNING) << "dropping too big answer query: size=" << data.size(); + td::actor::send_closure(SelfId, &AdnlPeerPairImpl::send_message, + OutboundAdnlMessage{adnlmessage::AdnlMessageQueryError{query_id}, flags}); } else { td::actor::send_closure(SelfId, &AdnlPeerPairImpl::send_message, OutboundAdnlMessage{adnlmessage::AdnlMessageAnswer{query_id, std::move(data)}, flags}); @@ -609,6 +613,18 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessagePart &messa } } +void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageQueryError &message) { + auto Q = out_queries_.find(message.query_id()); + + if (Q == out_queries_.end()) { + VLOG(ADNL_NOTICE) << this << ": dropping IN query error: unknown query id " << message.query_id(); + return; + } + + td::actor::send_closure_later(Q->second, &AdnlQuery::reject_query); + out_queries_.erase(Q); +} + void AdnlPeerPairImpl::delete_query(AdnlQueryId id) { auto Q = out_queries_.find(id); diff --git a/adnl/adnl-peer.hpp b/adnl/adnl-peer.hpp index 410c0f75f..ec7166ed6 100644 --- a/adnl/adnl-peer.hpp +++ b/adnl/adnl-peer.hpp @@ -104,6 +104,7 @@ class AdnlPeerPairImpl : public AdnlPeerPair { void process_message(const adnlmessage::AdnlMessageQuery &message); void process_message(const adnlmessage::AdnlMessageAnswer &message); void process_message(const adnlmessage::AdnlMessagePart &message); + void process_message(const adnlmessage::AdnlMessageQueryError &message); void process_message(const AdnlMessage::Empty &message) { UNREACHABLE(); } diff --git a/adnl/adnl-query.cpp b/adnl/adnl-query.cpp index 5bc767d2b..40e9be803 100644 --- a/adnl/adnl-query.cpp +++ b/adnl/adnl-query.cpp @@ -25,13 +25,17 @@ namespace ton { namespace adnl { void AdnlQuery::alarm() { - promise_.set_error(td::Status::Error(ErrorCode::timeout, "adnl query timeout")); + promise_.set_error(td::Status::Error(ErrorCode::timeout, PSTRING() << "timeout for adnl query " << name_)); stop(); } void AdnlQuery::result(td::BufferSlice data) { promise_.set_value(std::move(data)); stop(); } +void AdnlQuery::reject_query() { + promise_.set_error(td::Status::Error(ErrorCode::timeout, PSTRING() << "rejected adnl query " << name_)); + stop(); +} AdnlQueryId AdnlQuery::random_query_id() { AdnlQueryId q_id; diff --git a/adnl/adnl-query.h b/adnl/adnl-query.h index 6e24a49fe..2a78615a6 100644 --- a/adnl/adnl-query.h +++ b/adnl/adnl-query.h @@ -48,6 +48,7 @@ class AdnlQuery : public td::actor::Actor { } void alarm() override; void result(td::BufferSlice data); + void reject_query(); void start_up() override { alarm_timestamp() = timeout_; } diff --git a/rldp/rldp-in.hpp b/rldp/rldp-in.hpp index b49819993..8850ccf9d 100644 --- a/rldp/rldp-in.hpp +++ b/rldp/rldp-in.hpp @@ -78,6 +78,8 @@ class RldpIn : public RldpImpl { td::uint64 max_answer_size) override; void answer_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, adnl::AdnlQueryId query_id, TransferId transfer_id, td::BufferSlice data); + void reject_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, + adnl::AdnlQueryId query_id, TransferId transfer_id); void alarm_query(adnl::AdnlQueryId query_id, TransferId transfer_id); @@ -93,6 +95,8 @@ class RldpIn : public RldpImpl { ton_api::rldp_query &message); void process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, ton_api::rldp_answer &message); + void process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, + ton_api::rldp_queryError &message); void receive_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, td::BufferSlice data); diff --git a/rldp/rldp.cpp b/rldp/rldp.cpp index 9b38dcb8c..a1d459260 100644 --- a/rldp/rldp.cpp +++ b/rldp/rldp.cpp @@ -87,6 +87,13 @@ void RldpIn::answer_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, transfer(src, dst, timeout, std::move(B), transfer_id); } +void RldpIn::reject_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, + adnl::AdnlQueryId query_id, TransferId transfer_id) { + auto B = serialize_tl_object(create_tl_object(query_id), true); + + transfer(src, dst, timeout, std::move(B), transfer_id); +} + void RldpIn::alarm_query(adnl::AdnlQueryId query_id, TransferId transfer_id) { queries_.erase(query_id); max_size_.erase(transfer_id); @@ -199,12 +206,16 @@ void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort auto data = R.move_as_ok(); if (data.size() > max_answer_size) { VLOG(RLDP_NOTICE) << "rldp query failed: answer too big"; + td::actor::send_closure(SelfId, &RldpIn::reject_query, local_id, source, timeout, query_id, + transfer_id ^ TransferId::ones()); } else { td::actor::send_closure(SelfId, &RldpIn::answer_query, local_id, source, timeout, query_id, transfer_id ^ TransferId::ones(), std::move(data)); } } else { VLOG(RLDP_NOTICE) << "rldp query failed: " << R.move_as_error(); + td::actor::send_closure(SelfId, &RldpIn::reject_query, local_id, source, timeout, query_id, + transfer_id ^ TransferId::ones()); } }); VLOG(RLDP_DEBUG) << "delivering rldp query"; @@ -223,6 +234,17 @@ void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort } } +void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, + ton_api::rldp_queryError &message) { + auto it = queries_.find(message.query_id_); + if (it != queries_.end()) { + td::actor::send_closure(it->second, &adnl::AdnlQuery::reject_query); + queries_.erase(it); + } else { + VLOG(RLDP_INFO) << "received reject to unknown query " << message.query_id_; + } +} + void RldpIn::transfer_completed(TransferId transfer_id) { senders_.erase(transfer_id); VLOG(RLDP_DEBUG) << "rldp: completed transfer " << transfer_id << "; " << senders_.size() << " out transfer pending "; diff --git a/rldp2/rldp-in.hpp b/rldp2/rldp-in.hpp index c2e46d2ab..a815004c9 100644 --- a/rldp2/rldp-in.hpp +++ b/rldp2/rldp-in.hpp @@ -76,6 +76,8 @@ class RldpIn : public RldpImpl { td::uint64 max_answer_size) override; void answer_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, adnl::AdnlQueryId query_id, TransferId transfer_id, td::BufferSlice data); + void reject_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, + adnl::AdnlQueryId query_id, TransferId transfer_id); void receive_message_part(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, td::BufferSlice data); @@ -85,6 +87,8 @@ class RldpIn : public RldpImpl { ton_api::rldp_query &message); void process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, ton_api::rldp_answer &message); + void process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, + ton_api::rldp_queryError &message); void receive_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, td::Result data); diff --git a/rldp2/rldp.cpp b/rldp2/rldp.cpp index 765e38a5b..34abd78e0 100644 --- a/rldp2/rldp.cpp +++ b/rldp2/rldp.cpp @@ -118,6 +118,13 @@ void RldpIn::answer_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, send_closure(create_connection(src, dst), &RldpConnectionActor::send, transfer_id, std::move(B), timeout); } +void RldpIn::reject_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, + adnl::AdnlQueryId query_id, TransferId transfer_id) { + auto B = serialize_tl_object(create_tl_object(query_id), true); + + send_closure(create_connection(src, dst), &RldpConnectionActor::send, transfer_id, std::move(B), timeout); +} + void RldpIn::receive_message_part(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, td::BufferSlice data) { send_closure(create_connection(local_id, source), &RldpConnectionActor::receive_raw, std::move(data)); } @@ -174,12 +181,16 @@ void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort auto data = R.move_as_ok(); if (data.size() > max_answer_size) { VLOG(RLDP_NOTICE) << "rldp query failed: answer too big"; + td::actor::send_closure(SelfId, &RldpIn::reject_query, local_id, source, timeout, query_id, + transfer_id ^ TransferId::ones()); } else { td::actor::send_closure(SelfId, &RldpIn::answer_query, local_id, source, timeout, query_id, transfer_id ^ TransferId::ones(), std::move(data)); } } else { VLOG(RLDP_NOTICE) << "rldp query failed: " << R.move_as_error(); + td::actor::send_closure(SelfId, &RldpIn::reject_query, local_id, source, timeout, query_id, + transfer_id ^ TransferId::ones()); } }); VLOG(RLDP_DEBUG) << "delivering rldp query"; @@ -198,6 +209,17 @@ void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort } } +void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, + ton_api::rldp_queryError &message) { + auto it = queries_.find(transfer_id); + if (it != queries_.end()) { + it->second.set_error(td::Status::Error("rejected")); + queries_.erase(it); + } else { + VLOG(RLDP_INFO) << "received reject to unknown query " << message.query_id_; + } +} + void RldpIn::on_sent(TransferId transfer_id, td::Result state) { //TODO: completed transfer } diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 3a81ab585..9f1072ce7 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -136,6 +136,7 @@ adnl.message.reinit date:int = adnl.Message; adnl.message.query query_id:int256 query:bytes = adnl.Message; adnl.message.answer query_id:int256 answer:bytes = adnl.Message; +adnl.message.queryError query_id:int256 = adnl.Message; adnl.message.part hash:int256 total_size:int offset:int data:bytes = adnl.Message; @@ -161,6 +162,7 @@ rldp.complete transfer_id:int256 part:int = rldp.MessagePart; rldp.message id:int256 data:bytes = rldp.Message; rldp.query query_id:int256 max_answer_size:long timeout:int data:bytes = rldp.Message; rldp.answer query_id:int256 data:bytes = rldp.Message; +rldp.queryError query_id:int256 = rldp.Message; ---functions--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 479455feb679ece457175e95b6b44e8c7807cfa9..95a2144a69edb475430ee6965b8a84f5b497eddb 100644 GIT binary patch delta 481 zcmZo!%5r2G%LWx@(Y1dr9Og+($;;96O)V}?OiyKCVBnnSD84y>`HiIL`#%{~JViMv z1xT_in*(K&IT_tIZ&foFWdZ4*{K47>!f>$BfiMbe3LuOpHYyOtgM9{2HMS}eAYC_) z3HbmmV<>WF)8VjfI|w$Z##V(BCdoKi&_)}g=Y}oBlu33W5P=9gsId)pA*LV=Ad^7g zsvK90>V`ZOaj>6qA%4;;EKMz{bS)~%FG3jpbCvoeGY+WvP~#@qPXP;To?(B05h4}i zXafgLTmjo~`grj~`($*fStCpkQR24D}31U^CB}iZlSc*U)tU delta 322 zcmX@IjHP8M%LWx@(GH(|&v_D4@^bWiQ;Ule(^DB37&s<6if;~Jej_Pba%Wx$Pf<=v z0g^29=0MqGPDabxnT=2Ws+S8L?FTrYHR~c#Ks<>lVx)5 za`nj*>@&cMH~+CezzETHz|jWG*yQvig9YTd$qx0Jlh>|QhHxxa%Rm?rtLH!%KcFI$ S> Date: Mon, 1 Aug 2022 17:48:22 +0300 Subject: [PATCH 009/388] Get neighbors' msg queues from other nodes --- tl/generate/scheme/ton_api.tl | 4 + tl/generate/scheme/ton_api.tlo | Bin 70468 -> 70924 bytes validator/CMakeLists.txt | 1 + validator/full-node-shard.cpp | 53 +++++ validator/full-node-shard.h | 2 + validator/full-node-shard.hpp | 4 + validator/full-node.cpp | 17 ++ validator/full-node.hpp | 2 + validator/impl/CMakeLists.txt | 5 +- validator/impl/collator-impl.h | 5 +- validator/impl/collator.cpp | 70 +++--- validator/impl/out-msg-queue-proof.cpp | 245 +++++++++++++++++++++ validator/impl/out-msg-queue-proof.hpp | 109 +++++++++ validator/impl/proof.cpp | 35 +++ validator/interfaces/out-msg-queue-proof.h | 42 ++++ validator/interfaces/proof.h | 3 + validator/interfaces/validator-manager.h | 3 + validator/manager-disk.hpp | 8 + validator/manager-hardfork.hpp | 8 + validator/manager.cpp | 94 ++++++-- validator/manager.hpp | 7 + validator/validator.h | 7 + 22 files changed, 657 insertions(+), 67 deletions(-) create mode 100644 validator/impl/out-msg-queue-proof.cpp create mode 100644 validator/impl/out-msg-queue-proof.hpp create mode 100644 validator/interfaces/out-msg-queue-proof.h diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 9f1072ce7..f3ac4ee91 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -410,6 +410,9 @@ tonNode.success = tonNode.Success; tonNode.archiveNotFound = tonNode.ArchiveInfo; tonNode.archiveInfo id:long = tonNode.ArchiveInfo; +tonNode.outMsgQueueProof queue_proof:bytes block_state_proof:bytes = tonNode.OutMsgQueueProof; +tonNode.outMsgQueueProofEmpty = tonNode.OutMsgQueueProof; + ---functions--- tonNode.getNextBlockDescription prev_block:tonNode.blockIdExt = tonNode.BlockDescription; @@ -441,6 +444,7 @@ tonNode.downloadBlockProofLinks blocks:(vector tonNode.blockIdExt) = tonNode.Dat tonNode.downloadKeyBlockProofLinks blocks:(vector tonNode.blockIdExt) = tonNode.DataList; tonNode.getArchiveInfo masterchain_seqno:int = tonNode.ArchiveInfo; tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data; +tonNode.getOutMsgQueueProof block_id:tonNode.blockIdExt dst_workchain:int dst_shard:long = tonNode.OutMsgQueueProof; tonNode.getCapabilities = tonNode.Capabilities; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 95a2144a69edb475430ee6965b8a84f5b497eddb..2960fa72c13d97c0b874d959604f413a38afe08c 100644 GIT binary patch delta 326 zcmX@IjHPE43-6=Z`c@23aBd^-X+_y2t+*)?CHZ-N`6;P-{-q_p#p!{isimm_Mfv$@ zlO2VmH{Vda!@}sfd8=CUESmbFCBiC@ zz9MHf9S+;JgCKJ@>l{pwXH41L`@mmElZAIxjx@}m^wbh`55nBbJo#dx`g8|AMpl-T a;*$8u509uo7>wD promise) { + BlockIdExt block_id = create_block_id(query.block_id_); + ShardIdFull dst_shard(query.dst_workchain_, query.dst_shard_); + if (!block_id.is_valid_ext()) { + promise.set_error(td::Status::Error("invalid block_id")); + return; + } + if (!dst_shard.is_valid_ext()) { + promise.set_error(td::Status::Error("invalid shard")); + return; + } + auto P = td::PromiseCreator::lambda( + [promise = std::move(promise)](td::Result> R) mutable { + if (R.is_error()) { + promise.set_result(create_serialize_tl_object()); + } else { + promise.set_result(serialize_tl_object(R.move_as_ok(), true)); + } + }); + td::actor::create_actor("buildqueueproof", block_id, dst_shard, validator_manager_, + std::move(P)) + .release(); +} + void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise) { if (!active_) { @@ -791,6 +818,32 @@ void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, std::stri .release(); } +void FullNodeShardImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, + td::Promise> promise) { + // TODO: maybe more complex download (like other requests here) + // TODO: estimate max size + auto &b = choose_neighbour(); + auto P = td::PromiseCreator::lambda( + [=, promise = create_neighbour_promise(b, std::move(promise))](td::Result R) mutable { + if (R.is_error()) { + promise.set_result(R.move_as_error()); + return; + } + TRY_RESULT_PROMISE(promise, f, fetch_tl_object(R.move_as_ok(), true)); + ton_api::downcast_call(*f, td::overloaded( + [&](ton_api::tonNode_outMsgQueueProofEmpty &x) { + promise.set_error(td::Status::Error("node doesn't have this block")); + }, + [&](ton_api::tonNode_outMsgQueueProof &x) { + promise.set_result(OutMsgQueueProof::fetch(block_id, dst_shard, x)); + })); + }); + td::BufferSlice query = create_serialize_tl_object( + create_tl_block_id(block_id), dst_shard.workchain, dst_shard.shard); + td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, b.adnl_id, adnl_id_, overlay_id_, + "get_msg_queue", std::move(P), timeout, std::move(query), 1 << 20, rldp_); +} + void FullNodeShardImpl::set_handle(BlockHandle handle, td::Promise promise) { CHECK(!handle_); handle_ = std::move(handle); diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 2b2098a69..7da46af18 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -62,6 +62,8 @@ class FullNodeShard : public td::actor::Actor { td::Promise> promise) = 0; virtual void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) = 0; + virtual void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, + td::Promise> promise) = 0; virtual void set_handle(BlockHandle handle, td::Promise promise) = 0; diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 9a022cbf0..ead8cafd5 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -133,6 +133,8 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getOutMsgQueueProof &query, + td::Promise promise); // void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareNextKeyBlockProof &query, // td::Promise promise); void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise); @@ -164,6 +166,8 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise> promise) override; void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) override; + void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, + td::Promise> promise) override; void set_handle(BlockHandle handle, td::Promise promise) override; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 9c26cf3c4..9194b9a4c 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -356,6 +356,18 @@ void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, std::string tm std::move(promise)); } +void FullNodeImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, + td::Promise> promise) { + auto shard = get_shard(block_id.shard_full()); + if (shard.empty()) { + VLOG(FULL_NODE_WARNING) << "dropping download msg queue query to unknown shard"; + promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); + return; + } + td::actor::send_closure(shard, &FullNodeShard::download_out_msg_queue_proof, block_id, dst_shard, timeout, + std::move(promise)); +} + td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard, bool exact) { if (!exact) { ShardIdFull s = shard; @@ -546,6 +558,11 @@ void FullNodeImpl::start_up() { td::actor::send_closure(id_, &FullNodeImpl::download_archive, masterchain_seqno, std::move(tmp_dir), timeout, std::move(promise)); } + void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, + td::Promise> promise) override { + td::actor::send_closure(id_, &FullNodeImpl::download_out_msg_queue_proof, block_id, dst_shard, timeout, + std::move(promise)); + } void new_key_block(BlockHandle handle) override { td::actor::send_closure(id_, &FullNodeImpl::new_key_block, std::move(handle)); diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 47cca0f57..49dad3ba3 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -73,6 +73,8 @@ class FullNodeImpl : public FullNode { void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise); void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise); + void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, + td::Promise> promise); void got_key_block_proof(td::Ref proof); void got_zero_block_state(td::Ref state); diff --git a/validator/impl/CMakeLists.txt b/validator/impl/CMakeLists.txt index 224599f86..4e3fd04f1 100644 --- a/validator/impl/CMakeLists.txt +++ b/validator/impl/CMakeLists.txt @@ -15,6 +15,7 @@ set(TON_VALIDATOR_SOURCE ihr-message.cpp liteserver.cpp message-queue.cpp + out-msg-queue-proof.cpp proof.cpp shard.cpp signature-set.cpp @@ -32,13 +33,13 @@ set(TON_VALIDATOR_SOURCE ihr-message.hpp liteserver.hpp message-queue.hpp + out-msg-queue-proof.hpp proof.hpp shard.hpp signature-set.hpp top-shard-descr.hpp validate-query.hpp - validator-set.hpp -) + validator-set.hpp) add_library(ton_validator STATIC ${TON_VALIDATOR_SOURCE}) diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 8a17e9531..c19199c82 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -205,7 +205,7 @@ class Collator final : public td::actor::Actor { std::vector bad_ext_msgs_, delay_ext_msgs_; Ref shard_account_blocks_; // ShardAccountBlocks - std::map> blocks_with_state_proofs_; + std::map> block_state_proofs_; std::vector neighbor_proof_builders_; std::vector> collated_roots_; @@ -238,8 +238,7 @@ class Collator final : public td::actor::Actor { void after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res); bool fix_one_processed_upto(block::MsgProcessedUpto& proc, const ton::ShardIdFull& owner); bool fix_processed_upto(block::MsgProcessedUptoCollection& upto); - void got_neighbor_block_data(td::Result> res); - void got_neighbor_block_state(int i, td::Result> res); + void got_neighbor_msg_queue(unsigned i, td::Result> R); bool adjust_shard_config(); bool store_shard_fees(ShardIdFull shard, const block::CurrencyCollection& fees, const block::CurrencyCollection& created); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 49c945961..e116a9358 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -501,7 +501,13 @@ void Collator::after_get_block_data(int idx, td::Result> res) { prev_mc_block = prev_block_data[0]; mc_block_root = prev_mc_block->root_cell(); } - blocks_with_state_proofs_[prev_block_data[idx]->root_cell()->get_hash().bits()] = prev_block_data[idx]; + Ref root = prev_block_data[idx]->root_cell(); + auto proof = create_block_state_proof(root); + if (proof.is_error()) { + fatal_error(proof.move_as_error()); + return; + } + block_state_proofs_.emplace(root->get_hash().bits(), proof.move_as_ok()); } check_pending(); } @@ -613,54 +619,39 @@ bool Collator::request_neighbor_msg_queues() { } neighbors_.emplace_back(*shard_ptr); } - int i = 0; + unsigned i = 0; neighbor_proof_builders_.resize(neighbors_.size()); for (block::McShardDescr& descr : neighbors_) { LOG(DEBUG) << "neighbor #" << i << " : " << descr.blk_.to_str(); - if (descr.blk_.seqno() != 0) { - ++pending; - send_closure_later(manager, &ValidatorManager::wait_block_data_short, descr.blk_, priority(), timeout, - [self = get_self(), i](td::Result> res) { - LOG(DEBUG) << "got answer to wait_block_data for neighbor #" << i; - send_closure_later(std::move(self), &Collator::got_neighbor_block_data, std::move(res)); - }); - } ++pending; - send_closure_later(manager, &ValidatorManager::wait_block_state_short, descr.blk_, priority(), timeout, - [self = get_self(), i](td::Result> res) { - LOG(DEBUG) << "got answer to wait_block_state for neighbor #" << i; - send_closure_later(std::move(self), &Collator::got_neighbor_block_state, i, std::move(res)); + send_closure_later(manager, &ValidatorManager::wait_out_msg_queue_proof, descr.blk_, shard_, priority(), timeout, + [self = get_self(), i](td::Result> res) { + LOG(DEBUG) << "got msg queue for neighbor #" << i; + send_closure_later(std::move(self), &Collator::got_neighbor_msg_queue, i, std::move(res)); }); ++i; } return true; } -void Collator::got_neighbor_block_data(td::Result> res) { +void Collator::got_neighbor_msg_queue(unsigned i, td::Result> R) { --pending; - if (res.is_error()) { - fatal_error(res.move_as_error()); + if (R.is_error()) { + fatal_error(R.move_as_error()); return; } - auto block_data = res.move_as_ok(); - blocks_with_state_proofs_[block_data->root_cell()->get_hash().bits()] = block_data; - check_pending(); -} - -void Collator::got_neighbor_block_state(int i, td::Result> res) { - --pending; - if (res.is_error()) { - fatal_error(res.move_as_error()); - return; + auto res = R.move_as_ok(); + BlockIdExt block_id = neighbors_.at(i).blk_; + if (res->block_state_proof_.not_null()) { + block_state_proofs_.emplace(block_id.root_hash, res->block_state_proof_); } - Ref state = res.move_as_ok(); - neighbor_proof_builders_.at(i) = vm::MerkleProofBuilder{state->root_cell()}; - auto new_state = ShardStateQ::fetch(state->get_block_id(), {}, neighbor_proof_builders_.at(i).root()); - if (new_state.is_error()) { - fatal_error(new_state.move_as_error()); + neighbor_proof_builders_.at(i) = vm::MerkleProofBuilder{res->state_root_}; + auto state = ShardStateQ::fetch(block_id, {}, neighbor_proof_builders_.at(i).root()); + if (state.is_error()) { + fatal_error(state.move_as_error()); return; } - auto outq_descr_res = new_state.move_as_ok()->message_queue(); + auto outq_descr_res = state.move_as_ok()->message_queue(); if (outq_descr_res.is_error()) { fatal_error(outq_descr_res.move_as_error()); return; @@ -3992,17 +3983,8 @@ bool Collator::create_collated_data() { collated_roots_.push_back(std::move(cell)); } // 2. Proofs for hashes of states: previous states + neighbors - for (const auto& p : blocks_with_state_proofs_) { - vm::MerkleProofBuilder mpb{p.second->root_cell()}; - block::gen::Block::Record block; - if (!tlb::unpack_cell(mpb.root(), block) || block.state_update->load_cell().is_error()) { - return fatal_error("cannot generate Merkle proof for previous block"); - } - Ref proof = mpb.extract_proof(); - if (proof.is_null()) { - return fatal_error("cannot generate Merkle proof for previous block"); - } - collated_roots_.push_back(std::move(proof)); + for (const auto& p : block_state_proofs_) { + collated_roots_.push_back(p.second); } // 3. Previous state proof (only shadchains) std::map> proofs; diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp new file mode 100644 index 000000000..0c3c993cd --- /dev/null +++ b/validator/impl/out-msg-queue-proof.cpp @@ -0,0 +1,245 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "out-msg-queue-proof.hpp" +#include "interfaces/proof.h" +#include "shard.hpp" +#include "vm/cells/MerkleProof.h" +#include "common/delay.h" +#include "interfaces/validator-manager.h" + +namespace ton { + +namespace validator { + +td::Result> OutMsgQueueProof::fetch(BlockIdExt block_id, ShardIdFull dst_shard, + const ton_api::tonNode_outMsgQueueProof &f) { + Ref block_state_proof; + td::Bits256 state_root_hash; + if (block_id.seqno() == 0) { + if (!f.block_state_proof_.empty()) { + return td::Status::Error("expected empty block state proof"); + } + state_root_hash = block_id.root_hash; + } else { + TRY_RESULT_ASSIGN(block_state_proof, vm::std_boc_deserialize(f.block_state_proof_.as_slice())); + TRY_RESULT_ASSIGN(state_root_hash, unpack_block_state_proof(block_id, block_state_proof)); + } + + TRY_RESULT(queue_proof, vm::std_boc_deserialize(f.queue_proof_.as_slice())); + auto state_root = vm::MerkleProof::virtualize(queue_proof, 1); + if (state_root.is_null()) { + return td::Status::Error("invalid queue proof"); + } + if (state_root->get_hash().as_slice() != state_root_hash.as_slice()) { + return td::Status::Error("state root hash mismatch"); + } + + // TODO: validate + return Ref(true, std::move(state_root), std::move(block_state_proof)); +} + +td::Result> OutMsgQueueProof::serialize( + BlockIdExt block_id, ShardIdFull dst_shard, Ref state_root, Ref block_root) { + vm::MerkleProofBuilder mpb{std::move(state_root)}; + TRY_RESULT(state, ShardStateQ::fetch(block_id, {}, mpb.root())); + TRY_RESULT(outq_descr, state->message_queue()); + + // TODO: add only required part of msg queue + td::HashSet visited; + std::function)> dfs = [&](Ref cell) { + if (!visited.insert(cell->get_hash()).second) { + return; + } + vm::CellSlice cs(vm::NoVm(), cell); + for (unsigned i = 0; i < cs.size_refs(); i++) { + dfs(cs.prefetch_ref(i)); + } + }; + dfs(outq_descr->root_cell()); + + TRY_RESULT(queue_proof, vm::std_boc_serialize(mpb.extract_proof())); + + td::BufferSlice block_state_proof; + if (block_id.seqno() != 0) { + TRY_RESULT(proof, create_block_state_proof(std::move(block_root))); + TRY_RESULT_ASSIGN(block_state_proof, vm::std_boc_serialize(std::move(proof), 31)); + } + + return create_tl_object(std::move(queue_proof), std::move(block_state_proof)); +} + +void WaitOutMsgQueueProof::alarm() { + abort_query(td::Status::Error(ErrorCode::timeout, "timeout")); +} + +void WaitOutMsgQueueProof::abort_query(td::Status reason) { + if (promise_) { + if (priority_ > 0 || (reason.code() != ErrorCode::timeout && reason.code() != ErrorCode::notready)) { + LOG(WARNING) << "aborting wait msg queue query for " << block_id_.to_str() << " priority=" << priority_ << ": " + << reason; + } else { + LOG(DEBUG) << "aborting wait msg queue query for " << block_id_.to_str() << " priority=" << priority_ << ": " + << reason; + } + promise_.set_error( + reason.move_as_error_prefix(PSTRING() << "failed to get msg queue for " << block_id_.to_str() << ": ")); + } + stop(); +} + +void WaitOutMsgQueueProof::finish_query(Ref result) { + promise_.set_result(std::move(result)); + stop(); +} + +void WaitOutMsgQueueProof::start_up() { + alarm_timestamp() = timeout_; + if (local_) { + run_local(); + } else { + run_net(); + } +} + +void WaitOutMsgQueueProof::run_local() { + ++pending; + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, block_id_, priority_, timeout_, + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::abort_query, + R.move_as_error_prefix("failed to get shard state")); + } else { + td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::got_state_root, + R.move_as_ok()->root_cell()); + } + }); + if (block_id_.seqno() != 0) { + ++pending; + td::actor::send_closure(manager_, &ValidatorManager::wait_block_data_short, block_id_, priority_, timeout_, + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::abort_query, + R.move_as_error_prefix("failed to get block data")); + } else { + td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::got_block_root, + R.move_as_ok()->root_cell()); + } + }); + } +} + +void WaitOutMsgQueueProof::got_state_root(Ref root) { + state_root_ = std::move(root); + if (--pending == 0) { + run_local_cont(); + } +} + +void WaitOutMsgQueueProof::got_block_root(Ref root) { + block_root_ = std::move(root); + if (--pending == 0) { + run_local_cont(); + } +} + +void WaitOutMsgQueueProof::run_local_cont() { + Ref block_state_proof; + if (block_id_.seqno() != 0) { + auto R = create_block_state_proof(std::move(block_root_)); + if (R.is_error()) { + abort_query(R.move_as_error_prefix("failed to create block state proof")); + return; + } + block_state_proof = R.move_as_ok(); + } + finish_query(td::Ref(true, std::move(state_root_), std::move(block_state_proof))); +} + +void WaitOutMsgQueueProof::run_net() { + auto P = + td::PromiseCreator::lambda([SelfId = actor_id(this), block_id = block_id_](td::Result> R) { + if (R.is_error()) { + LOG(DEBUG) << "failed to get msg queue for " << block_id.to_str() << " from net: " << R.move_as_error(); + delay_action([SelfId]() mutable { td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::run_net); }, + td::Timestamp::in(0.1)); + } else { + td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::finish_query, R.move_as_ok()); + } + }); + + td::actor::send_closure(manager_, &ValidatorManager::send_get_out_msg_queue_proof_request, block_id_, dst_shard_, + priority_, std::move(P)); +} + +void BuildOutMsgQueueProof::abort_query(td::Status reason) { + if (promise_) { + LOG(WARNING) << "failed to build msg queue proof for " << block_id_.to_str() << ": " << reason; + promise_.set_error( + reason.move_as_error_prefix(PSTRING() << "failed to build msg queue proof for " << block_id_.to_str() << ": ")); + } + stop(); +} + +void BuildOutMsgQueueProof::start_up() { + ++pending; + td::actor::send_closure(manager_, &ValidatorManagerInterface::get_shard_state_from_db_short, block_id_, + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::abort_query, + R.move_as_error_prefix("failed to get shard state")); + } else { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_state_root, + R.move_as_ok()->root_cell()); + } + }); + if (block_id_.seqno() != 0) { + ++pending; + td::actor::send_closure(manager_, &ValidatorManagerInterface::get_block_data_from_db_short, block_id_, + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::abort_query, + R.move_as_error_prefix("failed to get block data")); + } else { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_block_root, + R.move_as_ok()->root_cell()); + } + }); + } +} + +void BuildOutMsgQueueProof::got_state_root(Ref root) { + state_root_ = std::move(root); + if (--pending == 0) { + build_proof(); + } +} + +void BuildOutMsgQueueProof::got_block_root(Ref root) { + block_root_ = std::move(root); + if (--pending == 0) { + build_proof(); + } +} + +void BuildOutMsgQueueProof::build_proof() { + promise_.set_result( + OutMsgQueueProof::serialize(block_id_, dst_shard_, std::move(state_root_), std::move(block_root_))); + stop(); +} + +} // namespace validator +} // namespace ton diff --git a/validator/impl/out-msg-queue-proof.hpp b/validator/impl/out-msg-queue-proof.hpp new file mode 100644 index 000000000..6a18ca6cb --- /dev/null +++ b/validator/impl/out-msg-queue-proof.hpp @@ -0,0 +1,109 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "vm/cells.h" +#include "ton/ton-types.h" +#include "auto/tl/ton_api.h" +#include "interfaces/out-msg-queue-proof.h" +#include "td/actor/actor.h" + +namespace ton { + +namespace validator { +using td::Ref; + +class ValidatorManager; +class ValidatorManagerInterface; + +class WaitOutMsgQueueProof : public td::actor::Actor { + public: + WaitOutMsgQueueProof(BlockIdExt block_id, ShardIdFull dst_shard, bool local, td::uint32 priority, + td::actor::ActorId manager, td::Timestamp timeout, + td::Promise> promise) + : block_id_(std::move(block_id)) + , dst_shard_(dst_shard) + , local_(local) + , priority_(priority) + , manager_(manager) + , timeout_(timeout) + , promise_(std::move(promise)) { + } + + void update_timeout(td::Timestamp timeout, td::uint32 priority) { + timeout_ = timeout; + alarm_timestamp() = timeout_; + priority_ = priority; + } + + void abort_query(td::Status reason); + void finish_query(Ref result); + void alarm() override; + + void start_up() override; + + void run_local(); + void got_state_root(Ref root); + void got_block_root(Ref root); + void run_local_cont(); + + void run_net(); + + + private: + BlockIdExt block_id_; + ShardIdFull dst_shard_; + bool local_; + td::uint32 priority_; + + td::actor::ActorId manager_; + td::Timestamp timeout_; + td::Promise> promise_; + + Ref state_root_, block_root_; + unsigned pending = 0; +}; + +class BuildOutMsgQueueProof : public td::actor::Actor { + public: + BuildOutMsgQueueProof(BlockIdExt block_id, ShardIdFull dst_shard, + td::actor::ActorId manager, + td::Promise> promise) + : block_id_(std::move(block_id)) + , dst_shard_(dst_shard) + , manager_(manager) + , promise_(std::move(promise)) { + } + + void abort_query(td::Status reason); + void start_up() override; + void got_state_root(Ref root); + void got_block_root(Ref root); + void build_proof(); + + private: + BlockIdExt block_id_; + ShardIdFull dst_shard_; + + td::actor::ActorId manager_; + td::Promise> promise_; + + Ref state_root_, block_root_; + unsigned pending = 0; +}; + +} // namespace validator +} // namespace ton diff --git a/validator/impl/proof.cpp b/validator/impl/proof.cpp index 033a1ab12..2297b48c1 100644 --- a/validator/impl/proof.cpp +++ b/validator/impl/proof.cpp @@ -162,5 +162,40 @@ td::Result> ProofQ::get_signatures_root() const { return proof.signatures->prefetch_ref(); } +td::Result> create_block_state_proof(td::Ref root) { + if (root.is_null()) { + return td::Status::Error("root is null"); + } + vm::MerkleProofBuilder mpb{std::move(root)}; + block::gen::Block::Record block; + if (!tlb::unpack_cell(mpb.root(), block) || block.state_update->load_cell().is_error()) { + return td::Status::Error("invalid block"); + } + Ref proof = mpb.extract_proof(); + if (proof.is_null()) { + return td::Status::Error("failed to create proof"); + } + return proof; +} + +td::Result unpack_block_state_proof(BlockIdExt block_id, td::Ref proof) { + auto virt_root = vm::MerkleProof::virtualize(proof, 1); + if (virt_root.is_null()) { + return td::Status::Error("invalid Merkle proof"); + } + if (virt_root->get_hash().as_slice() != block_id.root_hash.as_slice()) { + return td::Status::Error("hash mismatch"); + } + block::gen::Block::Record block; + if (!tlb::unpack_cell(virt_root, block)) { + return td::Status::Error("invalid block"); + } + vm::CellSlice upd_cs{vm::NoVmSpec(), block.state_update}; + if (!(upd_cs.is_special() && upd_cs.prefetch_long(8) == 4 && upd_cs.size_ext() == 0x20228)) { + return td::Status::Error("invalid Merkle update"); + } + return upd_cs.prefetch_ref(1)->get_hash(0).bits(); +} + } // namespace validator } // namespace ton diff --git a/validator/interfaces/out-msg-queue-proof.h b/validator/interfaces/out-msg-queue-proof.h new file mode 100644 index 000000000..1fece66a5 --- /dev/null +++ b/validator/interfaces/out-msg-queue-proof.h @@ -0,0 +1,42 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "vm/cells.h" +#include "ton/ton-types.h" +#include "auto/tl/ton_api.h" + +namespace ton { + +namespace validator { +using td::Ref; + +struct OutMsgQueueProof : public td::CntObject { + OutMsgQueueProof(Ref state_root, Ref block_state_proof) + : state_root_(std::move(state_root)), block_state_proof_(std::move(block_state_proof)) { + } + + Ref state_root_; + Ref block_state_proof_; + + static td::Result> fetch(BlockIdExt block_id, ShardIdFull dst_shard, + const ton_api::tonNode_outMsgQueueProof &f); + static td::Result> serialize( + BlockIdExt block_id, ShardIdFull dst_shard, Ref state_root, Ref block_root); +}; + +} // namespace validator +} // namespace ton diff --git a/validator/interfaces/proof.h b/validator/interfaces/proof.h index 99471a1ff..6665ad080 100644 --- a/validator/interfaces/proof.h +++ b/validator/interfaces/proof.h @@ -48,6 +48,9 @@ class Proof : virtual public ProofLink { virtual td::Result> export_as_proof_link() const = 0; }; +td::Result> create_block_state_proof(td::Ref root); +td::Result unpack_block_state_proof(BlockIdExt block_id, td::Ref proof); + } // namespace validator } // namespace ton diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 209f871f2..362c259f9 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -29,6 +29,7 @@ #include "liteserver.h" #include "crypto/vm/db/DynamicBagOfCellsDb.h" #include "validator-session/validator-session-types.h" +#include "impl/out-msg-queue-proof.hpp" namespace ton { @@ -128,6 +129,8 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void send_ihr_message(td::Ref message) = 0; virtual void send_top_shard_block_description(td::Ref desc) = 0; virtual void send_block_broadcast(BlockBroadcast broadcast) = 0; + virtual void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, td::uint32 priority, + td::Promise> promise) = 0; virtual void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_shard_client_state(bool from_db, td::Promise promise) = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 501ee078c..d8f75eef6 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -152,6 +152,10 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; + void wait_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) override { + UNREACHABLE(); + } void set_block_data(BlockHandle handle, td::Ref data, td::Promise promise) override; void wait_block_data(BlockHandle handle, td::uint32 priority, td::Timestamp, @@ -250,6 +254,10 @@ class ValidatorManagerImpl : public ValidatorManager { void send_top_shard_block_description(td::Ref desc) override; void send_block_broadcast(BlockBroadcast broadcast) override { } + void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, td::uint32 priority, + td::Promise> promise) override { + UNREACHABLE(); + } void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 677827714..87ec8c2ba 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -182,6 +182,10 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; + void wait_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) override { + UNREACHABLE(); + } void set_block_data(BlockHandle handle, td::Ref data, td::Promise promise) override { UNREACHABLE(); @@ -314,6 +318,10 @@ class ValidatorManagerImpl : public ValidatorManager { } void send_block_broadcast(BlockBroadcast broadcast) override { } + void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, td::uint32 priority, + td::Promise> promise) override { + UNREACHABLE(); + } void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override { UNREACHABLE(); diff --git a/validator/manager.cpp b/validator/manager.cpp index 7c5e7d3f1..64ee32734 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -453,24 +453,18 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refblock_id().shard_full(), desc->catchain_seqno()}] = desc; VLOG(VALIDATOR_DEBUG) << "new shard block descr for " << desc->block_id(); - if (opts_->need_monitor(desc->block_id().shard_full()) && last_masterchain_block_handle_ && - last_masterchain_seqno_ > 0 && desc->generated_at() < last_masterchain_block_handle_->unix_time() + 60) { - delay_action( - [SelfId = actor_id(this), desc]() { - auto P = td::PromiseCreator::lambda([](td::Result> R) { - if (R.is_error()) { - auto S = R.move_as_error(); - if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) { - VLOG(VALIDATOR_NOTICE) << "failed to get shard state: " << S; - } else { - VLOG(VALIDATOR_DEBUG) << "failed to get shard state: " << S; - } - } - }); - td::actor::send_closure(SelfId, &ValidatorManager::wait_block_state_short, desc->block_id(), 0, - td::Timestamp::in(60.0), std::move(P)); - }, - td::Timestamp::in(1.0)); + if (opts_->need_monitor(desc->block_id().shard_full())) { + auto P = td::PromiseCreator::lambda([](td::Result> R) { + if (R.is_error()) { + auto S = R.move_as_error(); + if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) { + VLOG(VALIDATOR_NOTICE) << "failed to get shard state: " << S; + } else { + VLOG(VALIDATOR_DEBUG) << "failed to get shard state: " << S; + } + } + }); + wait_block_state_short(desc->block_id(), 0, td::Timestamp::in(60.0), std::move(P)); } } } @@ -606,6 +600,30 @@ void ValidatorManagerImpl::wait_block_state_short(BlockIdExt block_id, td::uint3 get_block_handle(block_id, true, std::move(P)); } +void ValidatorManagerImpl::wait_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::uint32 priority, + td::Timestamp timeout, + td::Promise> promise) { + auto key = std::make_pair(block_id, dst_shard); + auto it = wait_out_msg_queue_proof_.find(key); + if (it == wait_out_msg_queue_proof_.end()) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), block_id, dst_shard](td::Result> R) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_msg_queue, block_id, dst_shard, + std::move(R)); + }); + auto id = td::actor::create_actor("waitmsgqueue", block_id, dst_shard, + opts_->need_monitor(block_id.shard_full()), priority, + actor_id(this), td::Timestamp::in(10.0), std::move(P)) + .release(); + wait_out_msg_queue_proof_[key].actor_ = id; + it = wait_out_msg_queue_proof_.find(key); + } + + it->second.waiting_.emplace_back(timeout, priority, std::move(promise)); + auto X = it->second.get_timeout(); + td::actor::send_closure(it->second.actor_, &WaitOutMsgQueueProof::update_timeout, X.first, X.second); +} + void ValidatorManagerImpl::wait_block_data(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) { auto it = wait_block_data_.find(handle->id()); @@ -1037,6 +1055,40 @@ void ValidatorManagerImpl::finished_wait_data(BlockHandle handle, td::Result> R) { + auto it = wait_out_msg_queue_proof_.find({block_id, dst_shard}); + if (it != wait_out_msg_queue_proof_.end()) { + if (R.is_error()) { + auto S = R.move_as_error(); + if (S.code() != ErrorCode::timeout) { + for (auto &X : it->second.waiting_) { + X.promise.set_error(S.clone()); + } + } else { + auto X = it->second.get_timeout(); + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), block_id, dst_shard](td::Result> R) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_msg_queue, block_id, dst_shard, + std::move(R)); + }); + auto id = td::actor::create_actor("waitmsgqueue", block_id, dst_shard, + opts_->need_monitor(block_id.shard_full()), X.second, + actor_id(this), X.first, std::move(P)) + .release(); + it->second.actor_ = id; + return; + } + } else { + auto r = R.move_as_ok(); + for (auto &X : it->second.waiting_) { + X.promise.set_result(r); + } + } + wait_out_msg_queue_proof_.erase(it); + } +} + void ValidatorManagerImpl::set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) { auto P = td::PromiseCreator::lambda( @@ -1410,6 +1462,12 @@ void ValidatorManagerImpl::send_block_broadcast(BlockBroadcast broadcast) { callback_->send_broadcast(std::move(broadcast)); } +void ValidatorManagerImpl::send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, + td::uint32 priority, + td::Promise> promise) { + callback_->download_out_msg_queue_proof(id, dst_shard, td::Timestamp::in(10.0), std::move(promise)); +} + void ValidatorManagerImpl::start_up() { db_ = create_db_actor(actor_id(this), db_root_); lite_server_cache_ = create_liteserver_cache_actor(actor_id(this), db_root_); diff --git a/validator/manager.hpp b/validator/manager.hpp index 1105b2b62..327aa3c51 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -183,6 +183,8 @@ class ValidatorManagerImpl : public ValidatorManager { }; std::map>> wait_state_; std::map>> wait_block_data_; + std::map, WaitList>> + wait_out_msg_queue_proof_; struct WaitBlockHandle { std::vector> waiting_; @@ -357,6 +359,8 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; + void wait_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) override; void set_block_data(BlockHandle handle, td::Ref data, td::Promise promise) override; void wait_block_data(BlockHandle handle, td::uint32 priority, td::Timestamp, @@ -444,6 +448,8 @@ class ValidatorManagerImpl : public ValidatorManager { void send_ihr_message(td::Ref message) override; void send_top_shard_block_description(td::Ref desc) override; void send_block_broadcast(BlockBroadcast broadcast) override; + void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, td::uint32 priority, + td::Promise> promise) override; void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; @@ -478,6 +484,7 @@ class ValidatorManagerImpl : public ValidatorManager { void finished_wait_state(BlockHandle handle, td::Result> R); void finished_wait_data(BlockHandle handle, td::Result> R); + void finished_wait_msg_queue(BlockIdExt block_id, ShardIdFull dst_shard, td::Result> R); void start_up() override; void started(ValidatorManagerInitResult result); diff --git a/validator/validator.h b/validator/validator.h index ab013cb08..6804ae804 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -34,6 +34,7 @@ #include "interfaces/proof.h" #include "interfaces/shard.h" #include "catchain/catchain-types.h" +#include "interfaces/out-msg-queue-proof.h" namespace ton { @@ -131,6 +132,8 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Promise> promise) = 0; virtual void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) = 0; + virtual void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, + td::Promise> promise) = 0; virtual void new_key_block(BlockHandle handle) = 0; }; @@ -212,6 +215,10 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) = 0; + virtual void wait_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::uint32 priority, + td::Timestamp timeout, + td::Promise> promise) = 0; + virtual void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) = 0; virtual void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise) = 0; From cffffbab79b29a8b91d6a908f099bc81c6072115 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 1 Aug 2022 18:19:50 +0300 Subject: [PATCH 010/388] Fix CE --- create-hardfork/create-hardfork.cpp | 3 +++ dht-server/dht-server.cpp | 2 +- test/test-ton-collator.cpp | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index 27260ee02..5455a984e 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -269,6 +269,9 @@ class HardforkCreator : public td::actor::Actor { td::Promise promise) override { } + void download_out_msg_queue_proof(ton::BlockIdExt block_id, ton::ShardIdFull dst_shard, td::Timestamp timeout, + td::Promise> promise) override { + } void new_key_block(ton::validator::BlockHandle handle) override { } diff --git a/dht-server/dht-server.cpp b/dht-server/dht-server.cpp index 212a6faee..24b7b7e30 100644 --- a/dht-server/dht-server.cpp +++ b/dht-server/dht-server.cpp @@ -146,7 +146,7 @@ ton::tl_object_ptr Config::tl() const { dht_vec.push_back(ton::create_tl_object(x.tl())); } - std::vector> val_vec; + std::vector> val_vec; std::vector> full_node_slaves_vec; std::vector> full_node_masters_vec; diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index b3d3f9db4..a68422d59 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -370,6 +370,9 @@ class TestNode : public td::actor::Actor { td::Promise promise) override { } + void download_out_msg_queue_proof(ton::BlockIdExt block_id, ton::ShardIdFull dst_shard, td::Timestamp timeout, + td::Promise> promise) override { + } void new_key_block(ton::validator::BlockHandle handle) override { } From ac7a33497b74371cd2b843942a7ee4176c3c5909 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 2 Aug 2022 18:22:56 +0300 Subject: [PATCH 011/388] Validate msg queue proof --- validator/impl/out-msg-queue-proof.cpp | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 0c3c993cd..4a8731d39 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -26,7 +26,7 @@ namespace ton { namespace validator { td::Result> OutMsgQueueProof::fetch(BlockIdExt block_id, ShardIdFull dst_shard, - const ton_api::tonNode_outMsgQueueProof &f) { + const ton_api::tonNode_outMsgQueueProof& f) { Ref block_state_proof; td::Bits256 state_root_hash; if (block_id.seqno() == 0) { @@ -40,20 +40,30 @@ td::Result> OutMsgQueueProof::fetch(BlockIdExt block_i } TRY_RESULT(queue_proof, vm::std_boc_deserialize(f.queue_proof_.as_slice())); - auto state_root = vm::MerkleProof::virtualize(queue_proof, 1); - if (state_root.is_null()) { + auto virtual_root = vm::MerkleProof::virtualize(queue_proof, 1); + if (virtual_root.is_null()) { return td::Status::Error("invalid queue proof"); } - if (state_root->get_hash().as_slice() != state_root_hash.as_slice()) { + if (virtual_root->get_hash().as_slice() != state_root_hash.as_slice()) { return td::Status::Error("state root hash mismatch"); } - // TODO: validate - return Ref(true, std::move(state_root), std::move(block_state_proof)); + // Validate proof + auto state_root = vm::CellSlice(vm::NoVm(), queue_proof).prefetch_ref(0); + TRY_RESULT_PREFIX(state, ShardStateQ::fetch(block_id, {}, state_root), "invalid proof: "); + TRY_RESULT_PREFIX(queue, state->message_queue(), "invalid proof: "); + auto queue_root = queue->root_cell(); + if (queue_root->get_level() != 0) { + return td::Status::Error("invalid proof: msg queue has prunned branches"); + } + + return Ref(true, std::move(virtual_root), std::move(block_state_proof)); } -td::Result> OutMsgQueueProof::serialize( - BlockIdExt block_id, ShardIdFull dst_shard, Ref state_root, Ref block_root) { +td::Result> OutMsgQueueProof::serialize(BlockIdExt block_id, + ShardIdFull dst_shard, + Ref state_root, + Ref block_root) { vm::MerkleProofBuilder mpb{std::move(state_root)}; TRY_RESULT(state, ShardStateQ::fetch(block_id, {}, mpb.root())); TRY_RESULT(outq_descr, state->message_queue()); From 212c07f2f0408c40efd9db442d6f9bb8657a685d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 3 Aug 2022 15:15:42 +0300 Subject: [PATCH 012/388] Change update_shard_configuration --- create-hardfork/create-hardfork.cpp | 3 +- test/test-ton-collator.cpp | 3 +- validator-engine/validator-engine.cpp | 2 +- validator/full-node.cpp | 74 ++++++++---------------- validator/full-node.h | 15 ++--- validator/full-node.hpp | 9 ++- validator/interfaces/validator-manager.h | 2 +- validator/manager-disk.hpp | 2 +- validator/manager-hardfork.hpp | 2 +- validator/manager.cpp | 16 ++--- validator/manager.hpp | 4 +- validator/shard-client.cpp | 53 ++++++++++++++++- validator/shard-client.hpp | 2 +- validator/validator.h | 3 +- 14 files changed, 106 insertions(+), 84 deletions(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index 5455a984e..a24b7fc18 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -236,7 +236,8 @@ class HardforkCreator : public td::actor::Actor { td::actor::send_closure(id_, &ton::validator::ValidatorManager::sync_complete, td::PromiseCreator::lambda([](td::Unit) {})); } - void update_shard_configuration(td::Ref state) override { + void update_shard_configuration(td::Ref state, + std::set shards_to_monitor) override { } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index a68422d59..d9ec82e5b 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -323,7 +323,8 @@ class TestNode : public td::actor::Actor { td::actor::send_closure(id_, &ton::validator::ValidatorManager::sync_complete, td::PromiseCreator::lambda([](td::Unit) {})); } - void update_shard_configuration(td::Ref state) override { + void update_shard_configuration(td::Ref state, + std::set shards_to_monitor) override { } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 1bb820a42..b2617769e 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1853,7 +1853,7 @@ void ValidatorEngine::start_full_node() { }); full_node_ = ton::validator::fullnode::FullNode::create( short_id, ton::adnl::AdnlNodeIdShort{config_.full_node}, validator_options_->zero_block_id().file_hash, - validator_options_, keyring_.get(), adnl_.get(), rldp_.get(), + keyring_.get(), adnl_.get(), rldp_.get(), default_dht_node_.is_zero() ? td::actor::ActorId{} : dht_nodes_[default_dht_node_].get(), overlay_manager_.get(), validator_manager_.get(), full_node_client_.get(), db_root_, std::move(P)); for (auto &v : config_.validators) { diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 9194b9a4c..63891502d 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -132,52 +132,31 @@ void FullNodeImpl::initial_read_complete(BlockHandle top_handle) { td::actor::send_closure(it->second.actor, &FullNodeShard::set_handle, top_handle, std::move(P)); } -void FullNodeImpl::update_shard_configuration(td::Ref state) { +void FullNodeImpl::update_shard_configuration(td::Ref state, + std::set shards_to_monitor) { std::map new_shards; std::set new_active; new_shards[ShardIdFull(masterchainId)] = state->get_block_id(); - new_active.insert(ShardIdFull(masterchainId)); std::set workchains; - auto cur_time = state->get_unix_time(); - auto set_active = [&](ShardIdFull shard) { while (new_active.insert(shard).second && shard.pfx_len() > 0) { shard = shard_parent(shard); } }; - for (auto &info : state->get_shards()) { - auto shard = info->shard(); - workchains.insert(shard.workchain); - new_shards[shard] = info->top_block_id(); - bool will_split = shard.pfx_len() < max_shard_pfx_len && ((info->fsm_state() == McShardHash::FsmState::fsm_split && - info->fsm_utime() < cur_time + 60) || info->before_split()); - bool will_merge = shard.pfx_len() > 0 && ((info->fsm_state() == McShardHash::FsmState::fsm_merge && - info->fsm_utime() < cur_time + 60) || info->before_merge()); - if (opts_->need_monitor(shard)) { - set_active(shard); - } - if (will_merge && opts_->need_monitor(shard_parent(shard))) { - set_active(shard); - set_active(shard_sibling(shard)); - } - for (int id = 0; id < 2; ++id) { - if (will_split && opts_->need_monitor(shard_child(shard, id))) { - set_active(shard_child(shard, id)); - } - } + workchains.insert(info->shard().workchain); + new_shards[info->shard()] = info->top_block_id(); } for (const auto &wpair : state->get_workchain_list()) { ton::WorkchainId wc = wpair.first; const block::WorkchainInfo *winfo = wpair.second.get(); - if (workchains.count(wc) == 0 && winfo->active && winfo->enabled_since <= cur_time) { - auto shard = ShardIdFull(wc); - new_shards[shard] = BlockIdExt(wc, shard.shard, 0, winfo->zerostate_root_hash, winfo->zerostate_file_hash); - if (opts_->need_monitor(shard)) { - set_active(shard); - } + if (workchains.count(wc) == 0 && winfo->active && winfo->enabled_since <= state->get_unix_time()) { + new_shards[ShardIdFull(wc)] = BlockIdExt(wc, shardIdAll, 0, winfo->zerostate_root_hash, winfo->zerostate_file_hash); } } + for (ShardIdFull shard : shards_to_monitor) { + set_active(shard); + } auto info_set_active = [&](ShardIdFull shard, ShardInfo& info, bool active) { if (info.active == active) { @@ -195,10 +174,6 @@ void FullNodeImpl::update_shard_configuration(td::Ref state) { for (auto shard : new_shards) { auto &info = shards_[shard.first]; info.exists = true; - if (!info.active && new_active.count(shard.first)) { - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::wait_block_state_short, shard.second, 0, - td::Timestamp::in(60.0), [](td::Result>){}); - } } for (auto& p : shards_) { @@ -511,8 +486,9 @@ void FullNodeImpl::start_up() { void initial_read_complete(BlockHandle handle) override { td::actor::send_closure(id_, &FullNodeImpl::initial_read_complete, handle); } - void update_shard_configuration(td::Ref state) override { - td::actor::send_closure(id_, &FullNodeImpl::update_shard_configuration, std::move(state)); + void update_shard_configuration(td::Ref state, std::set shards_to_monitor) override { + td::actor::send_closure(id_, &FullNodeImpl::update_shard_configuration, std::move(state), + std::move(shards_to_monitor)); } void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) override { td::actor::send_closure(id_, &FullNodeImpl::send_ihr_message, dst, std::move(data)); @@ -580,16 +556,15 @@ void FullNodeImpl::start_up() { } FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - td::Ref opts, td::actor::ActorId keyring, - td::actor::ActorId adnl, td::actor::ActorId rldp, - td::actor::ActorId dht, td::actor::ActorId overlays, + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId dht, + td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, std::string db_root, td::Promise started_promise) : local_id_(local_id) , adnl_id_(adnl_id) , zero_state_file_hash_(zero_state_file_hash) - , opts_(opts) , keyring_(keyring) , adnl_(adnl) , rldp_(rldp) @@ -602,17 +577,14 @@ FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id add_shard_actor(ShardIdFull{masterchainId}, true); } -td::actor::ActorOwn FullNode::create(ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, - FileHash zero_state_file_hash, td::Ref opts, - td::actor::ActorId keyring, - td::actor::ActorId adnl, td::actor::ActorId rldp, - td::actor::ActorId dht, - td::actor::ActorId overlays, - td::actor::ActorId validator_manager, - td::actor::ActorId client, std::string db_root, - td::Promise started_promise) { - return td::actor::create_actor("fullnode", local_id, adnl_id, zero_state_file_hash, opts, keyring, adnl, - rldp, dht, overlays, validator_manager, client, db_root, +td::actor::ActorOwn FullNode::create( + ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId dht, + td::actor::ActorId overlays, td::actor::ActorId validator_manager, + td::actor::ActorId client, std::string db_root, td::Promise started_promise) { + return td::actor::create_actor("fullnode", local_id, adnl_id, zero_state_file_hash, keyring, adnl, rldp, + dht, overlays, validator_manager, client, db_root, std::move(started_promise)); } diff --git a/validator/full-node.h b/validator/full-node.h index 7342a8229..9f268666c 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -72,15 +72,12 @@ class FullNode : public td::actor::Actor { return 4ull << 30; } - static td::actor::ActorOwn create(ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, - FileHash zero_state_file_hash, td::Ref opts, - td::actor::ActorId keyring, - td::actor::ActorId adnl, td::actor::ActorId rldp, - td::actor::ActorId dht, - td::actor::ActorId overlays, - td::actor::ActorId validator_manager, - td::actor::ActorId client, std::string db_root, - td::Promise started_promise); + static td::actor::ActorOwn create( + ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId dht, + td::actor::ActorId overlays, td::actor::ActorId validator_manager, + td::actor::ActorId client, std::string db_root, td::Promise started_promise); }; } // namespace fullnode diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 49dad3ba3..2f8254587 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -52,7 +52,7 @@ class FullNodeImpl : public FullNode { void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; - void update_shard_configuration(td::Ref state); + void update_shard_configuration(td::Ref state, std::set shards_to_monitor); void sync_completed(); @@ -83,9 +83,9 @@ class FullNodeImpl : public FullNode { void start_up() override; FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - td::Ref opts, td::actor::ActorId keyring, - td::actor::ActorId adnl, td::actor::ActorId rldp, - td::actor::ActorId dht, td::actor::ActorId overlays, + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId dht, + td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, std::string db_root, td::Promise started_promise); @@ -96,7 +96,6 @@ class FullNodeImpl : public FullNode { PublicKeyHash local_id_; adnl::AdnlNodeIdShort adnl_id_; FileHash zero_state_file_hash_; - td::Ref opts_; td::actor::ActorId get_shard(AccountIdPrefixFull dst); td::actor::ActorId get_shard(ShardIdFull shard, bool exact = false); diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 362c259f9..345ae679f 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -134,7 +134,7 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_shard_client_state(bool from_db, td::Promise promise) = 0; - virtual void update_shard_configuration(td::Ref state) = 0; + virtual void update_shard_configuration(td::Ref state, std::set shards_to_monitor) = 0; virtual void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) = 0; virtual void get_async_serializer_state(td::Promise promise) = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index d8f75eef6..2e331c720 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -261,7 +261,7 @@ class ValidatorManagerImpl : public ValidatorManager { void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; - void update_shard_configuration(td::Ref state) override { + void update_shard_configuration(td::Ref state, std::set shards_to_monitor) override { } void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) override { diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 87ec8c2ba..79e742c4b 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -329,7 +329,7 @@ class ValidatorManagerImpl : public ValidatorManager { void get_shard_client_state(bool from_db, td::Promise promise) override { UNREACHABLE(); } - void update_shard_configuration(td::Ref state) override { + void update_shard_configuration(td::Ref state, std::set shards_to_monitor) override { UNREACHABLE(); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 64ee32734..442796981 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -372,7 +372,7 @@ void ValidatorManagerImpl::new_external_message(td::BufferSlice data) { if (!is_validator()) { return; } - if( ext_messages_.size() > max_mempool_num() ) { + if ((double)ext_messages_.size() > max_mempool_num()) { return; } auto R = create_ext_message(std::move(data)); @@ -453,7 +453,7 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refblock_id().shard_full(), desc->catchain_seqno()}] = desc; VLOG(VALIDATOR_DEBUG) << "new shard block descr for " << desc->block_id(); - if (opts_->need_monitor(desc->block_id().shard_full())) { + if (shards_to_monitor_.count(desc->block_id().shard_full())) { auto P = td::PromiseCreator::lambda([](td::Result> R) { if (R.is_error()) { auto S = R.move_as_error(); @@ -612,7 +612,7 @@ void ValidatorManagerImpl::wait_out_msg_queue_proof(BlockIdExt block_id, ShardId std::move(R)); }); auto id = td::actor::create_actor("waitmsgqueue", block_id, dst_shard, - opts_->need_monitor(block_id.shard_full()), priority, + shards_to_monitor_.count(block_id.shard_full()), priority, actor_id(this), td::Timestamp::in(10.0), std::move(P)) .release(); wait_out_msg_queue_proof_[key].actor_ = id; @@ -1073,8 +1073,8 @@ void ValidatorManagerImpl::finished_wait_msg_queue(BlockIdExt block_id, ShardIdF std::move(R)); }); auto id = td::actor::create_actor("waitmsgqueue", block_id, dst_shard, - opts_->need_monitor(block_id.shard_full()), X.second, - actor_id(this), X.first, std::move(P)) + shards_to_monitor_.count(block_id.shard_full()), + X.second, actor_id(this), X.first, std::move(P)) .release(); it->second.actor_ = id; return; @@ -2425,8 +2425,10 @@ void ValidatorManagerImpl::get_shard_client_state(bool from_db, td::Promise state) { - callback_->update_shard_configuration(state); +void ValidatorManagerImpl::update_shard_configuration(td::Ref state, + std::set shards_to_monitor) { + shards_to_monitor_ = shards_to_monitor; + callback_->update_shard_configuration(std::move(state), std::move(shards_to_monitor)); } void ValidatorManagerImpl::update_async_serializer_state(AsyncSerializerState state, td::Promise promise) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 327aa3c51..6eb192a71 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -453,7 +453,7 @@ class ValidatorManagerImpl : public ValidatorManager { void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; - void update_shard_configuration(td::Ref state) override; + void update_shard_configuration(td::Ref state, std::set shards_to_monitor) override; void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) override; void get_async_serializer_state(td::Promise promise) override; @@ -619,6 +619,8 @@ class ValidatorManagerImpl : public ValidatorManager { std::map> collator_nodes_; bool collating_masterchain_ = false; + + std::set shards_to_monitor_ = {ShardIdFull(masterchainId)}; }; } // namespace validator diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index 50c6075d8..65adf34d4 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -91,7 +91,7 @@ void ShardClient::start_up_init_mode() { auto vec = masterchain_state_->get_shards(); for (auto &shard : vec) { - if (opts_->need_monitor(shard->shard())) { + if (shards_to_monitor_.count(shard->shard())) { auto P = td::PromiseCreator::lambda([promise = ig.get_promise()](td::Result> R) mutable { R.ensure(); promise.set_value(td::Unit()); @@ -192,7 +192,7 @@ void ShardClient::apply_all_shards() { auto vec = masterchain_state_->get_shards(); for (auto &shard : vec) { - if (opts_->need_monitor(shard->shard())) { + if (shards_to_monitor_.count(shard->shard())) { auto Q = td::PromiseCreator::lambda([SelfId = actor_id(this), promise = ig.get_promise(), shard = shard->shard()](td::Result> R) mutable { if (R.is_error()) { @@ -247,7 +247,54 @@ void ShardClient::get_processed_masterchain_block_id(td::Promise pro } void ShardClient::build_shard_overlays() { - td::actor::send_closure(manager_, &ValidatorManager::update_shard_configuration, masterchain_state_); + std::set new_shards_to_monitor; + std::set workchains; + auto cur_time = masterchain_state_->get_unix_time(); + new_shards_to_monitor.insert(ShardIdFull(masterchainId)); + for (const auto &info : masterchain_state_->get_shards()) { + auto shard = info->shard(); + workchains.insert(shard.workchain); + bool will_split = shard.pfx_len() < max_shard_pfx_len && + ((info->fsm_state() == McShardHash::FsmState::fsm_split && info->fsm_utime() < cur_time + 60) || + info->before_split()); + bool will_merge = shard.pfx_len() > 0 && + ((info->fsm_state() == McShardHash::FsmState::fsm_merge && info->fsm_utime() < cur_time + 60) || + info->before_merge()); + if (opts_->need_monitor(shard) || (will_merge && opts_->need_monitor(shard_parent(shard)))) { + new_shards_to_monitor.insert(shard); + } + if (will_merge && opts_->need_monitor(shard_parent(shard))) { + new_shards_to_monitor.insert(shard_parent(shard)); + } + if (will_split) { + for (int id = 0; id < 2; ++id) { + if (opts_->need_monitor(shard_child(shard, id))) { + new_shards_to_monitor.insert(shard_child(shard, id)); + } + } + } + } + + std::vector new_workchains; + for (const auto &wpair : masterchain_state_->get_workchain_list()) { + ton::WorkchainId wc = wpair.first; + const block::WorkchainInfo *winfo = wpair.second.get(); + auto shard = ShardIdFull(wc); + if (workchains.count(wc) == 0 && winfo->active && winfo->enabled_since <= cur_time && opts_->need_monitor(shard)) { + new_shards_to_monitor.insert(shard); + if (shards_to_monitor_.count(shard) == 0) { + new_workchains.push_back(BlockIdExt(wc, shardIdAll, 0, winfo->zerostate_root_hash, winfo->zerostate_file_hash)); + } + } + } + + td::actor::send_closure(manager_, &ValidatorManager::update_shard_configuration, masterchain_state_, + new_shards_to_monitor); + shards_to_monitor_ = std::move(new_shards_to_monitor); + for (BlockIdExt block_id : new_workchains) { + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, block_id, shard_client_priority(), + td::Timestamp::in(60.0), [](td::Result>) {}); + } } void ShardClient::force_update_shard_client(BlockHandle handle, td::Promise promise) { diff --git a/validator/shard-client.hpp b/validator/shard-client.hpp index 455b67e7c..701a387e8 100644 --- a/validator/shard-client.hpp +++ b/validator/shard-client.hpp @@ -42,7 +42,7 @@ class ShardClient : public td::actor::Actor { td::Promise promise_; - std::set created_overlays_; + std::set shards_to_monitor_ = {ShardIdFull(masterchainId)}; public: ShardClient(td::Ref opts, BlockHandle masterchain_block_handle, diff --git a/validator/validator.h b/validator/validator.h index 6804ae804..c5212abb4 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -112,7 +112,8 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual ~Callback() = default; virtual void initial_read_complete(BlockHandle top_masterchain_blocks) = 0; - virtual void update_shard_configuration(td::Ref state) = 0; + virtual void update_shard_configuration(td::Ref state, + std::set shards_to_monitor) = 0; virtual void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; From 5be10b1e6bfeb1f505eaf2d7b564221da5f411a4 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 4 Aug 2022 13:55:36 +0300 Subject: [PATCH 013/388] Add retries in validator group + bugfix --- validator/full-node.cpp | 4 +++- validator/validator-group.cpp | 39 ++++++++++++++++++++++++++++------- validator/validator-group.hpp | 5 +++-- 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 63891502d..e4a2cf576 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -407,7 +407,9 @@ void FullNodeImpl::got_key_block_proof(td::Ref proof) { CHECK(all_validators_.size() > 0); for (auto &shard : shards_) { - td::actor::send_closure(shard.second.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + if (!shard.second.actor.empty()) { + td::actor::send_closure(shard.second.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + } } } diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 8eae61332..6c0e242d5 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -263,16 +263,17 @@ void ValidatorGroup::create_session() { validatorsession::ValidatorSessionNode n; n.pub_key = ValidatorFullId{el.key}; n.weight = el.weight; - if (n.pub_key.compute_short_id() == local_id_) { - CHECK(!found); - found = true; - local_id_full_ = n.pub_key; - } if (el.addr.is_zero()) { n.adnl_id = adnl::AdnlNodeIdShort{n.pub_key.compute_short_id()}; } else { n.adnl_id = adnl::AdnlNodeIdShort{el.addr}; } + if (n.pub_key.compute_short_id() == local_id_) { + CHECK(!found); + found = true; + local_id_full_ = n.pub_key; + local_adnl_id_ = n.adnl_id; + } vec.emplace_back(std::move(n)); } CHECK(found); @@ -292,6 +293,8 @@ void ValidatorGroup::create_session() { if (started_) { td::actor::send_closure(session_, &validatorsession::ValidatorSession::start); } + + td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_adnl_id_); } void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterchain_block_id) { @@ -346,7 +349,11 @@ void ValidatorGroup::get_session_info( } void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeout, - td::Promise promise) { + td::Promise promise, unsigned max_retries) { + if (round_id < last_known_round_id_) { + promise.set_error(td::Status::Error("too old")); + return; + } adnl::AdnlNodeIdShort collator = adnl::AdnlNodeIdShort::zero(); // TODO: some other way for storing and choosing collators for real network int cnt = 0; @@ -356,12 +363,26 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo ++cnt; } } - if (collator.is_zero()) { promise.set_error(td::Status::Error(PSTRING() << "no collator for shard " << shard_.to_str())); return; } + if (max_retries > 0) { + promise = td::PromiseCreator::lambda( + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_ok()) { + promise.set_result(R.move_as_ok()); + } else if (timeout && timeout.is_in_past()) { + promise.set_result(R.move_as_error()); + } else { + LOG(WARNING) << "collate query error, retrying: " << R.move_as_error(); + td::actor::send_closure(SelfId, &ValidatorGroup::send_collate_query, round_id, timeout, std::move(promise), + max_retries - 1); + } + }); + } + std::vector> prev_blocks; for (const BlockIdExt &p : prev_block_ids_) { prev_blocks.push_back(create_tl_block_id(p)); @@ -381,7 +402,9 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo }); LOG(INFO) << "collate query for " << create_next_block_id_simple().to_str() << ": send query to " << collator; size_t max_answer_size = config_.max_block_size + config_.max_collated_data_size + 256; - td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, adnl::AdnlNodeIdShort(local_id_), collator, "collatequery", + td::Timestamp query_timeout = td::Timestamp::in(10.0); + query_timeout.relax(timeout); + td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_adnl_id_, collator, "collatequery", std::move(P), timeout, std::move(query), max_answer_size); } diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index d0a96d659..76e6ea992 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -55,7 +55,6 @@ class ValidatorGroup : public td::actor::Actor { init_ = false; create_session(); } - td::actor::send_closure(rldp_, &rldp::Rldp::add_id, adnl::AdnlNodeIdShort(local_id_)); } void get_session_info(td::Promise> promise); @@ -86,7 +85,8 @@ class ValidatorGroup : public td::actor::Actor { private: std::unique_ptr make_validator_session_callback(); - void send_collate_query(td::uint32 round_id, td::Timestamp timeout, td::Promise promise); + void send_collate_query(td::uint32 round_id, td::Timestamp timeout, td::Promise promise, + unsigned max_retries = 4); void receive_collate_query_response(td::uint32 round_id, td::BufferSlice data, td::Promise promise); struct PostponedAccept { @@ -120,6 +120,7 @@ class ValidatorGroup : public td::actor::Actor { std::string db_root_; td::actor::ActorId manager_; td::actor::ActorOwn session_; + adnl::AdnlNodeIdShort local_adnl_id_; bool init_ = false; bool started_ = false; From e5718c499f6ba6c47952efe311a989b5cebaf0d4 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 4 Aug 2022 23:21:55 +0300 Subject: [PATCH 014/388] Increase number of neighbours in BroadcastSimple::distribute (make it consistent with BroadcastFec) --- overlay/overlay-broadcast.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/overlay/overlay-broadcast.cpp b/overlay/overlay-broadcast.cpp index 03991b76b..ecd21e7af 100644 --- a/overlay/overlay-broadcast.cpp +++ b/overlay/overlay-broadcast.cpp @@ -68,7 +68,7 @@ td::Status BroadcastSimple::run_checks() { td::Status BroadcastSimple::distribute() { auto B = serialize(); - auto nodes = overlay_->get_neighbours(3); + auto nodes = overlay_->get_neighbours(5); auto manager = overlay_->overlay_manager(); for (auto &n : nodes) { From a792565afa4aaf33c078cdc14e32762cd4073105 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 5 Aug 2022 11:11:21 +0300 Subject: [PATCH 015/388] Fix MSVC compilation error --- validator/collator-node.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 958b5f136..84ef9eaa8 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -64,6 +64,7 @@ static td::BufferSlice serialize_error(td::Status error) { void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise) { + auto SelfId = actor_id(this); auto status = [&]() -> td::Status { TRY_RESULT(f, fetch_tl_object(std::move(data), true)); ShardIdFull shard(f->workchain_, f->shard_); @@ -92,7 +93,7 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data LOG(INFO) << "Query from " << src << ": shard=" << shard.to_str() << ", min_mc_id=" << min_mc_id.to_str(); - auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this), prev_blocks = std::move(prev_blocks), + auto P = td::PromiseCreator::lambda([=, prev_blocks = std::move(prev_blocks), promise = std::move(promise)](td::Result> R) mutable { if (R.is_error()) { LOG(WARNING) << "Query from " << src << ", error: " << R.error(); From 51e6885f2cc3da9bc5b82f1477fee91ae5f342bd Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 8 Aug 2022 12:21:15 +0300 Subject: [PATCH 016/388] Update validator config format; partial liteservers and their support in liteclient --- dht-server/dht-server.cpp | 15 +- dht-server/dht-server.hpp | 4 +- lite-client/lite-client.cpp | 701 +++++++++++------- lite-client/lite-client.h | 77 +- tl/generate/scheme/ton_api.tl | 20 +- tl/generate/scheme/ton_api.tlo | Bin 70924 -> 72088 bytes ton/ton-tl.hpp | 30 +- .../validator-engine-console-query.cpp | 22 +- .../validator-engine-console-query.h | 23 + .../validator-engine-console.cpp | 1 + validator-engine/validator-engine.cpp | 119 ++- validator-engine/validator-engine.hpp | 8 +- 12 files changed, 657 insertions(+), 363 deletions(-) diff --git a/dht-server/dht-server.cpp b/dht-server/dht-server.cpp index 24b7b7e30..d1f439534 100644 --- a/dht-server/dht-server.cpp +++ b/dht-server/dht-server.cpp @@ -38,6 +38,9 @@ #include "td/utils/TsFileLog.h" #include "td/utils/Random.h" +#include "ton/ton-tl.hpp" +#include "tl/tl_json.h" + #include "memprof/memprof.h" #if TD_DARWIN || TD_LINUX @@ -54,7 +57,7 @@ Config::Config() { out_port = 3278; } -Config::Config(ton::ton_api::engine_validator_config &config) { +Config::Config(const ton::ton_api::engine_validator_config_v2 &config) { out_port = static_cast(config.out_port_); if (!out_port) { out_port = 3278; @@ -121,7 +124,7 @@ Config::Config(ton::ton_api::engine_validator_config &config) { } } -ton::tl_object_ptr Config::tl() const { +ton::tl_object_ptr Config::tl() const { std::vector> addrs_vec; for (auto &x : addrs) { if (x.second.proxy) { @@ -146,7 +149,7 @@ ton::tl_object_ptr Config::tl() const { dht_vec.push_back(ton::create_tl_object(x.tl())); } - std::vector> val_vec; + std::vector> val_vec; std::vector> full_node_slaves_vec; std::vector> full_node_masters_vec; @@ -597,14 +600,14 @@ void DhtServer::load_config(td::Promise promise) { } auto conf_json = conf_json_R.move_as_ok(); - ton::ton_api::engine_validator_config conf; - auto S = ton::ton_api::from_json(conf, conf_json.get_object()); + ton::tl_object_ptr conf; + auto S = td::from_json(conf, std::move(conf_json)); if (S.is_error()) { promise.set_error(S.move_as_error_prefix("json does not fit TL scheme")); return; } - config_ = Config{conf}; + config_ = Config{*ton::unpack_engine_validator_config(std::move(conf))}; td::MultiPromise mp; auto ig = mp.init_guard(); diff --git a/dht-server/dht-server.hpp b/dht-server/dht-server.hpp index bf24d6216..724e7bbfa 100644 --- a/dht-server/dht-server.hpp +++ b/dht-server/dht-server.hpp @@ -91,10 +91,10 @@ struct Config { td::Result config_del_control_process(td::int32 port, ton::PublicKeyHash id); td::Result config_del_gc(ton::PublicKeyHash key); - ton::tl_object_ptr tl() const; + ton::tl_object_ptr tl() const; Config(); - Config(ton::ton_api::engine_validator_config &config); + Config(const ton::ton_api::engine_validator_config_v2 &config); }; class DhtServer : public td::actor::Actor { diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index aad3ae6e3..970cdad94 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -41,8 +41,6 @@ #include "td/utils/crypto.h" #include "td/utils/overloaded.h" #include "td/utils/port/signals.h" -#include "td/utils/port/stacktrace.h" -#include "td/utils/port/StdStreams.h" #include "td/utils/port/FileFd.h" #include "terminal/terminal.h" #include "ton/lite-tl.hpp" @@ -59,17 +57,14 @@ #include "vm/cp0.h" #include "vm/memo.h" #include "ton/ton-shard.h" -#include "openssl/rand.hpp" #include "crypto/vm/utils.h" #include "crypto/common/util.h" #include "common/checksum.h" #if TD_DARWIN || TD_LINUX #include -#include #endif #include -#include #include "git.h" using namespace std::literals::string_literals; @@ -77,22 +72,27 @@ using td::Ref; int verbosity; -std::unique_ptr TestNode::make_callback() { - class Callback : public ton::adnl::AdnlExtClient::Callback { - public: - void on_ready() override { - td::actor::send_closure(id_, &TestNode::conn_ready); - } - void on_stop_ready() override { - td::actor::send_closure(id_, &TestNode::conn_closed); - } - Callback(td::actor::ActorId id) : id_(std::move(id)) { +int TestNode::LiteServer::max_common_prefix(ton::ShardIdFull shard) const { + if (shard.is_masterchain()) { + return 0; + } + if (is_full) { + return shard.pfx_len(); + } + int res = -1; + for (const ton::ShardIdFull &our_shard : shards) { + if (shard.workchain == our_shard.workchain) { + int x = std::min({shard.pfx_len(), our_shard.pfx_len(), ton::count_matching_bits(shard.shard, our_shard.shard)}); + res = std::max(res, x); } + } + return res; +} - private: - td::actor::ActorId id_; - }; - return std::make_unique(actor_id(this)); +bool TestNode::LiteServer::supports_shard(ton::ShardIdFull shard) const { + return is_full || shard.is_masterchain() || + std::any_of(shards.begin(), shards.end(), + [&](const ton::ShardIdFull& our_shard) { return ton::shard_is_ancestor(shard, our_shard); }); } void TestNode::run() { @@ -110,31 +110,58 @@ void TestNode::run() { io_ = td::TerminalIO::create("> ", readline_enabled_, ex_mode_, std::make_unique(actor_id(this))); td::actor::send_closure(io_, &td::TerminalIO::set_log_interface); - if (remote_public_key_.empty()) { - auto G = td::read_file(global_config_).move_as_ok(); - auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); - ton::ton_api::liteclient_config_global gc; - ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); - CHECK(gc.liteservers_.size() > 0); - auto idx = liteserver_idx_ >= 0 ? liteserver_idx_ - : td::Random::fast(0, static_cast(gc.liteservers_.size() - 1)); - CHECK(idx >= 0 && static_cast(idx) <= gc.liteservers_.size()); - auto& cli = gc.liteservers_[idx]; - remote_addr_.init_host_port(td::IPAddress::ipv4_to_str(cli->ip_), cli->port_).ensure(); - remote_public_key_ = ton::PublicKey{cli->id_}; - td::TerminalIO::out() << "using liteserver " << idx << " with addr " << remote_addr_ << "\n"; - if (gc.validator_ && gc.validator_->zero_state_) { - zstate_id_.workchain = gc.validator_->zero_state_->workchain_; - if (zstate_id_.workchain != ton::workchainInvalid) { - zstate_id_.root_hash = gc.validator_->zero_state_->root_hash_; - zstate_id_.file_hash = gc.validator_->zero_state_->file_hash_; - td::TerminalIO::out() << "zerostate set to " << zstate_id_.to_str() << "\n"; - } + if (!single_remote_public_key_.empty()) { // Use single provided liteserver + LiteServer s; + s.addr = single_remote_addr_; + s.public_key = single_remote_public_key_; + single_liteserver_idx_ = 0; + servers_.push_back(std::move(s)); + run_init_queries(); + return; + } + + auto G = td::read_file(global_config_).move_as_ok(); + auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); + ton::ton_api::liteclient_config_global gc; + ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); + CHECK(gc.liteservers_.size() > 0); + + if (gc.validator_ && gc.validator_->zero_state_) { + zstate_id_.workchain = gc.validator_->zero_state_->workchain_; + if (zstate_id_.workchain != ton::workchainInvalid) { + zstate_id_.root_hash = gc.validator_->zero_state_->root_hash_; + zstate_id_.file_hash = gc.validator_->zero_state_->file_hash_; + td::TerminalIO::out() << "zerostate set to " << zstate_id_.to_str() << "\n"; } } - client_ = - ton::adnl::AdnlExtClient::create(ton::adnl::AdnlNodeIdFull{remote_public_key_}, remote_addr_, make_callback()); + for (auto& server : gc.liteservers_) { + LiteServer s; + ton::ton_api::downcast_call(*server, + td::overloaded( + [&](ton::ton_api::liteserver_desc& obj) { + s.addr.init_host_port(td::IPAddress::ipv4_to_str(obj.ip_), obj.port_).ensure(); + s.public_key = ton::PublicKey{obj.id_}; + }, + [&](ton::ton_api::liteserver_descV2& obj) { + s.addr.init_host_port(td::IPAddress::ipv4_to_str(obj.ip_), obj.port_).ensure(); + s.public_key = ton::PublicKey{obj.id_}; + s.is_full = false; + for (const auto& shard : obj.shards_) { + s.shards.emplace_back(shard->workchain_, shard->shard_); + CHECK(s.shards.back().is_valid_ext()); + } + })); + servers_.push_back(std::move(s)); + } + + if (single_liteserver_idx_ != -1) { // Use single liteserver from config + CHECK(single_liteserver_idx_ >= 0 && (size_t)single_liteserver_idx_ < gc.liteservers_.size()); + td::TerminalIO::out() << "using liteserver " << single_liteserver_idx_ << " with addr " + << servers_[single_liteserver_idx_].addr << "\n"; + } + + run_init_queries(); } void TestNode::got_result(td::Result R, td::Promise promise) { @@ -179,23 +206,156 @@ void TestNode::after_got_result(bool ok) { } } -bool TestNode::envelope_send_query(td::BufferSlice query, td::Promise promise) { - running_queries_++; - if (!ready_ || client_.empty()) { - got_result(td::Status::Error("failed to send query to server: not ready"), std::move(promise)); +bool TestNode::envelope_send_query_to_any(td::BufferSlice query, td::Promise promise) { + return envelope_send_query_to_shard(ton::ShardIdFull(ton::masterchainId), std::move(query), std::move(promise)); +} + +bool TestNode::envelope_send_query_to_account(ton::AccountIdPrefixFull prefix, td::BufferSlice query, + td::Promise promise) { + if (single_liteserver_idx_ >= 0) { + return envelope_send_query_to_server(single_liteserver_idx_, std::move(query), std::move(promise)); + } + // TODO: maybe use current shard configuration? + int max_prefix_len = -1; + for (const LiteServer &server : servers_) { + max_prefix_len = std::max(max_prefix_len, server.max_common_prefix(prefix.as_leaf_shard())); + } + max_prefix_len = std::min(max_prefix_len, ton::max_shard_pfx_len); + if (max_prefix_len == -1) { + running_queries_++; + got_result(td::Status::Error("failed to select a suitable server"), std::move(promise)); + return false; + } + ton::ShardIdFull shard = shard_prefix(prefix.as_leaf_shard(), max_prefix_len); + return envelope_send_query_to_shard(shard, std::move(query), std::move(promise)); +} + +bool TestNode::envelope_send_query_to_shard(ton::ShardIdFull shard, td::BufferSlice query, + td::Promise promise) { + if (single_liteserver_idx_ >= 0) { + return envelope_send_query_to_server(single_liteserver_idx_, std::move(query), std::move(promise)); + } + if (shard.is_masterchain() && mc_server_idx_ != -1) { + return envelope_send_query_to_server(mc_server_idx_, std::move(query), std::move(promise)); + } + auto it = shard_server_idx_cached_.find(shard); + if (it != shard_server_idx_cached_.end()) { + return envelope_send_query_to_server(it->second, std::move(query), std::move(promise)); + } + int server_idx = -1; + int random_idx = -1; + int cnt = 0; + bool selected_full = false; + for (int i = 0; i < (int)servers_.size(); ++i) { + const LiteServer &server = servers_[i]; + if (!server.supports_shard(shard)) { + continue; + } + if (server.is_full && !selected_full) { + selected_full = true; + server_idx = -1; + cnt = 0; + } + if (!server.is_full && selected_full) { + continue; + } + if (!server.client.empty()) { + server_idx = i; + } + if (td::Random::fast(0, cnt) == 0) { + random_idx = i; + } + ++cnt; + } + if (server_idx == -1) { + server_idx = random_idx; + } + if (server_idx == -1) { + running_queries_++; + got_result(td::Status::Error("failed to select a suitable server"), std::move(promise)); return false; } + shard_server_idx_cached_[shard] = server_idx; + if (shard.is_masterchain()) { + mc_server_idx_ = server_idx; + } + return envelope_send_query_to_server(server_idx, std::move(query), std::move(promise)); +} + +bool TestNode::envelope_send_query_to_server(td::int32 server_idx, td::BufferSlice query, + td::Promise promise) { + running_queries_++; + LiteServer &server = servers_.at(server_idx); + if (server.client.empty()) { + start_client(server_idx); + } + CHECK(!server.client.empty()); + auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { td::actor::send_closure(SelfId, &TestNode::got_result, std::move(R), std::move(promise)); }); td::BufferSlice b = ton::serialize_tl_object(ton::create_tl_object(std::move(query)), true); - td::actor::send_closure(client_, &ton::adnl::AdnlExtClient::send_query, "query", std::move(b), - td::Timestamp::in(10.0), std::move(P)); + if (server.client_ready) { + td::actor::send_closure(server.client, &ton::adnl::AdnlExtClient::send_query, "query", std::move(b), + td::Timestamp::in(10.0), std::move(P)); + } else { + server.wait_client_ready.push_back( + [client = server.client.get(), b = std::move(b), P = std::move(P)](td::Result R) mutable { + if (R.is_ok()) { + td::actor::send_closure(client, &ton::adnl::AdnlExtClient::send_query, "query", std::move(b), + td::Timestamp::in(10.0), std::move(P)); + } else { + P.set_error(R.move_as_error_prefix("failed to connect: ")); + } + }); + } return true; } +void TestNode::start_client(int server_idx) { + LiteServer &server = servers_[server_idx]; + CHECK(server.client.empty()); + class Callback : public ton::adnl::AdnlExtClient::Callback { + public: + void on_ready() override { + td::actor::send_closure(id_, &TestNode::conn_ready, server_idx_); + } + void on_stop_ready() override { + td::actor::send_closure(id_, &TestNode::conn_closed, server_idx_); + } + Callback(td::actor::ActorId id, int server_idx) : id_(std::move(id)), server_idx_(server_idx) { + } + + private: + td::actor::ActorId id_; + int server_idx_; + }; + server.client_ready = false; + server.wait_client_ready.clear(); + LOG(INFO) << "Connecting to " << server.addr << " (liteserver #" << server_idx << ")"; + server.client = ton::adnl::AdnlExtClient::create(ton::adnl::AdnlNodeIdFull{server.public_key}, server.addr, + std::make_unique(actor_id(this), server_idx)); +} + +void TestNode::conn_ready(int server_idx) { + LiteServer &server = servers_[server_idx]; + LOG(INFO) << "Connection to " << server.addr << " (liteserver #" << server_idx << ") is ready"; + server.client_ready = true; + for (auto &p : server.wait_client_ready) { + p.set_result(td::Unit()); + } + server.wait_client_ready.clear(); +} + +void TestNode::conn_closed(int server_idx) { + LiteServer &server = servers_[server_idx]; + LOG(INFO) << "Connection to " << server.addr << " (liteserver #" << server_idx << ") closed"; + server.client_ready = false; + server.wait_client_ready.clear(); +} + td::Promise TestNode::trivial_promise() { return td::PromiseCreator::lambda([Self = actor_id(this)](td::Result res) { if (res.is_error()) { @@ -310,7 +470,7 @@ bool TestNode::dump_cached_cell(td::Slice hash_pfx, td::Slice type_name) { bool TestNode::get_server_time() { auto b = ton::serialize_tl_object(ton::create_tl_object(), true); - return envelope_send_query(std::move(b), [&, Self = actor_id(this)](td::Result res) -> void { + return envelope_send_query_to_any(std::move(b), [&, Self = actor_id(this)](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "cannot get server time"; return; @@ -319,9 +479,10 @@ bool TestNode::get_server_time() { if (F.is_error()) { LOG(ERROR) << "cannot parse answer to liteServer.getTime"; } else { - server_time_ = F.move_as_ok()->now_; - server_time_got_at_ = now(); - LOG(INFO) << "server time is " << server_time_ << " (delta " << server_time_ - server_time_got_at_ << ")"; + mc_server_time_ = F.move_as_ok()->now_; + mc_server_time_got_at_ = now(); + LOG(INFO) << "server time is " << mc_server_time_ << " (delta " << mc_server_time_ - mc_server_time_got_at_ + << ")"; } } }); @@ -329,13 +490,13 @@ bool TestNode::get_server_time() { bool TestNode::get_server_version(int mode) { auto b = ton::serialize_tl_object(ton::create_tl_object(), true); - return envelope_send_query(std::move(b), [Self = actor_id(this), mode](td::Result res) { + return envelope_send_query_to_any(std::move(b), [Self = actor_id(this), mode](td::Result res) { td::actor::send_closure_later(Self, &TestNode::got_server_version, std::move(res), mode); }); }; void TestNode::got_server_version(td::Result res, int mode) { - server_ok_ = false; + mc_server_ok_ = false; if (res.is_error()) { LOG(ERROR) << "cannot get server version and time (server too old?)"; } else { @@ -344,11 +505,11 @@ void TestNode::got_server_version(td::Result res, int mode) { LOG(ERROR) << "cannot parse answer to liteServer.getVersion"; } else { auto a = F.move_as_ok(); - set_server_version(a->version_, a->capabilities_); - set_server_time(a->now_); + set_mc_server_version(a->version_, a->capabilities_); + set_mc_server_time(a->now_); } } - if (!server_ok_) { + if (!mc_server_ok_) { LOG(ERROR) << "server version is too old (at least " << (min_ls_version >> 8) << "." << (min_ls_version & 0xff) << " with capabilities " << min_ls_capabilities << " required), some queries are unavailable"; } @@ -357,27 +518,27 @@ void TestNode::got_server_version(td::Result res, int mode) { } } -void TestNode::set_server_version(td::int32 version, td::int64 capabilities) { - if (server_version_ != version || server_capabilities_ != capabilities) { - server_version_ = version; - server_capabilities_ = capabilities; - LOG(WARNING) << "server version is " << (server_version_ >> 8) << "." << (server_version_ & 0xff) - << ", capabilities " << server_capabilities_; +void TestNode::set_mc_server_version(td::int32 version, td::int64 capabilities) { + if (mc_server_version_ != version || mc_server_capabilities_ != capabilities) { + mc_server_version_ = version; + mc_server_capabilities_ = capabilities; + LOG(WARNING) << "server version is " << (mc_server_version_ >> 8) << "." << (mc_server_version_ & 0xff) + << ", capabilities " << mc_server_capabilities_; } - server_ok_ = (server_version_ >= min_ls_version) && !(~server_capabilities_ & min_ls_capabilities); + mc_server_ok_ = (mc_server_version_ >= min_ls_version) && !(~mc_server_capabilities_ & min_ls_capabilities); } -void TestNode::set_server_time(int server_utime) { - server_time_ = server_utime; - server_time_got_at_ = now(); - LOG(INFO) << "server time is " << server_time_ << " (delta " << server_time_ - server_time_got_at_ << ")"; +void TestNode::set_mc_server_time(int server_utime) { + mc_server_time_ = server_utime; + mc_server_time_got_at_ = now(); + LOG(INFO) << "server time is " << mc_server_time_ << " (delta " << mc_server_time_ - mc_server_time_got_at_ << ")"; } bool TestNode::get_server_mc_block_id() { - int mode = (server_capabilities_ & 2) ? 0 : -1; + int mode = (mc_server_capabilities_ & 2) ? 0 : -1; if (mode < 0) { auto b = ton::serialize_tl_object(ton::create_tl_object(), true); - return envelope_send_query(std::move(b), [Self = actor_id(this)](td::Result res) -> void { + return envelope_send_query_to_any(std::move(b), [Self = actor_id(this)](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "cannot get masterchain info from server"; return; @@ -397,24 +558,25 @@ bool TestNode::get_server_mc_block_id() { } else { auto b = ton::serialize_tl_object(ton::create_tl_object(mode), true); - return envelope_send_query(std::move(b), [Self = actor_id(this), mode](td::Result res) -> void { - if (res.is_error()) { - LOG(ERROR) << "cannot get extended masterchain info from server"; - return; - } else { - auto F = ton::fetch_tl_object(res.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getMasterchainInfoExt"; - } else { - auto f = F.move_as_ok(); - auto blk_id = create_block_id(f->last_); - auto zstate_id = create_zero_state_id(f->init_); - LOG(INFO) << "last masterchain block is " << blk_id.to_str(); - td::actor::send_closure_later(Self, &TestNode::got_server_mc_block_id_ext, blk_id, zstate_id, mode, - f->version_, f->capabilities_, f->last_utime_, f->now_); - } - } - }); + return envelope_send_query_to_any( + std::move(b), [Self = actor_id(this), mode](td::Result res) -> void { + if (res.is_error()) { + LOG(ERROR) << "cannot get extended masterchain info from server"; + return; + } else { + auto F = ton::fetch_tl_object(res.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getMasterchainInfoExt"; + } else { + auto f = F.move_as_ok(); + auto blk_id = create_block_id(f->last_); + auto zstate_id = create_zero_state_id(f->init_); + LOG(INFO) << "last masterchain block is " << blk_id.to_str(); + td::actor::send_closure_later(Self, &TestNode::got_server_mc_block_id_ext, blk_id, zstate_id, mode, + f->version_, f->capabilities_, f->last_utime_, f->now_); + } + } + }); } } @@ -448,8 +610,8 @@ void TestNode::got_server_mc_block_id(ton::BlockIdExt blkid, ton::ZeroStateIdExt void TestNode::got_server_mc_block_id_ext(ton::BlockIdExt blkid, ton::ZeroStateIdExt zstateid, int mode, int version, long long capabilities, int last_utime, int server_now) { - set_server_version(version, capabilities); - set_server_time(server_now); + set_mc_server_version(version, capabilities); + set_mc_server_time(server_now); if (last_utime > server_now) { LOG(WARNING) << "server claims to have a masterchain block " << blkid.to_str() << " created at " << last_utime << " (" << last_utime - server_now << " seconds in the future)"; @@ -457,10 +619,10 @@ void TestNode::got_server_mc_block_id_ext(ton::BlockIdExt blkid, ton::ZeroStateI LOG(WARNING) << "server appears to be out of sync: its newest masterchain block is " << blkid.to_str() << " created at " << last_utime << " (" << server_now - last_utime << " seconds ago according to the server's clock)"; - } else if (last_utime < server_time_got_at_ - 60) { + } else if (last_utime < mc_server_time_got_at_ - 60) { LOG(WARNING) << "either the server is out of sync, or the local clock is set incorrectly: the newest masterchain " "block known to server is " - << blkid.to_str() << " created at " << last_utime << " (" << server_now - server_time_got_at_ + << blkid.to_str() << " created at " << last_utime << " (" << server_now - mc_server_time_got_at_ << " seconds ago according to the local clock)"; } got_server_mc_block_id(blkid, zstateid, last_utime); @@ -469,52 +631,54 @@ void TestNode::got_server_mc_block_id_ext(ton::BlockIdExt blkid, ton::ZeroStateI bool TestNode::request_block(ton::BlockIdExt blkid) { auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); - return envelope_send_query(std::move(b), [Self = actor_id(this), blkid](td::Result res) -> void { - if (res.is_error()) { - LOG(ERROR) << "cannot obtain block " << blkid.to_str() << " from server"; - return; - } else { - auto F = ton::fetch_tl_object(res.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getBlock"; - } else { - auto f = F.move_as_ok(); - auto blk_id = ton::create_block_id(f->id_); - LOG(INFO) << "obtained block " << blk_id.to_str() << " from server"; - if (blk_id != blkid) { - LOG(ERROR) << "block id mismatch: expected data for block " << blkid.to_str() << ", obtained for " - << blk_id.to_str(); + return envelope_send_query_to_shard( + blkid.shard_full(), std::move(b), [Self = actor_id(this), blkid](td::Result res) -> void { + if (res.is_error()) { + LOG(ERROR) << "cannot obtain block " << blkid.to_str() << " from server"; + return; + } else { + auto F = ton::fetch_tl_object(res.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getBlock"; + } else { + auto f = F.move_as_ok(); + auto blk_id = ton::create_block_id(f->id_); + LOG(INFO) << "obtained block " << blk_id.to_str() << " from server"; + if (blk_id != blkid) { + LOG(ERROR) << "block id mismatch: expected data for block " << blkid.to_str() << ", obtained for " + << blk_id.to_str(); + } + td::actor::send_closure_later(Self, &TestNode::got_mc_block, blk_id, std::move(f->data_)); + } } - td::actor::send_closure_later(Self, &TestNode::got_mc_block, blk_id, std::move(f->data_)); - } - } - }); + }); } bool TestNode::request_state(ton::BlockIdExt blkid) { auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); - return envelope_send_query(std::move(b), [Self = actor_id(this), blkid](td::Result res) -> void { - if (res.is_error()) { - LOG(ERROR) << "cannot obtain state " << blkid.to_str() << " from server"; - return; - } else { - auto F = ton::fetch_tl_object(res.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getState"; - } else { - auto f = F.move_as_ok(); - auto blk_id = ton::create_block_id(f->id_); - LOG(INFO) << "obtained state " << blk_id.to_str() << " from server"; - if (blk_id != blkid) { - LOG(ERROR) << "block id mismatch: expected state for block " << blkid.to_str() << ", obtained for " - << blk_id.to_str(); + return envelope_send_query_to_shard( + blkid.shard_full(), std::move(b), [Self = actor_id(this), blkid](td::Result res) -> void { + if (res.is_error()) { + LOG(ERROR) << "cannot obtain state " << blkid.to_str() << " from server"; + return; + } else { + auto F = ton::fetch_tl_object(res.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getState"; + } else { + auto f = F.move_as_ok(); + auto blk_id = ton::create_block_id(f->id_); + LOG(INFO) << "obtained state " << blk_id.to_str() << " from server"; + if (blk_id != blkid) { + LOG(ERROR) << "block id mismatch: expected state for block " << blkid.to_str() << ", obtained for " + << blk_id.to_str(); + } + td::actor::send_closure_later(Self, &TestNode::got_mc_state, blk_id, f->root_hash_, f->file_hash_, + std::move(f->data_)); + } } - td::actor::send_closure_later(Self, &TestNode::got_mc_state, blk_id, f->root_hash_, f->file_hash_, - std::move(f->data_)); - } - } - }); + }); } void TestNode::got_mc_block(ton::BlockIdExt blkid, td::BufferSlice data) { @@ -1149,27 +1313,34 @@ td::Status TestNode::send_ext_msg_from_filename(std::string filename) { LOG(ERROR) << "failed to read file `" << filename << "`: " << err.to_string(); return err; } - if (ready_ && !client_.empty()) { - LOG(ERROR) << "sending query from file " << filename; - auto P = td::PromiseCreator::lambda([](td::Result R) { - if (R.is_error()) { - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.sendMessage"; - } else { - int status = F.move_as_ok()->status_; - LOG(INFO) << "external message status is " << status; - } - }); - auto b = - ton::serialize_tl_object(ton::create_tl_object(F.move_as_ok()), true); - return envelope_send_query(std::move(b), std::move(P)) ? td::Status::OK() - : td::Status::Error("cannot send query to server"); - } else { - return td::Status::Error("server connection not ready"); + LOG(ERROR) << "sending query from file " << filename; + + TRY_RESULT_PREFIX(root, vm::std_boc_deserialize(F.ok().as_slice()), "invalid boc: "); + block::gen::CommonMsgInfo::Record_ext_in_msg_info info; + if (!tlb::unpack_cell_inexact(root, info)) { + return td::Status::Error("failed to unpack external message header"); + } + auto dest_prefix = block::tlb::t_MsgAddressInt.get_prefix(info.dest); + if (!dest_prefix.is_valid()) { + return td::Status::Error("destination of the message is invalid"); } + + auto P = td::PromiseCreator::lambda([](td::Result R) { + if (R.is_error()) { + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.sendMessage"; + } else { + int status = F.move_as_ok()->status_; + LOG(INFO) << "external message status is " << status; + } + }); + auto b = + ton::serialize_tl_object(ton::create_tl_object(F.move_as_ok()), true); + return envelope_send_query_to_account(dest_prefix, std::move(b), std::move(P)) ? td::Status::OK() + : td::Status::Error("cannot send query to server"); } bool TestNode::get_account_state(ton::WorkchainId workchain, ton::StdSmcAddress addr, ton::BlockIdExt ref_blkid, @@ -1177,9 +1348,6 @@ bool TestNode::get_account_state(ton::WorkchainId workchain, ton::StdSmcAddress if (!ref_blkid.is_valid()) { return set_error("must obtain last block information before making other queries"); } - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } if (addr_ext) { return get_special_smc_addr(addr_ext, [this, ref_blkid, filename, mode](td::Result res) { if (res.is_error()) { @@ -1193,10 +1361,12 @@ bool TestNode::get_account_state(ton::WorkchainId workchain, ton::StdSmcAddress auto b = ton::serialize_tl_object(ton::create_tl_object( ton::create_tl_lite_block_id(ref_blkid), std::move(a)), true); + ton::AccountIdPrefixFull account_prefix(workchain, addr.bits().get_uint(64)); LOG(INFO) << "requesting account state for " << workchain << ":" << addr.to_hex() << " with respect to " << ref_blkid.to_str() << " with savefile `" << filename << "` and mode " << mode; - return envelope_send_query( - std::move(b), [Self = actor_id(this), workchain, addr, ref_blkid, filename, mode](td::Result R) { + return envelope_send_query_to_account( + account_prefix, std::move(b), + [Self = actor_id(this), workchain, addr, ref_blkid, filename, mode](td::Result R) { if (R.is_error()) { return; } @@ -1275,19 +1445,18 @@ bool TestNode::start_run_method(ton::WorkchainId workchain, ton::StdSmcAddress a if (!ref_blkid.is_valid()) { return set_error("must obtain last block information before making other queries"); } - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } auto a = ton::create_tl_object(workchain, addr); + ton::AccountIdPrefixFull account_prefix(workchain, addr.bits().get_uint(64)); if (!mode) { auto b = ton::serialize_tl_object(ton::create_tl_object( ton::create_tl_lite_block_id(ref_blkid), std::move(a)), true); LOG(INFO) << "requesting account state for " << workchain << ":" << addr.to_hex() << " with respect to " << ref_blkid.to_str() << " to run method " << method_name << " with " << params.size() << " parameters"; - return envelope_send_query( - std::move(b), [Self = actor_id(this), workchain, addr, ref_blkid, method_name, params = std::move(params), - promise = std::move(promise)](td::Result R) mutable { + return envelope_send_query_to_account( + account_prefix, std::move(b), + [Self = actor_id(this), workchain, addr, ref_blkid, method_name, params = std::move(params), + promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); return; @@ -1327,26 +1496,27 @@ bool TestNode::start_run_method(ton::WorkchainId workchain, ton::StdSmcAddress a LOG(INFO) << "requesting remote get-method execution for " << workchain << ":" << addr.to_hex() << " with respect to " << ref_blkid.to_str() << " to run method " << method_name << " with " << params.size() << " parameters"; - return envelope_send_query(std::move(b), [Self = actor_id(this), workchain, addr, ref_blkid, method_name, mode, - params = std::move(params), - promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.runSmcMethod"; - promise.set_error(td::Status::Error("cannot parse answer to liteServer.runSmcMethod")); - } else { - auto f = F.move_as_ok(); - td::actor::send_closure_later(Self, &TestNode::run_smc_method, mode, ref_blkid, ton::create_block_id(f->id_), - ton::create_block_id(f->shardblk_), std::move(f->shard_proof_), - std::move(f->proof_), std::move(f->state_proof_), workchain, addr, method_name, - std::move(params), std::move(f->init_c7_), std::move(f->lib_extras_), - std::move(f->result_), f->exit_code_, std::move(promise)); - } - }); + return envelope_send_query_to_account( + account_prefix, std::move(b), + [Self = actor_id(this), workchain, addr, ref_blkid, method_name, mode, params = std::move(params), + promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.runSmcMethod"; + promise.set_error(td::Status::Error("cannot parse answer to liteServer.runSmcMethod")); + } else { + auto f = F.move_as_ok(); + td::actor::send_closure_later( + Self, &TestNode::run_smc_method, mode, ref_blkid, ton::create_block_id(f->id_), + ton::create_block_id(f->shardblk_), std::move(f->shard_proof_), std::move(f->proof_), + std::move(f->state_proof_), workchain, addr, method_name, std::move(params), std::move(f->init_c7_), + std::move(f->lib_extras_), std::move(f->result_), f->exit_code_, std::move(promise)); + } + }); } } @@ -1577,7 +1747,7 @@ void TestNode::send_compute_complaint_price_query(ton::StdSmcAddress elector_add params.emplace_back(td::make_refint(refs)); params.emplace_back(td::make_refint(expires_in)); auto P = td::PromiseCreator::lambda( - [this, expires_in, bits, refs, chash, filename](td::Result> R) { + [expires_in, bits, refs, chash, filename](td::Result> R) { if (R.is_error()) { LOG(ERROR) << R.move_as_error(); return; @@ -1639,10 +1809,6 @@ bool TestNode::dns_resolve_start(ton::WorkchainId workchain, ton::StdSmcAddress return set_error("domain name too long"); } - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } - if (workchain == ton::workchainInvalid) { if (dns_root_queried_) { workchain = ton::masterchainId; @@ -1843,17 +2009,16 @@ bool TestNode::get_one_transaction(ton::BlockIdExt blkid, ton::WorkchainId workc if (!ton::shard_contains(blkid.shard_full(), ton::extract_addr_prefix(workchain, addr))) { return set_error("the shard of this block cannot contain this account"); } - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } auto a = ton::create_tl_object(workchain, addr); auto b = ton::serialize_tl_object(ton::create_tl_object( ton::create_tl_lite_block_id(blkid), std::move(a), lt), true); + ton::AccountIdPrefixFull account_prefix(workchain, addr.bits().get_uint(64)); LOG(INFO) << "requesting transaction " << lt << " of " << workchain << ":" << addr.to_hex() << " from block " << blkid.to_str(); - return envelope_send_query( - std::move(b), [Self = actor_id(this), workchain, addr, lt, blkid, dump](td::Result R) -> void { + return envelope_send_query_to_account( + account_prefix, std::move(b), + [Self = actor_id(this), workchain, addr, lt, blkid, dump](td::Result R) -> void { if (R.is_error()) { return; } @@ -1870,16 +2035,15 @@ bool TestNode::get_one_transaction(ton::BlockIdExt blkid, ton::WorkchainId workc bool TestNode::get_last_transactions(ton::WorkchainId workchain, ton::StdSmcAddress addr, ton::LogicalTime lt, ton::Bits256 hash, unsigned count, bool dump) { - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } auto a = ton::create_tl_object(workchain, addr); auto b = ton::serialize_tl_object( ton::create_tl_object(count, std::move(a), lt, hash), true); + ton::AccountIdPrefixFull account_prefix(workchain, addr.bits().get_uint(64)); LOG(INFO) << "requesting " << count << " last transactions from " << lt << ":" << hash.to_hex() << " of " << workchain << ":" << addr.to_hex(); - return envelope_send_query( - std::move(b), [Self = actor_id(this), workchain, addr, lt, hash, count, dump](td::Result R) { + return envelope_send_query_to_account( + account_prefix, std::move(b), + [Self = actor_id(this), workchain, addr, lt, hash, count, dump](td::Result R) { if (R.is_error()) { return; } @@ -2236,10 +2400,10 @@ void TestNode::got_one_transaction(ton::BlockIdExt req_blkid, ton::BlockIdExt bl << " but received data has " << root->get_hash().bits().to_hex(256); return; } - } catch (vm::VmError err) { + } catch (vm::VmError &err) { LOG(ERROR) << "error while traversing block transaction proof : " << err.get_msg(); return; - } catch (vm::VmVirtError err) { + } catch (vm::VmVirtError &err) { LOG(ERROR) << "virtualization error while traversing block transaction proof : " << err.get_msg(); return; } @@ -2418,32 +2582,30 @@ void TestNode::got_last_transactions(std::vector blkids, td::Bu bool TestNode::get_block_transactions(ton::BlockIdExt blkid, int mode, unsigned count, ton::Bits256 acc_addr, ton::LogicalTime lt) { - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } auto a = ton::create_tl_object(acc_addr, lt); auto b = ton::serialize_tl_object(ton::create_tl_object( ton::create_tl_lite_block_id(blkid), mode, count, std::move(a), false, false), true); LOG(INFO) << "requesting " << count << " transactions from block " << blkid.to_str() << " starting from account " << acc_addr.to_hex() << " lt " << lt; - return envelope_send_query(std::move(b), [Self = actor_id(this), mode](td::Result R) { - if (R.is_error()) { - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.listBlockTransactions"; - } else { - auto f = F.move_as_ok(); - std::vector transactions; - for (auto& id : f->ids_) { - transactions.emplace_back(id->account_, id->lt_, id->hash_); - } - td::actor::send_closure_later(Self, &TestNode::got_block_transactions, ton::create_block_id(f->id_), mode, - f->req_count_, f->incomplete_, std::move(transactions), std::move(f->proof_)); - } - }); + return envelope_send_query_to_shard( + blkid.shard_full(), std::move(b), [Self = actor_id(this), mode](td::Result R) { + if (R.is_error()) { + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.listBlockTransactions"; + } else { + auto f = F.move_as_ok(); + std::vector transactions; + for (auto& id : f->ids_) { + transactions.emplace_back(id->account_, id->lt_, id->hash_); + } + td::actor::send_closure_later(Self, &TestNode::got_block_transactions, ton::create_block_id(f->id_), mode, + f->req_count_, f->incomplete_, std::move(transactions), std::move(f->proof_)); + } + }); } void TestNode::got_block_transactions(ton::BlockIdExt blkid, int mode, unsigned req_count, bool incomplete, @@ -2469,25 +2631,23 @@ bool TestNode::get_all_shards(std::string filename, bool use_last, ton::BlockIdE if (!blkid.is_masterchain()) { return set_error("only masterchain blocks contain shard configuration"); } - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); LOG(INFO) << "requesting recent shard configuration"; - return envelope_send_query(std::move(b), [Self = actor_id(this), filename](td::Result R) -> void { - if (R.is_error()) { - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getAllShardsInfo"; - } else { - auto f = F.move_as_ok(); - td::actor::send_closure_later(Self, &TestNode::got_all_shards, ton::create_block_id(f->id_), std::move(f->proof_), - std::move(f->data_), filename); - } - }); + return envelope_send_query_to_any( + std::move(b), [Self = actor_id(this), filename](td::Result R) -> void { + if (R.is_error()) { + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getAllShardsInfo"; + } else { + auto f = F.move_as_ok(); + td::actor::send_closure_later(Self, &TestNode::got_all_shards, ton::create_block_id(f->id_), + std::move(f->proof_), std::move(f->data_), filename); + } + }); } void TestNode::got_all_shards(ton::BlockIdExt blk, td::BufferSlice proof, td::BufferSlice data, std::string filename) { @@ -2551,9 +2711,6 @@ bool TestNode::parse_get_config_params(ton::BlockIdExt blkid, int mode, std::str params.push_back(x); } } - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } if (!blkid.is_masterchain_ext()) { return set_error("only masterchain blocks contain configuration"); } @@ -2572,10 +2729,6 @@ bool TestNode::get_config_params(ton::BlockIdExt blkid, td::Promise promise, int mode, std::string filename, std::vector params) { - if (!(ready_ && !client_.empty())) { - promise.set_error(td::Status::Error("server connection not ready")); - return false; - } if (!blkid.is_masterchain_ext()) { promise.set_error(td::Status::Error("masterchain reference block expected")); return false; @@ -2593,11 +2746,12 @@ bool TestNode::get_config_params_ext(ton::BlockIdExt blkid, td::Promise R) mutable { - td::actor::send_closure_later(Self, &TestNode::got_config_params, blkid, mode, filename, std::move(params), - std::move(R), std::move(promise)); - }); + return envelope_send_query_to_any( + std::move(b), [Self = actor_id(this), mode, filename, blkid, params = std::move(params), + promise = std::move(promise)](td::Result R) mutable { + td::actor::send_closure_later(Self, &TestNode::got_config_params, blkid, mode, filename, std::move(params), + std::move(R), std::move(promise)); + }); } void TestNode::got_config_params(ton::BlockIdExt req_blkid, int mode, std::string filename, std::vector params, @@ -2785,8 +2939,8 @@ bool TestNode::get_block(ton::BlockIdExt blkid, bool dump) { LOG(INFO) << "got block download request for " << blkid.to_str(); auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); - return envelope_send_query( - std::move(b), [Self = actor_id(this), blkid, dump](td::Result res) -> void { + return envelope_send_query_to_shard( + blkid.shard_full(), std::move(b), [Self = actor_id(this), blkid, dump](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "cannot obtain block " << blkid.to_str() << " from server : " << res.move_as_error().to_string(); @@ -2814,8 +2968,8 @@ bool TestNode::get_state(ton::BlockIdExt blkid, bool dump) { LOG(INFO) << "got state download request for " << blkid.to_str(); auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); - return envelope_send_query( - std::move(b), [Self = actor_id(this), blkid, dump](td::Result res) -> void { + return envelope_send_query_to_shard( + blkid.shard_full(), std::move(b), [Self = actor_id(this), blkid, dump](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "cannot obtain state " << blkid.to_str() << " from server : " << res.move_as_error().to_string(); @@ -2952,7 +3106,7 @@ void TestNode::got_state(ton::BlockIdExt blkid, ton::RootHash root_hash, ton::Fi } bool TestNode::get_show_block_header(ton::BlockIdExt blkid, int mode) { - return get_block_header(blkid, mode, [this, blkid](td::Result R) { + return get_block_header(blkid, mode, [this](td::Result R) { if (R.is_error()) { LOG(ERROR) << "unable to fetch block header: " << R.move_as_error(); } else { @@ -2967,8 +3121,9 @@ bool TestNode::get_block_header(ton::BlockIdExt blkid, int mode, td::Promise(ton::create_tl_lite_block_id(blkid), mode), true); - return envelope_send_query( - std::move(b), [this, blkid, promise = std::move(promise)](td::Result R) mutable -> void { + return envelope_send_query_to_shard( + blkid.shard_full(), std::move(b), + [this, blkid, promise = std::move(promise)](td::Result R) mutable -> void { TRY_RESULT_PROMISE_PREFIX(promise, res, std::move(R), PSLICE() << "cannot obtain block header for " << blkid.to_str() << " from server :"); got_block_header_raw(std::move(res), std::move(promise), blkid); @@ -2994,8 +3149,8 @@ bool TestNode::lookup_block(ton::ShardIdFull shard, int mode, td::uint64 arg, auto b = ton::serialize_tl_object(ton::create_tl_object( mode, ton::create_tl_lite_block_id_simple(id), arg, (td::uint32)arg), true); - return envelope_send_query( - std::move(b), [this, id, mode, arg, promise = std::move(promise)](td::Result R) mutable -> void { + return envelope_send_query_to_shard( + shard, std::move(b), [this, id, mode, arg, promise = std::move(promise)](td::Result R) mutable -> void { TRY_RESULT_PROMISE_PREFIX(promise, res, std::move(R), PSLICE() << "cannot look up block header for " << id.to_str() << " with mode " << mode << " and argument " << arg << " from server :"); @@ -3109,9 +3264,9 @@ void TestNode::got_block_header(ton::BlockIdExt blkid, td::BufferSlice data, int return; } show_block_header(blkid, std::move(virt_root), mode); - } catch (vm::VmError err) { + } catch (vm::VmError &err) { LOG(ERROR) << "error processing header for " << blkid.to_str() << " : " << err.get_msg(); - } catch (vm::VmVirtError err) { + } catch (vm::VmVirtError &err) { LOG(ERROR) << "error processing header for " << blkid.to_str() << " : " << err.get_msg(); } show_new_blkids(); @@ -3140,14 +3295,15 @@ bool TestNode::get_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mod ton::serialize_tl_object(ton::create_tl_object( mode & 0xfff, ton::create_tl_lite_block_id(from), ton::create_tl_lite_block_id(to)), true); - return envelope_send_query(std::move(b), [Self = actor_id(this), from, to, mode](td::Result res) { - if (res.is_error()) { - LOG(ERROR) << "cannot obtain block proof for " << ((mode & 1) ? to.to_str() : "last masterchain block") - << " starting from " << from.to_str() << " from server : " << res.move_as_error().to_string(); - } else { - td::actor::send_closure_later(Self, &TestNode::got_block_proof, from, to, mode, res.move_as_ok()); - } - }); + return envelope_send_query_to_any( + std::move(b), [Self = actor_id(this), from, to, mode](td::Result res) { + if (res.is_error()) { + LOG(ERROR) << "cannot obtain block proof for " << ((mode & 1) ? to.to_str() : "last masterchain block") + << " starting from " << from.to_str() << " from server : " << res.move_as_error().to_string(); + } else { + td::actor::send_closure_later(Self, &TestNode::got_block_proof, from, to, mode, res.move_as_ok()); + } + }); } void TestNode::got_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mode, td::BufferSlice pchain) { @@ -3201,9 +3357,6 @@ void TestNode::got_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mod bool TestNode::get_creator_stats(ton::BlockIdExt blkid, int mode, unsigned req_count, ton::Bits256 start_after, ton::UnixTime min_utime) { - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } if (!blkid.is_masterchain_ext()) { return set_error("only masterchain blocks contain block creator statistics"); } @@ -3214,7 +3367,7 @@ bool TestNode::get_creator_stats(ton::BlockIdExt blkid, int mode, unsigned req_c auto& os = *osp; return get_creator_stats( blkid, mode, req_count, start_after, min_utime, - [min_utime, &os](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, + [&os](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, const block::DiscountedCounter& shard_cnt) -> bool { os << key.to_hex() << " mc_cnt:" << mc_cnt << " shard_cnt:" << shard_cnt << std::endl; return true; @@ -3244,10 +3397,6 @@ bool TestNode::get_creator_stats(ton::BlockIdExt blkid, int mode, unsigned req_c bool TestNode::get_creator_stats(ton::BlockIdExt blkid, unsigned req_count, ton::UnixTime min_utime, TestNode::creator_stats_func_t func, std::unique_ptr state, td::Promise> promise) { - if (!(ready_ && !client_.empty())) { - promise.set_error(td::Status::Error("server connection not ready")); - return false; - } if (!state) { promise.set_error(td::Status::Error("null CreatorStatsRes")); return false; @@ -3266,7 +3415,7 @@ bool TestNode::get_creator_stats(ton::BlockIdExt blkid, unsigned req_count, ton: LOG(INFO) << "requesting up to " << req_count << " block creator stats records with respect to masterchain block " << blkid.to_str() << " starting from validator public key " << state->last_key.to_hex() << " created after " << min_utime << " (mode=" << state->mode << ")"; - return envelope_send_query( + return envelope_send_query_to_any( std::move(b), [this, blkid, req_count, state = std::move(state), min_utime, func = std::move(func), promise = std::move(promise)](td::Result R) mutable { TRY_RESULT_PROMISE(promise, res, std::move(R)); @@ -3483,7 +3632,7 @@ bool TestNode::load_creator_stats(std::unique_ptr l ton::UnixTime min_utime = info.valid_since - 1000; return get_creator_stats( info.blk_id, 1000, min_utime, - [min_utime, &info](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, + [&info](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, const block::DiscountedCounter& shard_cnt) -> bool { info.store_record(key, mc_cnt, shard_cnt); return true; @@ -3878,7 +4027,7 @@ td::Result> TestNode::ValidatorLoadInfo::build_proof(int idx, td:: block::gen::ValidatorDescr::Record_validator_addr rec2; if (tlb::csr_unpack(entry, rec1)) { pk = std::move(rec1.public_key); - } else if (tlb::csr_unpack(std::move(entry), rec2)) { + } else if (tlb::csr_unpack(entry, rec2)) { pk = std::move(rec2.public_key); } else { return td::Status::Error("cannot unpack ValidatorDescr"); diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index e62e801ce..adfa9a04a 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -45,22 +45,41 @@ class TestNode : public td::actor::Actor { min_ls_version = 0x101, min_ls_capabilities = 1 }; // server version >= 1.1, capabilities at least +1 = build proof chains - td::actor::ActorOwn client_; td::actor::ActorOwn io_; + struct LiteServer { + td::IPAddress addr; + ton::PublicKey public_key; + bool is_full = true; + std::vector shards; + + td::actor::ActorOwn client; + bool client_ready = false; + std::vector> wait_client_ready; + + int max_common_prefix(ton::ShardIdFull shard) const; + bool supports_shard(ton::ShardIdFull shard) const; + }; + std::vector servers_; + + td::int32 single_liteserver_idx_ = -1; + td::IPAddress single_remote_addr_; + ton::PublicKey single_remote_public_key_; + + std::map shard_server_idx_cached_; + bool readline_enabled_ = true; - bool server_ok_ = false; - td::int32 liteserver_idx_ = -1; int print_limit_ = 1024; - bool ready_ = false; - bool inited_ = false; std::string db_root_; - int server_time_ = 0; - int server_time_got_at_ = 0; - int server_version_ = 0; - long long server_capabilities_ = 0; + // mc_server is the server for queries to masterchain + int mc_server_idx_ = -1; + int mc_server_time_ = 0; + int mc_server_time_got_at_ = 0; + int mc_server_version_ = 0; + long long mc_server_capabilities_ = 0; + bool mc_server_ok_ = false; ton::ZeroStateIdExt zstate_id_; ton::BlockIdExt mc_last_id_; @@ -75,9 +94,6 @@ class TestNode : public td::actor::Actor { const char *parse_ptr_, *parse_end_; td::Status error_; - td::IPAddress remote_addr_; - ton::PublicKey remote_public_key_; - std::vector known_blk_ids_; std::size_t shown_blk_ids_ = 0; @@ -88,8 +104,6 @@ class TestNode : public td::actor::Actor { std::map> cell_cache_; - std::unique_ptr make_callback(); - using creator_stats_func_t = std::function; @@ -182,8 +196,8 @@ class TestNode : public td::actor::Actor { void got_server_mc_block_id(ton::BlockIdExt blkid, ton::ZeroStateIdExt zstateid, int created_at); void got_server_mc_block_id_ext(ton::BlockIdExt blkid, ton::ZeroStateIdExt zstateid, int mode, int version, long long capabilities, int last_utime, int server_now); - void set_server_version(td::int32 version, td::int64 capabilities); - void set_server_time(int server_utime); + void set_mc_server_version(td::int32 version, td::int64 capabilities); + void set_mc_server_time(int server_utime); bool request_block(ton::BlockIdExt blkid); bool request_state(ton::BlockIdExt blkid); void got_mc_block(ton::BlockIdExt blkid, td::BufferSlice data); @@ -358,16 +372,6 @@ class TestNode : public td::actor::Actor { static const tlb::TypenameLookup& get_tlb_dict(); public: - void conn_ready() { - LOG(ERROR) << "conn ready"; - ready_ = true; - if (!inited_) { - run_init_queries(); - } - } - void conn_closed() { - ready_ = false; - } void set_global_config(std::string str) { global_config_ = str; } @@ -378,10 +382,10 @@ class TestNode : public td::actor::Actor { readline_enabled_ = value; } void set_liteserver_idx(td::int32 idx) { - liteserver_idx_ = idx; + single_liteserver_idx_ = idx; } void set_remote_addr(td::IPAddress addr) { - remote_addr_ = addr; + single_remote_addr_ = addr; } void set_public_key(td::BufferSlice file_name) { auto R = [&]() -> td::Result { @@ -392,7 +396,7 @@ class TestNode : public td::actor::Actor { if (R.is_error()) { LOG(FATAL) << "bad server public key: " << R.move_as_error(); } - remote_public_key_ = R.move_as_ok(); + single_remote_public_key_ = R.move_as_ok(); } void decode_public_key(td::BufferSlice b64_key) { auto R = [&]() -> td::Result { @@ -404,7 +408,7 @@ class TestNode : public td::actor::Actor { if (R.is_error()) { LOG(FATAL) << "bad b64 server public key: " << R.move_as_error(); } - remote_public_key_ = R.move_as_ok(); + single_remote_public_key_ = R.move_as_ok(); } void set_fail_timeout(td::Timestamp ts) { fail_timeout_ = ts; @@ -439,7 +443,18 @@ class TestNode : public td::actor::Actor { void got_result(td::Result R, td::Promise promise); void after_got_result(bool ok); - bool envelope_send_query(td::BufferSlice query, td::Promise promise); + bool envelope_send_query_to_any(td::BufferSlice query, td::Promise promise); + bool envelope_send_query_to_shard(ton::ShardIdFull shard, td::BufferSlice query, + td::Promise promise); + bool envelope_send_query_to_account(ton::AccountIdPrefixFull prefix, td::BufferSlice query, + td::Promise promise); + + bool envelope_send_query_to_server(td::int32 server_idx, td::BufferSlice query, td::Promise promise); + + void start_client(int server_idx); + void conn_ready(int server_idx); + void conn_closed(int server_idx); + void parse_line(td::BufferSlice data); TestNode() { diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index f3ac4ee91..187c4952c 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -364,6 +364,7 @@ tonNode.blockSignature who:int256 signature:bytes = tonNode.BlockSignature; tonNode.blockId workchain:int shard:long seqno:int = tonNode.BlockId; tonNode.blockIdExt workchain:int shard:long seqno:int root_hash:int256 file_hash:int256 = tonNode.BlockIdExt; tonNode.zeroStateIdExt workchain:int root_hash:int256 file_hash:int256 = tonNode.ZeroStateIdExt; +tonNode.shardId workchain:int shard:long = tonNode.ShardId; tonNode.blockDescriptionEmpty = tonNode.BlockDescription; tonNode.blockDescription id:tonNode.blockIdExt = tonNode.BlockDescription; @@ -568,7 +569,8 @@ validator.config.global zero_state:tonNode.blockIdExt init_block:tonNode.blockId config.global adnl:adnl.config.global dht:dht.config.global validator:validator.config.global = config.Global; liteserver.desc id:PublicKey ip:int port:int = liteserver.Desc; -liteclient.config.global liteservers:(vector liteserver.desc) validator:validator.config.global = liteclient.config.Global; +liteserver.descV2 id:PublicKey ip:int port:int shards:(vector tonNode.shardId) = liteserver.Desc; +liteclient.config.global liteservers:(vector liteserver.Desc) validator:validator.config.global = liteclient.config.Global; engine.adnl id:int256 category:int = engine.Adnl; engine.addr ip:int port:int categories:(vector int) priority_categories:(vector int) = engine.Addr; @@ -578,7 +580,7 @@ engine.dht id:int256 = engine.Dht; engine.validatorTempKey key:int256 expire_at:int = engine.ValidatorTempKey; engine.validatorAdnlAddress id:int256 expire_at:int = engine.ValidatorAdnlAddress; engine.validator id:int256 temp_keys:(vector engine.validatorTempKey) adnl_addrs:(vector engine.validatorAdnlAddress) election_date:int expire_at:int = engine.Validator; -engine.collator adnl_id:int256 workchain:int shard:long = engine.Validator; +engine.collator adnl_id:int256 shard:tonNode.shardId = engine.Collator; engine.liteServer id:int256 port:int = engine.LiteServer; engine.controlProcess id:int256 permissions:int = engine.ControlProcess; engine.controlInterface id:int256 port:int allowed:(vector engine.controlProcess) = engine.ControlInterface; @@ -592,7 +594,16 @@ engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector eng validators:(vector engine.Validator) fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave) fullnodemasters:(vector engine.validator.fullNodeMaster) liteservers:(vector engine.liteServer) control:(vector engine.controlInterface) - gc:engine.gc = engine.validator.Config; + gc:engine.gc = engine.validator.Config; + +engine.validator.config_v2 out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) + dht:(vector engine.dht) + validators:(vector engine.validator) collators:(vector engine.collator) + fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave) + fullnodemasters:(vector engine.validator.fullNodeMaster) + liteservers:(vector engine.liteServer) control:(vector engine.controlInterface) + shards_to_monitor:(vector tonNode.shardId) + gc:engine.gc = engine.validator.Config; ---functions--- ---types--- @@ -697,7 +708,8 @@ engine.validator.generateBlockCandidate block_id:tonNode.BlockId = db.Candidate; engine.validator.getRequiredBlockCandidates = engine.validator.RequiredBlockCandidates; engine.validator.importBlockCandidate block:db.candidate = engine.validator.Success; -engine.validator.addCollator adnl_id:int256 workchain:int shard:long = engine.validator.Success; +engine.validator.addCollator adnl_id:int256 shard:tonNode.shardId = engine.validator.Success; +engine.validator.addShard shard:tonNode.shardId = engine.validator.Success; ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 2960fa72c13d97c0b874d959604f413a38afe08c..2bb8d64c32f5762201a2aab2bffd1bed74b12bc8 100644 GIT binary patch delta 709 zcmeBK#WG_v3-6=Z`c@23aA6~_xu8gdlkGeH)V%b}yi`5s{G6P`lKi5H8zd&X2)AhCIhZ~`04qGNl{ zPyQikB4Ycceg=O|W=U#sYEfBgk)BIxaWY6%gj55ENIAzZ4gQk+Jiq*uRK4Jg#G(|> z6rjbMZzzedF#2qMtJWPRi|n{$sN+ClH;)PVFipPrTz#@khJXafBu1#Gik#VWIBeSv zf>dqJscm3_c;Z7n#5bEO8bz2{K)#qfq2&WuZgWek38N)Q2o^9`<+x&Ux(!}PVR$qd z?9uo#BL@Z^oD@~j~DFm7&b6=oC`ECzYD zIKCu5J~uxv6BvLDlh6CfvVgoXxu;DB?7Yo;+A@S8A*nG-2Ey=|Wx@&yQijQmvy34^ zJ7y`cfP^L+L`zIg@DZ4N!qPzptOn{CkY79l%whzQq9Fws4PizM3}B5HmTFA4@UUTJ z1G#Fl;!Sz1!3k2j*<*H(rYuUdfqmkc0uqBn$;apZV1H~rapZ;!W9sIG51H9mC0JI? zlbL+*rO4#H&xIM`@di<7@ZtdrOJu^O$CD4d6cqrc0|tifWluJP3}l?_m?jQU_u-9< F1^`Gk54r#V delta 367 zcmbQSnWbkH3-6=Z`c@23aBd^7x!~ji!55;BvK|@>q~@h(=B4U|CFW$NB$nhCP5!7L zy}3h(g^^{7aX`@I3BnSSA23U7J|LXH#`5DpFUMpRDHD;BC7wt4b23X(i&KlrQj7Fl zQj3!(ZjjjABGtgL*+5x@h0$}ft9p0XW{DI&bTVa6?3{_-)?@W z1CsM|azOTj)Z9EKE5RnU65|by)^O$@< zn*%JcSz>;V=4OSHXJi;tHotw!%*J~8>zNjr$%1diCUd_Lob2workchain_, B->root_hash_, B->file_hash_}; } +inline ShardIdFull create_shard_id(const tl_object_ptr &s) { + return ShardIdFull{s->workchain_, static_cast(s->shard_)}; +} + +inline tl_object_ptr create_tl_shard_id(const ShardIdFull &s) { + return create_tl_object(s.workchain, s.shard); +} + +inline tl_object_ptr unpack_engine_validator_config( + tl_object_ptr config) { + tl_object_ptr res; + ton_api::downcast_call(*config, td::overloaded( + [&](ton_api::engine_validator_config &c) { + res = create_tl_object( + c.out_port_, std::move(c.addrs_), std::move(c.adnl_), std::move(c.dht_), + std::move(c.validators_), + std::vector>(), c.fullnode_, + std::move(c.fullnodeslaves_), std::move(c.fullnodemasters_), + std::move(c.liteservers_), std::move(c.control_), + std::vector>(), std::move(c.gc_)); + }, + [&](ton_api::engine_validator_config_v2 &c) { + res = std::make_unique(std::move(c)); + })); + return res; +} + } // namespace ton diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 9c5b6572d..d1bc8d5ca 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1101,7 +1101,7 @@ td::Status AddCollatorQuery::run() { td::Status AddCollatorQuery::send() { auto b = ton::create_serialize_tl_object( - adnl_id_.tl(), wc_, shard_); + adnl_id_.tl(), ton::create_tl_shard_id(ton::ShardIdFull(wc_, shard_))); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -1112,3 +1112,23 @@ td::Status AddCollatorQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << "successfully added collator\n"; return td::Status::OK(); } + +td::Status AddShardQuery::run() { + TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); + return td::Status::OK(); +} + +td::Status AddShardQuery::send() { + auto b = ton::create_serialize_tl_object( + ton::create_tl_shard_id(ton::ShardIdFull(wc_, shard_))); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status AddShardQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "successfully added shard\n"; + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 34b2ede3e..daa19ab1a 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1187,3 +1187,26 @@ class AddCollatorQuery : public Query { td::int32 wc_; td::int64 shard_; }; + +class AddShardQuery : public Query { + public: + AddShardQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "addshard"; + } + static std::string get_help() { + return "addshard \tstart monitoring shard"; + } + std::string name() const override { + return get_name(); + } + + private: + td::int32 wc_; + td::int64 shard_; +}; diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index d70e2cf2e..83375b209 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -145,6 +145,7 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index b2617769e..fda610afe 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -54,6 +54,7 @@ #include "td/utils/Random.h" #include "auto/tl/lite_api.h" +#include "tl/tl_json.h" #include "memprof/memprof.h" @@ -74,7 +75,7 @@ Config::Config() { full_node = ton::PublicKeyHash::zero(); } -Config::Config(ton::ton_api::engine_validator_config &config) { +Config::Config(const ton::ton_api::engine_validator_config_v2 &config) { full_node = ton::PublicKeyHash::zero(); out_port = static_cast(config.out_port_); if (!out_port) { @@ -87,7 +88,7 @@ Config::Config(ton::ton_api::engine_validator_config &config) { std::vector categories; std::vector priority_categories; ton::ton_api::downcast_call( - *addr.get(), + *addr, td::overloaded( [&](const ton::ton_api::engine_addr &obj) { in_ip.init_ipv4_port(td::IPAddress::ipv4_to_str(obj.ip_), static_cast(obj.port_)).ensure(); @@ -125,24 +126,20 @@ Config::Config(ton::ton_api::engine_validator_config &config) { for (auto &dht : config.dht_) { config_add_dht_node(ton::PublicKeyHash{dht->id_}).ensure(); } - for (auto &v : config.validators_) { - ton::ton_api::downcast_call( - *v, td::overloaded( - [&](ton::ton_api::engine_validator &val) { - auto key = ton::PublicKeyHash{val.id_}; - config_add_validator_permanent_key(key, val.election_date_, val.expire_at_).ensure(); - for (auto &temp : val.temp_keys_) { - config_add_validator_temp_key(key, ton::PublicKeyHash{temp->key_}, temp->expire_at_).ensure(); - } - for (auto &adnl : val.adnl_addrs_) { - config_add_validator_adnl_id(key, ton::PublicKeyHash{adnl->id_}, adnl->expire_at_).ensure(); - } - }, - [&](ton::ton_api::engine_collator &col) { - auto key = ton::PublicKeyHash{col.adnl_id_}; - ton::ShardIdFull shard(col.workchain_, col.shard_); - config_add_collator(key, shard).ensure(); - })); + for (auto &val : config.validators_) { + auto key = ton::PublicKeyHash{val->id_}; + config_add_validator_permanent_key(key, val->election_date_, val->expire_at_).ensure(); + for (auto &temp : val->temp_keys_) { + config_add_validator_temp_key(key, ton::PublicKeyHash{temp->key_}, temp->expire_at_).ensure(); + } + for (auto &adnl : val->adnl_addrs_) { + config_add_validator_adnl_id(key, ton::PublicKeyHash{adnl->id_}, adnl->expire_at_).ensure(); + } + } + for (auto &col : config.collators_) { + auto key = ton::PublicKeyHash{col->adnl_id_}; + ton::ShardIdFull shard = ton::create_shard_id(col->shard_); + config_add_collator(key, shard).ensure(); } config_add_full_node_adnl_id(ton::PublicKeyHash{config.fullnode_}).ensure(); @@ -169,6 +166,10 @@ Config::Config(ton::ton_api::engine_validator_config &config) { } } + for (auto &shard : config.shards_to_monitor_) { + config_add_shard(ton::create_shard_id(shard)).ensure(); + } + if (config.gc_) { for (auto &gc : config.gc_->ids_) { config_add_gc(ton::PublicKeyHash{gc}).ensure(); @@ -176,7 +177,7 @@ Config::Config(ton::ton_api::engine_validator_config &config) { } } -ton::tl_object_ptr Config::tl() const { +ton::tl_object_ptr Config::tl() const { std::vector> addrs_vec; for (auto &x : addrs) { if (x.second.proxy) { @@ -201,7 +202,7 @@ ton::tl_object_ptr Config::tl() const { dht_vec.push_back(ton::create_tl_object(x.tl())); } - std::vector> val_vec; + std::vector> val_vec; for (auto &val : validators) { std::vector> temp_vec; for (auto &t : val.second.temp_keys) { @@ -214,9 +215,10 @@ ton::tl_object_ptr Config::tl() const { val_vec.push_back(ton::create_tl_object( val.first.tl(), std::move(temp_vec), std::move(adnl_val_vec), val.second.election_date, val.second.expire_at)); } + std::vector> col_vec; for (auto &col : collators) { - val_vec.push_back(ton::create_tl_object( - col.adnl_id.tl(), col.shard.workchain, col.shard.shard)); + col_vec.push_back( + ton::create_tl_object(col.adnl_id.tl(), ton::create_tl_shard_id(col.shard))); } std::vector> full_node_slaves_vec; @@ -245,14 +247,19 @@ ton::tl_object_ptr Config::tl() const { std::move(control_proc_vec))); } + std::vector> shards_vec; + for (auto &shard : shards_to_monitor) { + shards_vec.push_back(ton::create_tl_shard_id(shard)); + } + auto gc_vec = ton::create_tl_object(std::vector{}); for (auto &id : gc) { gc_vec->ids_.push_back(id.tl()); } - return ton::create_tl_object( - out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), full_node.tl(), - std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(liteserver_vec), - std::move(control_vec), std::move(gc_vec)); + return ton::create_tl_object( + out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec), + full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(liteserver_vec), + std::move(control_vec), std::move(shards_vec), std::move(gc_vec)); } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, @@ -527,6 +534,17 @@ td::Result Config::config_add_control_process(ton::PublicKeyHash key, td:: } } +td::Result Config::config_add_shard(ton::ShardIdFull shard) { + if (!shard.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard " << shard.to_str()); + } + if (std::find(shards_to_monitor.begin(), shards_to_monitor.end(), shard) != shards_to_monitor.end()) { + return false; + } + shards_to_monitor.push_back(shard); + return true; +} + td::Result Config::config_add_gc(ton::PublicKeyHash key) { return gc.insert(key).second; } @@ -1393,6 +1411,11 @@ void ValidatorEngine::init_validator_options() { for (const auto& c : config_.collators) { shards.push_back(c.shard); } + for (const auto& s : config_.shards_to_monitor) { + shards.push_back(s); + } + std::sort(shards.begin(), shards.end()); + shards.erase(std::unique(shards.begin(), shards.end()), shards.end()); validator_options_.write().set_shard_check_function( [shards = std::move(shards)](ton::ShardIdFull shard, ton::CatchainSeqno cc_seqno, ton::validator::ValidatorManagerOptions::ShardCheckMode mode) -> bool { @@ -1643,14 +1666,14 @@ void ValidatorEngine::load_config(td::Promise promise) { } auto conf_json = conf_json_R.move_as_ok(); - ton::ton_api::engine_validator_config conf; - auto S = ton::ton_api::from_json(conf, conf_json.get_object()); + ton::tl_object_ptr conf; + auto S = td::from_json(conf, std::move(conf_json)); if (S.is_error()) { promise.set_error(S.move_as_error_prefix("json does not fit TL scheme")); return; } - config_ = Config{conf}; + config_ = Config{*ton::unpack_engine_validator_config(std::move(conf))}; td::MultiPromise mp; auto ig = mp.init_guard(); @@ -3471,15 +3494,10 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollat } auto id = ton::PublicKeyHash{query.adnl_id_}; - auto shard = ton::ShardIdFull(query.workchain_, query.shard_); - if (!shard.is_valid_ext()) { - promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "invalid shard"))); - return; - } - + auto shard = ton::create_shard_id(query.shard_); auto R = config_.config_add_collator(id, shard); if (R.is_error()) { - promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + promise.set_value(create_control_query_error(R.move_as_error())); return; } if (!R.move_as_ok()) { @@ -3499,6 +3517,27 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollat }); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addShard &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + auto shard = ton::create_shard_id(query.shard_); + auto R = config_.config_add_shard(shard); + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { @@ -3532,7 +3571,7 @@ void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNode } auto f = F.move_as_ok(); - ton::ton_api::downcast_call(*f.get(), [&](auto &obj) { + ton::ton_api::downcast_call(*f, [&](auto &obj) { run_control_query(obj, std::move(data), src.pubkey_hash(), it->second, std::move(promise)); }); } @@ -3752,7 +3791,7 @@ int main(int argc, char *argv[]) { }); return td::Status::OK(); }); - p.add_option('M', "masterchain-only", "don't track shardchains", [&]() { + p.add_option('M', "not-all-shards", "monitor only a necessary set of shards instead of all", [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_masterchain_only); }); }); td::uint32 threads = 7; diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index bf47e2d79..46de7e86a 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -96,6 +96,7 @@ struct Config { std::map liteservers; std::map controls; std::set gc; + std::vector shards_to_monitor; void decref(ton::PublicKeyHash key); void incref(ton::PublicKeyHash key) { @@ -121,6 +122,7 @@ struct Config { td::Result config_add_control_interface(ton::PublicKeyHash key, td::int32 port); td::Result config_add_control_process(ton::PublicKeyHash key, td::int32 port, ton::PublicKeyHash id, td::uint32 permissions); + td::Result config_add_shard(ton::ShardIdFull shard); td::Result config_add_gc(ton::PublicKeyHash key); td::Result config_del_network_addr(td::IPAddress addr, std::vector cats, std::vector prio_cats); @@ -135,10 +137,10 @@ struct Config { td::Result config_del_control_process(td::int32 port, ton::PublicKeyHash id); td::Result config_del_gc(ton::PublicKeyHash key); - ton::tl_object_ptr tl() const; + ton::tl_object_ptr tl() const; Config(); - Config(ton::ton_api::engine_validator_config &config); + Config(const ton::ton_api::engine_validator_config_v2 &config); }; class ValidatorEngine : public td::actor::Actor { @@ -434,6 +436,8 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_addCollator &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_addShard &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { From 7241522de2c4ae243375c94cae8b44c0fe9cbb5e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 8 Aug 2022 12:41:28 +0300 Subject: [PATCH 017/388] Remove obsolete interface for importing blocks --- tl/generate/scheme/ton_api.tl | 4 - tl/generate/scheme/ton_api.tlo | Bin 72088 -> 71776 bytes .../validator-engine-console-query.cpp | 63 ---------- .../validator-engine-console-query.h | 70 ----------- .../validator-engine-console.cpp | 3 - validator-engine/validator-engine.cpp | 88 -------------- validator-engine/validator-engine.hpp | 6 - validator/interfaces/validator-manager.h | 2 - validator/manager-disk.hpp | 12 -- validator/manager-hardfork.hpp | 12 -- validator/manager.cpp | 113 ------------------ validator/manager.hpp | 8 -- validator/validator.h | 4 - 13 files changed, 385 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 187c4952c..837e57541 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -704,10 +704,6 @@ engine.validator.importShardOverlayCertificate workchain:int shard:long signed_k engine.validator.getValidatorSessionsInfo = engine.validator.ValidatorSessionsInfo; -engine.validator.generateBlockCandidate block_id:tonNode.BlockId = db.Candidate; -engine.validator.getRequiredBlockCandidates = engine.validator.RequiredBlockCandidates; -engine.validator.importBlockCandidate block:db.candidate = engine.validator.Success; - engine.validator.addCollator adnl_id:int256 shard:tonNode.shardId = engine.validator.Success; engine.validator.addShard shard:tonNode.shardId = engine.validator.Success; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 2bb8d64c32f5762201a2aab2bffd1bed74b12bc8..36c21e21f7ebcbdc819c4ab0da29bca718a36353 100644 GIT binary patch delta 38 wcmV+>0NMYTvjpI<1hDi7lc)(4vj_>~+yP>^V_GvVyp$P*mkNXY!rNGLvQg-eG<4d&+Os$$$Tfi=e7xVCY`()); - TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); - TRY_RESULT_ASSIGN(seqno_, tokenizer_.get_token()); - TRY_RESULT_ASSIGN(file_, tokenizer_.get_token()); - return td::Status::OK(); -} - -td::Status GenerateBlockCandidateQuery::send() { - auto b = ton::create_serialize_tl_object( - ton::create_tl_block_id_simple(ton::BlockId(wc_, shard_, seqno_))); - td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); - return td::Status::OK(); -} - -td::Status GenerateBlockCandidateQuery::receive(td::BufferSlice data) { - TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), - "received incorrect answer: "); - TRY_STATUS_PREFIX(td::write_file(file_, data.as_slice()), "failed to write block to file"); - td::TerminalIO::out() << "successfully written candidate to file\n"; - return td::Status::OK(); -} - -td::Status GetRequiredBlockCandidatesQuery::run() { - TRY_STATUS(tokenizer_.check_endl()); - return td::Status::OK(); -} - -td::Status GetRequiredBlockCandidatesQuery::send() { - auto b = ton::create_serialize_tl_object(); - td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); - return td::Status::OK(); -} - -td::Status GetRequiredBlockCandidatesQuery::receive(td::BufferSlice data) { - TRY_RESULT_PREFIX( - f, ton::fetch_tl_object(data.as_slice(), true), - "received incorrect answer: "); - td::TerminalIO::out() << td::json_encode(td::ToJson(*f), true); - return td::Status::OK(); -} - -td::Status ImportBlockCandidateQuery::run() { - TRY_RESULT_ASSIGN(file_, tokenizer_.get_token()); - return td::Status::OK(); -} - -td::Status ImportBlockCandidateQuery::send() { - TRY_RESULT(data, td::read_file(file_)); - TRY_RESULT_PREFIX(candidate, ton::fetch_tl_object(data.as_slice(), true), - "invalid file: "); - auto b = ton::create_serialize_tl_object(std::move(candidate)); - td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); - return td::Status::OK(); -} - -td::Status ImportBlockCandidateQuery::receive(td::BufferSlice data) { - TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), - "received incorrect answer: "); - td::TerminalIO::out() << "successfully imported a block candidate\n"; - return td::Status::OK(); -} - td::Status AddCollatorQuery::run() { TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index daa19ab1a..cfb8850d6 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1094,76 +1094,6 @@ class GetValidatorSessionsInfoQuery : public Query { } }; -class GenerateBlockCandidateQuery : public Query { - public: - GenerateBlockCandidateQuery(td::actor::ActorId console, Tokenizer tokenizer) - : Query(console, std::move(tokenizer)) { - } - td::Status run() override; - td::Status send() override; - td::Status receive(td::BufferSlice data) override; - static std::string get_name() { - return "genblock"; - } - static std::string get_help() { - return "genblock \t" - "generate a block candidate for a given shard (seqno mush match the next seqno for the shard), " - "candidate is saved to "; - } - std::string name() const override { - return get_name(); - } - - private: - td::int32 wc_; - td::int64 shard_; - td::int32 seqno_; - std::string file_; -}; - -class GetRequiredBlockCandidatesQuery : public Query { - public: - GetRequiredBlockCandidatesQuery(td::actor::ActorId console, Tokenizer tokenizer) - : Query(console, std::move(tokenizer)) { - } - td::Status run() override; - td::Status send() override; - td::Status receive(td::BufferSlice data) override; - static std::string get_name() { - return "getrequiredblockcandidates"; - } - static std::string get_help() { - return "getrequiredblockcandidates\t" - "get a list of block candidates that the validator is currently waiting for"; - } - std::string name() const override { - return get_name(); - } -}; - -class ImportBlockCandidateQuery : public Query { - public: - ImportBlockCandidateQuery(td::actor::ActorId console, Tokenizer tokenizer) - : Query(console, std::move(tokenizer)) { - } - td::Status run() override; - td::Status send() override; - td::Status receive(td::BufferSlice data) override; - static std::string get_name() { - return "importblockcandidate"; - } - static std::string get_help() { - return "importblockcandidate \t" - "load a block candidate from a given file"; - } - std::string name() const override { - return get_name(); - } - - private: - std::string file_; -}; - class AddCollatorQuery : public Query { public: AddCollatorQuery(td::actor::ActorId console, Tokenizer tokenizer) diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 83375b209..0ce81fb41 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -141,9 +141,6 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); - add_query_runner(std::make_unique>()); - add_query_runner(std::make_unique>()); - add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); } diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index fda610afe..9bafd347f 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -3393,94 +3393,6 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getValida std::move(P)); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_generateBlockCandidate &query, - td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, - td::Promise promise) { - if (!(perm & ValidatorEnginePermissions::vep_default)) { - promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); - return; - } - if (validator_manager_.empty()) { - promise.set_value( - create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "validator manager not started"))); - return; - } - ton::BlockId block_id = ton::create_block_id_simple(query.block_id_); - td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::generate_block_candidate, - block_id, [promise = std::move(promise)](td::Result R) mutable { - if (R.is_ok()) { - auto block = R.move_as_ok(); - auto result = ton::create_serialize_tl_object( - ton::PublicKey{ton::pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), - ton::create_tl_block_id(block.id), std::move(block.data), - std::move(block.collated_data)); - promise.set_result(std::move(result)); - } else { - promise.set_value(create_control_query_error(R.move_as_error())); - } - }); -} - -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getRequiredBlockCandidates &query, - td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, - td::Promise promise) { - if (!(perm & ValidatorEnginePermissions::vep_default)) { - promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); - return; - } - if (validator_manager_.empty()) { - promise.set_value( - create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "validator manager not started"))); - return; - } - td::actor::send_closure( - validator_manager_, &ton::validator::ValidatorManagerInterface::get_required_block_candidates, - [promise = std::move(promise)](td::Result> R) mutable { - if (R.is_ok()) { - std::vector> block_ids; - for (const ton::BlockId &block_id : R.move_as_ok()) { - block_ids.push_back(ton::create_tl_block_id_simple(block_id)); - } - auto result = ton::create_serialize_tl_object( - std::move(block_ids)); - promise.set_result(std::move(result)); - } else { - promise.set_value(create_control_query_error(R.move_as_error())); - } - }); -} - -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importBlockCandidate &query, - td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, - td::Promise promise) { - if (!(perm & ValidatorEnginePermissions::vep_modify)) { - promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); - return; - } - if (validator_manager_.empty()) { - promise.set_value( - create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "validator manager not started"))); - return; - } - - auto collated_data_hash = td::sha256_bits256(query.block_->collated_data_); - auto key = ton::PublicKey{query.block_->source_}; - auto e_key = ton::Ed25519_PublicKey{key.ed25519_value().raw()}; - ton::BlockCandidate candidate{e_key, ton::create_block_id(query.block_->id_), collated_data_hash, - std::move(query.block_->data_), std::move(query.block_->collated_data_)}; - - td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::import_block_candidate, - std::move(candidate), - [promise = std::move(promise)](td::Result R) mutable { - if (R.is_ok()) { - promise.set_result(ton::serialize_tl_object( - ton::create_tl_object(), true)); - } else { - promise.set_value(create_control_query_error(R.move_as_error())); - } - }); -} - void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollator &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 46de7e86a..dd1c8fcab 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -428,12 +428,6 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getValidatorSessionsInfo &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); - void run_control_query(ton::ton_api::engine_validator_generateBlockCandidate &query, td::BufferSlice data, - ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); - void run_control_query(ton::ton_api::engine_validator_getRequiredBlockCandidates &query, td::BufferSlice data, - ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); - void run_control_query(ton::ton_api::engine_validator_importBlockCandidate &query, td::BufferSlice data, - ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_addCollator &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_addShard &query, td::BufferSlice data, diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 345ae679f..46fe88709 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -168,8 +168,6 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) = 0; - virtual void wait_block_candidate(BlockId block_id, td::Timestamp timeout, td::Promise promise) = 0; - static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) { return ts / (1 << 17) != prev_ts / (1 << 17); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 2e331c720..3c45b48aa 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -379,18 +379,6 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { UNREACHABLE(); } - void generate_block_candidate(BlockId block_id, td::Promise promise) override { - UNREACHABLE(); - } - void get_required_block_candidates(td::Promise> promise) override { - UNREACHABLE(); - } - void import_block_candidate(BlockCandidate candidate, td::Promise promise) override { - UNREACHABLE(); - } - void wait_block_candidate(BlockId block_id, td::Timestamp timeout, td::Promise promise) override { - UNREACHABLE(); - } void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { UNREACHABLE(); diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 79e742c4b..bc8913b89 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -439,18 +439,6 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { UNREACHABLE(); } - void generate_block_candidate(BlockId block_id, td::Promise promise) override { - UNREACHABLE(); - } - void get_required_block_candidates(td::Promise> promise) override { - UNREACHABLE(); - } - void import_block_candidate(BlockCandidate candidate, td::Promise promise) override { - UNREACHABLE(); - } - void wait_block_candidate(BlockId block_id, td::Timestamp timeout, td::Promise promise) override { - UNREACHABLE(); - } void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { UNREACHABLE(); diff --git a/validator/manager.cpp b/validator/manager.cpp index 442796981..ab5968943 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2641,119 +2641,6 @@ void ValidatorManagerImpl::get_validator_sessions_info( IntermediateData::step({std::move(groups), {}, std::move(promise)}); } -void ValidatorManagerImpl::generate_block_candidate(BlockId block_id, td::Promise promise) { - if (!block_id.is_valid_full()) { - promise.set_error(td::Status::Error("invalid block id")); - return; - } - if (last_masterchain_state_.is_null()) { - promise.set_error(td::Status::Error("not started")); - return; - } - ShardIdFull shard_id = block_id.shard_full(); - std::vector prev; - auto shard = last_masterchain_state_->get_shard_from_config(shard_id); - if (shard.not_null()) { - if (shard->before_split()) { - promise.set_error(td::Status::Error("shard is before_split")); - return; - } - if (shard->before_merge()) { - promise.set_error(td::Status::Error("shard is before_merge")); - return; - } - prev.push_back(shard->top_block_id()); - } else { - auto parent = shard_id.pfx_len() == 0 ? td::Ref() - : last_masterchain_state_->get_shard_from_config(shard_parent(shard_id)); - if (parent.not_null() && parent->before_split()) { - prev.push_back(parent->top_block_id()); - } else { - auto child_l = last_masterchain_state_->get_shard_from_config(shard_child(shard_id, true)); - auto child_r = last_masterchain_state_->get_shard_from_config(shard_child(shard_id, false)); - if (child_l.not_null() && child_r.not_null() && child_l->before_merge() && child_r->before_merge()) { - prev.push_back(child_l->top_block_id()); - prev.push_back(child_r->top_block_id()); - } - } - if (prev.empty()) { - promise.set_error(td::Status::Error("no such shard")); - return; - } - } - - BlockSeqno next_seqno = 0; - for (const BlockIdExt& prev_id : prev) { - next_seqno = std::max(next_seqno, prev_id.seqno() + 1); - } - if (next_seqno != block_id.seqno) { - promise.set_error(td::Status::Error(PSTRING() << "seqno mismatch: asked for seqno " << block_id.seqno - << ", but actual next seqno is " << next_seqno)); - return; - } - - Ed25519_PublicKey local_id{Bits256::zero()}; - td::Ref validator_set = last_masterchain_state_->get_validator_set(shard_id); - if (validator_set.is_null()) { - promise.set_error(td::Status::Error("cannot get validator set")); - return; - } - run_collate_query(shard_id, last_masterchain_block_id_, std::move(prev), local_id, std::move(validator_set), - actor_id(this), td::Timestamp::in(10.0), std::move(promise)); -} - -void ValidatorManagerImpl::get_required_block_candidates(td::Promise> promise) { - std::vector block_ids; - for (const auto& p : pending_block_candidates_) { - block_ids.push_back(p.first); - } - promise.set_result(std::move(block_ids)); -} - -void ValidatorManagerImpl::import_block_candidate(BlockCandidate candidate, td::Promise promise) { - auto it = pending_block_candidates_.find(candidate.id.id); - if (it != pending_block_candidates_.end()) { - while (!it->second.empty()) { - auto promise = std::move(it->second.back().first); - it->second.pop_back(); - if (it->second.empty()) { - promise.set_result(std::move(candidate)); - } else { - promise.set_result(candidate.clone()); - } - } - pending_block_candidates_.erase(it); - } - promise.set_result(td::Unit()); -} - -void ValidatorManagerImpl::wait_block_candidate(BlockId block_id, td::Timestamp timeout, - td::Promise promise) { - pending_block_candidates_[block_id].emplace_back(std::move(promise), timeout); - delay_action([SelfId = actor_id(this), block_id, timeout]() { - td::actor::send_closure(SelfId, &ValidatorManagerImpl::cleanup_old_pending_candidates, block_id, timeout); - }, timeout); -} - -void ValidatorManagerImpl::cleanup_old_pending_candidates(BlockId block_id, td::Timestamp now) { - auto it = pending_block_candidates_.find(block_id); - if (it == pending_block_candidates_.end()) { - return; - } - it->second.erase(std::remove_if(it->second.begin(), it->second.end(), - [&](std::pair, td::Timestamp> &p) { - if (p.second.is_in_past(now)) { - p.first.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); - return true; - } - return false; - }), - it->second.end()); - if (it->second.empty()) { - pending_block_candidates_.erase(it); - } -} - void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) { auto it = collator_nodes_.find(id); if (it == collator_nodes_.end()) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 6eb192a71..5447c05e1 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -542,11 +542,6 @@ class ValidatorManagerImpl : public ValidatorManager { void get_validator_sessions_info( td::Promise> promise) override; - void generate_block_candidate(BlockId block_id, td::Promise promise) override; - void get_required_block_candidates(td::Promise> promise) override; - void import_block_candidate(BlockCandidate candidate, td::Promise promise) override; - void wait_block_candidate(BlockId block_id, td::Timestamp timeout, td::Promise promise) override; - void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; private: @@ -614,9 +609,6 @@ class ValidatorManagerImpl : public ValidatorManager { private: std::map> shard_client_waiters_; - std::map, td::Timestamp>>> pending_block_candidates_; - void cleanup_old_pending_candidates(BlockId block_id, td::Timestamp now); - std::map> collator_nodes_; bool collating_masterchain_ = false; diff --git a/validator/validator.h b/validator/validator.h index c5212abb4..363c837f7 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -229,10 +229,6 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_validator_sessions_info( td::Promise> promise) = 0; - virtual void generate_block_candidate(BlockId block_id, td::Promise promise) = 0; - virtual void get_required_block_candidates(td::Promise> promise) = 0; - virtual void import_block_candidate(BlockCandidate candidate, td::Promise promise) = 0; - virtual void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; }; From 5ba2a5571663fe5187a5615a60610223fb922990 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 8 Aug 2022 14:08:57 +0300 Subject: [PATCH 018/388] Add --lite-validator flag, rework is_validator checks --- tdutils/td/utils/OptionParser.cpp | 2 +- validator-engine/validator-engine.cpp | 33 ++++++++++----------------- validator-engine/validator-engine.hpp | 11 ++++++--- validator/manager.cpp | 23 ++++++++----------- validator/manager.hpp | 2 +- validator/validator-options.cpp | 2 +- validator/validator-options.hpp | 24 ++++++++++--------- validator/validator.h | 11 ++++----- 8 files changed, 50 insertions(+), 58 deletions(-) diff --git a/tdutils/td/utils/OptionParser.cpp b/tdutils/td/utils/OptionParser.cpp index b95848562..634570e1b 100644 --- a/tdutils/td/utils/OptionParser.cpp +++ b/tdutils/td/utils/OptionParser.cpp @@ -33,7 +33,7 @@ void OptionParser::set_description(string description) { void OptionParser::add_option(Option::Type type, char short_key, Slice long_key, Slice description, std::function callback) { for (auto &option : options_) { - if (option.short_key == short_key || (!long_key.empty() && long_key == option.long_key)) { + if ((short_key != '\0' && option.short_key == short_key) || (!long_key.empty() && long_key == option.long_key)) { LOG(ERROR) << "Ignore duplicated option '" << short_key << "' '" << long_key << "'"; } } diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 9bafd347f..26c485bdf 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1391,21 +1391,14 @@ td::Status ValidatorEngine::load_global_config() { h.push_back(b); } validator_options_.write().set_hardforks(std::move(h)); + validator_options_.write().set_validator_lite_mode(validator_lite_mode_); return td::Status::OK(); } void ValidatorEngine::init_validator_options() { - if (!masterchain_only_) { - validator_options_.write().set_shard_check_function( - [](ton::ShardIdFull shard, ton::CatchainSeqno cc_seqno, - ton::validator::ValidatorManagerOptions::ShardCheckMode mode) -> bool { - if (mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_monitor) { - return true; - } - CHECK(mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_validate); - return true; - }); + if (!not_all_shards_) { + validator_options_.write().set_shard_check_function([](ton::ShardIdFull shard) -> bool { return true; }); } else { std::vector shards = {ton::ShardIdFull(ton::masterchainId)}; for (const auto& c : config_.collators) { @@ -1417,18 +1410,13 @@ void ValidatorEngine::init_validator_options() { std::sort(shards.begin(), shards.end()); shards.erase(std::unique(shards.begin(), shards.end()), shards.end()); validator_options_.write().set_shard_check_function( - [shards = std::move(shards)](ton::ShardIdFull shard, ton::CatchainSeqno cc_seqno, - ton::validator::ValidatorManagerOptions::ShardCheckMode mode) -> bool { - if (mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_monitor) { - for (auto s : shards) { - if (shard_is_ancestor(shard, s)) { - return true; - } + [shards = std::move(shards)](ton::ShardIdFull shard) -> bool { + for (auto s : shards) { + if (shard_is_ancestor(shard, s)) { + return true; } - return false; } - CHECK(mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_validate); - return true; + return false; }); } } @@ -3704,7 +3692,10 @@ int main(int argc, char *argv[]) { return td::Status::OK(); }); p.add_option('M', "not-all-shards", "monitor only a necessary set of shards instead of all", [&]() { - acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_masterchain_only); }); + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_not_all_shards); }); + }); + p.add_option('\0', "lite-validator", "lite-mode validator (don't collate blocks, use collator nodes instead)", [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_validator_lite_mode); }); }); td::uint32 threads = 7; p.add_checked_option( diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index dd1c8fcab..8e333791a 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -214,7 +214,8 @@ class ValidatorEngine : public td::actor::Actor { bool started_ = false; ton::BlockSeqno truncate_seqno_{0}; std::string session_logs_file_; - bool masterchain_only_ = false; + bool not_all_shards_ = false; + bool validator_lite_mode_ = false; std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; @@ -266,9 +267,13 @@ class ValidatorEngine : public td::actor::Actor { void add_key_to_set(ton::PublicKey key) { keys_[key.compute_short_id()] = key; } - void set_masterchain_only() { - masterchain_only_ = true; + void set_not_all_shards() { + not_all_shards_ = true; } + void set_validator_lite_mode() { + validator_lite_mode_ = true; + } + void start_up() override; ValidatorEngine() { } diff --git a/validator/manager.cpp b/validator/manager.cpp index ab5968943..c2bb70619 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -369,7 +369,7 @@ void ValidatorManagerImpl::get_key_block_proof_link(BlockIdExt block_id, td::Pro } void ValidatorManagerImpl::new_external_message(td::BufferSlice data) { - if (!is_validator()) { + if (!is_collator()) { return; } if ((double)ext_messages_.size() > max_mempool_num()) { @@ -388,7 +388,7 @@ void ValidatorManagerImpl::add_external_message(td::Ref msg) { auto id = message->ext_id(); auto address = message->address(); unsigned long per_address_limit = 256; - if(ext_addr_messages_.count(address) < per_address_limit) { + if (ext_addr_messages_.count(address) < per_address_limit) { if (ext_messages_hashes_.count(id.hash) == 0) { ext_messages_.emplace(id, std::move(message)); ext_messages_hashes_.emplace(id.hash, id); @@ -401,7 +401,7 @@ void ValidatorManagerImpl::check_external_message(td::BufferSlice data, td::Prom } void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { - if (!is_validator()) { + if (!is_collator()) { return; } auto R = create_ihr_message(std::move(data)); @@ -418,7 +418,7 @@ void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { } void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { - if (!is_validator()) { + if (!is_collator()) { return; } if (!last_masterchain_block_handle_) { @@ -2086,7 +2086,7 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group "validatorgroup", shard, validator_id, session_id, validator_set, last_masterchain_state_->get_collator_set(), opts, keyring_, adnl_, rldp_, overlays_, db_root_, actor_id(this), init_session, - opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), true); + opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_->validator_lite_mode()); return G; } } @@ -2457,14 +2457,14 @@ void ValidatorManagerImpl::get_archive_slice(td::uint64 archive_id, td::uint64 o } bool ValidatorManagerImpl::is_validator() { - // TODO: change is_vaidator to other condition in some cases - return true; // temp_keys_.size() > 0 || permanent_keys_.size() > 0; + return temp_keys_.size() > 0 || permanent_keys_.size() > 0; +} + +bool ValidatorManagerImpl::is_collator() { + return !collator_nodes_.empty() || (!opts_->validator_lite_mode() && is_validator()); } PublicKeyHash ValidatorManagerImpl::get_validator(ShardIdFull shard, td::Ref val_set) { - if (!opts_->need_validate(shard, val_set->get_catchain_seqno())) { - return PublicKeyHash::zero(); - } for (auto &key : temp_keys_) { if (val_set->is_validator(key.bits256_value())) { return key; @@ -2648,9 +2648,6 @@ void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh it = collator_nodes_.emplace(id, std::move(actor)).first; } td::actor::send_closure(it->second, &CollatorNode::add_shard, shard); - if (shard.is_masterchain()) { - collating_masterchain_ = true; - } } td::actor::ActorOwn ValidatorManagerFactory::create( diff --git a/validator/manager.hpp b/validator/manager.hpp index 5447c05e1..9e55abf58 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -491,6 +491,7 @@ class ValidatorManagerImpl : public ValidatorManager { void read_gc_list(std::vector list); bool is_validator(); + bool is_collator(); PublicKeyHash get_validator(ShardIdFull shard, td::Ref val_set); ValidatorManagerImpl(td::Ref opts, std::string db_root, @@ -610,7 +611,6 @@ class ValidatorManagerImpl : public ValidatorManager { std::map> shard_client_waiters_; std::map> collator_nodes_; - bool collating_masterchain_ = false; std::set shards_to_monitor_ = {ShardIdFull(masterchainId)}; }; diff --git a/validator/validator-options.cpp b/validator/validator-options.cpp index 93fe05e6c..cb26fe44d 100644 --- a/validator/validator-options.cpp +++ b/validator/validator-options.cpp @@ -26,7 +26,7 @@ namespace validator { td::Ref ValidatorManagerOptions::create( BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard, bool allow_blockchain_init, + std::function check_shard, bool allow_blockchain_init, double sync_blocks_before, double block_ttl, double state_ttl, double max_mempool_num, double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) { return td::make_ref(zero_block_id, init_block_id, std::move(check_shard), diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index d23d8cc91..b4a94a39b 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -33,10 +33,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { return init_block_id_; } bool need_monitor(ShardIdFull shard) const override { - return check_shard_(shard, 0, ShardCheckMode::m_monitor); - } - bool need_validate(ShardIdFull shard, CatchainSeqno cc_seqno) const override { - return check_shard_(shard, cc_seqno, ShardCheckMode::m_validate); + return check_shard_(shard); } bool allow_blockchain_init() const override { return allow_blockchain_init_; @@ -114,6 +111,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { std::string get_session_logs_file() const override { return session_logs_file_; } + bool validator_lite_mode() const override { + return validator_lite_mode_; + } void set_zero_block_id(BlockIdExt block_id) override { zero_block_id_ = block_id; @@ -121,7 +121,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_init_block_id(BlockIdExt block_id) override { init_block_id_ = block_id; } - void set_shard_check_function(std::function check_shard) override { + void set_shard_check_function(std::function check_shard) override { check_shard_ = std::move(check_shard); } void set_allow_blockchain_init(bool value) override { @@ -167,17 +167,18 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_session_logs_file(std::string f) override { session_logs_file_ = std::move(f); } + void set_validator_lite_mode(bool value) override { + validator_lite_mode_ = value; + } ValidatorManagerOptionsImpl *make_copy() const override { return new ValidatorManagerOptionsImpl(*this); } ValidatorManagerOptionsImpl(BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard, - bool allow_blockchain_init, double sync_blocks_before, - double block_ttl, double state_ttl, double max_mempool_num, - double archive_ttl, double key_proof_ttl, - bool initial_sync_disabled) + std::function check_shard, bool allow_blockchain_init, + double sync_blocks_before, double block_ttl, double state_ttl, double max_mempool_num, + double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) : zero_block_id_(zero_block_id) , init_block_id_(init_block_id) , check_shard_(std::move(check_shard)) @@ -194,7 +195,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { private: BlockIdExt zero_block_id_; BlockIdExt init_block_id_; - std::function check_shard_; + std::function check_shard_; bool allow_blockchain_init_; double sync_blocks_before_; double block_ttl_; @@ -209,6 +210,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { BlockSeqno truncate_{0}; BlockSeqno sync_upto_{0}; std::string session_logs_file_; + bool validator_lite_mode_ = false; }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 363c837f7..545c8a81c 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -47,12 +47,9 @@ class DownloadToken { struct ValidatorManagerOptions : public td::CntObject { public: - enum class ShardCheckMode { m_monitor, m_validate }; - virtual BlockIdExt zero_block_id() const = 0; virtual BlockIdExt init_block_id() const = 0; virtual bool need_monitor(ShardIdFull shard) const = 0; - virtual bool need_validate(ShardIdFull shard, CatchainSeqno cc_seqno) const = 0; virtual bool allow_blockchain_init() const = 0; virtual double sync_blocks_before() const = 0; virtual double block_ttl() const = 0; @@ -75,11 +72,11 @@ struct ValidatorManagerOptions : public td::CntObject { virtual BlockSeqno get_truncate_seqno() const = 0; virtual BlockSeqno sync_upto() const = 0; virtual std::string get_session_logs_file() const = 0; + virtual bool validator_lite_mode() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; - virtual void set_shard_check_function( - std::function check_shard) = 0; + virtual void set_shard_check_function(std::function check_shard) = 0; virtual void set_allow_blockchain_init(bool value) = 0; virtual void set_sync_blocks_before(double value) = 0; virtual void set_block_ttl(double value) = 0; @@ -94,11 +91,11 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void truncate_db(BlockSeqno seqno) = 0; virtual void set_sync_upto(BlockSeqno seqno) = 0; virtual void set_session_logs_file(std::string f) = 0; + virtual void set_validator_lite_mode(bool value) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard = [](ShardIdFull, CatchainSeqno, - ShardCheckMode) { return true; }, + std::function check_shard = [](ShardIdFull) { return true; }, bool allow_blockchain_init = false, double sync_blocks_before = 86400, double block_ttl = 86400 * 7, double state_ttl = 3600, double archive_ttl = 86400 * 365, double key_proof_ttl = 86400 * 3650, double max_mempool_num = 999999, From 7749cbfa1ffb0de45f853cb39c744e061d3bf13e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 9 Aug 2022 17:37:01 +0300 Subject: [PATCH 019/388] Change config serialization for compatibility --- validator-engine/validator-engine.cpp | 18 +++++++++++++----- validator-engine/validator-engine.hpp | 2 +- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 26c485bdf..65fb0e8f2 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -177,7 +177,7 @@ Config::Config(const ton::ton_api::engine_validator_config_v2 &config) { } } -ton::tl_object_ptr Config::tl() const { +ton::tl_object_ptr Config::tl() const { std::vector> addrs_vec; for (auto &x : addrs) { if (x.second.proxy) { @@ -256,10 +256,18 @@ ton::tl_object_ptr Config::tl() const for (auto &id : gc) { gc_vec->ids_.push_back(id.tl()); } - return ton::create_tl_object( - out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec), - full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(liteserver_vec), - std::move(control_vec), std::move(shards_vec), std::move(gc_vec)); + + if (col_vec.empty() && shards_vec.empty()) { + return ton::create_tl_object( + out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), + full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(liteserver_vec), + std::move(control_vec), std::move(gc_vec)); + } else { + return ton::create_tl_object( + out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec), + full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(liteserver_vec), + std::move(control_vec), std::move(shards_vec), std::move(gc_vec)); + } } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 8e333791a..47d612b7b 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -137,7 +137,7 @@ struct Config { td::Result config_del_control_process(td::int32 port, ton::PublicKeyHash id); td::Result config_del_gc(ton::PublicKeyHash key); - ton::tl_object_ptr tl() const; + ton::tl_object_ptr tl() const; Config(); Config(const ton::ton_api::engine_validator_config_v2 &config); From 662435462e16b5e9489ecfb881a73e59ede9166b Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 10 Aug 2022 20:59:49 +0300 Subject: [PATCH 020/388] Validators temporary join shard overlays --- overlay/overlay-manager.cpp | 20 +++++--- overlay/overlay-manager.h | 3 +- overlay/overlay-peers.cpp | 2 +- overlay/overlay.cpp | 53 +++++++++++---------- overlay/overlay.h | 2 +- overlay/overlay.hpp | 8 +--- overlay/overlays.h | 7 ++- tl/generate/scheme/ton_api.tl | 2 + tl/generate/scheme/ton_api.tlo | Bin 71776 -> 71904 bytes validator/full-node-shard.cpp | 56 +++++++++++++---------- validator/full-node-shard.hpp | 1 + validator/interfaces/validator-manager.h | 2 + validator/manager-disk.hpp | 2 + validator/manager-hardfork.hpp | 2 + validator/manager.cpp | 30 +++++++++++- validator/manager.hpp | 9 ++++ validator/validator-group.cpp | 13 ++++-- validator/validator-group.hpp | 2 +- 18 files changed, 140 insertions(+), 74 deletions(-) diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index 62d98eb00..2134bbc0c 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -30,7 +30,6 @@ #include "td/utils/Status.h" #include "td/utils/overloaded.h" -#include "keys/encryptor.h" #include "td/utils/port/Poll.h" #include @@ -92,7 +91,6 @@ void OverlayManager::delete_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdSho void OverlayManager::create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) { CHECK(!dht_node_.empty()); - CHECK(callback != nullptr); auto id = overlay_id.compute_short_id(); register_overlay(local_id, id, Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), @@ -100,12 +98,13 @@ void OverlayManager::create_public_overlay(adnl::AdnlNodeIdShort local_id, Overl } void OverlayManager::create_public_overlay_external(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, - OverlayPrivacyRules rules, td::string scope) { + std::unique_ptr callback, OverlayPrivacyRules rules, + td::string scope) { CHECK(!dht_node_.empty()); auto id = overlay_id.compute_short_id(); register_overlay(local_id, id, - Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), nullptr, - std::move(rules), scope)); + Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), + std::move(callback), std::move(rules), scope, true)); } void OverlayManager::create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, @@ -155,16 +154,23 @@ void OverlayManager::receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdSh auto M = R.move_as_ok(); + OverlayIdShort overlay_id{M->overlay_}; + auto send_remove_peer = [&, this]() { + send_message(src, dst, overlay_id, create_serialize_tl_object()); + }; + auto it = overlays_.find(dst); if (it == overlays_.end()) { VLOG(OVERLAY_NOTICE) << this << ": query to unknown overlay " << M->overlay_ << "@" << dst << " from " << src; promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad local_id " << dst)); + send_remove_peer(); return; } - auto it2 = it->second.find(OverlayIdShort{M->overlay_}); + auto it2 = it->second.find(overlay_id); if (it2 == it->second.end()) { VLOG(OVERLAY_NOTICE) << this << ": query to localid not in overlay " << M->overlay_ << "@" << dst << " from " << src; promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad overlay_id " << M->overlay_)); + send_remove_peer(); return; } @@ -394,7 +400,7 @@ td::BufferSlice Certificate::to_sign(OverlayIdShort overlay_id, PublicKeyHash is } } -const PublicKeyHash Certificate::issuer_hash() const { +PublicKeyHash Certificate::issuer_hash() const { PublicKeyHash r; issued_by_.visit( td::overloaded([&](const PublicKeyHash &x) { r = x; }, [&](const PublicKey &x) { r = x.compute_short_id(); })); diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h index a3ed4a54a..ada37303d 100644 --- a/overlay/overlay-manager.h +++ b/overlay/overlay-manager.h @@ -53,7 +53,8 @@ class OverlayManager : public Overlays { void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) override; void create_public_overlay_external(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, - OverlayPrivacyRules rules, td::string scope) override; + std::unique_ptr callback, OverlayPrivacyRules rules, + td::string scope) override; void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules) override; diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index dbb6f32a8..e1e4a3ce8 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -142,7 +142,7 @@ void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::BufferSlic void OverlayImpl::send_random_peers_cont(adnl::AdnlNodeIdShort src, OverlayNode node, td::Promise promise) { std::vector> vec; - if (!is_external()) { + if (!is_external_) { vec.emplace_back(node.tl()); } diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index 8a655c6e5..ff86d837e 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -38,10 +38,10 @@ td::actor::ActorOwn Overlay::create(td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, - OverlayPrivacyRules rules, td::string scope) { + OverlayPrivacyRules rules, td::string scope, bool is_external) { auto R = td::actor::create_actor("overlay", keyring, adnl, manager, dht_node, local_id, std::move(overlay_id), true, std::vector(), - std::move(callback), std::move(rules), scope); + std::move(callback), std::move(rules), scope, is_external); return td::actor::ActorOwn(std::move(R)); } @@ -61,7 +61,7 @@ OverlayImpl::OverlayImpl(td::actor::ActorId keyring, td::actor td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, bool pub, std::vector nodes, std::unique_ptr callback, - OverlayPrivacyRules rules, td::string scope) + OverlayPrivacyRules rules, td::string scope, bool is_external) : keyring_(keyring) , adnl_(adnl) , manager_(manager) @@ -71,10 +71,11 @@ OverlayImpl::OverlayImpl(td::actor::ActorId keyring, td::actor , callback_(std::move(callback)) , public_(pub) , rules_(std::move(rules)) - , scope_(scope) { + , scope_(scope) + , is_external_(is_external) { overlay_id_ = id_full_.compute_short_id(); - if (is_external()) { + if (is_external_) { CHECK(public_); VLOG(OVERLAY_INFO) << this << ": creating public external"; } else { @@ -92,7 +93,6 @@ OverlayImpl::OverlayImpl(td::actor::ActorId keyring, td::actor void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeers &query, td::Promise promise) { - CHECK(!is_external()); if (public_) { VLOG(OVERLAY_DEBUG) << this << ": received " << query.peers_->nodes_.size() << " nodes from " << src << " in getRandomPeers query"; @@ -113,7 +113,6 @@ void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getR void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcast &query, td::Promise promise) { - CHECK(!is_external()); auto it = broadcasts_.find(query.hash_); if (it == broadcasts_.end()) { VLOG(OVERLAY_NOTICE) << this << ": received getBroadcastQuery(" << query.hash_ << ") from " << src @@ -135,15 +134,16 @@ void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getB void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcastList &query, td::Promise promise) { - CHECK(!is_external()); VLOG(OVERLAY_WARNING) << this << ": DROPPING getBroadcastList query"; promise.set_error(td::Status::Error(ErrorCode::protoviolation, "dropping get broadcast list query")); } void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise) { - if (is_external()) { + if (is_external_) { LOG(OVERLAY_WARNING) << "dropping query in external overlay " << overlay_id_; promise.set_error(td::Status::Error("overlay is external")); + td::actor::send_closure(manager_, &Overlays::send_message, src, local_id_, overlay_id_, + create_serialize_tl_object()); return; } if (!public_) { @@ -171,32 +171,27 @@ void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr bcast) { - CHECK(!is_external()); return BroadcastSimple::create(this, message_from, std::move(bcast)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr b) { - CHECK(!is_external()); return OverlayFecBroadcastPart::create(this, message_from, std::move(b)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr b) { - CHECK(!is_external()); return OverlayFecBroadcastPart::create(this, message_from, std::move(b)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr bcast) { - CHECK(!is_external()); return td::Status::Error(ErrorCode::protoviolation, PSTRING() << "received strange message broadcastNotFound from " << message_from); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg) { - CHECK(!is_external()); auto it = fec_broadcasts_.find(msg->hash_); if (it != fec_broadcasts_.end()) { VLOG(OVERLAY_DEBUG) << this << ": received fec opt-out message from " << message_from << " for broadcast " @@ -211,7 +206,6 @@ td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg) { - CHECK(!is_external()); auto it = fec_broadcasts_.find(msg->hash_); if (it != fec_broadcasts_.end()) { VLOG(OVERLAY_DEBUG) << this << ": received fec completed message from " << message_from << " for broadcast " @@ -226,17 +220,12 @@ td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg) { - CHECK(!is_external()); VLOG(OVERLAY_DEBUG) << this << ": received unicast from " << message_from; callback_->receive_message(message_from, overlay_id_, std::move(msg->data_)); return td::Status::OK(); } void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) { - if (is_external()) { - LOG(OVERLAY_WARNING) << "dropping message in external overlay " << overlay_id_; - return; - } if (!public_) { if (peers_.get(src) == nullptr) { VLOG(OVERLAY_WARNING) << this << ": received query in private overlay from unknown source " << src; @@ -244,13 +233,29 @@ void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice dat } } auto X = fetch_tl_object(data.clone(), true); + if (X.is_error()) { + auto Y = fetch_tl_object(data.clone(), true); + if (Y.is_ok()) { + VLOG(OVERLAY_DEBUG) << this << ": received removePeer message from " << src; + if (peers_.exists(src)) { + del_peer(src); + callback_->on_remove_peer(src); + } + return; + } + } + + if (is_external_) { + LOG(OVERLAY_WARNING) << "dropping message in external overlay " << overlay_id_; + return; + } if (X.is_error()) { VLOG(OVERLAY_DEBUG) << this << ": received custom message"; callback_->receive_message(src, overlay_id_, std::move(data)); return; } auto Q = X.move_as_ok(); - ton_api::downcast_call(*Q.get(), [Self = this, &Q, &src](auto &object) { + ton_api::downcast_call(*Q, [Self = this, &Q, &src](auto &object) { Self->process_broadcast(src, move_tl_object_as>(Q)); }); } @@ -344,7 +349,7 @@ void OverlayImpl::receive_dht_nodes(td::Result res, bool dummy) { VLOG(OVERLAY_NOTICE) << this << ": can not get value from DHT: " << res.move_as_error(); } - if (is_external()) { + if (is_external_) { return; } @@ -548,7 +553,7 @@ void OverlayImpl::send_new_fec_broadcast_part(PublicKeyHash local_id, Overlay::B } void OverlayImpl::deliver_broadcast(PublicKeyHash source, td::BufferSlice data) { - if (is_external()) { + if (is_external_) { return; } callback_->receive_broadcast(source, overlay_id_, std::move(data)); @@ -623,7 +628,7 @@ void OverlayImpl::set_privacy_rules(OverlayPrivacyRules rules) { } void OverlayImpl::check_broadcast(PublicKeyHash src, td::BufferSlice data, td::Promise promise) { - if (is_external()) { + if (is_external_) { promise.set_result(td::Unit()); return; } diff --git a/overlay/overlay.h b/overlay/overlay.h index 51eab1b43..146be7383 100644 --- a/overlay/overlay.h +++ b/overlay/overlay.h @@ -42,7 +42,7 @@ class Overlay : public td::actor::Actor { td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, - OverlayPrivacyRules rules, td::string scope); + OverlayPrivacyRules rules, td::string scope, bool is_external = false); static td::actor::ActorOwn create(td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId manager, diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 59b3ec06a..197864946 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -113,7 +113,7 @@ class OverlayImpl : public Overlay { td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, bool pub, std::vector nodes, std::unique_ptr callback, - OverlayPrivacyRules rules, td::string scope = "{ \"type\": \"undefined\" }"); + OverlayPrivacyRules rules, td::string scope = "{ \"type\": \"undefined\" }", bool is_external = false); void update_dht_node(td::actor::ActorId dht) override { dht_node_ = dht; } @@ -251,13 +251,8 @@ class OverlayImpl : public Overlay { } private: - bool is_external() const { - return callback_ == nullptr; - } - template void process_query(adnl::AdnlNodeIdShort src, T &query, td::Promise promise) { - CHECK(!is_external()); callback_->receive_query(src, overlay_id_, serialize_tl_object(&query, true), std::move(promise)); } @@ -357,6 +352,7 @@ class OverlayImpl : public Overlay { bool semi_public_ = false; OverlayPrivacyRules rules_; td::string scope_; + bool is_external_ = false; std::map> certs_; class CachedEncryptor : public td::ListNode { diff --git a/overlay/overlays.h b/overlay/overlays.h index 82617fab6..0a9c7765c 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -145,7 +145,7 @@ class Certificate { bool is_fec) const; tl_object_ptr tl() const; const PublicKey &issuer() const; - const PublicKeyHash issuer_hash() const; + PublicKeyHash issuer_hash() const; static td::Result> create(tl_object_ptr cert); static tl_object_ptr empty_tl(); @@ -170,6 +170,8 @@ class Overlays : public td::actor::Actor { td::Promise promise) { promise.set_value(td::Unit()); } + virtual void on_remove_peer(adnl::AdnlNodeIdShort src) { + } virtual ~Callback() = default; }; @@ -195,7 +197,8 @@ class Overlays : public td::actor::Actor { virtual void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) = 0; virtual void create_public_overlay_external(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, - OverlayPrivacyRules rules, td::string scope) = 0; + std::unique_ptr callback, OverlayPrivacyRules rules, + td::string scope) = 0; virtual void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules) = 0; diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 837e57541..733c034a3 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -211,6 +211,8 @@ overlay.message overlay:int256 = overlay.Message; //overlay.randomPeers peers:(vector adnl.node) = overlay.RandomPeers; overlay.broadcastList hashes:(vector int256) = overlay.BroadcastList; +overlay.message.removePeer = overlay.message.InternalMessage; + overlay.fec.received hash:int256 = overlay.Broadcast; overlay.fec.completed hash:int256 = overlay.Broadcast; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 36c21e21f7ebcbdc819c4ab0da29bca718a36353..c62bf28ef2a0a6bcd2314d8069c5af25cad09281 100644 GIT binary patch delta 132 zcmaE`f#tzQ7T!m*^{p77;NnJJZCTkTGgp6=&o4_Y%1Nx$%S|mVPE1eL^UNzrEy_#G zncOQazu7{Thl|m7v#a{=HVcqyDRk9EskuO*fYj6?29W&CV?sV4D~g=ibU19=4uZrs Jo2=8x0s!WpGt>Y8 delta 41 scmaE`k>$Y#7T!m*^{p77;KD{;ZQ0EwvOHXjKARiWf46NuuudTh030t4ApigX diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index a4eac57a9..fea4916df 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -25,7 +25,6 @@ #include "ton/ton-shard.h" #include "ton/ton-tl.hpp" -#include "ton/ton-io.hpp" #include "adnl/utils.hpp" #include "net/download-block-new.hpp" @@ -71,38 +70,41 @@ void Neighbour::update_roundtrip(double t) { } void FullNodeShardImpl::create_overlay() { - if (active_) { - class Callback : public overlay::Overlays::Callback { - public: - void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, - td::BufferSlice data) override { - // just ignore - } - void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, - td::Promise promise) override { - td::actor::send_closure(node_, &FullNodeShardImpl::receive_query, src, std::move(data), std::move(promise)); - } - void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { - td::actor::send_closure(node_, &FullNodeShardImpl::receive_broadcast, src, std::move(data)); - } - void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, - td::Promise promise) override { - td::actor::send_closure(node_, &FullNodeShardImpl::check_broadcast, src, std::move(data), std::move(promise)); - } - Callback(td::actor::ActorId node) : node_(node) { - } + class Callback : public overlay::Overlays::Callback { + public: + void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, + td::BufferSlice data) override { + // just ignore + } + void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + td::actor::send_closure(node_, &FullNodeShardImpl::receive_query, src, std::move(data), std::move(promise)); + } + void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { + td::actor::send_closure(node_, &FullNodeShardImpl::receive_broadcast, src, std::move(data)); + } + void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + td::actor::send_closure(node_, &FullNodeShardImpl::check_broadcast, src, std::move(data), std::move(promise)); + } + void on_remove_peer(adnl::AdnlNodeIdShort src) override { + td::actor::send_closure(node_, &FullNodeShardImpl::remove_neighbour, src); + } + Callback(td::actor::ActorId node) : node_(node) { + } - private: - td::actor::ActorId node_; - }; + private: + td::actor::ActorId node_; + }; + if (active_) { td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay, adnl_id_, overlay_id_full_.clone(), std::make_unique(actor_id(this)), rules_, PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() << ", \"workchain_id\": " << get_workchain() << " }"); } else { td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay_external, adnl_id_, - overlay_id_full_.clone(), rules_, + overlay_id_full_.clone(), std::make_unique(actor_id(this)), rules_, PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() << ", \"workchain_id\": " << get_workchain() << " }"); } @@ -124,6 +126,10 @@ void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broad std::move(q->message_->data_), std::move(promise)); } +void FullNodeShardImpl::remove_neighbour(adnl::AdnlNodeIdShort id) { + neighbours_.erase(id); +} + void FullNodeShardImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) { td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); adnl_id_ = adnl_id; diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index ead8cafd5..a9b761815 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -145,6 +145,7 @@ class FullNodeShardImpl : public FullNodeShard { void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); void receive_broadcast(PublicKeyHash src, td::BufferSlice query); void check_broadcast(PublicKeyHash src, td::BufferSlice query, td::Promise promise); + void remove_neighbour(adnl::AdnlNodeIdShort id); void send_ihr_message(td::BufferSlice data) override; void send_external_message(td::BufferSlice data) override; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 46fe88709..0477b6487 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -168,6 +168,8 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) = 0; + virtual void validated_new_block(BlockIdExt block_id) = 0; + static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) { return ts / (1 << 17) != prev_ts / (1 << 17); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 3c45b48aa..53467ff32 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -375,6 +375,8 @@ class ValidatorManagerImpl : public ValidatorManager { void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override { UNREACHABLE(); } + void validated_new_block(BlockIdExt block_id) override { + } void get_validator_sessions_info( td::Promise> promise) override { UNREACHABLE(); diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index bc8913b89..9aa302263 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -435,6 +435,8 @@ class ValidatorManagerImpl : public ValidatorManager { void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override { UNREACHABLE(); } + void validated_new_block(BlockIdExt block_id) override { + } void get_validator_sessions_info( td::Promise> promise) override { UNREACHABLE(); diff --git a/validator/manager.cpp b/validator/manager.cpp index c2bb70619..04edefdc6 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -418,7 +418,7 @@ void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { } void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { - if (!is_collator()) { + if (!is_collator() && !is_validator()) { return; } if (!last_masterchain_block_handle_) { @@ -1794,6 +1794,7 @@ void ValidatorManagerImpl::update_shards() { opts.proto_version = std::max(opts.proto_version, 1); } auto opts_hash = opts.get_hash(); + extra_active_shards_.clear(); std::map> new_shards; std::set future_shards; @@ -1848,6 +1849,11 @@ void ValidatorManagerImpl::update_shards() { default: LOG(FATAL) << "state=" << static_cast(v->fsm_state()); } + cleanup_last_validated_blocks(v->top_block_id().id); + } + + for (const auto& s : last_validated_blocks_) { + extra_active_shards_.insert(s.first); } new_shards.emplace(ShardIdFull{masterchainId, shardIdAll}, std::vector{last_masterchain_block_id_}); @@ -1920,6 +1926,7 @@ void ValidatorManagerImpl::update_shards() { auto validator_id = get_validator(shard, val_set); if (!validator_id.is_zero()) { + extra_active_shards_.insert(shard); auto val_group_id = get_validator_set_id(shard, val_set, opts_hash, key_seqno, opts); if (force_recover) { @@ -2012,7 +2019,25 @@ void ValidatorManagerImpl::update_shards() { }); td::actor::send_closure(db_, &Db::update_destroyed_validator_sessions, gc_list_, std::move(P)); } -} // namespace validator +} + +void ValidatorManagerImpl::cleanup_last_validated_blocks(BlockId new_block) { + auto process_shard = [&, this](ShardIdFull shard) { + auto it = last_validated_blocks_.find(shard); + if (it != last_validated_blocks_.end() && it->second < new_block.seqno) { + last_validated_blocks_.erase(it); + } + }; + ShardIdFull shard = new_block.shard_full(); + process_shard(shard); + if (shard.pfx_len() > 0) { + process_shard(shard_parent(shard)); + } + if (shard.pfx_len() < max_shard_pfx_len) { + process_shard(shard_child(shard, true)); + process_shard(shard_child(shard, false)); + } +} void ValidatorManagerImpl::written_destroyed_validator_sessions(std::vector> list) { for (auto &v : list) { @@ -2428,6 +2453,7 @@ void ValidatorManagerImpl::get_shard_client_state(bool from_db, td::Promise state, std::set shards_to_monitor) { shards_to_monitor_ = shards_to_monitor; + shards_to_monitor.insert(extra_active_shards_.begin(), extra_active_shards_.end()); callback_->update_shard_configuration(std::move(state), std::move(shards_to_monitor)); } diff --git a/validator/manager.hpp b/validator/manager.hpp index 9e55abf58..c65914aae 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -543,6 +543,11 @@ class ValidatorManagerImpl : public ValidatorManager { void get_validator_sessions_info( td::Promise> promise) override; + void validated_new_block(BlockIdExt block_id) override { + BlockSeqno &last = last_validated_blocks_[block_id.shard_full()]; + last = std::max(last, block_id.seqno()); + } + void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; private: @@ -606,13 +611,17 @@ class ValidatorManagerImpl : public ValidatorManager { double max_mempool_num() const { return opts_->max_mempool_num(); } + void cleanup_last_validated_blocks(BlockId new_block); private: + std::map> shard_client_waiters_; std::map> collator_nodes_; std::set shards_to_monitor_ = {ShardIdFull(masterchainId)}; + std::set extra_active_shards_; + std::map last_validated_blocks_; }; } // namespace validator diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 6c0e242d5..a72bded10 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -59,10 +59,14 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat approved_candidates_cache_round_ = round_id; approved_candidates_cache_.clear(); } + auto next_block_id = create_next_block_id(block.id.root_hash, block.id.file_hash); + block.id = next_block_id; + CacheKey cache_key = block_to_cache_key(block); auto it = approved_candidates_cache_.find(cache_key); if (it != approved_candidates_cache_.end()) { promise.set_result(it->second); + return; } auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), round_id, block = block.clone(), @@ -96,9 +100,7 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat P.set_error(td::Status::Error(ErrorCode::notready, "validator group not started")); return; } - auto next_block_id = create_next_block_id(block.id.root_hash, block.id.file_hash); VLOG(VALIDATOR_DEBUG) << "validating block candidate " << next_block_id; - block.id = next_block_id; run_validate_query(shard_, min_masterchain_block_id_, prev_block_ids_, std::move(block), validator_set_, manager_, td::Timestamp::in(10.0), std::move(P), lite_mode_ ? ValidateMode::lite : 0); } @@ -140,7 +142,10 @@ void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash s void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref block, std::vector prev, td::Ref sig_set, td::Ref approve_sig_set, - bool send_broadcast, td::Promise promise) { + bool send_broadcast, td::Promise promise, bool is_retry) { + if (!is_retry) { + td::actor::send_closure(manager_, &ValidatorManager::validated_new_block, block_id); + } auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { @@ -150,7 +155,7 @@ void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref } LOG_CHECK(R.error().code() == ErrorCode::timeout || R.error().code() == ErrorCode::notready) << R.move_as_error(); td::actor::send_closure(SelfId, &ValidatorGroup::accept_block_query, block_id, std::move(block), std::move(prev), - std::move(sig_set), std::move(approve_sig_set), false, std::move(promise)); + std::move(sig_set), std::move(approve_sig_set), false, std::move(promise), true); } else { promise.set_value(R.move_as_ok()); } diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 76e6ea992..4614ea12a 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -41,7 +41,7 @@ class ValidatorGroup : public td::actor::Actor { void skip_round(td::uint32 round); void accept_block_query(BlockIdExt block_id, td::Ref block, std::vector prev, td::Ref sigs, td::Ref approve_sigs, - bool send_broadcast, td::Promise promise); + bool send_broadcast, td::Promise promise, bool is_retry = false); void get_approved_candidate(PublicKey source, RootHash root_hash, FileHash file_hash, FileHash collated_data_file_hash, td::Promise promise); BlockId create_next_block_id_simple() const; From 910398da92af2cbabef208755e4cc90c0e572308 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 12 Aug 2022 12:10:46 +0300 Subject: [PATCH 021/388] Improved neighbor choosing in full-node-shard --- create-hardfork/create-hardfork.cpp | 3 +- dht-server/dht-server.cpp | 2 +- dht-server/dht-server.hpp | 2 +- overlay/overlay.cpp | 4 +- test/test-ton-collator.cpp | 3 +- tl/generate/scheme/ton_api.tl | 4 +- tl/generate/scheme/ton_api.tlo | Bin 71904 -> 72144 bytes ton/ton-tl.hpp | 10 +- validator-engine/validator-engine.cpp | 4 +- validator-engine/validator-engine.hpp | 2 +- validator/full-node-shard.cpp | 135 ++++++++++++++++++-------- validator/full-node-shard.h | 11 ++- validator/full-node-shard.hpp | 36 ++++--- validator/full-node.cpp | 61 ++++++------ validator/full-node.hpp | 7 +- validator/manager.cpp | 3 +- validator/validator.h | 3 +- 17 files changed, 185 insertions(+), 105 deletions(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index a24b7fc18..3fb57f41b 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -237,7 +237,8 @@ class HardforkCreator : public td::actor::Actor { td::PromiseCreator::lambda([](td::Unit) {})); } void update_shard_configuration(td::Ref state, - std::set shards_to_monitor) override { + std::set shards_to_monitor, + std::set temporary_shards) override { } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } diff --git a/dht-server/dht-server.cpp b/dht-server/dht-server.cpp index d1f439534..3bd65cc6e 100644 --- a/dht-server/dht-server.cpp +++ b/dht-server/dht-server.cpp @@ -57,7 +57,7 @@ Config::Config() { out_port = 3278; } -Config::Config(const ton::ton_api::engine_validator_config_v2 &config) { +Config::Config(const ton::ton_api::engine_validator_configV2 &config) { out_port = static_cast(config.out_port_); if (!out_port) { out_port = 3278; diff --git a/dht-server/dht-server.hpp b/dht-server/dht-server.hpp index 724e7bbfa..724232fa7 100644 --- a/dht-server/dht-server.hpp +++ b/dht-server/dht-server.hpp @@ -94,7 +94,7 @@ struct Config { ton::tl_object_ptr tl() const; Config(); - Config(const ton::ton_api::engine_validator_config_v2 &config); + Config(const ton::ton_api::engine_validator_configV2 &config); }; class DhtServer : public td::actor::Actor { diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index ff86d837e..8e35e2a25 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -235,12 +235,12 @@ void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice dat auto X = fetch_tl_object(data.clone(), true); if (X.is_error()) { auto Y = fetch_tl_object(data.clone(), true); - if (Y.is_ok()) { + if (Y.is_ok() && public_) { VLOG(OVERLAY_DEBUG) << this << ": received removePeer message from " << src; if (peers_.exists(src)) { del_peer(src); - callback_->on_remove_peer(src); } + callback_->on_remove_peer(src); return; } } diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index d9ec82e5b..8ef469b5f 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -324,7 +324,8 @@ class TestNode : public td::actor::Actor { td::PromiseCreator::lambda([](td::Unit) {})); } void update_shard_configuration(td::Ref state, - std::set shards_to_monitor) override { + std::set shards_to_monitor, + std::set temporary_shards) override { } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 733c034a3..39cfb5eec 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -407,6 +407,7 @@ tonNode.dataFull id:tonNode.blockIdExt proof:bytes block:bytes is_link:Bool = to tonNode.dataFullEmpty = tonNode.DataFull; tonNode.capabilities version:int capabilities:long = tonNode.Capabilities; +tonNode.capabilitiesV2 version:int capabilities:long have_state:Bool = tonNode.Capabilities; tonNode.success = tonNode.Success; @@ -450,6 +451,7 @@ tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data; tonNode.getOutMsgQueueProof block_id:tonNode.blockIdExt dst_workchain:int dst_shard:long = tonNode.OutMsgQueueProof; tonNode.getCapabilities = tonNode.Capabilities; +tonNode.getCapabilitiesV2 = tonNode.Capabilities; tonNode.slave.sendExtMessage message:tonNode.externalMessage = tonNode.Success; @@ -598,7 +600,7 @@ engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector eng liteservers:(vector engine.liteServer) control:(vector engine.controlInterface) gc:engine.gc = engine.validator.Config; -engine.validator.config_v2 out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) +engine.validator.configV2 out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) dht:(vector engine.dht) validators:(vector engine.validator) collators:(vector engine.collator) fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave) diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index c62bf28ef2a0a6bcd2314d8069c5af25cad09281..413fbd8769514b2bed2ffc52cc34f4679ea70c69 100644 GIT binary patch delta 319 zcmaE`k>$c>mJL0^EXSne_%=@v*5GBC(A5|+d5?m`LECku9oaHnOahZ!+2fX#|&U%&*>)74lvKUs#Dc`6%$&@U%+z9_i7?&FP~9q%1@5wPWh9oR z#ut|)mZVOWpCt=+_vQx&dsG;cH|IWEuNec=1+t~cnN5eow(TIgt%)9Nk0oKYrl*!T Rqgo8N8QqNSCIXCWH~=yFY=r;- delta 171 zcmcbxndQMomJL0^EI)Q6?Akm*Sc8}4K*Ij7llLe{OkTn%vH6X{6gEcR&5i2Sv0x>e zS~ZwNL24PG;Hn%~Ois7K3#rM1T_W6Rnd$LmMhsxXV%irlLG*Mrm~(pH;+YW-nwyOv-uHgUxU~NcV diff --git a/ton/ton-tl.hpp b/ton/ton-tl.hpp index f6bda4d92..5df54ec65 100644 --- a/ton/ton-tl.hpp +++ b/ton/ton-tl.hpp @@ -62,12 +62,12 @@ inline tl_object_ptr create_tl_shard_id(const ShardIdF return create_tl_object(s.workchain, s.shard); } -inline tl_object_ptr unpack_engine_validator_config( +inline tl_object_ptr unpack_engine_validator_config( tl_object_ptr config) { - tl_object_ptr res; + tl_object_ptr res; ton_api::downcast_call(*config, td::overloaded( [&](ton_api::engine_validator_config &c) { - res = create_tl_object( + res = create_tl_object( c.out_port_, std::move(c.addrs_), std::move(c.adnl_), std::move(c.dht_), std::move(c.validators_), std::vector>(), c.fullnode_, @@ -75,8 +75,8 @@ inline tl_object_ptr unpack_engine_validato std::move(c.liteservers_), std::move(c.control_), std::vector>(), std::move(c.gc_)); }, - [&](ton_api::engine_validator_config_v2 &c) { - res = std::make_unique(std::move(c)); + [&](ton_api::engine_validator_configV2 &c) { + res = std::make_unique(std::move(c)); })); return res; } diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 65fb0e8f2..c0f85d039 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -75,7 +75,7 @@ Config::Config() { full_node = ton::PublicKeyHash::zero(); } -Config::Config(const ton::ton_api::engine_validator_config_v2 &config) { +Config::Config(const ton::ton_api::engine_validator_configV2 &config) { full_node = ton::PublicKeyHash::zero(); out_port = static_cast(config.out_port_); if (!out_port) { @@ -263,7 +263,7 @@ ton::tl_object_ptr Config::tl() const { full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec)); } else { - return ton::create_tl_object( + return ton::create_tl_object( out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec), full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(liteserver_vec), std::move(control_vec), std::move(shards_vec), std::move(gc_vec)); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 47d612b7b..b5d770a4e 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -140,7 +140,7 @@ struct Config { ton::tl_object_ptr tl() const; Config(); - Config(const ton::ton_api::engine_validator_config_v2 &config); + Config(const ton::ton_api::engine_validator_configV2 &config); }; class ValidatorEngine : public td::actor::Actor { diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index fea4916df..4bd7bf362 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -46,11 +46,25 @@ namespace validator { namespace fullnode { +static const td::uint32 PROTO_CAPABILITIES_V2 = 2; + Neighbour Neighbour::zero = Neighbour{adnl::AdnlNodeIdShort::zero()}; -void Neighbour::update_proto_version(const ton_api::tonNode_capabilities &q) { - proto_version = q.version_; - capabilities = q.capabilities_; +void Neighbour::update_proto_version(ton_api::tonNode_Capabilities &q) { + ton_api::downcast_call(q, td::overloaded( + [&](ton_api::tonNode_capabilities &x) { + proto_version = x.version_; + capabilities = x.capabilities_; + if (capabilities < PROTO_CAPABILITIES_V2) { + has_state_known = has_state = true; + } + }, + [&](ton_api::tonNode_capabilitiesV2 &x) { + proto_version = x.version_; + capabilities = x.capabilities_; + has_state_known = true; + has_state = x.have_state_; + })); } void Neighbour::query_success(double t) { @@ -97,7 +111,7 @@ void FullNodeShardImpl::create_overlay() { td::actor::ActorId node_; }; - if (active_) { + if (is_active()) { td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay, adnl_id_, overlay_id_full_.clone(), std::make_unique(actor_id(this)), rules_, PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() @@ -137,13 +151,16 @@ void FullNodeShardImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promis create_overlay(); } -void FullNodeShardImpl::set_active(bool active) { - if (active_ == active || shard_.is_masterchain()) { +void FullNodeShardImpl::set_mode(FullNodeShardMode mode) { + if (shard_.is_masterchain()) { return; } - td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); - active_ = active; - create_overlay(); + bool was_active = is_active(); + mode_ = mode; + if (was_active != is_active()) { + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); + create_overlay(); + } } void FullNodeShardImpl::try_get_next_block(td::Timestamp timeout, td::Promise promise) { @@ -562,6 +579,12 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod promise.set_value(create_serialize_tl_object(proto_version(), proto_capabilities())); } +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilitiesV2 &query, + td::Promise promise) { + promise.set_value(create_serialize_tl_object(proto_version(), proto_capabilities(), + mode_ == FullNodeShardMode::active)); +} + void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, td::Promise promise) { auto P = td::PromiseCreator::lambda( @@ -609,7 +632,7 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise) { - if (!active_) { + if (!is_active()) { promise.set_error(td::Status::Error("shard is inactive")); return; } @@ -671,7 +694,7 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_bl } void FullNodeShardImpl::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { - if (!active_) { + if (!is_active()) { return; } auto B = fetch_tl_object(std::move(broadcast), true); @@ -801,7 +824,7 @@ void FullNodeShardImpl::download_block_proof_link(BlockIdExt block_id, td::uint3 td::Promise promise) { auto &b = choose_neighbour(); td::actor::create_actor("downloadproofreq", block_id, true, false, adnl_id_, overlay_id_, - adnl::AdnlNodeIdShort::zero(), priority, timeout, validator_manager_, rldp_, + b.adnl_id, priority, timeout, validator_manager_, rldp_, overlays_, adnl_, client_, create_neighbour_promise(b, std::move(promise))) .release(); } @@ -809,7 +832,7 @@ void FullNodeShardImpl::download_block_proof_link(BlockIdExt block_id, td::uint3 void FullNodeShardImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) { auto &b = choose_neighbour(); - td::actor::create_actor("next", block_id, 16, adnl_id_, overlay_id_, adnl::AdnlNodeIdShort::zero(), + td::actor::create_actor("next", block_id, 16, adnl_id_, overlay_id_, b.adnl_id, 1, timeout, validator_manager_, rldp_, overlays_, adnl_, client_, create_neighbour_promise(b, std::move(promise))) .release(); @@ -817,10 +840,10 @@ void FullNodeShardImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp t void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) { - auto &b = choose_neighbour(); + auto &b = choose_neighbour(true); td::actor::create_actor( - "archive", masterchain_seqno, std::move(tmp_dir), adnl_id_, overlay_id_, adnl::AdnlNodeIdShort::zero(), timeout, - validator_manager_, rldp_, overlays_, adnl_, client_, create_neighbour_promise(b, std::move(promise))) + "archive", masterchain_seqno, std::move(tmp_dir), adnl_id_, overlay_id_, b.adnl_id, timeout, + validator_manager_, rldp_, overlays_, adnl_, client_, create_neighbour_promise(b, std::move(promise), true)) .release(); } @@ -828,9 +851,9 @@ void FullNodeShardImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardI td::Promise> promise) { // TODO: maybe more complex download (like other requests here) // TODO: estimate max size - auto &b = choose_neighbour(); + auto &b = choose_neighbour(true); auto P = td::PromiseCreator::lambda( - [=, promise = create_neighbour_promise(b, std::move(promise))](td::Result R) mutable { + [=, promise = create_neighbour_promise(b, std::move(promise), true)](td::Result R) mutable { if (R.is_error()) { promise.set_result(R.move_as_error()); return; @@ -1026,11 +1049,21 @@ void FullNodeShardImpl::got_neighbours(std::vector vec) { continue; } if (neighbours_.size() == max_neighbours()) { + td::uint32 neighbours_with_state = 0; + for (const auto &n : neighbours_) { + if (n.second.has_state) { + ++neighbours_with_state; + } + } + adnl::AdnlNodeIdShort a = adnl::AdnlNodeIdShort::zero(); adnl::AdnlNodeIdShort b = adnl::AdnlNodeIdShort::zero(); td::uint32 cnt = 0; double u = 0; for (auto &n : neighbours_) { + if (neighbours_with_state <= min_neighbours_with_state() && n.second.has_state) { + continue; + } if (n.second.unreliability > u) { u = n.second.unreliability; a = n.first; @@ -1054,34 +1087,47 @@ void FullNodeShardImpl::got_neighbours(std::vector vec) { } } -const Neighbour &FullNodeShardImpl::choose_neighbour() const { +const Neighbour &FullNodeShardImpl::choose_neighbour(bool require_state) const { if (neighbours_.size() == 0) { return Neighbour::zero; } - const Neighbour *best = nullptr; - td::uint32 sum = 0; + for (int attempt = 0; attempt < (require_state ? 2 : 1); ++attempt) { + const Neighbour *best = nullptr; + td::uint32 sum = 0; - for (auto &x : neighbours_) { - td::uint32 unr = static_cast(x.second.unreliability); + for (auto &x : neighbours_) { + if (require_state) { + if (attempt == 0 && !x.second.has_state) { + continue; + } + if (attempt == 1 && x.second.has_state_known) { + continue; + } + } + auto unr = static_cast(x.second.unreliability); - if (x.second.proto_version < proto_version()) { - unr += 4; - } else if (x.second.proto_version == proto_version() && x.second.capabilities < proto_capabilities()) { - unr += 2; - } + if (x.second.proto_version < proto_version()) { + unr += 4; + } else if (x.second.proto_version == proto_version() && x.second.capabilities < proto_capabilities()) { + unr += 2; + } - auto f = static_cast(fail_unreliability()); + auto f = static_cast(fail_unreliability()); - if (unr <= f) { - auto w = 1 << (f - unr); - sum += w; - if (td::Random::fast(0, sum - 1) <= w - 1) { - best = &x.second; + if (unr <= f) { + auto w = 1 << (f - unr); + sum += w; + if (td::Random::fast(0, sum - 1) <= w - 1) { + best = &x.second; + } } } + if (best) { + return *best; + } } - return best ? *best : Neighbour::zero; + return Neighbour::zero; } void FullNodeShardImpl::update_neighbour_stats(adnl::AdnlNodeIdShort adnl_id, double t, bool success) { @@ -1100,11 +1146,11 @@ void FullNodeShardImpl::got_neighbour_capabilities(adnl::AdnlNodeIdShort adnl_id if (it == neighbours_.end()) { return; } - auto F = fetch_tl_object(std::move(data), true); + auto F = fetch_tl_object(std::move(data), true); if (F.is_error()) { it->second.query_failed(); } else { - it->second.update_proto_version(*F.move_as_ok().get()); + it->second.update_proto_version(*F.ok()); it->second.query_success(t); } } @@ -1133,7 +1179,12 @@ void FullNodeShardImpl::ping_neighbours() { td::Time::now() - start_time, R.move_as_ok()); } }); - auto q = create_serialize_tl_object(); + td::BufferSlice q; + if (it->second.capabilities >= PROTO_CAPABILITIES_V2) { + q = create_serialize_tl_object(); + } else { + q = create_serialize_tl_object(); + } td::actor::send_closure(overlays_, &overlay::Overlays::send_query, it->first, adnl_id_, overlay_id_, "get_prepare_block", std::move(P), td::Timestamp::in(1.0), std::move(q)); @@ -1148,7 +1199,7 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, bool active) + td::actor::ActorId client, FullNodeShardMode mode) : shard_(shard) , local_id_(local_id) , adnl_id_(adnl_id) @@ -1159,7 +1210,7 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, , overlays_(overlays) , validator_manager_(validator_manager) , client_(client) - , active_(shard.is_masterchain() || active) { + , mode_(shard.is_masterchain() ? FullNodeShardMode::active : mode) { } td::actor::ActorOwn FullNodeShard::create( @@ -1167,9 +1218,9 @@ td::actor::ActorOwn FullNodeShard::create( td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, - bool active) { + FullNodeShardMode mode) { return td::actor::create_actor("tonnode", shard, local_id, adnl_id, zero_state_file_hash, keyring, - adnl, rldp, overlays, validator_manager, client, active); + adnl, rldp, overlays, validator_manager, client, mode); } } // namespace fullnode diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 7da46af18..e24873aac 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -28,6 +28,13 @@ namespace validator { namespace fullnode { +enum FullNodeShardMode { + active, // Node can answer queries about the shard + active_temp, // Like 'active', but queries about shard state are not allowed (only blocks) + inactive // Node is not a part of the overlay (overlay is_external) +}; + + class FullNodeShard : public td::actor::Actor { public: virtual ~FullNodeShard() = default; @@ -36,7 +43,7 @@ class FullNodeShard : public td::actor::Actor { virtual ShardIdFull get_shard_full() const = 0; virtual void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) = 0; - virtual void set_active(bool active) = 0; + virtual void set_mode(FullNodeShardMode mode) = 0; virtual void send_ihr_message(td::BufferSlice data) = 0; virtual void send_external_message(td::BufferSlice data) = 0; @@ -74,7 +81,7 @@ class FullNodeShard : public td::actor::Actor { td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, - bool active = true); + FullNodeShardMode mode = FullNodeShardMode::active); }; } // namespace fullnode diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index a9b761815..cd7c20a95 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -36,10 +36,12 @@ struct Neighbour { double roundtrip_relax_at = 0; double roundtrip_weight = 0; double unreliability = 0; + bool has_state_known = false; + bool has_state = false; Neighbour(adnl::AdnlNodeIdShort adnl_id) : adnl_id(std::move(adnl_id)) { } - void update_proto_version(const ton_api::tonNode_capabilities &q); + void update_proto_version(ton_api::tonNode_Capabilities &q); void query_success(double t); void query_failed(); void update_roundtrip(double t); @@ -66,11 +68,14 @@ class FullNodeShardImpl : public FullNodeShard { return 2; } static constexpr td::uint64 proto_capabilities() { - return 1; + return 2; } static constexpr td::uint32 max_neighbours() { return 16; } + static constexpr td::uint32 min_neighbours_with_state() { + return 10; + } static constexpr double stop_unreliability() { return 5.0; } @@ -80,11 +85,8 @@ class FullNodeShardImpl : public FullNodeShard { void create_overlay(); void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; - void set_active(bool active) override; + void set_mode(FullNodeShardMode mode) override; - //td::Result fetch_block(td::BufferSlice data); - void prevalidate_block(BlockIdExt block_id, td::BufferSlice data, td::BufferSlice proof, - td::Promise promise); void try_get_next_block(td::Timestamp timestamp, td::Promise promise); void got_next_block(td::Result block); void get_next_block(); @@ -129,6 +131,8 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilitiesV2 &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, @@ -189,14 +193,16 @@ class FullNodeShardImpl : public FullNodeShard { void got_neighbours(std::vector res); void update_neighbour_stats(adnl::AdnlNodeIdShort adnl_id, double t, bool success); void got_neighbour_capabilities(adnl::AdnlNodeIdShort adnl_id, double t, td::BufferSlice data); - const Neighbour &choose_neighbour() const; + const Neighbour &choose_neighbour(bool require_state = false) const; template - td::Promise create_neighbour_promise(const Neighbour &x, td::Promise p) { - return td::PromiseCreator::lambda([id = x.adnl_id, SelfId = actor_id(this), p = std::move(p), - ts = td::Time::now()](td::Result R) mutable { + td::Promise create_neighbour_promise(const Neighbour &x, td::Promise p, bool require_state = false) { + return td::PromiseCreator::lambda([id = x.adnl_id, SelfId = actor_id(this), p = std::move(p), ts = td::Time::now(), + ignore_error = require_state && !x.has_state_known](td::Result R) mutable { if (R.is_error() && R.error().code() != ErrorCode::notready && R.error().code() != ErrorCode::cancelled) { - td::actor::send_closure(SelfId, &FullNodeShardImpl::update_neighbour_stats, id, td::Time::now() - ts, false); + if (!ignore_error) { + td::actor::send_closure(SelfId, &FullNodeShardImpl::update_neighbour_stats, id, td::Time::now() - ts, false); + } } else { td::actor::send_closure(SelfId, &FullNodeShardImpl::update_neighbour_stats, id, td::Time::now() - ts, true); } @@ -209,13 +215,17 @@ class FullNodeShardImpl : public FullNodeShard { td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, bool active = true); + td::actor::ActorId client, FullNodeShardMode mode = FullNodeShardMode::active); private: bool use_new_download() const { return false; } + bool is_active() const { + return mode_ != FullNodeShardMode::inactive; + } + ShardIdFull shard_; BlockHandle handle_; td::Promise promise_; @@ -247,7 +257,7 @@ class FullNodeShardImpl : public FullNodeShard { td::Timestamp ping_neighbours_at_; adnl::AdnlNodeIdShort last_pinged_neighbour_ = adnl::AdnlNodeIdShort::zero(); - bool active_ = false; + FullNodeShardMode mode_; }; } // namespace fullnode diff --git a/validator/full-node.cpp b/validator/full-node.cpp index e4a2cf576..6792f969f 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -132,14 +132,15 @@ void FullNodeImpl::initial_read_complete(BlockHandle top_handle) { td::actor::send_closure(it->second.actor, &FullNodeShard::set_handle, top_handle, std::move(P)); } -void FullNodeImpl::update_shard_configuration(td::Ref state, - std::set shards_to_monitor) { +void FullNodeImpl::update_shard_configuration(td::Ref state, std::set shards_to_monitor, + std::set temporary_shards) { + CHECK(shards_to_monitor.count(ShardIdFull(masterchainId))); std::map new_shards; - std::set new_active; + std::map new_active; new_shards[ShardIdFull(masterchainId)] = state->get_block_id(); std::set workchains; - auto set_active = [&](ShardIdFull shard) { - while (new_active.insert(shard).second && shard.pfx_len() > 0) { + auto set_active = [&](ShardIdFull shard, FullNodeShardMode mode) { + while (new_active.emplace(shard, mode).second && shard.pfx_len() > 0) { shard = shard_parent(shard); } }; @@ -155,20 +156,24 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, } } for (ShardIdFull shard : shards_to_monitor) { - set_active(shard); + set_active(shard, FullNodeShardMode::active); + } + for (ShardIdFull shard : temporary_shards) { + set_active(shard, FullNodeShardMode::active_temp); } - auto info_set_active = [&](ShardIdFull shard, ShardInfo& info, bool active) { - if (info.active == active) { + auto info_set_mode = [&](ShardIdFull shard, ShardInfo& info, FullNodeShardMode mode) { + if (info.mode == mode) { return; } if (info.actor.empty()) { - add_shard_actor(shard, active); + add_shard_actor(shard, mode); return; } - info.active = active; - td::actor::send_closure(info.actor, &FullNodeShard::set_active, active); - info.delete_at = active ? td::Timestamp::never() : td::Timestamp::in(INACTIVE_SHARD_TTL); + info.mode = mode; + td::actor::send_closure(info.actor, &FullNodeShard::set_mode, mode); + info.delete_at = + mode != FullNodeShardMode::inactive ? td::Timestamp::never() : td::Timestamp::in(INACTIVE_SHARD_TTL); }; for (auto shard : new_shards) { @@ -180,16 +185,17 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, ShardIdFull shard = p.first; ShardInfo &info = p.second; info.exists = new_shards.count(shard); - info_set_active(shard, info, new_active.count(shard)); + auto it = new_active.find(shard); + info_set_mode(shard, info, it == new_active.end() ? FullNodeShardMode::inactive : it->second); } - for (ShardIdFull shard : new_active) { - info_set_active(shard, shards_[shard], true); + for (const auto& s : new_active) { + info_set_mode(s.first, shards_[s.first], s.second); } auto it = shards_.begin(); while (it != shards_.end()) { - if (!it->second.active && it->second.delete_at && it->second.delete_at.is_in_past()) { + if (it->second.mode == FullNodeShardMode::inactive && it->second.delete_at && it->second.delete_at.is_in_past()) { it->second.actor.reset(); it->second.delete_at = td::Timestamp::never(); } @@ -201,15 +207,15 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, } } -void FullNodeImpl::add_shard_actor(ShardIdFull shard, bool active) { +void FullNodeImpl::add_shard_actor(ShardIdFull shard, FullNodeShardMode mode) { ShardInfo &info = shards_[shard]; if (!info.actor.empty()) { return; } info.actor = FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, keyring_, adnl_, rldp_, - overlays_, validator_manager_, client_, active); - info.active = active; - info.delete_at = active ? td::Timestamp::never() : td::Timestamp::in(INACTIVE_SHARD_TTL); + overlays_, validator_manager_, client_, mode); + info.mode = mode; + info.delete_at = mode != FullNodeShardMode::inactive ? td::Timestamp::never() : td::Timestamp::in(INACTIVE_SHARD_TTL); if (all_validators_.size() > 0) { td::actor::send_closure(info.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); } @@ -350,9 +356,9 @@ td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard, boo auto it = shards_.find(s); if (it != shards_.end() && it->second.exists) { if (it->second.actor.empty()) { - add_shard_actor(s, false); + add_shard_actor(s, FullNodeShardMode::inactive); } - if (!it->second.active) { + if (it->second.mode == FullNodeShardMode::inactive) { it->second.delete_at = td::Timestamp::in(INACTIVE_SHARD_TTL); } return it->second.actor.get(); @@ -365,9 +371,9 @@ td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard, boo } auto &info = shards_[shard]; if (info.actor.empty()) { - add_shard_actor(shard, false); + add_shard_actor(shard, FullNodeShardMode::inactive); } - if (!info.active) { + if (info.mode == FullNodeShardMode::inactive) { info.delete_at = td::Timestamp::in(INACTIVE_SHARD_TTL); } return info.actor.get(); @@ -488,9 +494,10 @@ void FullNodeImpl::start_up() { void initial_read_complete(BlockHandle handle) override { td::actor::send_closure(id_, &FullNodeImpl::initial_read_complete, handle); } - void update_shard_configuration(td::Ref state, std::set shards_to_monitor) override { + void update_shard_configuration(td::Ref state, std::set shards_to_monitor, + std::set temporary_shards) override { td::actor::send_closure(id_, &FullNodeImpl::update_shard_configuration, std::move(state), - std::move(shards_to_monitor)); + std::move(shards_to_monitor), std::move(temporary_shards)); } void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) override { td::actor::send_closure(id_, &FullNodeImpl::send_ihr_message, dst, std::move(data)); @@ -576,7 +583,7 @@ FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id , client_(client) , db_root_(db_root) , started_promise_(std::move(started_promise)) { - add_shard_actor(ShardIdFull{masterchainId}, true); + add_shard_actor(ShardIdFull{masterchainId}, FullNodeShardMode::active); } td::actor::ActorOwn FullNode::create( diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 2f8254587..fbe60d1b2 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -52,7 +52,8 @@ class FullNodeImpl : public FullNode { void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; - void update_shard_configuration(td::Ref state, std::set shards_to_monitor); + void update_shard_configuration(td::Ref state, std::set shards_to_monitor, + std::set temporary_shards); void sync_completed(); @@ -91,7 +92,7 @@ class FullNodeImpl : public FullNode { td::Promise started_promise); private: - void add_shard_actor(ShardIdFull shard, bool active); + void add_shard_actor(ShardIdFull shard, FullNodeShardMode mode); PublicKeyHash local_id_; adnl::AdnlNodeIdShort adnl_id_; @@ -103,7 +104,7 @@ class FullNodeImpl : public FullNode { struct ShardInfo { bool exists = false; td::actor::ActorOwn actor; - bool active = false; + FullNodeShardMode mode = FullNodeShardMode::inactive; td::Timestamp delete_at = td::Timestamp::never(); }; std::map shards_; diff --git a/validator/manager.cpp b/validator/manager.cpp index 04edefdc6..7e7e1e840 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2453,8 +2453,7 @@ void ValidatorManagerImpl::get_shard_client_state(bool from_db, td::Promise state, std::set shards_to_monitor) { shards_to_monitor_ = shards_to_monitor; - shards_to_monitor.insert(extra_active_shards_.begin(), extra_active_shards_.end()); - callback_->update_shard_configuration(std::move(state), std::move(shards_to_monitor)); + callback_->update_shard_configuration(std::move(state), std::move(shards_to_monitor), extra_active_shards_); } void ValidatorManagerImpl::update_async_serializer_state(AsyncSerializerState state, td::Promise promise) { diff --git a/validator/validator.h b/validator/validator.h index 545c8a81c..0595949fb 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -110,7 +110,8 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void initial_read_complete(BlockHandle top_masterchain_blocks) = 0; virtual void update_shard_configuration(td::Ref state, - std::set shards_to_monitor) = 0; + std::set shards_to_monitor, + std::set temporary_shards) = 0; virtual void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; From 597fd8443d877eae070aaebce85c22e2e4928730 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 12 Aug 2022 16:14:03 +0300 Subject: [PATCH 022/388] Collator nodes preload msg queues --- validator/collator-node.cpp | 52 +++++++++++++++++++++++++++++++------ validator/collator-node.hpp | 4 +++ validator/manager.cpp | 23 +++++++++++++++- validator/manager.hpp | 9 ++++++- 4 files changed, 78 insertions(+), 10 deletions(-) diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 84ef9eaa8..c11b01c82 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -58,6 +58,40 @@ void CollatorNode::add_shard(ShardIdFull shard) { shards_.push_back(shard); } +void CollatorNode::new_masterchain_block_notification(td::Ref state) { + std::vector top_blocks = {state->get_block_id()}; + std::vector next_shards; + if (collate_shard(ShardIdFull(masterchainId))) { + next_shards.push_back(ShardIdFull(masterchainId)); + } + for (const auto& desc : state->get_shards()) { + top_blocks.push_back(desc->top_block_id()); + ShardIdFull shard = desc->shard(); + if (desc->before_split()) { + if (collate_shard(shard_child(shard, true))) { + next_shards.push_back(shard_child(shard, true)); + } + if (collate_shard(shard_child(shard, false))) { + next_shards.push_back(shard_child(shard, false)); + } + } else if (desc->before_merge()) { + if (is_left_child(shard) && collate_shard(shard_parent(shard))) { + next_shards.push_back(shard_parent(shard)); + } + } else if (collate_shard(shard)) { + next_shards.push_back(shard); + } + } + for (const ShardIdFull& shard : next_shards) { + for (const BlockIdExt& neighbor : top_blocks) { + if (neighbor.shard_full() != shard && block::ShardConfig::is_neighbor(shard, neighbor.shard_full())) { + td::actor::send_closure(manager_, &ValidatorManager::wait_out_msg_queue_proof, neighbor, shard, 0, + td::Timestamp::in(10.0), [](td::Ref) {}); + } + } + } +} + static td::BufferSlice serialize_error(td::Status error) { return create_serialize_tl_object(error.code(), error.message().c_str()); } @@ -71,14 +105,7 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data if (!shard.is_valid_ext()) { return td::Status::Error(PSTRING() << "invalid shard " << shard.to_str()); } - bool found = false; - for (ShardIdFull our_shard : shards_) { - if (shard_is_ancestor(shard, our_shard)) { - found = true; - break; - } - } - if (!found) { + if (!collate_shard(shard)) { return td::Status::Error(PSTRING() << "this node doesn't collate shard " << shard.to_str()); } if (f->prev_blocks_.size() != 1 && f->prev_blocks_.size() != 2) { @@ -141,6 +168,15 @@ void CollatorNode::receive_query_cont(adnl::AdnlNodeIdShort src, ShardIdFull sha min_mc_state->get_validator_set(shard), manager_, td::Timestamp::in(10.0), std::move(P)); } +bool CollatorNode::collate_shard(ShardIdFull shard) const { + for (ShardIdFull our_shard : shards_) { + if (shard_is_ancestor(shard, our_shard)) { + return true; + } + } + return false; +} + } // namespace validator } // namespace ton diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp index 854e98ce2..b0a2ad903 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node.hpp @@ -33,12 +33,16 @@ class CollatorNode : public td::actor::Actor { void tear_down() override; void add_shard(ShardIdFull shard); + void new_masterchain_block_notification(td::Ref state); + private: void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); void receive_query_cont(adnl::AdnlNodeIdShort src, ShardIdFull shard, td::Ref min_mc_state, std::vector prev_blocks, Ed25519_PublicKey creator, td::Promise promise); + bool collate_shard(ShardIdFull shard) const; + adnl::AdnlNodeIdShort local_id_; td::actor::ActorId manager_; td::actor::ActorId adnl_; diff --git a/validator/manager.cpp b/validator/manager.cpp index 7e7e1e840..6323c4f2f 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -617,6 +617,9 @@ void ValidatorManagerImpl::wait_out_msg_queue_proof(BlockIdExt block_id, ShardId .release(); wait_out_msg_queue_proof_[key].actor_ = id; it = wait_out_msg_queue_proof_.find(key); + } else if (it->second.done_) { + promise.set_result(it->second.result_); + it->second.remove_at_ = td::Timestamp::in(30.0); } it->second.waiting_.emplace_back(timeout, priority, std::move(promise)); @@ -1079,13 +1082,16 @@ void ValidatorManagerImpl::finished_wait_msg_queue(BlockIdExt block_id, ShardIdF it->second.actor_ = id; return; } + wait_out_msg_queue_proof_.erase(it); } else { auto r = R.move_as_ok(); for (auto &X : it->second.waiting_) { X.promise.set_result(r); } + it->second.done_ = true; + it->second.result_ = std::move(r); + it->second.remove_at_ = td::Timestamp::in(30.0); } - wait_out_msg_queue_proof_.erase(it); } } @@ -1773,6 +1779,9 @@ void ValidatorManagerImpl::new_masterchain_block() { td::actor::send_closure(shard_client_, &ShardClient::new_masterchain_block_notification, last_masterchain_block_handle_, last_masterchain_state_); } + for (auto &c : collator_nodes_) { + td::actor::send_closure(c.second, &CollatorNode::new_masterchain_block_notification, last_masterchain_state_); + } if (last_masterchain_seqno_ % 1024 == 0) { LOG(WARNING) << "applied masterchain block " << last_masterchain_block_id_; @@ -2436,6 +2445,18 @@ void ValidatorManagerImpl::alarm() { } } alarm_timestamp().relax(check_shard_clients_); + if (cleanup_wait_caches_at_.is_in_past()) { + auto it = wait_out_msg_queue_proof_.begin(); + while (it != wait_out_msg_queue_proof_.end()) { + if (it->second.done_ && it->second.remove_at_.is_in_past()) { + it = wait_out_msg_queue_proof_.erase(it); + } else { + ++it; + } + } + cleanup_wait_caches_at_ = td::Timestamp::in(10.0); + } + alarm_timestamp().relax(cleanup_wait_caches_at_); } void ValidatorManagerImpl::update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) { diff --git a/validator/manager.hpp b/validator/manager.hpp index c65914aae..15598f4cd 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -181,10 +181,17 @@ class ValidatorManagerImpl : public ValidatorManager { waiting_.resize(j); } }; + template + struct WaitListCaching : public WaitList { + bool done_ = false; + ResType result_; + td::Timestamp remove_at_; + }; std::map>> wait_state_; std::map>> wait_block_data_; - std::map, WaitList>> + std::map, WaitListCaching>> wait_out_msg_queue_proof_; + td::Timestamp cleanup_wait_caches_at_ = td::Timestamp::now(); struct WaitBlockHandle { std::vector> waiting_; From 81d32ba5d64ffb6ab5bea99fb0ec794c2d93ebfd Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 15 Aug 2022 13:59:49 +0300 Subject: [PATCH 023/388] Out msg queue proof: send only the required part --- validator/full-node-shard.cpp | 4 ++ validator/impl/out-msg-queue-proof.cpp | 75 ++++++++++++++++++++++---- 2 files changed, 68 insertions(+), 11 deletions(-) diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 4bd7bf362..7bf44f33b 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -852,6 +852,10 @@ void FullNodeShardImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardI // TODO: maybe more complex download (like other requests here) // TODO: estimate max size auto &b = choose_neighbour(true); + if (b.adnl_id == adnl::AdnlNodeIdShort::zero()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "no nodes")); + return; + } auto P = td::PromiseCreator::lambda( [=, promise = create_neighbour_promise(b, std::move(promise), true)](td::Result R) mutable { if (R.is_error()) { diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 4a8731d39..4ba102c53 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -20,6 +20,8 @@ #include "vm/cells/MerkleProof.h" #include "common/delay.h" #include "interfaces/validator-manager.h" +#include "block/block-parse.h" +#include "block/block-auto.h" namespace ton { @@ -51,9 +53,31 @@ td::Result> OutMsgQueueProof::fetch(BlockIdExt block_i // Validate proof auto state_root = vm::CellSlice(vm::NoVm(), queue_proof).prefetch_ref(0); TRY_RESULT_PREFIX(state, ShardStateQ::fetch(block_id, {}, state_root), "invalid proof: "); - TRY_RESULT_PREFIX(queue, state->message_queue(), "invalid proof: "); - auto queue_root = queue->root_cell(); - if (queue_root->get_level() != 0) { + TRY_RESULT_PREFIX(outq_descr, state->message_queue(), "invalid proof: "); + + block::gen::OutMsgQueueInfo::Record qinfo; + if (!tlb::unpack_cell(outq_descr->root_cell(), qinfo)) { + return td::Status::Error("invalid proof: invalid message queue"); + } + td::Ref proc_info = qinfo.proc_info->prefetch_ref(0); + if (proc_info.not_null() && proc_info->get_level() != 0) { + return td::Status::Error("invalid proof: proc_info has prunned branches"); + } + td::Ref ihr_pending = qinfo.ihr_pending->prefetch_ref(0); + if (ihr_pending.not_null() && ihr_pending->get_level() != 0) { + return td::Status::Error("invalid proof: ihr_pending has prunned branches"); + } + auto queue = + std::make_unique(qinfo.out_queue->prefetch_ref(0), 352, block::tlb::aug_OutMsgQueue); + td::BitArray<96> prefix; + td::BitPtr ptr = prefix.bits(); + ptr.store_int(dst_shard.workchain, 32); + ptr.advance(32); + ptr.store_uint(dst_shard.shard, 64); + if (!queue->cut_prefix_subdict(prefix.bits(), 32 + dst_shard.pfx_len())) { + return td::Status::Error("invalid proof: failed to cut queue dict"); + } + if (queue->get_root_cell().not_null() && queue->get_root_cell()->get_level() != 0) { return td::Status::Error("invalid proof: msg queue has prunned branches"); } @@ -64,14 +88,20 @@ td::Result> OutMsgQueueProof::s ShardIdFull dst_shard, Ref state_root, Ref block_root) { + if (!dst_shard.is_valid_ext()) { + return td::Status::Error("invalid shard"); + } vm::MerkleProofBuilder mpb{std::move(state_root)}; TRY_RESULT(state, ShardStateQ::fetch(block_id, {}, mpb.root())); TRY_RESULT(outq_descr, state->message_queue()); + block::gen::OutMsgQueueInfo::Record qinfo; + if (!tlb::unpack_cell(outq_descr->root_cell(), qinfo)) { + return td::Status::Error("invalid message queue"); + } - // TODO: add only required part of msg queue td::HashSet visited; std::function)> dfs = [&](Ref cell) { - if (!visited.insert(cell->get_hash()).second) { + if (cell.is_null() || !visited.insert(cell->get_hash()).second) { return; } vm::CellSlice cs(vm::NoVm(), cell); @@ -79,16 +109,32 @@ td::Result> OutMsgQueueProof::s dfs(cs.prefetch_ref(i)); } }; - dfs(outq_descr->root_cell()); + auto dfs_cs = [&](const vm::CellSlice &cs) { + for (unsigned i = 0; i < cs.size_refs(); i++) { + dfs(cs.prefetch_ref(i)); + } + }; + dfs_cs(*qinfo.proc_info); + dfs_cs(*qinfo.ihr_pending); - TRY_RESULT(queue_proof, vm::std_boc_serialize(mpb.extract_proof())); + auto queue = + std::make_unique(qinfo.out_queue->prefetch_ref(0), 352, block::tlb::aug_OutMsgQueue); + td::BitArray<96> prefix; + td::BitPtr ptr = prefix.bits(); + ptr.store_int(dst_shard.workchain, 32); + ptr.advance(32); + ptr.store_uint(dst_shard.shard, 64); + if (!queue->cut_prefix_subdict(prefix.bits(), 32 + dst_shard.pfx_len())) { + return td::Status::Error("invalid message queue"); + } + dfs(queue->get_root_cell()); + TRY_RESULT(queue_proof, vm::std_boc_serialize(mpb.extract_proof())); td::BufferSlice block_state_proof; if (block_id.seqno() != 0) { TRY_RESULT(proof, create_block_state_proof(std::move(block_root))); TRY_RESULT_ASSIGN(block_state_proof, vm::std_boc_serialize(std::move(proof), 31)); } - return create_tl_object(std::move(queue_proof), std::move(block_state_proof)); } @@ -183,7 +229,11 @@ void WaitOutMsgQueueProof::run_net() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id = block_id_](td::Result> R) { if (R.is_error()) { - LOG(DEBUG) << "failed to get msg queue for " << block_id.to_str() << " from net: " << R.move_as_error(); + if (R.error().code() == ErrorCode::notready) { + LOG(DEBUG) << "failed to get msg queue for " << block_id.to_str() << " from net: " << R.move_as_error(); + } else { + LOG(WARNING) << "failed to get msg queue for " << block_id.to_str() << " from net: " << R.move_as_error(); + } delay_action([SelfId]() mutable { td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::run_net); }, td::Timestamp::in(0.1)); } else { @@ -246,8 +296,11 @@ void BuildOutMsgQueueProof::got_block_root(Ref root) { } void BuildOutMsgQueueProof::build_proof() { - promise_.set_result( - OutMsgQueueProof::serialize(block_id_, dst_shard_, std::move(state_root_), std::move(block_root_))); + auto result = OutMsgQueueProof::serialize(block_id_, dst_shard_, std::move(state_root_), std::move(block_root_)); + if (result.is_error()) { + LOG(ERROR) << "Failed to build msg queue proof: " << result.error(); + } + promise_.set_result(std::move(result)); stop(); } From 011e97f53c1610ead70e59f662f14d4a7be268d6 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 15 Aug 2022 19:27:19 +0300 Subject: [PATCH 024/388] Flag "full collated data" in mc config; fix accept-block --- crypto/block/block.tlb | 8 ++++---- crypto/block/mc-config.cpp | 24 ++++++++++++++++++++++++ crypto/block/mc-config.h | 11 +++++++++++ ton/ton-types.h | 5 ----- validator/fabric.h | 2 +- validator/impl/accept-block.cpp | 2 ++ validator/impl/accept-block.hpp | 1 + validator/impl/collator-impl.h | 1 + validator/impl/collator.cpp | 5 +++++ validator/impl/shard.cpp | 21 --------------------- validator/impl/shard.hpp | 4 +++- validator/impl/validate-query.cpp | 16 ++++++++-------- validator/impl/validate-query.hpp | 6 +++--- validator/interfaces/shard.h | 2 +- validator/manager.cpp | 8 ++++---- validator/validator-group.cpp | 9 +++++---- validator/validator-group.hpp | 6 +++--- 17 files changed, 76 insertions(+), 55 deletions(-) diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index c9a95535f..8cb6c2f0d 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -734,15 +734,15 @@ misbehaviour_punishment_config_v1#01 = MisbehaviourPunishmentConfig; _ MisbehaviourPunishmentConfig = ConfigParam 40; +// collator_nodes: each collator is (workchain:int32 shard:uint64 adnl_id:uint256) +colator_config#a0 full_collated_data:Bool collator_nodes:(HashmapE 352 Unit) = CollatorConfig; +_ CollatorConfig = ConfigParam 41; + oracle_bridge_params#_ bridge_address:bits256 oracle_mutlisig_address:bits256 oracles:(HashmapE 256 uint256) external_chain_address:bits256 = OracleBridgeParams; _ OracleBridgeParams = ConfigParam 71; // Ethereum bridge _ OracleBridgeParams = ConfigParam 72; // Binance Smart Chain bridge _ OracleBridgeParams = ConfigParam 73; // Polygon bridge -// Set of collators: each collator is (workchain:int32 shard:int64 adnl_id:int256) -colator_set#a0 collators:(HashmapE 352 Unit) = CollatorSet; -_ CollatorSet = ConfigParam 81; - // // PROOFS // diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index b5409a620..a1ddfb3ef 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2132,4 +2132,28 @@ Ref ConfigInfo::lookup_library(td::ConstBitPtr root_hash) const { return lib; } +CollatorConfig Config::get_collator_config(bool need_collator_nodes) const { + CollatorConfig collator_config; + gen::CollatorConfig::Record rec; + auto cell = get_config_param(41); + if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { + return collator_config; + } + collator_config.full_collated_data = rec.full_collated_data; + if (need_collator_nodes) { + vm::Dictionary dict{rec.collator_nodes->prefetch_ref(), 32 + 64 + 256}; + dict.check_for_each([&](Ref, td::ConstBitPtr key, int n) { + CHECK(n == 32 + 64 + 256); + auto workchain = (td::int32)key.get_int(32); + key.advance(32); + td::uint64 shard = key.get_uint(64); + key.advance(64); + td::Bits256 adnl_id(key); + collator_config.collator_nodes.push_back({ton::ShardIdFull(workchain, shard), adnl_id}); + return true; + }); + } + return collator_config; +} + } // namespace block diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 8be3f6e6f..7058117ba 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -482,6 +482,16 @@ class ShardConfig { bool set_shard_info(ton::ShardIdFull shard, Ref value); }; +struct CollatorNodeDescr { + ton::ShardIdFull shard; + ton::NodeIdShort adnl_id; +}; + +struct CollatorConfig { + bool full_collated_data = false; + std::vector collator_nodes; +}; + class Config { enum { default_mc_catchain_lifetime = 200, @@ -593,6 +603,7 @@ class Config { std::vector compute_validator_set(ton::ShardIdFull shard, ton::UnixTime time, ton::CatchainSeqno cc_seqno) const; std::vector compute_total_validator_set(int next) const; + CollatorConfig get_collator_config(bool need_collator_nodes) const; static std::vector do_compute_validator_set(const block::CatchainValidatorsConfig& ccv_conf, ton::ShardIdFull shard, const block::ValidatorSet& vset, ton::UnixTime time, diff --git a/ton/ton-types.h b/ton/ton-types.h index 7d0101d10..36c137b91 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -484,9 +484,4 @@ struct ValidatorSessionConfig { static const td::uint32 BLOCK_HASH_COVERS_DATA_FROM_VERSION = 2; }; -struct CollatorNodeDescr { - ShardIdFull shard; - NodeIdShort adnl_id; -}; - } // namespace ton diff --git a/validator/fabric.h b/validator/fabric.h index 12452ec55..069a0470b 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -25,7 +25,7 @@ namespace ton { namespace validator { -enum ValidateMode { fake = 1, lite = 2 }; +enum ValidateMode { fake = 1, full_collated_data = 2 }; td::actor::ActorOwn create_db_actor(td::actor::ActorId manager, std::string db_root_); td::actor::ActorOwn create_liteserver_cache_actor(td::actor::ActorId manager, diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index 6491a7116..c3e20c0e0 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -144,6 +144,7 @@ bool AcceptBlockQuery::precheck_header() { if (is_fork_ && !info.key_block) { return fatal_error("fork block is not a key block"); } + before_split_ = info.before_split; return true; } @@ -538,6 +539,7 @@ void AcceptBlockQuery::written_state(td::Ref upd_state) { } //handle_->set_masterchain_block(prev_[0]); + handle_->set_split(before_split_); handle_->set_state_root_hash(state_hash_); handle_->set_logical_time(lt_); handle_->set_unix_time(created_at_); diff --git a/validator/impl/accept-block.hpp b/validator/impl/accept-block.hpp index 1a1f2a543..e0e27ec20 100644 --- a/validator/impl/accept-block.hpp +++ b/validator/impl/accept-block.hpp @@ -115,6 +115,7 @@ class AcceptBlockQuery : public td::actor::Actor { UnixTime created_at_; RootHash state_keep_old_hash_, state_old_hash_, state_hash_; BlockIdExt mc_blkid_, prev_mc_blkid_; + bool before_split_; Ref last_mc_state_; BlockIdExt last_mc_id_; diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index c19199c82..b886173a7 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -128,6 +128,7 @@ class Collator final : public td::actor::Actor { void alarm() override; int verbosity{3 * 0}; int verify{1}; + bool full_collated_data_ = false; ton::LogicalTime start_lt, max_lt; ton::UnixTime now_; ton::UnixTime prev_now_; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index e116a9358..3468bc5de 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -572,6 +572,8 @@ bool Collator::unpack_last_mc_state() { << " have been enabled in global configuration, but we support only " << supported_version() << " (upgrade validator software?)"; } + full_collated_data_ = config_->get_collator_config(false).full_collated_data; + LOG(DEBUG) << "full_collated_data is " << full_collated_data_; // TODO: extract start_lt and end_lt from prev_mc_block as well // std::cerr << " block::gen::ShardState::print_ref(mc_state_root) = "; // block::gen::t_ShardState.print_ref(std::cerr, mc_state_root, 2); @@ -3982,6 +3984,9 @@ bool Collator::create_collated_data() { } collated_roots_.push_back(std::move(cell)); } + if (!full_collated_data_) { + return true; + } // 2. Proofs for hashes of states: previous states + neighbors for (const auto& p : block_state_proofs_) { collated_roots_.push_back(p.second); diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index dbf244002..019ecbeb0 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -511,27 +511,6 @@ bool MasterchainStateQ::check_old_mc_block_id(const ton::BlockIdExt& blkid, bool return config_ && config_->check_old_mc_block_id(blkid, strict); } -std::vector MasterchainStateQ::get_collator_set() const { - block::gen::CollatorSet::Record rec; - auto cell = config_->get_config_param(81); - if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { - return {}; - } - vm::Dictionary dict{rec.collators->prefetch_ref(), 32 + 64 + 256}; - std::vector collators; - dict.check_for_each([&](Ref, td::ConstBitPtr key, int n) { - CHECK(n == 32 + 64 + 256); - auto workchain = (td::int32)key.get_int(32); - key.advance(32); - td::uint64 shard = key.get_uint(64); - key.advance(64); - td::Bits256 adnl_id(key); - collators.push_back({ShardIdFull(workchain, shard), adnl_id}); - return true; - }); - return collators; -} - td::uint32 MasterchainStateQ::min_split_depth(WorkchainId workchain_id) const { if (!config_) { return 0; diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index 2b411d944..6b95c7e40 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -150,7 +150,9 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { block::WorkchainSet get_workchain_list() const override { return config_ ? config_->get_workchain_list() : block::WorkchainSet(); } - std::vector get_collator_set() const override; + block::CollatorConfig get_collator_config(bool need_collator_nodes) const override { + return config_ ? config_->get_collator_config(need_collator_nodes) : block::CollatorConfig(); + } private: ZeroStateIdExt zerostate_id_; diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 369f1f2a8..eeb2726c6 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -63,10 +63,9 @@ ValidateQuery::ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block , timeout(timeout) , main_promise(std::move(promise)) , is_fake_(mode & ValidateMode::fake) - , is_lite_(mode & ValidateMode::lite) + , full_collated_data_(mode & ValidateMode::full_collated_data) , shard_pfx_(shard_.shard) , shard_pfx_len_(ton::shard_prefix_length(shard_)) { - proc_hash_.zero(); } void ValidateQuery::alarm() { @@ -158,6 +157,7 @@ void ValidateQuery::finish_query() { void ValidateQuery::start_up() { LOG(INFO) << "validate query for " << block_candidate.id.to_str() << " started"; + LOG(DEBUG) << "full_collated_data is " << full_collated_data_; alarm_timestamp() = timeout; rand_seed_.set_zero(); created_by_ = block_candidate.pubkey; @@ -258,9 +258,9 @@ void ValidateQuery::start_up() { td::actor::send_closure_later( std::move(self), &ValidateQuery::after_get_latest_mc_state, std::move(res)); }); - // 3. load state(s) corresponding to previous block(s) (non-lite mode or masterchain) + // 3. load state(s) corresponding to previous block(s) (not full-collaoted-data or masterchain) prev_states.resize(prev_blocks.size()); - if (is_masterchain() || !is_lite_) { + if (is_masterchain() || !full_collated_data_) { for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { // 3.1. load state LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; @@ -991,8 +991,8 @@ bool ValidateQuery::check_this_shard_mc_info() { */ bool ValidateQuery::compute_prev_state() { - if (!is_masterchain() && is_lite_) { - return compute_prev_state_lite_mode(); + if (!is_masterchain() && full_collated_data_) { + return compute_prev_state_from_collated_data(); } CHECK(prev_states.size() == 1u + after_merge_); prev_state_root_ = prev_states[0]->root_cell(); @@ -1012,7 +1012,7 @@ bool ValidateQuery::compute_prev_state() { return true; } -bool ValidateQuery::compute_prev_state_lite_mode() { +bool ValidateQuery::compute_prev_state_from_collated_data() { td::Bits256 state_hash; if (id_.seqno() == 1) { if (prev_blocks.size() != 1) { @@ -1255,7 +1255,7 @@ bool ValidateQuery::request_neighbor_queues() { neighbors_.emplace_back(*shard_ptr); } int i = 0; - if (is_lite_) { + if (full_collated_data_) { for (block::McShardDescr& descr : neighbors_) { LOG(DEBUG) << "getting outbound queue of neighbor #" << i << " from collated data : " << descr.blk_.to_str(); td::Bits256 state_root_hash; diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 6fe8010f5..48c20e6ce 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -141,7 +141,7 @@ class ValidateQuery : public td::actor::Actor { bool is_key_block_{false}; bool update_shard_cc_{false}; bool is_fake_{false}; - bool is_lite_{false}; + bool full_collated_data_{false}; bool prev_key_block_exists_{false}; bool debug_checks_{false}; bool outq_cleanup_partial_{false}; @@ -221,7 +221,7 @@ class ValidateQuery : public td::actor::Actor { td::RefInt256 import_fees_; ton::LogicalTime proc_lt_{0}, claimed_proc_lt_{0}, min_enq_lt_{~0ULL}; - ton::Bits256 proc_hash_, claimed_proc_hash_, min_enq_hash_; + ton::Bits256 proc_hash_ = ton::Bits256::zero(), claimed_proc_hash_, min_enq_hash_; bool inbound_queues_empty_{false}; std::vector> msg_proc_lt_; @@ -286,7 +286,7 @@ class ValidateQuery : public td::actor::Actor { bool extract_collated_data(); bool try_validate(); bool compute_prev_state(); - bool compute_prev_state_lite_mode(); + bool compute_prev_state_from_collated_data(); bool compute_next_state(); bool unpack_merge_prev_state(); bool unpack_prev_state(); diff --git a/validator/interfaces/shard.h b/validator/interfaces/shard.h index ff458d604..15caa7b27 100644 --- a/validator/interfaces/shard.h +++ b/validator/interfaces/shard.h @@ -82,7 +82,7 @@ class MasterchainState : virtual public ShardState { virtual bool check_old_mc_block_id(const ton::BlockIdExt& blkid, bool strict = false) const = 0; virtual td::Result> get_config_holder() const = 0; virtual block::WorkchainSet get_workchain_list() const = 0; - virtual std::vector get_collator_set() const = 0; + virtual block::CollatorConfig get_collator_config(bool need_collator_nodes) const = 0; virtual td::Status prepare() { return td::Status::OK(); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 6323c4f2f..10f0ca488 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2117,10 +2117,10 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group auto validator_id = get_validator(shard, validator_set); CHECK(!validator_id.is_zero()); auto G = td::actor::create_actor( - "validatorgroup", shard, validator_id, session_id, validator_set, last_masterchain_state_->get_collator_set(), - opts, keyring_, adnl_, rldp_, overlays_, - db_root_, actor_id(this), init_session, - opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_->validator_lite_mode()); + "validatorgroup", shard, validator_id, session_id, validator_set, + last_masterchain_state_->get_collator_config(true), opts, keyring_, adnl_, rldp_, overlays_, db_root_, + actor_id(this), init_session, opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), + opts_->validator_lite_mode()); return G; } } diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index a72bded10..88a473ff6 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -42,7 +42,7 @@ void ValidatorGroup::generate_block_candidate(td::uint32 round_id, td::Promise> promise); ValidatorGroup(ShardIdFull shard, PublicKeyHash local_id, ValidatorSessionId session_id, - td::Ref validator_set, std::vector collator_set, + td::Ref validator_set, block::CollatorConfig collator_config, validatorsession::ValidatorSessionOptions config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, @@ -70,7 +70,7 @@ class ValidatorGroup : public td::actor::Actor { , local_id_(std::move(local_id)) , session_id_(session_id) , validator_set_(std::move(validator_set)) - , collator_set_(std::move(collator_set)) + , collator_config_(std::move(collator_config)) , config_(std::move(config)) , keyring_(keyring) , adnl_(adnl) @@ -110,7 +110,7 @@ class ValidatorGroup : public td::actor::Actor { BlockIdExt min_masterchain_block_id_; td::Ref validator_set_; - std::vector collator_set_; + block::CollatorConfig collator_config_; validatorsession::ValidatorSessionOptions config_; td::actor::ActorId keyring_; From be2169e523398564fbb112b85bf124ec2b142e81 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 16 Aug 2022 13:54:15 +0300 Subject: [PATCH 025/388] Reload validator options when needed --- validator-engine/validator-engine.cpp | 20 +++++++++++++++++--- validator-engine/validator-engine.hpp | 2 +- validator/manager-disk.hpp | 4 ++++ validator/manager-hardfork.hpp | 3 +++ validator/manager.cpp | 10 ++++++++++ validator/manager.hpp | 1 + validator/shard-client.cpp | 4 ++++ validator/shard-client.hpp | 2 ++ validator/state-serializer.hpp | 4 ++++ validator/validator.h | 1 + 10 files changed, 47 insertions(+), 4 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index c0f85d039..e56dfb7be 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1404,7 +1404,7 @@ td::Status ValidatorEngine::load_global_config() { return td::Status::OK(); } -void ValidatorEngine::init_validator_options() { +void ValidatorEngine::set_shard_check_function() { if (!not_all_shards_) { validator_options_.write().set_shard_check_function([](ton::ShardIdFull shard) -> bool { return true; }); } else { @@ -1711,7 +1711,7 @@ void ValidatorEngine::got_key(ton::PublicKey key) { } void ValidatorEngine::start() { - init_validator_options(); + set_shard_check_function(); read_config_ = true; start_adnl(); } @@ -3412,7 +3412,10 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollat promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); return; } + set_shard_check_function(); if (!validator_manager_.empty()) { + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_collator, ton::adnl::AdnlNodeIdShort(id), shard); } @@ -3443,7 +3446,18 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addShard promise.set_value(create_control_query_error(R.move_as_error())); return; } - promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + set_shard_check_function(); + if (!validator_manager_.empty()) { + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + } + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); } void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index b5d770a4e..40a04c703 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -283,7 +283,7 @@ class ValidatorEngine : public td::actor::Actor { void load_empty_local_config(td::Promise promise); void load_local_config(td::Promise promise); void load_config(td::Promise promise); - void init_validator_options(); + void set_shard_check_function(); void start(); diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 53467ff32..ea443d466 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -386,6 +386,10 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } + void update_options(td::Ref opts) override { + opts_ = std::move(opts); + } + private: PublicKeyHash local_id_; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 9aa302263..ce92f8a7b 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -445,6 +445,9 @@ class ValidatorManagerImpl : public ValidatorManager { void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { UNREACHABLE(); } + void update_options(td::Ref opts) override { + opts_ = std::move(opts); + } private: td::Ref opts_; diff --git a/validator/manager.cpp b/validator/manager.cpp index 10f0ca488..710edec56 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2696,6 +2696,16 @@ void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh td::actor::send_closure(it->second, &CollatorNode::add_shard, shard); } +void ValidatorManagerImpl::update_options(td::Ref opts) { + if (!shard_client_.empty()) { + td::actor::send_closure(shard_client_, &ShardClient::update_options, opts); + } + if (!serializer_.empty()) { + td::actor::send_closure(serializer_, &AsyncStateSerializer::update_options, opts); + } + opts_ = std::move(opts); +} + td::actor::ActorOwn ValidatorManagerFactory::create( td::Ref opts, std::string db_root, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, diff --git a/validator/manager.hpp b/validator/manager.hpp index 15598f4cd..d61351f29 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -556,6 +556,7 @@ class ValidatorManagerImpl : public ValidatorManager { } void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; + void update_options(td::Ref opts) override; private: td::Timestamp resend_shard_blocks_at_; diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index 65adf34d4..eff8aced3 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -331,6 +331,10 @@ void ShardClient::force_update_shard_client_ex(BlockHandle handle, td::Ref opts) { + opts_ = std::move(opts); +} + } // namespace validator } // namespace ton diff --git a/validator/shard-client.hpp b/validator/shard-client.hpp index 701a387e8..d7330b368 100644 --- a/validator/shard-client.hpp +++ b/validator/shard-client.hpp @@ -97,6 +97,8 @@ class ShardClient : public td::actor::Actor { void force_update_shard_client(BlockHandle handle, td::Promise promise); void force_update_shard_client_ex(BlockHandle handle, td::Ref state, td::Promise promise); + + void update_options(td::Ref opts); }; } // namespace validator diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index e971a3f84..89db2fc6c 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -91,6 +91,10 @@ class AsyncStateSerializer : public td::actor::Actor { void fail_handler(td::Status reason); void fail_handler_cont(); void success_handler(); + + void update_options(td::Ref opts) { + opts_ = std::move(opts); + } }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 0595949fb..0daccc1ae 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -228,6 +228,7 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Promise> promise) = 0; virtual void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; + virtual void update_options(td::Ref opts) = 0; }; } // namespace validator From ea7a5776fecdab61301106e462ea98e55c38ca7c Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 17 Aug 2022 16:29:50 +0300 Subject: [PATCH 026/388] Download persistent states when syncing new shards --- tl/generate/scheme/ton_api.tl | 5 ++ tl/generate/scheme/ton_api.tlo | Bin 72144 -> 73056 bytes ton/ton-types.h | 10 +++ validator/db/rootdb.cpp | 8 ++ validator/db/rootdb.hpp | 3 + validator/db/statedb.cpp | 95 +++++++++++++++++++++ validator/db/statedb.hpp | 4 +- validator/downloaders/wait-block-state.cpp | 14 +++ validator/downloaders/wait-block-state.hpp | 15 +++- validator/interfaces/db.h | 5 ++ validator/interfaces/validator-manager.h | 2 + validator/manager-disk.hpp | 3 + validator/manager-hardfork.hpp | 2 + validator/manager.cpp | 67 +++++++++++++-- validator/manager.hpp | 10 ++- validator/shard-client.cpp | 10 +-- validator/state-serializer.cpp | 35 +++++++- validator/state-serializer.hpp | 2 + 18 files changed, 273 insertions(+), 17 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 39cfb5eec..ff03764a7 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -514,6 +514,9 @@ db.state.shardClient block:tonNode.blockIdExt = db.state.ShardClient; db.state.asyncSerializer block:tonNode.blockIdExt last:tonNode.blockIdExt last_ts:int = db.state.AsyncSerializer; db.state.hardforks blocks:(vector tonNode.blockIdExt) = db.state.Hardforks; db.state.dbVersion version:int = db.state.DbVersion; +db.state.persistentStateDescriptionShards shard_blocks:(vector tonNode.blockIdExt) = db.state.PersistentStateDescriptionShards; +db.state.persistentStateDescriptionHeader masterchain_id:tonNode.blockIdExt start_time:int end_time:int = db.state.PersistentStateDescriptionHeader; +db.state.persistentStateDescriptionsList list:(vector db.state.persistentStateDescriptionHeader) = db.state.PersistentStateDescriptionsList; db.state.key.destroyedSessions = db.state.Key; db.state.key.initBlockId = db.state.Key; @@ -522,6 +525,8 @@ db.state.key.shardClient = db.state.Key; db.state.key.asyncSerializer = db.state.Key; db.state.key.hardforks = db.state.Key; db.state.key.dbVersion = db.state.Key; +db.state.key.persistentStateDescriptionShards masterchain_seqno:int = db.state.Key; +db.state.key.persistentStateDescriptionsList = db.state.Key; db.lt.el.key workchain:int shard:long idx:int = db.lt.Key; db.lt.desc.key workchain:int shard:long = db.lt.Key; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 413fbd8769514b2bed2ffc52cc34f4679ea70c69..12399020f6b0a164ee8f20686f7629f4a83f5545 100644 GIT binary patch delta 915 zcmcbxndQMQ7T!m*^{p77;L1kc-@Kv$m3$9*Qj+wFOA<>`^}JInfwG(v9mVxRlvy@v z!X*Myi;6RgOH%Vnfxf9mPICyC*N?l-_K@=Of7&ytz^RzNZAp={JuF`GEXWy2AH{w(-(>}icG$^Nx2>*glz1oO7SQiB8>$HHBd9mSdcl$fs1S|Y7&7v)eJ`( zVPF7h0L2(H&=l!s_ci%)6M<=}C^;iBGcSH}Vx%-T7cdsMS2hpi~&2l#`#FT|D{WcKPWE+>9bFAY+lO oK~15!t$_s_Nd5E2vV_`qxi%!0QlWLng9R* delta 265 zcmaE`i{-**7T!m*^{p77;NnK!-@Kwi$D6cxQj+wFOA<>`^}JInfwJrq9mO~E@cT$I z`fZNXxbG_ku#ePhi%(Ikl5saA|2e4Q;IZjOJ2dJQ=-@ax6XuOJjO1- LBe{7-*@ shard_blocks; + UnixTime start_time, end_time; + + virtual CntObject* make_copy() const { + return new PersistentStateDescription(*this); + } +}; + } // namespace ton diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index a7a1becf3..1c8c8e591 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -497,6 +497,14 @@ void RootDb::set_async_mode(bool mode, td::Promise promise) { td::actor::send_closure(archive_db_, &ArchiveManager::set_async_mode, mode, std::move(promise)); } +void RootDb::add_persistent_state_description(td::Ref desc, td::Promise promise) { + td::actor::send_closure(state_db_, &StateDb::add_persistent_state_description, std::move(desc), std::move(promise)); +} + +void RootDb::get_persistent_state_descriptions(td::Promise>> promise) { + td::actor::send_closure(state_db_, &StateDb::get_persistent_state_descriptions, std::move(promise)); +} + void RootDb::run_gc(UnixTime ts, UnixTime archive_ttl) { td::actor::send_closure(archive_db_, &ArchiveManager::run_gc, ts, archive_ttl); } diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 9b0d52a6b..6206bbfe6 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -132,6 +132,9 @@ class RootDb : public Db { td::Promise promise) override; void set_async_mode(bool mode, td::Promise promise) override; + void add_persistent_state_description(td::Ref desc, td::Promise promise) override; + void get_persistent_state_descriptions(td::Promise>> promise) override; + void run_gc(UnixTime ts, UnixTime archive_ttl) override; private: diff --git a/validator/db/statedb.cpp b/validator/db/statedb.cpp index 5d49ae2bd..fe3b9d735 100644 --- a/validator/db/statedb.cpp +++ b/validator/db/statedb.cpp @@ -240,6 +240,101 @@ void StateDb::start_up() { } } +void StateDb::add_persistent_state_description(td::Ref desc, + td::Promise promise) { + std::string value; + auto list_key = create_hash_tl_object(); + auto R = kv_->get(list_key.as_slice(), value); + R.ensure(); + tl_object_ptr list; + if (R.ok() == td::KeyValue::GetStatus::Ok) { + auto F = fetch_tl_object(value, true); + F.ensure(); + list = F.move_as_ok(); + } else { + list = create_tl_object( + std::vector>()); + } + for (const auto& obj : list->list_) { + if ((BlockSeqno)obj->masterchain_id_->seqno_ == desc->masterchain_id.seqno()) { + promise.set_error(td::Status::Error("duplicate masterchain seqno")); + return; + } + } + + auto now = (UnixTime)td::Clocks::system(); + size_t new_size = 0; + kv_->begin_write_batch().ensure(); + for (auto& obj : list->list_) { + auto end_time = (UnixTime)obj->end_time_; + if (end_time <= now) { + auto key = + create_hash_tl_object(obj->masterchain_id_->seqno_); + kv_->erase(key.as_slice()).ensure(); + } else { + list->list_[new_size++] = std::move(obj); + } + } + list->list_.resize(new_size); + + std::vector> shard_blocks; + for (const BlockIdExt& block_id : desc->shard_blocks) { + shard_blocks.push_back(create_tl_block_id(block_id)); + } + auto key = + create_hash_tl_object(desc->masterchain_id.seqno()); + kv_->set(key.as_slice(), + create_serialize_tl_object(std::move(shard_blocks)) + .as_slice()) + .ensure(); + + list->list_.push_back(create_tl_object( + create_tl_block_id(desc->masterchain_id), desc->start_time, desc->end_time)); + kv_->set(list_key.as_slice(), serialize_tl_object(list, true).as_slice()).ensure(); + + kv_->commit_write_batch().ensure(); + + promise.set_result(td::Unit()); +} + +void StateDb::get_persistent_state_descriptions(td::Promise>> promise) { + std::string value; + auto R = kv_->get(create_hash_tl_object().as_slice(), value); + R.ensure(); + if (R.ok() == td::KeyValue::GetStatus::NotFound) { + promise.set_value({}); + return; + } + auto F = fetch_tl_object(value, true); + F.ensure(); + std::vector> result; + auto now = (UnixTime)td::Clocks::system(); + for (const auto& obj : F.ok()->list_) { + auto end_time = (UnixTime)obj->end_time_; + if (end_time <= now) { + continue; + } + PersistentStateDescription desc; + desc.start_time = (UnixTime)obj->start_time_; + desc.end_time = end_time; + desc.masterchain_id = create_block_id(obj->masterchain_id_); + auto key = + create_hash_tl_object(desc.masterchain_id.seqno()); + auto R2 = kv_->get(key.as_slice(), value); + R2.ensure(); + if (R2.ok() == td::KeyValue::GetStatus::NotFound) { + continue; + } + auto F2 = fetch_tl_object(value, true); + F2.ensure(); + for (const auto& block_id : F2.ok()->shard_blocks_) { + desc.shard_blocks.push_back(create_block_id(block_id)); + } + result.push_back(td::Ref(true, std::move(desc))); + } + promise.set_result(std::move(result)); +} + void StateDb::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise) { { auto key = create_hash_tl_object(); diff --git a/validator/db/statedb.hpp b/validator/db/statedb.hpp index 75382d61c..fe23898f4 100644 --- a/validator/db/statedb.hpp +++ b/validator/db/statedb.hpp @@ -50,8 +50,8 @@ class StateDb : public td::actor::Actor { void update_hardforks(std::vector blocks, td::Promise promise); void get_hardforks(td::Promise> promise); - void update_db_version(td::uint32 version, td::Promise promise); - void get_db_version(td::Promise promise); + void add_persistent_state_description(td::Ref desc, td::Promise promise); + void get_persistent_state_descriptions(td::Promise>> promise); StateDb(td::actor::ActorId root_db, std::string path); diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index 56137fc39..8d033c112 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -21,6 +21,7 @@ #include "ton/ton-io.hpp" #include "common/checksum.h" #include "common/delay.h" +#include "validator/downloaders/download-state.hpp" namespace ton { @@ -106,6 +107,19 @@ void WaitBlockState::start() { }); td::actor::send_closure(manager_, &ValidatorManager::send_get_zero_state_request, handle_->id(), priority_, std::move(P)); + } else if (check_persistent_state_desc()) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + LOG(WARNING) << "failed to get persistent state: " << R.move_as_error(); + td::actor::send_closure(SelfId, &WaitBlockState::start); + } else { + td::actor::send_closure(SelfId, &WaitBlockState::written_state, R.move_as_ok()); + } + }); + BlockIdExt masterchain_id = persistent_state_desc_->masterchain_id; + td::actor::create_actor("downloadstate", handle_->id(), masterchain_id, priority_, manager_, + timeout_, std::move(P)) + .release(); } else if (!handle_->inited_prev() || (!handle_->inited_proof() && !handle_->inited_proof_link())) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result R) { if (R.is_error()) { diff --git a/validator/downloaders/wait-block-state.hpp b/validator/downloaders/wait-block-state.hpp index a93f29717..fc25eeec4 100644 --- a/validator/downloaders/wait-block-state.hpp +++ b/validator/downloaders/wait-block-state.hpp @@ -27,12 +27,14 @@ namespace validator { class WaitBlockState : public td::actor::Actor { public: WaitBlockState(BlockHandle handle, td::uint32 priority, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise> promise) + td::Timestamp timeout, td::Promise> promise, + td::Ref persistent_state_desc = {}) : handle_(std::move(handle)) , priority_(priority) , manager_(manager) , timeout_(timeout) - , promise_(std::move(promise)) { + , promise_(std::move(promise)) + , persistent_state_desc_(std::move(persistent_state_desc)) { } void abort_query(td::Status reason); @@ -73,6 +75,7 @@ class WaitBlockState : public td::actor::Actor { td::actor::ActorId manager_; td::Timestamp timeout_; td::Promise> promise_; + td::Ref persistent_state_desc_; td::Ref prev_state_; td::Ref block_; @@ -81,6 +84,14 @@ class WaitBlockState : public td::actor::Actor { td::Timestamp next_static_file_attempt_; //td::PerfWarningTimer perf_timer_{"waitstate", 1.0}; + + bool check_persistent_state_desc() const { + if (persistent_state_desc_.is_null()) { + return false; + } + auto now = (UnixTime)td::Clocks::system(); + return persistent_state_desc_->end_time > now + 3600 && persistent_state_desc_->start_time < now - 6 * 3600; + } }; } // namespace validator diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index ba4d9dda1..2df3da591 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -119,6 +119,11 @@ class Db : public td::actor::Actor { td::Promise promise) = 0; virtual void set_async_mode(bool mode, td::Promise promise) = 0; + virtual void add_persistent_state_description(td::Ref desc, + td::Promise promise) = 0; + virtual void get_persistent_state_descriptions( + td::Promise>> promise) = 0; + virtual void run_gc(UnixTime ts, UnixTime archive_ttl) = 0; }; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 0477b6487..38de74676 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -170,6 +170,8 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void validated_new_block(BlockIdExt block_id) = 0; + virtual void add_persistent_state_description(td::Ref desc) = 0; + static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) { return ts / (1 << 17) != prev_ts / (1 << 17); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index ea443d466..1ab6fb092 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -377,6 +377,9 @@ class ValidatorManagerImpl : public ValidatorManager { } void validated_new_block(BlockIdExt block_id) override { } + void add_persistent_state_description(td::Ref desc) override { + } + void get_validator_sessions_info( td::Promise> promise) override { UNREACHABLE(); diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index ce92f8a7b..3d20e494c 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -437,6 +437,8 @@ class ValidatorManagerImpl : public ValidatorManager { } void validated_new_block(BlockIdExt block_id) override { } + void add_persistent_state_description(td::Ref desc) override { + } void get_validator_sessions_info( td::Promise> promise) override { UNREACHABLE(); diff --git a/validator/manager.cpp b/validator/manager.cpp index 710edec56..2f2e5f024 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -574,9 +574,10 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); }); - auto id = td::actor::create_actor("waitstate", handle, priority, actor_id(this), - td::Timestamp::in(10.0), std::move(P)) - .release(); + auto id = + td::actor::create_actor("waitstate", handle, priority, actor_id(this), td::Timestamp::in(10.0), + std::move(P), get_block_persistent_state(handle->id())) + .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); } @@ -1013,7 +1014,7 @@ void ValidatorManagerImpl::finished_wait_state(BlockHandle handle, td::Result("waitstate", handle, X.second, actor_id(this), X.first, - std::move(P)) + std::move(P), get_block_persistent_state(handle->id())) .release(); it->second.actor_ = id; return; @@ -1568,8 +1569,17 @@ void ValidatorManagerImpl::started(ValidatorManagerInitResult R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::read_gc_list, R.move_as_ok()); } }); - td::actor::send_closure(db_, &Db::get_destroyed_validator_sessions, std::move(P)); + + auto Q = td::PromiseCreator::lambda( + [SelfId = actor_id(this)](td::Result>> R) { + if (R.is_error()) { + LOG(FATAL) << "db error: " << R.move_as_error(); + } else { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::got_persistent_state_descriptions, R.move_as_ok()); + } + }); + td::actor::send_closure(db_, &Db::get_persistent_state_descriptions, std::move(Q)); } void ValidatorManagerImpl::read_gc_list(std::vector list) { @@ -2706,6 +2716,53 @@ void ValidatorManagerImpl::update_options(td::Ref opts) opts_ = std::move(opts); } +void ValidatorManagerImpl::add_persistent_state_description(td::Ref desc) { + auto now = (UnixTime)td::Clocks::system(); + if (desc->end_time <= now) { + return; + } + td::actor::send_closure(db_, &Db::add_persistent_state_description, desc, [](td::Result) {}); + auto it = persistent_state_descriptions_.begin(); + while (it != persistent_state_descriptions_.end()) { + const auto &prev_desc = it->second; + if (prev_desc->end_time <= now) { + for (const BlockIdExt &block_id : prev_desc->shard_blocks) { + persistent_state_blocks_.erase(block_id); + } + it = persistent_state_descriptions_.erase(it); + } else { + ++it; + } + } + add_persistent_state_description_impl(std::move(desc)); +} + +void ValidatorManagerImpl::add_persistent_state_description_impl(td::Ref desc) { + if (!persistent_state_descriptions_.emplace(desc->masterchain_id.seqno(), desc).second) { + return; + } + LOG(DEBUG) << "Add persistent state description for mc block " << desc->masterchain_id.to_str() + << " start_time=" << desc->start_time << " end_time=" << desc->end_time; + for (const BlockIdExt &block_id : desc->shard_blocks) { + persistent_state_blocks_[block_id] = desc; + LOG(DEBUG) << "Persistent state description: shard block " << block_id.to_str(); + } +} + +void ValidatorManagerImpl::got_persistent_state_descriptions(std::vector> descs) { + for (auto &desc : descs) { + add_persistent_state_description_impl(std::move(desc)); + } +} + +td::Ref ValidatorManagerImpl::get_block_persistent_state(BlockIdExt block_id) { + auto it = persistent_state_blocks_.find(block_id); + if (it == persistent_state_blocks_.end()) { + return {}; + } + return it->second; +} + td::actor::ActorOwn ValidatorManagerFactory::create( td::Ref opts, std::string db_root, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, diff --git a/validator/manager.hpp b/validator/manager.hpp index d61351f29..ddef34414 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -555,6 +555,8 @@ class ValidatorManagerImpl : public ValidatorManager { last = std::max(last, block_id.seqno()); } + void add_persistent_state_description(td::Ref desc) override; + void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; void update_options(td::Ref opts) override; @@ -621,8 +623,11 @@ class ValidatorManagerImpl : public ValidatorManager { } void cleanup_last_validated_blocks(BlockId new_block); - private: + void got_persistent_state_descriptions(std::vector> descs); + void add_persistent_state_description_impl(td::Ref desc); + td::Ref get_block_persistent_state(BlockIdExt block_id); + private: std::map> shard_client_waiters_; std::map> collator_nodes_; @@ -630,6 +635,9 @@ class ValidatorManagerImpl : public ValidatorManager { std::set shards_to_monitor_ = {ShardIdFull(masterchainId)}; std::set extra_active_shards_; std::map last_validated_blocks_; + + std::map> persistent_state_descriptions_; + std::map> persistent_state_blocks_; }; } // namespace validator diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index eff8aced3..431d77216 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -202,7 +202,7 @@ void ShardClient::apply_all_shards() { } }); td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, shard->top_block_id(), - shard_client_priority(), td::Timestamp::in(600), std::move(Q)); + shard_client_priority(), td::Timestamp::in(1500), std::move(Q)); } } } @@ -255,11 +255,9 @@ void ShardClient::build_shard_overlays() { auto shard = info->shard(); workchains.insert(shard.workchain); bool will_split = shard.pfx_len() < max_shard_pfx_len && - ((info->fsm_state() == McShardHash::FsmState::fsm_split && info->fsm_utime() < cur_time + 60) || - info->before_split()); - bool will_merge = shard.pfx_len() > 0 && - ((info->fsm_state() == McShardHash::FsmState::fsm_merge && info->fsm_utime() < cur_time + 60) || - info->before_merge()); + (info->fsm_state() == McShardHash::FsmState::fsm_split || info->before_split()); + bool will_merge = + shard.pfx_len() > 0 && (info->fsm_state() == McShardHash::FsmState::fsm_merge || info->before_merge()); if (opts_->need_monitor(shard) || (will_merge && opts_->need_monitor(shard_parent(shard)))) { new_shards_to_monitor.insert(shard); } diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index d5d2f0009..fc8cd29aa 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -18,7 +18,6 @@ */ #include "state-serializer.hpp" #include "td/utils/Random.h" -#include "adnl/utils.hpp" #include "ton/ton-io.hpp" #include "common/delay.h" @@ -122,6 +121,21 @@ void AsyncStateSerializer::next_iteration() { CHECK(masterchain_handle_->id() == last_block_id_); if (attempt_ < max_attempt() && last_key_block_id_.id.seqno < last_block_id_.id.seqno && need_serialize(masterchain_handle_)) { + if (!stored_persistent_state_description_) { + LOG(INFO) << "storing persistent state description for " << masterchain_handle_->id().id; + running_ = true; + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &AsyncStateSerializer::fail_handler, + R.move_as_error_prefix("failed to get masterchain state: ")); + } else { + td::actor::send_closure(SelfId, &AsyncStateSerializer::store_persistent_state_description, + td::Ref(R.move_as_ok())); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, masterchain_handle_, std::move(P)); + return; + } if (!cell_db_reader_) { running_ = true; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { @@ -175,6 +189,7 @@ void AsyncStateSerializer::next_iteration() { if (masterchain_handle_->inited_next_left()) { last_block_id_ = masterchain_handle_->one_next(true); have_masterchain_state_ = false; + stored_persistent_state_description_ = false; masterchain_handle_ = nullptr; saved_to_db_ = false; shards_.clear(); @@ -189,6 +204,24 @@ void AsyncStateSerializer::got_top_masterchain_handle(BlockIdExt block_id) { } } +void AsyncStateSerializer::store_persistent_state_description(td::Ref state) { + stored_persistent_state_description_ = true; + attempt_ = 0; + running_ = false; + + PersistentStateDescription desc; + desc.masterchain_id = state->get_block_id(); + desc.start_time = state->get_unix_time(); + desc.end_time = ValidatorManager::persistent_state_ttl(desc.start_time); + for (const auto &v : state->get_shards()) { + desc.shard_blocks.push_back(v->top_block_id()); + } + td::actor::send_closure(manager_, &ValidatorManager::add_persistent_state_description, + td::Ref(true, std::move(desc))); + + next_iteration(); +} + void AsyncStateSerializer::got_cell_db_reader(std::shared_ptr cell_db_reader) { cell_db_reader_ = std::move(cell_db_reader); running_ = false; diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index 89db2fc6c..0d6e95505 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -44,6 +44,7 @@ class AsyncStateSerializer : public td::actor::Actor { std::shared_ptr cell_db_reader_ = nullptr; BlockHandle masterchain_handle_; + bool stored_persistent_state_description_ = false; bool have_masterchain_state_ = false; std::vector shards_; @@ -71,6 +72,7 @@ class AsyncStateSerializer : public td::actor::Actor { void next_iteration(); void got_top_masterchain_handle(BlockIdExt block_id); + void store_persistent_state_description(td::Ref state); void got_cell_db_reader(std::shared_ptr cell_db_reader); void got_masterchain_handle(BlockHandle handle_); void got_masterchain_state(td::Ref state); From 1efd4254031e8d1d37d98a81e0f47d42180120b3 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 19 Aug 2022 18:20:16 +0300 Subject: [PATCH 027/388] Fix compatibility --- dht-server/dht-server.cpp | 19 ++++++++-------- dht-server/dht-server.hpp | 4 ++-- lite-client/lite-client.cpp | 30 ++++++++++++-------------- tl/generate/scheme/ton_api.tl | 12 +++-------- tl/generate/scheme/ton_api.tlo | Bin 73056 -> 72560 bytes ton/ton-tl.hpp | 19 ---------------- validator-engine/validator-engine.cpp | 25 ++++++++------------- validator-engine/validator-engine.hpp | 4 ++-- validator/full-node-shard.cpp | 10 ++++----- 9 files changed, 44 insertions(+), 79 deletions(-) diff --git a/dht-server/dht-server.cpp b/dht-server/dht-server.cpp index 3bd65cc6e..01ee105d2 100644 --- a/dht-server/dht-server.cpp +++ b/dht-server/dht-server.cpp @@ -38,9 +38,6 @@ #include "td/utils/TsFileLog.h" #include "td/utils/Random.h" -#include "ton/ton-tl.hpp" -#include "tl/tl_json.h" - #include "memprof/memprof.h" #if TD_DARWIN || TD_LINUX @@ -57,7 +54,7 @@ Config::Config() { out_port = 3278; } -Config::Config(const ton::ton_api::engine_validator_configV2 &config) { +Config::Config(const ton::ton_api::engine_validator_config &config) { out_port = static_cast(config.out_port_); if (!out_port) { out_port = 3278; @@ -124,7 +121,7 @@ Config::Config(const ton::ton_api::engine_validator_configV2 &config) { } } -ton::tl_object_ptr Config::tl() const { +ton::tl_object_ptr Config::tl() const { std::vector> addrs_vec; for (auto &x : addrs) { if (x.second.proxy) { @@ -150,6 +147,7 @@ ton::tl_object_ptr Config::tl() const { } std::vector> val_vec; + std::vector> col_vec; std::vector> full_node_slaves_vec; std::vector> full_node_masters_vec; @@ -165,15 +163,16 @@ ton::tl_object_ptr Config::tl() const { control_vec.push_back(ton::create_tl_object(x.second.key.tl(), x.first, std::move(control_proc_vec))); } + std::vector> shard_vec; auto gc_vec = ton::create_tl_object(std::vector{}); for (auto &id : gc) { gc_vec->ids_.push_back(id.tl()); } return ton::create_tl_object( - out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), + out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec), ton::PublicKeyHash::zero().tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), - std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec)); + std::move(liteserver_vec), std::move(control_vec), std::move(shard_vec), std::move(gc_vec)); } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, @@ -600,14 +599,14 @@ void DhtServer::load_config(td::Promise promise) { } auto conf_json = conf_json_R.move_as_ok(); - ton::tl_object_ptr conf; - auto S = td::from_json(conf, std::move(conf_json)); + ton::ton_api::engine_validator_config conf; + auto S = ton::ton_api::from_json(conf, conf_json.get_object()); if (S.is_error()) { promise.set_error(S.move_as_error_prefix("json does not fit TL scheme")); return; } - config_ = Config{*ton::unpack_engine_validator_config(std::move(conf))}; + config_ = Config{conf}; td::MultiPromise mp; auto ig = mp.init_guard(); diff --git a/dht-server/dht-server.hpp b/dht-server/dht-server.hpp index 724232fa7..bf8152026 100644 --- a/dht-server/dht-server.hpp +++ b/dht-server/dht-server.hpp @@ -91,10 +91,10 @@ struct Config { td::Result config_del_control_process(td::int32 port, ton::PublicKeyHash id); td::Result config_del_gc(ton::PublicKeyHash key); - ton::tl_object_ptr tl() const; + ton::tl_object_ptr tl() const; Config(); - Config(const ton::ton_api::engine_validator_configV2 &config); + Config(const ton::ton_api::engine_validator_config &config); }; class DhtServer : public td::actor::Actor { diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index 970cdad94..7165ca53c 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -124,7 +124,7 @@ void TestNode::run() { auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); ton::ton_api::liteclient_config_global gc; ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); - CHECK(gc.liteservers_.size() > 0); + CHECK(gc.liteservers_.size() + gc.liteservers_v2_.size() > 0); if (gc.validator_ && gc.validator_->zero_state_) { zstate_id_.workchain = gc.validator_->zero_state_->workchain_; @@ -137,21 +137,19 @@ void TestNode::run() { for (auto& server : gc.liteservers_) { LiteServer s; - ton::ton_api::downcast_call(*server, - td::overloaded( - [&](ton::ton_api::liteserver_desc& obj) { - s.addr.init_host_port(td::IPAddress::ipv4_to_str(obj.ip_), obj.port_).ensure(); - s.public_key = ton::PublicKey{obj.id_}; - }, - [&](ton::ton_api::liteserver_descV2& obj) { - s.addr.init_host_port(td::IPAddress::ipv4_to_str(obj.ip_), obj.port_).ensure(); - s.public_key = ton::PublicKey{obj.id_}; - s.is_full = false; - for (const auto& shard : obj.shards_) { - s.shards.emplace_back(shard->workchain_, shard->shard_); - CHECK(s.shards.back().is_valid_ext()); - } - })); + s.addr.init_host_port(td::IPAddress::ipv4_to_str(server->ip_), server->port_).ensure(); + s.public_key = ton::PublicKey{server->id_}; + servers_.push_back(std::move(s)); + } + for (auto& server : gc.liteservers_v2_) { + LiteServer s; + s.addr.init_host_port(td::IPAddress::ipv4_to_str(server->ip_), server->port_).ensure(); + s.public_key = ton::PublicKey{server->id_}; + s.is_full = false; + for (const auto& shard : server->shards_) { + s.shards.emplace_back(shard->workchain_, shard->shard_); + CHECK(s.shards.back().is_valid_ext()); + } servers_.push_back(std::move(s)); } diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index ff03764a7..67b123c8b 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -578,8 +578,8 @@ validator.config.global zero_state:tonNode.blockIdExt init_block:tonNode.blockId config.global adnl:adnl.config.global dht:dht.config.global validator:validator.config.global = config.Global; liteserver.desc id:PublicKey ip:int port:int = liteserver.Desc; -liteserver.descV2 id:PublicKey ip:int port:int shards:(vector tonNode.shardId) = liteserver.Desc; -liteclient.config.global liteservers:(vector liteserver.Desc) validator:validator.config.global = liteclient.config.Global; +liteserver.descV2 id:PublicKey ip:int port:int shards:(vector tonNode.shardId) = liteserver.DescV2; +liteclient.config.global liteservers:(vector liteserver.desc) liteservers_v2:(vector liteserver.descV2) validator:validator.config.global = liteclient.config.Global; engine.adnl id:int256 category:int = engine.Adnl; engine.addr ip:int port:int categories:(vector int) priority_categories:(vector int) = engine.Addr; @@ -598,14 +598,8 @@ engine.gc ids:(vector int256) = engine.Gc; engine.dht.config dht:(vector engine.dht) gc:engine.gc = engine.dht.Config; engine.validator.fullNodeMaster port:int adnl:int256 = engine.validator.FullNodeMaster; engine.validator.fullNodeSlave ip:int port:int adnl:PublicKey = engine.validator.FullNodeSlave; -engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) - dht:(vector engine.dht) - validators:(vector engine.Validator) fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave) - fullnodemasters:(vector engine.validator.fullNodeMaster) - liteservers:(vector engine.liteServer) control:(vector engine.controlInterface) - gc:engine.gc = engine.validator.Config; -engine.validator.configV2 out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) +engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) dht:(vector engine.dht) validators:(vector engine.validator) collators:(vector engine.collator) fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave) diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 12399020f6b0a164ee8f20686f7629f4a83f5545..25b64d2125744eb1c87f0ccc2bca800b5edbe326 100644 GIT binary patch delta 308 zcmaE`i{-;M7T!m*^{p77;Oa)+a1l|1+w1R$r{<++=B4VDCFW$NB$nhCO@1gWzj=m8 z0vpTP`py@V|45mLlq~T)!k?2_l3JWvRF+z#=aO2SJaL1Bn6iVtz95=-n9;;_Qk!F> z|L`ydZI09sPhbVxGI?RI2zy#)I*7CRP1gb@7BFi6*M@VTKNf0r##PR}8vi dQ-GcbGh$!>8?m86V{*=-60mujpDe1W0ss;Jb=3d> delta 314 zcmeycjpe~E7T!m*^{p77;L1kca1ohfQgVFasd?#{d8vA3i8+}mi6!|(dd~TIX_@I0 z9V9l-5J_NTS#)gg`N@BzOhjzI)X(70$t+1NPAw`+Ez)yIElvih>XH7#!x+4|Q$sv~ zb&7F7koe@r?;<=vBS6OAJSODBJ-M)5d-8{lBa952-*yT!PA=$B6amV?9CKBUD<(+O zFGdoiksGKn%!q*jq create_tl_shard_id(const ShardIdF return create_tl_object(s.workchain, s.shard); } -inline tl_object_ptr unpack_engine_validator_config( - tl_object_ptr config) { - tl_object_ptr res; - ton_api::downcast_call(*config, td::overloaded( - [&](ton_api::engine_validator_config &c) { - res = create_tl_object( - c.out_port_, std::move(c.addrs_), std::move(c.adnl_), std::move(c.dht_), - std::move(c.validators_), - std::vector>(), c.fullnode_, - std::move(c.fullnodeslaves_), std::move(c.fullnodemasters_), - std::move(c.liteservers_), std::move(c.control_), - std::vector>(), std::move(c.gc_)); - }, - [&](ton_api::engine_validator_configV2 &c) { - res = std::make_unique(std::move(c)); - })); - return res; -} - } // namespace ton diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index e56dfb7be..2aaff6741 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -75,7 +75,7 @@ Config::Config() { full_node = ton::PublicKeyHash::zero(); } -Config::Config(const ton::ton_api::engine_validator_configV2 &config) { +Config::Config(const ton::ton_api::engine_validator_config &config) { full_node = ton::PublicKeyHash::zero(); out_port = static_cast(config.out_port_); if (!out_port) { @@ -177,7 +177,7 @@ Config::Config(const ton::ton_api::engine_validator_configV2 &config) { } } -ton::tl_object_ptr Config::tl() const { +ton::tl_object_ptr Config::tl() const { std::vector> addrs_vec; for (auto &x : addrs) { if (x.second.proxy) { @@ -257,17 +257,10 @@ ton::tl_object_ptr Config::tl() const { gc_vec->ids_.push_back(id.tl()); } - if (col_vec.empty() && shards_vec.empty()) { - return ton::create_tl_object( - out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), - full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(liteserver_vec), - std::move(control_vec), std::move(gc_vec)); - } else { - return ton::create_tl_object( - out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec), - full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(liteserver_vec), - std::move(control_vec), std::move(shards_vec), std::move(gc_vec)); - } + return ton::create_tl_object( + out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec), + full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(liteserver_vec), + std::move(control_vec), std::move(shards_vec), std::move(gc_vec)); } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, @@ -1662,14 +1655,14 @@ void ValidatorEngine::load_config(td::Promise promise) { } auto conf_json = conf_json_R.move_as_ok(); - ton::tl_object_ptr conf; - auto S = td::from_json(conf, std::move(conf_json)); + ton::ton_api::engine_validator_config conf; + auto S = ton::ton_api::from_json(conf, conf_json.get_object()); if (S.is_error()) { promise.set_error(S.move_as_error_prefix("json does not fit TL scheme")); return; } - config_ = Config{*ton::unpack_engine_validator_config(std::move(conf))}; + config_ = Config{conf}; td::MultiPromise mp; auto ig = mp.init_guard(); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 40a04c703..69ef5cb00 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -137,10 +137,10 @@ struct Config { td::Result config_del_control_process(td::int32 port, ton::PublicKeyHash id); td::Result config_del_gc(ton::PublicKeyHash key); - ton::tl_object_ptr tl() const; + ton::tl_object_ptr tl() const; Config(); - Config(const ton::ton_api::engine_validator_configV2 &config); + Config(const ton::ton_api::engine_validator_config &config); }; class ValidatorEngine : public td::actor::Actor { diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 7bf44f33b..2101435fd 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -663,11 +663,11 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ne void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { BlockIdExt block_id = create_block_id(query.id_); - if (block_id.shard_full() != shard_) { - LOG(FULL_NODE_WARNING) << "dropping block broadcast: shard mismatch. overlay=" << shard_.to_str() - << " block=" << block_id.to_str(); - return; - } + //if (block_id.shard_full() != shard_) { + // LOG(FULL_NODE_WARNING) << "dropping block broadcast: shard mismatch. overlay=" << shard_.to_str() + // << " block=" << block_id.to_str(); + // return; + //} std::vector signatures; for (auto &sig : query.signatures_) { From c2e7d0b0dec18b26c8f6d751a36f48109e879731 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 23 Aug 2022 17:31:37 +0300 Subject: [PATCH 028/388] Bugfixes in manager.cpp --- validator/manager.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/validator/manager.cpp b/validator/manager.cpp index 2f2e5f024..2b6018364 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -214,6 +214,10 @@ void ValidatorManagerImpl::prevalidate_block(BlockBroadcast broadcast, td::Promi promise.set_error(td::Status::Error(ErrorCode::notready, "node not started")); return; } + if (!shards_to_monitor_.count(broadcast.block_id.shard_full())) { + promise.set_error(td::Status::Error("not monitoring shard")); + return; + } td::actor::create_actor("broadcast", std::move(broadcast), last_masterchain_block_handle_, last_masterchain_state_, last_known_key_block_handle_, actor_id(this), td::Timestamp::in(2.0), std::move(promise)) @@ -575,8 +579,9 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); }); auto id = - td::actor::create_actor("waitstate", handle, priority, actor_id(this), td::Timestamp::in(10.0), - std::move(P), get_block_persistent_state(handle->id())) + td::actor::create_actor("waitstate", handle, priority, actor_id(this), + td::Timestamp::at(timeout.at() + 10.0), std::move(P), + get_block_persistent_state(handle->id())) .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); @@ -614,7 +619,8 @@ void ValidatorManagerImpl::wait_out_msg_queue_proof(BlockIdExt block_id, ShardId }); auto id = td::actor::create_actor("waitmsgqueue", block_id, dst_shard, shards_to_monitor_.count(block_id.shard_full()), priority, - actor_id(this), td::Timestamp::in(10.0), std::move(P)) + actor_id(this), td::Timestamp::at(timeout.at() + 10.0), + std::move(P)) .release(); wait_out_msg_queue_proof_[key].actor_ = id; it = wait_out_msg_queue_proof_.find(key); @@ -636,7 +642,7 @@ void ValidatorManagerImpl::wait_block_data(BlockHandle handle, td::uint32 priori td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_data, handle, std::move(R)); }); auto id = td::actor::create_actor("waitdata", handle, priority, actor_id(this), - td::Timestamp::in(10.0), std::move(P)) + td::Timestamp::at(timeout.at() + 10.0), std::move(P)) .release(); wait_block_data_[handle->id()].actor_ = id; it = wait_block_data_.find(handle->id()); From 86250706b8ce8529c8e9f3e24baf6b69aff523b7 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 13 Sep 2022 16:02:55 +0300 Subject: [PATCH 029/388] Change format of specifying shards to monitor --- lite-client/lite-client.cpp | 5 ++++- validator-engine/validator-engine.cpp | 2 +- validator/collator-node.cpp | 2 +- validator/validator-group.cpp | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index 7165ca53c..2b559bc78 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -81,6 +81,9 @@ int TestNode::LiteServer::max_common_prefix(ton::ShardIdFull shard) const { } int res = -1; for (const ton::ShardIdFull &our_shard : shards) { + if (ton::shard_is_ancestor(our_shard, shard)) { + return shard.pfx_len(); + } if (shard.workchain == our_shard.workchain) { int x = std::min({shard.pfx_len(), our_shard.pfx_len(), ton::count_matching_bits(shard.shard, our_shard.shard)}); res = std::max(res, x); @@ -92,7 +95,7 @@ int TestNode::LiteServer::max_common_prefix(ton::ShardIdFull shard) const { bool TestNode::LiteServer::supports_shard(ton::ShardIdFull shard) const { return is_full || shard.is_masterchain() || std::any_of(shards.begin(), shards.end(), - [&](const ton::ShardIdFull& our_shard) { return ton::shard_is_ancestor(shard, our_shard); }); + [&](const ton::ShardIdFull& our_shard) { return ton::shard_intersects(shard, our_shard); }); } void TestNode::run() { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 2aaff6741..bb3bd0590 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1413,7 +1413,7 @@ void ValidatorEngine::set_shard_check_function() { validator_options_.write().set_shard_check_function( [shards = std::move(shards)](ton::ShardIdFull shard) -> bool { for (auto s : shards) { - if (shard_is_ancestor(shard, s)) { + if (shard_intersects(shard, s)) { return true; } } diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index c11b01c82..0a2a050eb 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -170,7 +170,7 @@ void CollatorNode::receive_query_cont(adnl::AdnlNodeIdShort src, ShardIdFull sha bool CollatorNode::collate_shard(ShardIdFull shard) const { for (ShardIdFull our_shard : shards_) { - if (shard_is_ancestor(shard, our_shard)) { + if (shard_intersects(shard, our_shard)) { return true; } } diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 88a473ff6..21517a205 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -364,7 +364,7 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo // TODO: some way to choose node (similar to "unreliability" in full-node) int cnt = 0; for (const block::CollatorNodeDescr& c : collator_config_.collator_nodes) { - if (shard_is_ancestor(shard_, c.shard) && td::Random::fast(0, cnt) == 0) { + if (shard_intersects(shard_, c.shard) && td::Random::fast(0, cnt) == 0) { collator = adnl::AdnlNodeIdShort(c.adnl_id); ++cnt; } From 62a412ac13627162a22cb5887b7d68572e87c55b Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 15 Sep 2022 20:14:34 +0300 Subject: [PATCH 030/388] Add some required collated data --- validator/full-node-shard.cpp | 3 ++ validator/impl/collator-impl.h | 7 +++- validator/impl/collator.cpp | 57 ++++++++++++++++++++++++++++--- validator/impl/validate-query.cpp | 2 +- 4 files changed, 62 insertions(+), 7 deletions(-) diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 2101435fd..56e0c1db1 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -130,6 +130,9 @@ void FullNodeShardImpl::create_overlay() { } void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broadcast, td::Promise promise) { + if (mode_ != FullNodeShardMode::active) { + return promise.set_error(td::Status::Error("cannot check broadcast: shard is not active")); + } auto B = fetch_tl_object(std::move(broadcast), true); if (B.is_error()) { return promise.set_error(B.move_as_error_prefix("failed to parse external message broadcast: ")); diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index b886173a7..58f87fcee 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -197,7 +197,8 @@ class Collator final : public td::actor::Actor { std::vector, ExtMessage::Hash>> ext_msg_list_; std::priority_queue, std::greater> new_msgs; std::pair last_proc_int_msg_, first_unproc_int_msg_; - std::unique_ptr in_msg_dict, out_msg_dict, out_msg_queue_, sibling_out_msg_queue_; + std::unique_ptr in_msg_dict, out_msg_dict, old_out_msg_queue_, out_msg_queue_, + sibling_out_msg_queue_; std::unique_ptr ihr_pending; std::shared_ptr processed_upto_, sibling_processed_upto_; std::unique_ptr block_create_stats_; @@ -292,6 +293,7 @@ class Collator final : public td::actor::Actor { bool process_new_messages(bool enqueue_only = false); int process_one_new_message(block::NewOutMsg msg, bool enqueue_only = false, Ref* is_special = nullptr); bool process_inbound_internal_messages(); + bool precheck_inbound_message(Ref msg, ton::LogicalTime lt); bool process_inbound_message(Ref msg, ton::LogicalTime lt, td::ConstBitPtr key, const block::McShardDescr& src_nb); bool process_inbound_external_messages(); @@ -333,8 +335,11 @@ class Collator final : public td::actor::Actor { bool update_cc); bool create_mc_block_extra(Ref& mc_block_extra); bool create_block(); + Ref collate_shard_block_descr_set(); + bool prepare_msg_queue_proof(); bool create_collated_data(); + bool create_block_candidate(); void return_block_candidate(td::Result saved); bool update_last_proc_int_msg(const std::pair& new_lt_hash); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 3468bc5de..98f0799c5 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -825,6 +825,7 @@ bool Collator::import_shard_state_data(block::ShardState& ss) { total_validator_fees_ = std::move(ss.total_validator_fees_); old_global_balance_ = std::move(ss.global_balance_); out_msg_queue_ = std::move(ss.out_msg_queue_); + old_out_msg_queue_ = std::make_unique(*out_msg_queue_); processed_upto_ = std::move(ss.processed_upto_); ihr_pending = std::move(ss.ihr_pending_); block_create_stats_ = std::move(ss.block_create_stats_); @@ -2619,8 +2620,7 @@ bool Collator::delete_out_msg_queue_msg(td::ConstBitPtr key) { return register_out_msg_queue_op(); } -bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalTime lt, td::ConstBitPtr key, - const block::McShardDescr& src_nb) { +bool Collator::precheck_inbound_message(Ref enq_msg, ton::LogicalTime lt) { ton::LogicalTime enqueued_lt = 0; if (enq_msg.is_null() || enq_msg->size_ext() != 0x10040 || (enqueued_lt = enq_msg->prefetch_ulong(64)) < /* 0 */ 1 * lt) { // DEBUG @@ -2646,6 +2646,13 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT LOG(ERROR) << "inbound internal MsgEnvelope is invalid according to automated checks"; return false; } + return true; +} + +bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalTime lt, td::ConstBitPtr key, + const block::McShardDescr& src_nb) { + ton::LogicalTime enqueued_lt = enq_msg->prefetch_ulong(64); + auto msg_env = enq_msg->prefetch_ref(); // 1. unpack MsgEnvelope block::tlb::MsgEnvelope::Record_std env; if (!tlb::unpack_cell(msg_env, env)) { @@ -2784,14 +2791,22 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT } bool Collator::process_inbound_internal_messages() { - while (!block_full_ && !nb_out_msgs_->is_eof()) { + while (!nb_out_msgs_->is_eof()) { block_full_ = !block_limit_status_->fits(block::ParamLimits::cl_normal); + auto kv = nb_out_msgs_->extract_cur(); + CHECK(kv && kv->msg.not_null()); + if (!precheck_inbound_message(kv->msg, kv->lt)) { + if (verbosity > 1) { + std::cerr << "invalid inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() + << " msg="; + block::gen::t_EnqueuedMsg.print(std::cerr, *(kv->msg)); + } + return fatal_error("error processing inbound internal message"); + } if (block_full_) { LOG(INFO) << "BLOCK FULL, stop processing inbound internal messages"; break; } - auto kv = nb_out_msgs_->extract_cur(); - CHECK(kv && kv->msg.not_null()); LOG(DEBUG) << "processing inbound message with (lt,hash)=(" << kv->lt << "," << kv->key.to_hex() << ") from neighbor #" << kv->source; if (verbosity > 2) { @@ -3974,6 +3989,34 @@ Ref Collator::collate_shard_block_descr_set() { return cell; } +bool Collator::prepare_msg_queue_proof() { + auto res = old_out_msg_queue_->scan_diff( + *out_msg_queue_, + [this](td::ConstBitPtr key, int key_len, Ref old_value, Ref new_value) { + old_value = old_out_msg_queue_->extract_value(std::move(old_value)); + new_value = out_msg_queue_->extract_value(std::move(new_value)); + if (new_value.not_null()) { + if (!block::gen::t_EnqueuedMsg.validate_csr(new_value)) { + return false; + } + if (!block::tlb::t_EnqueuedMsg.validate_csr(new_value)) { + return false; + } + } + if (old_value.not_null()) { + if (!block::gen::t_EnqueuedMsg.validate_csr(old_value)) { + return false; + } + if (!block::tlb::t_EnqueuedMsg.validate_csr(old_value)) { + return false; + } + } + return true; + }, + 3); + return res; +} + bool Collator::create_collated_data() { // 1. store the set of used shard block descriptions if (!used_shard_block_descr_.empty()) { @@ -3994,6 +4037,10 @@ bool Collator::create_collated_data() { // 3. Previous state proof (only shadchains) std::map> proofs; if (!is_masterchain()) { + if (!prepare_msg_queue_proof()) { + return fatal_error("cannot prepare message queue proof"); + } + state_usage_tree_->set_use_mark_for_is_loaded(false); Ref state_proof = vm::MerkleProof::generate(prev_state_root_, state_usage_tree_.get()); if (state_proof.is_null()) { diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index eeb2726c6..aa342fc44 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -2678,7 +2678,7 @@ bool ValidateQuery::precheck_one_message_queue_update(td::ConstBitPtr out_msg_id } ton::LogicalTime enqueued_lt = old_value->prefetch_ulong(64); if (enqueued_lt >= start_lt_) { - return reject_query(PSTRING() << "new EnqueuedMsg with key "s + out_msg_id.to_hex(352) + " has enqueued_lt=" + return reject_query(PSTRING() << "old EnqueuedMsg with key "s + out_msg_id.to_hex(352) + " has enqueued_lt=" << enqueued_lt << " greater than or equal to this block's start_lt=" << start_lt_); } } From b0c2c6c525f58f882c3b3f3cb9b9639c8da0d802 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 23 Sep 2022 17:23:00 +0300 Subject: [PATCH 031/388] Don't send proof of masterchain message queue --- validator/impl/collator.cpp | 28 +++++++++++++++++----------- validator/impl/validate-query.cpp | 9 +++++++++ 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 98f0799c5..5d0d2972d 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -500,14 +500,15 @@ void Collator::after_get_block_data(int idx, td::Result> res) { CHECK(!idx); prev_mc_block = prev_block_data[0]; mc_block_root = prev_mc_block->root_cell(); + } else { + Ref root = prev_block_data[idx]->root_cell(); + auto proof = create_block_state_proof(root); + if (proof.is_error()) { + fatal_error(proof.move_as_error()); + return; + } + block_state_proofs_.emplace(root->get_hash().bits(), proof.move_as_ok()); } - Ref root = prev_block_data[idx]->root_cell(); - auto proof = create_block_state_proof(root); - if (proof.is_error()) { - fatal_error(proof.move_as_error()); - return; - } - block_state_proofs_.emplace(root->get_hash().bits(), proof.move_as_ok()); } check_pending(); } @@ -622,7 +623,6 @@ bool Collator::request_neighbor_msg_queues() { neighbors_.emplace_back(*shard_ptr); } unsigned i = 0; - neighbor_proof_builders_.resize(neighbors_.size()); for (block::McShardDescr& descr : neighbors_) { LOG(DEBUG) << "neighbor #" << i << " : " << descr.blk_.to_str(); ++pending; @@ -644,11 +644,17 @@ void Collator::got_neighbor_msg_queue(unsigned i, td::Resultblock_state_proof_.not_null()) { + if (res->block_state_proof_.not_null() && !block_id.is_masterchain()) { block_state_proofs_.emplace(block_id.root_hash, res->block_state_proof_); } - neighbor_proof_builders_.at(i) = vm::MerkleProofBuilder{res->state_root_}; - auto state = ShardStateQ::fetch(block_id, {}, neighbor_proof_builders_.at(i).root()); + Ref state_root; + if (block_id.is_masterchain()) { + state_root = res->state_root_; + } else { + neighbor_proof_builders_.push_back(vm::MerkleProofBuilder{res->state_root_}); + state_root = neighbor_proof_builders_.back().root(); + } + auto state = ShardStateQ::fetch(block_id, {}, state_root); if (state.is_error()) { fatal_error(state.move_as_error()); return; diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index aa342fc44..f8a2708d8 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -1258,6 +1258,15 @@ bool ValidateQuery::request_neighbor_queues() { if (full_collated_data_) { for (block::McShardDescr& descr : neighbors_) { LOG(DEBUG) << "getting outbound queue of neighbor #" << i << " from collated data : " << descr.blk_.to_str(); + if (descr.blk_.is_masterchain()) { + if (descr.blk_ != mc_state_->get_block_id()) { + return fatal_error("neighbor from masterchain is not the last mc block"); + } + ++pending; + send_closure_later(get_self(), &ValidateQuery::got_neighbor_out_queue, i, mc_state_->message_queue()); + ++i; + continue; + } td::Bits256 state_root_hash; if (descr.blk_.seqno() == 0) { state_root_hash = descr.blk_.root_hash; From 69cee95abbe64d621f6e35d3ec88095b7e08d989 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 4 Oct 2022 11:07:38 +0300 Subject: [PATCH 032/388] Bugfixes in rldp-http-proxy and http parser --- http/http-inbound-connection.cpp | 2 +- http/http-outbound-connection.cpp | 2 +- http/http.cpp | 2 +- rldp-http-proxy/rldp-http-proxy.cpp | 6 ++++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/http/http-inbound-connection.cpp b/http/http-inbound-connection.cpp index 89c9c1230..57064dddc 100644 --- a/http/http-inbound-connection.cpp +++ b/http/http-inbound-connection.cpp @@ -79,10 +79,10 @@ td::Status HttpInboundConnection::receive(td::ChainBufferReader &input) { send_client_error(); return td::Status::OK(); } + cur_request_ = R.move_as_ok(); if (exit_loop) { return td::Status::OK(); } - cur_request_ = R.move_as_ok(); } auto payload = cur_request_->create_empty_payload().move_as_ok(); diff --git a/http/http-outbound-connection.cpp b/http/http-outbound-connection.cpp index a98efbc93..63621df87 100644 --- a/http/http-outbound-connection.cpp +++ b/http/http-outbound-connection.cpp @@ -42,10 +42,10 @@ td::Status HttpOutboundConnection::receive(td::ChainBufferReader &input) { answer_error(HttpStatusCode::status_bad_request, "", std::move(promise_)); return td::Status::OK(); } + cur_response_ = R.move_as_ok(); if (exit_loop) { return td::Status::OK(); } - cur_response_ = R.move_as_ok(); } if (cur_response_->code() == 100) { diff --git a/http/http.cpp b/http/http.cpp index cefe1a475..cc1fd889d 100644 --- a/http/http.cpp +++ b/http/http.cpp @@ -587,7 +587,7 @@ tl_object_ptr HttpPayload::store_tl(size_t max_size) max_size -= s.size(); } obj->data_.truncate(obj->data_.size() - S.size()); - if (chunks_.size() != 0) { + if (chunks_.size() != 0 || !parse_completed()) { return obj; } if (!written_zero_chunk_) { diff --git a/rldp-http-proxy/rldp-http-proxy.cpp b/rldp-http-proxy/rldp-http-proxy.cpp index a5faa002c..0b679c271 100644 --- a/rldp-http-proxy/rldp-http-proxy.cpp +++ b/rldp-http-proxy/rldp-http-proxy.cpp @@ -117,7 +117,7 @@ class HttpRemote : public td::actor::Actor { } }); td::actor::send_closure(client_, &ton::http::HttpClient::send_request, std::move(request), std::move(payload), - td::Timestamp::in(30.0), std::move(P)); + td::Timestamp::never(), std::move(P)); } else { ton::http::answer_error(ton::http::HttpStatusCode::status_bad_request, "", std::move(promise)); } @@ -801,6 +801,7 @@ class RldpToTcpRequestSender : public td::actor::Actor { , dst_(dst) , request_(std::move(request)) , request_payload_(std::move(request_payload)) + , proto_version_(request_->proto_version()) , promise_(std::move(promise)) , adnl_(adnl) , rldp_(rldp) @@ -836,7 +837,7 @@ class RldpToTcpRequestSender : public td::actor::Actor { void abort_query(td::Status error) { LOG(INFO) << "aborting http over rldp query: " << error; - promise_.set_result(create_error_response(request_->proto_version(), 502, "Bad Gateway")); + promise_.set_result(create_error_response(proto_version_, 502, "Bad Gateway")); stop(); } @@ -848,6 +849,7 @@ class RldpToTcpRequestSender : public td::actor::Actor { std::unique_ptr request_; std::shared_ptr request_payload_; + std::string proto_version_; td::Promise promise_; From 33a079962fca393b06168cc497cf2fcdbf3ca854 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 4 Oct 2022 15:12:13 +0300 Subject: [PATCH 033/388] Tonlib: change liteservers on query timeout or connection close --- adnl/adnl-ext-client.hpp | 3 ++ adnl/adnl-query.cpp | 7 ++- adnl/adnl-query.h | 1 + rldp-http-proxy/DNSResolver.cpp | 13 ++++- rldp-http-proxy/DNSResolver.h | 1 + tonlib/tonlib/ExtClient.cpp | 6 +-- tonlib/tonlib/ExtClient.h | 9 +++- tonlib/tonlib/ExtClientLazy.cpp | 77 +++++++++++++++++++++++------ tonlib/tonlib/ExtClientLazy.h | 11 +++-- tonlib/tonlib/ExtClientOutbound.cpp | 3 ++ tonlib/tonlib/ExtClientOutbound.h | 4 +- tonlib/tonlib/LastBlock.cpp | 1 + tonlib/tonlib/LastConfig.cpp | 1 + tonlib/tonlib/TonlibClient.cpp | 13 +++-- tonlib/tonlib/TonlibClient.h | 2 +- tonlib/tonlib/tonlib-cli.cpp | 2 +- 16 files changed, 117 insertions(+), 37 deletions(-) diff --git a/adnl/adnl-ext-client.hpp b/adnl/adnl-ext-client.hpp index a4df818e7..133397255 100644 --- a/adnl/adnl-ext-client.hpp +++ b/adnl/adnl-ext-client.hpp @@ -80,6 +80,9 @@ class AdnlExtClientImpl : public AdnlExtClient { if (!conn_.empty() && conn_.get() == conn) { callback_->on_stop_ready(); conn_ = {}; + for (auto& q : out_queries_) { + td::actor::send_closure(q.second, &AdnlQuery::set_error, td::Status::Error(ErrorCode::cancelled)); + } alarm_timestamp() = next_create_at_; try_stop(); } diff --git a/adnl/adnl-query.cpp b/adnl/adnl-query.cpp index 5bc767d2b..e098c1344 100644 --- a/adnl/adnl-query.cpp +++ b/adnl/adnl-query.cpp @@ -25,13 +25,16 @@ namespace ton { namespace adnl { void AdnlQuery::alarm() { - promise_.set_error(td::Status::Error(ErrorCode::timeout, "adnl query timeout")); - stop(); + set_error(td::Status::Error(ErrorCode::timeout, "adnl query timeout")); } void AdnlQuery::result(td::BufferSlice data) { promise_.set_value(std::move(data)); stop(); } +void AdnlQuery::set_error(td::Status error) { + promise_.set_error(std::move(error)); + stop(); +} AdnlQueryId AdnlQuery::random_query_id() { AdnlQueryId q_id; diff --git a/adnl/adnl-query.h b/adnl/adnl-query.h index 6e24a49fe..3db8c754c 100644 --- a/adnl/adnl-query.h +++ b/adnl/adnl-query.h @@ -48,6 +48,7 @@ class AdnlQuery : public td::actor::Actor { } void alarm() override; void result(td::BufferSlice data); + void set_error(td::Status error); void start_up() override { alarm_timestamp() = timeout_; } diff --git a/rldp-http-proxy/DNSResolver.cpp b/rldp-http-proxy/DNSResolver.cpp index a75197420..1fb197a57 100644 --- a/rldp-http-proxy/DNSResolver.cpp +++ b/rldp-http-proxy/DNSResolver.cpp @@ -25,6 +25,7 @@ */ #include "DNSResolver.h" #include "td/utils/overloaded.h" +#include "common/delay.h" static const double CACHE_TIMEOUT_HARD = 300.0; static const double CACHE_TIMEOUT_SOFT = 270.0; @@ -33,8 +34,18 @@ DNSResolver::DNSResolver(td::actor::ActorId tonlib_client) : tonli } void DNSResolver::start_up() { + sync(); +} + +void DNSResolver::sync() { auto obj = tonlib_api::make_object(); - auto P = td::PromiseCreator::lambda([](td::Result>) {}); + auto P = td::PromiseCreator::lambda([SelfId = + actor_id(this)](td::Result> R) { + if (R.is_error()) { + LOG(WARNING) << "Sync error: " << R.move_as_error(); + ton::delay_action([SelfId]() { td::actor::send_closure(SelfId, &DNSResolver::sync); }, td::Timestamp::in(5.0)); + } + }); td::actor::send_closure(tonlib_client_, &TonlibClient::send_request, std::move(obj), std::move(P)); } diff --git a/rldp-http-proxy/DNSResolver.h b/rldp-http-proxy/DNSResolver.h index f87b5e5a8..ba52a67e3 100644 --- a/rldp-http-proxy/DNSResolver.h +++ b/rldp-http-proxy/DNSResolver.h @@ -37,6 +37,7 @@ class DNSResolver : public td::actor::Actor { void resolve(std::string host, td::Promise promise); private: + void sync(); void save_to_cache(std::string host, ton::adnl::AdnlNodeIdShort id); td::actor::ActorId tonlib_client_; diff --git a/tonlib/tonlib/ExtClient.cpp b/tonlib/tonlib/ExtClient.cpp index 8c8b5d074..30a29b59c 100644 --- a/tonlib/tonlib/ExtClient.cpp +++ b/tonlib/tonlib/ExtClient.cpp @@ -35,7 +35,7 @@ void ExtClient::with_last_config(td::Promise promise) { self->last_config_queries_.extract(query_id).set_result(std::move(result)); }); }; - if (client_.last_block_actor_.empty()) { + if (client_.last_config_actor_.empty()) { return P.set_error(TonlibError::NoLiteServers()); } td::actor::send_closure(client_.last_config_actor_, &LastConfig::get_last_config, std::move(P)); @@ -62,10 +62,10 @@ void ExtClient::send_raw_query(td::BufferSlice query, td::Promisequeries_.extract(query_id).set_result(std::move(result)); }); }; - if (client_.andl_ext_client_.empty()) { + if (client_.adnl_ext_client_.empty()) { return P.set_error(TonlibError::NoLiteServers()); } - td::actor::send_closure(client_.andl_ext_client_, &ton::adnl::AdnlExtClient::send_query, "query", std::move(query), + td::actor::send_closure(client_.adnl_ext_client_, &ton::adnl::AdnlExtClient::send_query, "query", std::move(query), td::Timestamp::in(10.0), std::move(P)); } } // namespace tonlib diff --git a/tonlib/tonlib/ExtClient.h b/tonlib/tonlib/ExtClient.h index 882b4794a..707ca74bd 100644 --- a/tonlib/tonlib/ExtClient.h +++ b/tonlib/tonlib/ExtClient.h @@ -28,6 +28,7 @@ #include "td/utils/Container.h" #include "td/utils/Random.h" +#include "ExtClientLazy.h" #include "TonlibError.h" #include "utils.h" @@ -37,7 +38,7 @@ class LastConfig; struct LastBlockState; struct LastConfigState; struct ExtClientRef { - td::actor::ActorId andl_ext_client_; + td::actor::ActorId adnl_ext_client_; td::actor::ActorId last_block_actor_; td::actor::ActorId last_config_actor_; }; @@ -94,6 +95,12 @@ class ExtClient { }); } + void force_change_liteserver() { + if (!client_.adnl_ext_client_.empty()) { + td::actor::send_closure(client_.adnl_ext_client_, &ExtClientLazy::force_change_liteserver); + } + } + private: ExtClientRef client_; td::Container> queries_; diff --git a/tonlib/tonlib/ExtClientLazy.cpp b/tonlib/tonlib/ExtClientLazy.cpp index 1ab7a24f8..335a0ff96 100644 --- a/tonlib/tonlib/ExtClientLazy.cpp +++ b/tonlib/tonlib/ExtClientLazy.cpp @@ -18,13 +18,20 @@ */ #include "ExtClientLazy.h" #include "TonlibError.h" +#include "td/utils/Random.h" namespace tonlib { -class ExtClientLazyImp : public ton::adnl::AdnlExtClient { +class ExtClientLazyImp : public ExtClientLazy { public: - ExtClientLazyImp(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, + ExtClientLazyImp(std::vector> servers, td::unique_ptr callback) - : dst_(std::move(dst)), dst_addr_(std::move(dst_addr)), callback_(std::move(callback)) { + : servers_(std::move(servers)), callback_(std::move(callback)) { + CHECK(!servers_.empty()); + } + + void start_up() override { + td::Random::Fast rnd; + td::random_shuffle(td::as_mutable_span(servers_), rnd); } void check_ready(td::Promise promise) override { @@ -41,37 +48,66 @@ class ExtClientLazyImp : public ton::adnl::AdnlExtClient { if (client_.empty()) { return promise.set_error(TonlibError::Cancelled()); } + td::Promise P = [SelfId = actor_id(this), idx = cur_server_idx_, + promise = std::move(promise)](td::Result R) mutable { + if (R.is_error() && + (R.error().code() == ton::ErrorCode::timeout || R.error().code() == ton::ErrorCode::cancelled)) { + td::actor::send_closure(SelfId, &ExtClientLazyImp::set_server_bad, idx, true); + } + promise.set_result(std::move(R)); + }; send_closure(client_, &ton::adnl::AdnlExtClient::send_query, std::move(name), std::move(data), timeout, - std::move(promise)); + std::move(P)); + } + + void force_change_liteserver() override { + if (servers_.size() == 1) { + return; + } + cur_server_bad_ = cur_server_bad_force_ = true; } + private: void before_query() { if (is_closing_) { return; } - if (!client_.empty()) { - alarm_timestamp() = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT); + alarm_timestamp() = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT); + if (cur_server_bad_) { + ++cur_server_idx_; + } else if (!client_.empty()) { return; } class Callback : public ton::adnl::AdnlExtClient::Callback { public: - explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { + explicit Callback(td::actor::ActorShared parent, size_t idx) + : parent_(std::move(parent)), idx_(idx) { } void on_ready() override { + td::actor::send_closure(parent_, &ExtClientLazyImp::set_server_bad, idx_, false); } void on_stop_ready() override { + td::actor::send_closure(parent_, &ExtClientLazyImp::set_server_bad, idx_, true); } private: - td::actor::ActorShared<> parent_; + td::actor::ActorShared parent_; + size_t idx_; }; ref_cnt_++; - client_ = ton::adnl::AdnlExtClient::create(dst_, dst_addr_, std::make_unique(td::actor::actor_shared())); + cur_server_bad_ = false; + cur_server_bad_force_ = false; + const auto& s = servers_[cur_server_idx_ % servers_.size()]; + LOG(INFO) << "Connecting to liteserver " << s.second; + client_ = ton::adnl::AdnlExtClient::create( + s.first, s.second, std::make_unique(td::actor::actor_shared(this), cur_server_idx_)); } - private: - ton::adnl::AdnlNodeIdFull dst_; - td::IPAddress dst_addr_; + std::vector> servers_; + size_t cur_server_idx_ = 0; + bool cur_server_bad_ = false; + bool cur_server_bad_force_ = false; + td::actor::ActorOwn client_; td::unique_ptr callback_; static constexpr double MAX_NO_QUERIES_TIMEOUT = 100; @@ -79,6 +115,11 @@ class ExtClientLazyImp : public ton::adnl::AdnlExtClient { bool is_closing_{false}; td::uint32 ref_cnt_{1}; + void set_server_bad(size_t idx, bool bad) { + if (idx == cur_server_idx_ && servers_.size() > 1 && !cur_server_bad_force_) { + cur_server_bad_ = bad; + } + } void alarm() override { client_.reset(); } @@ -99,9 +140,13 @@ class ExtClientLazyImp : public ton::adnl::AdnlExtClient { } }; -td::actor::ActorOwn ExtClientLazy::create(ton::adnl::AdnlNodeIdFull dst, - td::IPAddress dst_addr, - td::unique_ptr callback) { - return td::actor::create_actor("ExtClientLazy", dst, dst_addr, std::move(callback)); +td::actor::ActorOwn ExtClientLazy::create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, + td::unique_ptr callback) { + return create({std::make_pair(dst, dst_addr)}, std::move(callback)); +} + +td::actor::ActorOwn ExtClientLazy::create( + std::vector> servers, td::unique_ptr callback) { + return td::actor::create_actor("ExtClientLazy", std::move(servers), std::move(callback)); } } // namespace tonlib diff --git a/tonlib/tonlib/ExtClientLazy.h b/tonlib/tonlib/ExtClientLazy.h index 6014abd3d..dc4490b3a 100644 --- a/tonlib/tonlib/ExtClientLazy.h +++ b/tonlib/tonlib/ExtClientLazy.h @@ -22,15 +22,20 @@ #include "adnl/adnl-ext-client.h" namespace tonlib { -class ExtClientLazy { +class ExtClientLazy : public ton::adnl::AdnlExtClient { public: class Callback { public: virtual ~Callback() { } }; - static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, - td::unique_ptr callback); + + virtual void force_change_liteserver() = 0; + + static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, + td::unique_ptr callback); + static td::actor::ActorOwn create( + std::vector> servers, td::unique_ptr callback); }; } // namespace tonlib diff --git a/tonlib/tonlib/ExtClientOutbound.cpp b/tonlib/tonlib/ExtClientOutbound.cpp index fcccc4d4b..025ba848e 100644 --- a/tonlib/tonlib/ExtClientOutbound.cpp +++ b/tonlib/tonlib/ExtClientOutbound.cpp @@ -38,6 +38,9 @@ class ExtClientOutboundImp : public ExtClientOutbound { callback_->request(query_id, data.as_slice().str()); } + void force_change_liteserver() override { + } + void on_query_result(td::int64 id, td::Result r_data, td::Promise promise) override { auto it = queries_.find(id); if (it == queries_.end()) { diff --git a/tonlib/tonlib/ExtClientOutbound.h b/tonlib/tonlib/ExtClientOutbound.h index 7bab49030..87b73b9e0 100644 --- a/tonlib/tonlib/ExtClientOutbound.h +++ b/tonlib/tonlib/ExtClientOutbound.h @@ -19,10 +19,10 @@ #pragma once #include "td/actor/actor.h" -#include "adnl/adnl-ext-client.h" +#include "ExtClientLazy.h" namespace tonlib { -class ExtClientOutbound : public ton::adnl::AdnlExtClient { +class ExtClientOutbound : public ExtClientLazy { public: class Callback { public: diff --git a/tonlib/tonlib/LastBlock.cpp b/tonlib/tonlib/LastBlock.cpp index bc2b74ba8..10a4d10fb 100644 --- a/tonlib/tonlib/LastBlock.cpp +++ b/tonlib/tonlib/LastBlock.cpp @@ -374,6 +374,7 @@ void LastBlock::on_sync_error(td::Status status) { promise.set_error(status.clone()); } promises_.clear(); + client_.force_change_liteserver(); } void LastBlock::on_fatal_error(td::Status status) { VLOG(last_block) << "sync: fatal error " << status; diff --git a/tonlib/tonlib/LastConfig.cpp b/tonlib/tonlib/LastConfig.cpp index 0111095bc..960d59946 100644 --- a/tonlib/tonlib/LastConfig.cpp +++ b/tonlib/tonlib/LastConfig.cpp @@ -141,6 +141,7 @@ void LastConfig::on_error(td::Status status) { promise.set_error(status.clone()); } promises_.clear(); + get_config_state_ = QueryState::Empty; } void LastConfig::tear_down() { diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 842ea5fdf..f9e984bb9 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -1649,7 +1649,7 @@ void TonlibClient::hangup() { ExtClientRef TonlibClient::get_client_ref() { ExtClientRef ref; - ref.andl_ext_client_ = raw_client_.get(); + ref.adnl_ext_client_ = raw_client_.get(); ref.last_block_actor_ = raw_last_block_.get(); ref.last_config_actor_ = raw_last_config_.get(); @@ -1683,10 +1683,10 @@ void TonlibClient::init_ext_client() { ext_client_outbound_ = client.get(); raw_client_ = std::move(client); } else { - auto lite_clients_size = config_.lite_clients.size(); - CHECK(lite_clients_size != 0); - auto lite_client_id = td::Random::fast(0, td::narrow_cast(lite_clients_size) - 1); - auto& lite_client = config_.lite_clients[lite_client_id]; + std::vector> servers; + for (const auto& s : config_.lite_clients) { + servers.emplace_back(s.adnl_id, s.address); + } class Callback : public ExtClientLazy::Callback { public: explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { @@ -1697,8 +1697,7 @@ void TonlibClient::init_ext_client() { }; ext_client_outbound_ = {}; ref_cnt_++; - raw_client_ = ExtClientLazy::create(lite_client.adnl_id, lite_client.address, - td::make_unique(td::actor::actor_shared())); + raw_client_ = ExtClientLazy::create(std::move(servers), td::make_unique(td::actor::actor_shared())); } } diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index 800d8c806..e78192909 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -110,7 +110,7 @@ class TonlibClient : public td::actor::Actor { vm::Dictionary libraries{256}; // network - td::actor::ActorOwn raw_client_; + td::actor::ActorOwn raw_client_; td::actor::ActorId ext_client_outbound_; td::actor::ActorOwn raw_last_block_; td::actor::ActorOwn raw_last_config_; diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index e889234ac..0a042eb1d 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -174,7 +174,7 @@ class TonlibCli : public td::actor::Actor { std::map>> query_handlers_; - td::actor::ActorOwn raw_client_; + td::actor::ActorOwn raw_client_; bool is_closing_{false}; td::uint32 ref_cnt_{1}; From 836184e566de6a3e6ee44e0a06f9c5f85b98dfbf Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 4 Oct 2022 18:25:25 +0300 Subject: [PATCH 034/388] Increase maximum size of http request --- rldp-http-proxy/rldp-http-proxy.cpp | 9 ++++----- rldp/rldp-in.hpp | 7 ++++++- rldp/rldp.cpp | 4 ++-- rldp/rldp.h | 6 ++---- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/rldp-http-proxy/rldp-http-proxy.cpp b/rldp-http-proxy/rldp-http-proxy.cpp index 0b679c271..a45e6433d 100644 --- a/rldp-http-proxy/rldp-http-proxy.cpp +++ b/rldp-http-proxy/rldp-http-proxy.cpp @@ -825,11 +825,9 @@ class RldpToTcpRequestSender : public td::actor::Actor { } void got_result(std::pair, std::shared_ptr> R) { - if (R.first->need_payload()) { - td::actor::create_actor("HttpPayloadSender(R)", std::move(R.second), id_, local_id_, adnl_, - rldp_) - .release(); - } + td::actor::create_actor("HttpPayloadSender(R)", std::move(R.second), id_, local_id_, adnl_, + rldp_) + .release(); auto f = ton::serialize_tl_object(R.first->store_tl(), true); promise_.set_value(std::move(f)); stop(); @@ -1092,6 +1090,7 @@ class RldpHttpProxy : public td::actor::Actor { } rldp_ = ton::rldp::Rldp::create(adnl_.get()); + td::actor::send_closure(rldp_, &ton::rldp::Rldp::set_default_mtu, 16 << 10); td::actor::send_closure(rldp_, &ton::rldp::Rldp::add_id, local_id_); for (auto &serv_id : server_ids_) { td::actor::send_closure(rldp_, &ton::rldp::Rldp::add_id, serv_id); diff --git a/rldp/rldp-in.hpp b/rldp/rldp-in.hpp index b49819993..9640fd98e 100644 --- a/rldp/rldp-in.hpp +++ b/rldp/rldp-in.hpp @@ -71,7 +71,7 @@ class RldpIn : public RldpImpl { void send_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice data) override { - send_query_ex(src, dst, name, std::move(promise), timeout, std::move(data), default_mtu()); + send_query_ex(src, dst, name, std::move(promise), timeout, std::move(data), default_mtu_); } void send_query_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice data, @@ -101,6 +101,10 @@ class RldpIn : public RldpImpl { void add_id(adnl::AdnlNodeIdShort local_id) override; void get_conn_ip_str(adnl::AdnlNodeIdShort l_id, adnl::AdnlNodeIdShort p_id, td::Promise promise) override; + void set_default_mtu(td::uint64 mtu) override { + default_mtu_ = mtu; + } + RldpIn(td::actor::ActorId adnl) : adnl_(adnl) { } @@ -116,6 +120,7 @@ class RldpIn : public RldpImpl { std::set lru_set_; RldpLru lru_; td::uint32 lru_size_ = 0; + td::uint64 default_mtu_ = adnl::Adnl::get_mtu(); std::map max_size_; diff --git a/rldp/rldp.cpp b/rldp/rldp.cpp index 9b38dcb8c..1a772a688 100644 --- a/rldp/rldp.cpp +++ b/rldp/rldp.cpp @@ -116,9 +116,9 @@ void RldpIn::process_message_part(adnl::AdnlNodeIdShort source, adnl::AdnlNodeId } auto ite = max_size_.find(part.transfer_id_); if (ite == max_size_.end()) { - if (static_cast(part.total_size_) > default_mtu()) { + if (static_cast(part.total_size_) > default_mtu_) { VLOG(RLDP_NOTICE) << "dropping too big rldp packet of size=" << part.total_size_ - << " default_mtu=" << default_mtu(); + << " default_mtu=" << default_mtu_; return; } } else { diff --git a/rldp/rldp.h b/rldp/rldp.h index edb19d7a4..b8ce3623b 100644 --- a/rldp/rldp.h +++ b/rldp/rldp.h @@ -28,15 +28,13 @@ class Rldp : public adnl::AdnlSenderInterface { public: virtual ~Rldp() = default; - static constexpr td::uint64 default_mtu() { - return adnl::Adnl::get_mtu(); - } - virtual void add_id(adnl::AdnlNodeIdShort local_id) = 0; virtual void send_message_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, td::BufferSlice data) = 0; + virtual void set_default_mtu(td::uint64 mtu) = 0; + static td::actor::ActorOwn create(td::actor::ActorId adnl); }; From 5500acd1ba9fcc6782107ff0d10fc69ec48ae2c6 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 5 Oct 2022 23:05:23 +0300 Subject: [PATCH 035/388] Minor bugfixes in http --- http/http.cpp | 45 +++++++++++++++++++++------------------------ 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/http/http.cpp b/http/http.cpp index cc1fd889d..dd8ab8093 100644 --- a/http/http.cpp +++ b/http/http.cpp @@ -279,25 +279,20 @@ td::Status HttpPayload::parse(td::ChainBufferReader &input) { } break; case ParseState::reading_chunk_data: { if (cur_chunk_size_ == 0) { - switch (type_) { - case PayloadType::pt_empty: - UNREACHABLE(); - case PayloadType::pt_eof: - case PayloadType::pt_tunnel: - cur_chunk_size_ = 1LL << 60; - break; - case PayloadType::pt_chunked: - state_ = ParseState::reading_crlf; - break; - case PayloadType::pt_content_length: { - LOG(INFO) << "payload parse success"; - const std::lock_guard lock{mutex_}; - state_ = ParseState::completed; - run_callbacks(); - return td::Status::OK(); - } break; + if (type_ == PayloadType::pt_eof || type_ == PayloadType::pt_tunnel) { + cur_chunk_size_ = 1LL << 60; + } else if (type_ == PayloadType::pt_chunked) { + state_ = ParseState::reading_crlf; + break; + } else if (type_ == PayloadType::pt_content_length) { + LOG(INFO) << "payload parse success"; + const std::lock_guard lock{mutex_}; + state_ = ParseState::completed; + run_callbacks(); + return td::Status::OK(); + } else { + UNREACHABLE(); } - break; } if (input.size() == 0) { return td::Status::OK(); @@ -502,7 +497,7 @@ bool HttpPayload::store_http(td::ChainBufferWriter &output, size_t max_size, Htt char buf[64]; ::sprintf(buf, "%lx\r\n", s.size()); auto slice = td::Slice(buf, strlen(buf)); - wrote |= !slice.empty(); + wrote = true; output.append(slice); } @@ -514,7 +509,8 @@ bool HttpPayload::store_http(td::ChainBufferWriter &output, size_t max_size, Htt wrote = true; } } - if (chunks_.size() != 0 || !parse_completed()) { + auto cur_state = state_.load(std::memory_order_consume); + if (chunks_.size() != 0 || (cur_state != ParseState::reading_trailer && cur_state != ParseState::completed)) { return wrote; } if (!written_zero_chunk_) { @@ -531,7 +527,7 @@ bool HttpPayload::store_http(td::ChainBufferWriter &output, size_t max_size, Htt } while (max_size > 0) { - auto cur_state = state_.load(std::memory_order_consume); + cur_state = state_.load(std::memory_order_consume); HttpHeader h = get_header(); if (h.empty()) { if (cur_state != ParseState::completed) { @@ -587,7 +583,8 @@ tl_object_ptr HttpPayload::store_tl(size_t max_size) max_size -= s.size(); } obj->data_.truncate(obj->data_.size() - S.size()); - if (chunks_.size() != 0 || !parse_completed()) { + auto cur_state = state_.load(std::memory_order_consume); + if (chunks_.size() != 0 || (cur_state != ParseState::reading_trailer && cur_state != ParseState::completed)) { return obj; } if (!written_zero_chunk_) { @@ -597,7 +594,7 @@ tl_object_ptr HttpPayload::store_tl(size_t max_size) LOG(INFO) << "data completed"; while (max_size > 0) { - auto cur_state = state_.load(std::memory_order_consume); + cur_state = state_.load(std::memory_order_consume); HttpHeader h = get_header(); if (h.empty()) { if (cur_state != ParseState::completed) { @@ -869,7 +866,7 @@ td::Status HttpHeader::basic_check() { } for (auto &c : value) { if (c == '\r' || c == '\n') { - return td::Status::Error("bad character in header name"); + return td::Status::Error("bad character in header value"); } } return td::Status::OK(); From 699a56b951b196aa96d819889e08ea9a2fd83563 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 27 Sep 2022 16:33:27 +0300 Subject: [PATCH 036/388] Send block broadcasts directly to collators --- crypto/block/block.tlb | 3 ++- crypto/block/mc-config.cpp | 9 +++++++-- crypto/block/mc-config.h | 1 + overlay/overlay-broadcast.cpp | 6 +++++- overlay/overlay-broadcast.hpp | 6 ++++-- overlay/overlay-fec-broadcast.cpp | 20 +++++++++++++------- overlay/overlay-fec-broadcast.hpp | 15 ++++++++++----- overlay/overlay-manager.cpp | 13 +++++++++++++ overlay/overlay-manager.h | 3 +++ overlay/overlay.h | 1 + overlay/overlay.hpp | 19 ++++++++++++++++++- overlay/overlays.h | 3 +++ validator/full-node-shard.cpp | 13 +++++++++++++ validator/full-node-shard.h | 1 + validator/full-node-shard.hpp | 2 ++ validator/full-node.cpp | 30 ++++++++++++++++++++++++++++++ validator/full-node.hpp | 18 +++++++++++------- validator/impl/shard.cpp | 4 ++++ validator/impl/shard.hpp | 1 + validator/interfaces/shard.h | 1 + 20 files changed, 143 insertions(+), 26 deletions(-) diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 8cb6c2f0d..f1a1c9485 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -735,7 +735,8 @@ misbehaviour_punishment_config_v1#01 _ MisbehaviourPunishmentConfig = ConfigParam 40; // collator_nodes: each collator is (workchain:int32 shard:uint64 adnl_id:uint256) -colator_config#a0 full_collated_data:Bool collator_nodes:(HashmapE 352 Unit) = CollatorConfig; +collator_info#0 full_node_id:(Maybe uint256) = CollatorInfo; +colator_config#a0 full_collated_data:Bool collator_nodes:(HashmapE 352 CollatorInfo) = CollatorConfig; _ CollatorConfig = ConfigParam 41; oracle_bridge_params#_ bridge_address:bits256 oracle_mutlisig_address:bits256 oracles:(HashmapE 256 uint256) external_chain_address:bits256 = OracleBridgeParams; diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index a1ddfb3ef..ed79c8b63 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2142,14 +2142,19 @@ CollatorConfig Config::get_collator_config(bool need_collator_nodes) const { collator_config.full_collated_data = rec.full_collated_data; if (need_collator_nodes) { vm::Dictionary dict{rec.collator_nodes->prefetch_ref(), 32 + 64 + 256}; - dict.check_for_each([&](Ref, td::ConstBitPtr key, int n) { + dict.check_for_each([&](Ref value, td::ConstBitPtr key, int n) { CHECK(n == 32 + 64 + 256); auto workchain = (td::int32)key.get_int(32); key.advance(32); td::uint64 shard = key.get_uint(64); key.advance(64); td::Bits256 adnl_id(key); - collator_config.collator_nodes.push_back({ton::ShardIdFull(workchain, shard), adnl_id}); + td::Bits256 full_node_id = td::Bits256::zero(); + gen::CollatorInfo::Record info; + if (tlb::csr_unpack(std::move(value), info) && info.full_node_id->size() == 257) { + full_node_id = td::Bits256(info.full_node_id->data_bits() + 1); + } + collator_config.collator_nodes.push_back({ton::ShardIdFull(workchain, shard), adnl_id, full_node_id}); return true; }); } diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 7058117ba..e00a045ef 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -485,6 +485,7 @@ class ShardConfig { struct CollatorNodeDescr { ton::ShardIdFull shard; ton::NodeIdShort adnl_id; + ton::NodeIdShort full_node_id; }; struct CollatorConfig { diff --git a/overlay/overlay-broadcast.cpp b/overlay/overlay-broadcast.cpp index ecd21e7af..bcb7c2858 100644 --- a/overlay/overlay-broadcast.cpp +++ b/overlay/overlay-broadcast.cpp @@ -69,6 +69,10 @@ td::Status BroadcastSimple::run_checks() { td::Status BroadcastSimple::distribute() { auto B = serialize(); auto nodes = overlay_->get_neighbours(5); + if (is_ours_) { + auto priority_nodes = overlay_->get_priority_broadcast_receivers(3); + nodes.insert(nodes.end(), priority_nodes.begin(), priority_nodes.end()); + } auto manager = overlay_->overlay_manager(); for (auto &n : nodes) { @@ -140,7 +144,7 @@ td::Status BroadcastSimple::create_new(td::actor::ActorId overlay, auto date = static_cast(td::Clocks::system()); auto B = std::make_unique(broadcast_hash, PublicKey{}, nullptr, flags, std::move(data), date, - td::BufferSlice{}, false, nullptr, adnl::AdnlNodeIdShort::zero()); + td::BufferSlice{}, false, nullptr, adnl::AdnlNodeIdShort::zero(), true); auto to_sign = B->to_sign(); auto P = td::PromiseCreator::lambda( diff --git a/overlay/overlay-broadcast.hpp b/overlay/overlay-broadcast.hpp index e7b39eecc..906a94a0d 100644 --- a/overlay/overlay-broadcast.hpp +++ b/overlay/overlay-broadcast.hpp @@ -46,6 +46,7 @@ class BroadcastSimple : public td::ListNode { td::uint32 date_; td::BufferSlice signature_; bool is_valid_{false}; + bool is_ours_{false}; OverlayImpl *overlay_; @@ -63,7 +64,7 @@ class BroadcastSimple : public td::ListNode { public: BroadcastSimple(Overlay::BroadcastHash broadcast_hash, PublicKey source, std::shared_ptr cert, td::uint32 flags, td::BufferSlice data, td::uint32 date, td::BufferSlice signature, bool is_valid, - OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id) + OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, bool is_ours = false) : broadcast_hash_(broadcast_hash) , source_(std::move(source)) , cert_(std::move(cert)) @@ -73,7 +74,8 @@ class BroadcastSimple : public td::ListNode { , signature_(std::move(signature)) , is_valid_(is_valid) , overlay_(overlay) - , src_peer_id_(src_peer_id) { + , src_peer_id_(src_peer_id) + , is_ours_(is_ours) { } Overlay::BroadcastHash get_hash() const { diff --git a/overlay/overlay-fec-broadcast.cpp b/overlay/overlay-fec-broadcast.cpp index 7ff08309d..4297ea0cc 100644 --- a/overlay/overlay-fec-broadcast.cpp +++ b/overlay/overlay-fec-broadcast.cpp @@ -26,8 +26,8 @@ namespace overlay { td::Result> BroadcastFec::create(Overlay::BroadcastHash hash, PublicKey src, Overlay::BroadcastDataHash data_hash, td::uint32 flags, - td::uint32 date, fec::FecType fec_type) { - auto F = std::make_unique(hash, std::move(src), data_hash, flags, date, std::move(fec_type)); + td::uint32 date, fec::FecType fec_type, bool is_ours) { + auto F = std::make_unique(hash, std::move(src), data_hash, flags, date, std::move(fec_type), is_ours); TRY_STATUS(F->init_fec_type()); TRY_STATUS(F->run_checks()); return std::move(F); @@ -94,12 +94,12 @@ void BroadcastFec::broadcast_checked(td::Result R) { overlay_->deliver_broadcast(get_source().compute_short_id(), data_.clone()); auto manager = overlay_->overlay_manager(); while (!parts_.empty()) { - distribute_part(parts_.begin()->first); + distribute_part(parts_.begin()->first); } } // Do we need status here?? -td::Status BroadcastFec::distribute_part(td::uint32 seqno) { +td::Status BroadcastFec::distribute_part(td::uint32 seqno) { auto i = parts_.find(seqno); if (i == parts_.end()) { // should not get here @@ -111,8 +111,12 @@ td::Status BroadcastFec::distribute_part(td::uint32 seqno) { td::BufferSlice data = std::move(tls.second); auto nodes = overlay_->get_neighbours(5); - auto manager = overlay_->overlay_manager(); + if (is_ours_) { + auto priority_nodes = overlay_->get_priority_broadcast_receivers(3); + nodes.insert(nodes.end(), priority_nodes.begin(), priority_nodes.end()); + } + auto manager = overlay_->overlay_manager(); for (auto &n : nodes) { if (neighbour_completed(n)) { continue; @@ -140,7 +144,8 @@ td::Status OverlayFecBroadcastPart::apply() { if (is_short_) { return td::Status::Error(ErrorCode::protoviolation, "short broadcast part for incomplete broadcast"); } - TRY_RESULT(B, BroadcastFec::create(broadcast_hash_, source_, broadcast_data_hash_, flags_, date_, fec_type_)); + TRY_RESULT( + B, BroadcastFec::create(broadcast_hash_, source_, broadcast_data_hash_, flags_, date_, fec_type_, is_ours_)); bcast_ = B.get(); overlay_->register_fec_broadcast(std::move(B)); } @@ -304,7 +309,8 @@ td::Status OverlayFecBroadcastPart::create_new(OverlayImpl *overlay, td::actor:: auto B = std::make_unique( broadcast_hash, part_hash, PublicKey{}, overlay->get_certificate(local_id), data_hash, size, flags, - part_data_hash, std::move(part), seqno, std::move(fec_type), date, td::BufferSlice{}, false, nullptr, overlay, adnl::AdnlNodeIdShort::zero()); + part_data_hash, std::move(part), seqno, std::move(fec_type), date, td::BufferSlice{}, false, nullptr, overlay, + adnl::AdnlNodeIdShort::zero(), true); auto to_sign = B->to_sign(); auto P = td::PromiseCreator::lambda( diff --git a/overlay/overlay-fec-broadcast.hpp b/overlay/overlay-fec-broadcast.hpp index 612af22fb..7f038b41d 100644 --- a/overlay/overlay-fec-broadcast.hpp +++ b/overlay/overlay-fec-broadcast.hpp @@ -131,18 +131,19 @@ class BroadcastFec : public td::ListNode { td::Status run_checks(); BroadcastFec(Overlay::BroadcastHash hash, PublicKey src, Overlay::BroadcastDataHash data_hash, td::uint32 flags, - td::uint32 date, fec::FecType fec_type) + td::uint32 date, fec::FecType fec_type, bool is_ours = false) : hash_(hash) , data_hash_(data_hash) , flags_(flags) , date_(date) , src_(std::move(src)) - , fec_type_(std::move(fec_type)) { + , fec_type_(std::move(fec_type)) + , is_ours_(is_ours) { } static td::Result> create(Overlay::BroadcastHash hash, PublicKey src, Overlay::BroadcastDataHash data_hash, td::uint32 flags, - td::uint32 date, fec::FecType fec_type); + td::uint32 date, fec::FecType fec_type, bool is_ours); bool neighbour_received(adnl::AdnlNodeIdShort id) const { return received_neighbours_.find(id) != received_neighbours_.end(); @@ -225,6 +226,7 @@ class BroadcastFec : public td::ListNode { OverlayImpl *overlay_; adnl::AdnlNodeIdShort src_peer_id_ = adnl::AdnlNodeIdShort::zero(); td::BufferSlice data_; + bool is_ours_ = false; }; class OverlayFecBroadcastPart : public td::ListNode { @@ -246,6 +248,7 @@ class OverlayFecBroadcastPart : public td::ListNode { bool is_short_; bool untrusted_{false}; + bool is_ours_; BroadcastFec *bcast_; OverlayImpl *overlay_; @@ -265,7 +268,8 @@ class OverlayFecBroadcastPart : public td::ListNode { std::shared_ptr cert, Overlay::BroadcastDataHash data_hash, td::uint32 data_size, td::uint32 flags, Overlay::BroadcastDataHash part_data_hash, td::BufferSlice data, td::uint32 seqno, fec::FecType fec_type, td::uint32 date, td::BufferSlice signature, - bool is_short, BroadcastFec *bcast, OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id) + bool is_short, BroadcastFec *bcast, OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, + bool is_ours = false) : broadcast_hash_(broadcast_hash) , part_hash_(part_hash) , source_(std::move(source)) @@ -282,7 +286,8 @@ class OverlayFecBroadcastPart : public td::ListNode { , is_short_(is_short) , bcast_(bcast) , overlay_(overlay) - , src_peer_id_(src_peer_id) { + , src_peer_id_(src_peer_id) + , is_ours_(is_ours) { } td::uint32 data_size() const { diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index 2134bbc0c..96e9fc2a9 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -359,6 +359,19 @@ void OverlayManager::get_stats(td::Promise nodes) { + auto it = overlays_.find(local_id); + if (it == overlays_.end()) { + return; + } + auto it2 = it->second.find(overlay); + if (it2 == it->second.end()) { + return; + } + td::actor::send_closure(it2->second, &Overlay::set_priority_broadcast_receivers, std::move(nodes)); +} + Certificate::Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, td::BufferSlice signature) : issued_by_(issued_by) diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h index ada37303d..ed9da23b5 100644 --- a/overlay/overlay-manager.h +++ b/overlay/overlay-manager.h @@ -96,6 +96,9 @@ class OverlayManager : public Overlays { td::actor::ActorOwn overlay); void get_stats(td::Promise> promise) override; + void set_priority_broadcast_receivers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, + std::vector nodes) override; + struct PrintId {}; PrintId print_id() const { diff --git a/overlay/overlay.h b/overlay/overlay.h index 146be7383..fcf61ea84 100644 --- a/overlay/overlay.h +++ b/overlay/overlay.h @@ -69,6 +69,7 @@ class Overlay : public td::actor::Actor { virtual void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) = 0; //virtual void receive_broadcast(td::BufferSlice data) = 0; //virtual void subscribe(std::unique_ptr callback) = 0; + virtual void set_priority_broadcast_receivers(std::vector nodes) = 0; }; } // namespace overlay diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 197864946..07003e098 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -192,7 +192,19 @@ class OverlayImpl : public Overlay { } else { std::vector vec; for (td::uint32 i = 0; i < max_size; i++) { - vec.push_back(neighbours_[td::Random::fast(0, static_cast(neighbours_.size()))]); + vec.push_back(neighbours_[td::Random::fast(0, static_cast(neighbours_.size()) - 1)]); + } + return vec; + } + } + std::vector get_priority_broadcast_receivers(td::uint32 max_size = 0) const { + if (max_size == 0 || max_size >= priority_broadcast_receivers_.size()) { + return priority_broadcast_receivers_; + } else { + std::vector vec; + for (td::uint32 i = 0; i < max_size; i++) { + vec.push_back(priority_broadcast_receivers_[td::Random::fast( + 0, static_cast(priority_broadcast_receivers_.size()) - 1)]); } return vec; } @@ -250,6 +262,10 @@ class OverlayImpl : public Overlay { } } + void set_priority_broadcast_receivers(std::vector nodes) override { + priority_broadcast_receivers_ = std::move(nodes); + } + private: template void process_query(adnl::AdnlNodeIdShort src, T &query, td::Promise promise) { @@ -309,6 +325,7 @@ class OverlayImpl : public Overlay { std::queue bcast_lru_; std::map> out_fec_bcasts_; + std::vector priority_broadcast_receivers_; void bcast_gc(); diff --git a/overlay/overlays.h b/overlay/overlays.h index 0a9c7765c..a7b29faf8 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -238,6 +238,9 @@ class Overlays : public td::actor::Actor { virtual void get_overlay_random_peers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, td::uint32 max_peers, td::Promise> promise) = 0; virtual void get_stats(td::Promise> promise) = 0; + + virtual void set_priority_broadcast_receivers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, + std::vector nodes) = 0; }; } // namespace overlay diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 56e0c1db1..e491061ed 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -127,6 +127,10 @@ void FullNodeShardImpl::create_overlay() { if (cert_) { td::actor::send_closure(overlays_, &overlay::Overlays::update_certificate, adnl_id_, overlay_id_, local_id_, cert_); } + if (!collator_nodes_.empty()) { + td::actor::send_closure(overlays_, &overlay::Overlays::set_priority_broadcast_receivers, adnl_id_, overlay_id_, + collator_nodes_); + } } void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broadcast, td::Promise promise) { @@ -1031,6 +1035,15 @@ void FullNodeShardImpl::update_validators(std::vector public_key_ } } +void FullNodeShardImpl::update_collators(std::vector nodes) { + if (!client_.empty()) { + return; + } + collator_nodes_ = std::move(nodes); + td::actor::send_closure(overlays_, &overlay::Overlays::set_priority_broadcast_receivers, adnl_id_, overlay_id_, + collator_nodes_); +} + void FullNodeShardImpl::reload_neighbours() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index e24873aac..847091c65 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -75,6 +75,7 @@ class FullNodeShard : public td::actor::Actor { virtual void set_handle(BlockHandle handle, td::Promise promise) = 0; virtual void update_validators(std::vector public_key_hashes, PublicKeyHash local_hash) = 0; + virtual void update_collators(std::vector nodes) = 0; static td::actor::ActorOwn create( ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index cd7c20a95..1cb2957de 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -181,6 +181,7 @@ class FullNodeShardImpl : public FullNodeShard { void alarm() override; void update_validators(std::vector public_key_hashes, PublicKeyHash local_hash) override; + void update_collators(std::vector nodes) override; void sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) override; void import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) override; @@ -258,6 +259,7 @@ class FullNodeShardImpl : public FullNodeShard { adnl::AdnlNodeIdShort last_pinged_neighbour_ = adnl::AdnlNodeIdShort::zero(); FullNodeShardMode mode_; + std::vector collator_nodes_; }; } // namespace fullnode diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 6792f969f..52d3cfb97 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -205,6 +205,33 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s ++it; } } + + if (!collators_inited_ || state->is_key_state()) { + update_collators(state); + } +} + +void FullNodeImpl::update_collators(td::Ref state) { + collators_inited_ = true; + collator_config_ = state->get_collator_config(true); + for (auto& s : shards_) { + update_shard_collators(s.first, s.second); + } +} + +void FullNodeImpl::update_shard_collators(ShardIdFull shard, ShardInfo& info) { + if (info.actor.empty()) { + return; + } + std::vector nodes; + for (const block::CollatorNodeDescr& desc : collator_config_.collator_nodes) { + if (!desc.full_node_id.is_zero() && shard_intersects(shard, desc.shard)) { + nodes.emplace_back(desc.full_node_id); + } + } + std::sort(nodes.begin(), nodes.end()); + nodes.erase(std::unique(nodes.begin(), nodes.end()), nodes.end()); + td::actor::send_closure(info.actor, &FullNodeShard::update_collators, std::move(nodes)); } void FullNodeImpl::add_shard_actor(ShardIdFull shard, FullNodeShardMode mode) { @@ -219,6 +246,9 @@ void FullNodeImpl::add_shard_actor(ShardIdFull shard, FullNodeShardMode mode) { if (all_validators_.size() > 0) { td::actor::send_closure(info.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); } + if (collators_inited_) { + update_shard_collators(shard, info); + } } void FullNodeImpl::sync_completed() { diff --git a/validator/full-node.hpp b/validator/full-node.hpp index fbe60d1b2..8cc8994b1 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -92,7 +92,16 @@ class FullNodeImpl : public FullNode { td::Promise started_promise); private: + struct ShardInfo { + bool exists = false; + td::actor::ActorOwn actor; + FullNodeShardMode mode = FullNodeShardMode::inactive; + td::Timestamp delete_at = td::Timestamp::never(); + }; + void add_shard_actor(ShardIdFull shard, FullNodeShardMode mode); + void update_collators(td::Ref state); + void update_shard_collators(ShardIdFull shard, ShardInfo& info); PublicKeyHash local_id_; adnl::AdnlNodeIdShort adnl_id_; @@ -100,13 +109,6 @@ class FullNodeImpl : public FullNode { td::actor::ActorId get_shard(AccountIdPrefixFull dst); td::actor::ActorId get_shard(ShardIdFull shard, bool exact = false); - - struct ShardInfo { - bool exists = false; - td::actor::ActorOwn actor; - FullNodeShardMode mode = FullNodeShardMode::inactive; - td::Timestamp delete_at = td::Timestamp::never(); - }; std::map shards_; td::actor::ActorId keyring_; @@ -125,6 +127,8 @@ class FullNodeImpl : public FullNode { std::set local_keys_; td::Promise started_promise_; + bool collators_inited_ = false; + block::CollatorConfig collator_config_; }; } // namespace fullnode diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index 019ecbeb0..a09bd1df8 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -556,5 +556,9 @@ BlockIdExt MasterchainStateQ::prev_key_block_id(BlockSeqno seqno) const { return block_id; } +bool MasterchainStateQ::is_key_state() const { + return config_ ? config_->is_key_state() : false; +} + } // namespace validator } // namespace ton diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index 6b95c7e40..c726447fc 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -129,6 +129,7 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { BlockIdExt last_key_block_id() const override; BlockIdExt next_key_block_id(BlockSeqno seqno) const override; BlockIdExt prev_key_block_id(BlockSeqno seqno) const override; + bool is_key_state() const override; MasterchainStateQ* make_copy() const override; static td::Result> fetch(const BlockIdExt& _id, td::BufferSlice _data, diff --git a/validator/interfaces/shard.h b/validator/interfaces/shard.h index 15caa7b27..59211a855 100644 --- a/validator/interfaces/shard.h +++ b/validator/interfaces/shard.h @@ -77,6 +77,7 @@ class MasterchainState : virtual public ShardState { virtual BlockIdExt last_key_block_id() const = 0; virtual BlockIdExt next_key_block_id(BlockSeqno seqno) const = 0; virtual BlockIdExt prev_key_block_id(BlockSeqno seqno) const = 0; + virtual bool is_key_state() const = 0; virtual bool get_old_mc_block_id(ton::BlockSeqno seqno, ton::BlockIdExt& blkid, ton::LogicalTime* end_lt = nullptr) const = 0; virtual bool check_old_mc_block_id(const ton::BlockIdExt& blkid, bool strict = false) const = 0; From 9fb986f6f5a65ef573ba383503649565cae9c9f1 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 28 Sep 2022 13:16:22 +0300 Subject: [PATCH 037/388] Disable downloading archives during sync --- validator/manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/manager.cpp b/validator/manager.cpp index 2b6018364..95ef6bf06 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1680,7 +1680,7 @@ void ValidatorManagerImpl::prestart_sync() { } void ValidatorManagerImpl::download_next_archive() { - if (!out_of_sync()) { + if (true) { finish_prestart_sync(); return; } From c2dde00459183caf39941bb8c45fb687b0e8c2ff Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 28 Sep 2022 14:33:17 +0300 Subject: [PATCH 038/388] Improve retries in overlay.cpp --- overlay/overlay.cpp | 66 +++++++++++++++++++++++---------------------- overlay/overlay.hpp | 2 ++ 2 files changed, 36 insertions(+), 32 deletions(-) diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index 8e35e2a25..3e10daec2 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -421,48 +421,50 @@ void OverlayImpl::bcast_gc() { CHECK(delivered_broadcasts_.size() == bcast_lru_.size()); } -void OverlayImpl::send_message_to_neighbours(td::BufferSlice data) { - if (neighbours_.empty()) { - // TODO: limit retries +void OverlayImpl::wait_neighbours_not_empty(td::Promise promise, int max_retries) { + if (!neighbours_.empty()) { + promise.set_result(td::Unit()); + } else if (max_retries > 0) { delay_action( - [SelfId = actor_id(this), data = std::move(data)]() mutable { - td::actor::send_closure(SelfId, &OverlayImpl::send_message_to_neighbours, std::move(data)); + [SelfId = actor_id(this), promise = std::move(promise), max_retries]() mutable { + td::actor::send_closure(SelfId, &OverlayImpl::wait_neighbours_not_empty, std::move(promise), max_retries - 1); }, td::Timestamp::in(0.5)); - return; - } - for (auto &n : neighbours_) { - td::actor::send_closure(manager_, &OverlayManager::send_message, n, local_id_, overlay_id_, data.clone()); + } else { + promise.set_error(td::Status::Error(ErrorCode::timeout)); } } +void OverlayImpl::send_message_to_neighbours(td::BufferSlice data) { + wait_neighbours_not_empty([this, data = std::move(data)](td::Result R) { + if (R.is_error()) { + return; + } + for (auto &n : neighbours_) { + td::actor::send_closure(manager_, &OverlayManager::send_message, n, local_id_, overlay_id_, data.clone()); + } + }); +} + void OverlayImpl::send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) { - if (neighbours_.empty()) { - // TODO: limit retries - delay_action( - [SelfId = actor_id(this), send_as, flags, data = std::move(data)]() mutable { - td::actor::send_closure(SelfId, &OverlayImpl::send_broadcast, send_as, flags, std::move(data)); - }, - td::Timestamp::in(0.5)); - return; - } - auto S = BroadcastSimple::create_new(actor_id(this), keyring_, send_as, std::move(data), flags); - if (S.is_error()) { - LOG(WARNING) << "failed to send broadcast: " << S; - } + wait_neighbours_not_empty([this, send_as, flags, data = std::move(data)](td::Result R) mutable { + if (R.is_error()) { + return; + } + auto S = BroadcastSimple::create_new(actor_id(this), keyring_, send_as, std::move(data), flags); + if (S.is_error()) { + LOG(WARNING) << "failed to send broadcast: " << S; + } + }); } void OverlayImpl::send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) { - if (neighbours_.empty()) { - // TODO: limit retries - delay_action( - [SelfId = actor_id(this), send_as, flags, data = std::move(data)]() mutable { - td::actor::send_closure(SelfId, &OverlayImpl::send_broadcast_fec, send_as, flags, std::move(data)); - }, - td::Timestamp::in(0.5)); - return; - } - OverlayOutboundFecBroadcast::create(std::move(data), flags, actor_id(this), send_as); + wait_neighbours_not_empty([this, send_as, flags, data = std::move(data)](td::Result R) mutable { + if (R.is_error()) { + return; + } + OverlayOutboundFecBroadcast::create(std::move(data), flags, actor_id(this), send_as); + }); } void OverlayImpl::print(td::StringBuilder &sb) { diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 07003e098..f52de86d6 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -266,6 +266,8 @@ class OverlayImpl : public Overlay { priority_broadcast_receivers_ = std::move(nodes); } + void wait_neighbours_not_empty(td::Promise promise, int max_retries = 10); + private: template void process_query(adnl::AdnlNodeIdShort src, T &query, td::Promise promise) { From 81c0e920c5d13764b8bf40a7e5e69efa296eb16d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 3 Oct 2022 17:55:37 +0300 Subject: [PATCH 039/388] Simplify selecting shards for monitor --- validator/full-node-shard.cpp | 2 +- validator/full-node.cpp | 59 +++++++++++++++++++-------------- validator/full-node.hpp | 2 +- validator/manager-init.cpp | 2 +- validator/manager.cpp | 16 ++++----- validator/manager.hpp | 5 ++- validator/shard-client.cpp | 23 +++---------- validator/state-serializer.cpp | 24 +++++--------- validator/state-serializer.hpp | 1 - validator/validator-options.hpp | 5 +-- validator/validator.h | 2 +- 11 files changed, 66 insertions(+), 75 deletions(-) diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index e491061ed..d048f6112 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -670,7 +670,7 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ne void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { BlockIdExt block_id = create_block_id(query.id_); - //if (block_id.shard_full() != shard_) { + //if (!shard_is_ancestor(shard_, block_id.shard_full())) { // LOG(FULL_NODE_WARNING) << "dropping block broadcast: shard mismatch. overlay=" << shard_.to_str() // << " block=" << block_id.to_str(); // return; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 52d3cfb97..72af6fe8e 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -135,10 +135,18 @@ void FullNodeImpl::initial_read_complete(BlockHandle top_handle) { void FullNodeImpl::update_shard_configuration(td::Ref state, std::set shards_to_monitor, std::set temporary_shards) { CHECK(shards_to_monitor.count(ShardIdFull(masterchainId))); - std::map new_shards; + std::set new_shards; std::map new_active; - new_shards[ShardIdFull(masterchainId)] = state->get_block_id(); + new_shards.insert(ShardIdFull(masterchainId)); std::set workchains; + auto cut_shard = [&](ShardIdFull shard) -> ShardIdFull { + unsigned pfx_len = shard.pfx_len(); + unsigned min_split = state->soft_min_split_depth(shard.workchain); + if (min_split < pfx_len) { + return shard_prefix(shard, min_split); + } + return shard; + }; auto set_active = [&](ShardIdFull shard, FullNodeShardMode mode) { while (new_active.emplace(shard, mode).second && shard.pfx_len() > 0) { shard = shard_parent(shard); @@ -146,20 +154,20 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s }; for (auto &info : state->get_shards()) { workchains.insert(info->shard().workchain); - new_shards[info->shard()] = info->top_block_id(); + new_shards.insert(cut_shard(info->shard())); } for (const auto &wpair : state->get_workchain_list()) { ton::WorkchainId wc = wpair.first; const block::WorkchainInfo *winfo = wpair.second.get(); if (workchains.count(wc) == 0 && winfo->active && winfo->enabled_since <= state->get_unix_time()) { - new_shards[ShardIdFull(wc)] = BlockIdExt(wc, shardIdAll, 0, winfo->zerostate_root_hash, winfo->zerostate_file_hash); + new_shards.insert(ShardIdFull(wc)); } } for (ShardIdFull shard : shards_to_monitor) { - set_active(shard, FullNodeShardMode::active); + set_active(cut_shard(shard), FullNodeShardMode::active); } for (ShardIdFull shard : temporary_shards) { - set_active(shard, FullNodeShardMode::active_temp); + set_active(cut_shard(shard), FullNodeShardMode::active_temp); } auto info_set_mode = [&](ShardIdFull shard, ShardInfo& info, FullNodeShardMode mode) { @@ -177,7 +185,7 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s }; for (auto shard : new_shards) { - auto &info = shards_[shard.first]; + auto &info = shards_[shard]; info.exists = true; } @@ -283,7 +291,7 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s } void FullNodeImpl::send_broadcast(BlockBroadcast broadcast) { - auto shard = get_shard(broadcast.block_id.shard_full(), true); + auto shard = get_shard(broadcast.block_id.shard_full()); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; return; @@ -379,27 +387,28 @@ void FullNodeImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull std::move(promise)); } -td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard, bool exact) { - if (!exact) { - ShardIdFull s = shard; - while (true) { - auto it = shards_.find(s); - if (it != shards_.end() && it->second.exists) { - if (it->second.actor.empty()) { - add_shard_actor(s, FullNodeShardMode::inactive); - } - if (it->second.mode == FullNodeShardMode::inactive) { - it->second.delete_at = td::Timestamp::in(INACTIVE_SHARD_TTL); - } - return it->second.actor.get(); +td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { + while (true) { + auto it = shards_.find(shard); + if (it != shards_.end() && it->second.exists) { + if (it->second.actor.empty()) { + add_shard_actor(shard, FullNodeShardMode::inactive); } - if (s.pfx_len() == 0) { - break; + if (it->second.mode == FullNodeShardMode::inactive) { + it->second.delete_at = td::Timestamp::in(INACTIVE_SHARD_TTL); } - s = shard_parent(s); + return it->second.actor.get(); + } + if (shard.pfx_len() == 0) { + break; } + shard = shard_parent(shard); + } + auto it = shards_.find(shard); + if (it == shards_.end()) { + it = shards_.emplace(shard = ShardIdFull(shard.workchain), ShardInfo{}).first; } - auto &info = shards_[shard]; + auto &info = it->second; if (info.actor.empty()) { add_shard_actor(shard, FullNodeShardMode::inactive); } diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 8cc8994b1..503ba2866 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -108,7 +108,7 @@ class FullNodeImpl : public FullNode { FileHash zero_state_file_hash_; td::actor::ActorId get_shard(AccountIdPrefixFull dst); - td::actor::ActorId get_shard(ShardIdFull shard, bool exact = false); + td::actor::ActorId get_shard(ShardIdFull shard); std::map shards_; td::actor::ActorId keyring_; diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index 1a322c02c..8935664e3 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -566,7 +566,7 @@ void ValidatorManagerMasterchainStarter::truncated() { truncate_shard_next(handle_->id(), ig.get_promise()); auto s = state_->get_shards(); for (auto &shard : s) { - if (opts_->need_monitor(shard->shard())) { + if (opts_->need_monitor(shard->shard(), state_)) { truncate_shard_next(shard->top_block_id(), ig.get_promise()); } } diff --git a/validator/manager.cpp b/validator/manager.cpp index 95ef6bf06..2a17af5ed 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -214,7 +214,7 @@ void ValidatorManagerImpl::prevalidate_block(BlockBroadcast broadcast, td::Promi promise.set_error(td::Status::Error(ErrorCode::notready, "node not started")); return; } - if (!shards_to_monitor_.count(broadcast.block_id.shard_full())) { + if (!need_monitor(broadcast.block_id.shard_full())) { promise.set_error(td::Status::Error("not monitoring shard")); return; } @@ -457,7 +457,7 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refblock_id().shard_full(), desc->catchain_seqno()}] = desc; VLOG(VALIDATOR_DEBUG) << "new shard block descr for " << desc->block_id(); - if (shards_to_monitor_.count(desc->block_id().shard_full())) { + if (need_monitor(desc->block_id().shard_full())) { auto P = td::PromiseCreator::lambda([](td::Result> R) { if (R.is_error()) { auto S = R.move_as_error(); @@ -617,10 +617,9 @@ void ValidatorManagerImpl::wait_out_msg_queue_proof(BlockIdExt block_id, ShardId td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_msg_queue, block_id, dst_shard, std::move(R)); }); - auto id = td::actor::create_actor("waitmsgqueue", block_id, dst_shard, - shards_to_monitor_.count(block_id.shard_full()), priority, - actor_id(this), td::Timestamp::at(timeout.at() + 10.0), - std::move(P)) + auto id = td::actor::create_actor( + "waitmsgqueue", block_id, dst_shard, need_monitor(block_id.shard_full()), priority, actor_id(this), + td::Timestamp::at(timeout.at() + 10.0), std::move(P)) .release(); wait_out_msg_queue_proof_[key].actor_ = id; it = wait_out_msg_queue_proof_.find(key); @@ -1083,8 +1082,8 @@ void ValidatorManagerImpl::finished_wait_msg_queue(BlockIdExt block_id, ShardIdF std::move(R)); }); auto id = td::actor::create_actor("waitmsgqueue", block_id, dst_shard, - shards_to_monitor_.count(block_id.shard_full()), - X.second, actor_id(this), X.first, std::move(P)) + need_monitor(block_id.shard_full()), X.second, + actor_id(this), X.first, std::move(P)) .release(); it->second.actor_ = id; return; @@ -2489,7 +2488,6 @@ void ValidatorManagerImpl::get_shard_client_state(bool from_db, td::Promise state, std::set shards_to_monitor) { - shards_to_monitor_ = shards_to_monitor; callback_->update_shard_configuration(std::move(state), std::move(shards_to_monitor), extra_active_shards_); } diff --git a/validator/manager.hpp b/validator/manager.hpp index ddef34414..3b32cfcdf 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -628,11 +628,14 @@ class ValidatorManagerImpl : public ValidatorManager { td::Ref get_block_persistent_state(BlockIdExt block_id); private: + bool need_monitor(ShardIdFull shard) const { + return opts_->need_monitor(shard, last_masterchain_state_); + } + std::map> shard_client_waiters_; std::map> collator_nodes_; - std::set shards_to_monitor_ = {ShardIdFull(masterchainId)}; std::set extra_active_shards_; std::map last_validated_blocks_; diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index 431d77216..5c25b7830 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -91,7 +91,7 @@ void ShardClient::start_up_init_mode() { auto vec = masterchain_state_->get_shards(); for (auto &shard : vec) { - if (shards_to_monitor_.count(shard->shard())) { + if (opts_->need_monitor(shard->shard(), masterchain_state_)) { auto P = td::PromiseCreator::lambda([promise = ig.get_promise()](td::Result> R) mutable { R.ensure(); promise.set_value(td::Unit()); @@ -192,7 +192,7 @@ void ShardClient::apply_all_shards() { auto vec = masterchain_state_->get_shards(); for (auto &shard : vec) { - if (shards_to_monitor_.count(shard->shard())) { + if (opts_->need_monitor(shard->shard(), masterchain_state_)) { auto Q = td::PromiseCreator::lambda([SelfId = actor_id(this), promise = ig.get_promise(), shard = shard->shard()](td::Result> R) mutable { if (R.is_error()) { @@ -254,23 +254,9 @@ void ShardClient::build_shard_overlays() { for (const auto &info : masterchain_state_->get_shards()) { auto shard = info->shard(); workchains.insert(shard.workchain); - bool will_split = shard.pfx_len() < max_shard_pfx_len && - (info->fsm_state() == McShardHash::FsmState::fsm_split || info->before_split()); - bool will_merge = - shard.pfx_len() > 0 && (info->fsm_state() == McShardHash::FsmState::fsm_merge || info->before_merge()); - if (opts_->need_monitor(shard) || (will_merge && opts_->need_monitor(shard_parent(shard)))) { + if (opts_->need_monitor(shard, masterchain_state_)) { new_shards_to_monitor.insert(shard); } - if (will_merge && opts_->need_monitor(shard_parent(shard))) { - new_shards_to_monitor.insert(shard_parent(shard)); - } - if (will_split) { - for (int id = 0; id < 2; ++id) { - if (opts_->need_monitor(shard_child(shard, id))) { - new_shards_to_monitor.insert(shard_child(shard, id)); - } - } - } } std::vector new_workchains; @@ -278,7 +264,8 @@ void ShardClient::build_shard_overlays() { ton::WorkchainId wc = wpair.first; const block::WorkchainInfo *winfo = wpair.second.get(); auto shard = ShardIdFull(wc); - if (workchains.count(wc) == 0 && winfo->active && winfo->enabled_since <= cur_time && opts_->need_monitor(shard)) { + if (workchains.count(wc) == 0 && winfo->active && winfo->enabled_since <= cur_time && + opts_->need_monitor(shard, masterchain_state_)) { new_shards_to_monitor.insert(shard); if (shards_to_monitor_.count(shard) == 0) { new_workchains.push_back(BlockIdExt(wc, shardIdAll, 0, winfo->zerostate_root_hash, winfo->zerostate_file_hash)); diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index fc8cd29aa..1c8d677c2 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -159,16 +159,12 @@ void AsyncStateSerializer::next_iteration() { return; } while (next_idx_ < shards_.size()) { - if (!need_monitor(shards_[next_idx_].shard_full())) { - next_idx_++; - } else { - // block next attempts immediately, but send actual request later - running_ = true; - delay_action( - [SelfId = actor_id(this), shard = shards_[next_idx_]]() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_shard_state, shard); }, - td::Timestamp::in(td::Random::fast(0, 4 * 3600))); - return; - } + // block next attempts immediately, but send actual request later + running_ = true; + delay_action( + [SelfId = actor_id(this), shard = shards_[next_idx_]]() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_shard_state, shard); }, + td::Timestamp::in(td::Random::fast(0, 4 * 3600))); + return; } LOG(INFO) << "finished serializing persistent state for " << masterchain_handle_->id().id; last_key_block_ts_ = masterchain_handle_->unix_time(); @@ -245,7 +241,9 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state auto vec = state->get_shards(); for (auto &v : vec) { - shards_.push_back(v->top_block_id()); + if (opts_->need_monitor(v->shard(), state)) { + shards_.push_back(v->top_block_id()); + } } auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader = cell_db_reader_] (td::FileFd& fd) { @@ -311,10 +309,6 @@ void AsyncStateSerializer::success_handler() { next_iteration(); } -bool AsyncStateSerializer::need_monitor(ShardIdFull shard) { - return opts_->need_monitor(shard); -} - bool AsyncStateSerializer::need_serialize(BlockHandle handle) { if (handle->id().id.seqno == 0 || !handle->is_key_block()) { return false; diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index 0d6e95505..1472c3b4b 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -60,7 +60,6 @@ class AsyncStateSerializer : public td::actor::Actor { } bool need_serialize(BlockHandle handle); - bool need_monitor(ShardIdFull shard); void alarm() override; void start_up() override; diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index b4a94a39b..e96292ad3 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -32,8 +32,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { BlockIdExt init_block_id() const override { return init_block_id_; } - bool need_monitor(ShardIdFull shard) const override { - return check_shard_(shard); + bool need_monitor(ShardIdFull shard, const td::Ref& state) const override { + td::uint32 min_split = state->min_split_depth(shard.workchain); + return check_shard_((td::uint32)shard.pfx_len() <= min_split ? shard : shard_prefix(shard, min_split)); } bool allow_blockchain_init() const override { return allow_blockchain_init_; diff --git a/validator/validator.h b/validator/validator.h index 0daccc1ae..c52cbad7d 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -49,7 +49,7 @@ struct ValidatorManagerOptions : public td::CntObject { public: virtual BlockIdExt zero_block_id() const = 0; virtual BlockIdExt init_block_id() const = 0; - virtual bool need_monitor(ShardIdFull shard) const = 0; + virtual bool need_monitor(ShardIdFull shard, const td::Ref& state) const = 0; virtual bool allow_blockchain_init() const = 0; virtual double sync_blocks_before() const = 0; virtual double block_ttl() const = 0; From 89771e2a2c354a4ead5ee160876b4e29e9ba5ab7 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 6 Oct 2022 13:53:53 +0300 Subject: [PATCH 040/388] Generate tlo and other changes after merge --- adnl/adnl-peer.cpp | 2 +- overlay/overlay-broadcast.hpp | 4 ++-- overlay/overlay-fec-broadcast.hpp | 6 +++--- rldp/rldp.cpp | 2 +- tl/generate/scheme/ton_api.tlo | Bin 69060 -> 73376 bytes 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp index 1d73b8f6c..46a097d95 100644 --- a/adnl/adnl-peer.cpp +++ b/adnl/adnl-peer.cpp @@ -621,7 +621,7 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageQueryError return; } - td::actor::send_closure_later(Q->second, &AdnlQuery::reject_query); + td::actor::send_closure_later(Q->second, &AdnlQuery::set_error, td::Status::Error("adnl query rejected")); out_queries_.erase(Q); } diff --git a/overlay/overlay-broadcast.hpp b/overlay/overlay-broadcast.hpp index 906a94a0d..8a5d13249 100644 --- a/overlay/overlay-broadcast.hpp +++ b/overlay/overlay-broadcast.hpp @@ -73,9 +73,9 @@ class BroadcastSimple : public td::ListNode { , date_(date) , signature_(std::move(signature)) , is_valid_(is_valid) + , is_ours_(is_ours) , overlay_(overlay) - , src_peer_id_(src_peer_id) - , is_ours_(is_ours) { + , src_peer_id_(src_peer_id) { } Overlay::BroadcastHash get_hash() const { diff --git a/overlay/overlay-fec-broadcast.hpp b/overlay/overlay-fec-broadcast.hpp index 7f038b41d..c97ad8858 100644 --- a/overlay/overlay-fec-broadcast.hpp +++ b/overlay/overlay-fec-broadcast.hpp @@ -284,10 +284,10 @@ class OverlayFecBroadcastPart : public td::ListNode { , date_(date) , signature_(std::move(signature)) , is_short_(is_short) + , is_ours_(is_ours) , bcast_(bcast) - , overlay_(overlay) - , src_peer_id_(src_peer_id) - , is_ours_(is_ours) { + , overlay_(overlay) + , src_peer_id_(src_peer_id) { } td::uint32 data_size() const { diff --git a/rldp/rldp.cpp b/rldp/rldp.cpp index 952a7d692..1834aac3f 100644 --- a/rldp/rldp.cpp +++ b/rldp/rldp.cpp @@ -238,7 +238,7 @@ void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort ton_api::rldp_queryError &message) { auto it = queries_.find(message.query_id_); if (it != queries_.end()) { - td::actor::send_closure(it->second, &adnl::AdnlQuery::reject_query); + td::actor::send_closure(it->second, &adnl::AdnlQuery::set_error, td::Status::Error("adnl query rejected")); queries_.erase(it); } else { VLOG(RLDP_INFO) << "received reject to unknown query " << message.query_id_; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index d89402883f7b46eb1700a121d1d38ff367ee2df2..d535668aa423585bb2be3319dbf46d3f4f610efc 100644 GIT binary patch delta 3357 zcma)8e{2(V6z^@t$5uTfk7#5#1JH`guD{elGL0uCgu$a=@J%4?%a&P%l zxDfF!ms7rHD$KSlo>6tX0C9~G4*6=5H*4#($+X##MOXa%`4!nc)K8CLTzC%Gz_%SL z8FO*Zp39^DS6c*i*=c)+q7SZq?{~bfxZYM)J#Cyh8X>Nc(2N8hM03%2old@6d?u^( z#z&vsck)CYWm-Hl#jB3+bl|<>R3aXOasYyx2)Zj$004Vq_J~Md9 z$WG2}Rmwt(---khT4o6XFFZmw?0*(_PRvbu(C|8^wkr~O_Oz2EXK+2^biXR9oqlad z<@r}Akx1`SOH$KLkq?th#E zBP+)T8!W94(6EqQ!P6x+DB;Xtf^0MI9dHlNm7?(|pCimWA;rzy{*wO9nb@ zdK_B6?PwdD>I=(Ji!VN4`%#`Hi8kPF`GltYt6`On_FaqhHSbfRiM6E35#y&hW#0yw zI^lEuc1R@XbYK8^$|er%H-GWu#~9L4g>VxS;X-&B52(k|HsLof}GM#IYf4 zZYx^?rs;R;+iO$4A9J0G4(ZLxTTHq(iqM2Z%b*eBO<@VG~G$x_B^l?Eoh*h z?u3q#VB)s0(+0<2N_^xVbiTLTt8orpKmCzK|{*K0ZqB} z&8D1@(EhEDHRLeQHAEU=5AaO5Yt0y3z4n~Jpt-_-1f8^kLa(z>L(EEO#E`^?Vmuci zyn$~|dWHtq*tp)Sl8k?SYu=0flZ?2#nAPUvK&P2q$f5ZYFAt>yr_EqKH?e()Mn3b?1+Y0Br&Dk&5fcs3D6 zxM!^sx6Zdz186dRN`LL2`Z5osP@fKTadm8h4L~yQ2%GZcw)z~E9JgHvhn#bcilS*lzPrN2LLACL2-99{%P}wA*f?Np(X6BaRRqy%-<*3;OoPJ*u zSjfk>mKRJ08ZODCVFAU6D1b;g!#-d`T(mb0GTgoQC=Kkqd1g7T-b3S)`&@4LPY+;Z zUdzV83VSQWq=S7-@8ABIV)l}UyYm!9h7(sJK~e0is?dS*INW-vzYmY51f<^if1GlE`8ip&6S&U>fflW0j3y#gjYcA+>o0w)4BJB$A zIO0@5&L;K}lS!tVS+Tf8##qf2*l7s@nf%t5?O96lQ?rfoDCV$Ajqhv- zD#b3-UEL0fm9Ji%pXKqK>pfJWH4J60>cjaduGeEz!(5XgMjCo79Ez~SHLhH2HRi_8?*&giL4l!rDQh}yLTHXhFFl84+RB1NG}CRR4Dln)c$vy2%`U|Kflla`~Cm^zx&?! zb82anD*7^7cRDY-R%VJyNqfTo;;TgnD>Bz6s;UqXGnBsXD@6e^O`!lipKrIl9yo3Z zcX6sFXd-uB?8_WEgpXK?tt2{Ya}d;2*{fC5Q{txJ2J;d5%Vj`qr|;2lp(2F*j#&b_ zD|x&2`Ek$^m&37LSCOc{@*h!_4!zte(W1(QXjtT$g_zJK$&x0CDx4^&K(j}K0D9Rs z=u~wH1)9*g+Io8dZv4}-%}>CS^_7sHqhYeKio#X+zR|UVNgW3U^5F?jtAVe3SYtEp z;j;n{%jEp7k%c=qV8t(vO8`73QvzgFHajfc^4w!BVP?ndz}W_*n_;jEL!tr z@89-~=!em8QyLhuV2j%*jkK)Wkle9Q&fxLmhwg3+*PvH*?GJCO&i&!iK}uTV_g4nA z18j#6fGE*RnGtoX(^QSaQYtDDNfEhgf-HpvEgI*#qsnC|f*F3S8UEUG!I{B#Xnu$W zkRPlvGREFbm{G&Sn*g~-gQi*>GA5H0v0IizMa!CS5jol=$eDp*ZiHm`X!ekQzL#@^aYbL Date: Thu, 6 Oct 2022 15:13:10 +0300 Subject: [PATCH 041/388] Tonlib uses partial liteservers --- tl/generate/scheme/tonlib_api.tl | 2 +- tl/generate/scheme/tonlib_api.tlo | Bin 30836 -> 30904 bytes tonlib/CMakeLists.txt | 4 +- tonlib/tonlib/Config.cpp | 51 +++-- tonlib/tonlib/Config.h | 2 + tonlib/tonlib/ExtClient.cpp | 6 +- tonlib/tonlib/ExtClient.h | 15 +- tonlib/tonlib/ExtClientLazy.cpp | 24 +-- tonlib/tonlib/ExtClientMulti.cpp | 169 +++++++++++++++ tonlib/tonlib/ExtClientMulti.h | 61 ++++++ tonlib/tonlib/ExtClientOutbound.cpp | 8 +- tonlib/tonlib/ExtClientOutbound.h | 6 +- .../{ExtClientLazy.h => ExtClientRaw.h} | 15 +- tonlib/tonlib/QueryTraits.h | 202 ++++++++++++++++++ tonlib/tonlib/TonlibClient.cpp | 38 ++-- tonlib/tonlib/TonlibClient.h | 4 +- tonlib/tonlib/tonlib-cli.cpp | 31 ++- 17 files changed, 555 insertions(+), 83 deletions(-) create mode 100644 tonlib/tonlib/ExtClientMulti.cpp create mode 100644 tonlib/tonlib/ExtClientMulti.h rename tonlib/tonlib/{ExtClientLazy.h => ExtClientRaw.h} (68%) create mode 100644 tonlib/tonlib/QueryTraits.h diff --git a/tl/generate/scheme/tonlib_api.tl b/tl/generate/scheme/tonlib_api.tl index 79185d5bc..e83ee7842 100644 --- a/tl/generate/scheme/tonlib_api.tl +++ b/tl/generate/scheme/tonlib_api.tl @@ -183,7 +183,7 @@ smc.runResult gas_used:int53 stack:vector exit_code:int32 = smc. smc.libraryEntry hash:int256 data:bytes = smc.LibraryEntry; smc.libraryResult result:(vector smc.libraryEntry) = smc.LibraryResult; -updateSendLiteServerQuery id:int64 data:bytes = Update; +updateSendLiteServerQuery id:int64 data:bytes workchain:int32 shard:int64 = Update; updateSyncState sync_state:SyncState = Update; //@class LogStream @description Describes a stream to which tonlib internal log is written diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index ee457609da33cca7655480b0531417fd19ccf958..5f3970a1197f2a1c2cdabfbcb6e8638fcc202586 100644 GIT binary patch delta 94 zcmezJfpN!2#tjSDc<1 Config::parse(std::string str) { if (json.type() != td::JsonValue::Type::Object) { return td::Status::Error("Invalid config (1)"); } - //TRY_RESULT(main_type, td::get_json_object_string_field(json.get_object(), "@type", false)); - //if (main_type != "config.global") { - //return td::Status::Error("Invalid config (3)"); - //} + td::JsonArray empty_array; TRY_RESULT(lite_clients_obj, - td::get_json_object_field(json.get_object(), "liteservers", td::JsonValue::Type::Array, false)); - auto &lite_clients = lite_clients_obj.get_array(); + td::get_json_object_field(json.get_object(), "liteservers", td::JsonValue::Type::Array, true)); + auto &lite_clients = + lite_clients_obj.type() == td::JsonValue::Type::Array ? lite_clients_obj.get_array() : empty_array; + TRY_RESULT(lite_clients_v2_obj, + td::get_json_object_field(json.get_object(), "liteservers_v2", td::JsonValue::Type::Array, true)); + auto &lite_clients_v2 = + lite_clients_v2_obj.type() == td::JsonValue::Type::Array ? lite_clients_v2_obj.get_array() : empty_array; - Config res; - for (auto &value : lite_clients) { + auto parse_desc = [&](td::JsonValue& value) -> td::Result { if (value.type() != td::JsonValue::Type::Object) { return td::Status::Error("Invalid config (2)"); } auto &object = value.get_object(); - //TRY_RESULT(value_type, td::get_json_object_string_field(object, "@type", false)); - //if (value_type != "liteclient.config.global") { - //return td::Status::Error("Invalid config (4)"); - //} TRY_RESULT(ip, td::get_json_object_long_field(object, "ip", false)); TRY_RESULT(port, td::get_json_object_int_field(object, "port", false)); @@ -93,15 +90,41 @@ td::Result Config::parse(std::string str) { auto &id = id_obj.get_object(); TRY_RESULT(id_type, td::get_json_object_string_field(id, "@type", false)); if (id_type != "pub.ed25519") { - return td::Status::Error("Invalid config (5)"); + return td::Status::Error("Invalid config (3)"); } TRY_RESULT(key_base64, td::get_json_object_string_field(id, "key", false)); TRY_RESULT(key, td::base64_decode(key_base64)); if (key.size() != 32) { - return td::Status::Error("Invalid config (6)"); + return td::Status::Error("Invalid config (4)"); } client.adnl_id = ton::adnl::AdnlNodeIdFull(ton::pubkeys::Ed25519(td::Bits256(td::Slice(key).ubegin()))); + return client; + }; + + Config res; + for (auto &value : lite_clients) { + TRY_RESULT(client, parse_desc(value)); + res.lite_clients.push_back(std::move(client)); + } + for (auto &value : lite_clients_v2) { + TRY_RESULT(client, parse_desc(value)); + client.is_full = false; + TRY_RESULT(shards_obj, + td::get_json_object_field(value.get_object(), "shards", td::JsonValue::Type::Array, false)); + for (auto &shard : shards_obj.get_array()) { + if (shard.type() != td::JsonValue::Type::Object) { + return td::Status::Error("Invalid config (5)"); + } + auto &shard_obj = shard.get_object(); + TRY_RESULT(workchain, td::get_json_object_int_field(shard_obj, "workchain", false)); + TRY_RESULT(shard_id, td::get_json_object_long_field(shard_obj, "shard", false)); + if (shard_id == 0) { + return td::Status::Error("Invalid config (6)"); + } + client.shards.emplace_back(workchain, shard_id); + } + res.lite_clients.push_back(std::move(client)); } diff --git a/tonlib/tonlib/Config.h b/tonlib/tonlib/Config.h index 3902c3419..420889dfd 100644 --- a/tonlib/tonlib/Config.h +++ b/tonlib/tonlib/Config.h @@ -26,6 +26,8 @@ struct Config { struct LiteClient { ton::adnl::AdnlNodeIdFull adnl_id; td::IPAddress address; + bool is_full = true; + std::vector shards; }; ton::BlockIdExt zero_state_id; ton::BlockIdExt init_block_id; diff --git a/tonlib/tonlib/ExtClient.cpp b/tonlib/tonlib/ExtClient.cpp index 30a29b59c..fdd1606df 100644 --- a/tonlib/tonlib/ExtClient.cpp +++ b/tonlib/tonlib/ExtClient.cpp @@ -54,7 +54,7 @@ void ExtClient::with_last_block(td::Promise promise) { td::actor::send_closure(client_.last_block_actor_, &LastBlock::get_last_block, std::move(P)); } -void ExtClient::send_raw_query(td::BufferSlice query, td::Promise promise) { +void ExtClient::send_raw_query(td::BufferSlice query, ton::ShardIdFull shard, td::Promise promise) { auto query_id = queries_.create(std::move(promise)); td::Promise P = [query_id, self = this, actor_id = td::actor::actor_id()](td::Result result) { @@ -62,10 +62,10 @@ void ExtClient::send_raw_query(td::BufferSlice query, td::Promisequeries_.extract(query_id).set_result(std::move(result)); }); }; - if (client_.adnl_ext_client_.empty()) { + if (client_.raw_client_.empty()) { return P.set_error(TonlibError::NoLiteServers()); } - td::actor::send_closure(client_.adnl_ext_client_, &ton::adnl::AdnlExtClient::send_query, "query", std::move(query), + td::actor::send_closure(client_.raw_client_, &ExtClientRaw::send_query, "query", std::move(query), shard, td::Timestamp::in(10.0), std::move(P)); } } // namespace tonlib diff --git a/tonlib/tonlib/ExtClient.h b/tonlib/tonlib/ExtClient.h index 707ca74bd..488186222 100644 --- a/tonlib/tonlib/ExtClient.h +++ b/tonlib/tonlib/ExtClient.h @@ -28,9 +28,11 @@ #include "td/utils/Container.h" #include "td/utils/Random.h" -#include "ExtClientLazy.h" +#include "ExtClientRaw.h" #include "TonlibError.h" #include "utils.h" +#include "QueryTraits.h" +#include "ExtClientRaw.h" namespace tonlib { class LastBlock; @@ -38,7 +40,7 @@ class LastConfig; struct LastBlockState; struct LastConfigState; struct ExtClientRef { - td::actor::ActorId adnl_ext_client_; + td::actor::ActorId raw_client_; td::actor::ActorId last_block_actor_; td::actor::ActorId last_config_actor_; }; @@ -64,6 +66,7 @@ class ExtClient { template void send_query(QueryT query, td::Promise promise, td::int32 seq_no = -1) { + ton::ShardIdFull shard = QueryTraits::get_shard(query); auto raw_query = ton::serialize_tl_object(&query, true); td::uint32 tag = td::Random::fast_uint32(); VLOG(lite_server) << "send query to liteserver: " << tag << " " << to_string(query); @@ -77,7 +80,7 @@ class ExtClient { ton::serialize_tl_object(ton::create_tl_object(std::move(raw_query)), true); send_raw_query( - std::move(liteserver_query), [promise = std::move(promise), tag](td::Result R) mutable { + std::move(liteserver_query), shard, [promise = std::move(promise), tag](td::Result R) mutable { auto res = [&]() -> td::Result { TRY_RESULT_PREFIX(data, std::move(R), TonlibError::LiteServerNetwork()); auto r_error = ton::fetch_tl_object(data.clone(), true); @@ -96,8 +99,8 @@ class ExtClient { } void force_change_liteserver() { - if (!client_.adnl_ext_client_.empty()) { - td::actor::send_closure(client_.adnl_ext_client_, &ExtClientLazy::force_change_liteserver); + if (!client_.raw_client_.empty()) { + td::actor::send_closure(client_.raw_client_, &ExtClientRaw::force_change_liteserver); } } @@ -107,6 +110,6 @@ class ExtClient { td::Container> last_block_queries_; td::Container> last_config_queries_; - void send_raw_query(td::BufferSlice query, td::Promise promise); + void send_raw_query(td::BufferSlice query, ton::ShardIdFull shard, td::Promise promise); }; } // namespace tonlib diff --git a/tonlib/tonlib/ExtClientLazy.cpp b/tonlib/tonlib/ExtClientLazy.cpp index 335a0ff96..625d7eaa7 100644 --- a/tonlib/tonlib/ExtClientLazy.cpp +++ b/tonlib/tonlib/ExtClientLazy.cpp @@ -16,15 +16,15 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "ExtClientLazy.h" +#include "ExtClientRaw.h" #include "TonlibError.h" #include "td/utils/Random.h" namespace tonlib { -class ExtClientLazyImp : public ExtClientLazy { +class ExtClientLazyImp : public ExtClientRaw { public: ExtClientLazyImp(std::vector> servers, - td::unique_ptr callback) + td::unique_ptr callback) : servers_(std::move(servers)), callback_(std::move(callback)) { CHECK(!servers_.empty()); } @@ -34,15 +34,7 @@ class ExtClientLazyImp : public ExtClientLazy { td::random_shuffle(td::as_mutable_span(servers_), rnd); } - void check_ready(td::Promise promise) override { - before_query(); - if (client_.empty()) { - return promise.set_error(TonlibError::Cancelled()); - } - send_closure(client_, &ton::adnl::AdnlExtClient::check_ready, std::move(promise)); - } - - void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, + void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, td::Promise promise) override { before_query(); if (client_.empty()) { @@ -109,7 +101,7 @@ class ExtClientLazyImp : public ExtClientLazy { bool cur_server_bad_force_ = false; td::actor::ActorOwn client_; - td::unique_ptr callback_; + td::unique_ptr callback_; static constexpr double MAX_NO_QUERIES_TIMEOUT = 100; bool is_closing_{false}; @@ -140,12 +132,12 @@ class ExtClientLazyImp : public ExtClientLazy { } }; -td::actor::ActorOwn ExtClientLazy::create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, - td::unique_ptr callback) { +td::actor::ActorOwn ExtClientRaw::create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, + td::unique_ptr callback) { return create({std::make_pair(dst, dst_addr)}, std::move(callback)); } -td::actor::ActorOwn ExtClientLazy::create( +td::actor::ActorOwn ExtClientRaw::create( std::vector> servers, td::unique_ptr callback) { return td::actor::create_actor("ExtClientLazy", std::move(servers), std::move(callback)); } diff --git a/tonlib/tonlib/ExtClientMulti.cpp b/tonlib/tonlib/ExtClientMulti.cpp new file mode 100644 index 000000000..3a6cdf595 --- /dev/null +++ b/tonlib/tonlib/ExtClientMulti.cpp @@ -0,0 +1,169 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "ExtClientMulti.h" +#include "ton/ton-shard.h" +#include "td/utils/Random.h" + +namespace tonlib { + +static const double MAX_NO_QUERIES_TIMEOUT = 120; + +ExtClientMulti::ExtClientMulti(std::vector clients, td::unique_ptr callback) + : callback_(std::move(callback)) { + for (auto &desc : clients) { + servers_.emplace_back(); + servers_.back().desc = std::move(desc); + } +} + +void ExtClientMulti::start_up() { + alarm_timestamp() = td::Timestamp::in(60.0); +} + +void ExtClientMulti::send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, + td::Promise promise) { + if (shard.is_masterchain() && mc_server_idx_ != -1 && !servers_[mc_server_idx_].is_bad()) { + send_query_to_server(std::move(name), std::move(data), mc_server_idx_, timeout, std::move(promise)); + return; + } + auto it = shard_server_idx_cached_.find(shard); + if (it != shard_server_idx_cached_.end() && !servers_[it->second].is_bad()) { + send_query_to_server(std::move(name), std::move(data), it->second, timeout, std::move(promise)); + return; + } + int server_idx = -1; + int random_idx = -1; + int cnt = 0; + int best_prefix = -1; + for (int i = 0; i < (int)servers_.size(); ++i) { + const Server &server = servers_[i]; + if (server.is_bad()) { + continue; + } + int len = server.desc.is_full ? 65 : server.max_supported_prefix(shard); + if (len > best_prefix) { + best_prefix = len; + server_idx = -1; + random_idx = -1; + cnt = 0; + } else if (len < best_prefix) { + continue; + } + if (!server.client.empty()) { + server_idx = i; + } + if (td::Random::fast(0, cnt) == 0) { + random_idx = i; + } + ++cnt; + } + if (server_idx == -1) { + server_idx = random_idx; + } + if (server_idx == -1) { + promise.set_error(td::Status::Error("failed to select a suitable server")); + return; + } + if (shard.pfx_len() <= ton::max_shard_pfx_len) { + shard_server_idx_cached_[shard] = server_idx; + } + if (shard.is_masterchain() || servers_[server_idx].desc.is_full) { + mc_server_idx_ = server_idx; + } + send_query_to_server(std::move(name), std::move(data), server_idx, timeout, std::move(promise)); +} + +void ExtClientMulti::send_query_to_server(std::string name, td::BufferSlice data, int server_idx, td::Timestamp timeout, + td::Promise promise) { + Server &server = servers_.at(server_idx); + if (server.client.empty()) { + start_client(server_idx); + } + server.ttl = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT); + td::Promise P = [SelfId = actor_id(this), idx = server_idx, + promise = std::move(promise)](td::Result R) mutable { + if (R.is_error() && + (R.error().code() == ton::ErrorCode::timeout || R.error().code() == ton::ErrorCode::cancelled)) { + td::actor::send_closure(SelfId, &ExtClientMulti::set_server_bad, idx); + } + promise.set_result(std::move(R)); + }; + send_closure(server.client, &ton::adnl::AdnlExtClient::send_query, std::move(name), std::move(data), timeout, + std::move(P)); +} + +void ExtClientMulti::force_change_liteserver() { + if (mc_server_idx_ != -1) { + set_server_bad(mc_server_idx_); + mc_server_idx_ = -1; + } +} + +void ExtClientMulti::start_client(int server_idx) { + class Callback : public ton::adnl::AdnlExtClient::Callback { + public: + Callback(td::actor::ActorId parent, int idx) : parent_(std::move(parent)), idx_(idx) { + } + void on_ready() override { + } + void on_stop_ready() override { + td::actor::send_closure(parent_, &ExtClientMulti::set_server_bad, idx_); + } + + private: + td::actor::ActorId parent_; + int idx_; + }; + Server &server = servers_.at(server_idx); + server.client = ton::adnl::AdnlExtClient::create(server.desc.adnl_id, server.desc.address, + std::make_unique(actor_id(this), server_idx)); +} + +void ExtClientMulti::alarm() { + for (Server& server : servers_) { + if (server.ttl && server.ttl.is_in_past()) { + server.client.reset(); + } + } + alarm_timestamp() = td::Timestamp::in(60.0); +} + +void ExtClientMulti::set_server_bad(int idx) { + Server& server = servers_.at(idx); + server.client.reset(); + server.ttl = td::Timestamp::never(); + server.ignore_until = td::Timestamp::in(10.0); +} + +int ExtClientMulti::Server::max_supported_prefix(ton::ShardIdFull shard) const { + if (desc.is_full || shard.is_masterchain()) { + return shard.pfx_len(); + } + int res = -1; + for (const ton::ShardIdFull &our_shard : desc.shards) { + if (ton::shard_is_ancestor(our_shard, shard)) { + return shard.pfx_len(); + } + if (shard.workchain == our_shard.workchain) { + int x = std::min({shard.pfx_len(), our_shard.pfx_len(), ton::count_matching_bits(shard.shard, our_shard.shard)}); + res = std::max(res, x); + } + } + return res; +} + +} // namespace tonlib diff --git a/tonlib/tonlib/ExtClientMulti.h b/tonlib/tonlib/ExtClientMulti.h new file mode 100644 index 000000000..4c93b00ee --- /dev/null +++ b/tonlib/tonlib/ExtClientMulti.h @@ -0,0 +1,61 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "td/actor/actor.h" +#include "adnl/adnl-ext-client.h" +#include "Config.h" +#include "ExtClientRaw.h" +#include + +namespace tonlib { + +class ExtClientMulti : public ExtClientRaw { + public: + ExtClientMulti(std::vector clients, td::unique_ptr callback); + + void start_up() override; + void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, + td::Promise promise) override; + void alarm() override; + void force_change_liteserver() override; + + private: + void send_query_to_server(std::string name, td::BufferSlice data, int server_idx, td::Timestamp timeout, + td::Promise promise); + void start_client(int server_idx); + + struct Server { + Config::LiteClient desc; + td::actor::ActorOwn client; + td::Timestamp ttl; + td::Timestamp ignore_until = td::Timestamp::never(); + + int max_supported_prefix(ton::ShardIdFull shard) const; + bool is_bad() const { + return ignore_until && !ignore_until.is_in_past(); + } + }; + + void set_server_bad(int idx); + + td::unique_ptr callback_; + std::vector servers_; + int mc_server_idx_ = -1; + std::map shard_server_idx_cached_; +}; + +} // namespace tonlib diff --git a/tonlib/tonlib/ExtClientOutbound.cpp b/tonlib/tonlib/ExtClientOutbound.cpp index 025ba848e..e057368d7 100644 --- a/tonlib/tonlib/ExtClientOutbound.cpp +++ b/tonlib/tonlib/ExtClientOutbound.cpp @@ -27,15 +27,11 @@ class ExtClientOutboundImp : public ExtClientOutbound { ExtClientOutboundImp(td::unique_ptr callback) : callback_(std::move(callback)) { } - void check_ready(td::Promise promise) override { - promise.set_error(td::Status::Error("Not supported")); - } - - void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, + void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, td::Promise promise) override { auto query_id = next_query_id_++; queries_[query_id] = std::move(promise); - callback_->request(query_id, data.as_slice().str()); + callback_->request(query_id, data.as_slice().str(), shard); } void force_change_liteserver() override { diff --git a/tonlib/tonlib/ExtClientOutbound.h b/tonlib/tonlib/ExtClientOutbound.h index 87b73b9e0..912178684 100644 --- a/tonlib/tonlib/ExtClientOutbound.h +++ b/tonlib/tonlib/ExtClientOutbound.h @@ -19,16 +19,16 @@ #pragma once #include "td/actor/actor.h" -#include "ExtClientLazy.h" +#include "ExtClientRaw.h" namespace tonlib { -class ExtClientOutbound : public ExtClientLazy { +class ExtClientOutbound : public ExtClientRaw { public: class Callback { public: virtual ~Callback() { } - virtual void request(td::int64 id, std::string data) = 0; + virtual void request(td::int64 id, std::string data, ton::ShardIdFull shard) = 0; }; virtual void on_query_result(td::int64 id, td::Result r_data, td::Promise promise) = 0; static td::actor::ActorOwn create(td::unique_ptr callback); diff --git a/tonlib/tonlib/ExtClientLazy.h b/tonlib/tonlib/ExtClientRaw.h similarity index 68% rename from tonlib/tonlib/ExtClientLazy.h rename to tonlib/tonlib/ExtClientRaw.h index dc4490b3a..0a55bdc17 100644 --- a/tonlib/tonlib/ExtClientLazy.h +++ b/tonlib/tonlib/ExtClientRaw.h @@ -18,23 +18,24 @@ */ #pragma once #include "td/actor/actor.h" - #include "adnl/adnl-ext-client.h" +#include "ton/ton-types.h" namespace tonlib { -class ExtClientLazy : public ton::adnl::AdnlExtClient { +class ExtClientRaw : public td::actor::Actor { public: class Callback { public: - virtual ~Callback() { - } + virtual ~Callback() = default; }; + virtual void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, + td::Promise promise) = 0; virtual void force_change_liteserver() = 0; - static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, - td::unique_ptr callback); - static td::actor::ActorOwn create( + static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, + td::unique_ptr callback); + static td::actor::ActorOwn create( std::vector> servers, td::unique_ptr callback); }; diff --git a/tonlib/tonlib/QueryTraits.h b/tonlib/tonlib/QueryTraits.h new file mode 100644 index 000000000..bf480f64c --- /dev/null +++ b/tonlib/tonlib/QueryTraits.h @@ -0,0 +1,202 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "ton/ton-types.h" +#include "auto/tl/lite_api.h" +#include "vm/boc.h" +#include "vm/cellslice.h" +#include "block/block-auto.h" +#include "block/block-parse.h" + +namespace tonlib { + +template +struct QueryTraits { + static ton::ShardIdFull get_shard(const Query& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getMasterchainInfo& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getMasterchainInfoExt& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getTime& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getVersion& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getBlock& q) { + return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getState& q) { + return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getBlockHeader& q) { + return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_sendMessage& q) { + auto shard = [&]() -> td::Result { + vm::BagOfCells boc; + TRY_STATUS(boc.deserialize(q.body_.as_slice())); + if (boc.get_root_count() != 1) { + return td::Status::Error("external message is not a valid bag of cells"); + } + block::gen::CommonMsgInfo::Record_ext_in_msg_info info; + if (!tlb::unpack_cell_inexact(boc.get_root_cell(), info)) { + return td::Status::Error("cannot unpack external message header"); + } + auto dest_prefix = block::tlb::t_MsgAddressInt.get_prefix(info.dest); + if (!dest_prefix.is_valid()) { + return td::Status::Error("destination of an inbound external message is an invalid blockchain address"); + } + return dest_prefix.as_leaf_shard(); + }(); + if (shard.is_error()) { + LOG(DEBUG) << "Failed to get shard from query liteServer.sendMessage: " << shard.move_as_error(); + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } + return shard.move_as_ok(); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getAccountState& q) { + return ton::AccountIdPrefixFull(q.account_->workchain_, q.account_->id_.bits().get_uint(64)).as_leaf_shard(); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_runSmcMethod& q) { + return ton::AccountIdPrefixFull(q.account_->workchain_, q.account_->id_.bits().get_uint(64)).as_leaf_shard(); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getShardInfo& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getAllShardsInfo& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getOneTransaction& q) { + return ton::AccountIdPrefixFull(q.account_->workchain_, q.account_->id_.bits().get_uint(64)).as_leaf_shard(); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getTransactions& q) { + return ton::AccountIdPrefixFull(q.account_->workchain_, q.account_->id_.bits().get_uint(64)).as_leaf_shard(); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_lookupBlock& q) { + return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_listBlockTransactions& q) { + return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getBlockProof& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getConfigAll& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getConfigParams& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getValidatorStats& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getLibraries& q) { + return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); + } +}; + + +} // namespace tonlib diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index f9e984bb9..b4b386978 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -18,7 +18,7 @@ */ #include "TonlibClient.h" -#include "tonlib/ExtClientLazy.h" +#include "tonlib/ExtClientRaw.h" #include "tonlib/ExtClientOutbound.h" #include "tonlib/LastBlock.h" #include "tonlib/LastConfig.h" @@ -27,6 +27,7 @@ #include "tonlib/keys/Mnemonic.h" #include "tonlib/keys/SimpleEncryption.h" #include "tonlib/TonlibError.h" +#include "tonlib/ExtClientMulti.h" #include "smc-envelope/GenericAccount.h" #include "smc-envelope/ManualDns.h" @@ -1649,15 +1650,16 @@ void TonlibClient::hangup() { ExtClientRef TonlibClient::get_client_ref() { ExtClientRef ref; - ref.adnl_ext_client_ = raw_client_.get(); + ref.raw_client_ = raw_client_.get(); ref.last_block_actor_ = raw_last_block_.get(); ref.last_config_actor_ = raw_last_config_.get(); return ref; } -void TonlibClient::proxy_request(td::int64 query_id, std::string data) { - on_update(tonlib_api::make_object(query_id, data)); +void TonlibClient::proxy_request(td::int64 query_id, std::string data, ton::ShardIdFull shard) { + on_update( + tonlib_api::make_object(query_id, data, shard.workchain, shard.shard)); } void TonlibClient::init_ext_client() { @@ -1668,9 +1670,9 @@ void TonlibClient::init_ext_client() { : parent_(std::move(parent)), config_generation_(config_generation) { } - void request(td::int64 id, std::string data) override { + void request(td::int64 id, std::string data, ton::ShardIdFull shard) override { send_closure(parent_, &TonlibClient::proxy_request, (id << 16) | (config_generation_ & 0xffff), - std::move(data)); + std::move(data), shard); } private: @@ -1683,21 +1685,29 @@ void TonlibClient::init_ext_client() { ext_client_outbound_ = client.get(); raw_client_ = std::move(client); } else { - std::vector> servers; - for (const auto& s : config_.lite_clients) { - servers.emplace_back(s.adnl_id, s.address); - } - class Callback : public ExtClientLazy::Callback { + class Callback : public ExtClientRaw::Callback { public: explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { } - private: td::actor::ActorShared<> parent_; }; + std::vector> full_servers; + for (const auto& s : config_.lite_clients) { + if (s.is_full) { + full_servers.emplace_back(s.adnl_id, s.address); + } + } + if (!full_servers.empty()) { + raw_client_ = + ExtClientRaw::create(std::move(full_servers), td::make_unique(td::actor::actor_shared())); + } else { + CHECK(!config_.lite_clients.empty()); + raw_client_ = td::actor::create_actor("ExtClientMulti", config_.lite_clients, + td::make_unique(td::actor::actor_shared())); + } ext_client_outbound_ = {}; ref_cnt_++; - raw_client_ = ExtClientLazy::create(std::move(servers), td::make_unique(td::actor::actor_shared())); } } @@ -2826,7 +2836,7 @@ td::Status TonlibClient::do_request(tonlib_api::raw_getTransactionsV2& request, auto actor_id = actor_id_++; actors_[actor_id] = td::actor::create_actor( - "GetTransactionHistory", client_.get_client(), account_address, lt, hash, count, actor_shared(this, actor_id), + "GetTransactionHistory", client_.get_client(), account_address, lt, hash, count, actor_shared(this, actor_id), promise.wrap([private_key = std::move(private_key), try_decode_messages = request.try_decode_messages_](auto&& x) mutable { return ToRawTransactions(std::move(private_key), try_decode_messages).to_raw_transactions(std::move(x)); })); diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index e78192909..8c0768762 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -110,7 +110,7 @@ class TonlibClient : public td::actor::Actor { vm::Dictionary libraries{256}; // network - td::actor::ActorOwn raw_client_; + td::actor::ActorOwn raw_client_; td::actor::ActorId ext_client_outbound_; td::actor::ActorOwn raw_last_block_; td::actor::ActorOwn raw_last_config_; @@ -372,7 +372,7 @@ class TonlibClient : public td::actor::Actor { td::Status do_request(const tonlib_api::getConfigParam& request, td::Promise>&& promise); - void proxy_request(td::int64 query_id, std::string data); + void proxy_request(td::int64 query_id, std::string data, ton::ShardIdFull shard); void load_libs_from_disk(); void store_libs_to_disk(); diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index 0a042eb1d..c256bf1ee 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -47,7 +47,7 @@ #include "tonlib/TonlibClient.h" #include "tonlib/TonlibCallback.h" -#include "tonlib/ExtClientLazy.h" +#include "tonlib/ExtClientRaw.h" #include "smc-envelope/ManualDns.h" #include "smc-envelope/PaymentChannel.h" @@ -62,6 +62,7 @@ #include #include #include "git.h" +#include "ExtClientMulti.h" using tonlib_api::make_object; @@ -174,7 +175,7 @@ class TonlibCli : public td::actor::Actor { std::map>> query_handlers_; - td::actor::ActorOwn raw_client_; + td::actor::ActorOwn raw_client_; bool is_closing_{false}; td::uint32 ref_cnt_{1}; @@ -223,11 +224,7 @@ class TonlibCli : public td::actor::Actor { if (options_.use_callbacks_for_network) { auto config = tonlib::Config::parse(options_.config).move_as_ok(); - auto lite_clients_size = config.lite_clients.size(); - CHECK(lite_clients_size != 0); - auto lite_client_id = td::Random::fast(0, td::narrow_cast(lite_clients_size) - 1); - auto& lite_client = config.lite_clients[lite_client_id]; - class Callback : public tonlib::ExtClientLazy::Callback { + class Callback : public tonlib::ExtClientRaw::Callback { public: explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { } @@ -235,9 +232,22 @@ class TonlibCli : public td::actor::Actor { private: td::actor::ActorShared<> parent_; }; + std::vector> full_servers; + int lite_client_id = -1, cnt = 0; + for (const auto& s : config.lite_clients) { + if (s.is_full) { + full_servers.emplace_back(s.adnl_id, s.address); + } + } + if (!full_servers.empty()) { + raw_client_ = tonlib::ExtClientRaw::create(std::move(full_servers), + td::make_unique(td::actor::actor_shared())); + } else { + CHECK(!config.lite_clients.empty()); + raw_client_ = td::actor::create_actor( + "ExtClientMulti", config.lite_clients, td::make_unique(td::actor::actor_shared())); + } ref_cnt_++; - raw_client_ = tonlib::ExtClientLazy::create(lite_client.adnl_id, lite_client.address, - td::make_unique(td::actor::actor_shared())); } auto config = !options_.config.empty() @@ -1534,7 +1544,8 @@ class TonlibCli : public td::actor::Actor { auto update = tonlib_api::move_object_as(std::move(result)); CHECK(!raw_client_.empty()); snd_bytes_ += update->data_.size(); - send_closure(raw_client_, &ton::adnl::AdnlExtClient::send_query, "query", td::BufferSlice(update->data_), + ton::ShardIdFull shard(update->workchain_, update->shard_); + send_closure(raw_client_, &tonlib::ExtClientRaw::send_query, "query", td::BufferSlice(update->data_), shard, td::Timestamp::in(5), [actor_id = actor_id(this), id = update->id_](td::Result res) { send_closure(actor_id, &TonlibCli::on_adnl_result, id, std::move(res)); From 23dbf2d834256ef7fafca6f536eead1ebb594c78 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Sat, 15 Oct 2022 11:58:01 +0300 Subject: [PATCH 042/388] Move CollatorConfig to -41 Config Param --- crypto/block/mc-config.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index ed79c8b63..6b3e51ffa 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2135,7 +2135,7 @@ Ref ConfigInfo::lookup_library(td::ConstBitPtr root_hash) const { CollatorConfig Config::get_collator_config(bool need_collator_nodes) const { CollatorConfig collator_config; gen::CollatorConfig::Record rec; - auto cell = get_config_param(41); + auto cell = get_config_param(-41); if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { return collator_config; } From d4339b839c966dd423862bd328216da73df41169 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 17 Oct 2022 13:24:59 +0300 Subject: [PATCH 043/388] Add "delshard", "delcollator" to validator-engine-console --- tl/generate/scheme/ton_api.tl | 2 + tl/generate/scheme/ton_api.tlo | Bin 73376 -> 73612 bytes .../validator-engine-console-query.cpp | 41 +++++++ .../validator-engine-console-query.h | 47 ++++++++ validator-engine/validator-engine.cpp | 104 ++++++++++++++++++ validator-engine/validator-engine.hpp | 6 + validator/collator-node.cpp | 10 ++ validator/collator-node.hpp | 1 + validator/manager-disk.hpp | 3 + validator/manager-hardfork.hpp | 3 + validator/manager.cpp | 21 +++- validator/manager.hpp | 7 +- validator/validator.h | 1 + 13 files changed, 241 insertions(+), 5 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index f04f7863c..3a2ad1fc0 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -715,6 +715,8 @@ engine.validator.getValidatorSessionsInfo = engine.validator.ValidatorSessionsIn engine.validator.addCollator adnl_id:int256 shard:tonNode.shardId = engine.validator.Success; engine.validator.addShard shard:tonNode.shardId = engine.validator.Success; +engine.validator.delCollator adnl_id:int256 shard:tonNode.shardId = engine.validator.Success; +engine.validator.delShard shard:tonNode.shardId = engine.validator.Success; ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index d535668aa423585bb2be3319dbf46d3f4f610efc..2820a7898588afc696e5aef67cb15bbddf789087 100644 GIT binary patch delta 53 zcmV-50LuTMy#$QE1h9PT0cNv(>rW8{T*-tH9Fx)f6qA|pZ?nw&XaWSgb}CPk(EJsX L;r2_jH2-WLyuuo? delta 26 icmeC#&$3`I%Z8fQj47LIU;7Jg4*8qGv{ivIM;-vFhzoZB diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index cab347774..7bad3c974 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1119,3 +1119,44 @@ td::Status AddShardQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << "successfully added shard\n"; return td::Status::OK(); } + +td::Status DelCollatorQuery::run() { + TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); + return td::Status::OK(); +} + +td::Status DelCollatorQuery::send() { + auto b = ton::create_serialize_tl_object( + adnl_id_.tl(), ton::create_tl_shard_id(ton::ShardIdFull(wc_, shard_))); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status DelCollatorQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "successfully removed collator\n"; + return td::Status::OK(); +} + +td::Status DelShardQuery::run() { + TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); + return td::Status::OK(); +} + +td::Status DelShardQuery::send() { + auto b = ton::create_serialize_tl_object( + ton::create_tl_shard_id(ton::ShardIdFull(wc_, shard_))); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status DelShardQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "successfully removed shard\n"; + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 316cc5e77..3c2c21939 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1162,3 +1162,50 @@ class AddShardQuery : public Query { td::int32 wc_; td::int64 shard_; }; + +class DelCollatorQuery : public Query { + public: + DelCollatorQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "delcollator"; + } + static std::string get_help() { + return "delcollator \tremove collator with given adnl_id and shard"; + } + std::string name() const override { + return get_name(); + } + + private: + ton::PublicKeyHash adnl_id_; + td::int32 wc_; + td::int64 shard_; +}; + +class DelShardQuery : public Query { + public: + DelShardQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "delshard"; + } + static std::string get_help() { + return "delshard \tstop monitoring shard"; + } + std::string name() const override { + return get_name(); + } + + private: + td::int32 wc_; + td::int64 shard_; +}; diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 14a6961fa..cd14ae1ed 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -418,6 +418,19 @@ td::Result Config::config_add_collator(ton::PublicKeyHash addr, ton::Shard return true; } +td::Result Config::config_del_collator(ton::PublicKeyHash addr, ton::ShardIdFull shard) { + if (!shard.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard: " << shard.to_str()); + } + Collator c{addr, shard}; + auto it = std::find(collators.begin(), collators.end(), c); + if (it == collators.end()) { + return false; + } + collators.erase(it); + return true; +} + td::Result Config::config_add_full_node_adnl_id(ton::PublicKeyHash id) { if (full_node == id) { return false; @@ -548,6 +561,18 @@ td::Result Config::config_add_shard(ton::ShardIdFull shard) { return true; } +td::Result Config::config_del_shard(ton::ShardIdFull shard) { + if (!shard.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard " << shard.to_str()); + } + auto it = std::find(shards_to_monitor.begin(), shards_to_monitor.end(), shard); + if (it == shards_to_monitor.end()) { + return false; + } + shards_to_monitor.erase(it); + return true; +} + td::Result Config::config_add_gc(ton::PublicKeyHash key) { return gc.insert(key).second; } @@ -3506,6 +3531,85 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addShard }); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCollator &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + auto id = ton::PublicKeyHash{query.adnl_id_}; + auto shard = ton::create_shard_id(query.shard_); + auto R = config_.config_del_collator(id, shard); + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + if (!R.move_as_ok()) { + promise.set_value(create_control_query_error(td::Status::Error("No such collator"))); + return; + } + if (!R.move_as_ok()) { + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + return; + } + set_shard_check_function(); + if (!validator_manager_.empty()) { + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::del_collator, + ton::adnl::AdnlNodeIdShort(id), shard); + } + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delShard &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + auto shard = ton::create_shard_id(query.shard_); + auto R = config_.config_del_shard(shard); + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + if (!R.move_as_ok()) { + promise.set_value(create_control_query_error(td::Status::Error("No such shard"))); + return; + } + set_shard_check_function(); + if (!validator_manager_.empty()) { + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + } + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 71276e5df..49010bc47 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -115,6 +115,7 @@ struct Config { td::Result config_add_validator_adnl_id(ton::PublicKeyHash perm_key, ton::PublicKeyHash adnl_id, ton::UnixTime expire_at); td::Result config_add_collator(ton::PublicKeyHash addr, ton::ShardIdFull shard); + td::Result config_del_collator(ton::PublicKeyHash addr, ton::ShardIdFull shard); td::Result config_add_full_node_adnl_id(ton::PublicKeyHash id); td::Result config_add_full_node_slave(td::IPAddress addr, ton::PublicKey id); td::Result config_add_full_node_master(td::int32 port, ton::PublicKeyHash id); @@ -123,6 +124,7 @@ struct Config { td::Result config_add_control_process(ton::PublicKeyHash key, td::int32 port, ton::PublicKeyHash id, td::uint32 permissions); td::Result config_add_shard(ton::ShardIdFull shard); + td::Result config_del_shard(ton::ShardIdFull shard); td::Result config_add_gc(ton::PublicKeyHash key); td::Result config_del_network_addr(td::IPAddress addr, std::vector cats, std::vector prio_cats); @@ -437,6 +439,10 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_addShard &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_delCollator &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_delShard &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getPerfTimerStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 0a2a050eb..c6afa5418 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -54,10 +54,20 @@ void CollatorNode::tear_down() { } void CollatorNode::add_shard(ShardIdFull shard) { + if (std::find(shards_.begin(), shards_.end(), shard) != shards_.end()) { + return; + } LOG(INFO) << "Collator node: local_id=" << local_id_ << " , shard=" << shard.to_str(); shards_.push_back(shard); } +void CollatorNode::del_shard(ShardIdFull shard) { + auto it = std::find(shards_.begin(), shards_.end(), shard); + if (it != shards_.end()) { + shards_.erase(it); + } +} + void CollatorNode::new_masterchain_block_notification(td::Ref state) { std::vector top_blocks = {state->get_block_id()}; std::vector next_shards; diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp index b0a2ad903..b3b232cf9 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node.hpp @@ -32,6 +32,7 @@ class CollatorNode : public td::actor::Actor { void start_up() override; void tear_down() override; void add_shard(ShardIdFull shard); + void del_shard(ShardIdFull shard); void new_masterchain_block_notification(td::Ref state); diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index c4403e941..e0aad62ed 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -395,6 +395,9 @@ class ValidatorManagerImpl : public ValidatorManager { void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { UNREACHABLE(); } + void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { + UNREACHABLE(); + } void update_options(td::Ref opts) override { opts_ = std::move(opts); diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index bd30b1de3..e8fa327b0 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -454,6 +454,9 @@ class ValidatorManagerImpl : public ValidatorManager { void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { UNREACHABLE(); } + void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { + UNREACHABLE(); + } void update_options(td::Ref opts) override { opts_ = std::move(opts); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 183dc8c1e..f56d69017 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1795,7 +1795,7 @@ void ValidatorManagerImpl::new_masterchain_block() { last_masterchain_block_handle_, last_masterchain_state_); } for (auto &c : collator_nodes_) { - td::actor::send_closure(c.second, &CollatorNode::new_masterchain_block_notification, last_masterchain_state_); + td::actor::send_closure(c.second.actor, &CollatorNode::new_masterchain_block_notification, last_masterchain_state_); } if (last_masterchain_seqno_ % 1024 == 0) { @@ -2722,10 +2722,23 @@ void ValidatorManagerImpl::get_validator_sessions_info( void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) { auto it = collator_nodes_.find(id); if (it == collator_nodes_.end()) { - auto actor = td::actor::create_actor("collatornode", id, actor_id(this), adnl_, rldp_); - it = collator_nodes_.emplace(id, std::move(actor)).first; + it = collator_nodes_.emplace(id, Collator()).first; + it->second.actor = td::actor::create_actor("collatornode", id, actor_id(this), adnl_, rldp_); + } + it->second.shards.insert(shard); + td::actor::send_closure(it->second.actor, &CollatorNode::add_shard, shard); +} + +void ValidatorManagerImpl::del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) { + auto it = collator_nodes_.find(id); + if (it == collator_nodes_.end()) { + return; + } + td::actor::send_closure(it->second.actor, &CollatorNode::del_shard, shard); + it->second.shards.erase(shard); + if (it->second.shards.empty()) { + collator_nodes_.erase(it); } - td::actor::send_closure(it->second, &CollatorNode::add_shard, shard); } void ValidatorManagerImpl::update_options(td::Ref opts) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 136e5e8ac..6fca11785 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -563,6 +563,7 @@ class ValidatorManagerImpl : public ValidatorManager { void add_persistent_state_description(td::Ref desc) override; void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; + void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; void update_options(td::Ref opts) override; private: @@ -639,7 +640,11 @@ class ValidatorManagerImpl : public ValidatorManager { std::map> shard_client_waiters_; - std::map> collator_nodes_; + struct Collator { + td::actor::ActorOwn actor; + std::set shards; + }; + std::map collator_nodes_; std::set extra_active_shards_; std::map last_validated_blocks_; diff --git a/validator/validator.h b/validator/validator.h index c387caffc..0a5a22b5f 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -238,6 +238,7 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Promise> promise) = 0; virtual void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; + virtual void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; virtual void update_options(td::Ref opts) = 0; }; From 2ea17ec03b8b116e24c800b3389e34b5ae2e21f9 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 11 Jan 2023 19:14:03 +0300 Subject: [PATCH 044/388] Use partial liteservers in tonlib and lite-client --- lite-client/lite-client.cpp | 757 ++++++++++-------- lite-client/lite-client.h | 9 +- tl/generate/scheme/ton_api.tlo | Bin 83196 -> 87748 bytes tl/generate/scheme/tonlib_api.tlo | Bin 31016 -> 31084 bytes tonlib/CMakeLists.txt | 4 +- tonlib/tonlib/Config.cpp | 41 +- tonlib/tonlib/Config.h | 4 +- tonlib/tonlib/ExtClient.cpp | 6 +- tonlib/tonlib/ExtClient.h | 1 - tonlib/tonlib/ExtClientLazy.cpp | 174 ++-- .../{ExtClientRaw.h => ExtClientLazy.h} | 14 +- tonlib/tonlib/ExtClientMulti.cpp | 169 ---- tonlib/tonlib/ExtClientMulti.h | 61 -- tonlib/tonlib/ExtClientOutbound.cpp | 10 +- tonlib/tonlib/ExtClientOutbound.h | 1 + tonlib/tonlib/TonlibClient.cpp | 581 +++++++------- tonlib/tonlib/tonlib-cli.cpp | 33 +- 17 files changed, 891 insertions(+), 974 deletions(-) rename tonlib/tonlib/{ExtClientRaw.h => ExtClientLazy.h} (72%) delete mode 100644 tonlib/tonlib/ExtClientMulti.cpp delete mode 100644 tonlib/tonlib/ExtClientMulti.h diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index feedbe40b..c6988ef3e 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -29,22 +29,16 @@ #include "lite-client-common.h" -#include "adnl/adnl-ext-client.h" #include "tl-utils/lite-utils.hpp" #include "auto/tl/ton_api_json.h" #include "auto/tl/lite_api.hpp" #include "td/utils/OptionParser.h" #include "td/utils/Time.h" #include "td/utils/filesystem.h" -#include "td/utils/format.h" #include "td/utils/Random.h" #include "td/utils/crypto.h" -#include "td/utils/overloaded.h" #include "td/utils/port/signals.h" -#include "td/utils/port/stacktrace.h" -#include "td/utils/port/StdStreams.h" #include "td/utils/port/FileFd.h" -#include "terminal/terminal.h" #include "ton/lite-tl.hpp" #include "block/block-db.h" #include "block/block.h" @@ -58,18 +52,14 @@ #include "vm/vm.h" #include "vm/cp0.h" #include "vm/memo.h" -#include "ton/ton-shard.h" -#include "openssl/rand.hpp" #include "crypto/vm/utils.h" #include "crypto/common/util.h" #include "common/checksum.h" #if TD_DARWIN || TD_LINUX #include -#include #endif #include -#include #include "git.h" using namespace std::literals::string_literals; @@ -77,22 +67,10 @@ using td::Ref; int verbosity; -std::unique_ptr TestNode::make_callback() { - class Callback : public ton::adnl::AdnlExtClient::Callback { - public: - void on_ready() override { - td::actor::send_closure(id_, &TestNode::conn_ready); - } - void on_stop_ready() override { - td::actor::send_closure(id_, &TestNode::conn_closed); - } - Callback(td::actor::ActorId id) : id_(std::move(id)) { - } - - private: - td::actor::ActorId id_; - }; - return std::make_unique(actor_id(this)); +bool TestNode::LiteServer::supports(ton::ShardIdFull shard) const { + return is_full || shard.is_masterchain() || + std::any_of(shards.begin(), shards.end(), + [&](const ton::ShardIdFull& our_shard) { return ton::shard_intersects(shard, our_shard); }); } void TestNode::run() { @@ -110,31 +88,57 @@ void TestNode::run() { io_ = td::TerminalIO::create("> ", readline_enabled_, ex_mode_, std::make_unique(actor_id(this))); td::actor::send_closure(io_, &td::TerminalIO::set_log_interface); - if (remote_public_key_.empty()) { - auto G = td::read_file(global_config_).move_as_ok(); - auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); - ton::ton_api::liteclient_config_global gc; - ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); - CHECK(gc.liteservers_.size() > 0); - auto idx = liteserver_idx_ >= 0 ? liteserver_idx_ - : td::Random::fast(0, static_cast(gc.liteservers_.size() - 1)); - CHECK(idx >= 0 && static_cast(idx) <= gc.liteservers_.size()); - auto& cli = gc.liteservers_[idx]; - remote_addr_.init_host_port(td::IPAddress::ipv4_to_str(cli->ip_), cli->port_).ensure(); - remote_public_key_ = ton::PublicKey{cli->id_}; - td::TerminalIO::out() << "using liteserver " << idx << " with addr " << remote_addr_ << "\n"; - if (gc.validator_ && gc.validator_->zero_state_) { - zstate_id_.workchain = gc.validator_->zero_state_->workchain_; - if (zstate_id_.workchain != ton::workchainInvalid) { - zstate_id_.root_hash = gc.validator_->zero_state_->root_hash_; - zstate_id_.file_hash = gc.validator_->zero_state_->file_hash_; - td::TerminalIO::out() << "zerostate set to " << zstate_id_.to_str() << "\n"; - } + if (!single_remote_public_key_.empty()) { // Use single provided liteserver + LiteServer s; + s.addr = single_remote_addr_; + s.public_key = single_remote_public_key_; + single_liteserver_idx_ = 0; + td::TerminalIO::out() << "using liteserver " << s.addr << "\n"; + servers_.push_back(std::move(s)); + run_init_queries(); + return; + } + + auto G = td::read_file(global_config_).move_as_ok(); + auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); + ton::ton_api::liteclient_config_global gc; + ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); + CHECK(gc.liteservers_.size() + gc.liteservers_v2_.size() > 0); + + if (gc.validator_ && gc.validator_->zero_state_) { + zstate_id_.workchain = gc.validator_->zero_state_->workchain_; + if (zstate_id_.workchain != ton::workchainInvalid) { + zstate_id_.root_hash = gc.validator_->zero_state_->root_hash_; + zstate_id_.file_hash = gc.validator_->zero_state_->file_hash_; + td::TerminalIO::out() << "zerostate set to " << zstate_id_.to_str() << "\n"; } } - client_ = - ton::adnl::AdnlExtClient::create(ton::adnl::AdnlNodeIdFull{remote_public_key_}, remote_addr_, make_callback()); + for (auto& server : gc.liteservers_) { + LiteServer s; + s.addr.init_host_port(td::IPAddress::ipv4_to_str(server->ip_), server->port_).ensure(); + s.public_key = ton::PublicKey{server->id_}; + servers_.push_back(std::move(s)); + } + for (auto& server : gc.liteservers_v2_) { + LiteServer s; + s.addr.init_host_port(td::IPAddress::ipv4_to_str(server->ip_), server->port_).ensure(); + s.public_key = ton::PublicKey{server->id_}; + s.is_full = false; + for (const auto& shard : server->shards_) { + s.shards.emplace_back(shard->workchain_, shard->shard_); + CHECK(s.shards.back().is_valid_ext()); + } + servers_.push_back(std::move(s)); + } + + if (single_liteserver_idx_ != -1) { // Use single liteserver from config + CHECK(single_liteserver_idx_ >= 0 && (size_t)single_liteserver_idx_ < gc.liteservers_.size()); + td::TerminalIO::out() << "using liteserver " << single_liteserver_idx_ << " with addr " + << servers_[single_liteserver_idx_].addr << "\n"; + } + + run_init_queries(); } void TestNode::got_result(td::Result R, td::Promise promise) { @@ -179,23 +183,142 @@ void TestNode::after_got_result(bool ok) { } } -bool TestNode::envelope_send_query(td::BufferSlice query, td::Promise promise) { - running_queries_++; - if (!ready_ || client_.empty()) { - got_result(td::Status::Error("failed to send query to server: not ready"), std::move(promise)); +bool TestNode::envelope_send_query_to_any(td::BufferSlice query, td::Promise promise) { + return envelope_send_query_to_shard(ton::ShardIdFull(ton::masterchainId), std::move(query), std::move(promise)); +} + +bool TestNode::envelope_send_query_to_account(ton::AccountIdPrefixFull prefix, td::BufferSlice query, + td::Promise promise) { + return envelope_send_query_to_shard(prefix.as_leaf_shard(), std::move(query), std::move(promise)); +} + +bool TestNode::envelope_send_query_to_shard(ton::ShardIdFull shard, td::BufferSlice query, + td::Promise promise) { + if (single_liteserver_idx_ >= 0) { + return envelope_send_query_to_server(single_liteserver_idx_, std::move(query), std::move(promise)); + } + if (shard.is_masterchain() && mc_server_idx_ != -1) { + return envelope_send_query_to_server(mc_server_idx_, std::move(query), std::move(promise)); + } + auto it = shard_server_idx_cached_.find(shard); + if (it != shard_server_idx_cached_.end()) { + return envelope_send_query_to_server(it->second, std::move(query), std::move(promise)); + } + int server_idx = -1; + int random_idx = -1; + int cnt = 0; + bool selected_full = false; + for (int i = 0; i < (int)servers_.size(); ++i) { + const LiteServer& server = servers_[i]; + if (!server.supports(shard)) { + continue; + } + if (server.is_full && !selected_full) { + selected_full = true; + server_idx = -1; + cnt = 0; + } + if (!server.is_full && selected_full) { + continue; + } + if (!server.client.empty()) { + server_idx = i; + } + if (td::Random::fast(0, cnt) == 0) { + random_idx = i; + } + ++cnt; + } + if (server_idx == -1) { + server_idx = random_idx; + } + if (server_idx == -1) { + running_queries_++; + got_result(td::Status::Error(PSTRING() << "failed to select a suitable server for " << shard.to_str()), + std::move(promise)); return false; } + shard_server_idx_cached_[shard] = server_idx; + if (shard.is_masterchain()) { + mc_server_idx_ = server_idx; + } + return envelope_send_query_to_server(server_idx, std::move(query), std::move(promise)); +} + +bool TestNode::envelope_send_query_to_server(td::int32 server_idx, td::BufferSlice query, + td::Promise promise) { + running_queries_++; + LiteServer& server = servers_.at(server_idx); + if (server.client.empty()) { + start_client(server_idx); + } + CHECK(!server.client.empty()); + auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { td::actor::send_closure(SelfId, &TestNode::got_result, std::move(R), std::move(promise)); }); td::BufferSlice b = ton::serialize_tl_object(ton::create_tl_object(std::move(query)), true); - td::actor::send_closure(client_, &ton::adnl::AdnlExtClient::send_query, "query", std::move(b), - td::Timestamp::in(10.0), std::move(P)); + if (server.client_ready) { + td::actor::send_closure(server.client, &ton::adnl::AdnlExtClient::send_query, "query", std::move(b), + td::Timestamp::in(10.0), std::move(P)); + } else { + server.wait_client_ready.push_back( + [client = server.client.get(), b = std::move(b), P = std::move(P)](td::Result R) mutable { + if (R.is_ok()) { + td::actor::send_closure(client, &ton::adnl::AdnlExtClient::send_query, "query", std::move(b), + td::Timestamp::in(10.0), std::move(P)); + } else { + P.set_error(R.move_as_error_prefix("failed to connect: ")); + } + }); + } return true; } +void TestNode::start_client(int server_idx) { + LiteServer& server = servers_[server_idx]; + CHECK(server.client.empty()); + class Callback : public ton::adnl::AdnlExtClient::Callback { + public: + void on_ready() override { + td::actor::send_closure(id_, &TestNode::conn_ready, server_idx_); + } + void on_stop_ready() override { + td::actor::send_closure(id_, &TestNode::conn_closed, server_idx_); + } + Callback(td::actor::ActorId id, int server_idx) : id_(std::move(id)), server_idx_(server_idx) { + } + + private: + td::actor::ActorId id_; + int server_idx_; + }; + server.client_ready = false; + server.wait_client_ready.clear(); + LOG(INFO) << "Connecting to " << server.addr << " (liteserver #" << server_idx << ")"; + server.client = ton::adnl::AdnlExtClient::create(ton::adnl::AdnlNodeIdFull{server.public_key}, server.addr, + std::make_unique(actor_id(this), server_idx)); +} + +void TestNode::conn_ready(int server_idx) { + LiteServer& server = servers_[server_idx]; + LOG(INFO) << "Connection to " << server.addr << " (liteserver #" << server_idx << ") is ready"; + server.client_ready = true; + for (auto& p : server.wait_client_ready) { + p.set_result(td::Unit()); + } + server.wait_client_ready.clear(); +} + +void TestNode::conn_closed(int server_idx) { + LiteServer& server = servers_[server_idx]; + LOG(INFO) << "Connection to " << server.addr << " (liteserver #" << server_idx << ") closed"; + server.client_ready = false; + server.wait_client_ready.clear(); +} + td::Promise TestNode::trivial_promise() { return td::PromiseCreator::lambda([Self = actor_id(this)](td::Result res) { if (res.is_error()) { @@ -310,7 +433,7 @@ bool TestNode::dump_cached_cell(td::Slice hash_pfx, td::Slice type_name) { bool TestNode::get_server_time() { auto b = ton::serialize_tl_object(ton::create_tl_object(), true); - return envelope_send_query(std::move(b), [&, Self = actor_id(this)](td::Result res) -> void { + return envelope_send_query_to_any(std::move(b), [&, Self = actor_id(this)](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "cannot get server time"; return; @@ -319,9 +442,10 @@ bool TestNode::get_server_time() { if (F.is_error()) { LOG(ERROR) << "cannot parse answer to liteServer.getTime"; } else { - server_time_ = F.move_as_ok()->now_; - server_time_got_at_ = now(); - LOG(INFO) << "server time is " << server_time_ << " (delta " << server_time_ - server_time_got_at_ << ")"; + mc_server_time_ = F.move_as_ok()->now_; + mc_server_time_got_at_ = now(); + LOG(INFO) << "server time is " << mc_server_time_ << " (delta " << mc_server_time_ - mc_server_time_got_at_ + << ")"; } } }); @@ -329,13 +453,13 @@ bool TestNode::get_server_time() { bool TestNode::get_server_version(int mode) { auto b = ton::serialize_tl_object(ton::create_tl_object(), true); - return envelope_send_query(std::move(b), [Self = actor_id(this), mode](td::Result res) { + return envelope_send_query_to_any(std::move(b), [Self = actor_id(this), mode](td::Result res) { td::actor::send_closure_later(Self, &TestNode::got_server_version, std::move(res), mode); }); }; void TestNode::got_server_version(td::Result res, int mode) { - server_ok_ = false; + mc_server_ok_ = false; if (res.is_error()) { LOG(ERROR) << "cannot get server version and time (server too old?)"; } else { @@ -344,11 +468,11 @@ void TestNode::got_server_version(td::Result res, int mode) { LOG(ERROR) << "cannot parse answer to liteServer.getVersion"; } else { auto a = F.move_as_ok(); - set_server_version(a->version_, a->capabilities_); - set_server_time(a->now_); + set_mc_server_version(a->version_, a->capabilities_); + set_mc_server_time(a->now_); } } - if (!server_ok_) { + if (!mc_server_ok_) { LOG(ERROR) << "server version is too old (at least " << (min_ls_version >> 8) << "." << (min_ls_version & 0xff) << " with capabilities " << min_ls_capabilities << " required), some queries are unavailable"; } @@ -357,27 +481,27 @@ void TestNode::got_server_version(td::Result res, int mode) { } } -void TestNode::set_server_version(td::int32 version, td::int64 capabilities) { - if (server_version_ != version || server_capabilities_ != capabilities) { - server_version_ = version; - server_capabilities_ = capabilities; - LOG(WARNING) << "server version is " << (server_version_ >> 8) << "." << (server_version_ & 0xff) - << ", capabilities " << server_capabilities_; +void TestNode::set_mc_server_version(td::int32 version, td::int64 capabilities) { + if (mc_server_version_ != version || mc_server_capabilities_ != capabilities) { + mc_server_version_ = version; + mc_server_capabilities_ = capabilities; + LOG(WARNING) << "server version is " << (mc_server_version_ >> 8) << "." << (mc_server_version_ & 0xff) + << ", capabilities " << mc_server_capabilities_; } - server_ok_ = (server_version_ >= min_ls_version) && !(~server_capabilities_ & min_ls_capabilities); + mc_server_ok_ = (mc_server_version_ >= min_ls_version) && !(~mc_server_capabilities_ & min_ls_capabilities); } -void TestNode::set_server_time(int server_utime) { - server_time_ = server_utime; - server_time_got_at_ = now(); - LOG(INFO) << "server time is " << server_time_ << " (delta " << server_time_ - server_time_got_at_ << ")"; +void TestNode::set_mc_server_time(int server_utime) { + mc_server_time_ = server_utime; + mc_server_time_got_at_ = now(); + LOG(INFO) << "server time is " << mc_server_time_ << " (delta " << mc_server_time_ - mc_server_time_got_at_ << ")"; } bool TestNode::get_server_mc_block_id() { - int mode = (server_capabilities_ & 2) ? 0 : -1; + int mode = (mc_server_capabilities_ & 2) ? 0 : -1; if (mode < 0) { auto b = ton::serialize_tl_object(ton::create_tl_object(), true); - return envelope_send_query(std::move(b), [Self = actor_id(this)](td::Result res) -> void { + return envelope_send_query_to_any(std::move(b), [Self = actor_id(this)](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "cannot get masterchain info from server"; return; @@ -397,24 +521,25 @@ bool TestNode::get_server_mc_block_id() { } else { auto b = ton::serialize_tl_object(ton::create_tl_object(mode), true); - return envelope_send_query(std::move(b), [Self = actor_id(this), mode](td::Result res) -> void { - if (res.is_error()) { - LOG(ERROR) << "cannot get extended masterchain info from server"; - return; - } else { - auto F = ton::fetch_tl_object(res.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getMasterchainInfoExt"; - } else { - auto f = F.move_as_ok(); - auto blk_id = create_block_id(f->last_); - auto zstate_id = create_zero_state_id(f->init_); - LOG(INFO) << "last masterchain block is " << blk_id.to_str(); - td::actor::send_closure_later(Self, &TestNode::got_server_mc_block_id_ext, blk_id, zstate_id, mode, - f->version_, f->capabilities_, f->last_utime_, f->now_); - } - } - }); + return envelope_send_query_to_any( + std::move(b), [Self = actor_id(this), mode](td::Result res) -> void { + if (res.is_error()) { + LOG(ERROR) << "cannot get extended masterchain info from server"; + return; + } else { + auto F = ton::fetch_tl_object(res.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getMasterchainInfoExt"; + } else { + auto f = F.move_as_ok(); + auto blk_id = create_block_id(f->last_); + auto zstate_id = create_zero_state_id(f->init_); + LOG(INFO) << "last masterchain block is " << blk_id.to_str(); + td::actor::send_closure_later(Self, &TestNode::got_server_mc_block_id_ext, blk_id, zstate_id, mode, + f->version_, f->capabilities_, f->last_utime_, f->now_); + } + } + }); } } @@ -448,8 +573,8 @@ void TestNode::got_server_mc_block_id(ton::BlockIdExt blkid, ton::ZeroStateIdExt void TestNode::got_server_mc_block_id_ext(ton::BlockIdExt blkid, ton::ZeroStateIdExt zstateid, int mode, int version, long long capabilities, int last_utime, int server_now) { - set_server_version(version, capabilities); - set_server_time(server_now); + set_mc_server_version(version, capabilities); + set_mc_server_time(server_now); if (last_utime > server_now) { LOG(WARNING) << "server claims to have a masterchain block " << blkid.to_str() << " created at " << last_utime << " (" << last_utime - server_now << " seconds in the future)"; @@ -457,10 +582,10 @@ void TestNode::got_server_mc_block_id_ext(ton::BlockIdExt blkid, ton::ZeroStateI LOG(WARNING) << "server appears to be out of sync: its newest masterchain block is " << blkid.to_str() << " created at " << last_utime << " (" << server_now - last_utime << " seconds ago according to the server's clock)"; - } else if (last_utime < server_time_got_at_ - 60) { + } else if (last_utime < mc_server_time_got_at_ - 60) { LOG(WARNING) << "either the server is out of sync, or the local clock is set incorrectly: the newest masterchain " "block known to server is " - << blkid.to_str() << " created at " << last_utime << " (" << server_now - server_time_got_at_ + << blkid.to_str() << " created at " << last_utime << " (" << server_now - mc_server_time_got_at_ << " seconds ago according to the local clock)"; } got_server_mc_block_id(blkid, zstateid, last_utime); @@ -469,52 +594,54 @@ void TestNode::got_server_mc_block_id_ext(ton::BlockIdExt blkid, ton::ZeroStateI bool TestNode::request_block(ton::BlockIdExt blkid) { auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); - return envelope_send_query(std::move(b), [Self = actor_id(this), blkid](td::Result res) -> void { - if (res.is_error()) { - LOG(ERROR) << "cannot obtain block " << blkid.to_str() << " from server"; - return; - } else { - auto F = ton::fetch_tl_object(res.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getBlock"; - } else { - auto f = F.move_as_ok(); - auto blk_id = ton::create_block_id(f->id_); - LOG(INFO) << "obtained block " << blk_id.to_str() << " from server"; - if (blk_id != blkid) { - LOG(ERROR) << "block id mismatch: expected data for block " << blkid.to_str() << ", obtained for " - << blk_id.to_str(); + return envelope_send_query_to_shard( + blkid.shard_full(), std::move(b), [Self = actor_id(this), blkid](td::Result res) -> void { + if (res.is_error()) { + LOG(ERROR) << "cannot obtain block " << blkid.to_str() << " from server"; + return; + } else { + auto F = ton::fetch_tl_object(res.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getBlock"; + } else { + auto f = F.move_as_ok(); + auto blk_id = ton::create_block_id(f->id_); + LOG(INFO) << "obtained block " << blk_id.to_str() << " from server"; + if (blk_id != blkid) { + LOG(ERROR) << "block id mismatch: expected data for block " << blkid.to_str() << ", obtained for " + << blk_id.to_str(); + } + td::actor::send_closure_later(Self, &TestNode::got_mc_block, blk_id, std::move(f->data_)); + } } - td::actor::send_closure_later(Self, &TestNode::got_mc_block, blk_id, std::move(f->data_)); - } - } - }); + }); } bool TestNode::request_state(ton::BlockIdExt blkid) { auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); - return envelope_send_query(std::move(b), [Self = actor_id(this), blkid](td::Result res) -> void { - if (res.is_error()) { - LOG(ERROR) << "cannot obtain state " << blkid.to_str() << " from server"; - return; - } else { - auto F = ton::fetch_tl_object(res.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getState"; - } else { - auto f = F.move_as_ok(); - auto blk_id = ton::create_block_id(f->id_); - LOG(INFO) << "obtained state " << blk_id.to_str() << " from server"; - if (blk_id != blkid) { - LOG(ERROR) << "block id mismatch: expected state for block " << blkid.to_str() << ", obtained for " - << blk_id.to_str(); + return envelope_send_query_to_shard( + blkid.shard_full(), std::move(b), [Self = actor_id(this), blkid](td::Result res) -> void { + if (res.is_error()) { + LOG(ERROR) << "cannot obtain state " << blkid.to_str() << " from server"; + return; + } else { + auto F = ton::fetch_tl_object(res.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getState"; + } else { + auto f = F.move_as_ok(); + auto blk_id = ton::create_block_id(f->id_); + LOG(INFO) << "obtained state " << blk_id.to_str() << " from server"; + if (blk_id != blkid) { + LOG(ERROR) << "block id mismatch: expected state for block " << blkid.to_str() << ", obtained for " + << blk_id.to_str(); + } + td::actor::send_closure_later(Self, &TestNode::got_mc_state, blk_id, f->root_hash_, f->file_hash_, + std::move(f->data_)); + } } - td::actor::send_closure_later(Self, &TestNode::got_mc_state, blk_id, f->root_hash_, f->file_hash_, - std::move(f->data_)); - } - } - }); + }); } void TestNode::got_mc_block(ton::BlockIdExt blkid, td::BufferSlice data) { @@ -1150,27 +1277,34 @@ td::Status TestNode::send_ext_msg_from_filename(std::string filename) { LOG(ERROR) << "failed to read file `" << filename << "`: " << err.to_string(); return err; } - if (ready_ && !client_.empty()) { - LOG(ERROR) << "sending query from file " << filename; - auto P = td::PromiseCreator::lambda([](td::Result R) { - if (R.is_error()) { - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.sendMessage"; - } else { - int status = F.move_as_ok()->status_; - LOG(INFO) << "external message status is " << status; - } - }); - auto b = - ton::serialize_tl_object(ton::create_tl_object(F.move_as_ok()), true); - return envelope_send_query(std::move(b), std::move(P)) ? td::Status::OK() - : td::Status::Error("cannot send query to server"); - } else { - return td::Status::Error("server connection not ready"); + LOG(ERROR) << "sending query from file " << filename; + + TRY_RESULT_PREFIX(root, vm::std_boc_deserialize(F.ok().as_slice()), "invalid boc: "); + block::gen::CommonMsgInfo::Record_ext_in_msg_info info; + if (!tlb::unpack_cell_inexact(root, info)) { + return td::Status::Error("failed to unpack external message header"); + } + auto dest_prefix = block::tlb::MsgAddressInt::get_prefix(info.dest); + if (!dest_prefix.is_valid()) { + return td::Status::Error("destination of the message is invalid"); } + + auto P = td::PromiseCreator::lambda([](td::Result R) { + if (R.is_error()) { + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.sendMessage"; + } else { + int status = F.move_as_ok()->status_; + LOG(INFO) << "external message status is " << status; + } + }); + auto b = ton::serialize_tl_object(ton::create_tl_object(F.move_as_ok()), true); + return envelope_send_query_to_account(dest_prefix, std::move(b), std::move(P)) + ? td::Status::OK() + : td::Status::Error("cannot send query to server"); } bool TestNode::get_account_state(ton::WorkchainId workchain, ton::StdSmcAddress addr, ton::BlockIdExt ref_blkid, @@ -1178,9 +1312,6 @@ bool TestNode::get_account_state(ton::WorkchainId workchain, ton::StdSmcAddress if (!ref_blkid.is_valid()) { return set_error("must obtain last block information before making other queries"); } - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } if (addr_ext) { return get_special_smc_addr( addr_ext, [this, ref_blkid, filename, mode, prunned](td::Result res) { @@ -1202,24 +1333,26 @@ bool TestNode::get_account_state(ton::WorkchainId workchain, ton::StdSmcAddress ton::create_tl_lite_block_id(ref_blkid), std::move(a)), true); } + ton::AccountIdPrefixFull account_prefix(workchain, addr.bits().get_uint(64)); LOG(INFO) << "requesting " << (prunned ? "prunned " : "") << "account state for " << workchain << ":" << addr.to_hex() << " with respect to " << ref_blkid.to_str() << " with savefile `" << filename << "` and mode " << mode; - return envelope_send_query(std::move(b), [Self = actor_id(this), workchain, addr, ref_blkid, filename, mode, - prunned](td::Result R) { - if (R.is_error()) { - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getAccountState"; - } else { - auto f = F.move_as_ok(); - td::actor::send_closure_later(Self, &TestNode::got_account_state, ref_blkid, ton::create_block_id(f->id_), - ton::create_block_id(f->shardblk_), std::move(f->shard_proof_), - std::move(f->proof_), std::move(f->state_), workchain, addr, filename, mode, - prunned); - } - }); + return envelope_send_query_to_account( + account_prefix, std::move(b), + [Self = actor_id(this), workchain, addr, ref_blkid, filename, mode, prunned](td::Result R) { + if (R.is_error()) { + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getAccountState"; + } else { + auto f = F.move_as_ok(); + td::actor::send_closure_later(Self, &TestNode::got_account_state, ref_blkid, ton::create_block_id(f->id_), + ton::create_block_id(f->shardblk_), std::move(f->shard_proof_), + std::move(f->proof_), std::move(f->state_), workchain, addr, filename, mode, + prunned); + } + }); } td::int64 TestNode::compute_method_id(std::string method) { @@ -1285,19 +1418,18 @@ bool TestNode::start_run_method(ton::WorkchainId workchain, ton::StdSmcAddress a if (!ref_blkid.is_valid()) { return set_error("must obtain last block information before making other queries"); } - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } auto a = ton::create_tl_object(workchain, addr); + ton::AccountIdPrefixFull account_prefix(workchain, addr.bits().get_uint(64)); if (!mode) { auto b = ton::serialize_tl_object(ton::create_tl_object( ton::create_tl_lite_block_id(ref_blkid), std::move(a)), true); LOG(INFO) << "requesting account state for " << workchain << ":" << addr.to_hex() << " with respect to " << ref_blkid.to_str() << " to run method " << method_name << " with " << params.size() << " parameters"; - return envelope_send_query( - std::move(b), [Self = actor_id(this), workchain, addr, ref_blkid, method_name, params = std::move(params), - promise = std::move(promise)](td::Result R) mutable { + return envelope_send_query_to_account( + account_prefix, std::move(b), + [Self = actor_id(this), workchain, addr, ref_blkid, method_name, params = std::move(params), + promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); return; @@ -1337,26 +1469,27 @@ bool TestNode::start_run_method(ton::WorkchainId workchain, ton::StdSmcAddress a LOG(INFO) << "requesting remote get-method execution for " << workchain << ":" << addr.to_hex() << " with respect to " << ref_blkid.to_str() << " to run method " << method_name << " with " << params.size() << " parameters"; - return envelope_send_query(std::move(b), [Self = actor_id(this), workchain, addr, ref_blkid, method_name, mode, - params = std::move(params), - promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.runSmcMethod"; - promise.set_error(td::Status::Error("cannot parse answer to liteServer.runSmcMethod")); - } else { - auto f = F.move_as_ok(); - td::actor::send_closure_later(Self, &TestNode::run_smc_method, mode, ref_blkid, ton::create_block_id(f->id_), - ton::create_block_id(f->shardblk_), std::move(f->shard_proof_), - std::move(f->proof_), std::move(f->state_proof_), workchain, addr, method_name, - std::move(params), std::move(f->init_c7_), std::move(f->lib_extras_), - std::move(f->result_), f->exit_code_, std::move(promise)); - } - }); + return envelope_send_query_to_account( + account_prefix, std::move(b), + [Self = actor_id(this), workchain, addr, ref_blkid, method_name, mode, params = std::move(params), + promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.runSmcMethod"; + promise.set_error(td::Status::Error("cannot parse answer to liteServer.runSmcMethod")); + } else { + auto f = F.move_as_ok(); + td::actor::send_closure_later( + Self, &TestNode::run_smc_method, mode, ref_blkid, ton::create_block_id(f->id_), + ton::create_block_id(f->shardblk_), std::move(f->shard_proof_), std::move(f->proof_), + std::move(f->state_proof_), workchain, addr, method_name, std::move(params), std::move(f->init_c7_), + std::move(f->lib_extras_), std::move(f->result_), f->exit_code_, std::move(promise)); + } + }); } } @@ -1586,8 +1719,8 @@ void TestNode::send_compute_complaint_price_query(ton::StdSmcAddress elector_add params.emplace_back(td::make_refint(bits)); params.emplace_back(td::make_refint(refs)); params.emplace_back(td::make_refint(expires_in)); - auto P = td::PromiseCreator::lambda( - [this, expires_in, bits, refs, chash, filename](td::Result> R) { + auto P = + td::PromiseCreator::lambda([expires_in, bits, refs, chash, filename](td::Result> R) { if (R.is_error()) { LOG(ERROR) << R.move_as_error(); return; @@ -1649,10 +1782,6 @@ bool TestNode::dns_resolve_start(ton::WorkchainId workchain, ton::StdSmcAddress return set_error("domain name too long"); } - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } - if (workchain == ton::workchainInvalid) { if (dns_root_queried_) { workchain = ton::masterchainId; @@ -1861,17 +1990,16 @@ bool TestNode::get_one_transaction(ton::BlockIdExt blkid, ton::WorkchainId workc if (!ton::shard_contains(blkid.shard_full(), ton::extract_addr_prefix(workchain, addr))) { return set_error("the shard of this block cannot contain this account"); } - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } auto a = ton::create_tl_object(workchain, addr); auto b = ton::serialize_tl_object(ton::create_tl_object( ton::create_tl_lite_block_id(blkid), std::move(a), lt), true); + ton::AccountIdPrefixFull account_prefix(workchain, addr.bits().get_uint(64)); LOG(INFO) << "requesting transaction " << lt << " of " << workchain << ":" << addr.to_hex() << " from block " << blkid.to_str(); - return envelope_send_query( - std::move(b), [Self = actor_id(this), workchain, addr, lt, blkid, dump](td::Result R) -> void { + return envelope_send_query_to_account( + account_prefix, std::move(b), + [Self = actor_id(this), workchain, addr, lt, blkid, dump](td::Result R) -> void { if (R.is_error()) { return; } @@ -1888,16 +2016,15 @@ bool TestNode::get_one_transaction(ton::BlockIdExt blkid, ton::WorkchainId workc bool TestNode::get_last_transactions(ton::WorkchainId workchain, ton::StdSmcAddress addr, ton::LogicalTime lt, ton::Bits256 hash, unsigned count, bool dump) { - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } auto a = ton::create_tl_object(workchain, addr); auto b = ton::serialize_tl_object( ton::create_tl_object(count, std::move(a), lt, hash), true); + ton::AccountIdPrefixFull account_prefix(workchain, addr.bits().get_uint(64)); LOG(INFO) << "requesting " << count << " last transactions from " << lt << ":" << hash.to_hex() << " of " << workchain << ":" << addr.to_hex(); - return envelope_send_query( - std::move(b), [Self = actor_id(this), workchain, addr, lt, hash, count, dump](td::Result R) { + return envelope_send_query_to_account( + account_prefix, std::move(b), + [Self = actor_id(this), workchain, addr, lt, hash, count, dump](td::Result R) { if (R.is_error()) { return; } @@ -2257,10 +2384,10 @@ void TestNode::got_one_transaction(ton::BlockIdExt req_blkid, ton::BlockIdExt bl << " but received data has " << root->get_hash().bits().to_hex(256); return; } - } catch (vm::VmError err) { + } catch (vm::VmError& err) { LOG(ERROR) << "error while traversing block transaction proof : " << err.get_msg(); return; - } catch (vm::VmVirtError err) { + } catch (vm::VmVirtError& err) { LOG(ERROR) << "virtualization error while traversing block transaction proof : " << err.get_msg(); return; } @@ -2439,32 +2566,30 @@ void TestNode::got_last_transactions(std::vector blkids, td::Bu bool TestNode::get_block_transactions(ton::BlockIdExt blkid, int mode, unsigned count, ton::Bits256 acc_addr, ton::LogicalTime lt) { - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } auto a = ton::create_tl_object(acc_addr, lt); auto b = ton::serialize_tl_object(ton::create_tl_object( ton::create_tl_lite_block_id(blkid), mode, count, std::move(a), false, false), true); LOG(INFO) << "requesting " << count << " transactions from block " << blkid.to_str() << " starting from account " << acc_addr.to_hex() << " lt " << lt; - return envelope_send_query(std::move(b), [Self = actor_id(this), mode](td::Result R) { - if (R.is_error()) { - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.listBlockTransactions"; - } else { - auto f = F.move_as_ok(); - std::vector transactions; - for (auto& id : f->ids_) { - transactions.emplace_back(id->account_, id->lt_, id->hash_); - } - td::actor::send_closure_later(Self, &TestNode::got_block_transactions, ton::create_block_id(f->id_), mode, - f->req_count_, f->incomplete_, std::move(transactions), std::move(f->proof_)); - } - }); + return envelope_send_query_to_shard( + blkid.shard_full(), std::move(b), [Self = actor_id(this), mode](td::Result R) { + if (R.is_error()) { + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.listBlockTransactions"; + } else { + auto f = F.move_as_ok(); + std::vector transactions; + for (auto& id : f->ids_) { + transactions.emplace_back(id->account_, id->lt_, id->hash_); + } + td::actor::send_closure_later(Self, &TestNode::got_block_transactions, ton::create_block_id(f->id_), mode, + f->req_count_, f->incomplete_, std::move(transactions), std::move(f->proof_)); + } + }); } void TestNode::got_block_transactions(ton::BlockIdExt blkid, int mode, unsigned req_count, bool incomplete, @@ -2490,25 +2615,23 @@ bool TestNode::get_all_shards(std::string filename, bool use_last, ton::BlockIdE if (!blkid.is_masterchain()) { return set_error("only masterchain blocks contain shard configuration"); } - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); LOG(INFO) << "requesting recent shard configuration"; - return envelope_send_query(std::move(b), [Self = actor_id(this), filename](td::Result R) -> void { - if (R.is_error()) { - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getAllShardsInfo"; - } else { - auto f = F.move_as_ok(); - td::actor::send_closure_later(Self, &TestNode::got_all_shards, ton::create_block_id(f->id_), std::move(f->proof_), - std::move(f->data_), filename); - } - }); + return envelope_send_query_to_any( + std::move(b), [Self = actor_id(this), filename](td::Result R) -> void { + if (R.is_error()) { + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getAllShardsInfo"; + } else { + auto f = F.move_as_ok(); + td::actor::send_closure_later(Self, &TestNode::got_all_shards, ton::create_block_id(f->id_), + std::move(f->proof_), std::move(f->data_), filename); + } + }); } void TestNode::got_all_shards(ton::BlockIdExt blk, td::BufferSlice proof, td::BufferSlice data, std::string filename) { @@ -2572,9 +2695,6 @@ bool TestNode::parse_get_config_params(ton::BlockIdExt blkid, int mode, std::str params.push_back(x); } } - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } if (!blkid.is_masterchain_ext()) { return set_error("only masterchain blocks contain configuration"); } @@ -2593,10 +2713,6 @@ bool TestNode::get_config_params(ton::BlockIdExt blkid, td::Promise promise, int mode, std::string filename, std::vector params) { - if (!(ready_ && !client_.empty())) { - promise.set_error(td::Status::Error("server connection not ready")); - return false; - } if (!blkid.is_masterchain_ext()) { promise.set_error(td::Status::Error("masterchain reference block expected")); return false; @@ -2614,11 +2730,12 @@ bool TestNode::get_config_params_ext(ton::BlockIdExt blkid, td::Promise R) mutable { - td::actor::send_closure_later(Self, &TestNode::got_config_params, blkid, mode, filename, std::move(params), - std::move(R), std::move(promise)); - }); + return envelope_send_query_to_any( + std::move(b), [Self = actor_id(this), mode, filename, blkid, params = std::move(params), + promise = std::move(promise)](td::Result R) mutable { + td::actor::send_closure_later(Self, &TestNode::got_config_params, blkid, mode, filename, std::move(params), + std::move(R), std::move(promise)); + }); } void TestNode::got_config_params(ton::BlockIdExt req_blkid, int mode, std::string filename, std::vector params, @@ -2806,8 +2923,8 @@ bool TestNode::get_block(ton::BlockIdExt blkid, bool dump) { LOG(INFO) << "got block download request for " << blkid.to_str(); auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); - return envelope_send_query( - std::move(b), [Self = actor_id(this), blkid, dump](td::Result res) -> void { + return envelope_send_query_to_shard( + blkid.shard_full(), std::move(b), [Self = actor_id(this), blkid, dump](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "cannot obtain block " << blkid.to_str() << " from server : " << res.move_as_error().to_string(); @@ -2835,8 +2952,8 @@ bool TestNode::get_state(ton::BlockIdExt blkid, bool dump) { LOG(INFO) << "got state download request for " << blkid.to_str(); auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); - return envelope_send_query( - std::move(b), [Self = actor_id(this), blkid, dump](td::Result res) -> void { + return envelope_send_query_to_shard( + blkid.shard_full(), std::move(b), [Self = actor_id(this), blkid, dump](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "cannot obtain state " << blkid.to_str() << " from server : " << res.move_as_error().to_string(); @@ -2973,7 +3090,7 @@ void TestNode::got_state(ton::BlockIdExt blkid, ton::RootHash root_hash, ton::Fi } bool TestNode::get_show_block_header(ton::BlockIdExt blkid, int mode) { - return get_block_header(blkid, mode, [this, blkid](td::Result R) { + return get_block_header(blkid, mode, [this](td::Result R) { if (R.is_error()) { LOG(ERROR) << "unable to fetch block header: " << R.move_as_error(); } else { @@ -2988,8 +3105,9 @@ bool TestNode::get_block_header(ton::BlockIdExt blkid, int mode, td::Promise(ton::create_tl_lite_block_id(blkid), mode), true); - return envelope_send_query( - std::move(b), [this, blkid, promise = std::move(promise)](td::Result R) mutable -> void { + return envelope_send_query_to_shard( + blkid.shard_full(), std::move(b), + [this, blkid, promise = std::move(promise)](td::Result R) mutable -> void { TRY_RESULT_PROMISE_PREFIX(promise, res, std::move(R), PSLICE() << "cannot obtain block header for " << blkid.to_str() << " from server :"); got_block_header_raw(std::move(res), std::move(promise), blkid); @@ -3015,8 +3133,9 @@ bool TestNode::lookup_block(ton::ShardIdFull shard, int mode, td::uint64 arg, auto b = ton::serialize_tl_object(ton::create_tl_object( mode, ton::create_tl_lite_block_id_simple(id), arg, (td::uint32)arg), true); - return envelope_send_query( - std::move(b), [this, id, mode, arg, promise = std::move(promise)](td::Result R) mutable -> void { + return envelope_send_query_to_shard( + shard, std::move(b), + [this, id, mode, arg, promise = std::move(promise)](td::Result R) mutable -> void { TRY_RESULT_PROMISE_PREFIX(promise, res, std::move(R), PSLICE() << "cannot look up block header for " << id.to_str() << " with mode " << mode << " and argument " << arg << " from server :"); @@ -3130,9 +3249,9 @@ void TestNode::got_block_header(ton::BlockIdExt blkid, td::BufferSlice data, int return; } show_block_header(blkid, std::move(virt_root), mode); - } catch (vm::VmError err) { + } catch (vm::VmError& err) { LOG(ERROR) << "error processing header for " << blkid.to_str() << " : " << err.get_msg(); - } catch (vm::VmVirtError err) { + } catch (vm::VmVirtError& err) { LOG(ERROR) << "error processing header for " << blkid.to_str() << " : " << err.get_msg(); } show_new_blkids(); @@ -3161,14 +3280,15 @@ bool TestNode::get_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mod ton::serialize_tl_object(ton::create_tl_object( mode & 0xfff, ton::create_tl_lite_block_id(from), ton::create_tl_lite_block_id(to)), true); - return envelope_send_query(std::move(b), [Self = actor_id(this), from, to, mode](td::Result res) { - if (res.is_error()) { - LOG(ERROR) << "cannot obtain block proof for " << ((mode & 1) ? to.to_str() : "last masterchain block") - << " starting from " << from.to_str() << " from server : " << res.move_as_error().to_string(); - } else { - td::actor::send_closure_later(Self, &TestNode::got_block_proof, from, to, mode, res.move_as_ok()); - } - }); + return envelope_send_query_to_any( + std::move(b), [Self = actor_id(this), from, to, mode](td::Result res) { + if (res.is_error()) { + LOG(ERROR) << "cannot obtain block proof for " << ((mode & 1) ? to.to_str() : "last masterchain block") + << " starting from " << from.to_str() << " from server : " << res.move_as_error().to_string(); + } else { + td::actor::send_closure_later(Self, &TestNode::got_block_proof, from, to, mode, res.move_as_ok()); + } + }); } void TestNode::got_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mode, td::BufferSlice pchain) { @@ -3222,9 +3342,6 @@ void TestNode::got_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mod bool TestNode::get_creator_stats(ton::BlockIdExt blkid, int mode, unsigned req_count, ton::Bits256 start_after, ton::UnixTime min_utime) { - if (!(ready_ && !client_.empty())) { - return set_error("server connection not ready"); - } if (!blkid.is_masterchain_ext()) { return set_error("only masterchain blocks contain block creator statistics"); } @@ -3235,8 +3352,8 @@ bool TestNode::get_creator_stats(ton::BlockIdExt blkid, int mode, unsigned req_c auto& os = *osp; return get_creator_stats( blkid, mode, req_count, start_after, min_utime, - [min_utime, &os](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, - const block::DiscountedCounter& shard_cnt) -> bool { + [&os](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, + const block::DiscountedCounter& shard_cnt) -> bool { os << key.to_hex() << " mc_cnt:" << mc_cnt << " shard_cnt:" << shard_cnt << std::endl; return true; }, @@ -3265,10 +3382,6 @@ bool TestNode::get_creator_stats(ton::BlockIdExt blkid, int mode, unsigned req_c bool TestNode::get_creator_stats(ton::BlockIdExt blkid, unsigned req_count, ton::UnixTime min_utime, TestNode::creator_stats_func_t func, std::unique_ptr state, td::Promise> promise) { - if (!(ready_ && !client_.empty())) { - promise.set_error(td::Status::Error("server connection not ready")); - return false; - } if (!state) { promise.set_error(td::Status::Error("null CreatorStatsRes")); return false; @@ -3287,7 +3400,7 @@ bool TestNode::get_creator_stats(ton::BlockIdExt blkid, unsigned req_count, ton: LOG(INFO) << "requesting up to " << req_count << " block creator stats records with respect to masterchain block " << blkid.to_str() << " starting from validator public key " << state->last_key.to_hex() << " created after " << min_utime << " (mode=" << state->mode << ")"; - return envelope_send_query( + return envelope_send_query_to_any( std::move(b), [this, blkid, req_count, state = std::move(state), min_utime, func = std::move(func), promise = std::move(promise)](td::Result R) mutable { TRY_RESULT_PROMISE(promise, res, std::move(R)); @@ -3504,8 +3617,8 @@ bool TestNode::load_creator_stats(std::unique_ptr l ton::UnixTime min_utime = info.valid_since - 1000; return get_creator_stats( info.blk_id, 1000, min_utime, - [min_utime, &info](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, - const block::DiscountedCounter& shard_cnt) -> bool { + [&info](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, + const block::DiscountedCounter& shard_cnt) -> bool { info.store_record(key, mc_cnt, shard_cnt); return true; }, @@ -3708,7 +3821,7 @@ bool compute_punishment_default(int interval, bool severe, td::RefInt256& fine, fine = td::make_refint(101 * 1000000000LL); // 101 fine_part = 0; - return true; // todo: (tolya-yanot) temporary reduction of fine + return true; // todo: (tolya-yanot) temporary reduction of fine if (severe) { fine = td::make_refint(2500 * 1000000000LL); // GR$2500 @@ -3730,41 +3843,49 @@ bool compute_punishment_default(int interval, bool severe, td::RefInt256& fine, return true; } -bool compute_punishment(int interval, bool severe, td::RefInt256& fine, unsigned& fine_part, Ref punishment_params) { - if(punishment_params.is_null()) { +bool compute_punishment(int interval, bool severe, td::RefInt256& fine, unsigned& fine_part, + Ref punishment_params) { + if (punishment_params.is_null()) { return compute_punishment_default(interval, severe, fine, fine_part); } block::gen::MisbehaviourPunishmentConfig::Record rec; if (!tlb::unpack_cell(punishment_params, rec)) { - return false; + return false; } - if(interval <= rec.unpunishable_interval) { - return false; + if (interval <= rec.unpunishable_interval) { + return false; } fine = block::tlb::t_Grams.as_integer(rec.default_flat_fine); fine_part = rec.default_proportional_fine; if (severe) { - fine = fine * rec.severity_flat_mult; fine >>= 8; - fine_part = fine_part * rec.severity_proportional_mult; fine_part >>= 8; + fine = fine * rec.severity_flat_mult; + fine >>= 8; + fine_part = fine_part * rec.severity_proportional_mult; + fine_part >>= 8; } if (interval >= rec.long_interval) { - fine = fine * rec.long_flat_mult; fine >>= 8; - fine_part = fine_part * rec.long_proportional_mult; fine_part >>= 8; + fine = fine * rec.long_flat_mult; + fine >>= 8; + fine_part = fine_part * rec.long_proportional_mult; + fine_part >>= 8; return true; } if (interval >= rec.medium_interval) { - fine = fine * rec.medium_flat_mult; fine >>= 8; - fine_part = fine_part * rec.medium_proportional_mult; fine_part >>= 8; + fine = fine * rec.medium_flat_mult; + fine >>= 8; + fine_part = fine_part * rec.medium_proportional_mult; + fine_part >>= 8; return true; } return true; } -bool check_punishment(int interval, bool severe, td::RefInt256 fine, unsigned fine_part, Ref punishment_params) { +bool check_punishment(int interval, bool severe, td::RefInt256 fine, unsigned fine_part, + Ref punishment_params) { td::RefInt256 computed_fine; unsigned computed_fine_part; return compute_punishment(interval, severe, computed_fine, computed_fine_part, punishment_params) && @@ -3800,7 +3921,7 @@ td::Status TestNode::write_val_create_proof(TestNode::ValidatorLoadInfo& info1, int severity = (severe ? 2 : 1); td::RefInt256 fine = td::make_refint(101000000000); - unsigned fine_part = 0; // todo: (tolya-yanot) temporary reduction of fine // 0xffffffff / 16; // 1/16 + unsigned fine_part = 0; // todo: (tolya-yanot) temporary reduction of fine // 0xffffffff / 16; // 1/16 if (!compute_punishment(interval, severe, fine, fine_part, punishment_params)) { return td::Status::Error("cannot compute adequate punishment"); } @@ -3899,7 +4020,7 @@ td::Result> TestNode::ValidatorLoadInfo::build_proof(int idx, td:: block::gen::ValidatorDescr::Record_validator_addr rec2; if (tlb::csr_unpack(entry, rec1)) { pk = std::move(rec1.public_key); - } else if (tlb::csr_unpack(std::move(entry), rec2)) { + } else if (tlb::csr_unpack(entry, rec2)) { pk = std::move(rec2.public_key); } else { return td::Status::Error("cannot unpack ValidatorDescr"); @@ -4115,7 +4236,8 @@ td::Status TestNode::continue_check_validator_load_proof(std::unique_ptrconfig->get_config_param(40))) { + if (!check_punishment(interval, severe, suggested_fine, rec.suggested_fine_part, + info2->config->get_config_param(40))) { LOG(ERROR) << "proposed punishment (fine " << td::dec_string(suggested_fine) << ", fine_part=" << (double)rec.suggested_fine_part / (1LL << 32) << " is too harsh"; show_vote(root->get_hash().bits(), false); @@ -4264,8 +4386,9 @@ int main(int argc, char* argv[]) { return (verbosity >= 0 && verbosity <= 9) ? td::Status::OK() : td::Status::Error("verbosity must be 0..9"); }); p.add_option('V', "version", "shows lite-client build information", [&]() { - std::cout << "lite-client build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; - + std::cout << "lite-client build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::exit(0); }); p.add_option('i', "idx", "set liteserver idx", [&](td::Slice arg) { diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index 34d02011d..018a9298a 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -57,8 +57,7 @@ class TestNode : public td::actor::Actor { bool client_ready = false; std::vector> wait_client_ready; - int max_common_prefix(ton::ShardIdFull shard) const; - bool supports_shard(ton::ShardIdFull shard) const; + bool supports(ton::ShardIdFull shard) const; }; std::vector servers_; @@ -352,9 +351,6 @@ class TestNode : public td::actor::Actor { bool parse_shard_id(ton::ShardIdFull& shard); bool parse_block_id_ext(ton::BlockIdExt& blkid, bool allow_incomplete = false); bool parse_block_id_ext(std::string blk_id_string, ton::BlockIdExt& blkid, bool allow_incomplete = false) const; - bool parse_stack_value(td::Slice str, vm::StackEntry& value); - bool parse_stack_value(vm::StackEntry& value); - bool parse_stack_values(std::vector& values); bool register_blkid(const ton::BlockIdExt& blkid); bool show_new_blkids(bool all = false); bool complete_blkid(ton::BlockId partial_blkid, ton::BlockIdExt& complete_blkid) const; @@ -458,8 +454,7 @@ class TestNode : public td::actor::Actor { void parse_line(td::BufferSlice data); - TestNode() { - } + TestNode() = default; void run(); }; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 6a2e18f1640f473c854f3eb2dabdf8f076537438..97c2bda2df4125bec8281603581b892696097361 100644 GIT binary patch delta 3518 zcma)8Z){Ul6z^SMXUo>D!;ZCUYl|zcV_P>6lqkc=fNe6QIzp68y0^Z|R_)%audf>= z7$F(~K^hgPp{{m z^E-d;x#xFo?lGJ^W+=KeSoeS~{nMl2qIvq#V>idYC}cT4;$MejF}54iW7GbJ%jAH5 zj43}^&;Q^F36TgZ3F4yyhyBf%$093XGAI8Y*<$>}P#Q=SO%-xo{+L9?{Rr_{YXPR{@ zzeS)qSD!t4^uj&=s~rP%a#3%WkJuuJIrjXXYs6aKIj^Gep}LYtn7YSe9~S*k){ToP zt}K<0H3e-j&JyTa6Am?pf_=a5fTZ9uNBg_{rBRF(B{XlR>-Wzw7!-8R$o{W3P`iuJNY+oLpUF0_piL z=eFs|RD+w;?=leAn&#ATkE%DYrao=3#%ogzpxMQy{2F7!OD8~fpxK)NL!@N5I%U$K z5$BPqk!mv1oOVBP+tHF1?^3`5Z1|nMYfi_4A>Lc2JR`Nr&&IKsSS^ZzsK$aQaOHs& z#dc{zP$w!u_M4q*ia~Ewsd!}aowAa|3evL{iw#P#u zdc`^&yjrLhO1cnov|V`mJiKb=s@w)| zA3On+3Jo&y-PW4ihqi<-_u&(X|FArisLySA`M+Sj+k73lqSY2+_g91aYcm7o8JzG( za}F-0hiSb5H#zYjTK+tIY00Wn+Mr+_|eKUttPz`W2fAC<6@5$rEL$n%G_) zH=5F*WXCEGqnJ#*YDIHPl*D5}NeK1|e3W(du@3AmXq}mS#F@#5 zrV0Am5}TCb2E!cLxyo(p9zHXgod{US1B3g}+L77%NNGKF znL6UWVJ96oJf@61IgF6IxQNGFrC?MPxOj-ZDDu#giy8q+d*3s2VRje_7wSNlSo@dQ z03*~H5!#H-VRT&WIuxV3@=jw~bNEr$Hmr32KA9R)`6 z!6UxnWkA^F*@P|PUMdP;Eb8DK;Gt@2PlJpM$)7O5;|KQol4EY-AJXg&8yD+TYkk_-8-`o~%_5GW=jawP{7|#3zxUnCayqco&{F~%9VkGP4D)2?w?8L6 zW>jivhk<=d8<9nPSxPftL1P9($JZ~GP;mRKz<)1AyYmd6_AOj*S4?uFCutm>(XBQV zVGM&WIIdPDStqjAWpC4|nV<{>$VGcCFbhU$k*%H%wdJ(O*)7Yq z#^xPTIgBhn4)k(N){sfqyhCOS)8rNM2BIZ*=7sPS<)joKS;Y)8hebhvNAf_z{;wh> z`FVc%DXDtSi3N#CnK_vynW@E-7jjB(_RtAooP0(1#O4!vA}ow0o9`Nxi?V>+KKa8W zjmZrjDiF>J4;@aB=*?q7KFpH^9p$G#P-JAC{#Ke%8KKC>Q-v9*X!3eTL#QrK6$zLw zQ0Npnv*~cywjBhU|HKn&HjkG9M7e{P3fvN=$%+TGC(rPLsyTomCE*Q~S^{I7@ty)U zXLE|r0Y-?48h$nqp$xwVLYp0u_DD&90v+TFWcPr?CX3YS;FgT2)xa$|1E0=2wFbC# zs?^~zHU*F5=9oqf-pzOVV%We*WU|dv2~JR20r`x3a_rRKnHK!wE`EAU#ZzC$6=h%(G4dB4x2|&gMDmW%wum*|&xTl)NSv zOwyP>!GKW(!uzvd2EvdzFbBfea{wxG=RgQpWb=fB2edgsi4S7_!?)g(N;z0S0-Iyr zJz(A}@wtO(v&^?MGK|&RUkWk$Gi|pKW0YXoen*P&0LwN7#s#d~HPje0n6`h>V9a6J ezD1XjLqP&&9&+q~#I{TLF-~CI9uvTr!3F^N)?rWp diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index 075c5a3abdf705679b48dd79a799a1f70a9d67a9..1f4dab07974697338e4e299f46e17cda26580488 100644 GIT binary patch delta 101 zcmZ4SiSf-R#tjSDc<1Iw@Fk`cgwt*i2F8LvG diff --git a/tonlib/CMakeLists.txt b/tonlib/CMakeLists.txt index d9984368b..f27e00f21 100644 --- a/tonlib/CMakeLists.txt +++ b/tonlib/CMakeLists.txt @@ -8,7 +8,6 @@ set(TONLIB_SOURCE tonlib/Client.cpp tonlib/Config.cpp tonlib/ExtClient.cpp - tonlib/ExtClientMulti.cpp tonlib/ExtClientLazy.cpp tonlib/ExtClientOutbound.cpp tonlib/KeyStorage.cpp @@ -24,8 +23,7 @@ set(TONLIB_SOURCE tonlib/Client.h tonlib/Config.h tonlib/ExtClient.h - tonlib/ExtClientMulti.h - tonlib/ExtClientRaw.h + tonlib/ExtClientLazy.h tonlib/ExtClientOutbound.h tonlib/KeyStorage.h tonlib/KeyValue.h diff --git a/tonlib/tonlib/Config.cpp b/tonlib/tonlib/Config.cpp index dfa9e3f75..73d4e033c 100644 --- a/tonlib/tonlib/Config.cpp +++ b/tonlib/tonlib/Config.cpp @@ -66,16 +66,16 @@ td::Result Config::parse(std::string str) { return td::Status::Error("Invalid config (1)"); } td::JsonArray empty_array; - TRY_RESULT(lite_clients_obj, + TRY_RESULT(lite_servers_obj, td::get_json_object_field(json.get_object(), "liteservers", td::JsonValue::Type::Array, true)); - auto &lite_clients = - lite_clients_obj.type() == td::JsonValue::Type::Array ? lite_clients_obj.get_array() : empty_array; - TRY_RESULT(lite_clients_v2_obj, + auto &lite_servers = + lite_servers_obj.type() == td::JsonValue::Type::Array ? lite_servers_obj.get_array() : empty_array; + TRY_RESULT(lite_servers_v2_obj, td::get_json_object_field(json.get_object(), "liteservers_v2", td::JsonValue::Type::Array, true)); - auto &lite_clients_v2 = - lite_clients_v2_obj.type() == td::JsonValue::Type::Array ? lite_clients_v2_obj.get_array() : empty_array; + auto &lite_servers_v2 = + lite_servers_v2_obj.type() == td::JsonValue::Type::Array ? lite_servers_v2_obj.get_array() : empty_array; - auto parse_desc = [&](td::JsonValue& value) -> td::Result { + auto parse_desc = [&](td::JsonValue &value) -> td::Result { if (value.type() != td::JsonValue::Type::Object) { return td::Status::Error("Invalid config (2)"); } @@ -83,8 +83,8 @@ td::Result Config::parse(std::string str) { TRY_RESULT(ip, td::get_json_object_long_field(object, "ip", false)); TRY_RESULT(port, td::get_json_object_int_field(object, "port", false)); - Config::LiteClient client; - TRY_STATUS(client.address.init_host_port(td::IPAddress::ipv4_to_str(static_cast(ip)), port)); + Config::LiteServer server; + TRY_STATUS(server.address.init_host_port(td::IPAddress::ipv4_to_str(static_cast(ip)), port)); TRY_RESULT(id_obj, td::get_json_object_field(object, "id", td::JsonValue::Type::Object, false)); auto &id = id_obj.get_object(); @@ -98,20 +98,19 @@ td::Result Config::parse(std::string str) { return td::Status::Error("Invalid config (4)"); } - client.adnl_id = ton::adnl::AdnlNodeIdFull(ton::pubkeys::Ed25519(td::Bits256(td::Slice(key).ubegin()))); - return client; + server.adnl_id = ton::adnl::AdnlNodeIdFull(ton::pubkeys::Ed25519(td::Bits256(td::Slice(key).ubegin()))); + return server; }; Config res; - for (auto &value : lite_clients) { - TRY_RESULT(client, parse_desc(value)); - res.lite_clients.push_back(std::move(client)); + for (auto &value : lite_servers) { + TRY_RESULT(server, parse_desc(value)); + res.lite_servers.push_back(std::move(server)); } - for (auto &value : lite_clients_v2) { - TRY_RESULT(client, parse_desc(value)); - client.is_full = false; - TRY_RESULT(shards_obj, - td::get_json_object_field(value.get_object(), "shards", td::JsonValue::Type::Array, false)); + for (auto &value : lite_servers_v2) { + TRY_RESULT(server, parse_desc(value)); + server.is_full = false; + TRY_RESULT(shards_obj, td::get_json_object_field(value.get_object(), "shards", td::JsonValue::Type::Array, false)); for (auto &shard : shards_obj.get_array()) { if (shard.type() != td::JsonValue::Type::Object) { return td::Status::Error("Invalid config (5)"); @@ -122,10 +121,10 @@ td::Result Config::parse(std::string str) { if (shard_id == 0) { return td::Status::Error("Invalid config (6)"); } - client.shards.emplace_back(workchain, shard_id); + server.shards.emplace_back(workchain, shard_id); } - res.lite_clients.push_back(std::move(client)); + res.lite_servers.push_back(std::move(server)); } TRY_RESULT(validator_obj, diff --git a/tonlib/tonlib/Config.h b/tonlib/tonlib/Config.h index 420889dfd..7af1b1c5a 100644 --- a/tonlib/tonlib/Config.h +++ b/tonlib/tonlib/Config.h @@ -23,7 +23,7 @@ namespace tonlib { struct Config { - struct LiteClient { + struct LiteServer { ton::adnl::AdnlNodeIdFull adnl_id; td::IPAddress address; bool is_full = true; @@ -32,7 +32,7 @@ struct Config { ton::BlockIdExt zero_state_id; ton::BlockIdExt init_block_id; std::vector hardforks; - std::vector lite_clients; + std::vector lite_servers; std::string name; static td::Result parse(std::string str); }; diff --git a/tonlib/tonlib/ExtClient.cpp b/tonlib/tonlib/ExtClient.cpp index 30a29b59c..755be170d 100644 --- a/tonlib/tonlib/ExtClient.cpp +++ b/tonlib/tonlib/ExtClient.cpp @@ -54,7 +54,7 @@ void ExtClient::with_last_block(td::Promise promise) { td::actor::send_closure(client_.last_block_actor_, &LastBlock::get_last_block, std::move(P)); } -void ExtClient::send_raw_query(td::BufferSlice query, td::Promise promise) { +void ExtClient::send_raw_query(td::BufferSlice query, ton::ShardIdFull shard, td::Promise promise) { auto query_id = queries_.create(std::move(promise)); td::Promise P = [query_id, self = this, actor_id = td::actor::actor_id()](td::Result result) { @@ -65,7 +65,7 @@ void ExtClient::send_raw_query(td::BufferSlice query, td::Promise + namespace tonlib { -class ExtClientLazyImp : public ExtClientLazy { +class ExtClientLazyImpl : public ExtClientLazy { public: - ExtClientLazyImp(std::vector> servers, - td::unique_ptr callback) - : servers_(std::move(servers)), callback_(std::move(callback)) { - CHECK(!servers_.empty()); + ExtClientLazyImpl(std::vector servers, td::unique_ptr callback) + : callback_(std::move(callback)) { + CHECK(!servers.empty()); + servers_.resize(servers.size()); + for (size_t i = 0; i < servers_.size(); ++i) { + servers_[i].s = std::move(servers[i]); + if (!servers_[i].s.is_full) { + for (auto shard : servers_[i].s.shards) { + CHECK(shard.is_valid_ext()); + max_server_shard_depth_ = std::max(max_server_shard_depth_, shard.pfx_len()); + } + } + } } void start_up() override { @@ -34,29 +46,21 @@ class ExtClientLazyImp : public ExtClientLazy { td::random_shuffle(td::as_mutable_span(servers_), rnd); } - void check_ready(td::Promise promise) override { - before_query(); - if (client_.empty()) { - return promise.set_error(TonlibError::Cancelled()); - } - send_closure(client_, &ton::adnl::AdnlExtClient::check_ready, std::move(promise)); - } - - void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, + void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, td::Promise promise) override { - before_query(); - if (client_.empty()) { - return promise.set_error(TonlibError::Cancelled()); - } - td::Promise P = [SelfId = actor_id(this), idx = cur_server_idx_, + TRY_RESULT_PROMISE(promise, server_idx, before_query(shard)); + auto& server = servers_[server_idx]; + CHECK(!server.client.empty()); + alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT)); + td::Promise P = [SelfId = actor_id(this), server_idx, promise = std::move(promise)](td::Result R) mutable { if (R.is_error() && (R.error().code() == ton::ErrorCode::timeout || R.error().code() == ton::ErrorCode::cancelled)) { - td::actor::send_closure(SelfId, &ExtClientLazyImp::set_server_bad, idx, true); + td::actor::send_closure(SelfId, &ExtClientLazyImpl::set_server_bad, server_idx); } promise.set_result(std::move(R)); }; - send_closure(client_, &ton::adnl::AdnlExtClient::send_query, std::move(name), std::move(data), timeout, + send_closure(server.client, &ton::adnl::AdnlExtClient::send_query, std::move(name), std::move(data), timeout, std::move(P)); } @@ -64,64 +68,124 @@ class ExtClientLazyImp : public ExtClientLazy { if (servers_.size() == 1) { return; } - cur_server_bad_ = cur_server_bad_force_ = true; + auto it = shard_to_server_.find(ton::ShardIdFull(ton::masterchainId)); + if (it != shard_to_server_.end()) { + set_server_bad(it->second); + } } private: - void before_query() { + td::Result before_query(ton::ShardIdFull shard) { + if (!shard.is_valid_ext()) { + return td::Status::Error("Invalid shard"); + } if (is_closing_) { - return; + return td::Status::Error("Client is closing"); } - alarm_timestamp() = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT); - if (cur_server_bad_) { - ++cur_server_idx_; - } else if (!client_.empty()) { - return; + if (shard.pfx_len() > max_server_shard_depth_) { + shard = shard_prefix(shard, max_server_shard_depth_); + } + auto it = shard_to_server_.find(shard); + if (it != shard_to_server_.end()) { + size_t server_idx = it->second; + if (!servers_[server_idx].client.empty()) { + return server_idx; + } + shard_to_server_.erase(it); + } + + size_t server_idx = servers_.size(); + int cnt = 0; + int best_priority = -1; + for (size_t i = 0; i < servers_.size(); ++i) { + Server& server = servers_[i]; + if (!server.supports(shard)) { + continue; + } + int priority = 0; + priority += (server.client.empty() ? 0 : 100); + priority += (server.ignore_until && !server.ignore_until.is_in_past() ? 0 : 10); + priority += (server.s.is_full ? 1 : 0); + if (priority < best_priority) { + continue; + } + if (priority > best_priority) { + best_priority = priority; + cnt = 0; + } + if (td::Random::fast(0, cnt) == 0) { + server_idx = i; + } + ++cnt; } + if (server_idx == servers_.size()) { + return td::Status::Error(PSTRING() << "No liteserver for shard " << shard.to_str()); + } + Server& server = servers_[server_idx]; + if (!server.client.empty()) { + return server_idx; + } + class Callback : public ton::adnl::AdnlExtClient::Callback { public: - explicit Callback(td::actor::ActorShared parent, size_t idx) + explicit Callback(td::actor::ActorShared parent, size_t idx) : parent_(std::move(parent)), idx_(idx) { } void on_ready() override { - td::actor::send_closure(parent_, &ExtClientLazyImp::set_server_bad, idx_, false); } void on_stop_ready() override { - td::actor::send_closure(parent_, &ExtClientLazyImp::set_server_bad, idx_, true); + td::actor::send_closure(parent_, &ExtClientLazyImpl::set_server_bad, idx_); } private: - td::actor::ActorShared parent_; + td::actor::ActorShared parent_; size_t idx_; }; ref_cnt_++; - cur_server_bad_ = false; - cur_server_bad_force_ = false; - const auto& s = servers_[cur_server_idx_ % servers_.size()]; - LOG(INFO) << "Connecting to liteserver " << s.second; - client_ = ton::adnl::AdnlExtClient::create( - s.first, s.second, std::make_unique(td::actor::actor_shared(this), cur_server_idx_)); + if (shard.is_masterchain()) { + LOG(INFO) << "Connecting to liteserver " << server.s.address << " for masterchain"; + } else { + LOG(INFO) << "Connecting to liteserver " << server.s.address << " for shard " << shard.to_str(); + } + server.client = ton::adnl::AdnlExtClient::create( + server.s.adnl_id, server.s.address, std::make_unique(td::actor::actor_shared(this), server_idx)); + alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT)); + return server_idx; } - std::vector> servers_; - size_t cur_server_idx_ = 0; - bool cur_server_bad_ = false; - bool cur_server_bad_force_ = false; + struct Server { + Config::LiteServer s; + td::actor::ActorOwn client; + td::Timestamp timeout = td::Timestamp::never(); + td::Timestamp ignore_until = td::Timestamp::never(); + + bool supports(const ton::ShardIdFull& shard) const { + return s.is_full || shard.is_masterchain() || + std::any_of(s.shards.begin(), s.shards.end(), + [&](const ton::ShardIdFull s_shard) { return ton::shard_intersects(shard, s_shard); }); + } + }; + std::vector servers_; + std::map shard_to_server_; + int max_server_shard_depth_ = 0; - td::actor::ActorOwn client_; td::unique_ptr callback_; static constexpr double MAX_NO_QUERIES_TIMEOUT = 100; bool is_closing_{false}; td::uint32 ref_cnt_{1}; - void set_server_bad(size_t idx, bool bad) { - if (idx == cur_server_idx_ && servers_.size() > 1 && !cur_server_bad_force_) { - cur_server_bad_ = bad; + void alarm() override { + for (Server& server : servers_) { + if (server.timeout && server.timeout.is_in_past()) { + server.client.reset(); + } } } - void alarm() override { - client_.reset(); + void set_server_bad(size_t idx) { + servers_[idx].client.reset(); + servers_[idx].timeout = td::Timestamp::never(); + servers_[idx].ignore_until = td::Timestamp::in(60.0); } void hangup_shared() override { ref_cnt_--; @@ -130,7 +194,9 @@ class ExtClientLazyImp : public ExtClientLazy { void hangup() override { is_closing_ = true; ref_cnt_--; - client_.reset(); + for (Server& server : servers_) { + server.client.reset(); + } try_stop(); } void try_stop() { @@ -142,11 +208,11 @@ class ExtClientLazyImp : public ExtClientLazy { td::actor::ActorOwn ExtClientLazy::create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, td::unique_ptr callback) { - return create({std::make_pair(dst, dst_addr)}, std::move(callback)); + return create({Config::LiteServer{dst, dst_addr, true, {}}}, std::move(callback)); } -td::actor::ActorOwn ExtClientLazy::create( - std::vector> servers, td::unique_ptr callback) { - return td::actor::create_actor("ExtClientLazy", std::move(servers), std::move(callback)); +td::actor::ActorOwn ExtClientLazy::create(std::vector servers, + td::unique_ptr callback) { + return td::actor::create_actor("ExtClientLazy", std::move(servers), std::move(callback)); } } // namespace tonlib diff --git a/tonlib/tonlib/ExtClientRaw.h b/tonlib/tonlib/ExtClientLazy.h similarity index 72% rename from tonlib/tonlib/ExtClientRaw.h rename to tonlib/tonlib/ExtClientLazy.h index 612995af0..6383cec55 100644 --- a/tonlib/tonlib/ExtClientRaw.h +++ b/tonlib/tonlib/ExtClientLazy.h @@ -18,23 +18,27 @@ */ #pragma once #include "td/actor/actor.h" -#include "adnl/adnl-ext-client.h" #include "ton/ton-types.h" +#include "adnl/adnl-ext-client.h" +#include "Config.h" namespace tonlib { -class ExtClientLazy : public ton::adnl::AdnlExtClient { +class ExtClientLazy : public td::actor::Actor { public: class Callback { public: - virtual ~Callback() = default; + virtual ~Callback() { + } }; + virtual void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, + td::Promise promise) = 0; virtual void force_change_liteserver() = 0; static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, td::unique_ptr callback); - static td::actor::ActorOwn create( - std::vector> servers, td::unique_ptr callback); + static td::actor::ActorOwn create(std::vector servers, + td::unique_ptr callback); }; } // namespace tonlib diff --git a/tonlib/tonlib/ExtClientMulti.cpp b/tonlib/tonlib/ExtClientMulti.cpp deleted file mode 100644 index 3a6cdf595..000000000 --- a/tonlib/tonlib/ExtClientMulti.cpp +++ /dev/null @@ -1,169 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . -*/ -#include "ExtClientMulti.h" -#include "ton/ton-shard.h" -#include "td/utils/Random.h" - -namespace tonlib { - -static const double MAX_NO_QUERIES_TIMEOUT = 120; - -ExtClientMulti::ExtClientMulti(std::vector clients, td::unique_ptr callback) - : callback_(std::move(callback)) { - for (auto &desc : clients) { - servers_.emplace_back(); - servers_.back().desc = std::move(desc); - } -} - -void ExtClientMulti::start_up() { - alarm_timestamp() = td::Timestamp::in(60.0); -} - -void ExtClientMulti::send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, - td::Promise promise) { - if (shard.is_masterchain() && mc_server_idx_ != -1 && !servers_[mc_server_idx_].is_bad()) { - send_query_to_server(std::move(name), std::move(data), mc_server_idx_, timeout, std::move(promise)); - return; - } - auto it = shard_server_idx_cached_.find(shard); - if (it != shard_server_idx_cached_.end() && !servers_[it->second].is_bad()) { - send_query_to_server(std::move(name), std::move(data), it->second, timeout, std::move(promise)); - return; - } - int server_idx = -1; - int random_idx = -1; - int cnt = 0; - int best_prefix = -1; - for (int i = 0; i < (int)servers_.size(); ++i) { - const Server &server = servers_[i]; - if (server.is_bad()) { - continue; - } - int len = server.desc.is_full ? 65 : server.max_supported_prefix(shard); - if (len > best_prefix) { - best_prefix = len; - server_idx = -1; - random_idx = -1; - cnt = 0; - } else if (len < best_prefix) { - continue; - } - if (!server.client.empty()) { - server_idx = i; - } - if (td::Random::fast(0, cnt) == 0) { - random_idx = i; - } - ++cnt; - } - if (server_idx == -1) { - server_idx = random_idx; - } - if (server_idx == -1) { - promise.set_error(td::Status::Error("failed to select a suitable server")); - return; - } - if (shard.pfx_len() <= ton::max_shard_pfx_len) { - shard_server_idx_cached_[shard] = server_idx; - } - if (shard.is_masterchain() || servers_[server_idx].desc.is_full) { - mc_server_idx_ = server_idx; - } - send_query_to_server(std::move(name), std::move(data), server_idx, timeout, std::move(promise)); -} - -void ExtClientMulti::send_query_to_server(std::string name, td::BufferSlice data, int server_idx, td::Timestamp timeout, - td::Promise promise) { - Server &server = servers_.at(server_idx); - if (server.client.empty()) { - start_client(server_idx); - } - server.ttl = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT); - td::Promise P = [SelfId = actor_id(this), idx = server_idx, - promise = std::move(promise)](td::Result R) mutable { - if (R.is_error() && - (R.error().code() == ton::ErrorCode::timeout || R.error().code() == ton::ErrorCode::cancelled)) { - td::actor::send_closure(SelfId, &ExtClientMulti::set_server_bad, idx); - } - promise.set_result(std::move(R)); - }; - send_closure(server.client, &ton::adnl::AdnlExtClient::send_query, std::move(name), std::move(data), timeout, - std::move(P)); -} - -void ExtClientMulti::force_change_liteserver() { - if (mc_server_idx_ != -1) { - set_server_bad(mc_server_idx_); - mc_server_idx_ = -1; - } -} - -void ExtClientMulti::start_client(int server_idx) { - class Callback : public ton::adnl::AdnlExtClient::Callback { - public: - Callback(td::actor::ActorId parent, int idx) : parent_(std::move(parent)), idx_(idx) { - } - void on_ready() override { - } - void on_stop_ready() override { - td::actor::send_closure(parent_, &ExtClientMulti::set_server_bad, idx_); - } - - private: - td::actor::ActorId parent_; - int idx_; - }; - Server &server = servers_.at(server_idx); - server.client = ton::adnl::AdnlExtClient::create(server.desc.adnl_id, server.desc.address, - std::make_unique(actor_id(this), server_idx)); -} - -void ExtClientMulti::alarm() { - for (Server& server : servers_) { - if (server.ttl && server.ttl.is_in_past()) { - server.client.reset(); - } - } - alarm_timestamp() = td::Timestamp::in(60.0); -} - -void ExtClientMulti::set_server_bad(int idx) { - Server& server = servers_.at(idx); - server.client.reset(); - server.ttl = td::Timestamp::never(); - server.ignore_until = td::Timestamp::in(10.0); -} - -int ExtClientMulti::Server::max_supported_prefix(ton::ShardIdFull shard) const { - if (desc.is_full || shard.is_masterchain()) { - return shard.pfx_len(); - } - int res = -1; - for (const ton::ShardIdFull &our_shard : desc.shards) { - if (ton::shard_is_ancestor(our_shard, shard)) { - return shard.pfx_len(); - } - if (shard.workchain == our_shard.workchain) { - int x = std::min({shard.pfx_len(), our_shard.pfx_len(), ton::count_matching_bits(shard.shard, our_shard.shard)}); - res = std::max(res, x); - } - } - return res; -} - -} // namespace tonlib diff --git a/tonlib/tonlib/ExtClientMulti.h b/tonlib/tonlib/ExtClientMulti.h deleted file mode 100644 index 4c93b00ee..000000000 --- a/tonlib/tonlib/ExtClientMulti.h +++ /dev/null @@ -1,61 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . -*/ -#pragma once -#include "td/actor/actor.h" -#include "adnl/adnl-ext-client.h" -#include "Config.h" -#include "ExtClientRaw.h" -#include - -namespace tonlib { - -class ExtClientMulti : public ExtClientRaw { - public: - ExtClientMulti(std::vector clients, td::unique_ptr callback); - - void start_up() override; - void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, - td::Promise promise) override; - void alarm() override; - void force_change_liteserver() override; - - private: - void send_query_to_server(std::string name, td::BufferSlice data, int server_idx, td::Timestamp timeout, - td::Promise promise); - void start_client(int server_idx); - - struct Server { - Config::LiteClient desc; - td::actor::ActorOwn client; - td::Timestamp ttl; - td::Timestamp ignore_until = td::Timestamp::never(); - - int max_supported_prefix(ton::ShardIdFull shard) const; - bool is_bad() const { - return ignore_until && !ignore_until.is_in_past(); - } - }; - - void set_server_bad(int idx); - - td::unique_ptr callback_; - std::vector servers_; - int mc_server_idx_ = -1; - std::map shard_server_idx_cached_; -}; - -} // namespace tonlib diff --git a/tonlib/tonlib/ExtClientOutbound.cpp b/tonlib/tonlib/ExtClientOutbound.cpp index d2715b8b5..eb923734e 100644 --- a/tonlib/tonlib/ExtClientOutbound.cpp +++ b/tonlib/tonlib/ExtClientOutbound.cpp @@ -20,11 +20,12 @@ #include "ExtClientOutbound.h" #include "TonlibError.h" #include + namespace tonlib { -class ExtClientOutboundImp : public ExtClientOutbound { +class ExtClientOutboundImpl : public ExtClientOutbound { public: - ExtClientOutboundImp(td::unique_ptr callback) : callback_(std::move(callback)) { + ExtClientOutboundImpl(td::unique_ptr callback) : callback_(std::move(callback)) { } void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, @@ -37,9 +38,6 @@ class ExtClientOutboundImp : public ExtClientOutbound { void force_change_liteserver() override { } - void force_change_liteserver() override { - } - void on_query_result(td::int64 id, td::Result r_data, td::Promise promise) override { auto it = queries_.find(id); if (it == queries_.end()) { @@ -65,6 +63,6 @@ class ExtClientOutboundImp : public ExtClientOutbound { }; td::actor::ActorOwn ExtClientOutbound::create(td::unique_ptr callback) { - return td::actor::create_actor("ExtClientOutbound", std::move(callback)); + return td::actor::create_actor("ExtClientOutbound", std::move(callback)); } } // namespace tonlib diff --git a/tonlib/tonlib/ExtClientOutbound.h b/tonlib/tonlib/ExtClientOutbound.h index 4721e8729..31f76b977 100644 --- a/tonlib/tonlib/ExtClientOutbound.h +++ b/tonlib/tonlib/ExtClientOutbound.h @@ -30,6 +30,7 @@ class ExtClientOutbound : public ExtClientLazy { } virtual void request(td::int64 id, std::string data, ton::ShardIdFull shard) = 0; }; + virtual void on_query_result(td::int64 id, td::Result r_data, td::Promise promise) = 0; static td::actor::ActorOwn create(td::unique_ptr callback); }; diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 0680cebc6..05406d509 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -27,7 +27,6 @@ #include "tonlib/keys/Mnemonic.h" #include "tonlib/keys/SimpleEncryption.h" #include "tonlib/TonlibError.h" -#include "tonlib/ExtClientMulti.h" #include "smc-envelope/GenericAccount.h" #include "smc-envelope/ManualDns.h" @@ -863,7 +862,9 @@ class Query { } return res; } - td::Result>> estimate_fees(bool ignore_chksig, std::shared_ptr& cfg, vm::Dictionary& libraries) { + td::Result>> estimate_fees(bool ignore_chksig, + std::shared_ptr& cfg, + vm::Dictionary& libraries) { // gas fees bool is_masterchain = raw_.source->get_address().workchain == ton::masterchainId; TRY_RESULT(gas_limits_prices, cfg->get_gas_limits_prices(is_masterchain)); @@ -892,7 +893,8 @@ class Query { .set_now(raw_.source->get_sync_time()) .set_ignore_chksig(ignore_chksig) .set_address(raw_.source->get_address()) - .set_config(cfg).set_libraries(libraries)); + .set_config(cfg) + .set_libraries(libraries)); td::int64 fwd_fee = 0; if (res.success) { LOG(DEBUG) << "output actions:\n" @@ -962,8 +964,9 @@ td::Result to_balance(td::Ref balance_ref) { class GetTransactionHistory : public td::actor::Actor { public: - GetTransactionHistory(ExtClientRef ext_client_ref, block::StdAddress address, ton::LogicalTime lt, ton::Bits256 hash, td::int32 count, - td::actor::ActorShared<> parent, td::Promise promise) + GetTransactionHistory(ExtClientRef ext_client_ref, block::StdAddress address, ton::LogicalTime lt, ton::Bits256 hash, + td::int32 count, td::actor::ActorShared<> parent, + td::Promise promise) : address_(std::move(address)) , lt_(std::move(lt)) , hash_(std::move(hash)) @@ -1671,8 +1674,8 @@ void TonlibClient::init_ext_client() { } void request(td::int64 id, std::string data, ton::ShardIdFull shard) override { - send_closure(parent_, &TonlibClient::proxy_request, (id << 16) | (config_generation_ & 0xffff), - std::move(data), shard); + send_closure(parent_, &TonlibClient::proxy_request, (id << 16) | (config_generation_ & 0xffff), std::move(data), + shard); } private: @@ -1685,34 +1688,17 @@ void TonlibClient::init_ext_client() { ext_client_outbound_ = client.get(); raw_client_ = std::move(client); } else { - std::vector> servers; - for (const auto& s : config_.lite_clients) { - servers.emplace_back(s.adnl_id, s.address); - } class Callback : public ExtClientLazy::Callback { public: explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { } + private: td::actor::ActorShared<> parent_; }; - std::vector> full_servers; - for (const auto& s : config_.lite_clients) { - if (s.is_full) { - full_servers.emplace_back(s.adnl_id, s.address); - } - } - if (!full_servers.empty()) { - raw_client_ = - ExtClientRaw::create(std::move(full_servers), td::make_unique(td::actor::actor_shared())); - } else { - CHECK(!config_.lite_clients.empty()); - raw_client_ = td::actor::create_actor("ExtClientMulti", config_.lite_clients, - td::make_unique(td::actor::actor_shared())); - } ext_client_outbound_ = {}; ref_cnt_++; - raw_client_ = ExtClientLazy::create(std::move(servers), td::make_unique(td::actor::actor_shared())); + raw_client_ = ExtClientLazy::create(config_.lite_servers, td::make_unique(td::actor::actor_shared())); } } @@ -2360,7 +2346,7 @@ td::Result TonlibClient::validate_config(tonlib_api::o TRY_RESULT_PREFIX(new_config, Config::parse(std::move(config->config_)), TonlibError::InvalidConfig("can't parse config")); - if (new_config.lite_clients.empty() && !config->use_callbacks_for_network_) { + if (new_config.lite_servers.empty() && !config->use_callbacks_for_network_) { return TonlibError::InvalidConfig("no lite clients"); } td::optional o_master_config; @@ -2427,8 +2413,7 @@ td::Result TonlibClient::validate_config(tonlib_api::o state.vert_seqno = vert_seqno; bool user_defined_init_block = false; - if (new_config.init_block_id.is_valid() && - state.last_key_block_id.id.seqno < new_config.init_block_id.id.seqno) { + if (new_config.init_block_id.is_valid() && state.last_key_block_id.id.seqno < new_config.init_block_id.id.seqno) { state.last_key_block_id = new_config.init_block_id; user_defined_init_block = true; LOG(INFO) << "Use init block from USER config: " << new_config.init_block_id.to_str(); @@ -2524,8 +2509,7 @@ td::Result to_std_address(td::Ref cs) { } struct ToRawTransactions { explicit ToRawTransactions(td::optional private_key, bool try_decode_messages = true) - : private_key_(std::move(private_key)) - , try_decode_messages_(try_decode_messages) { + : private_key_(std::move(private_key)), try_decode_messages_(try_decode_messages) { } td::optional private_key_; @@ -2588,7 +2572,8 @@ struct ToRawTransactions { } } if (!data) { - data = tonlib_api::make_object(to_bytes(std::move(body_cell)), to_bytes(std::move(init_state_cell))); + data = tonlib_api::make_object(to_bytes(std::move(body_cell)), + to_bytes(std::move(init_state_cell))); } return data; }; @@ -2636,7 +2621,8 @@ struct ToRawTransactions { auto created_lt = static_cast(msg_info.created_lt); return tonlib_api::make_object( tonlib_api::make_object(src), - tonlib_api::make_object(), 0, 0, 0, created_lt, std::move(body_hash), get_data(src)); + tonlib_api::make_object(), 0, 0, 0, created_lt, std::move(body_hash), + get_data(src)); } } @@ -2746,10 +2732,9 @@ td::Status TonlibClient::do_request(const tonlib_api::raw_sendMessageReturnHash& td::Promise>&& promise) { TRY_RESULT_PREFIX(body, vm::std_boc_deserialize(request.body_), TonlibError::InvalidBagOfCells("body")); auto hash = body->get_hash().as_slice().str(); - make_request(int_api::SendMessage{std::move(body)}, - promise.wrap([hash = std::move(hash)](auto res) { - return tonlib_api::make_object(std::move(hash)); - })); + make_request(int_api::SendMessage{std::move(body)}, promise.wrap([hash = std::move(hash)](auto res) { + return tonlib_api::make_object(std::move(hash)); + })); return td::Status::OK(); } @@ -2839,7 +2824,7 @@ td::Status TonlibClient::do_request(tonlib_api::raw_getTransactions& request, } td::Status TonlibClient::do_request(tonlib_api::raw_getTransactionsV2& request, - td::Promise>&& promise) { + td::Promise>&& promise) { if (!request.account_address_) { return TonlibError::EmptyField("account_address"); } @@ -2875,9 +2860,10 @@ td::Status TonlibClient::do_request(tonlib_api::raw_getTransactionsV2& request, auto actor_id = actor_id_++; actors_[actor_id] = td::actor::create_actor( "GetTransactionHistory", client_.get_client(), account_address, lt, hash, count, actor_shared(this, actor_id), - promise.wrap([private_key = std::move(private_key), try_decode_messages = request.try_decode_messages_](auto&& x) mutable { - return ToRawTransactions(std::move(private_key), try_decode_messages).to_raw_transactions(std::move(x)); - })); + promise.wrap( + [private_key = std::move(private_key), try_decode_messages = request.try_decode_messages_](auto&& x) mutable { + return ToRawTransactions(std::move(private_key), try_decode_messages).to_raw_transactions(std::move(x)); + })); return td::Status::OK(); } @@ -3044,26 +3030,25 @@ class GenericCreateSendGrams : public TonlibQueryActor { td::Result to_dns_action(tonlib_api::dns_Action& action) { using R = td::Result; - return downcast_call2(action, - td::overloaded( - [&](tonlib_api::dns_actionDeleteAll& del_all) -> R { - return ton::ManualDns::Action{"", td::Bits256::zero(), {}}; - }, - [&](tonlib_api::dns_actionDelete& del) -> R { - return ton::ManualDns::Action{del.name_, del.category_, {}}; - }, - [&](tonlib_api::dns_actionSet& set) -> R { - if (!set.entry_) { - return TonlibError::EmptyField("entry"); - } - if (!set.entry_->entry_) { - return TonlibError::EmptyField("entry.entry"); - } - TRY_RESULT(entry_data, to_dns_entry_data(*set.entry_->entry_)); - TRY_RESULT(data_cell, entry_data.as_cell()); - return ton::ManualDns::Action{set.entry_->name_, set.entry_->category_, - std::move(data_cell)}; - })); + return downcast_call2( + action, td::overloaded( + [&](tonlib_api::dns_actionDeleteAll& del_all) -> R { + return ton::ManualDns::Action{"", td::Bits256::zero(), {}}; + }, + [&](tonlib_api::dns_actionDelete& del) -> R { + return ton::ManualDns::Action{del.name_, del.category_, {}}; + }, + [&](tonlib_api::dns_actionSet& set) -> R { + if (!set.entry_) { + return TonlibError::EmptyField("entry"); + } + if (!set.entry_->entry_) { + return TonlibError::EmptyField("entry.entry"); + } + TRY_RESULT(entry_data, to_dns_entry_data(*set.entry_->entry_)); + TRY_RESULT(data_cell, entry_data.as_cell()); + return ton::ManualDns::Action{set.entry_->name_, set.entry_->category_, std::move(data_cell)}; + })); } td::Status parse_action(tonlib_api::Action& action) { @@ -3471,9 +3456,9 @@ class GenericCreateSendGrams : public TonlibQueryActor { } } -// if (!o_public_key) { // todo: (tolya-yanot) temporary disable msg comment encryption (The exchanges/payment services needs to read the comment of incoming messages). This will be uncommented when a general standard is developed. - return TonlibError::MessageEncryption("Get public key (in destination)"); -// } + // if (!o_public_key) { // todo: (tolya-yanot) temporary disable msg comment encryption (The exchanges/payment services needs to read the comment of incoming messages). This will be uncommented when a general standard is developed. + return TonlibError::MessageEncryption("Get public key (in destination)"); + // } auto addr = source_->get_address(); addr.bounceable = true; @@ -3868,8 +3853,8 @@ td::Result from_tonlib_api(tonlib_api::tvm_StackEntry& entry) { })); } -void deep_library_search(std::set& set, std::set& visited, - vm::Dictionary& libs, td::Ref cell, int depth) { +void deep_library_search(std::set& set, std::set& visited, vm::Dictionary& libs, + td::Ref cell, int depth) { if (depth <= 0 || set.size() >= 16 || visited.size() >= 256) { return; } @@ -3895,7 +3880,7 @@ void deep_library_search(std::set& set, std::set& v } return; } - for (unsigned int i=0; iget_refs_cnt(); i++) { + for (unsigned int i = 0; i < loaded_cell.data_cell->get_refs_cnt(); i++) { deep_library_search(set, visited, libs, loaded_cell.data_cell->get_ref(i), depth - 1); } } @@ -3919,36 +3904,38 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_getLibraries& request, return td::Status::OK(); } - client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(not_cached_hashes)), - promise.wrap([self=this, result_entries = std::move(result_entries)] - (td::Result> r_libraries) mutable - { - if (r_libraries.is_error()) { - LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string(); - } else { - auto libraries = r_libraries.move_as_ok(); - bool updated = false; - for (auto& lr : libraries->result_) { - auto contents = vm::std_boc_deserialize(lr->data_); - if (contents.is_ok() && contents.ok().not_null()) { - if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { - LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); - continue; + client_.send_query( + ton::lite_api::liteServer_getLibraries(std::move(not_cached_hashes)), + promise.wrap( + [self = this, result_entries = std::move(result_entries)]( + td::Result> r_libraries) mutable { + if (r_libraries.is_error()) { + LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string(); + } else { + auto libraries = r_libraries.move_as_ok(); + bool updated = false; + for (auto& lr : libraries->result_) { + auto contents = vm::std_boc_deserialize(lr->data_); + if (contents.is_ok() && contents.ok().not_null()) { + if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { + LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); + continue; + } + result_entries.push_back( + tonlib_api::make_object(lr->hash_, lr->data_.as_slice().str())); + self->libraries.set_ref(lr->hash_, contents.move_as_ok()); + updated = true; + LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); + } else { + LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); + } + if (updated) { + self->store_libs_to_disk(); + } + } } - result_entries.push_back(tonlib_api::make_object(lr->hash_, lr->data_.as_slice().str())); - self->libraries.set_ref(lr->hash_, contents.move_as_ok()); - updated = true; - LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); - } else { - LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); - } - if (updated) { - self->store_libs_to_disk(); - } - } - } - return tonlib_api::make_object(std::move(result_entries)); - })); + return tonlib_api::make_object(std::move(result_entries)); + })); return td::Status::OK(); } @@ -3975,8 +3962,8 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, args.set_now(it->second->get_sync_time()); args.set_address(it->second->get_address()); - client_.with_last_config([self = this, smc = std::move(smc), args = std::move(args), promise = std::move(promise) - ](td::Result r_state) mutable { + client_.with_last_config([self = this, smc = std::move(smc), args = std::move(args), + promise = std::move(promise)](td::Result r_state) mutable { TRY_RESULT_PROMISE(promise, state, std::move(r_state)); args.set_config(state.config); @@ -3988,41 +3975,39 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, std::vector libraryList{librarySet.begin(), librarySet.end()}; if (libraryList.size() > 0) { LOG(DEBUG) << "Requesting found libraries in code (" << libraryList.size() << ")"; - self->client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(libraryList)), - [self, smc = std::move(smc), args = std::move(args), promise = std::move(promise)] - (td::Result> r_libraries) mutable - { - if (r_libraries.is_error()) { - LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string(); - } else { - auto libraries = r_libraries.move_as_ok(); - bool updated = false; - for (auto& lr : libraries->result_) { - auto contents = vm::std_boc_deserialize(lr->data_); - if (contents.is_ok() && contents.ok().not_null()) { - if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { - LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); - continue; - } - self->libraries.set_ref(lr->hash_, contents.move_as_ok()); - updated = true; - LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); + self->client_.send_query( + ton::lite_api::liteServer_getLibraries(std::move(libraryList)), + [self, smc = std::move(smc), args = std::move(args), promise = std::move(promise)]( + td::Result> r_libraries) mutable { + if (r_libraries.is_error()) { + LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string(); } else { - LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); + auto libraries = r_libraries.move_as_ok(); + bool updated = false; + for (auto& lr : libraries->result_) { + auto contents = vm::std_boc_deserialize(lr->data_); + if (contents.is_ok() && contents.ok().not_null()) { + if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { + LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); + continue; + } + self->libraries.set_ref(lr->hash_, contents.move_as_ok()); + updated = true; + LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); + } else { + LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); + } + } + if (updated) { + self->store_libs_to_disk(); + } } - } - if (updated) { - self->store_libs_to_disk(); - } - } - self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); - }); - } - else { + self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); + }); + } else { self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); } - } - else { + } else { self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); } }); @@ -4031,7 +4016,6 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, void TonlibClient::perform_smc_execution(td::Ref smc, ton::SmartContract::Args args, td::Promise>&& promise) { - args.set_libraries(libraries); auto res = smc->run_get_method(args); @@ -4046,45 +4030,46 @@ void TonlibClient::perform_smc_execution(td::Ref smc, ton::S td::Bits256 hash = res.missing_library; LOG(DEBUG) << "Requesting missing library: " << hash.to_hex(); std::vector req = {std::move(hash)}; - client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(req)), - [self = this, res = std::move(res), res_stack = std::move(res_stack), hash = std::move(hash), - smc = std::move(smc), args = std::move(args), promise = std::move(promise)] - (td::Result> r_libraries) mutable - { - if (r_libraries.is_error()) { - LOG(WARNING) << "cannot obtain missing library: " << r_libraries.move_as_error().to_string(); - promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); - return; - } - bool found = false, updated = false; - auto libraries = r_libraries.move_as_ok(); - for (auto& lr : libraries->result_) { - auto contents = vm::std_boc_deserialize(lr->data_); - if (contents.is_ok() && contents.ok().not_null()) { - if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { - LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); - continue; + client_.send_query( + ton::lite_api::liteServer_getLibraries(std::move(req)), + [self = this, res = std::move(res), res_stack = std::move(res_stack), hash = std::move(hash), + smc = std::move(smc), args = std::move(args), promise = std::move(promise)]( + td::Result> r_libraries) mutable { + if (r_libraries.is_error()) { + LOG(WARNING) << "cannot obtain missing library: " << r_libraries.move_as_error().to_string(); + promise.set_value( + tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); + return; } - found |= (lr->hash_ == hash); - updated = true; - self->libraries.set_ref(lr->hash_, contents.move_as_ok()); - LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); - } else { - LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); - } - } - if (updated) { - self->store_libs_to_disk(); - } - if (!found) { - LOG(WARNING) << "cannot obtain library " << hash.to_hex() << ", it may not exist"; - promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); - } else { - self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); - } - }); - } - else { + bool found = false, updated = false; + auto libraries = r_libraries.move_as_ok(); + for (auto& lr : libraries->result_) { + auto contents = vm::std_boc_deserialize(lr->data_); + if (contents.is_ok() && contents.ok().not_null()) { + if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { + LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); + continue; + } + found |= (lr->hash_ == hash); + updated = true; + self->libraries.set_ref(lr->hash_, contents.move_as_ok()); + LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); + } else { + LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); + } + } + if (updated) { + self->store_libs_to_disk(); + } + if (!found) { + LOG(WARNING) << "cannot obtain library " << hash.to_hex() << ", it may not exist"; + promise.set_value( + tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); + } else { + self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); + } + }); + } else { promise.set_value(tonlib_api::make_object(res.gas_used, std::move(res_stack), res.code)); } } @@ -4160,9 +4145,8 @@ void TonlibClient::do_dns_request(std::string name, td::Bits256 category, td::in td::optional block_id, block::StdAddress address, td::Promise>&& promise) { auto block_id_copy = block_id.copy(); - td::Promise new_promise = - promise.send_closure(actor_id(this), &TonlibClient::finish_dns_resolve, name, category, ttl, std::move(block_id), - address); + td::Promise new_promise = promise.send_closure(actor_id(this), &TonlibClient::finish_dns_resolve, name, + category, ttl, std::move(block_id), address); if (0) { make_request(int_api::GetAccountState{address, std::move(block_id_copy), {}}, @@ -4200,8 +4184,7 @@ td::Status TonlibClient::do_request(const tonlib_api::dns_resolve& request, name += '.'; } TRY_RESULT(account_address, get_account_address(request.account_address_->account_address_)); - do_dns_request(name, request.category_, request.ttl_, std::move(block_id), account_address, - std::move(promise)); + do_dns_request(name, request.category_, request.ttl_, std::move(block_id), account_address, std::move(promise)); return td::Status::OK(); } @@ -4639,15 +4622,15 @@ auto to_tonlib_api(const ton::lite_api::tonNode_blockIdExt& blk) -> tonlib_api_p auto to_tonlib_api(const ton::lite_api::tonNode_zeroStateIdExt& zeroStateId) -> tonlib_api_ptr { - return tonlib_api::make_object( //TODO check wether shard indeed 0??? + return tonlib_api::make_object( //TODO check wether shard indeed 0??? zeroStateId.workchain_, 0, 0, zeroStateId.root_hash_.as_slice().str(), zeroStateId.file_hash_.as_slice().str()); } auto to_lite_api(const tonlib_api::ton_blockIdExt& blk) -> td::Result> { TRY_RESULT(root_hash, to_bits256(blk.root_hash_, "blk.root_hash")) TRY_RESULT(file_hash, to_bits256(blk.file_hash_, "blk.file_hash")) - return ton::lite_api::make_object( - blk.workchain_, blk.shard_, blk.seqno_, root_hash, file_hash); + return ton::lite_api::make_object(blk.workchain_, blk.shard_, blk.seqno_, + root_hash, file_hash); } td::Result to_block_id(const tonlib_api::ton_blockIdExt& blk) { @@ -4657,33 +4640,34 @@ td::Result to_block_id(const tonlib_api::ton_blockIdExt& blk) { } td::Status TonlibClient::do_request(const tonlib_api::getConfigParam& request, - td::Promise>&& promise) { + td::Promise>&& promise) { TRY_RESULT(lite_block, to_lite_api(*request.id_)) auto block = create_block_id(std::move(lite_block)); auto param = request.param_; - std::vector params = { param }; + std::vector params = {param}; client_.send_query(ton::lite_api::liteServer_getConfigParams(0, std::move(lite_block), std::move(params)), promise.wrap([block, param](auto r_config) { - auto state = block::check_extract_state_proof(block, r_config->state_proof_.as_slice(), - r_config->config_proof_.as_slice()); - if (state.is_error()) { - LOG(ERROR) << "block::check_extract_state_proof failed: " << state.error(); - } - auto config = block::Config::extract_from_state(std::move(state.move_as_ok()), 0); - if (config.is_error()) { - LOG(ERROR) << "block::Config::extract_from_state failed: " << config.error(); - } - tonlib_api::configInfo config_result; - config_result.config_ = tonlib_api::make_object(to_bytes(config.move_as_ok()->get_config_param(param))); - return tonlib_api::make_object(std::move(config_result)); - })); + auto state = block::check_extract_state_proof(block, r_config->state_proof_.as_slice(), + r_config->config_proof_.as_slice()); + if (state.is_error()) { + LOG(ERROR) << "block::check_extract_state_proof failed: " << state.error(); + } + auto config = block::Config::extract_from_state(std::move(state.move_as_ok()), 0); + if (config.is_error()) { + LOG(ERROR) << "block::Config::extract_from_state failed: " << config.error(); + } + tonlib_api::configInfo config_result; + config_result.config_ = tonlib_api::make_object( + to_bytes(config.move_as_ok()->get_config_param(param))); + return tonlib_api::make_object(std::move(config_result)); + })); return td::Status::OK(); } td::Status TonlibClient::do_request(const tonlib_api::blocks_getMasterchainInfo& masterchain_info, - td::Promise>&& promise) { + td::Promise>&& promise) { client_.send_query(ton::lite_api::liteServer_getMasterchainInfo(), promise.wrap([](lite_api_ptr&& masterchain_info) { return tonlib_api::make_object( @@ -4728,133 +4712,126 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getShards& request, return td::Status::OK(); } - td::Status TonlibClient::do_request(const tonlib_api::blocks_lookupBlock& request, - td::Promise>&& promise) { + td::Promise>&& promise) { client_.send_query(ton::lite_api::liteServer_lookupBlock( - request.mode_, - ton::lite_api::make_object((*request.id_).workchain_, (*request.id_).shard_, (*request.id_).seqno_), - (td::uint64)(request.lt_), - (td::uint32)(request.utime_)), + request.mode_, + ton::lite_api::make_object( + (*request.id_).workchain_, (*request.id_).shard_, (*request.id_).seqno_), + (td::uint64)(request.lt_), (td::uint32)(request.utime_)), promise.wrap([](lite_api_ptr&& header) { - const auto& id = header->id_; - return to_tonlib_api(*id); - //tonlib_api::make_object( - // ton::tonlib_api::ton_blockIdExt(id->workchain_, id->) - //); + const auto& id = header->id_; + return to_tonlib_api(*id); + //tonlib_api::make_object( + // ton::tonlib_api::ton_blockIdExt(id->workchain_, id->) + //); })); return td::Status::OK(); } auto to_tonlib_api(const ton::lite_api::liteServer_transactionId& txid) -> tonlib_api_ptr { - return tonlib_api::make_object( - txid.mode_, txid.account_.as_slice().str(), txid.lt_, txid.hash_.as_slice().str()); + return tonlib_api::make_object(txid.mode_, txid.account_.as_slice().str(), txid.lt_, + txid.hash_.as_slice().str()); } td::Status TonlibClient::do_request(const tonlib_api::blocks_getTransactions& request, - td::Promise>&& promise) { + td::Promise>&& promise) { TRY_RESULT(block, to_lite_api(*request.id_)) TRY_RESULT(account, to_bits256((*request.after_).account_, "account")); auto after = ton::lite_api::make_object(account, (*request.after_).lt_); - client_.send_query(ton::lite_api::liteServer_listBlockTransactions( - std::move(block), - request.mode_, - request.count_, - std::move(after), - false, - false), + client_.send_query(ton::lite_api::liteServer_listBlockTransactions(std::move(block), request.mode_, request.count_, + std::move(after), false, false), promise.wrap([](lite_api_ptr&& bTxes) { - const auto& id = bTxes->id_; - //for (auto id : ids) { - tonlib_api::blocks_transactions r; - r.id_ = to_tonlib_api(*id); - r.req_count_ = bTxes->req_count_; - r.incomplete_ = bTxes->incomplete_; - for (auto& id: bTxes->ids_) { - //tonlib_api::blocks_shortTxId txid = tonlib_api::blocks_shortTxId(id->mode_, id->account_.as_slice().str(), id->lt_, id->hash_.as_slice().str()); - //r.transactions_.push_back(txid); - r.transactions_.push_back(to_tonlib_api(*id)); - } - return tonlib_api::make_object(std::move(r)); + const auto& id = bTxes->id_; + //for (auto id : ids) { + tonlib_api::blocks_transactions r; + r.id_ = to_tonlib_api(*id); + r.req_count_ = bTxes->req_count_; + r.incomplete_ = bTxes->incomplete_; + for (auto& id : bTxes->ids_) { + //tonlib_api::blocks_shortTxId txid = tonlib_api::blocks_shortTxId(id->mode_, id->account_.as_slice().str(), id->lt_, id->hash_.as_slice().str()); + //r.transactions_.push_back(txid); + r.transactions_.push_back(to_tonlib_api(*id)); + } + return tonlib_api::make_object(std::move(r)); })); return td::Status::OK(); } td::Status TonlibClient::do_request(const tonlib_api::blocks_getBlockHeader& request, - td::Promise>&& promise) { + td::Promise>&& promise) { TRY_RESULT(block, to_lite_api(*request.id_)) - client_.send_query(ton::lite_api::liteServer_getBlockHeader( - std::move(block), - 0xffff), - promise.wrap([](lite_api_ptr&& hdr) { - auto blk_id = ton::create_block_id(hdr->id_); - auto R = vm::std_boc_deserialize(std::move(hdr->header_proof_)); - tonlib_api::blocks_header header; - if (R.is_error()) { - LOG(WARNING) << "R.is_error() "; - } else { - auto root = R.move_as_ok(); - try { - ton::RootHash vhash{root->get_hash().bits()}; - auto virt_root = vm::MerkleProof::virtualize(root, 1); - if (virt_root.is_null()) { - LOG(WARNING) << "virt root is null"; - } else { - std::vector prev; - ton::BlockIdExt mc_blkid; - bool after_split; - auto res = block::unpack_block_prev_blk_ext(virt_root, blk_id, prev, mc_blkid, after_split); - if (res.is_error()) { - LOG(WARNING) << "res.is_error() "; - } else { - block::gen::Block::Record blk; - block::gen::BlockInfo::Record info; - if (!(tlb::unpack_cell(virt_root, blk) && tlb::unpack_cell(blk.info, info))) { - LOG(WARNING) << "unpack failed"; - } else { - header.id_ = to_tonlib_api(blk_id); - header.global_id_ = blk.global_id; - header.version_ = info.version; - header.flags_ = info.flags; - header.after_merge_ = info.after_merge; - header.after_split_ = info.after_split; - header.before_split_ = info.before_split; - header.want_merge_ = info.want_merge; - header.want_split_ = info.want_split; - header.validator_list_hash_short_ = info.gen_validator_list_hash_short; - header.catchain_seqno_ = info.gen_catchain_seqno; - header.min_ref_mc_seqno_ = info.min_ref_mc_seqno; - header.start_lt_ = info.start_lt; - header.end_lt_ = info.end_lt; - header.gen_utime_ = info.gen_utime; - header.is_key_block_ = info.key_block; - header.vert_seqno_ = info.vert_seq_no; - if(!info.not_master) { - header.prev_key_block_seqno_ = info.prev_key_block_seqno; - } - for (auto id : prev) { - header.prev_blocks_.push_back(to_tonlib_api(id)); - } - //if(info.before_split) { - //} else { - //} - return tonlib_api::make_object(std::move(header)); - } - } - } - } catch (vm::VmError& err) { - auto E = err.as_status(PSLICE() << "error processing header for " << blk_id.to_str() << " :"); - LOG(ERROR) << std::move(E); - } catch (vm::VmVirtError& err) { - auto E = err.as_status(PSLICE() << "error processing header for " << blk_id.to_str() << " :"); - LOG(ERROR) << std::move(E); - } catch (...) { - LOG(WARNING) << "exception catched "; - } - } - return tonlib_api::make_object(std::move(header)); - })); + client_.send_query( + ton::lite_api::liteServer_getBlockHeader(std::move(block), 0xffff), + promise.wrap([](lite_api_ptr&& hdr) { + auto blk_id = ton::create_block_id(hdr->id_); + auto R = vm::std_boc_deserialize(std::move(hdr->header_proof_)); + tonlib_api::blocks_header header; + if (R.is_error()) { + LOG(WARNING) << "R.is_error() "; + } else { + auto root = R.move_as_ok(); + try { + ton::RootHash vhash{root->get_hash().bits()}; + auto virt_root = vm::MerkleProof::virtualize(root, 1); + if (virt_root.is_null()) { + LOG(WARNING) << "virt root is null"; + } else { + std::vector prev; + ton::BlockIdExt mc_blkid; + bool after_split; + auto res = block::unpack_block_prev_blk_ext(virt_root, blk_id, prev, mc_blkid, after_split); + if (res.is_error()) { + LOG(WARNING) << "res.is_error() "; + } else { + block::gen::Block::Record blk; + block::gen::BlockInfo::Record info; + if (!(tlb::unpack_cell(virt_root, blk) && tlb::unpack_cell(blk.info, info))) { + LOG(WARNING) << "unpack failed"; + } else { + header.id_ = to_tonlib_api(blk_id); + header.global_id_ = blk.global_id; + header.version_ = info.version; + header.flags_ = info.flags; + header.after_merge_ = info.after_merge; + header.after_split_ = info.after_split; + header.before_split_ = info.before_split; + header.want_merge_ = info.want_merge; + header.want_split_ = info.want_split; + header.validator_list_hash_short_ = info.gen_validator_list_hash_short; + header.catchain_seqno_ = info.gen_catchain_seqno; + header.min_ref_mc_seqno_ = info.min_ref_mc_seqno; + header.start_lt_ = info.start_lt; + header.end_lt_ = info.end_lt; + header.gen_utime_ = info.gen_utime; + header.is_key_block_ = info.key_block; + header.vert_seqno_ = info.vert_seq_no; + if (!info.not_master) { + header.prev_key_block_seqno_ = info.prev_key_block_seqno; + } + for (auto id : prev) { + header.prev_blocks_.push_back(to_tonlib_api(id)); + } + //if(info.before_split) { + //} else { + //} + return tonlib_api::make_object(std::move(header)); + } + } + } + } catch (vm::VmError& err) { + auto E = err.as_status(PSLICE() << "error processing header for " << blk_id.to_str() << " :"); + LOG(ERROR) << std::move(E); + } catch (vm::VmVirtError& err) { + auto E = err.as_status(PSLICE() << "error processing header for " << blk_id.to_str() << " :"); + LOG(ERROR) << std::move(E); + } catch (...) { + LOG(WARNING) << "exception catched "; + } + } + return tonlib_api::make_object(std::move(header)); + })); return td::Status::OK(); } @@ -4890,15 +4867,17 @@ void TonlibClient::load_libs_from_disk() { if (r_dict.is_error()) { return; } - libraries = vm::Dictionary(vm::load_cell_slice(vm::CellBuilder().append_cellslice(vm::load_cell_slice( - r_dict.move_as_ok())).finalize()), 256); + libraries = vm::Dictionary( + vm::load_cell_slice(vm::CellBuilder().append_cellslice(vm::load_cell_slice(r_dict.move_as_ok())).finalize()), + 256); // int n = 0; for (auto&& lr : libraries) n++; LOG(DEBUG) << "loaded libraries from disk cache"; } void TonlibClient::store_libs_to_disk() { // NB: Dictionary.get_root_cell does not compute_root, and it is protected - kv_->set("tonlib.libcache", vm::std_boc_serialize(vm::CellBuilder().append_cellslice(libraries.get_root()) - .finalize()).move_as_ok().as_slice()); + kv_->set("tonlib.libcache", vm::std_boc_serialize(vm::CellBuilder().append_cellslice(libraries.get_root()).finalize()) + .move_as_ok() + .as_slice()); // int n = 0; for (auto&& lr : libraries) n++; LOG(DEBUG) << "stored libraries to disk cache"; } diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index 9430b6725..1f4d8f5b4 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -47,7 +47,7 @@ #include "tonlib/TonlibClient.h" #include "tonlib/TonlibCallback.h" -#include "tonlib/ExtClientRaw.h" +#include "tonlib/ExtClientLazy.h" #include "smc-envelope/ManualDns.h" #include "smc-envelope/PaymentChannel.h" @@ -62,7 +62,6 @@ #include #include #include "git.h" -#include "ExtClientMulti.h" using tonlib_api::make_object; @@ -224,7 +223,7 @@ class TonlibCli : public td::actor::Actor { if (options_.use_callbacks_for_network) { auto config = tonlib::Config::parse(options_.config).move_as_ok(); - class Callback : public tonlib::ExtClientRaw::Callback { + class Callback : public tonlib::ExtClientLazy::Callback { public: explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { } @@ -232,28 +231,15 @@ class TonlibCli : public td::actor::Actor { private: td::actor::ActorShared<> parent_; }; - std::vector> full_servers; - int lite_client_id = -1, cnt = 0; - for (const auto& s : config.lite_clients) { - if (s.is_full) { - full_servers.emplace_back(s.adnl_id, s.address); - } - } - if (!full_servers.empty()) { - raw_client_ = tonlib::ExtClientRaw::create(std::move(full_servers), - td::make_unique(td::actor::actor_shared())); - } else { - CHECK(!config.lite_clients.empty()); - raw_client_ = td::actor::create_actor( - "ExtClientMulti", config.lite_clients, td::make_unique(td::actor::actor_shared())); - } ref_cnt_++; + raw_client_ = tonlib::ExtClientLazy::create(config.lite_servers, + td::make_unique(td::actor::actor_shared())); } auto config = !options_.config.empty() - ? make_object(options_.config, options_.name, - options_.use_callbacks_for_network, options_.ignore_cache) - : nullptr; + ? make_object(options_.config, options_.name, + options_.use_callbacks_for_network, options_.ignore_cache) + : nullptr; tonlib_api::object_ptr ks_type; if (options_.in_memory) { @@ -1544,9 +1530,8 @@ class TonlibCli : public td::actor::Actor { auto update = tonlib_api::move_object_as(std::move(result)); CHECK(!raw_client_.empty()); snd_bytes_ += update->data_.size(); - ton::ShardIdFull shard(update->workchain_, update->shard_); - send_closure(raw_client_, &tonlib::ExtClientRaw::send_query, "query", td::BufferSlice(update->data_), shard, - td::Timestamp::in(5), + send_closure(raw_client_, &tonlib::ExtClientLazy::send_query, "query", td::BufferSlice(update->data_), + ton::ShardIdFull(update->workchain_, update->shard_), td::Timestamp::in(5), [actor_id = actor_id(this), id = update->id_](td::Result res) { send_closure(actor_id, &TonlibCli::on_adnl_result, id, std::move(res)); }); From 29851c38ef91ce42215f5e8965084c9a3f9aca7c Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 12 Jan 2023 17:32:59 +0300 Subject: [PATCH 045/388] Various changes after merge --- crypto/block/mc-config.cpp | 2 +- overlay/overlay-manager.cpp | 34 ++++++++---------- overlay/overlay-manager.h | 1 + overlay/overlay.cpp | 2 +- overlay/overlay.h | 1 + overlay/overlay.hpp | 6 +++- overlay/overlays.h | 5 ++- tl/generate/scheme/ton_api.tl | 4 +-- tl/generate/scheme/ton_api.tlo | Bin 87748 -> 87728 bytes tonlib/tonlib/QueryTraits.h | 14 ++++++++ .../validator-engine-console.cpp | 2 ++ validator/full-node-shard.cpp | 25 ++++++++----- validator/full-node-shard.h | 2 +- validator/full-node-shard.hpp | 1 + validator/full-node.cpp | 12 +++---- validator/manager.cpp | 9 +++++ validator/validator-group.cpp | 29 ++++++++------- 17 files changed, 93 insertions(+), 56 deletions(-) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 31826acf0..8d592c210 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2203,7 +2203,7 @@ Ref ConfigInfo::lookup_library(td::ConstBitPtr root_hash) const { CollatorConfig Config::get_collator_config(bool need_collator_nodes) const { CollatorConfig collator_config; gen::CollatorConfig::Record rec; - auto cell = get_config_param(-41); + auto cell = get_config_param(41, -41); if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { return collator_config; } diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index b7ca1270e..f9a798781 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -105,16 +105,6 @@ void OverlayManager::create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, Ov std::move(callback), std::move(rules), scope, announce_self)); } -void OverlayManager::create_public_overlay_external(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, - std::unique_ptr callback, OverlayPrivacyRules rules, - td::string scope) { - CHECK(!dht_node_.empty()); - auto id = overlay_id.compute_short_id(); - register_overlay(local_id, id, - Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), - std::move(callback), std::move(rules), scope, true)); -} - void OverlayManager::create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules) { @@ -162,23 +152,16 @@ void OverlayManager::receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdSh auto M = R.move_as_ok(); - OverlayIdShort overlay_id{M->overlay_}; - auto send_remove_peer = [&, this]() { - send_message(src, dst, overlay_id, create_serialize_tl_object()); - }; - auto it = overlays_.find(dst); if (it == overlays_.end()) { VLOG(OVERLAY_NOTICE) << this << ": query to unknown overlay " << M->overlay_ << "@" << dst << " from " << src; promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad local_id " << dst)); - send_remove_peer(); return; } - auto it2 = it->second.find(overlay_id); + auto it2 = it->second.find(OverlayIdShort{M->overlay_}); if (it2 == it->second.end()) { VLOG(OVERLAY_NOTICE) << this << ": query to localid not in overlay " << M->overlay_ << "@" << dst << " from " << src; promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad overlay_id " << M->overlay_)); - send_remove_peer(); return; } @@ -380,6 +363,19 @@ void OverlayManager::set_priority_broadcast_receivers(adnl::AdnlNodeIdShort loca td::actor::send_closure(it2->second, &Overlay::set_priority_broadcast_receivers, std::move(nodes)); } +void OverlayManager::forget_peer(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, + adnl::AdnlNodeIdShort peer_id) { + auto it = overlays_.find(local_id); + if (it == overlays_.end()) { + return; + } + auto it2 = it->second.find(overlay); + if (it2 == it->second.end()) { + return; + } + td::actor::send_closure(it2->second, &Overlay::forget_peer, peer_id); +} + Certificate::Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, td::BufferSlice signature) : issued_by_(issued_by) @@ -421,7 +417,7 @@ td::BufferSlice Certificate::to_sign(OverlayIdShort overlay_id, PublicKeyHash is } } -PublicKeyHash Certificate::issuer_hash() const { +const PublicKeyHash Certificate::issuer_hash() const { PublicKeyHash r; issued_by_.visit( td::overloaded([&](const PublicKeyHash &x) { r = x; }, [&](const PublicKey &x) { r = x.compute_short_id(); })); diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h index f7ed32bac..5b59b3715 100644 --- a/overlay/overlay-manager.h +++ b/overlay/overlay-manager.h @@ -98,6 +98,7 @@ class OverlayManager : public Overlays { void set_priority_broadcast_receivers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, std::vector nodes) override; + void forget_peer(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, adnl::AdnlNodeIdShort peer_id) override; struct PrintId {}; diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index f0bd3b9ca..c4cf8428b 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -231,7 +231,7 @@ void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice dat return; } auto Q = X.move_as_ok(); - ton_api::downcast_call(*Q, [Self = this, &Q, &src](auto &object) { + ton_api::downcast_call(*Q.get(), [Self = this, &Q, &src](auto &object) { Self->process_broadcast(src, move_tl_object_as>(Q)); }); } diff --git a/overlay/overlay.h b/overlay/overlay.h index 4289142cb..5141ae538 100644 --- a/overlay/overlay.h +++ b/overlay/overlay.h @@ -70,6 +70,7 @@ class Overlay : public td::actor::Actor { //virtual void receive_broadcast(td::BufferSlice data) = 0; //virtual void subscribe(std::unique_ptr callback) = 0; virtual void set_priority_broadcast_receivers(std::vector nodes) = 0; + virtual void forget_peer(adnl::AdnlNodeIdShort peer_id) = 0; }; } // namespace overlay diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 208aba7f3..a8fd7f303 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -278,7 +278,11 @@ class OverlayImpl : public Overlay { priority_broadcast_receivers_ = std::move(nodes); } - void wait_neighbours_not_empty(td::Promise promise, int max_retries = 10); + void forget_peer(adnl::AdnlNodeIdShort peer_id) override { + del_peer(peer_id); + } + + void wait_neighbours_not_empty(td::Promise promise, int max_retries = 20); private: template diff --git a/overlay/overlays.h b/overlay/overlays.h index def80d758..1141efee2 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -145,7 +145,7 @@ class Certificate { bool is_fec) const; tl_object_ptr tl() const; const PublicKey &issuer() const; - PublicKeyHash issuer_hash() const; + const PublicKeyHash issuer_hash() const; static td::Result> create(tl_object_ptr cert); static tl_object_ptr empty_tl(); @@ -170,8 +170,6 @@ class Overlays : public td::actor::Actor { td::Promise promise) { promise.set_value(td::Unit()); } - virtual void on_remove_peer(adnl::AdnlNodeIdShort src) { - } virtual ~Callback() = default; }; @@ -242,6 +240,7 @@ class Overlays : public td::actor::Actor { virtual void set_priority_broadcast_receivers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, std::vector nodes) = 0; + virtual void forget_peer(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, adnl::AdnlNodeIdShort peer_id) = 0; }; } // namespace overlay diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 004b1dd29..08d4c9dfb 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -218,8 +218,6 @@ overlay.message overlay:int256 = overlay.Message; //overlay.randomPeers peers:(vector adnl.node) = overlay.RandomPeers; overlay.broadcastList hashes:(vector int256) = overlay.BroadcastList; -overlay.message.removePeer = overlay.message.InternalMessage; - overlay.fec.received hash:int256 = overlay.Broadcast; overlay.fec.completed hash:int256 = overlay.Broadcast; @@ -424,6 +422,8 @@ tonNode.archiveInfo id:long = tonNode.ArchiveInfo; tonNode.outMsgQueueProof queue_proof:bytes block_state_proof:bytes = tonNode.OutMsgQueueProof; tonNode.outMsgQueueProofEmpty = tonNode.OutMsgQueueProof; +tonNode.forgetPeer = tonNode.ForgetPeer; + ---functions--- tonNode.getNextBlockDescription prev_block:tonNode.blockIdExt = tonNode.BlockDescription; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 97c2bda2df4125bec8281603581b892696097361..9dc8c51b2a5dad856a6abbf39aeedffcc767d724 100644 GIT binary patch delta 106 zcmX@ImUROV<;ZVdA)ml2-uOjnwNOcZo?m`Ss-9bZQF>}gKx%5ylnhOx*lu mUyq>yimEiYDh81F&0|773_wui%%;O(+jbBnwz=kmM;ZV##4DTt delta 120 zcmdn6mi5S5)(tuGvQK8N{wkkemRgjPSgDtrT3no%o~q}WSCU$kmzXp8psf7n9Qg#^ z&3|+@Otb*0l|olrl$r|^3P?>YVgSkCJSODB00c$OY&smaZ3jVOn-}bxVYs>EgH0L$ DeYiDE diff --git a/tonlib/tonlib/QueryTraits.h b/tonlib/tonlib/QueryTraits.h index bf480f64c..b583cd05b 100644 --- a/tonlib/tonlib/QueryTraits.h +++ b/tonlib/tonlib/QueryTraits.h @@ -114,6 +114,13 @@ struct QueryTraits { } }; +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getAccountStatePrunned& q) { + return ton::AccountIdPrefixFull(q.account_->workchain_, q.account_->id_.bits().get_uint(64)).as_leaf_shard(); + } +}; + template<> struct QueryTraits { static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_runSmcMethod& q) { @@ -198,5 +205,12 @@ struct QueryTraits { } }; +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getShardBlockProof& q) { + return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); + } +}; + } // namespace tonlib diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index a1a0135b5..97e7690a3 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -144,6 +144,8 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 35a9b8518..69804a07a 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -88,7 +88,7 @@ void FullNodeShardImpl::create_overlay() { public: void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { - // just ignore + td::actor::send_closure(node_, &FullNodeShardImpl::receive_message, src, std::move(data)); } void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, td::Promise promise) override { @@ -101,26 +101,23 @@ void FullNodeShardImpl::create_overlay() { td::Promise promise) override { td::actor::send_closure(node_, &FullNodeShardImpl::check_broadcast, src, std::move(data), std::move(promise)); } - void on_remove_peer(adnl::AdnlNodeIdShort src) override { - td::actor::send_closure(node_, &FullNodeShardImpl::remove_neighbour, src); - } Callback(td::actor::ActorId node) : node_(node) { } private: td::actor::ActorId node_; }; - if (is_active()) { td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay, adnl_id_, overlay_id_full_.clone(), std::make_unique(actor_id(this)), rules_, PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() << ", \"workchain_id\": " << get_workchain() << " }"); } else { - td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay_external, adnl_id_, - overlay_id_full_.clone(), std::make_unique(actor_id(this)), rules_, + td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay_ex, adnl_id_, overlay_id_full_.clone(), + std::make_unique(actor_id(this)), rules_, PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() - << ", \"workchain_id\": " << get_workchain() << " }"); + << ", \"workchain_id\": " << get_workchain() << " }", + false); } td::actor::send_closure(rldp_, &rldp::Rldp::add_id, adnl_id_); @@ -647,6 +644,8 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise) { if (!is_active()) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_message, src, adnl_id_, overlay_id_, + create_serialize_tl_object()); promise.set_error(td::Status::Error("shard is inactive")); return; } @@ -658,6 +657,16 @@ void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice ton_api::downcast_call(*B.move_as_ok().get(), [&](auto &obj) { this->process_query(src, obj, std::move(promise)); }); } +void FullNodeShardImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) { + auto B = fetch_tl_object(std::move(data), true); + if (B.is_error()) { + return; + } + VLOG(FULL_NODE_DEBUG) << "Got tonNode.forgetPeer from " << src; + neighbours_.erase(src); + td::actor::send_closure(overlays_, &overlay::Overlays::forget_peer, adnl_id_, overlay_id_, src); +} + void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ihrMessageBroadcast &query) { td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_ihr_message, std::move(query.message_->data_)); diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 847091c65..633c93409 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -31,7 +31,7 @@ namespace fullnode { enum FullNodeShardMode { active, // Node can answer queries about the shard active_temp, // Like 'active', but queries about shard state are not allowed (only blocks) - inactive // Node is not a part of the overlay (overlay is_external) + inactive // Node is not a part of the overlay }; diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 1cb2957de..7d80792bf 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -142,6 +142,7 @@ class FullNodeShardImpl : public FullNodeShard { // void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareNextKeyBlockProof &query, // td::Promise promise); void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise); + void receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_ihrMessageBroadcast &query); diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 72af6fe8e..42e8511fe 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -17,10 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "full-node.hpp" -#include "ton/ton-shard.h" #include "ton/ton-io.hpp" #include "td/actor/MultiPromise.h" -#include "ton/ton-types.h" namespace ton { @@ -140,12 +138,8 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s new_shards.insert(ShardIdFull(masterchainId)); std::set workchains; auto cut_shard = [&](ShardIdFull shard) -> ShardIdFull { - unsigned pfx_len = shard.pfx_len(); - unsigned min_split = state->soft_min_split_depth(shard.workchain); - if (min_split < pfx_len) { - return shard_prefix(shard, min_split); - } - return shard; + int min_split = state->soft_min_split_depth(shard.workchain); + return min_split < shard.pfx_len() ? shard_prefix(shard, min_split) : shard; }; auto set_active = [&](ShardIdFull shard, FullNodeShardMode mode) { while (new_active.emplace(shard, mode).second && shard.pfx_len() > 0) { @@ -388,6 +382,7 @@ void FullNodeImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull } td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { + ShardIdFull shard0 = shard; while (true) { auto it = shards_.find(shard); if (it != shards_.end() && it->second.exists) { @@ -404,6 +399,7 @@ td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { } shard = shard_parent(shard); } + shard = shard0; auto it = shards_.find(shard); if (it == shards_.end()) { it = shards_.emplace(shard = ShardIdFull(shard.workchain), ShardInfo{}).first; diff --git a/validator/manager.cpp b/validator/manager.cpp index 3015e8827..c19e96f41 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -583,6 +583,10 @@ void ValidatorManagerImpl::run_ext_query(td::BufferSlice data, td::Promise> promise) { + if (last_masterchain_state_.not_null() && !opts_->need_monitor(handle->id().shard_full(), last_masterchain_state_)) { + return promise.set_error( + td::Status::Error(PSTRING() << "not monitoring shard " << handle->id().shard_full().to_str())); + } auto it = wait_state_.find(handle->id()); if (it == wait_state_.end()) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { @@ -678,6 +682,10 @@ void ValidatorManagerImpl::wait_block_data_short(BlockIdExt block_id, td::uint32 void ValidatorManagerImpl::wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) { + if (last_masterchain_state_.not_null() && !opts_->need_monitor(left_id.shard_full(), last_masterchain_state_)) { + return promise.set_error( + td::Status::Error(PSTRING() << "not monitoring shard " << left_id.shard_full().to_str())); + } td::actor::create_actor("merge", left_id, right_id, priority, actor_id(this), timeout, std::move(promise)) .release(); @@ -1604,6 +1612,7 @@ void ValidatorManagerImpl::started(ValidatorManagerInitResult R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::read_gc_list, R.move_as_ok()); } }); + td::actor::send_closure(db_, &Db::get_destroyed_validator_sessions, std::move(P)); auto Q = td::PromiseCreator::lambda( diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 601fc5264..b4f665c39 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -139,6 +139,13 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat collator_config_.full_collated_data ? ValidateMode::full_collated_data : 0); } +void ValidatorGroup::update_approve_cache(td::uint32 round_id, CacheKey key, UnixTime value) { + if (approved_candidates_cache_round_ != round_id) { + return; + } + approved_candidates_cache_[key] = value; +} + void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash src, td::BufferSlice block_data, RootHash root_hash, FileHash file_hash, std::vector signatures, @@ -382,8 +389,8 @@ void ValidatorGroup::get_session_info( td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_session_info, std::move(P)); } -void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeout, - td::Promise promise, unsigned max_retries) { +void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeout, td::Promise promise, + unsigned max_retries) { if (round_id < last_known_round_id_) { promise.set_error(td::Status::Error("too old")); return; @@ -391,9 +398,11 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo adnl::AdnlNodeIdShort collator = adnl::AdnlNodeIdShort::zero(); // TODO: some way to choose node (similar to "unreliability" in full-node) int cnt = 0; - for (const block::CollatorNodeDescr& c : collator_config_.collator_nodes) { - if (shard_intersects(shard_, c.shard) && td::Random::fast(0, cnt) == 0) { - collator = adnl::AdnlNodeIdShort(c.adnl_id); + for (const block::CollatorNodeDescr &c : collator_config_.collator_nodes) { + if (shard_intersects(shard_, c.shard)) { + if (td::Random::fast(0, cnt) == 0) { + collator = adnl::AdnlNodeIdShort(c.adnl_id); + } ++cnt; } } @@ -438,8 +447,8 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo size_t max_answer_size = config_.max_block_size + config_.max_collated_data_size + 256; td::Timestamp query_timeout = td::Timestamp::in(10.0); query_timeout.relax(timeout); - td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_adnl_id_, collator, "collatequery", - std::move(P), timeout, std::move(query), max_answer_size); + td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_adnl_id_, collator, "collatequery", std::move(P), + timeout, std::move(query), max_answer_size); } void ValidatorGroup::receive_collate_query_response(td::uint32 round_id, td::BufferSlice data, @@ -460,15 +469,11 @@ void ValidatorGroup::receive_collate_query_response(td::uint32 round_id, td::Buf return; } auto key = PublicKey{b->source_}; - if (!key.is_ed25519()) { + if (key != local_id_full_) { promise.set_error(td::Status::Error("collate query: block candidate source mismatch")); return; } auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; - if (e_key != Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}) { - promise.set_error(td::Status::Error("collate query: block candidate source mismatch")); - return; - } auto block_id = ton::create_block_id(b->id_); if (block_id.shard_full() != shard_) { promise.set_error(td::Status::Error("collate query: shard mismatch")); From d324fa58208e69179b473ccb1cc2a8fc1c5e6ce0 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 13 Jan 2023 19:04:11 +0300 Subject: [PATCH 046/388] Bugfix --- overlay/overlay-peers.cpp | 4 +++- validator/impl/out-msg-queue-proof.cpp | 10 +++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index 409f09935..28dac8f05 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -24,7 +24,9 @@ namespace overlay { void OverlayImpl::del_peer(adnl::AdnlNodeIdShort id) { auto P = peers_.get(id); - CHECK(P != nullptr); + if (P == nullptr) { + return; + } VLOG(OVERLAY_DEBUG) << this << ": deleting peer " << id; diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 4ba102c53..7591ab179 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -177,7 +177,7 @@ void WaitOutMsgQueueProof::run_local() { [SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::abort_query, - R.move_as_error_prefix("failed to get shard state")); + R.move_as_error_prefix("failed to get shard state: ")); } else { td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::got_state_root, R.move_as_ok()->root_cell()); @@ -189,7 +189,7 @@ void WaitOutMsgQueueProof::run_local() { [SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::abort_query, - R.move_as_error_prefix("failed to get block data")); + R.move_as_error_prefix("failed to get block data: ")); } else { td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::got_block_root, R.move_as_ok()->root_cell()); @@ -247,7 +247,7 @@ void WaitOutMsgQueueProof::run_net() { void BuildOutMsgQueueProof::abort_query(td::Status reason) { if (promise_) { - LOG(WARNING) << "failed to build msg queue proof for " << block_id_.to_str() << ": " << reason; + LOG(DEBUG) << "failed to build msg queue proof for " << block_id_.to_str() << ": " << reason; promise_.set_error( reason.move_as_error_prefix(PSTRING() << "failed to build msg queue proof for " << block_id_.to_str() << ": ")); } @@ -260,7 +260,7 @@ void BuildOutMsgQueueProof::start_up() { [SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::abort_query, - R.move_as_error_prefix("failed to get shard state")); + R.move_as_error_prefix("failed to get shard state: ")); } else { td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_state_root, R.move_as_ok()->root_cell()); @@ -272,7 +272,7 @@ void BuildOutMsgQueueProof::start_up() { [SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::abort_query, - R.move_as_error_prefix("failed to get block data")); + R.move_as_error_prefix("failed to get block data: ")); } else { td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_block_root, R.move_as_ok()->root_cell()); From 225f71238b61694ca42fc7af95415098b9914d0e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 16 Jan 2023 19:30:15 +0300 Subject: [PATCH 047/388] Option for collating masterchain on validators --- validator-engine/validator-engine.cpp | 19 ++++++++++++---- validator-engine/validator-engine.hpp | 7 +++--- validator/collator-node.cpp | 26 +++++++++++----------- validator/manager.cpp | 31 +++++++++++++++++++-------- validator/manager.hpp | 1 + validator/validator-group.cpp | 6 ++++-- validator/validator-group.hpp | 7 +++--- validator/validator-options.hpp | 10 ++++----- validator/validator.h | 6 ++++-- 9 files changed, 72 insertions(+), 41 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index cd14ae1ed..4ed6ae56b 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1419,7 +1419,7 @@ td::Status ValidatorEngine::load_global_config() { h.push_back(b); } validator_options_.write().set_hardforks(std::move(h)); - validator_options_.write().set_validator_lite_mode(validator_lite_mode_); + validator_options_.write().set_validator_mode(validator_mode_); return td::Status::OK(); } @@ -3866,9 +3866,20 @@ int main(int argc, char *argv[]) { p.add_option('M', "not-all-shards", "monitor only a necessary set of shards instead of all", [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_not_all_shards); }); }); - p.add_option('\0', "lite-validator", "lite-mode validator (don't collate blocks, use collator nodes instead)", [&]() { - acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_validator_lite_mode); }); - }); + p.add_option('\0', "lite-validator-shards", + "lite-mode validator for shard blocks (don't collate blocks, use collator nodes instead)", [&]() { + acts.push_back([&x]() { + td::actor::send_closure(x, &ValidatorEngine::set_validator_mode, + ton::validator::ValidatorManagerOptions::validator_lite_shards); + }); + }); + p.add_option('\0', "lite-validator-all", + "lite-mode validator for all blocks (don't collate blocks, use collator nodes instead)", [&]() { + acts.push_back([&x]() { + td::actor::send_closure(x, &ValidatorEngine::set_validator_mode, + ton::validator::ValidatorManagerOptions::validator_lite_all); + }); + }); td::uint32 threads = 7; p.add_checked_option( 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice fname) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 49010bc47..7e0db6614 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -217,7 +217,8 @@ class ValidatorEngine : public td::actor::Actor { ton::BlockSeqno truncate_seqno_{0}; std::string session_logs_file_; bool not_all_shards_ = false; - bool validator_lite_mode_ = false; + ton::validator::ValidatorManagerOptions::ValidatorMode validator_mode_ = + ton::validator::ValidatorManagerOptions::validator_normal; std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; @@ -272,8 +273,8 @@ class ValidatorEngine : public td::actor::Actor { void set_not_all_shards() { not_all_shards_ = true; } - void set_validator_lite_mode() { - validator_lite_mode_ = true; + void set_validator_mode(ton::validator::ValidatorManagerOptions::ValidatorMode mode) { + validator_mode_ = mode; } void start_up() override; diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index c6afa5418..193613c74 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -160,19 +160,19 @@ void CollatorNode::receive_query_cont(adnl::AdnlNodeIdShort src, ShardIdFull sha td::Ref min_mc_state, std::vector prev_blocks, Ed25519_PublicKey creator, td::Promise promise) { auto P = td::PromiseCreator::lambda([promise = std::move(promise), src](td::Result R) mutable { - if (R.is_error()) { - LOG(WARNING) << "Query from " << src << ", error: " << R.error(); - promise.set_result(serialize_error(R.move_as_error())); - } else { - LOG(INFO) << "Query from " << src << ", success"; - auto block = R.move_as_ok(); - auto result = create_serialize_tl_object( - create_tl_object(PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), - create_tl_block_id(block.id), std::move(block.data), - std::move(block.collated_data))); - promise.set_result(std::move(result)); - } - }); + if (R.is_error()) { + LOG(WARNING) << "Query from " << src << ", error: " << R.error(); + promise.set_result(serialize_error(R.move_as_error())); + } else { + LOG(INFO) << "Query from " << src << ", success"; + auto block = R.move_as_ok(); + auto result = create_serialize_tl_object( + create_tl_object(PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), + create_tl_block_id(block.id), std::move(block.data), + std::move(block.collated_data))); + promise.set_result(std::move(result)); + } + }); run_collate_query(shard, min_mc_state->get_block_id(), std::move(prev_blocks), creator, min_mc_state->get_validator_set(shard), manager_, td::Timestamp::in(10.0), std::move(P)); diff --git a/validator/manager.cpp b/validator/manager.cpp index c19e96f41..07266fb14 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -18,11 +18,9 @@ */ #include "manager.hpp" #include "validator-group.hpp" -#include "adnl/utils.hpp" #include "downloaders/wait-block-state.hpp" #include "downloaders/wait-block-state-merge.hpp" #include "downloaders/wait-block-data.hpp" -#include "validator-group.hpp" #include "fabric.h" #include "manager.h" #include "validate-broadcast.hpp" @@ -199,7 +197,7 @@ void ValidatorManagerImpl::validate_block(ReceivedBlock block, td::Promise R) mutable { + [SelfId = actor_id(this), promise = std::move(promise), id = blkid](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); } else { @@ -1241,7 +1239,7 @@ void ValidatorManagerImpl::write_handle(BlockHandle handle, td::Promise promise) { bool received = handle->received(); bool inited_state = handle->received_state(); - bool inited_proof = handle->id().is_masterchain() ? handle->inited_proof() : handle->inited_proof(); + bool inited_proof = handle->inited_proof(); if (handle->need_flush()) { handle->flush(actor_id(this), handle, std::move(promise)); @@ -1712,13 +1710,14 @@ bool ValidatorManagerImpl::out_of_sync() { void ValidatorManagerImpl::prestart_sync() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); - td::actor::send_closure(SelfId, &ValidatorManagerImpl::download_next_archive); + // Don't download archives + td::actor::send_closure(SelfId, &ValidatorManagerImpl::finish_prestart_sync); }); td::actor::send_closure(db_, &Db::set_async_mode, false, std::move(P)); } void ValidatorManagerImpl::download_next_archive() { - if (true) { + if (!out_of_sync()) { finish_prestart_sync(); return; } @@ -1836,6 +1835,13 @@ void ValidatorManagerImpl::new_masterchain_block() { for (auto &c : collator_nodes_) { td::actor::send_closure(c.second.actor, &CollatorNode::new_masterchain_block_notification, last_masterchain_state_); } + if (opts_->validator_mode() == ValidatorManagerOptions::validator_lite_shards && validating_masterchain()) { + // Prepare neighboours' queues for collating masterchain + for (const auto &desc : last_masterchain_state_->get_shards()) { + wait_out_msg_queue_proof(desc->top_block_id(), ShardIdFull(masterchainId), 0, td::Timestamp::in(10.0), + [](td::Ref) {}); + } + } if (last_masterchain_seqno_ % 1024 == 0) { LOG(WARNING) << "applied masterchain block " << last_masterchain_block_id_; @@ -2174,7 +2180,7 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group "validatorgroup", shard, validator_id, session_id, validator_set, last_masterchain_state_->get_collator_config(true), opts, keyring_, adnl_, rldp_, overlays_, db_root_, actor_id(this), init_session, opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), - opts_->validator_lite_mode()); + opts_->validator_mode()); return G; } } @@ -2348,7 +2354,7 @@ void ValidatorManagerImpl::allow_block_state_gc(BlockIdExt block_id, td::Promise return; } auto shards = gc_masterchain_state_->get_shards(); - for (auto shard : shards) { + for (const auto &shard : shards) { if (shard_intersects(shard->shard(), block_id.shard_full())) { promise.set_result(block_id.id.seqno < shard->top_block_id().id.seqno); return; @@ -2564,7 +2570,14 @@ bool ValidatorManagerImpl::is_validator() { } bool ValidatorManagerImpl::is_collator() { - return !collator_nodes_.empty() || (!opts_->validator_lite_mode() && is_validator()); + return !collator_nodes_.empty() || + (opts_->validator_mode() != ValidatorManagerOptions::validator_lite_all && is_validator()); +} + +bool ValidatorManagerImpl::validating_masterchain() { + return !get_validator(ShardIdFull(masterchainId), + last_masterchain_state_->get_validator_set(ShardIdFull(masterchainId))) + .is_zero(); } PublicKeyHash ValidatorManagerImpl::get_validator(ShardIdFull shard, td::Ref val_set) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 835f7de07..3debccb39 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -506,6 +506,7 @@ class ValidatorManagerImpl : public ValidatorManager { bool is_validator(); bool is_collator(); + bool validating_masterchain(); PublicKeyHash get_validator(ShardIdFull shard, td::Ref val_set); ValidatorManagerImpl(td::Ref opts, std::string db_root, diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index b4f665c39..7cc92f804 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -50,7 +50,8 @@ void ValidatorGroup::generate_block_candidate(td::uint32 round_id, td::Promise R) { td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_candidate, std::move(cache), std::move(R)); }; - if (lite_mode_) { + if (mode_ == ValidatorManagerOptions::validator_lite_all || + (mode_ == ValidatorManagerOptions::validator_lite_shards && !shard_.is_masterchain())) { send_collate_query(round_id, td::Timestamp::in(10.0), std::move(P)); return; } @@ -197,7 +198,8 @@ void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref }); run_accept_block_query(block_id, std::move(block), std::move(prev), validator_set_, std::move(sig_set), - std::move(approve_sig_set), send_broadcast, shard_.is_masterchain() || !lite_mode_, manager_, + std::move(approve_sig_set), send_broadcast, + shard_.is_masterchain() || mode_ == ValidatorManagerOptions::validator_normal, manager_, std::move(P)); } diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index c7964ae54..6a78827d5 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -65,7 +65,8 @@ class ValidatorGroup : public td::actor::Actor { td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, td::actor::ActorId validator_manager, bool create_session, - bool allow_unsafe_self_blocks_resync, bool lite_mode = false) + bool allow_unsafe_self_blocks_resync, + ValidatorManagerOptions::ValidatorMode mode = ValidatorManagerOptions::validator_normal) : shard_(shard) , local_id_(std::move(local_id)) , session_id_(session_id) @@ -80,7 +81,7 @@ class ValidatorGroup : public td::actor::Actor { , manager_(validator_manager) , init_(create_session) , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) - , lite_mode_(lite_mode) { + , mode_(mode) { } private: @@ -125,7 +126,7 @@ class ValidatorGroup : public td::actor::Actor { bool init_ = false; bool started_ = false; bool allow_unsafe_self_blocks_resync_; - bool lite_mode_ = false; + ValidatorManagerOptions::ValidatorMode mode_; td::uint32 last_known_round_id_ = 0; struct CachedCollatedBlock { diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index e96292ad3..ea100a50b 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -112,8 +112,8 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { std::string get_session_logs_file() const override { return session_logs_file_; } - bool validator_lite_mode() const override { - return validator_lite_mode_; + ValidatorMode validator_mode() const override { + return validator_mode_; } void set_zero_block_id(BlockIdExt block_id) override { @@ -168,8 +168,8 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_session_logs_file(std::string f) override { session_logs_file_ = std::move(f); } - void set_validator_lite_mode(bool value) override { - validator_lite_mode_ = value; + void set_validator_mode(ValidatorMode value) override { + validator_mode_ = value; } ValidatorManagerOptionsImpl *make_copy() const override { @@ -211,7 +211,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { BlockSeqno truncate_{0}; BlockSeqno sync_upto_{0}; std::string session_logs_file_; - bool validator_lite_mode_ = false; + ValidatorMode validator_mode_ = validator_normal; }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 343de0824..4ce3f4d02 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -54,6 +54,8 @@ struct PerfTimerStats { struct ValidatorManagerOptions : public td::CntObject { public: + enum ValidatorMode { validator_normal, validator_lite_shards, validator_lite_all }; + virtual BlockIdExt zero_block_id() const = 0; virtual BlockIdExt init_block_id() const = 0; virtual bool need_monitor(ShardIdFull shard, const td::Ref& state) const = 0; @@ -79,7 +81,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual BlockSeqno get_truncate_seqno() const = 0; virtual BlockSeqno sync_upto() const = 0; virtual std::string get_session_logs_file() const = 0; - virtual bool validator_lite_mode() const = 0; + virtual ValidatorMode validator_mode() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; @@ -98,7 +100,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void truncate_db(BlockSeqno seqno) = 0; virtual void set_sync_upto(BlockSeqno seqno) = 0; virtual void set_session_logs_file(std::string f) = 0; - virtual void set_validator_lite_mode(bool value) = 0; + virtual void set_validator_mode(ValidatorMode value) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, From 5ad7f8c246646afa819ea95b30fe026183019bf3 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 17 Jan 2023 21:02:47 +0300 Subject: [PATCH 048/388] Allow queries to collator from validators only --- validator/collator-node.cpp | 19 +++++++++++++++++++ validator/collator-node.hpp | 1 + 2 files changed, 20 insertions(+) diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 193613c74..654e4e742 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -100,6 +100,22 @@ void CollatorNode::new_masterchain_block_notification(td::Ref } } } + if (validators_.empty() || state->is_key_state()) { + validators_.clear(); + for (int next : {-1, 0, 1}) { + td::Ref vals = state->get_total_validator_set(next); + if (vals.not_null()) { + for (const ValidatorDescr& descr : vals->export_vector()) { + if (descr.addr.is_zero()) { + validators_.insert( + adnl::AdnlNodeIdShort(PublicKey(pubkeys::Ed25519{descr.key.as_bits256()}).compute_short_id())); + } else { + validators_.insert(adnl::AdnlNodeIdShort(descr.addr)); + } + } + } + } + } } static td::BufferSlice serialize_error(td::Status error) { @@ -110,6 +126,9 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data td::Promise promise) { auto SelfId = actor_id(this); auto status = [&]() -> td::Status { + if (!validators_.count(src)) { + return td::Status::Error("src is not a validator"); + } TRY_RESULT(f, fetch_tl_object(std::move(data), true)); ShardIdFull shard(f->workchain_, f->shard_); if (!shard.is_valid_ext()) { diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp index b3b232cf9..46848fcfe 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node.hpp @@ -49,6 +49,7 @@ class CollatorNode : public td::actor::Actor { td::actor::ActorId adnl_; td::actor::ActorId rldp_; std::vector shards_; + std::set validators_; }; } // namespace validator From 5dd0c15d072d016b15b8952c9bbd119953a0accf Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 4 Jul 2023 23:34:34 +0300 Subject: [PATCH 049/388] Limit imported msg queue size --- crypto/block/block.tlb | 1 + crypto/block/output-queue-merger.cpp | 42 +++-- crypto/block/output-queue-merger.h | 18 +- tl/generate/scheme/ton_api.tl | 2 +- tl/generate/scheme/ton_api.tlo | Bin 90648 -> 90684 bytes validator/impl/collator-impl.h | 2 + validator/impl/collator.cpp | 29 ++- validator/impl/out-msg-queue-proof.cpp | 208 +++++++++++++-------- validator/impl/out-msg-queue-proof.hpp | 10 +- validator/impl/validate-query.cpp | 21 ++- validator/impl/validate-query.hpp | 1 + validator/interfaces/out-msg-queue-proof.h | 13 +- 12 files changed, 226 insertions(+), 121 deletions(-) diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index cb7860497..b316d1c64 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -829,6 +829,7 @@ top_block_descr#d5 proof_for:BlockIdExt signatures:(Maybe ^BlockSignatures) // COLLATED DATA // top_block_descr_set#4ac789f3 collection:(HashmapE 96 ^TopBlockDescr) = TopBlockDescrSet; +neighbor_msg_queue_limits#7e549333 neighbors:(HashmapE 96 int32) = NeighborMsgQueueLimits; // // VALIDATOR MISBEHAVIOR COMPLAINTS diff --git a/crypto/block/output-queue-merger.cpp b/crypto/block/output-queue-merger.cpp index 1084bb1a8..ca7021a89 100644 --- a/crypto/block/output-queue-merger.cpp +++ b/crypto/block/output-queue-merger.cpp @@ -134,34 +134,33 @@ bool OutputQueueMerger::MsgKeyValue::split(MsgKeyValue& second) { return true; } -bool OutputQueueMerger::add_root(int src, Ref outmsg_root) { +void OutputQueueMerger::add_root(int src, Ref outmsg_root, td::int32 msg_limit) { if (outmsg_root.is_null()) { - return true; + return; } //block::gen::HashmapAug{352, block::gen::t_EnqueuedMsg, block::gen::t_uint64}.print_ref(std::cerr, outmsg_root); auto kv = std::make_unique(src, std::move(outmsg_root)); if (kv->replace_by_prefix(common_pfx.cbits(), common_pfx_len)) { heap.push_back(std::move(kv)); } - return true; -} - -OutputQueueMerger::OutputQueueMerger(ton::ShardIdFull _queue_for, std::vector _neighbors) - : queue_for(_queue_for), neighbors(std::move(_neighbors)), eof(false), failed(false) { - init(); + if ((int)src_remaining_msgs_.size() < src + 1) { + src_remaining_msgs_.resize(src + 1); + } + src_remaining_msgs_[src] = msg_limit; } -void OutputQueueMerger::init() { +OutputQueueMerger::OutputQueueMerger(ton::ShardIdFull queue_for, std::vector neighbors) + : eof(false), failed(false) { common_pfx.bits().store_int(queue_for.workchain, 32); int l = queue_for.pfx_len(); td::bitstring::bits_store_long_top(common_pfx.bits() + 32, queue_for.shard, l); common_pfx_len = 32 + l; int i = 0; - for (block::McShardDescr& neighbor : neighbors) { - if (!neighbor.is_disabled()) { - LOG(DEBUG) << "adding " << (neighbor.outmsg_root.is_null() ? "" : "non-") << "empty output queue for neighbor #" - << i << " (" << neighbor.blk_.to_str() << ")"; - add_root(i++, neighbor.outmsg_root); + for (Neighbor& neighbor : neighbors) { + if (!neighbor.disabled_) { + LOG(DEBUG) << "adding " << (neighbor.outmsg_root_.is_null() ? "" : "non-") << "empty output queue for neighbor #" + << i << " (" << neighbor.block_id_.to_str() << ")"; + add_root(i++, neighbor.outmsg_root_, neighbor.msg_limit_); } else { LOG(DEBUG) << "skipping output queue for disabled neighbor #" << i; i++; @@ -200,6 +199,10 @@ bool OutputQueueMerger::load() { unsigned long long lt = heap[0]->lt; std::size_t orig_size = msg_list.size(); do { + if (src_remaining_msgs_[heap[0]->source] == 0) { + std::pop_heap(heap.begin(), heap.end(), MsgKeyValue::greater); + continue; + } while (heap[0]->is_fork()) { auto other = std::make_unique(); if (!heap[0]->split(*other)) { @@ -215,6 +218,17 @@ bool OutputQueueMerger::load() { heap.pop_back(); } while (!heap.empty() && heap[0]->lt <= lt); std::sort(msg_list.begin() + orig_size, msg_list.end(), MsgKeyValue::less); + size_t j = orig_size; + for (size_t i = orig_size; i < msg_list.size(); ++i) { + td::int32 &remaining = src_remaining_msgs_[msg_list[i]->source]; + if (remaining != 0) { + if (remaining > 0) { + --remaining; + } + msg_list[j++] = std::move(msg_list[i]); + } + } + msg_list.resize(j); return true; } diff --git a/crypto/block/output-queue-merger.h b/crypto/block/output-queue-merger.h index bf3d85868..f5c15ab4d 100644 --- a/crypto/block/output-queue-merger.h +++ b/crypto/block/output-queue-merger.h @@ -52,12 +52,20 @@ struct OutputQueueMerger { bool split(MsgKeyValue& second); }; // - ton::ShardIdFull queue_for; std::vector> msg_list; - std::vector neighbors; public: - OutputQueueMerger(ton::ShardIdFull _queue_for, std::vector _neighbors); + struct Neighbor { + ton::BlockIdExt block_id_; + td::Ref outmsg_root_; + bool disabled_; + td::int32 msg_limit_; // -1 - unlimited + Neighbor(ton::BlockIdExt block_id, td::Ref outmsg_root, bool disabled = false, td::int32 msg_limit = -1) + : block_id_(block_id), outmsg_root_(std::move(outmsg_root)), disabled_(disabled), msg_limit_(msg_limit) { + } + }; + + OutputQueueMerger(ton::ShardIdFull queue_for, std::vector neighbors); bool is_eof() const { return eof; } @@ -70,10 +78,10 @@ struct OutputQueueMerger { int common_pfx_len; std::vector> heap; std::size_t pos{0}; + std::vector src_remaining_msgs_; bool eof; bool failed; - void init(); - bool add_root(int src, Ref outmsg_root); + void add_root(int src, Ref outmsg_root, td::int32 msg_limit); bool load(); }; diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 2a1181972..6d52b8454 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -415,7 +415,7 @@ tonNode.success = tonNode.Success; tonNode.archiveNotFound = tonNode.ArchiveInfo; tonNode.archiveInfo id:long = tonNode.ArchiveInfo; -tonNode.outMsgQueueProof queue_proof:bytes block_state_proof:bytes = tonNode.OutMsgQueueProof; +tonNode.outMsgQueueProof queue_proof:bytes block_state_proof:bytes msg_count:int = tonNode.OutMsgQueueProof; tonNode.outMsgQueueProofEmpty = tonNode.OutMsgQueueProof; tonNode.forgetPeer = tonNode.ForgetPeer; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index f85ddc6b49657a9b9477515e82ad3d1775fe4197..1eed452846aa645ab1b76b678dde81080e579487 100644 GIT binary patch delta 138 zcmbPngmupm)(txPEI$+_I5!*UPdKG1rQfttq9i}hFFz$!FTb?Jw>UkpG_^D}peR2- zje&syr1Iu5As^<+FMkG4R`6$G;mj>gpRBN21#HM X_|=mDRyJKBny~@Q+P);3v4jl(pQkWp delta 119 zcmdmUgmuOd)(txPEJ<2%Q#KpuPdLTepmM}SVsh;-2_6Op29WU0V?sVmlVAP}o~+=; z0aiFU=a+#5ObV!qp~#s{hr_n*AlQ^Gzj_ilVY1AV8}qcMmqarLfOT#^5zSb_1^{`V BEK2|Y diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index dab3f6a95..80a8e9a7d 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -133,6 +133,8 @@ class Collator final : public td::actor::Actor { std::unique_ptr config_; std::unique_ptr shard_conf_; std::map> aux_mc_states_; + std::vector neighbor_queues_; + vm::Dictionary neighbor_msg_queues_limits_{32 + 64}; std::vector neighbors_; std::unique_ptr nb_out_msgs_; std::vector special_smcs; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index a2f3f7bd3..c6ac6f2fb 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -77,7 +77,7 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_mastercha } void Collator::start_up() { - LOG(DEBUG) << "Collator for shard " << shard_.to_str() << " started"; + LOG(INFO) << "Collator for shard " << shard_.to_str() << " started"; LOG(DEBUG) << "Previous block #1 is " << prev_blocks.at(0).to_str(); if (prev_blocks.size() > 1) { LOG(DEBUG) << "Previous block #2 is " << prev_blocks.at(1).to_str(); @@ -690,7 +690,15 @@ void Collator::got_neighbor_msg_queue(unsigned i, td::Resultprefetch_ref(0)); + auto queue_root = qinfo.out_queue->prefetch_ref(0); + descr.set_queue_root(queue_root); + neighbor_queues_.emplace_back(descr.top_block_id(), queue_root, descr.is_disabled(), res->msg_count_); + if (res->msg_count_ != -1) { + td::BitArray<96> key; + key.bits().store_int(block_id.id.workchain, 32); + (key.bits() + 32).store_uint(block_id.id.shard, 64); + neighbor_msg_queues_limits_.set_builder(key, vm::CellBuilder().store_long(res->msg_count_, 32)); + } // comment the next two lines in the future when the output queues become huge // CHECK(block::gen::t_OutMsgQueueInfo.validate_ref(1000000, outq_descr->root_cell())); // CHECK(block::tlb::t_OutMsgQueueInfo.validate_ref(1000000, outq_descr->root_cell())); @@ -1723,7 +1731,7 @@ bool Collator::do_collate() { // 1.3. create OutputQueueMerger from adjusted neighbors CHECK(!nb_out_msgs_); LOG(DEBUG) << "creating OutputQueueMerger"; - nb_out_msgs_ = std::make_unique(shard_, neighbors_); + nb_out_msgs_ = std::make_unique(shard_, neighbor_queues_); // 1.4. compute created / minted / recovered if (!init_value_create()) { return fatal_error("cannot compute the value to be created / minted / recovered"); @@ -4013,18 +4021,25 @@ bool Collator::create_collated_data() { auto cell = collate_shard_block_descr_set(); if (cell.is_null()) { return true; - return fatal_error("cannot collate the collection of used shard block descriptions"); + // return fatal_error("cannot collate the collection of used shard block descriptions"); } collated_roots_.push_back(std::move(cell)); } + // 2. Message count for neighbors' out queues + if (!neighbor_msg_queues_limits_.is_empty()) { + vm::CellBuilder cb; + cb.store_long(block::gen::t_NeighborMsgQueueLimits.cons_tag[0], 32); + cb.store_maybe_ref(neighbor_msg_queues_limits_.get_root_cell()); + collated_roots_.push_back(cb.finalize_novm()); + } if (!full_collated_data_) { return true; } - // 2. Proofs for hashes of states: previous states + neighbors + // 3. Proofs for hashes of states: previous states + neighbors for (const auto& p : block_state_proofs_) { collated_roots_.push_back(p.second); } - // 3. Previous state proof (only shadchains) + // 4. Previous state proof (only shadchains) std::map> proofs; if (!is_masterchain()) { if (!prepare_msg_queue_proof()) { @@ -4038,7 +4053,7 @@ bool Collator::create_collated_data() { } proofs[prev_state_root_->get_hash().bits()] = std::move(state_proof); } - // 4. Proofs for message queues + // 5. Proofs for message queues for (vm::MerkleProofBuilder &mpb : neighbor_proof_builders_) { auto r_proof = mpb.extract_proof(); if (r_proof.is_error()) { diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 56a4edd34..329da607d 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -22,94 +22,45 @@ #include "interfaces/validator-manager.h" #include "block/block-parse.h" #include "block/block-auto.h" +#include "output-queue-merger.h" namespace ton { namespace validator { -td::Result> OutMsgQueueProof::fetch(BlockIdExt block_id, ShardIdFull dst_shard, - const ton_api::tonNode_outMsgQueueProof& f) { - Ref block_state_proof; - td::Bits256 state_root_hash; - if (block_id.seqno() == 0) { - if (!f.block_state_proof_.empty()) { - return td::Status::Error("expected empty block state proof"); - } - state_root_hash = block_id.root_hash; - } else { - TRY_RESULT_ASSIGN(block_state_proof, vm::std_boc_deserialize(f.block_state_proof_.as_slice())); - TRY_RESULT_ASSIGN(state_root_hash, unpack_block_state_proof(block_id, block_state_proof)); - } - - TRY_RESULT(queue_proof, vm::std_boc_deserialize(f.queue_proof_.as_slice())); - auto virtual_root = vm::MerkleProof::virtualize(queue_proof, 1); - if (virtual_root.is_null()) { - return td::Status::Error("invalid queue proof"); +static td::Status check_no_prunned(const Ref& cell) { + if (cell.is_null()) { + return td::Status::OK(); } - if (virtual_root->get_hash().as_slice() != state_root_hash.as_slice()) { - return td::Status::Error("state root hash mismatch"); + TRY_RESULT(loaded_cell, cell->load_cell()); + if (loaded_cell.data_cell->get_level() > 0) { + return td::Status::Error("prunned branch"); } + return td::Status::OK(); +} - // Validate proof - auto state_root = vm::CellSlice(vm::NoVm(), queue_proof).prefetch_ref(0); - TRY_RESULT_PREFIX(state, ShardStateQ::fetch(block_id, {}, state_root), "invalid proof: "); - TRY_RESULT_PREFIX(outq_descr, state->message_queue(), "invalid proof: "); - - block::gen::OutMsgQueueInfo::Record qinfo; - if (!tlb::unpack_cell(outq_descr->root_cell(), qinfo)) { - return td::Status::Error("invalid proof: invalid message queue"); - } - td::Ref proc_info = qinfo.proc_info->prefetch_ref(0); - if (proc_info.not_null() && proc_info->get_level() != 0) { - return td::Status::Error("invalid proof: proc_info has prunned branches"); +static td::Status check_no_prunned(const vm::CellSlice& cs) { + for (unsigned i = 0; i < cs.size_refs(); ++i) { + TRY_STATUS(check_no_prunned(cs.prefetch_ref(i))); } - td::Ref ihr_pending = qinfo.ihr_pending->prefetch_ref(0); - if (ihr_pending.not_null() && ihr_pending->get_level() != 0) { - return td::Status::Error("invalid proof: ihr_pending has prunned branches"); - } - auto queue = - std::make_unique(qinfo.out_queue->prefetch_ref(0), 352, block::tlb::aug_OutMsgQueue); - td::BitArray<96> prefix; - td::BitPtr ptr = prefix.bits(); - ptr.store_int(dst_shard.workchain, 32); - ptr.advance(32); - ptr.store_uint(dst_shard.shard, 64); - if (!queue->cut_prefix_subdict(prefix.bits(), 32 + dst_shard.pfx_len())) { - return td::Status::Error("invalid proof: failed to cut queue dict"); - } - if (queue->get_root_cell().not_null() && queue->get_root_cell()->get_level() != 0) { - return td::Status::Error("invalid proof: msg queue has prunned branches"); - } - - return Ref(true, std::move(virtual_root), std::move(block_state_proof)); + return td::Status::OK(); } -td::Result> OutMsgQueueProof::serialize(BlockIdExt block_id, - ShardIdFull dst_shard, - Ref state_root, - Ref block_root) { - if (!dst_shard.is_valid_ext()) { - return td::Status::Error("invalid shard"); - } - vm::MerkleProofBuilder mpb{std::move(state_root)}; - TRY_RESULT(state, ShardStateQ::fetch(block_id, {}, mpb.root())); - TRY_RESULT(outq_descr, state->message_queue()); - block::gen::OutMsgQueueInfo::Record qinfo; - if (!tlb::unpack_cell(outq_descr->root_cell(), qinfo)) { - return td::Status::Error("invalid message queue"); - } +static td::Result process_queue(BlockIdExt block_id, ShardIdFull dst_shard, + const block::gen::OutMsgQueueInfo::Record& qinfo) { + td::uint64 estimated_proof_size = 0; td::HashSet visited; - std::function)> dfs = [&](Ref cell) { + std::function dfs_cs; + auto dfs = [&](Ref cell) { if (cell.is_null() || !visited.insert(cell->get_hash()).second) { return; } - vm::CellSlice cs(vm::NoVm(), cell); - for (unsigned i = 0; i < cs.size_refs(); i++) { - dfs(cs.prefetch_ref(i)); - } + dfs_cs(vm::CellSlice(vm::NoVm(), cell)); }; - auto dfs_cs = [&](const vm::CellSlice &cs) { + dfs_cs = [&](const vm::CellSlice& cs) { + // Based on BlockLimitStatus::estimate_block_size + estimated_proof_size += 12 + (cs.size() + 7) / 8 + cs.size_refs() * 3; for (unsigned i = 0; i < cs.size_refs(); i++) { dfs(cs.prefetch_ref(i)); } @@ -117,17 +68,65 @@ td::Result> OutMsgQueueProof::s dfs_cs(*qinfo.proc_info); dfs_cs(*qinfo.ihr_pending); - auto queue = - std::make_unique(qinfo.out_queue->prefetch_ref(0), 352, block::tlb::aug_OutMsgQueue); - td::BitArray<96> prefix; - td::BitPtr ptr = prefix.bits(); - ptr.store_int(dst_shard.workchain, 32); - ptr.advance(32); - ptr.store_uint(dst_shard.shard, 64); - if (!queue->cut_prefix_subdict(prefix.bits(), 32 + dst_shard.pfx_len())) { + block::OutputQueueMerger queue_merger{ + dst_shard, {block::OutputQueueMerger::Neighbor{block_id, qinfo.out_queue->prefetch_ref()}}}; + td::int32 msg_count = 0; + bool limit_reached = false; + + while (!queue_merger.is_eof()) { + auto kv = queue_merger.extract_cur(); + queue_merger.next(); + ++msg_count; + + // TODO: Get processed_upto from destination shard (in request?) + /* + // Parse message to check if it was processed (as in Collator::process_inbound_message) + ton::LogicalTime enqueued_lt = kv->msg->prefetch_ulong(64); + auto msg_env = kv->msg->prefetch_ref(); + block::tlb::MsgEnvelope::Record_std env; + if (!tlb::unpack_cell(msg_env, env)) { + return td::Status::Error("cannot unpack MsgEnvelope of an internal message"); + } + vm::CellSlice cs{vm::NoVmOrd{}, env.msg}; + block::gen::CommonMsgInfo::Record_int_msg_info info; + if (!tlb::unpack(cs, info)) { + return td::Status::Error("cannot unpack CommonMsgInfo of an internal message"); + } + auto src_prefix = block::tlb::MsgAddressInt::get_prefix(info.src); + auto dest_prefix = block::tlb::MsgAddressInt::get_prefix(info.dest); + auto cur_prefix = block::interpolate_addr(src_prefix, dest_prefix, env.cur_addr); + auto next_prefix = block::interpolate_addr(src_prefix, dest_prefix, env.next_addr); + block::EnqueuedMsgDescr descr{cur_prefix, next_prefix, kv->lt, enqueued_lt, env.msg->get_hash().bits()}; + if (dst_processed_upto->already_processed(descr)) { + } else { + }*/ + + dfs_cs(*kv->msg); + TRY_STATUS_PREFIX(check_no_prunned(*kv->msg), "invalid message proof: ") + if (estimated_proof_size > OutMsgQueueProof::QUEUE_SIZE_THRESHOLD) { + limit_reached = true; + break; + } + } + return limit_reached ? msg_count : -1; +} + +td::Result> OutMsgQueueProof::build(BlockIdExt block_id, + ShardIdFull dst_shard, + Ref state_root, + Ref block_root) { + if (!dst_shard.is_valid_ext()) { + return td::Status::Error("invalid shard"); + } + + vm::MerkleProofBuilder mpb{std::move(state_root)}; + TRY_RESULT(state, ShardStateQ::fetch(block_id, {}, mpb.root())); + TRY_RESULT(outq_descr, state->message_queue()); + block::gen::OutMsgQueueInfo::Record qinfo; + if (!tlb::unpack_cell(outq_descr->root_cell(), qinfo)) { return td::Status::Error("invalid message queue"); } - dfs(queue->get_root_cell()); + TRY_RESULT(cnt, process_queue(block_id, dst_shard, qinfo)); TRY_RESULT(queue_proof, mpb.extract_proof_boc()); td::BufferSlice block_state_proof; @@ -135,7 +134,52 @@ td::Result> OutMsgQueueProof::s TRY_RESULT(proof, create_block_state_proof(std::move(block_root))); TRY_RESULT_ASSIGN(block_state_proof, vm::std_boc_serialize(std::move(proof), 31)); } - return create_tl_object(std::move(queue_proof), std::move(block_state_proof)); + return create_tl_object(std::move(queue_proof), std::move(block_state_proof), + cnt); +} + +td::Result> OutMsgQueueProof::fetch(BlockIdExt block_id, ShardIdFull dst_shard, + const ton_api::tonNode_outMsgQueueProof& f) { + try { + Ref block_state_proof; + td::Bits256 state_root_hash; + if (block_id.seqno() == 0) { + if (!f.block_state_proof_.empty()) { + return td::Status::Error("expected empty block state proof"); + } + state_root_hash = block_id.root_hash; + } else { + TRY_RESULT_ASSIGN(block_state_proof, vm::std_boc_deserialize(f.block_state_proof_.as_slice())); + TRY_RESULT_ASSIGN(state_root_hash, unpack_block_state_proof(block_id, block_state_proof)); + } + + TRY_RESULT(queue_proof, vm::std_boc_deserialize(f.queue_proof_.as_slice())); + auto virtual_root = vm::MerkleProof::virtualize(queue_proof, 1); + if (virtual_root.is_null()) { + return td::Status::Error("invalid queue proof"); + } + if (virtual_root->get_hash().as_slice() != state_root_hash.as_slice()) { + return td::Status::Error("state root hash mismatch"); + } + + // Validate proof + TRY_RESULT_PREFIX(state, ShardStateQ::fetch(block_id, {}, virtual_root), "invalid proof: "); + TRY_RESULT_PREFIX(outq_descr, state->message_queue(), "invalid proof: "); + + block::gen::OutMsgQueueInfo::Record qinfo; + if (!tlb::unpack_cell(outq_descr->root_cell(), qinfo)) { + return td::Status::Error("invalid proof: invalid message queue"); + } + TRY_STATUS_PREFIX(check_no_prunned(qinfo.proc_info->prefetch_ref(0)), "invalid proc_info: ") + TRY_STATUS_PREFIX(check_no_prunned(qinfo.ihr_pending->prefetch_ref(0)), "invalid ihr_pending: ") + TRY_RESULT(cnt, process_queue(block_id, dst_shard, qinfo)); + if (cnt != f.msg_count_) { + return td::Status::Error(PSTRING() << "invalid msg_count: expected=" << f.msg_count_ << ", found=" << cnt); + } + return Ref(true, std::move(virtual_root), std::move(block_state_proof), cnt); + } catch (vm::VmVirtError& err) { + return td::Status::Error(PSTRING() << "invalid proof: " << err.get_msg()); + } } void WaitOutMsgQueueProof::alarm() { @@ -296,7 +340,7 @@ void BuildOutMsgQueueProof::got_block_root(Ref root) { } void BuildOutMsgQueueProof::build_proof() { - auto result = OutMsgQueueProof::serialize(block_id_, dst_shard_, std::move(state_root_), std::move(block_root_)); + auto result = OutMsgQueueProof::build(block_id_, dst_shard_, std::move(state_root_), std::move(block_root_)); if (result.is_error()) { LOG(ERROR) << "Failed to build msg queue proof: " << result.error(); } diff --git a/validator/impl/out-msg-queue-proof.hpp b/validator/impl/out-msg-queue-proof.hpp index 6a18ca6cb..e94f805f4 100644 --- a/validator/impl/out-msg-queue-proof.hpp +++ b/validator/impl/out-msg-queue-proof.hpp @@ -62,7 +62,6 @@ class WaitOutMsgQueueProof : public td::actor::Actor { void run_net(); - private: BlockIdExt block_id_; ShardIdFull dst_shard_; @@ -80,12 +79,9 @@ class WaitOutMsgQueueProof : public td::actor::Actor { class BuildOutMsgQueueProof : public td::actor::Actor { public: BuildOutMsgQueueProof(BlockIdExt block_id, ShardIdFull dst_shard, - td::actor::ActorId manager, - td::Promise> promise) - : block_id_(std::move(block_id)) - , dst_shard_(dst_shard) - , manager_(manager) - , promise_(std::move(promise)) { + td::actor::ActorId manager, + td::Promise> promise) + : block_id_(std::move(block_id)), dst_shard_(dst_shard), manager_(manager), promise_(std::move(promise)) { } void abort_query(td::Status reason); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index f43bad297..3329e4fd5 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -522,6 +522,13 @@ bool ValidateQuery::extract_collated_data_from(Ref croot, int idx) { top_shard_descr_dict_ = std::make_unique(cs.prefetch_ref(), 96); return true; } + if (block::gen::t_NeighborMsgQueueLimits.has_valid_tag(cs)) { + LOG(DEBUG) << "collated datum # " << idx << " is a NeighborMsgQueueLimits"; + if (!block::gen::t_NeighborMsgQueueLimits.validate_upto(10000, cs)) { + return reject_query("invalid NeighborMsgQueueLimits"); + } + neighbor_msg_queues_limits_ = vm::Dictionary{cs.prefetch_ref(0), 32 + 64}; + } LOG(WARNING) << "collated datum # " << idx << " has unknown type (magic " << cs.prefetch_ulong(32) << "), ignoring"; return true; } @@ -4106,7 +4113,19 @@ bool ValidateQuery::check_neighbor_outbound_message(Ref enq_msg, } bool ValidateQuery::check_in_queue() { - block::OutputQueueMerger nb_out_msgs(shard_, neighbors_); + std::vector neighbor_queues; + for (const auto& descr : neighbors_) { + td::BitArray<96> key; + key.bits().store_int(descr.workchain(), 32); + (key.bits() + 32).store_uint(descr.shard().shard, 64); + auto r = neighbor_msg_queues_limits_.lookup(key); + td::int32 msg_limit = r.is_null() ? -1 : (td::int32)r->prefetch_long(32); + if (msg_limit < -1) { + return reject_query("invalid value in NeighborMsgQueueLimits"); + } + neighbor_queues.emplace_back(descr.top_block_id(), descr.outmsg_root, descr.disabled_, msg_limit); + } + block::OutputQueueMerger nb_out_msgs(shard_, std::move(neighbor_queues)); while (!nb_out_msgs.is_eof()) { auto kv = nb_out_msgs.extract_cur(); CHECK(kv && kv->msg.not_null()); diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 942b32a5f..cd14504e6 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -206,6 +206,7 @@ class ValidateQuery : public td::actor::Actor { block::ActionPhaseConfig action_phase_cfg_; td::RefInt256 masterchain_create_fee_, basechain_create_fee_; + vm::Dictionary neighbor_msg_queues_limits_{32 + 64}; std::vector neighbors_; std::map> aux_mc_states_; diff --git a/validator/interfaces/out-msg-queue-proof.h b/validator/interfaces/out-msg-queue-proof.h index 1fece66a5..5d7924913 100644 --- a/validator/interfaces/out-msg-queue-proof.h +++ b/validator/interfaces/out-msg-queue-proof.h @@ -18,6 +18,7 @@ #include "vm/cells.h" #include "ton/ton-types.h" #include "auto/tl/ton_api.h" +#include "block/block.h" namespace ton { @@ -25,17 +26,21 @@ namespace validator { using td::Ref; struct OutMsgQueueProof : public td::CntObject { - OutMsgQueueProof(Ref state_root, Ref block_state_proof) - : state_root_(std::move(state_root)), block_state_proof_(std::move(block_state_proof)) { + OutMsgQueueProof(Ref state_root, Ref block_state_proof, td::int32 msg_count = -1) + : state_root_(std::move(state_root)), block_state_proof_(std::move(block_state_proof)), msg_count_(msg_count) { } Ref state_root_; Ref block_state_proof_; + td::int32 msg_count_; // -1 - up to end of queue static td::Result> fetch(BlockIdExt block_id, ShardIdFull dst_shard, const ton_api::tonNode_outMsgQueueProof &f); - static td::Result> serialize( - BlockIdExt block_id, ShardIdFull dst_shard, Ref state_root, Ref block_root); + static td::Result> build(BlockIdExt block_id, ShardIdFull dst_shard, + Ref state_root, + Ref block_root); + + static const td::uint64 QUEUE_SIZE_THRESHOLD = 128 * 1024; }; } // namespace validator From f10c7f54a82bc8ead6aaae9e6fd01db02493e3c0 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 6 Jul 2023 11:44:21 +0300 Subject: [PATCH 050/388] Bugfix in processing message queues; improve out_msg_queue_cleanup --- crypto/block/output-queue-merger.cpp | 2 +- crypto/block/output-queue-merger.h | 1 + validator/impl/collator-impl.h | 5 +- validator/impl/collator.cpp | 69 ++++++++++++++++++++++------ validator/impl/validate-query.cpp | 8 ++++ 5 files changed, 68 insertions(+), 17 deletions(-) diff --git a/crypto/block/output-queue-merger.cpp b/crypto/block/output-queue-merger.cpp index ca7021a89..2955b3e01 100644 --- a/crypto/block/output-queue-merger.cpp +++ b/crypto/block/output-queue-merger.cpp @@ -229,7 +229,7 @@ bool OutputQueueMerger::load() { } } msg_list.resize(j); - return true; + return msg_list.size() > orig_size; } } // namespace block diff --git a/crypto/block/output-queue-merger.h b/crypto/block/output-queue-merger.h index f5c15ab4d..3eb4b3799 100644 --- a/crypto/block/output-queue-merger.h +++ b/crypto/block/output-queue-merger.h @@ -60,6 +60,7 @@ struct OutputQueueMerger { td::Ref outmsg_root_; bool disabled_; td::int32 msg_limit_; // -1 - unlimited + Neighbor() = default; Neighbor(ton::BlockIdExt block_id, td::Ref outmsg_root, bool disabled = false, td::int32 msg_limit = -1) : block_id_(block_id), outmsg_root_(std::move(outmsg_root)), disabled_(disabled), msg_limit_(msg_limit) { } diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 80a8e9a7d..598118966 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -133,8 +133,8 @@ class Collator final : public td::actor::Actor { std::unique_ptr config_; std::unique_ptr shard_conf_; std::map> aux_mc_states_; - std::vector neighbor_queues_; - vm::Dictionary neighbor_msg_queues_limits_{32 + 64}; + std::map neighbor_msg_queues_limits_; + vm::Dictionary neighbor_msg_queues_limits_dict_{32 + 64}; std::vector neighbors_; std::unique_ptr nb_out_msgs_; std::vector special_smcs; @@ -241,6 +241,7 @@ class Collator final : public td::actor::Actor { bool register_shard_block_creators(std::vector creator_list); bool init_block_limits(); bool compute_minted_amount(block::CurrencyCollection& to_mint); + bool create_output_queue_merger(); bool init_value_create(); bool try_collate(); bool do_preinit(); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index c6ac6f2fb..1ce0a4b4c 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -672,9 +672,9 @@ void Collator::got_neighbor_msg_queue(unsigned i, td::Result outq_descr = outq_descr_res.move_as_ok(); block::McShardDescr& descr = neighbors_.at(i); + LOG(DEBUG) << "obtained outbound queue for neighbor #" << i << "(" << descr.shard().to_str() << ")"; if (outq_descr->get_block_id() != descr.blk_) { LOG(DEBUG) << "outq_descr->id = " << outq_descr->get_block_id().to_str() << " ; descr.id = " << descr.blk_.to_str(); fatal_error( @@ -692,12 +692,13 @@ void Collator::got_neighbor_msg_queue(unsigned i, td::Resultprefetch_ref(0); descr.set_queue_root(queue_root); - neighbor_queues_.emplace_back(descr.top_block_id(), queue_root, descr.is_disabled(), res->msg_count_); if (res->msg_count_ != -1) { + LOG(DEBUG) << "Neighbor " << descr.shard().to_str() << " has msg_limit=" << res->msg_count_; td::BitArray<96> key; key.bits().store_int(block_id.id.workchain, 32); (key.bits() + 32).store_uint(block_id.id.shard, 64); - neighbor_msg_queues_limits_.set_builder(key, vm::CellBuilder().store_long(res->msg_count_, 32)); + neighbor_msg_queues_limits_dict_.set_builder(key, vm::CellBuilder().store_long(res->msg_count_, 32)); + neighbor_msg_queues_limits_[block_id.shard_full()] = res->msg_count_; } // comment the next two lines in the future when the output queues become huge // CHECK(block::gen::t_OutMsgQueueInfo.validate_ref(1000000, outq_descr->root_cell())); @@ -1674,6 +1675,17 @@ bool Collator::compute_minted_amount(block::CurrencyCollection& to_mint) { return true; } +bool Collator::create_output_queue_merger() { + std::vector neighbor_queues; + for (const auto& descr : neighbors_) { + auto it = neighbor_msg_queues_limits_.find(descr.shard()); + td::int32 msg_limit = it == neighbor_msg_queues_limits_.end() ? -1 : it->second; + neighbor_queues.emplace_back(descr.top_block_id(), descr.outmsg_root, descr.disabled_, msg_limit); + } + nb_out_msgs_ = std::make_unique(shard_, neighbor_queues); + return true; +} + bool Collator::init_value_create() { value_flow_.created.set_zero(); value_flow_.minted.set_zero(); @@ -1731,7 +1743,9 @@ bool Collator::do_collate() { // 1.3. create OutputQueueMerger from adjusted neighbors CHECK(!nb_out_msgs_); LOG(DEBUG) << "creating OutputQueueMerger"; - nb_out_msgs_ = std::make_unique(shard_, neighbor_queues_); + if (!create_output_queue_merger()) { + return fatal_error("cannot compute the value to be created / minted / recovered"); + } // 1.4. compute created / minted / recovered if (!init_value_create()) { return fatal_error("cannot compute the value to be created / minted / recovered"); @@ -1861,18 +1875,34 @@ bool Collator::out_msg_queue_cleanup() { } } - auto res = out_msg_queue_->filter([&](vm::CellSlice& cs, td::ConstBitPtr key, int n) -> int { + auto queue_root = out_msg_queue_->get_root_cell(); + if (queue_root.is_null()) { + LOG(DEBUG) << "out_msg_queue is empty"; + return true; + } + // Unwrap UsageCell: don't build proof for visiting output queue (unless something is deleted) + auto r_cell = queue_root->load_cell(); + if (r_cell.is_error()) { + return fatal_error(r_cell.move_as_error()); + } + auto pure_out_msg_queue = + std::make_unique(r_cell.move_as_ok().data_cell, 352, block::tlb::aug_OutMsgQueue); + + int deleted = 0; + bool fail = false; + pure_out_msg_queue->check_for_each([&](Ref value, td::ConstBitPtr key, int n) -> bool { assert(n == 352); + vm::CellSlice& cs = value.write(); // LOG(DEBUG) << "key is " << key.to_hex(n); if (queue_cleanup_timeout_.is_in_past(td::Timestamp::now())) { LOG(WARNING) << "cleaning up outbound queue takes too long, ending"; outq_cleanup_partial_ = true; - return (1 << 30) + 1; // retain all remaining outbound queue entries including this one without processing + return false; // retain all remaining outbound queue entries including this one without processing } if (block_full_) { LOG(WARNING) << "BLOCK FULL while cleaning up outbound queue, cleanup completed only partially"; outq_cleanup_partial_ = true; - return (1 << 30) + 1; // retain all remaining outbound queue entries including this one without processing + return false; // retain all remaining outbound queue entries including this one without processing } block::EnqueuedMsgDescr enq_msg_descr; unsigned long long created_lt; @@ -1881,7 +1911,8 @@ bool Collator::out_msg_queue_cleanup() { && enq_msg_descr.check_key(key) // check key && enq_msg_descr.lt_ == created_lt)) { LOG(ERROR) << "cannot unpack EnqueuedMsg with key " << key.to_hex(n); - return -1; + fail = true; + return false; } LOG(DEBUG) << "scanning outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," << enq_msg_descr.hash_.to_hex() << ") enqueued_lt=" << enq_msg_descr.enqueued_lt_; @@ -1899,20 +1930,30 @@ bool Collator::out_msg_queue_cleanup() { if (delivered) { LOG(DEBUG) << "outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," << enq_msg_descr.hash_.to_hex() << ") enqueued_lt=" << enq_msg_descr.enqueued_lt_ << " has been already delivered, dequeueing"; + // Get value from out_msg_queue_ instead of pure_out_msg_queue (for proof) + auto value2 = out_msg_queue_->lookup_delete_with_extra(key, n); + CHECK(value2.not_null()); + vm::CellSlice& cs2 = value2.write(); + CHECK(cs2.fetch_ulong_bool(64, created_lt) // augmentation + && enq_msg_descr.unpack(cs2) // unpack EnqueuedMsg + && enq_msg_descr.check_key(key) // check key + && enq_msg_descr.lt_ == created_lt); + if (!dequeue_message(std::move(enq_msg_descr.msg_env_), deliver_lt)) { fatal_error(PSTRING() << "cannot dequeue outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," << enq_msg_descr.hash_.to_hex() << ") by inserting a msg_export_deq record"); - return -1; + fail = true; + return false; } register_out_msg_queue_op(); if (!block_limit_status_->fits(block::ParamLimits::cl_normal)) { block_full_ = true; } } - return !delivered; + return true; }); - LOG(DEBUG) << "deleted " << res << " messages from out_msg_queue"; - if (res < 0) { + LOG(DEBUG) << "deleted " << deleted << " messages from out_msg_queue"; + if (fail) { return fatal_error("error scanning/updating OutMsgQueue"); } auto rt = out_msg_queue_->get_root(); @@ -4026,10 +4067,10 @@ bool Collator::create_collated_data() { collated_roots_.push_back(std::move(cell)); } // 2. Message count for neighbors' out queues - if (!neighbor_msg_queues_limits_.is_empty()) { + if (!neighbor_msg_queues_limits_dict_.is_empty()) { vm::CellBuilder cb; cb.store_long(block::gen::t_NeighborMsgQueueLimits.cons_tag[0], 32); - cb.store_maybe_ref(neighbor_msg_queues_limits_.get_root_cell()); + cb.store_maybe_ref(neighbor_msg_queues_limits_dict_.get_root_cell()); collated_roots_.push_back(cb.finalize_novm()); } if (!full_collated_data_) { diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 3329e4fd5..f9e5be275 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -528,6 +528,7 @@ bool ValidateQuery::extract_collated_data_from(Ref croot, int idx) { return reject_query("invalid NeighborMsgQueueLimits"); } neighbor_msg_queues_limits_ = vm::Dictionary{cs.prefetch_ref(0), 32 + 64}; + return true; } LOG(WARNING) << "collated datum # " << idx << " has unknown type (magic " << cs.prefetch_ulong(32) << "), ignoring"; return true; @@ -4123,7 +4124,14 @@ bool ValidateQuery::check_in_queue() { if (msg_limit < -1) { return reject_query("invalid value in NeighborMsgQueueLimits"); } + LOG(DEBUG) << "Neighbor " << descr.shard().to_str() << " has msg_limit=" << msg_limit; neighbor_queues.emplace_back(descr.top_block_id(), descr.outmsg_root, descr.disabled_, msg_limit); + if (msg_limit != -1 && descr.shard().is_masterchain()) { + return reject_query("masterchain out message queue cannot be limited"); + } + if (msg_limit != -1 && shard_intersects(descr.shard(), shard_)) { + return reject_query("prev block out message queue cannot be limited"); + } } block::OutputQueueMerger nb_out_msgs(shard_, std::move(neighbor_queues)); while (!nb_out_msgs.is_eof()) { From d5a56b7c2bcd57c08a27fedf45b701e96ce86634 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 12 Jul 2023 09:01:15 +0300 Subject: [PATCH 051/388] Changes in CollatorNode --- validator/collator-node.cpp | 225 +++++++++++++++++++++++++----------- validator/collator-node.hpp | 33 +++++- 2 files changed, 187 insertions(+), 71 deletions(-) diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 654e4e742..516555fae 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -17,6 +17,8 @@ #include "collator-node.hpp" #include "ton/ton-tl.hpp" #include "fabric.h" +#include "block-auto.h" +#include "block-db.h" namespace ton { @@ -69,33 +71,35 @@ void CollatorNode::del_shard(ShardIdFull shard) { } void CollatorNode::new_masterchain_block_notification(td::Ref state) { - std::vector top_blocks = {state->get_block_id()}; + last_masterchain_block_ = state->get_block_id(); + last_top_blocks_.clear(); + last_top_blocks_[ShardIdFull{masterchainId, shardIdAll}] = last_masterchain_block_; std::vector next_shards; - if (collate_shard(ShardIdFull(masterchainId))) { + if (can_collate_shard(ShardIdFull(masterchainId))) { next_shards.push_back(ShardIdFull(masterchainId)); } for (const auto& desc : state->get_shards()) { - top_blocks.push_back(desc->top_block_id()); + last_top_blocks_[desc->shard()] = desc->top_block_id(); ShardIdFull shard = desc->shard(); if (desc->before_split()) { - if (collate_shard(shard_child(shard, true))) { + if (can_collate_shard(shard_child(shard, true))) { next_shards.push_back(shard_child(shard, true)); } - if (collate_shard(shard_child(shard, false))) { + if (can_collate_shard(shard_child(shard, false))) { next_shards.push_back(shard_child(shard, false)); } } else if (desc->before_merge()) { - if (is_left_child(shard) && collate_shard(shard_parent(shard))) { + if (is_left_child(shard) && can_collate_shard(shard_parent(shard))) { next_shards.push_back(shard_parent(shard)); } - } else if (collate_shard(shard)) { + } else if (can_collate_shard(shard)) { next_shards.push_back(shard); } } for (const ShardIdFull& shard : next_shards) { - for (const BlockIdExt& neighbor : top_blocks) { - if (neighbor.shard_full() != shard && block::ShardConfig::is_neighbor(shard, neighbor.shard_full())) { - td::actor::send_closure(manager_, &ValidatorManager::wait_out_msg_queue_proof, neighbor, shard, 0, + for (const auto& neighbor : last_top_blocks_) { + if (neighbor.first != shard && block::ShardConfig::is_neighbor(shard, neighbor.first)) { + td::actor::send_closure(manager_, &ValidatorManager::wait_out_msg_queue_proof, neighbor.second, shard, 0, td::Timestamp::in(10.0), [](td::Ref) {}); } } @@ -116,88 +120,173 @@ void CollatorNode::new_masterchain_block_notification(td::Ref } } } + // Remove old cache entries + auto it = cache_.begin(); + while (it != cache_.end()) { + auto prev_block_id = std::get<2>(it->first)[0]; + auto top_block = get_shard_top_block(prev_block_id.shard_full()); + if (top_block && top_block.value().seqno() > prev_block_id.seqno()) { + it = cache_.erase(it); + } else { + ++it; + } + } } static td::BufferSlice serialize_error(td::Status error) { return create_serialize_tl_object(error.code(), error.message().c_str()); } +static td::BufferSlice serialize_response(BlockCandidate block) { + return create_serialize_tl_object(create_tl_object( + PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), std::move(block.data), + std::move(block.collated_data))); +} + +static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey creator) { + CHECK(!block.id.is_masterchain()); + if (block.pubkey == creator) { + return block; + } + auto root = vm::std_boc_deserialize(block.data).move_as_ok(); + block::gen::Block::Record blk; + block::gen::BlockExtra::Record extra; + CHECK(tlb::unpack_cell(root, blk)); + CHECK(tlb::unpack_cell(blk.extra, extra)); + extra.created_by = creator.as_bits256(); + CHECK(tlb::pack_cell(blk.extra, extra)); + CHECK(tlb::pack_cell(root, blk)); + block.data = vm::std_boc_serialize(root, 31).move_as_ok(); + + block.id.root_hash = root->get_hash().bits(); + block.id.file_hash = block::compute_file_hash(block.data.as_slice()); + block.pubkey = creator; + return block; +} + void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise) { - auto SelfId = actor_id(this); - auto status = [&]() -> td::Status { - if (!validators_.count(src)) { - return td::Status::Error("src is not a validator"); - } - TRY_RESULT(f, fetch_tl_object(std::move(data), true)); - ShardIdFull shard(f->workchain_, f->shard_); - if (!shard.is_valid_ext()) { - return td::Status::Error(PSTRING() << "invalid shard " << shard.to_str()); - } - if (!collate_shard(shard)) { - return td::Status::Error(PSTRING() << "this node doesn't collate shard " << shard.to_str()); + td::Promise new_promise = [promise = std::move(promise), src](td::Result R) mutable { + if (R.is_error()) { + LOG(WARNING) << "Query from " << src << ", error: " << R.error(); + promise.set_result(serialize_error(R.move_as_error())); + } else { + LOG(INFO) << "Query from " << src << ", success"; + promise.set_result(serialize_response(R.move_as_ok())); } - if (f->prev_blocks_.size() != 1 && f->prev_blocks_.size() != 2) { - return td::Status::Error(PSTRING() << "invalid size of prev_blocks: " << f->prev_blocks_.size()); + }; + if (!last_masterchain_block_.is_valid()) { + new_promise.set_error(td::Status::Error("not ready")); + return; + } + if (!validators_.count(src)) { + new_promise.set_error(td::Status::Error("src is not a validator")); + } + TRY_RESULT_PROMISE(new_promise, f, fetch_tl_object(std::move(data), true)); + ShardIdFull shard(f->workchain_, f->shard_); + BlockIdExt min_mc_id = create_block_id(f->min_mc_id_); + LOG(INFO) << "Got query from " << src << ": shard=" << shard.to_str() << ", min_mc_seqno=" << min_mc_id.seqno(); + if (!shard.is_valid_ext()) { + new_promise.set_error(td::Status::Error(PSTRING() << "invalid shard " << shard.to_str())); + return; + } + if (!can_collate_shard(shard)) { + new_promise.set_error(td::Status::Error(PSTRING() << "this node doesn't collate shard " << shard.to_str())); + return; + } + if (f->prev_blocks_.size() != 1 && f->prev_blocks_.size() != 2) { + new_promise.set_error(td::Status::Error(PSTRING() << "invalid size of prev_blocks: " << f->prev_blocks_.size())); + return; + } + if (!min_mc_id.is_masterchain_ext()) { + new_promise.set_error(td::Status::Error("min_mc_id is not form masterchain")); + return; + } + std::vector prev_blocks; + for (const auto& b : f->prev_blocks_) { + auto id = create_block_id(b); + if (!id.is_valid_full()) { + new_promise.set_error(td::Status::Error("invalid prev_block")); + return; } - std::vector prev_blocks; - for (const auto& b : f->prev_blocks_) { - prev_blocks.push_back(create_block_id(b)); + auto top_block = get_shard_top_block(id.shard_full()); + if (top_block && top_block.value().seqno() > id.seqno()) { + new_promise.set_error(td::Status::Error("cannot collate block: already exists in blockchain")); + return; } - BlockIdExt min_mc_id = create_block_id(f->min_mc_id_); - Ed25519_PublicKey creator(f->creator_); - - LOG(INFO) << "Query from " << src << ": shard=" << shard.to_str() << ", min_mc_id=" << min_mc_id.to_str(); - - auto P = td::PromiseCreator::lambda([=, prev_blocks = std::move(prev_blocks), - promise = std::move(promise)](td::Result> R) mutable { + prev_blocks.push_back(id); + } + Ed25519_PublicKey creator(f->creator_); + if (!shard.is_masterchain()) { + // Collation of masterchain cannot be cached because changing "created_by" in masterchain is hard + // It does not really matter because validators can collate masterchain themselves + new_promise = [promise = std::move(new_promise), creator](td::Result R) mutable { if (R.is_error()) { - LOG(WARNING) << "Query from " << src << ", error: " << R.error(); - promise.set_result(serialize_error(R.move_as_error_prefix("failed to get masterchain state: "))); + promise.set_error(R.move_as_error()); } else { - td::Ref state(R.move_as_ok()); - if (state.is_null()) { - LOG(WARNING) << "Query from " << src << ", error: failed to get masterchain state"; - promise.set_result(serialize_error(R.move_as_error_prefix("failed to get masterchain state: "))); - return; - } - td::actor::send_closure(SelfId, &CollatorNode::receive_query_cont, src, shard, std::move(state), - std::move(prev_blocks), creator, std::move(promise)); + promise.set_result(change_creator(R.move_as_ok(), creator)); } - }); - td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, min_mc_id, 1, td::Timestamp::in(5.0), - std::move(P)); - return td::Status::OK(); - }(); - if (status.is_error()) { - LOG(WARNING) << "Query from " << src << ", error: " << status; - promise.set_result(serialize_error(std::move(status))); + }; + auto cache_key = std::make_tuple(min_mc_id.seqno(), shard, prev_blocks); + auto cache_entry = cache_[cache_key]; + if (cache_entry == nullptr) { + cache_entry = cache_[cache_key] = std::make_shared(); + } + if (cache_entry->result) { + new_promise.set_result(cache_entry->result.value().clone()); + return; + } + cache_entry->promises.push_back(std::move(new_promise)); + if (cache_entry->started) { + return; + } + cache_entry->started = true; + new_promise = [SelfId = actor_id(this), cache_entry](td::Result R) mutable { + td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R)); + }; } -} -void CollatorNode::receive_query_cont(adnl::AdnlNodeIdShort src, ShardIdFull shard, - td::Ref min_mc_state, std::vector prev_blocks, - Ed25519_PublicKey creator, td::Promise promise) { - auto P = td::PromiseCreator::lambda([promise = std::move(promise), src](td::Result R) mutable { + auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this), prev_blocks = std::move(prev_blocks), + promise = std::move(new_promise)](td::Result> R) mutable { if (R.is_error()) { - LOG(WARNING) << "Query from " << src << ", error: " << R.error(); - promise.set_result(serialize_error(R.move_as_error())); + promise.set_error(R.move_as_error_prefix("failed to get masterchain state: ")); } else { - LOG(INFO) << "Query from " << src << ", success"; - auto block = R.move_as_ok(); - auto result = create_serialize_tl_object( - create_tl_object(PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), - create_tl_block_id(block.id), std::move(block.data), - std::move(block.collated_data))); - promise.set_result(std::move(result)); + td::Ref state(R.move_as_ok()); + if (state.is_null()) { + promise.set_error(R.move_as_error_prefix("failed to get masterchain state: ")); + return; + } + td::actor::send_closure(SelfId, &CollatorNode::receive_query_cont, shard, std::move(state), + std::move(prev_blocks), creator, std::move(promise)); } }); + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, min_mc_id, 1, td::Timestamp::in(5.0), + std::move(P)); +} +void CollatorNode::receive_query_cont(ShardIdFull shard, td::Ref min_mc_state, + std::vector prev_blocks, Ed25519_PublicKey creator, + td::Promise promise) { run_collate_query(shard, min_mc_state->get_block_id(), std::move(prev_blocks), creator, - min_mc_state->get_validator_set(shard), manager_, td::Timestamp::in(10.0), std::move(P)); + min_mc_state->get_validator_set(shard), manager_, td::Timestamp::in(10.0), std::move(promise)); +} + +void CollatorNode::process_result(std::shared_ptr cache_entry, td::Result R) { + if (R.is_error()) { + cache_entry->started = false; + for (auto& p : cache_entry->promises) { + p.set_error(R.error().clone()); + } + } else { + cache_entry->result = R.move_as_ok(); + for (auto& p : cache_entry->promises) { + p.set_result(cache_entry->result.value().clone()); + } + } + cache_entry->promises.clear(); } -bool CollatorNode::collate_shard(ShardIdFull shard) const { +bool CollatorNode::can_collate_shard(ShardIdFull shard) const { for (ShardIdFull our_shard : shards_) { if (shard_intersects(shard, our_shard)) { return true; diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp index 46848fcfe..d640244a7 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node.hpp @@ -18,6 +18,7 @@ #include "interfaces/validator-manager.h" #include "rldp/rldp.h" +#include namespace ton { @@ -38,11 +39,11 @@ class CollatorNode : public td::actor::Actor { private: void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); - void receive_query_cont(adnl::AdnlNodeIdShort src, ShardIdFull shard, td::Ref min_mc_state, + void receive_query_cont(ShardIdFull shard, td::Ref min_mc_state, std::vector prev_blocks, Ed25519_PublicKey creator, - td::Promise promise); + td::Promise promise); - bool collate_shard(ShardIdFull shard) const; + bool can_collate_shard(ShardIdFull shard) const; adnl::AdnlNodeIdShort local_id_; td::actor::ActorId manager_; @@ -50,6 +51,32 @@ class CollatorNode : public td::actor::Actor { td::actor::ActorId rldp_; std::vector shards_; std::set validators_; + + BlockIdExt last_masterchain_block_{}; + std::map last_top_blocks_; + + struct CacheEntry { + bool started = false; + td::optional result; + std::vector> promises; + }; + std::map>, std::shared_ptr> cache_; + + td::optional get_shard_top_block(ShardIdFull shard) const { + auto it = last_top_blocks_.lower_bound(shard); + if (it != last_top_blocks_.end() && shard_intersects(it->first, shard)) { + return it->second; + } + if (it != last_top_blocks_.begin()) { + --it; + if (shard_intersects(it->first, shard)) { + return it->second; + } + } + return {}; + } + + void process_result(std::shared_ptr cache_entry, td::Result R); }; } // namespace validator From 21ce145af2aeb628c154ac5d805ac5f5b7c20941 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 12 Jul 2023 13:29:59 +0300 Subject: [PATCH 052/388] Limit collated data size in collator --- crypto/block/block.cpp | 23 ++++++++++++++++------- crypto/block/block.h | 7 +++++-- crypto/vm/boc.cpp | 31 +++++++++++++++++++++++++++++++ crypto/vm/boc.h | 13 +++++++++++++ crypto/vm/cells/CellUsageTree.cpp | 12 +++++++++--- crypto/vm/cells/CellUsageTree.h | 13 +++++++++++-- crypto/vm/cells/MerkleProof.h | 4 ++++ crypto/vm/cells/UsageCell.h | 2 +- validator/impl/collator.cpp | 28 ++++++++++++++++++++++++++-- 9 files changed, 116 insertions(+), 17 deletions(-) diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp index 1131213c6..b9ecfe60c 100644 --- a/crypto/block/block.cpp +++ b/crypto/block/block.cpp @@ -701,12 +701,19 @@ int BlockLimits::classify_lt(ton::LogicalTime lt) const { return lt_delta.classify(lt - start_lt); } -int BlockLimits::classify(td::uint64 size, td::uint64 gas, ton::LogicalTime lt) const { - return std::max(std::max(classify_size(size), classify_gas(gas)), classify_lt(lt)); +int BlockLimits::classify_collated_data_size(td::uint64 size) const { + return bytes.classify(size); // TODO: Maybe separate limits in config } -bool BlockLimits::fits(unsigned cls, td::uint64 size, td::uint64 gas_value, ton::LogicalTime lt) const { - return bytes.fits(cls, size) && gas.fits(cls, gas_value) && lt_delta.fits(cls, lt - start_lt); +int BlockLimits::classify(td::uint64 size, td::uint64 gas, ton::LogicalTime lt, td::uint64 collated_size) const { + return std::max( + {classify_size(size), classify_gas(gas), classify_lt(lt), classify_collated_data_size(collated_size)}); +} + +bool BlockLimits::fits(unsigned cls, td::uint64 size, td::uint64 gas_value, ton::LogicalTime lt, + td::uint64 collated_size) const { + return bytes.fits(cls, size) && gas.fits(cls, gas_value) && lt_delta.fits(cls, lt - start_lt) && + bytes.fits(cls, collated_size); } td::uint64 BlockLimitStatus::estimate_block_size(const vm::NewCellStorageStat::Stat* extra) const { @@ -719,20 +726,22 @@ td::uint64 BlockLimitStatus::estimate_block_size(const vm::NewCellStorageStat::S } int BlockLimitStatus::classify() const { - return limits.classify(estimate_block_size(), gas_used, cur_lt); + return limits.classify(estimate_block_size(), gas_used, cur_lt, collated_data_stat.estimate_proof_size()); } bool BlockLimitStatus::fits(unsigned cls) const { return cls >= ParamLimits::limits_cnt || (limits.gas.fits(cls, gas_used) && limits.lt_delta.fits(cls, cur_lt - limits.start_lt) && - limits.bytes.fits(cls, estimate_block_size())); + limits.bytes.fits(cls, estimate_block_size()) && + limits.bytes.fits(cls, collated_data_stat.estimate_proof_size())); } bool BlockLimitStatus::would_fit(unsigned cls, ton::LogicalTime end_lt, td::uint64 more_gas, const vm::NewCellStorageStat::Stat* extra) const { return cls >= ParamLimits::limits_cnt || (limits.gas.fits(cls, gas_used + more_gas) && limits.lt_delta.fits(cls, std::max(cur_lt, end_lt) - limits.start_lt) && - limits.bytes.fits(cls, estimate_block_size(extra))); + limits.bytes.fits(cls, estimate_block_size(extra)) && + limits.bytes.fits(cls, collated_data_stat.estimate_proof_size())); } // SETS: account_dict, shard_libraries_, mc_state_extra diff --git a/crypto/block/block.h b/crypto/block/block.h index 19d99e6a0..b6b46a3d6 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -252,8 +252,9 @@ struct BlockLimits { int classify_size(td::uint64 size) const; int classify_gas(td::uint64 gas) const; int classify_lt(ton::LogicalTime lt) const; - int classify(td::uint64 size, td::uint64 gas, ton::LogicalTime lt) const; - bool fits(unsigned cls, td::uint64 size, td::uint64 gas, ton::LogicalTime lt) const; + int classify_collated_data_size(td::uint64 size) const; + int classify(td::uint64 size, td::uint64 gas, ton::LogicalTime lt, td::uint64 collated_size) const; + bool fits(unsigned cls, td::uint64 size, td::uint64 gas, ton::LogicalTime lt, td::uint64 collated_size) const; }; struct BlockLimitStatus { @@ -262,6 +263,7 @@ struct BlockLimitStatus { td::uint64 gas_used{}; vm::NewCellStorageStat st_stat; unsigned accounts{}, transactions{}, extra_out_msgs{}; + vm::ProofStorageStat collated_data_stat; BlockLimitStatus(const BlockLimits& limits_, ton::LogicalTime lt = 0) : limits(limits_), cur_lt(std::max(limits_.start_lt, lt)) { } @@ -271,6 +273,7 @@ struct BlockLimitStatus { transactions = accounts = 0; gas_used = 0; extra_out_msgs = 0; + collated_data_stat = {}; } td::uint64 estimate_block_size(const vm::NewCellStorageStat::Stat* extra = nullptr) const; int classify() const; diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index 11583ede6..0d84c1189 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -1214,4 +1214,35 @@ bool VmStorageStat::add_storage(const CellSlice& cs) { return true; } +static td::uint64 estimate_prunned_size() { + return 41; +} + +static td::uint64 estimate_serialized_size(const Ref& cell) { + return cell->get_serialized_size() + cell->size_refs() * 3 + 3; +} + +void ProofStorageStat::add_cell(const Ref& cell) { + auto& status = cells_[cell->get_hash()]; + if (status == c_loaded) { + return; + } + if (status == c_prunned) { + proof_size_ -= estimate_prunned_size(); + } + status = c_loaded; + proof_size_ += estimate_serialized_size(cell); + for (unsigned i = 0; i < cell->size_refs(); ++i) { + auto& child_status = cells_[cell->get_ref(i)->get_hash()]; + if (child_status == c_none) { + child_status = c_prunned; + proof_size_ += estimate_prunned_size(); + } + } +} + +td::uint64 ProofStorageStat::estimate_proof_size() const { + return proof_size_; +} + } // namespace vm diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index c7a1810d7..d6a7f9adc 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -41,6 +41,7 @@ class NewCellStorageStat { Stat(td::uint64 cells_, td::uint64 bits_, td::uint64 internal_refs_ = 0, td::uint64 external_refs_ = 0) : cells(cells_), bits(bits_), internal_refs(internal_refs_), external_refs(external_refs_) { } + Stat(const Stat&) = default; td::uint64 cells{0}; td::uint64 bits{0}; td::uint64 internal_refs{0}; @@ -160,6 +161,18 @@ struct VmStorageStat { } }; +class ProofStorageStat { + public: + void add_cell(const Ref& cell); + td::uint64 estimate_proof_size() const; + private: + enum CellStatus { + c_none = 0, c_prunned = 1, c_loaded = 2 + }; + std::map cells_; + td::uint64 proof_size_ = 0; +}; + struct CellSerializationInfo { bool special; Cell::LevelMask level_mask; diff --git a/crypto/vm/cells/CellUsageTree.cpp b/crypto/vm/cells/CellUsageTree.cpp index 3f43ec6bd..410b3fcd6 100644 --- a/crypto/vm/cells/CellUsageTree.cpp +++ b/crypto/vm/cells/CellUsageTree.cpp @@ -22,12 +22,12 @@ namespace vm { // // CellUsageTree::NodePtr // -bool CellUsageTree::NodePtr::on_load() const { +bool CellUsageTree::NodePtr::on_load(const td::Ref& cell) const { auto tree = tree_weak_.lock(); if (!tree) { return false; } - tree->on_load(node_id_); + tree->on_load(node_id_, cell); return true; } @@ -111,8 +111,14 @@ void CellUsageTree::set_use_mark_for_is_loaded(bool use_mark) { use_mark_ = use_mark; } -void CellUsageTree::on_load(NodeId node_id) { +void CellUsageTree::on_load(NodeId node_id, const td::Ref& cell) { + if (nodes_[node_id].is_loaded) { + return; + } nodes_[node_id].is_loaded = true; + if (cell_load_callback_) { + cell_load_callback_(cell); + } } CellUsageTree::NodeId CellUsageTree::create_child(NodeId node_id, unsigned ref_id) { diff --git a/crypto/vm/cells/CellUsageTree.h b/crypto/vm/cells/CellUsageTree.h index 150dd2bda..af0f21f53 100644 --- a/crypto/vm/cells/CellUsageTree.h +++ b/crypto/vm/cells/CellUsageTree.h @@ -22,8 +22,12 @@ #include "td/utils/int_types.h" #include "td/utils/logging.h" +#include namespace vm { + +class DataCell; + class CellUsageTree : public std::enable_shared_from_this { public: using NodeId = td::uint32; @@ -38,7 +42,7 @@ class CellUsageTree : public std::enable_shared_from_this { return node_id_ == 0 || tree_weak_.expired(); } - bool on_load() const; + bool on_load(const td::Ref& cell) const; NodePtr create_child(unsigned ref_id) const; bool mark_path(CellUsageTree* master_tree) const; bool is_from_tree(const CellUsageTree* master_tree) const; @@ -59,6 +63,10 @@ class CellUsageTree : public std::enable_shared_from_this { void set_use_mark_for_is_loaded(bool use_mark = true); NodeId create_child(NodeId node_id, unsigned ref_id); + void set_cell_load_callback(std::function&)> f) { + cell_load_callback_ = std::move(f); + } + private: struct Node { bool is_loaded{false}; @@ -68,8 +76,9 @@ class CellUsageTree : public std::enable_shared_from_this { }; bool use_mark_{false}; std::vector nodes_{2}; + std::function&)> cell_load_callback_; - void on_load(NodeId node_id); + void on_load(NodeId node_id, const td::Ref& cell); NodeId create_node(NodeId parent); }; } // namespace vm diff --git a/crypto/vm/cells/MerkleProof.h b/crypto/vm/cells/MerkleProof.h index 9c50fd078..fc2cb6ebd 100644 --- a/crypto/vm/cells/MerkleProof.h +++ b/crypto/vm/cells/MerkleProof.h @@ -66,6 +66,10 @@ class MerkleProofBuilder { td::Result> extract_proof() const; bool extract_proof_to(Ref &proof_root) const; td::Result extract_proof_boc() const; + + void set_cell_load_callback(std::function&)> f) { + usage_tree->set_cell_load_callback(std::move(f)); + } }; } // namespace vm diff --git a/crypto/vm/cells/UsageCell.h b/crypto/vm/cells/UsageCell.h index bf15bb56f..3e6e88982 100644 --- a/crypto/vm/cells/UsageCell.h +++ b/crypto/vm/cells/UsageCell.h @@ -39,7 +39,7 @@ class UsageCell : public Cell { // load interface td::Result load_cell() const override { TRY_RESULT(loaded_cell, cell_->load_cell()); - if (tree_node_.on_load()) { + if (tree_node_.on_load(loaded_cell.data_cell)) { CHECK(loaded_cell.tree_node.empty()); loaded_cell.tree_node = tree_node_; } diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 1ce0a4b4c..fc31d2645 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -661,6 +661,13 @@ void Collator::got_neighbor_msg_queue(unsigned i, td::Resultstate_root_}); state_root = neighbor_proof_builders_.back().root(); + if (full_collated_data_ && !block_id.is_masterchain()) { + neighbor_proof_builders_.back().set_cell_load_callback([&](const td::Ref& cell) { + if (block_limit_status_) { + block_limit_status_->collated_data_stat.add_cell(cell); + } + }); + } } auto state = ShardStateQ::fetch(block_id, {}, state_root); if (state.is_error()) { @@ -745,6 +752,13 @@ bool Collator::unpack_merge_last_state() { } // 1. prepare for creating a MerkleUpdate based on previous state state_usage_tree_ = std::make_shared(); + if (full_collated_data_ && !is_masterchain()) { + state_usage_tree_->set_cell_load_callback([&](const td::Ref& cell) { + if (block_limit_status_) { + block_limit_status_->collated_data_stat.add_cell(cell); + } + }); + } prev_state_root_ = vm::UsageCell::create(prev_state_root_pure_, state_usage_tree_->root_ptr()); // 2. extract back slightly virtualized roots of the two original states Ref root0, root1; @@ -783,6 +797,13 @@ bool Collator::unpack_last_state() { prev_state_root_pure_ = prev_states.at(0)->root_cell(); // prepare for creating a MerkleUpdate based on previous state state_usage_tree_ = std::make_shared(); + if (full_collated_data_ && !is_masterchain()) { + state_usage_tree_->set_cell_load_callback([&](const td::Ref& cell) { + if (block_limit_status_) { + block_limit_status_->collated_data_stat.add_cell(cell); + } + }); + } prev_state_root_ = vm::UsageCell::create(prev_state_root_pure_, state_usage_tree_->root_ptr()); // unpack previous state block::ShardState ss; @@ -3547,7 +3568,8 @@ bool Collator::check_block_overload() { block_size_estimate_ = block_limit_status_->estimate_block_size(); LOG(INFO) << "block load statistics: gas=" << block_limit_status_->gas_used << " lt_delta=" << block_limit_status_->cur_lt - block_limit_status_->limits.start_lt - << " size_estimate=" << block_size_estimate_; + << " size_estimate=" << block_size_estimate_ + << " collated_size_estimate=" << block_limit_status_->collated_data_stat.estimate_proof_size(); auto cl = block_limit_status_->classify(); if (cl <= block::ParamLimits::cl_underload) { underload_history_ |= 1; @@ -4153,11 +4175,13 @@ bool Collator::create_block_candidate() { cdata_slice = cdata_res.move_as_ok(); } LOG(INFO) << "serialized block size " << blk_slice.size() << " bytes (preliminary estimate was " - << block_size_estimate_ << "), collated data " << cdata_slice.size() << " bytes"; + << block_size_estimate_ << ")"; auto st = block_limit_status_->st_stat.get_total_stat(); LOG(INFO) << "size regression stats: " << blk_slice.size() << " " << st.cells << " " << st.bits << " " << st.internal_refs << " " << st.external_refs << " " << block_limit_status_->accounts << " " << block_limit_status_->transactions; + LOG(INFO) << "serialized collated data size " << cdata_slice.size() << " bytes (preliminary estimate was " + << block_limit_status_->collated_data_stat.estimate_proof_size() << ")"; // 3. create a BlockCandidate block_candidate = std::make_unique( created_by_, From d115807f6e4d5fc67481a3f9ccb40c19bacb07d0 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 14 Jul 2023 13:13:00 +0300 Subject: [PATCH 053/388] Move ExtClientLazy and QueryTraits to lite-client-common --- lite-client/CMakeLists.txt | 4 +-- {tonlib/tonlib => lite-client}/QueryTraits.h | 8 ++++- .../ext-client.cpp | 35 +++++++++---------- .../ext-client.h | 24 +++++++------ tonlib/CMakeLists.txt | 2 -- tonlib/tonlib/Config.h | 8 ++--- tonlib/tonlib/ExtClient.cpp | 2 +- tonlib/tonlib/ExtClient.h | 10 +++--- tonlib/tonlib/ExtClientOutbound.h | 5 ++- tonlib/tonlib/TonlibClient.cpp | 5 ++- tonlib/tonlib/TonlibClient.h | 3 +- tonlib/tonlib/tonlib-cli.cpp | 11 +++--- 12 files changed, 57 insertions(+), 60 deletions(-) rename {tonlib/tonlib => lite-client}/QueryTraits.h (97%) rename tonlib/tonlib/ExtClientLazy.cpp => lite-client/ext-client.cpp (84%) rename tonlib/tonlib/ExtClientLazy.h => lite-client/ext-client.h (67%) diff --git a/lite-client/CMakeLists.txt b/lite-client/CMakeLists.txt index b84495162..fd5c812cf 100644 --- a/lite-client/CMakeLists.txt +++ b/lite-client/CMakeLists.txt @@ -1,9 +1,9 @@ cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) -add_library(lite-client-common lite-client-common.cpp lite-client-common.h) +add_library(lite-client-common lite-client-common.cpp lite-client-common.h ext-client.cpp ext-client.h QueryTraits.h) target_link_libraries(lite-client-common PUBLIC tdutils tdactor adnllite tl_api tl_lite_api tl-lite-utils ton_crypto ton_block) -add_executable(lite-client lite-client.cpp lite-client.h) +add_executable(lite-client lite-client.cpp lite-client.h ext-client.h ext-client.cpp) target_link_libraries(lite-client tdutils tdactor adnllite tl_api tl_lite_api tl-lite-utils ton_crypto ton_block terminal lite-client-common git) diff --git a/tonlib/tonlib/QueryTraits.h b/lite-client/QueryTraits.h similarity index 97% rename from tonlib/tonlib/QueryTraits.h rename to lite-client/QueryTraits.h index b583cd05b..ffa683e62 100644 --- a/tonlib/tonlib/QueryTraits.h +++ b/lite-client/QueryTraits.h @@ -17,12 +17,14 @@ #pragma once #include "ton/ton-types.h" #include "auto/tl/lite_api.h" +#include "auto/tl/lite_api.hpp" #include "vm/boc.h" #include "vm/cellslice.h" #include "block/block-auto.h" #include "block/block-parse.h" +#include "auto/tl/lite_api.hpp" -namespace tonlib { +namespace liteclient { template struct QueryTraits { @@ -212,5 +214,9 @@ struct QueryTraits { } }; +template +inline ton::ShardIdFull get_query_shard(const Query& q) { + return QueryTraits::get_shard(q); +} } // namespace tonlib diff --git a/tonlib/tonlib/ExtClientLazy.cpp b/lite-client/ext-client.cpp similarity index 84% rename from tonlib/tonlib/ExtClientLazy.cpp rename to lite-client/ext-client.cpp index 027198de4..e79ae9a4f 100644 --- a/tonlib/tonlib/ExtClientLazy.cpp +++ b/lite-client/ext-client.cpp @@ -13,20 +13,17 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP */ -#include "ExtClientLazy.h" -#include "TonlibError.h" +#include "ext-client.h" #include "td/utils/Random.h" #include "ton/ton-shard.h" #include -namespace tonlib { +namespace liteclient { -class ExtClientLazyImpl : public ExtClientLazy { +class ExtClientImpl : public ExtClient { public: - ExtClientLazyImpl(std::vector servers, td::unique_ptr callback) + ExtClientImpl(std::vector servers, td::unique_ptr callback) : callback_(std::move(callback)) { CHECK(!servers.empty()); servers_.resize(servers.size()); @@ -53,10 +50,10 @@ class ExtClientLazyImpl : public ExtClientLazy { CHECK(!server.client.empty()); alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT)); td::Promise P = [SelfId = actor_id(this), server_idx, - promise = std::move(promise)](td::Result R) mutable { + promise = std::move(promise)](td::Result R) mutable { if (R.is_error() && (R.error().code() == ton::ErrorCode::timeout || R.error().code() == ton::ErrorCode::cancelled)) { - td::actor::send_closure(SelfId, &ExtClientLazyImpl::set_server_bad, server_idx); + td::actor::send_closure(SelfId, &ExtClientImpl::set_server_bad, server_idx); } promise.set_result(std::move(R)); }; @@ -128,17 +125,17 @@ class ExtClientLazyImpl : public ExtClientLazy { class Callback : public ton::adnl::AdnlExtClient::Callback { public: - explicit Callback(td::actor::ActorShared parent, size_t idx) + explicit Callback(td::actor::ActorShared parent, size_t idx) : parent_(std::move(parent)), idx_(idx) { } void on_ready() override { } void on_stop_ready() override { - td::actor::send_closure(parent_, &ExtClientLazyImpl::set_server_bad, idx_); + td::actor::send_closure(parent_, &ExtClientImpl::set_server_bad, idx_); } private: - td::actor::ActorShared parent_; + td::actor::ActorShared parent_; size_t idx_; }; ref_cnt_++; @@ -154,7 +151,7 @@ class ExtClientLazyImpl : public ExtClientLazy { } struct Server { - Config::LiteServer s; + LiteServer s; td::actor::ActorOwn client; td::Timestamp timeout = td::Timestamp::never(); td::Timestamp ignore_until = td::Timestamp::never(); @@ -169,7 +166,7 @@ class ExtClientLazyImpl : public ExtClientLazy { std::map shard_to_server_; int max_server_shard_depth_ = 0; - td::unique_ptr callback_; + td::unique_ptr callback_; static constexpr double MAX_NO_QUERIES_TIMEOUT = 100; bool is_closing_{false}; @@ -206,13 +203,13 @@ class ExtClientLazyImpl : public ExtClientLazy { } }; -td::actor::ActorOwn ExtClientLazy::create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, +td::actor::ActorOwn ExtClient::create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, td::unique_ptr callback) { - return create({Config::LiteServer{dst, dst_addr, true, {}}}, std::move(callback)); + return create({LiteServer{dst, dst_addr, true, {}}}, std::move(callback)); } -td::actor::ActorOwn ExtClientLazy::create(std::vector servers, +td::actor::ActorOwn ExtClient::create(std::vector servers, td::unique_ptr callback) { - return td::actor::create_actor("ExtClientLazy", std::move(servers), std::move(callback)); + return td::actor::create_actor("ExtClient", std::move(servers), std::move(callback)); } -} // namespace tonlib +} // namespace liteclient diff --git a/tonlib/tonlib/ExtClientLazy.h b/lite-client/ext-client.h similarity index 67% rename from tonlib/tonlib/ExtClientLazy.h rename to lite-client/ext-client.h index 6383cec55..c8569964f 100644 --- a/tonlib/tonlib/ExtClientLazy.h +++ b/lite-client/ext-client.h @@ -13,18 +13,22 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP */ #pragma once #include "td/actor/actor.h" #include "ton/ton-types.h" #include "adnl/adnl-ext-client.h" -#include "Config.h" -namespace tonlib { -class ExtClientLazy : public td::actor::Actor { +namespace liteclient { +class ExtClient : public td::actor::Actor { public: + struct LiteServer { + ton::adnl::AdnlNodeIdFull adnl_id; + td::IPAddress address; + bool is_full = true; + std::vector shards; + }; + class Callback { public: virtual ~Callback() { @@ -35,10 +39,8 @@ class ExtClientLazy : public td::actor::Actor { td::Promise promise) = 0; virtual void force_change_liteserver() = 0; - static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, - td::unique_ptr callback); - static td::actor::ActorOwn create(std::vector servers, - td::unique_ptr callback); + static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, + td::unique_ptr callback); + static td::actor::ActorOwn create(std::vector servers, td::unique_ptr callback); }; - -} // namespace tonlib +} // namespace liteclient diff --git a/tonlib/CMakeLists.txt b/tonlib/CMakeLists.txt index 72c59c4da..8ee8fcc69 100644 --- a/tonlib/CMakeLists.txt +++ b/tonlib/CMakeLists.txt @@ -10,7 +10,6 @@ set(TONLIB_SOURCE tonlib/Client.cpp tonlib/Config.cpp tonlib/ExtClient.cpp - tonlib/ExtClientLazy.cpp tonlib/ExtClientOutbound.cpp tonlib/KeyStorage.cpp tonlib/KeyValue.cpp @@ -25,7 +24,6 @@ set(TONLIB_SOURCE tonlib/Client.h tonlib/Config.h tonlib/ExtClient.h - tonlib/ExtClientLazy.h tonlib/ExtClientOutbound.h tonlib/KeyStorage.h tonlib/KeyValue.h diff --git a/tonlib/tonlib/Config.h b/tonlib/tonlib/Config.h index 7af1b1c5a..5ae304ea8 100644 --- a/tonlib/tonlib/Config.h +++ b/tonlib/tonlib/Config.h @@ -20,15 +20,11 @@ #include "adnl/adnl-node-id.hpp" #include "td/utils/port/IPAddress.h" #include "ton/ton-types.h" +#include "lite-client/ext-client.h" namespace tonlib { struct Config { - struct LiteServer { - ton::adnl::AdnlNodeIdFull adnl_id; - td::IPAddress address; - bool is_full = true; - std::vector shards; - }; + using LiteServer = liteclient::ExtClient::LiteServer; ton::BlockIdExt zero_state_id; ton::BlockIdExt init_block_id; std::vector hardforks; diff --git a/tonlib/tonlib/ExtClient.cpp b/tonlib/tonlib/ExtClient.cpp index 755be170d..f0c84d616 100644 --- a/tonlib/tonlib/ExtClient.cpp +++ b/tonlib/tonlib/ExtClient.cpp @@ -65,7 +65,7 @@ void ExtClient::send_raw_query(td::BufferSlice query, ton::ShardIdFull shard, td if (client_.adnl_ext_client_.empty()) { return P.set_error(TonlibError::NoLiteServers()); } - td::actor::send_closure(client_.adnl_ext_client_, &ExtClientLazy::send_query, "query", std::move(query), + td::actor::send_closure(client_.adnl_ext_client_, &liteclient::ExtClient::send_query, "query", std::move(query), shard, td::Timestamp::in(10.0), std::move(P)); } } // namespace tonlib diff --git a/tonlib/tonlib/ExtClient.h b/tonlib/tonlib/ExtClient.h index 5cd7f4fd6..9db040bb1 100644 --- a/tonlib/tonlib/ExtClient.h +++ b/tonlib/tonlib/ExtClient.h @@ -28,10 +28,10 @@ #include "td/utils/Container.h" #include "td/utils/Random.h" -#include "ExtClientLazy.h" +#include "lite-client/ext-client.h" #include "TonlibError.h" #include "utils.h" -#include "QueryTraits.h" +#include "lite-client/QueryTraits.h" namespace tonlib { class LastBlock; @@ -39,7 +39,7 @@ class LastConfig; struct LastBlockState; struct LastConfigState; struct ExtClientRef { - td::actor::ActorId adnl_ext_client_; + td::actor::ActorId adnl_ext_client_; td::actor::ActorId last_block_actor_; td::actor::ActorId last_config_actor_; }; @@ -65,7 +65,7 @@ class ExtClient { template void send_query(QueryT query, td::Promise promise, td::int32 seq_no = -1) { - ton::ShardIdFull shard = QueryTraits::get_shard(query); + ton::ShardIdFull shard = liteclient::QueryTraits::get_shard(query); auto raw_query = ton::serialize_tl_object(&query, true); td::uint32 tag = td::Random::fast_uint32(); VLOG(lite_server) << "send query to liteserver: " << tag << " " << to_string(query); @@ -99,7 +99,7 @@ class ExtClient { void force_change_liteserver() { if (!client_.adnl_ext_client_.empty()) { - td::actor::send_closure(client_.adnl_ext_client_, &ExtClientLazy::force_change_liteserver); + td::actor::send_closure(client_.adnl_ext_client_, &liteclient::ExtClient::force_change_liteserver); } } diff --git a/tonlib/tonlib/ExtClientOutbound.h b/tonlib/tonlib/ExtClientOutbound.h index 31f76b977..6eb6aa983 100644 --- a/tonlib/tonlib/ExtClientOutbound.h +++ b/tonlib/tonlib/ExtClientOutbound.h @@ -18,11 +18,10 @@ */ #pragma once #include "td/actor/actor.h" - -#include "ExtClientLazy.h" +#include "lite-client/ext-client.h" namespace tonlib { -class ExtClientOutbound : public ExtClientLazy { +class ExtClientOutbound : public liteclient::ExtClient { public: class Callback { public: diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 1682c7803..3bb3f8a21 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -18,7 +18,6 @@ */ #include "TonlibClient.h" -#include "tonlib/ExtClientLazy.h" #include "tonlib/ExtClientOutbound.h" #include "tonlib/LastBlock.h" #include "tonlib/LastConfig.h" @@ -2022,7 +2021,7 @@ void TonlibClient::init_ext_client() { ext_client_outbound_ = client.get(); raw_client_ = std::move(client); } else { - class Callback : public ExtClientLazy::Callback { + class Callback : public liteclient::ExtClient::Callback { public: explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { } @@ -2032,7 +2031,7 @@ void TonlibClient::init_ext_client() { }; ext_client_outbound_ = {}; ref_cnt_++; - raw_client_ = ExtClientLazy::create(config_.lite_servers, td::make_unique(td::actor::actor_shared())); + raw_client_ = liteclient::ExtClient::create(config_.lite_servers, td::make_unique(td::actor::actor_shared())); } } diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index e31653693..b1d649efe 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -33,6 +33,7 @@ #include "td/utils/optional.h" #include "smc-envelope/ManualDns.h" +#include "lite-client/ext-client.h" #include @@ -112,7 +113,7 @@ class TonlibClient : public td::actor::Actor { vm::Dictionary libraries{256}; // network - td::actor::ActorOwn raw_client_; + td::actor::ActorOwn raw_client_; td::actor::ActorId ext_client_outbound_; td::actor::ActorOwn raw_last_block_; td::actor::ActorOwn raw_last_config_; diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index 92749d089..8669dda32 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -47,8 +47,6 @@ #include "tonlib/TonlibClient.h" #include "tonlib/TonlibCallback.h" -#include "tonlib/ExtClientLazy.h" - #include "smc-envelope/ManualDns.h" #include "smc-envelope/PaymentChannel.h" @@ -57,6 +55,7 @@ #include "crypto/util/Miner.h" #include "vm/boc.h" #include "vm/cells/CellBuilder.h" +#include "lite-client/ext-client.h" #include #include @@ -174,7 +173,7 @@ class TonlibCli : public td::actor::Actor { std::map>> query_handlers_; - td::actor::ActorOwn raw_client_; + td::actor::ActorOwn raw_client_; bool is_closing_{false}; td::uint32 ref_cnt_{1}; @@ -223,7 +222,7 @@ class TonlibCli : public td::actor::Actor { if (options_.use_callbacks_for_network) { auto config = tonlib::Config::parse(options_.config).move_as_ok(); - class Callback : public tonlib::ExtClientLazy::Callback { + class Callback : public liteclient::ExtClient::Callback { public: explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { } @@ -232,7 +231,7 @@ class TonlibCli : public td::actor::Actor { td::actor::ActorShared<> parent_; }; ref_cnt_++; - raw_client_ = tonlib::ExtClientLazy::create(config.lite_servers, + raw_client_ = liteclient::ExtClient::create(config.lite_servers, td::make_unique(td::actor::actor_shared())); } @@ -1533,7 +1532,7 @@ class TonlibCli : public td::actor::Actor { auto update = tonlib_api::move_object_as(std::move(result)); CHECK(!raw_client_.empty()); snd_bytes_ += update->data_.size(); - send_closure(raw_client_, &tonlib::ExtClientLazy::send_query, "query", td::BufferSlice(update->data_), + send_closure(raw_client_, &liteclient::ExtClient::send_query, "query", td::BufferSlice(update->data_), ton::ShardIdFull(update->workchain_, update->shard_), td::Timestamp::in(5), [actor_id = actor_id(this), id = update->id_](td::Result res) { send_closure(actor_id, &TonlibCli::on_adnl_result, id, std::move(res)); From b422d95b95d5389cc432e42e289687df7b6c56d4 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 14 Jul 2023 13:15:10 +0300 Subject: [PATCH 054/388] Add missing query to QueryTraits --- lite-client/QueryTraits.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lite-client/QueryTraits.h b/lite-client/QueryTraits.h index ffa683e62..54190244e 100644 --- a/lite-client/QueryTraits.h +++ b/lite-client/QueryTraits.h @@ -172,6 +172,13 @@ struct QueryTraits { } }; +template<> +struct QueryTraits { + static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_listBlockTransactionsExt& q) { + return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); + } +}; + template<> struct QueryTraits { static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getBlockProof& q) { From ac6cc3bafd5f45f97956db820f88a77806ec3726 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 14 Jul 2023 15:51:27 +0300 Subject: [PATCH 055/388] Use partial liteservers in blockchain-explorer --- blockchain-explorer/CMakeLists.txt | 4 +- .../blockchain-explorer-query.cpp | 145 ++++++++---------- blockchain-explorer/blockchain-explorer.cpp | 93 +++++------ blockchain-explorer/blockchain-explorer.hpp | 2 +- 4 files changed, 112 insertions(+), 132 deletions(-) diff --git a/blockchain-explorer/CMakeLists.txt b/blockchain-explorer/CMakeLists.txt index 11328a7a3..b1979e70e 100644 --- a/blockchain-explorer/CMakeLists.txt +++ b/blockchain-explorer/CMakeLists.txt @@ -17,11 +17,11 @@ if (NIX) find_package(PkgConfig REQUIRED) pkg_check_modules(MHD libmicrohttpd) target_include_directories(blockchain-explorer PUBLIC ${MHD_INCLUDE_DIRS} ${MHD_STATIC_INCLUDE_DIRS}) - target_link_libraries(blockchain-explorer tdutils tdactor adnllite tl_lite_api tl-lite-utils ton_crypto ton_block ${MHD_LIBRARIES} ${MHD_STATIC_LIBRARIES}) + target_link_libraries(blockchain-explorer tdutils tdactor adnllite tl_lite_api tl-lite-utils ton_crypto ton_block lite-client-common ${MHD_LIBRARIES} ${MHD_STATIC_LIBRARIES}) else() find_package(MHD) target_include_directories(blockchain-explorer PUBLIC ${MHD_INCLUDE_DIRS}) - target_link_libraries(blockchain-explorer tdutils tdactor adnllite tl_lite_api tl-lite-utils ton_crypto ton_block ${MHD_LIBRARIES}) + target_link_libraries(blockchain-explorer tdutils tdactor adnllite tl_lite_api tl-lite-utils ton_crypto ton_block lite-client-common ${MHD_LIBRARIES}) endif() install(TARGETS blockchain-explorer RUNTIME DESTINATION bin) diff --git a/blockchain-explorer/blockchain-explorer-query.cpp b/blockchain-explorer/blockchain-explorer-query.cpp index b53e79696..9f8ea7419 100644 --- a/blockchain-explorer/blockchain-explorer-query.cpp +++ b/blockchain-explorer/blockchain-explorer-query.cpp @@ -43,6 +43,7 @@ #include "block/block-auto.h" #include "crypto/vm/utils.h" #include "td/utils/crypto.h" +#include "lite-client/QueryTraits.h" #include "vm/boc.h" #include "vm/cellops.h" @@ -237,8 +238,7 @@ void HttpQueryBlockData::finish_query() { } void HttpQueryBlockData::start_up() { - auto query = ton::serialize_tl_object( - ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)), true); + auto query = ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { @@ -249,7 +249,7 @@ void HttpQueryBlockData::start_up() { }); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query), std::move(P)); + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); } void HttpQueryBlockData::got_block_data(td::BufferSlice data) { @@ -300,8 +300,7 @@ void HttpQueryBlockView::finish_query() { } void HttpQueryBlockView::start_up_query() { - auto query = ton::serialize_tl_object( - ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)), true); + auto query = ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { @@ -312,7 +311,7 @@ void HttpQueryBlockView::start_up_query() { }); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query), std::move(P)); + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); } void HttpQueryBlockView::got_block_data(td::BufferSlice data) { @@ -348,11 +347,10 @@ void HttpQueryBlockInfo::start_up_query() { td::actor::send_closure(SelfId, &HttpQueryBlockInfo::got_block_header, R.move_as_ok()); } }); - auto query = ton::serialize_tl_object( - ton::create_tl_object(ton::create_tl_lite_block_id(block_id_), 0), - true); + auto query = + ton::create_tl_object(ton::create_tl_lite_block_id(block_id_), 0); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query), std::move(P)); + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); pending_queries_ = 1; if (block_id_.is_masterchain()) { @@ -364,16 +362,15 @@ void HttpQueryBlockInfo::start_up_query() { td::actor::send_closure(SelfId, &HttpQueryBlockInfo::got_shard_info, R.move_as_ok()); } }); - auto query_2 = ton::serialize_tl_object( - ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)), - true); + auto query_2 = + ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query_2), std::move(P_2)); + ton::serialize_tl_object(query_2, true), liteclient::get_query_shard(*query_2), + std::move(P_2)); pending_queries_++; } - auto query_3 = ton::serialize_tl_object(ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), 7, 1024, nullptr, false, false), - true); + auto query_3 = ton::create_tl_object( + ton::create_tl_lite_block_id(block_id_), 7, 1024, nullptr, false, false); auto P_3 = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &HttpQueryBlockInfo::abort_query, R.move_as_error_prefix("litequery failed: ")); @@ -382,7 +379,8 @@ void HttpQueryBlockInfo::start_up_query() { } }); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query_3), std::move(P_3)); + ton::serialize_tl_object(query_3, true), liteclient::get_query_shard(*query_3), + std::move(P_3)); pending_queries_++; } @@ -435,11 +433,9 @@ void HttpQueryBlockInfo::got_transactions(td::BufferSlice data) { if (f->incomplete_ && transactions_.size() > 0) { const auto &T = *transactions_.rbegin(); - auto query_3 = ton::serialize_tl_object( - ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), 7 + 128, 1024, - ton::create_tl_object(T.addr.addr, T.lt), false, false), - true); + auto query_3 = ton::create_tl_object( + ton::create_tl_lite_block_id(block_id_), 7 + 128, 1024, + ton::create_tl_object(T.addr.addr, T.lt), false, false); auto P_3 = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &HttpQueryBlockInfo::abort_query, R.move_as_error_prefix("litequery failed: ")); @@ -448,7 +444,8 @@ void HttpQueryBlockInfo::got_transactions(td::BufferSlice data) { } }); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query_3), std::move(P_3)); + ton::serialize_tl_object(query_3, true), liteclient::get_query_shard(*query_3), + std::move(P_3)); } else { if (!--pending_queries_) { finish_query(); @@ -568,14 +565,13 @@ void HttpQueryBlockSearch::start_up_query() { td::actor::send_closure(SelfId, &HttpQueryBlockSearch::got_block_header, R.move_as_ok()); } }); - auto query = ton::serialize_tl_object(ton::create_tl_object( - mode_, - ton::create_tl_lite_block_id_simple(ton::BlockId{ - account_prefix_.workchain, account_prefix_.account_id_prefix, seqno_}), - lt_, utime_), - true); + auto query = ton::create_tl_object( + mode_, + ton::create_tl_lite_block_id_simple( + ton::BlockId{account_prefix_.workchain, account_prefix_.account_id_prefix, seqno_}), + lt_, utime_); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query), std::move(P)); + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); } void HttpQueryBlockSearch::got_block_header(td::BufferSlice data) { @@ -597,17 +593,16 @@ void HttpQueryBlockSearch::got_block_header(td::BufferSlice data) { td::actor::send_closure(SelfId, &HttpQueryBlockSearch::got_shard_info, R.move_as_ok()); } }); - auto query_2 = ton::serialize_tl_object( - ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)), - true); + auto query_2 = + ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query_2), std::move(P_2)); + ton::serialize_tl_object(query_2, true), liteclient::get_query_shard(*query_2), + std::move(P_2)); pending_queries_++; } - auto query_3 = ton::serialize_tl_object(ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), 7, 1024, nullptr, false, false), - true); + auto query_3 = ton::create_tl_object( + ton::create_tl_lite_block_id(block_id_), 7, 1024, nullptr, false, false); auto P_3 = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &HttpQueryBlockSearch::abort_query, R.move_as_error_prefix("litequery failed: ")); @@ -616,7 +611,8 @@ void HttpQueryBlockSearch::got_block_header(td::BufferSlice data) { } }); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query_3), std::move(P_3)); + ton::serialize_tl_object(query_3, true), liteclient::get_query_shard(*query_3), + std::move(P_3)); pending_queries_++; } @@ -656,11 +652,9 @@ void HttpQueryBlockSearch::got_transactions(td::BufferSlice data) { if (f->incomplete_ && transactions_.size() > 0) { const auto &T = *transactions_.rbegin(); - auto query_3 = ton::serialize_tl_object( - ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), 7 + 128, 1024, - ton::create_tl_object(T.addr.addr, T.lt), false, false), - true); + auto query_3 = ton::create_tl_object( + ton::create_tl_lite_block_id(block_id_), 7 + 128, 1024, + ton::create_tl_object(T.addr.addr, T.lt), false, false); auto P_3 = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &HttpQueryBlockSearch::abort_query, @@ -670,7 +664,8 @@ void HttpQueryBlockSearch::got_transactions(td::BufferSlice data) { } }); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query_3), std::move(P_3)); + ton::serialize_tl_object(query_3, true), liteclient::get_query_shard(*query_3), + std::move(P_3)); } else { if (!--pending_queries_) { finish_query(); @@ -758,11 +753,10 @@ void HttpQueryViewAccount::start_up_query() { } }); auto a = ton::create_tl_object(addr_.workchain, addr_.addr); - auto query = ton::serialize_tl_object(ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), std::move(a)), - true); + auto query = ton::create_tl_object(ton::create_tl_lite_block_id(block_id_), + std::move(a)); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query), std::move(P)); + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); } void HttpQueryViewAccount::got_account(td::BufferSlice data) { @@ -855,10 +849,9 @@ void HttpQueryViewTransaction::start_up_query() { } }); auto a = ton::create_tl_object(addr_.workchain, addr_.addr); - auto query = ton::serialize_tl_object( - ton::create_tl_object(1, std::move(a), lt_, hash_), true); + auto query = ton::create_tl_object(1, std::move(a), lt_, hash_); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query), std::move(P)); + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); } void HttpQueryViewTransaction::got_transaction(td::BufferSlice data) { @@ -946,11 +939,10 @@ void HttpQueryViewTransaction2::start_up_query() { } }); auto a = ton::create_tl_object(addr_.workchain, addr_.addr); - auto query = ton::serialize_tl_object(ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), std::move(a), lt_), - true); + auto query = ton::create_tl_object( + ton::create_tl_lite_block_id(block_id_), std::move(a), lt_); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query), std::move(P)); + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); } void HttpQueryViewTransaction2::got_transaction(td::BufferSlice data) { @@ -1009,9 +1001,9 @@ void HttpQueryViewLastBlock::start_up() { } }); - auto query = ton::serialize_tl_object(ton::create_tl_object(), true); + auto query = ton::create_tl_object(); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query), std::move(P)); + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); } void HttpQueryViewLastBlock::got_result(td::BufferSlice data) { @@ -1075,9 +1067,9 @@ void HttpQueryConfig::start_up() { } }); - auto query = ton::serialize_tl_object(ton::create_tl_object(), true); + auto query = ton::create_tl_object(); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query), std::move(P)); + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); } } @@ -1101,16 +1093,17 @@ void HttpQueryConfig::send_main_query() { td::actor::send_closure(SelfId, &HttpQueryConfig::got_result, R.move_as_ok()); } }); - auto query = - params_.size() > 0 - ? ton::serialize_tl_object(ton::create_tl_object( - 0, ton::create_tl_lite_block_id(block_id_), std::vector(params_)), - true) - : ton::serialize_tl_object(ton::create_tl_object( - 0, ton::create_tl_lite_block_id(block_id_)), - true); - td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query), std::move(P)); + if (params_.size() > 0) { + auto query = ton::create_tl_object( + 0, ton::create_tl_lite_block_id(block_id_), std::vector(params_)); + td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + } else { + auto query = + ton::create_tl_object(0, ton::create_tl_lite_block_id(block_id_)); + td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + } } void HttpQueryConfig::got_result(td::BufferSlice data) { @@ -1252,10 +1245,9 @@ void HttpQuerySend::start_up() { td::actor::send_closure(SelfId, &HttpQuerySend::got_result, R.move_as_ok()); } }); - auto query = - ton::serialize_tl_object(ton::create_tl_object(std::move(data_)), true); + auto query = ton::create_tl_object(std::move(data_)); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query), std::move(P)); + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); } void HttpQuerySend::got_result(td::BufferSlice data) { @@ -1347,11 +1339,10 @@ void HttpQueryRunMethod::start_up_query() { } }); auto a = ton::create_tl_object(addr_.workchain, addr_.addr); - auto query = ton::serialize_tl_object(ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), std::move(a)), - true); + auto query = ton::create_tl_object(ton::create_tl_lite_block_id(block_id_), + std::move(a)); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - std::move(query), std::move(P)); + ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); } void HttpQueryRunMethod::got_account(td::BufferSlice data) { diff --git a/blockchain-explorer/blockchain-explorer.cpp b/blockchain-explorer/blockchain-explorer.cpp index 035b84ea7..9baee7d8d 100644 --- a/blockchain-explorer/blockchain-explorer.cpp +++ b/blockchain-explorer/blockchain-explorer.cpp @@ -57,6 +57,7 @@ #include "auto/tl/lite_api.h" #include "ton/lite-tl.hpp" #include "tl-utils/lite-utils.hpp" +#include "lite-client/ext-client.h" #include @@ -126,7 +127,7 @@ class CoreActor : public CoreActorInterface { private: std::string global_config_ = "ton-global.config"; - std::vector> clients_; + td::actor::ActorOwn client_; td::uint32 http_port_ = 80; MHD_Daemon* daemon_ = nullptr; @@ -136,24 +137,17 @@ class CoreActor : public CoreActorInterface { bool hide_ips_ = false; - std::unique_ptr make_callback(td::uint32 idx) { - class Callback : public ton::adnl::AdnlExtClient::Callback { + td::unique_ptr make_callback() { + class Callback : public liteclient::ExtClient::Callback { public: - void on_ready() override { - td::actor::send_closure(id_, &CoreActor::conn_ready, idx_); - } - void on_stop_ready() override { - td::actor::send_closure(id_, &CoreActor::conn_closed, idx_); - } - Callback(td::actor::ActorId id, td::uint32 idx) : id_(std::move(id)), idx_(idx) { + Callback(td::actor::ActorId id) : id_(std::move(id)) { } private: td::actor::ActorId id_; - td::uint32 idx_; }; - return std::make_unique(actor_id(this), idx); + return td::make_unique(actor_id(this)); } std::shared_ptr new_result_; @@ -162,9 +156,8 @@ class CoreActor : public CoreActorInterface { std::vector ready_; - void run_queries(); + //void run_queries(); void got_result(td::uint32 idx, td::int32 attempt, td::Result data); - void send_query(td::uint32 idx); void add_result() { if (new_result_) { @@ -183,7 +176,7 @@ class CoreActor : public CoreActorInterface { add_result(); } attempt_ = t; - run_queries(); + //run_queries(); alarm_timestamp() = td::Timestamp::at_unix((attempt_ + 1) * 60); } @@ -225,10 +218,7 @@ class CoreActor : public CoreActorInterface { hide_ips_ = value; } - void send_lite_query(td::uint32 idx, td::BufferSlice query, td::Promise promise); - void send_lite_query(td::BufferSlice data, td::Promise promise) override { - return send_lite_query(0, std::move(data), std::move(promise)); - } + void send_lite_query(td::BufferSlice query, ton::ShardIdFull shard, td::Promise promise) override; void get_last_result(td::Promise> promise) override { } void get_results(td::uint32 max, td::Promise promise) override { @@ -448,23 +438,39 @@ class CoreActor : public CoreActorInterface { } void run() { + std::vector servers; if (remote_public_key_.empty()) { auto G = td::read_file(global_config_).move_as_ok(); auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); ton::ton_api::liteclient_config_global gc; ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); - CHECK(gc.liteservers_.size() > 0); - td::uint32 size = static_cast(gc.liteservers_.size()); + size_t size = gc.liteservers_.size() + gc.liteservers_v2_.size(); + CHECK(size > 0); ready_.resize(size, false); - for (td::uint32 i = 0; i < size; i++) { - auto& cli = gc.liteservers_[i]; + for (auto& s : gc.liteservers_) { td::IPAddress addr; - addr.init_host_port(td::IPAddress::ipv4_to_str(cli->ip_), cli->port_).ensure(); + addr.init_host_port(td::IPAddress::ipv4_to_str(s->ip_), s->port_).ensure(); addrs_.push_back(addr); - clients_.emplace_back(ton::adnl::AdnlExtClient::create(ton::adnl::AdnlNodeIdFull::create(cli->id_).move_as_ok(), - addr, make_callback(i))); + liteclient::ExtClient::LiteServer serv; + serv.address = addr; + serv.adnl_id = ton::adnl::AdnlNodeIdFull::create(s->id_).move_as_ok(); + servers.push_back(std::move(serv)); + } + for (auto& s : gc.liteservers_v2_) { + td::IPAddress addr; + addr.init_host_port(td::IPAddress::ipv4_to_str(s->ip_), s->port_).ensure(); + addrs_.push_back(addr); + liteclient::ExtClient::LiteServer serv; + serv.address = addr; + serv.adnl_id = ton::adnl::AdnlNodeIdFull::create(s->id_).move_as_ok(); + serv.is_full = false; + for (auto& shard : s->shards_) { + serv.shards.emplace_back(shard->workchain_, (ton::ShardId)shard->shard_); + CHECK(serv.shards.back().is_valid_ext()); + } + servers.push_back(std::move(serv)); } } else { if (!remote_addr_.is_valid()) { @@ -472,9 +478,12 @@ class CoreActor : public CoreActorInterface { } ready_.resize(1, false); addrs_.push_back(remote_addr_); - clients_.emplace_back(ton::adnl::AdnlExtClient::create(ton::adnl::AdnlNodeIdFull{remote_public_key_}, - remote_addr_, make_callback(0))); + liteclient::ExtClient::LiteServer serv; + serv.address = remote_addr_; + serv.adnl_id = ton::adnl::AdnlNodeIdFull{remote_public_key_}; + servers.push_back(std::move(serv)); } + client_ = liteclient::ExtClient::create(std::move(servers), make_callback()); daemon_ = MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, static_cast(http_port_), nullptr, nullptr, &process_http_request, nullptr, MHD_OPTION_NOTIFY_COMPLETED, request_completed, nullptr, MHD_OPTION_THREAD_POOL_SIZE, 16, MHD_OPTION_END); @@ -523,23 +532,7 @@ void CoreActor::got_result(td::uint32 idx, td::int32 attempt, td::Result(); - auto q = ton::create_tl_object(serialize_tl_object(query, true)); - - auto P = - td::PromiseCreator::lambda([SelfId = actor_id(this), idx, attempt = attempt_](td::Result R) { - td::actor::send_closure(SelfId, &CoreActor::got_result, idx, attempt, std::move(R)); - }); - td::actor::send_closure(clients_[idx], &ton::adnl::AdnlExtClient::send_query, "query", serialize_tl_object(q, true), - td::Timestamp::in(10.0), std::move(P)); -} - -void CoreActor::run_queries() { +/*void CoreActor::run_queries() { waiting_ = 0; new_result_ = std::make_shared(ready_.size(), td::Timestamp::at_unix(attempt_ * 60)); for (td::uint32 i = 0; i < ready_.size(); i++) { @@ -549,13 +542,9 @@ void CoreActor::run_queries() { if (waiting_ == 0) { add_result(); } -} +}*/ -void CoreActor::send_lite_query(td::uint32 idx, td::BufferSlice query, td::Promise promise) { - if (!ready_[idx]) { - promise.set_error(td::Status::Error(ton::ErrorCode::notready, "ext conn not ready")); - return; - } +void CoreActor::send_lite_query(td::BufferSlice query, ton::ShardIdFull shard, td::Promise promise) { auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); @@ -573,7 +562,7 @@ void CoreActor::send_lite_query(td::uint32 idx, td::BufferSlice query, td::Promi promise.set_value(std::move(B)); }); auto q = ton::create_tl_object(std::move(query)); - td::actor::send_closure(clients_[idx], &ton::adnl::AdnlExtClient::send_query, "query", serialize_tl_object(q, true), + td::actor::send_closure(client_, &liteclient::ExtClient::send_query, "query", serialize_tl_object(q, true), shard, td::Timestamp::in(10.0), std::move(P)); } diff --git a/blockchain-explorer/blockchain-explorer.hpp b/blockchain-explorer/blockchain-explorer.hpp index 1bae362de..d3c53e451 100644 --- a/blockchain-explorer/blockchain-explorer.hpp +++ b/blockchain-explorer/blockchain-explorer.hpp @@ -64,7 +64,7 @@ class CoreActorInterface : public td::actor::Actor { }; virtual ~CoreActorInterface() = default; - virtual void send_lite_query(td::BufferSlice data, td::Promise promise) = 0; + virtual void send_lite_query(td::BufferSlice data, ton::ShardIdFull shard, td::Promise promise) = 0; virtual void get_last_result(td::Promise> promise) = 0; virtual void get_results(td::uint32 max, td::Promise promise) = 0; From 96afdc12ac547a5b12541f0432f50e5ab83610ab Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 14 Jul 2023 17:16:49 +0300 Subject: [PATCH 056/388] Increase INACTIVE_SHARD_TTL Node in overlay should be available until peer is expired --- overlay/overlay-peers.cpp | 4 ++-- overlay/overlays.h | 4 ++++ validator/full-node.cpp | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index 28dac8f05..8c4f5d228 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -100,7 +100,7 @@ void OverlayImpl::add_peer_in(OverlayNode node) { return; } auto t = td::Clocks::system(); - if (node.version() + 600 < t || node.version() > t + 60) { + if (node.version() + Overlays::overlay_peer_ttl() < t || node.version() > t + 60) { VLOG(OVERLAY_INFO) << this << ": ignoring node of too old version " << node.version(); return; } @@ -231,7 +231,7 @@ void OverlayImpl::update_neighbours(td::uint32 nodes_to_change) { continue; } - if (X->get_version() <= td::Clocks::system() - 600) { + if (X->get_version() <= td::Clocks::system() - Overlays::overlay_peer_ttl()) { if (X->is_neighbour()) { bool found = false; for (auto &n : neighbours_) { diff --git a/overlay/overlays.h b/overlay/overlays.h index 1141efee2..cf153c3a5 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -187,6 +187,10 @@ class Overlays : public td::actor::Actor { return 1; } + static constexpr td::uint32 overlay_peer_ttl() { + return 600; + } + static td::actor::ActorOwn create(std::string db_root, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId dht); diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 669e2fd97..cfe493667 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -27,7 +27,7 @@ namespace validator { namespace fullnode { -static const double INACTIVE_SHARD_TTL = 120.0; +static const double INACTIVE_SHARD_TTL = (double)overlay::Overlays::overlay_peer_ttl() + 60.0; void FullNodeImpl::add_permanent_key(PublicKeyHash key, td::Promise promise) { if (local_keys_.count(key)) { From b8f347231fd4246736da8300d81df0ff73861f08 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 17 Jul 2023 18:45:32 +0300 Subject: [PATCH 057/388] Fix shard overlays --- validator/full-node.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/full-node.cpp b/validator/full-node.cpp index cfe493667..3897626f0 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -147,7 +147,7 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s new_shards.insert(ShardIdFull(masterchainId)); std::set workchains; auto cut_shard = [&](ShardIdFull shard) -> ShardIdFull { - int min_split = state->soft_min_split_depth(shard.workchain); + int min_split = state->min_split_depth(shard.workchain); return min_split < shard.pfx_len() ? shard_prefix(shard, min_split) : shard; }; auto set_active = [&](ShardIdFull shard, FullNodeShardMode mode) { From 3265e397f27d50b6d45b6e0d9e6b20f724467ba1 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 17 Jul 2023 19:14:39 +0300 Subject: [PATCH 058/388] Rename actual_min_split and soft_mis_split to avoid confusion --- crypto/block/block.tlb | 8 ++++---- crypto/block/mc-config.cpp | 2 +- crypto/block/mc-config.h | 2 +- validator/full-node.cpp | 2 +- validator/impl/shard.cpp | 6 +++--- validator/impl/shard.hpp | 2 +- validator/interfaces/shard.h | 2 +- validator/validator-options.hpp | 2 +- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 14ec54f47..f2f11fc71 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -646,15 +646,15 @@ wc_split_merge_timings#0 //workchain#a5 enabled_since:uint32 min_split:(## 8) max_split:(## 8) // { min_split <= max_split } { max_split <= 60 } -workchain#a6 enabled_since:uint32 actual_min_split:(## 8) - min_split:(## 8) max_split:(## 8) { actual_min_split <= min_split } +workchain#a6 enabled_since:uint32 monitor_min_split:(## 8) + min_split:(## 8) max_split:(## 8) { monitor_min_split <= min_split } basic:(## 1) active:Bool accept_msgs:Bool flags:(## 13) { flags = 0 } zerostate_root_hash:bits256 zerostate_file_hash:bits256 version:uint32 format:(WorkchainFormat basic) = WorkchainDescr; -workchain_v2#a7 enabled_since:uint32 actual_min_split:(## 8) - min_split:(## 8) max_split:(## 8) { actual_min_split <= min_split } +workchain_v2#a7 enabled_since:uint32 monitor_min_split:(## 8) + min_split:(## 8) max_split:(## 8) { monitor_min_split <= min_split } basic:(## 1) active:Bool accept_msgs:Bool flags:(## 13) { flags = 0 } zerostate_root_hash:bits256 zerostate_file_hash:bits256 version:uint32 format:(WorkchainFormat basic) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index b4de693cf..1ab907cf5 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2005,7 +2005,7 @@ bool WorkchainInfo::unpack(ton::WorkchainId wc, vm::CellSlice& cs) { } auto unpack_v1 = [this](auto& info) { enabled_since = info.enabled_since; - actual_min_split = info.actual_min_split; + monitor_min_split = info.monitor_min_split; min_split = info.min_split; max_split = info.max_split; basic = info.basic; diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 8eb8a7f9f..bf5350b50 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -407,7 +407,7 @@ struct CatchainValidatorsConfig { struct WorkchainInfo : public td::CntObject { ton::WorkchainId workchain{ton::workchainInvalid}; ton::UnixTime enabled_since; - td::uint32 actual_min_split; + td::uint32 monitor_min_split; td::uint32 min_split, max_split; bool basic; bool active; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 3897626f0..3b275ea99 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -147,7 +147,7 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s new_shards.insert(ShardIdFull(masterchainId)); std::set workchains; auto cut_shard = [&](ShardIdFull shard) -> ShardIdFull { - int min_split = state->min_split_depth(shard.workchain); + int min_split = state->monitor_min_split_depth(shard.workchain); return min_split < shard.pfx_len() ? shard_prefix(shard, min_split) : shard; }; auto set_active = [&](ShardIdFull shard, FullNodeShardMode mode) { diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index a09bd1df8..a7ff371ea 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -511,15 +511,15 @@ bool MasterchainStateQ::check_old_mc_block_id(const ton::BlockIdExt& blkid, bool return config_ && config_->check_old_mc_block_id(blkid, strict); } -td::uint32 MasterchainStateQ::min_split_depth(WorkchainId workchain_id) const { +td::uint32 MasterchainStateQ::monitor_min_split_depth(WorkchainId workchain_id) const { if (!config_) { return 0; } auto wc_info = config_->get_workchain_info(workchain_id); - return wc_info.not_null() ? wc_info->actual_min_split : 0; + return wc_info.not_null() ? wc_info->monitor_min_split : 0; } -td::uint32 MasterchainStateQ::soft_min_split_depth(WorkchainId workchain_id) const { +td::uint32 MasterchainStateQ::min_split_depth(WorkchainId workchain_id) const { if (!config_) { return 0; } diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index b55cac35c..83aa1f9a4 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -116,8 +116,8 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { bool has_workchain(WorkchainId workchain) const { return config_ && config_->has_workchain(workchain); } + td::uint32 monitor_min_split_depth(WorkchainId workchain_id) const override; td::uint32 min_split_depth(WorkchainId workchain_id) const override; - td::uint32 soft_min_split_depth(WorkchainId workchain_id) const override; BlockSeqno min_ref_masterchain_seqno() const override; td::Status prepare() override; ZeroStateIdExt get_zerostate_id() const { diff --git a/validator/interfaces/shard.h b/validator/interfaces/shard.h index ac355d7dc..9ccfa2fb4 100644 --- a/validator/interfaces/shard.h +++ b/validator/interfaces/shard.h @@ -69,8 +69,8 @@ class MasterchainState : virtual public ShardState { virtual std::vector> get_shards() const = 0; virtual td::Ref get_shard_from_config(ShardIdFull shard) const = 0; virtual bool workchain_is_active(WorkchainId workchain_id) const = 0; + virtual td::uint32 monitor_min_split_depth(WorkchainId workchain_id) const = 0; virtual td::uint32 min_split_depth(WorkchainId workchain_id) const = 0; - virtual td::uint32 soft_min_split_depth(WorkchainId workchain_id) const = 0; virtual BlockSeqno min_ref_masterchain_seqno() const = 0; virtual bool ancestor_is_valid(BlockIdExt id) const = 0; virtual ValidatorSessionConfig get_consensus_config() const = 0; diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index ea100a50b..fa7eb5f9b 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -33,7 +33,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { return init_block_id_; } bool need_monitor(ShardIdFull shard, const td::Ref& state) const override { - td::uint32 min_split = state->min_split_depth(shard.workchain); + td::uint32 min_split = state->monitor_min_split_depth(shard.workchain); return check_shard_((td::uint32)shard.pfx_len() <= min_split ? shard : shard_prefix(shard, min_split)); } bool allow_blockchain_init() const override { From aa4f5769cac9707365e7ad79eab8bbae3a8010a2 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 18 Jul 2023 17:06:06 +0300 Subject: [PATCH 059/388] More logs for collators and validators --- tdutils/td/utils/Timer.cpp | 4 ++++ tdutils/td/utils/Timer.h | 1 + validator/collator-node.cpp | 2 ++ validator/full-node-shard.cpp | 1 - validator/impl/collator.cpp | 2 ++ validator/impl/validate-query.cpp | 4 ++++ validator/validator-group.cpp | 36 +++++++++++++++++-------------- 7 files changed, 33 insertions(+), 17 deletions(-) diff --git a/tdutils/td/utils/Timer.cpp b/tdutils/td/utils/Timer.cpp index 0f6a7d6ed..1f72fba96 100644 --- a/tdutils/td/utils/Timer.cpp +++ b/tdutils/td/utils/Timer.cpp @@ -87,4 +87,8 @@ void PerfWarningTimer::reset() { start_at_ = 0; } +double PerfWarningTimer::elapsed() const { + return Time::now() - start_at_; +} + } // namespace td diff --git a/tdutils/td/utils/Timer.h b/tdutils/td/utils/Timer.h index 18ea35d7c..3e0cafbf5 100644 --- a/tdutils/td/utils/Timer.h +++ b/tdutils/td/utils/Timer.h @@ -53,6 +53,7 @@ class PerfWarningTimer { PerfWarningTimer &operator=(PerfWarningTimer &&) = delete; ~PerfWarningTimer(); void reset(); + double elapsed() const; private: string name_; diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 516555fae..2cc06b340 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -233,11 +233,13 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data cache_entry = cache_[cache_key] = std::make_shared(); } if (cache_entry->result) { + LOG(INFO) << "Using cached result"; new_promise.set_result(cache_entry->result.value().clone()); return; } cache_entry->promises.push_back(std::move(new_promise)); if (cache_entry->started) { + LOG(INFO) << "Collating of this block is already in progress, waiting"; return; } cache_entry->started = true; diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 02bf0e39d..b34432a93 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -927,7 +927,6 @@ void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, std::stri void FullNodeShardImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, td::Promise> promise) { // TODO: maybe more complex download (like other requests here) - // TODO: estimate max size auto &b = choose_neighbour(true); if (b.adnl_id == adnl::AdnlNodeIdShort::zero()) { promise.set_error(td::Status::Error(ErrorCode::notready, "no nodes")); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index fc31d2645..2d91489e0 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -279,6 +279,7 @@ bool Collator::fatal_error(td::Status error) { error.ensure_error(); LOG(ERROR) << "cannot generate block candidate for " << show_shard(shard_) << " : " << error.to_string(); if (busy_) { + LOG(INFO) << "collation took " << perf_timer_.elapsed() << " s"; main_promise(std::move(error)); busy_ = false; } @@ -4226,6 +4227,7 @@ void Collator::return_block_candidate(td::Result saved) { } else { CHECK(block_candidate); LOG(INFO) << "sending new BlockCandidate to Promise"; + LOG(INFO) << "collation took " << perf_timer_.elapsed() << " s"; main_promise(block_candidate->clone()); busy_ = false; stop(); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index f9e5be275..4c7384ee9 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -88,6 +88,7 @@ bool ValidateQuery::reject_query(std::string error, td::BufferSlice reason) { << " collated_data=" << block_candidate.collated_file_hash.to_hex()); errorlog::ErrorLog::log_file(block_candidate.data.clone()); errorlog::ErrorLog::log_file(block_candidate.collated_data.clone()); + LOG(INFO) << "validation took " << perf_timer_.elapsed() << " s"; main_promise.set_result(CandidateReject{std::move(error), std::move(reason)}); } stop(); @@ -108,6 +109,7 @@ bool ValidateQuery::soft_reject_query(std::string error, td::BufferSlice reason) << " collated_data=" << block_candidate.collated_file_hash.to_hex()); errorlog::ErrorLog::log_file(block_candidate.data.clone()); errorlog::ErrorLog::log_file(block_candidate.collated_data.clone()); + LOG(INFO) << "validation took " << perf_timer_.elapsed() << " s"; main_promise.set_result(CandidateReject{std::move(error), std::move(reason)}); } stop(); @@ -126,6 +128,7 @@ bool ValidateQuery::fatal_error(td::Status error) { errorlog::ErrorLog::log_file(block_candidate.data.clone()); errorlog::ErrorLog::log_file(block_candidate.collated_data.clone()); } + LOG(INFO) << "validation took " << perf_timer_.elapsed() << " s"; main_promise(std::move(error)); } stop(); @@ -147,6 +150,7 @@ bool ValidateQuery::fatal_error(std::string err_msg, int err_code) { void ValidateQuery::finish_query() { if (main_promise) { + LOG(INFO) << "validation took " << perf_timer_.elapsed() << " s"; main_promise.set_result(now_); } stop(); diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 7cc92f804..bc6a9812c 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -133,7 +133,7 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat P.set_error(td::Status::Error(ErrorCode::notready, "validator group not started")); return; } - VLOG(VALIDATOR_DEBUG) << "validating block candidate " << next_block_id; + VLOG(VALIDATOR_DEBUG) << "validating block candidate " << next_block_id.to_str(); block.id = next_block_id; run_validate_query(shard_, min_masterchain_block_id_, prev_block_ids_, std::move(block), validator_set_, manager_, td::Timestamp::in(15.0), std::move(P), @@ -397,6 +397,7 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo promise.set_error(td::Status::Error("too old")); return; } + BlockId next_block_id = create_next_block_id_simple(); adnl::AdnlNodeIdShort collator = adnl::AdnlNodeIdShort::zero(); // TODO: some way to choose node (similar to "unreliability" in full-node) int cnt = 0; @@ -413,20 +414,23 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo return; } - if (max_retries > 0) { - promise = td::PromiseCreator::lambda( - [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - if (R.is_ok()) { - promise.set_result(R.move_as_ok()); - } else if (timeout && timeout.is_in_past()) { - promise.set_result(R.move_as_error()); - } else { - LOG(WARNING) << "collate query error, retrying: " << R.move_as_error(); - td::actor::send_closure(SelfId, &ValidatorGroup::send_collate_query, round_id, timeout, std::move(promise), - max_retries - 1); - } - }); - } + promise = td::PromiseCreator::lambda([=, SelfId = actor_id(this), promise = std::move(promise), + timer = td::Timer()](td::Result R) mutable { + if (R.is_ok()) { + LOG(WARNING) << "collate query for " << next_block_id.to_str() << ": success, time=" << timer.elapsed() << "s"; + promise.set_result(R.move_as_ok()); + return; + } + bool retry = (!timeout || !timeout.is_in_past()) && max_retries > 0; + LOG(WARNING) << "collate query for " << next_block_id.to_str() << ": " << R.error() << ", time=" << timer.elapsed() + << "s, " << (retry ? "retrying" : "giving up"); + if (retry) { + td::actor::send_closure(SelfId, &ValidatorGroup::send_collate_query, round_id, timeout, std::move(promise), + max_retries - 1); + } else { + promise.set_result(td::Status::Error(ErrorCode::timeout, "timeout")); + } + }); std::vector> prev_blocks; for (const BlockIdExt &p : prev_block_ids_) { @@ -445,7 +449,7 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo td::actor::send_closure(SelfId, &ValidatorGroup::receive_collate_query_response, round_id, R.move_as_ok(), std::move(promise)); }); - LOG(INFO) << "collate query for " << create_next_block_id_simple().to_str() << ": send query to " << collator; + LOG(INFO) << "sending collate query for " << next_block_id.to_str() << ": send to " << collator; size_t max_answer_size = config_.max_block_size + config_.max_collated_data_size + 256; td::Timestamp query_timeout = td::Timestamp::in(10.0); query_timeout.relax(timeout); From e6b77ef71dcd71bbbc0e46b93e9c0a3d47dbc05e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 19 Jul 2023 14:53:10 +0300 Subject: [PATCH 060/388] Remove excessive check in check_neighbor_outbound_message --- validator/impl/validate-query.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 4c7384ee9..fcd869b1a 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -3983,14 +3983,6 @@ bool ValidateQuery::check_neighbor_outbound_message(Ref enq_msg, td::ConstBitPtr key, const block::McShardDescr& nb, bool& unprocessed) { unprocessed = false; - if (!block::gen::t_EnqueuedMsg.validate_csr(enq_msg)) { - return reject_query("EnqueuedMsg with key "s + key.to_hex(352) + " in outbound queue of our neighbor " + - nb.blk_.to_str() + " failed to pass automated validity tests"); - } - if (!block::tlb::t_EnqueuedMsg.validate_csr(enq_msg)) { - return reject_query("EnqueuedMsg with key "s + key.to_hex(352) + " in outbound queue of our neighbor " + - nb.blk_.to_str() + " failed to pass hand-written validity tests"); - } block::EnqueuedMsgDescr enq; if (!enq.unpack(enq_msg.write())) { // unpack EnqueuedMsg return reject_query("cannot unpack EnqueuedMsg with key "s + key.to_hex(352) + From 869c6fe675bb7ee936a5284d229eadb0ae0e40df Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 20 Jul 2023 17:48:52 +0300 Subject: [PATCH 061/388] Rework limiting imported msg queues --- crypto/block/block.tlb | 1 - crypto/block/output-queue-merger.cpp | 14 +++++--------- crypto/block/output-queue-merger.h | 2 ++ validator/impl/collator-impl.h | 1 - validator/impl/collator.cpp | 25 ++++++++++--------------- validator/impl/out-msg-queue-proof.cpp | 13 +++++++++++-- validator/impl/validate-query.cpp | 22 +--------------------- validator/impl/validate-query.hpp | 1 - validator/validator-group.cpp | 2 +- 9 files changed, 30 insertions(+), 51 deletions(-) diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index f2f11fc71..2176c0adb 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -829,7 +829,6 @@ top_block_descr#d5 proof_for:BlockIdExt signatures:(Maybe ^BlockSignatures) // COLLATED DATA // top_block_descr_set#4ac789f3 collection:(HashmapE 96 ^TopBlockDescr) = TopBlockDescrSet; -neighbor_msg_queue_limits#7e549333 neighbors:(HashmapE 96 int32) = NeighborMsgQueueLimits; // // VALIDATOR MISBEHAVIOR COMPLAINTS diff --git a/crypto/block/output-queue-merger.cpp b/crypto/block/output-queue-merger.cpp index 2955b3e01..f3ec61430 100644 --- a/crypto/block/output-queue-merger.cpp +++ b/crypto/block/output-queue-merger.cpp @@ -199,10 +199,6 @@ bool OutputQueueMerger::load() { unsigned long long lt = heap[0]->lt; std::size_t orig_size = msg_list.size(); do { - if (src_remaining_msgs_[heap[0]->source] == 0) { - std::pop_heap(heap.begin(), heap.end(), MsgKeyValue::greater); - continue; - } while (heap[0]->is_fork()) { auto other = std::make_unique(); if (!heap[0]->split(*other)) { @@ -218,17 +214,17 @@ bool OutputQueueMerger::load() { heap.pop_back(); } while (!heap.empty() && heap[0]->lt <= lt); std::sort(msg_list.begin() + orig_size, msg_list.end(), MsgKeyValue::less); - size_t j = orig_size; for (size_t i = orig_size; i < msg_list.size(); ++i) { td::int32 &remaining = src_remaining_msgs_[msg_list[i]->source]; - if (remaining != 0) { - if (remaining > 0) { + if (remaining != -1) { + if (remaining == 0) { + limit_exceeded = true; + } else { --remaining; } - msg_list[j++] = std::move(msg_list[i]); } + msg_list[i]->limit_exceeded = limit_exceeded; } - msg_list.resize(j); return msg_list.size() > orig_size; } diff --git a/crypto/block/output-queue-merger.h b/crypto/block/output-queue-merger.h index 3eb4b3799..26deb7ee4 100644 --- a/crypto/block/output-queue-merger.h +++ b/crypto/block/output-queue-merger.h @@ -32,6 +32,7 @@ struct OutputQueueMerger { int source; int key_len{0}; td::BitArray key; + bool limit_exceeded{false}; MsgKeyValue() = default; MsgKeyValue(int src, Ref node); MsgKeyValue(td::ConstBitPtr key_pfx, int key_pfx_len, int src, Ref node); @@ -82,6 +83,7 @@ struct OutputQueueMerger { std::vector src_remaining_msgs_; bool eof; bool failed; + bool limit_exceeded{false}; void add_root(int src, Ref outmsg_root, td::int32 msg_limit); bool load(); }; diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 59be8d4b4..2839a0e5b 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -134,7 +134,6 @@ class Collator final : public td::actor::Actor { std::unique_ptr shard_conf_; std::map> aux_mc_states_; std::map neighbor_msg_queues_limits_; - vm::Dictionary neighbor_msg_queues_limits_dict_{32 + 64}; std::vector neighbors_; std::unique_ptr nb_out_msgs_; std::vector special_smcs; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 2d91489e0..52d9c1c40 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -701,11 +701,7 @@ void Collator::got_neighbor_msg_queue(unsigned i, td::Resultprefetch_ref(0); descr.set_queue_root(queue_root); if (res->msg_count_ != -1) { - LOG(DEBUG) << "Neighbor " << descr.shard().to_str() << " has msg_limit=" << res->msg_count_; - td::BitArray<96> key; - key.bits().store_int(block_id.id.workchain, 32); - (key.bits() + 32).store_uint(block_id.id.shard, 64); - neighbor_msg_queues_limits_dict_.set_builder(key, vm::CellBuilder().store_long(res->msg_count_, 32)); + LOG(INFO) << "neighbor " << descr.shard().to_str() << " has msg_limit=" << res->msg_count_; neighbor_msg_queues_limits_[block_id.shard_full()] = res->msg_count_; } // comment the next two lines in the future when the output queues become huge @@ -2833,6 +2829,12 @@ bool Collator::process_inbound_internal_messages() { block_full_ = !block_limit_status_->fits(block::ParamLimits::cl_normal); auto kv = nb_out_msgs_->extract_cur(); CHECK(kv && kv->msg.not_null()); + if (kv->limit_exceeded) { + LOG(INFO) << "limit for imported messages is reached, stop processing inbound internal messages"; + block::EnqueuedMsgDescr enq; + enq.unpack(kv->msg.write()); // Visit cells to include it in proof + break; + } if (!precheck_inbound_message(kv->msg, kv->lt)) { if (verbosity > 1) { std::cerr << "invalid inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() @@ -4089,21 +4091,14 @@ bool Collator::create_collated_data() { } collated_roots_.push_back(std::move(cell)); } - // 2. Message count for neighbors' out queues - if (!neighbor_msg_queues_limits_dict_.is_empty()) { - vm::CellBuilder cb; - cb.store_long(block::gen::t_NeighborMsgQueueLimits.cons_tag[0], 32); - cb.store_maybe_ref(neighbor_msg_queues_limits_dict_.get_root_cell()); - collated_roots_.push_back(cb.finalize_novm()); - } if (!full_collated_data_) { return true; } - // 3. Proofs for hashes of states: previous states + neighbors + // 2. Proofs for hashes of states: previous states + neighbors for (const auto& p : block_state_proofs_) { collated_roots_.push_back(p.second); } - // 4. Previous state proof (only shadchains) + // 3. Previous state proof (only shadchains) std::map> proofs; if (!is_masterchain()) { if (!prepare_msg_queue_proof()) { @@ -4117,7 +4112,7 @@ bool Collator::create_collated_data() { } proofs[prev_state_root_->get_hash().bits()] = std::move(state_proof); } - // 5. Proofs for message queues + // 4. Proofs for message queues for (vm::MerkleProofBuilder &mpb : neighbor_proof_builders_) { auto r_proof = mpb.extract_proof(); if (r_proof.is_error()) { diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 329da607d..59aa8aed6 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -52,7 +52,7 @@ static td::Result process_queue(BlockIdExt block_id, ShardIdFull dst_ td::HashSet visited; std::function dfs_cs; - auto dfs = [&](Ref cell) { + auto dfs = [&](const Ref& cell) { if (cell.is_null() || !visited.insert(cell->get_hash()).second) { return; } @@ -65,6 +65,8 @@ static td::Result process_queue(BlockIdExt block_id, ShardIdFull dst_ dfs(cs.prefetch_ref(i)); } }; + TRY_STATUS_PREFIX(check_no_prunned(*qinfo.proc_info), "invalid proc_info proof: ") + TRY_STATUS_PREFIX(check_no_prunned(*qinfo.ihr_pending), "invalid ihr_pending proof: ") dfs_cs(*qinfo.proc_info); dfs_cs(*qinfo.ihr_pending); @@ -76,6 +78,14 @@ static td::Result process_queue(BlockIdExt block_id, ShardIdFull dst_ while (!queue_merger.is_eof()) { auto kv = queue_merger.extract_cur(); queue_merger.next(); + block::EnqueuedMsgDescr enq; + auto msg = kv->msg; + if (!enq.unpack(msg.write())) { + return td::Status::Error("cannot unpack EnqueuedMsgDescr"); + } + if (limit_reached) { + break; + } ++msg_count; // TODO: Get processed_upto from destination shard (in request?) @@ -105,7 +115,6 @@ static td::Result process_queue(BlockIdExt block_id, ShardIdFull dst_ TRY_STATUS_PREFIX(check_no_prunned(*kv->msg), "invalid message proof: ") if (estimated_proof_size > OutMsgQueueProof::QUEUE_SIZE_THRESHOLD) { limit_reached = true; - break; } } return limit_reached ? msg_count : -1; diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index fcd869b1a..4f8c464c9 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -526,14 +526,6 @@ bool ValidateQuery::extract_collated_data_from(Ref croot, int idx) { top_shard_descr_dict_ = std::make_unique(cs.prefetch_ref(), 96); return true; } - if (block::gen::t_NeighborMsgQueueLimits.has_valid_tag(cs)) { - LOG(DEBUG) << "collated datum # " << idx << " is a NeighborMsgQueueLimits"; - if (!block::gen::t_NeighborMsgQueueLimits.validate_upto(10000, cs)) { - return reject_query("invalid NeighborMsgQueueLimits"); - } - neighbor_msg_queues_limits_ = vm::Dictionary{cs.prefetch_ref(0), 32 + 64}; - return true; - } LOG(WARNING) << "collated datum # " << idx << " has unknown type (magic " << cs.prefetch_ulong(32) << "), ignoring"; return true; } @@ -4115,19 +4107,7 @@ bool ValidateQuery::check_in_queue() { td::BitArray<96> key; key.bits().store_int(descr.workchain(), 32); (key.bits() + 32).store_uint(descr.shard().shard, 64); - auto r = neighbor_msg_queues_limits_.lookup(key); - td::int32 msg_limit = r.is_null() ? -1 : (td::int32)r->prefetch_long(32); - if (msg_limit < -1) { - return reject_query("invalid value in NeighborMsgQueueLimits"); - } - LOG(DEBUG) << "Neighbor " << descr.shard().to_str() << " has msg_limit=" << msg_limit; - neighbor_queues.emplace_back(descr.top_block_id(), descr.outmsg_root, descr.disabled_, msg_limit); - if (msg_limit != -1 && descr.shard().is_masterchain()) { - return reject_query("masterchain out message queue cannot be limited"); - } - if (msg_limit != -1 && shard_intersects(descr.shard(), shard_)) { - return reject_query("prev block out message queue cannot be limited"); - } + neighbor_queues.emplace_back(descr.top_block_id(), descr.outmsg_root, descr.disabled_); } block::OutputQueueMerger nb_out_msgs(shard_, std::move(neighbor_queues)); while (!nb_out_msgs.is_eof()) { diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index cd14504e6..942b32a5f 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -206,7 +206,6 @@ class ValidateQuery : public td::actor::Actor { block::ActionPhaseConfig action_phase_cfg_; td::RefInt256 masterchain_create_fee_, basechain_create_fee_; - vm::Dictionary neighbor_msg_queues_limits_{32 + 64}; std::vector neighbors_; std::map> aux_mc_states_; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index bc6a9812c..5c7bef5b3 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -417,7 +417,7 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo promise = td::PromiseCreator::lambda([=, SelfId = actor_id(this), promise = std::move(promise), timer = td::Timer()](td::Result R) mutable { if (R.is_ok()) { - LOG(WARNING) << "collate query for " << next_block_id.to_str() << ": success, time=" << timer.elapsed() << "s"; + LOG(INFO) << "collate query for " << next_block_id.to_str() << ": success, time=" << timer.elapsed() << "s"; promise.set_result(R.move_as_ok()); return; } From 32b3fe748ab1799843b73595e1f9ebad597934ee Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 21 Jul 2023 19:29:29 +0300 Subject: [PATCH 062/388] Fix validating inbound msg queues --- validator/impl/collator.cpp | 3 +++ validator/impl/validate-query.cpp | 43 +++++++------------------------ validator/impl/validate-query.hpp | 2 +- 3 files changed, 14 insertions(+), 34 deletions(-) diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 52d9c1c40..f11bd1169 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -3805,6 +3805,9 @@ bool Collator::store_master_ref(vm::CellBuilder& cb) { bool Collator::update_processed_upto() { auto ref_mc_seqno = is_masterchain() ? new_block_seqno : prev_mc_block_seqno; update_min_mc_seqno(ref_mc_seqno); + if (in_msg_dict->is_empty()) { + return true; + } if (last_proc_int_msg_.first) { if (!processed_upto_->insert(ref_mc_seqno, last_proc_int_msg_.first, last_proc_int_msg_.second.cbits())) { return fatal_error("cannot update our ProcessedUpto to reflect processed inbound message"); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 4f8c464c9..8d0b51731 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -3927,6 +3927,7 @@ bool ValidateQuery::check_processed_upto() { if (!ok) { return reject_query("new ProcessedInfo is not obtained from old ProcessedInfo by adding at most one new entry"); } + processed_upto_updated_ = upd; if (upd) { if (upd->shard != shard_.shard) { return reject_query("newly-added ProcessedInfo entry refers to shard "s + @@ -4007,37 +4008,8 @@ bool ValidateQuery::check_neighbor_outbound_message(Ref enq_msg, key.to_hex(352) + " of neighbor " + nb.blk_.to_str()); } if (shard_contains(shard_, enq.cur_prefix_)) { - // if this message comes from our own outbound queue, we must have dequeued it - if (out_entry.is_null()) { - return reject_query("our old outbound queue contains EnqueuedMsg with key "s + key.to_hex(352) + - " already processed by this shard, but there is no ext_message_deq OutMsg record for this " - "message in this block"); - } - int tag = block::gen::t_OutMsg.get_tag(*out_entry); - if (tag == block::gen::OutMsg::msg_export_deq_short) { - block::gen::OutMsg::Record_msg_export_deq_short deq; - if (!tlb::csr_unpack(std::move(out_entry), deq)) { - return reject_query( - "cannot unpack msg_export_deq_short OutMsg record for already processed EnqueuedMsg with key "s + - key.to_hex(352) + " of old outbound queue"); - } - if (deq.msg_env_hash != enq.msg_env_->get_hash().bits()) { - return reject_query("unpack ext_message_deq OutMsg record for already processed EnqueuedMsg with key "s + - key.to_hex(352) + " of old outbound queue refers to MsgEnvelope with different hash " + - deq.msg_env_hash.to_hex()); - } - } else { - block::gen::OutMsg::Record_msg_export_deq deq; - if (!tlb::csr_unpack(std::move(out_entry), deq)) { - return reject_query( - "cannot unpack msg_export_deq OutMsg record for already processed EnqueuedMsg with key "s + - key.to_hex(352) + " of old outbound queue"); - } - if (deq.out_msg->get_hash() != enq.msg_env_->get_hash()) { - return reject_query("unpack ext_message_deq OutMsg record for already processed EnqueuedMsg with key "s + - key.to_hex(352) + " of old outbound queue contains a different MsgEnvelope"); - } - } + // this message couldn't come from our own outbound queue because processed messages from our queue don't stay here + return fatal_error("have an already processed EnqueuedMsg from our shard: "s + key.to_hex(352)); } // next check is incorrect after a merge, when ns_.processed_upto has > 1 entries // we effectively comment it out @@ -4110,6 +4082,13 @@ bool ValidateQuery::check_in_queue() { neighbor_queues.emplace_back(descr.top_block_id(), descr.outmsg_root, descr.disabled_); } block::OutputQueueMerger nb_out_msgs(shard_, std::move(neighbor_queues)); + if (in_msg_dict_->is_empty()) { + LOG(DEBUG) << "in_msg_dict is empty, skip checking neighbors' message queues"; + if (processed_upto_updated_) { + return reject_query("processed_upto was updated, but no messages were processed"); + } + return true; + } while (!nb_out_msgs.is_eof()) { auto kv = nb_out_msgs.extract_cur(); CHECK(kv && kv->msg.not_null()); @@ -4130,12 +4109,10 @@ bool ValidateQuery::check_in_queue() { neighbors_.at(kv->source).blk_.to_str()); } if (unprocessed) { - inbound_queues_empty_ = false; return true; } nb_out_msgs.next(); } - inbound_queues_empty_ = true; return true; } diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 942b32a5f..b1d528197 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -210,6 +210,7 @@ class ValidateQuery : public td::actor::Actor { std::map> aux_mc_states_; block::ShardState ps_, ns_; + bool processed_upto_updated_{false}; std::unique_ptr sibling_out_msg_queue_; std::shared_ptr sibling_processed_upto_; @@ -223,7 +224,6 @@ class ValidateQuery : public td::actor::Actor { ton::LogicalTime proc_lt_{0}, claimed_proc_lt_{0}, min_enq_lt_{~0ULL}; ton::Bits256 proc_hash_ = ton::Bits256::zero(), claimed_proc_hash_, min_enq_hash_; - bool inbound_queues_empty_{false}; std::vector> msg_proc_lt_; From f1e62d00757ad585c77b3a2b7d09496a55113dd3 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 24 Jul 2023 15:29:55 +0300 Subject: [PATCH 063/388] Move msg queue limits to config --- crypto/block/block.cpp | 20 ++++++++++++++++---- crypto/block/block.h | 8 ++++++++ crypto/block/block.tlb | 6 +++++- tl/generate/scheme/ton_api.tl | 4 +++- tl/generate/scheme/ton_api.tlo | Bin 90684 -> 90912 bytes validator/full-node-shard.cpp | 17 ++++++++++++----- validator/full-node-shard.h | 3 ++- validator/full-node-shard.hpp | 4 ++-- validator/full-node.cpp | 11 ++++++----- validator/full-node.hpp | 4 ++-- validator/impl/out-msg-queue-proof.cpp | 19 ++++++++++--------- validator/impl/out-msg-queue-proof.hpp | 15 +++++++++++---- validator/impl/shard.hpp | 7 +++++++ validator/interfaces/out-msg-queue-proof.h | 3 ++- validator/interfaces/shard.h | 1 + validator/interfaces/validator-manager.h | 3 ++- validator/manager-disk.hpp | 3 ++- validator/manager-hardfork.hpp | 3 ++- validator/manager.cpp | 20 ++++++++++++-------- validator/manager.hpp | 3 ++- validator/validator.h | 3 ++- 21 files changed, 109 insertions(+), 48 deletions(-) diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp index b9ecfe60c..14f2c18b0 100644 --- a/crypto/block/block.cpp +++ b/crypto/block/block.cpp @@ -655,6 +655,12 @@ bool EnqueuedMsgDescr::check_key(td::ConstBitPtr key) const { hash_ == key + 96; } +bool ImportedMsgQueueLimits::deserialize(vm::CellSlice& cs) { + return cs.fetch_ulong(8) == 0xd3 // imported_msg_queue_limits#d3 + && cs.fetch_uint_to(32, max_bytes) // max_bytes:# + && cs.fetch_uint_to(32, max_msgs); // max_msgs:# +} + bool ParamLimits::deserialize(vm::CellSlice& cs) { return cs.fetch_ulong(8) == 0xc3 // param_limits#c3 && cs.fetch_uint_to(32, limits_[0]) // underload:uint32 @@ -666,10 +672,16 @@ bool ParamLimits::deserialize(vm::CellSlice& cs) { } bool BlockLimits::deserialize(vm::CellSlice& cs) { - return cs.fetch_ulong(8) == 0x5d // block_limits#5d - && bytes.deserialize(cs) // bytes:ParamLimits - && gas.deserialize(cs) // gas:ParamLimits - && lt_delta.deserialize(cs); // lt_delta:ParamLimits + auto tag = cs.fetch_ulong(8); + if (tag != 0x5d && tag != 0x5e) { + return false; + } + // block_limits#5d + // block_limits_v2#5e + return bytes.deserialize(cs) // bytes:ParamLimits + && gas.deserialize(cs) // gas:ParamLimits + && lt_delta.deserialize(cs) // lt_delta:ParamLimits + && (tag == 0x5d || imported_msg_queue.deserialize(cs)); // imported_msg_queue:ImportedMsgQueueLimits } int ParamLimits::classify(td::uint64 value) const { diff --git a/crypto/block/block.h b/crypto/block/block.h index b6b46a3d6..5f3f31630 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -216,6 +216,13 @@ static inline std::ostream& operator<<(std::ostream& os, const MsgProcessedUptoC return proc_coll.print(os); } +struct ImportedMsgQueueLimits { + // Default values + td::uint32 max_bytes = 1 << 18; + td::uint32 max_msgs = 40; + bool deserialize(vm::CellSlice& cs); +}; + struct ParamLimits { enum { limits_cnt = 4 }; enum { cl_underload = 0, cl_normal = 1, cl_soft = 2, cl_medium = 3, cl_hard = 4 }; @@ -247,6 +254,7 @@ struct ParamLimits { struct BlockLimits { ParamLimits bytes, gas, lt_delta; ton::LogicalTime start_lt{0}; + ImportedMsgQueueLimits imported_msg_queue; const vm::CellUsageTree* usage_tree{nullptr}; bool deserialize(vm::CellSlice& cs); int classify_size(td::uint64 size) const; diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 2176c0adb..9e542999a 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -704,9 +704,13 @@ config_gas_prices#_ GasLimitsPrices = ConfigParam 21; param_limits#c3 underload:# soft_limit:# { underload <= soft_limit } hard_limit:# { soft_limit <= hard_limit } = ParamLimits; +imported_msg_queue_limits#d3 max_bytes:# max_msgs:# = ImportedMsgQueueLimits; block_limits#5d bytes:ParamLimits gas:ParamLimits lt_delta:ParamLimits = BlockLimits; - +block_limits_v2#5e bytes:ParamLimits gas:ParamLimits lt_delta:ParamLimits + imported_msg_queue:ImportedMsgQueueLimits + = BlockLimits; + config_mc_block_limits#_ BlockLimits = ConfigParam 22; config_block_limits#_ BlockLimits = ConfigParam 23; diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 6d52b8454..5d636e9c3 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -415,6 +415,7 @@ tonNode.success = tonNode.Success; tonNode.archiveNotFound = tonNode.ArchiveInfo; tonNode.archiveInfo id:long = tonNode.ArchiveInfo; +tonNode.importedMsgQueueLimits max_bytes:int max_msgs:int = ImportedMsgQueueLimits; tonNode.outMsgQueueProof queue_proof:bytes block_state_proof:bytes msg_count:int = tonNode.OutMsgQueueProof; tonNode.outMsgQueueProofEmpty = tonNode.OutMsgQueueProof; @@ -451,7 +452,8 @@ tonNode.downloadBlockProofLinks blocks:(vector tonNode.blockIdExt) = tonNode.Dat tonNode.downloadKeyBlockProofLinks blocks:(vector tonNode.blockIdExt) = tonNode.DataList; tonNode.getArchiveInfo masterchain_seqno:int = tonNode.ArchiveInfo; tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data; -tonNode.getOutMsgQueueProof block_id:tonNode.blockIdExt dst_workchain:int dst_shard:long = tonNode.OutMsgQueueProof; +tonNode.getOutMsgQueueProof block_id:tonNode.blockIdExt dst_workchain:int dst_shard:long + limits:tonNode.importedMsgQueueLimits = tonNode.OutMsgQueueProof; tonNode.getCapabilities = tonNode.Capabilities; tonNode.getCapabilitiesV2 = tonNode.Capabilities; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 1eed452846aa645ab1b76b678dde81080e579487..60e942922e5b98a51f11eb69ef9e47aaf47f22ce 100644 GIT binary patch delta 213 zcmdmUgmu9&R^CUm^{p77fO{hEV`=WkXTOVi<`(1^m87Ql7N-Z6rk1AqWaeg;6i-|) zx$#4dG2?{IxhCJatwAc~O7ipk@>5dvGO_Al0I9ioOvr}`Xod8%`vw2kaEEi#crD4uZ5zj`*dp{Yo@r1{(k?AyH2N delta 83 zcmZ2*jCIcuR^CUm^{p77fNLV}EzI^8mc(S1@m$7G$~0^4`QFlMj;09oxDF#rGn diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index b34432a93..3c7b80638 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -666,6 +666,7 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod td::Promise promise) { BlockIdExt block_id = create_block_id(query.block_id_); ShardIdFull dst_shard(query.dst_workchain_, query.dst_shard_); + block::ImportedMsgQueueLimits limits{(td::uint32)query.limits_->max_bytes_, (td::uint32)query.limits_->max_msgs_}; if (!block_id.is_valid_ext()) { promise.set_error(td::Status::Error("invalid block_id")); return; @@ -674,6 +675,10 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod promise.set_error(td::Status::Error("invalid shard")); return; } + if (limits.max_bytes > (1 << 21)) { + promise.set_error(td::Status::Error("max_bytes is too big")); + return; + } auto P = td::PromiseCreator::lambda( [promise = std::move(promise)](td::Result> R) mutable { if (R.is_error()) { @@ -684,7 +689,7 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod }); VLOG(FULL_NODE_DEBUG) << "Got query getOutMsgQueueProof " << block_id.to_str() << " " << dst_shard.to_str() << " from " << src; - td::actor::create_actor("buildqueueproof", block_id, dst_shard, validator_manager_, + td::actor::create_actor("buildqueueproof", block_id, dst_shard, limits, validator_manager_, std::move(P)) .release(); } @@ -924,7 +929,8 @@ void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, std::stri .release(); } -void FullNodeShardImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, +void FullNodeShardImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, td::Promise> promise) { // TODO: maybe more complex download (like other requests here) auto &b = choose_neighbour(true); @@ -944,13 +950,14 @@ void FullNodeShardImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardI promise.set_error(td::Status::Error("node doesn't have this block")); }, [&](ton_api::tonNode_outMsgQueueProof &x) { - promise.set_result(OutMsgQueueProof::fetch(block_id, dst_shard, x)); + promise.set_result(OutMsgQueueProof::fetch(block_id, dst_shard, limits, x)); })); }); td::BufferSlice query = create_serialize_tl_object( - create_tl_block_id(block_id), dst_shard.workchain, dst_shard.shard); + create_tl_block_id(block_id), dst_shard.workchain, dst_shard.shard, + create_tl_object(limits.max_bytes, limits.max_msgs)); td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, b.adnl_id, adnl_id_, overlay_id_, - "get_msg_queue", std::move(P), timeout, std::move(query), 1 << 20, rldp_); + "get_msg_queue", std::move(P), timeout, std::move(query), 1 << 22, rldp_); } void FullNodeShardImpl::set_handle(BlockHandle handle, td::Promise promise) { diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 59d00e264..fca17bf4b 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -70,7 +70,8 @@ class FullNodeShard : public td::actor::Actor { td::Promise> promise) = 0; virtual void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) = 0; - virtual void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, + virtual void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, td::Promise> promise) = 0; virtual void set_handle(BlockHandle handle, td::Promise promise) = 0; diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index c0c7dd4bb..b2be91081 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -186,8 +186,8 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise> promise) override; void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) override; - void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, - td::Promise> promise) override; + void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, + td::Timestamp timeout, td::Promise> promise) override; void set_handle(BlockHandle handle, td::Promise promise) override; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 3b275ea99..eebc49f8b 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -378,7 +378,8 @@ void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, std::string tm std::move(promise)); } -void FullNodeImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, +void FullNodeImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, td::Promise> promise) { auto shard = get_shard(block_id.shard_full()); if (shard.empty()) { @@ -386,7 +387,7 @@ void FullNodeImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); return; } - td::actor::send_closure(shard, &FullNodeShard::download_out_msg_queue_proof, block_id, dst_shard, timeout, + td::actor::send_closure(shard, &FullNodeShard::download_out_msg_queue_proof, block_id, dst_shard, limits, timeout, std::move(promise)); } @@ -587,9 +588,9 @@ void FullNodeImpl::start_up() { td::actor::send_closure(id_, &FullNodeImpl::download_archive, masterchain_seqno, std::move(tmp_dir), timeout, std::move(promise)); } - void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, - td::Promise> promise) override { - td::actor::send_closure(id_, &FullNodeImpl::download_out_msg_queue_proof, block_id, dst_shard, timeout, + void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, + td::Timestamp timeout, td::Promise> promise) override { + td::actor::send_closure(id_, &FullNodeImpl::download_out_msg_queue_proof, block_id, dst_shard, limits, timeout, std::move(promise)); } diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 01bd19e4c..99ea49cc3 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -74,8 +74,8 @@ class FullNodeImpl : public FullNode { void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise); void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise); - void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, - td::Promise> promise); + void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, + td::Timestamp timeout, td::Promise> promise); void got_key_block_proof(td::Ref proof); void got_zero_block_state(td::Ref state); diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 59aa8aed6..cc8c7e179 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -47,6 +47,7 @@ static td::Status check_no_prunned(const vm::CellSlice& cs) { } static td::Result process_queue(BlockIdExt block_id, ShardIdFull dst_shard, + block::ImportedMsgQueueLimits limits, const block::gen::OutMsgQueueInfo::Record& qinfo) { td::uint64 estimated_proof_size = 0; @@ -113,17 +114,16 @@ static td::Result process_queue(BlockIdExt block_id, ShardIdFull dst_ dfs_cs(*kv->msg); TRY_STATUS_PREFIX(check_no_prunned(*kv->msg), "invalid message proof: ") - if (estimated_proof_size > OutMsgQueueProof::QUEUE_SIZE_THRESHOLD) { + if (estimated_proof_size >= limits.max_bytes || msg_count >= (long long)limits.max_msgs) { limit_reached = true; } } return limit_reached ? msg_count : -1; } -td::Result> OutMsgQueueProof::build(BlockIdExt block_id, - ShardIdFull dst_shard, - Ref state_root, - Ref block_root) { +td::Result> OutMsgQueueProof::build( + BlockIdExt block_id, ShardIdFull dst_shard, + block::ImportedMsgQueueLimits limits, Ref state_root, Ref block_root) { if (!dst_shard.is_valid_ext()) { return td::Status::Error("invalid shard"); } @@ -135,7 +135,7 @@ td::Result> OutMsgQueueProof::b if (!tlb::unpack_cell(outq_descr->root_cell(), qinfo)) { return td::Status::Error("invalid message queue"); } - TRY_RESULT(cnt, process_queue(block_id, dst_shard, qinfo)); + TRY_RESULT(cnt, process_queue(block_id, dst_shard, limits, qinfo)); TRY_RESULT(queue_proof, mpb.extract_proof_boc()); td::BufferSlice block_state_proof; @@ -148,6 +148,7 @@ td::Result> OutMsgQueueProof::b } td::Result> OutMsgQueueProof::fetch(BlockIdExt block_id, ShardIdFull dst_shard, + block::ImportedMsgQueueLimits limits, const ton_api::tonNode_outMsgQueueProof& f) { try { Ref block_state_proof; @@ -181,7 +182,7 @@ td::Result> OutMsgQueueProof::fetch(BlockIdExt block_i } TRY_STATUS_PREFIX(check_no_prunned(qinfo.proc_info->prefetch_ref(0)), "invalid proc_info: ") TRY_STATUS_PREFIX(check_no_prunned(qinfo.ihr_pending->prefetch_ref(0)), "invalid ihr_pending: ") - TRY_RESULT(cnt, process_queue(block_id, dst_shard, qinfo)); + TRY_RESULT(cnt, process_queue(block_id, dst_shard, limits, qinfo)); if (cnt != f.msg_count_) { return td::Status::Error(PSTRING() << "invalid msg_count: expected=" << f.msg_count_ << ", found=" << cnt); } @@ -295,7 +296,7 @@ void WaitOutMsgQueueProof::run_net() { }); td::actor::send_closure(manager_, &ValidatorManager::send_get_out_msg_queue_proof_request, block_id_, dst_shard_, - priority_, std::move(P)); + limits_, priority_, std::move(P)); } void BuildOutMsgQueueProof::abort_query(td::Status reason) { @@ -349,7 +350,7 @@ void BuildOutMsgQueueProof::got_block_root(Ref root) { } void BuildOutMsgQueueProof::build_proof() { - auto result = OutMsgQueueProof::build(block_id_, dst_shard_, std::move(state_root_), std::move(block_root_)); + auto result = OutMsgQueueProof::build(block_id_, dst_shard_, limits_, std::move(state_root_), std::move(block_root_)); if (result.is_error()) { LOG(ERROR) << "Failed to build msg queue proof: " << result.error(); } diff --git a/validator/impl/out-msg-queue-proof.hpp b/validator/impl/out-msg-queue-proof.hpp index e94f805f4..9c5acfef9 100644 --- a/validator/impl/out-msg-queue-proof.hpp +++ b/validator/impl/out-msg-queue-proof.hpp @@ -31,11 +31,12 @@ class ValidatorManagerInterface; class WaitOutMsgQueueProof : public td::actor::Actor { public: - WaitOutMsgQueueProof(BlockIdExt block_id, ShardIdFull dst_shard, bool local, td::uint32 priority, - td::actor::ActorId manager, td::Timestamp timeout, + WaitOutMsgQueueProof(BlockIdExt block_id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, bool local, + td::uint32 priority, td::actor::ActorId manager, td::Timestamp timeout, td::Promise> promise) : block_id_(std::move(block_id)) , dst_shard_(dst_shard) + , limits_(limits) , local_(local) , priority_(priority) , manager_(manager) @@ -65,6 +66,7 @@ class WaitOutMsgQueueProof : public td::actor::Actor { private: BlockIdExt block_id_; ShardIdFull dst_shard_; + block::ImportedMsgQueueLimits limits_; bool local_; td::uint32 priority_; @@ -78,10 +80,14 @@ class WaitOutMsgQueueProof : public td::actor::Actor { class BuildOutMsgQueueProof : public td::actor::Actor { public: - BuildOutMsgQueueProof(BlockIdExt block_id, ShardIdFull dst_shard, + BuildOutMsgQueueProof(BlockIdExt block_id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, td::actor::ActorId manager, td::Promise> promise) - : block_id_(std::move(block_id)), dst_shard_(dst_shard), manager_(manager), promise_(std::move(promise)) { + : block_id_(std::move(block_id)) + , dst_shard_(dst_shard) + , limits_(limits) + , manager_(manager) + , promise_(std::move(promise)) { } void abort_query(td::Status reason); @@ -93,6 +99,7 @@ class BuildOutMsgQueueProof : public td::actor::Actor { private: BlockIdExt block_id_; ShardIdFull dst_shard_; + block::ImportedMsgQueueLimits limits_; td::actor::ActorId manager_; td::Promise> promise_; diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index 83aa1f9a4..a59f26d18 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -130,6 +130,13 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { auto R = config_->get_size_limits_config(); return R.is_error() ? block::SizeLimitsConfig::ExtMsgLimits() : R.ok_ref().ext_msg_limits; } + block::ImportedMsgQueueLimits get_imported_msg_queue_limits(bool is_masterchain) const override { + auto R = config_->get_block_limits(is_masterchain); + if (R.is_ok() && R.ok()) { + return R.ok()->imported_msg_queue; + } + return {}; + } BlockIdExt last_key_block_id() const override; BlockIdExt next_key_block_id(BlockSeqno seqno) const override; BlockIdExt prev_key_block_id(BlockSeqno seqno) const override; diff --git a/validator/interfaces/out-msg-queue-proof.h b/validator/interfaces/out-msg-queue-proof.h index 5d7924913..524168d0d 100644 --- a/validator/interfaces/out-msg-queue-proof.h +++ b/validator/interfaces/out-msg-queue-proof.h @@ -35,12 +35,13 @@ struct OutMsgQueueProof : public td::CntObject { td::int32 msg_count_; // -1 - up to end of queue static td::Result> fetch(BlockIdExt block_id, ShardIdFull dst_shard, + block::ImportedMsgQueueLimits limits, const ton_api::tonNode_outMsgQueueProof &f); static td::Result> build(BlockIdExt block_id, ShardIdFull dst_shard, + block::ImportedMsgQueueLimits limits, Ref state_root, Ref block_root); - static const td::uint64 QUEUE_SIZE_THRESHOLD = 128 * 1024; }; } // namespace validator diff --git a/validator/interfaces/shard.h b/validator/interfaces/shard.h index 9ccfa2fb4..2e3437470 100644 --- a/validator/interfaces/shard.h +++ b/validator/interfaces/shard.h @@ -88,6 +88,7 @@ class MasterchainState : virtual public ShardState { return td::Status::OK(); } virtual block::SizeLimitsConfig::ExtMsgLimits get_ext_msg_limits() const = 0; + virtual block::ImportedMsgQueueLimits get_imported_msg_queue_limits(bool is_masterchain) const = 0; }; } // namespace validator diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index bcc5fd53e..a4a1dd167 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -129,7 +129,8 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void send_ihr_message(td::Ref message) = 0; virtual void send_top_shard_block_description(td::Ref desc) = 0; virtual void send_block_broadcast(BlockBroadcast broadcast) = 0; - virtual void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, td::uint32 priority, + virtual void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, + block::ImportedMsgQueueLimits limits, td::uint32 priority, td::Promise> promise) = 0; virtual void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index d41a6e342..17a1bffae 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -255,7 +255,8 @@ class ValidatorManagerImpl : public ValidatorManager { void send_top_shard_block_description(td::Ref desc) override; void send_block_broadcast(BlockBroadcast broadcast) override { } - void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, td::uint32 priority, + void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, + td::uint32 priority, td::Promise> promise) override { UNREACHABLE(); } diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 0a9e8a2b7..65d2b3c0d 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -321,7 +321,8 @@ class ValidatorManagerImpl : public ValidatorManager { } void send_block_broadcast(BlockBroadcast broadcast) override { } - void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, td::uint32 priority, + void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, + td::uint32 priority, td::Promise> promise) override { UNREACHABLE(); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 5c3691bd6..638b741f9 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -630,16 +630,18 @@ void ValidatorManagerImpl::wait_out_msg_queue_proof(BlockIdExt block_id, ShardId std::move(R)); }); auto id = td::actor::create_actor( - "waitmsgqueue", block_id, dst_shard, need_monitor(block_id.shard_full()), priority, actor_id(this), - td::Timestamp::at(timeout.at() + 10.0), std::move(P)) + "waitmsgqueue", block_id, dst_shard, + last_masterchain_state_->get_imported_msg_queue_limits(block_id.is_masterchain()), + need_monitor(block_id.shard_full()), priority, actor_id(this), td::Timestamp::at(timeout.at() + 10.0), + std::move(P)) .release(); wait_out_msg_queue_proof_[key].actor_ = id; it = wait_out_msg_queue_proof_.find(key); } else if (it->second.done_) { promise.set_result(it->second.result_); it->second.remove_at_ = td::Timestamp::in(30.0); + return; } - it->second.waiting_.emplace_back(timeout, priority, std::move(promise)); auto X = it->second.get_timeout(); td::actor::send_closure(it->second.actor_, &WaitOutMsgQueueProof::update_timeout, X.first, X.second); @@ -1097,9 +1099,10 @@ void ValidatorManagerImpl::finished_wait_msg_queue(BlockIdExt block_id, ShardIdF td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_msg_queue, block_id, dst_shard, std::move(R)); }); - auto id = td::actor::create_actor("waitmsgqueue", block_id, dst_shard, - need_monitor(block_id.shard_full()), X.second, - actor_id(this), X.first, std::move(P)) + auto id = td::actor::create_actor( + "waitmsgqueue", block_id, dst_shard, + last_masterchain_state_->get_imported_msg_queue_limits(block_id.is_masterchain()), + need_monitor(block_id.shard_full()), X.second, actor_id(this), X.first, std::move(P)) .release(); it->second.actor_ = id; return; @@ -1511,9 +1514,10 @@ void ValidatorManagerImpl::send_block_broadcast(BlockBroadcast broadcast) { } void ValidatorManagerImpl::send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, + block::ImportedMsgQueueLimits limits, td::uint32 priority, td::Promise> promise) { - callback_->download_out_msg_queue_proof(id, dst_shard, td::Timestamp::in(10.0), std::move(promise)); + callback_->download_out_msg_queue_proof(id, dst_shard, limits, td::Timestamp::in(10.0), std::move(promise)); } void ValidatorManagerImpl::start_up() { @@ -1839,7 +1843,7 @@ void ValidatorManagerImpl::new_masterchain_block() { // Prepare neighboours' queues for collating masterchain for (const auto &desc : last_masterchain_state_->get_shards()) { wait_out_msg_queue_proof(desc->top_block_id(), ShardIdFull(masterchainId), 0, td::Timestamp::in(10.0), - [](td::Ref) {}); + [](td::Result>) {}); } } diff --git a/validator/manager.hpp b/validator/manager.hpp index 3debccb39..a37fcb4e0 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -462,7 +462,8 @@ class ValidatorManagerImpl : public ValidatorManager { void send_ihr_message(td::Ref message) override; void send_top_shard_block_description(td::Ref desc) override; void send_block_broadcast(BlockBroadcast broadcast) override; - void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, td::uint32 priority, + void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, + td::uint32 priority, td::Promise> promise) override; void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; diff --git a/validator/validator.h b/validator/validator.h index 4ce3f4d02..5fa23467f 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -140,7 +140,8 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Promise> promise) = 0; virtual void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) = 0; - virtual void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::Timestamp timeout, + virtual void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, td::Promise> promise) = 0; virtual void new_key_block(BlockHandle handle) = 0; From 8c4bc5b3f15ce0d137a8b4e0298f9d1304291d96 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 26 Jul 2023 12:21:19 +0300 Subject: [PATCH 064/388] Fix sending msg queue queries --- validator/collator-node.cpp | 52 +++++++++++++------------- validator/impl/out-msg-queue-proof.cpp | 28 +++++++------- validator/manager.cpp | 3 +- 3 files changed, 43 insertions(+), 40 deletions(-) diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 2cc06b340..f5900e0a9 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -74,33 +74,35 @@ void CollatorNode::new_masterchain_block_notification(td::Ref last_masterchain_block_ = state->get_block_id(); last_top_blocks_.clear(); last_top_blocks_[ShardIdFull{masterchainId, shardIdAll}] = last_masterchain_block_; - std::vector next_shards; - if (can_collate_shard(ShardIdFull(masterchainId))) { - next_shards.push_back(ShardIdFull(masterchainId)); - } - for (const auto& desc : state->get_shards()) { - last_top_blocks_[desc->shard()] = desc->top_block_id(); - ShardIdFull shard = desc->shard(); - if (desc->before_split()) { - if (can_collate_shard(shard_child(shard, true))) { - next_shards.push_back(shard_child(shard, true)); - } - if (can_collate_shard(shard_child(shard, false))) { - next_shards.push_back(shard_child(shard, false)); - } - } else if (desc->before_merge()) { - if (is_left_child(shard) && can_collate_shard(shard_parent(shard))) { - next_shards.push_back(shard_parent(shard)); + if (state->get_unix_time() > (td::uint32)td::Clocks::system() - 20) { + std::vector next_shards; + if (can_collate_shard(ShardIdFull(masterchainId))) { + next_shards.push_back(ShardIdFull(masterchainId)); + } + for (const auto& desc : state->get_shards()) { + last_top_blocks_[desc->shard()] = desc->top_block_id(); + ShardIdFull shard = desc->shard(); + if (desc->before_split()) { + if (can_collate_shard(shard_child(shard, true))) { + next_shards.push_back(shard_child(shard, true)); + } + if (can_collate_shard(shard_child(shard, false))) { + next_shards.push_back(shard_child(shard, false)); + } + } else if (desc->before_merge()) { + if (is_left_child(shard) && can_collate_shard(shard_parent(shard))) { + next_shards.push_back(shard_parent(shard)); + } + } else if (can_collate_shard(shard)) { + next_shards.push_back(shard); } - } else if (can_collate_shard(shard)) { - next_shards.push_back(shard); } - } - for (const ShardIdFull& shard : next_shards) { - for (const auto& neighbor : last_top_blocks_) { - if (neighbor.first != shard && block::ShardConfig::is_neighbor(shard, neighbor.first)) { - td::actor::send_closure(manager_, &ValidatorManager::wait_out_msg_queue_proof, neighbor.second, shard, 0, - td::Timestamp::in(10.0), [](td::Ref) {}); + for (const ShardIdFull& shard : next_shards) { + for (const auto& neighbor : last_top_blocks_) { + if (neighbor.first != shard && block::ShardConfig::is_neighbor(shard, neighbor.first)) { + td::actor::send_closure(manager_, &ValidatorManager::wait_out_msg_queue_proof, neighbor.second, shard, 0, + td::Timestamp::in(10.0), [](td::Ref) {}); + } } } } diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index cc8c7e179..52c59afa0 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -280,20 +280,20 @@ void WaitOutMsgQueueProof::run_local_cont() { } void WaitOutMsgQueueProof::run_net() { - auto P = - td::PromiseCreator::lambda([SelfId = actor_id(this), block_id = block_id_](td::Result> R) { - if (R.is_error()) { - if (R.error().code() == ErrorCode::notready) { - LOG(DEBUG) << "failed to get msg queue for " << block_id.to_str() << " from net: " << R.move_as_error(); - } else { - LOG(WARNING) << "failed to get msg queue for " << block_id.to_str() << " from net: " << R.move_as_error(); - } - delay_action([SelfId]() mutable { td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::run_net); }, - td::Timestamp::in(0.1)); - } else { - td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::finish_query, R.move_as_ok()); - } - }); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id = block_id_, + retry_after = td::Timestamp::in(0.5)](td::Result> R) { + if (R.is_error()) { + if (R.error().code() == ErrorCode::notready) { + LOG(DEBUG) << "failed to get msg queue for " << block_id.to_str() << " from net: " << R.move_as_error(); + } else { + LOG(WARNING) << "failed to get msg queue for " << block_id.to_str() << " from net: " << R.move_as_error(); + } + delay_action([SelfId]() mutable { td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::run_net); }, + retry_after); + } else { + td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::finish_query, R.move_as_ok()); + } + }); td::actor::send_closure(manager_, &ValidatorManager::send_get_out_msg_queue_proof_request, block_id_, dst_shard_, limits_, priority_, std::move(P)); diff --git a/validator/manager.cpp b/validator/manager.cpp index 638b741f9..d419f80ee 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1839,7 +1839,8 @@ void ValidatorManagerImpl::new_masterchain_block() { for (auto &c : collator_nodes_) { td::actor::send_closure(c.second.actor, &CollatorNode::new_masterchain_block_notification, last_masterchain_state_); } - if (opts_->validator_mode() == ValidatorManagerOptions::validator_lite_shards && validating_masterchain()) { + if (opts_->validator_mode() == ValidatorManagerOptions::validator_lite_shards && validating_masterchain() && + last_masterchain_state_->get_unix_time() > (td::uint32)td::Clocks::system() - 20) { // Prepare neighboours' queues for collating masterchain for (const auto &desc : last_masterchain_state_->get_shards()) { wait_out_msg_queue_proof(desc->top_block_id(), ShardIdFull(masterchainId), 0, td::Timestamp::in(10.0), From da137fecf55b670d79df25390096396264da380c Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 26 Jul 2023 13:05:16 +0300 Subject: [PATCH 065/388] Extra shard overlay stats --- overlay/overlay.cpp | 7 +++- overlay/overlays.h | 3 ++ tl/generate/scheme/ton_api.tl | 7 +++- tl/generate/scheme/ton_api.tlo | Bin 90912 -> 91588 bytes .../validator-engine-console-query.cpp | 14 ++++++-- validator/full-node-shard.cpp | 34 ++++++++++++++++++ validator/full-node-shard.hpp | 1 + 7 files changed, 62 insertions(+), 4 deletions(-) diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index c4cf8428b..88518eb7e 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -661,7 +661,12 @@ void OverlayImpl::get_stats(td::Promisestats_.push_back( create_tl_object("neighbours_cnt", PSTRING() << neighbours_.size())); - promise.set_value(std::move(res)); + callback_->get_stats_extra([promise = std::move(promise), res = std::move(res)](td::Result R) mutable { + if (R.is_ok()) { + res->extra_ = R.move_as_ok(); + } + promise.set_value(std::move(res)); + }); } } // namespace overlay diff --git a/overlay/overlays.h b/overlay/overlays.h index cf153c3a5..ee225c3b8 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -170,6 +170,9 @@ class Overlays : public td::actor::Actor { td::Promise promise) { promise.set_value(td::Unit()); } + virtual void get_stats_extra(td::Promise promise) { + promise.set_result(""); + } virtual ~Callback() = default; }; diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 5d636e9c3..0a4134e5f 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -656,9 +656,14 @@ engine.validator.dhtServersStatus servers:(vector engine.validator.dhtServerStat engine.validator.overlayStatsNode adnl_id:int256 ip_addr:string bdcst_errors:int fec_bdcst_errors:int last_in_query:int last_out_query:int t_out_bytes:int t_in_bytes:int t_out_pckts:int t_in_pckts:int = engine.validator.OverlayStatsNode; -engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 scope:string nodes:(vector engine.validator.overlayStatsNode) stats:(vector engine.validator.oneStat) = engine.validator.OverlayStats; +engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 scope:string nodes:(vector engine.validator.overlayStatsNode) stats:(vector engine.validator.oneStat) extra:string = engine.validator.OverlayStats; engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.OverlaysStats; +engine.validator.shardOverlayStats.neighbour id:string proto_verison:int capabilities:long + roundtrip:double unreliability:double has_state:string = engine.validator.shardOverlayStats.Neighbour; +engine.validator.shardOverlayStats shard:string mode:string + neighbours:(vector engine.validator.shardOverlayStats.neighbour) = engine.validator.ShardOverlayStats; + engine.validator.onePerfTimerStat time:int min:double avg:double max:double = engine.validator.OnePerfTimerStat; engine.validator.perfTimerStatsByName name:string stats:(vector engine.validator.OnePerfTimerStat) = engine.validator.PerfTimerStatsByName; engine.validator.perfTimerStats stats:(vector engine.validator.PerfTimerStatsByName) = engine.validator.PerfTimerStats; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 60e942922e5b98a51f11eb69ef9e47aaf47f22ce..974e5ad6d42377302c711e0636525175d496910f 100644 GIT binary patch delta 407 zcmZ2*jP=NAR^CUm^{p77fOjMBWpP$M59vn zu`;+Mv80%Raq>YX@y&N66Bt?7T0Zg9nY?$IxG0hWJ-^h<^o*qZ(xS;1`K348NT*0L zPTIWCBsqr#Wc*}~g$kS?#?50wKJ1eX7k=lFes*7zHMOFoD3O6-@`PD3V0D{K7HKd+ zSP4sxFhSh1XPE@DRSX~pg3Mu_Tv#GLdBq|D7M9%nl*t#)%W!k$!JJV%Io@AZ0_0Xk zsAr3u*>pH;+YW-XZM&T46;w)MgaAJ^iTRM0kVXbx1cD$BtIUgG_yE=vSFF@ zNLXKqhaNS;evw0NxLf5&!@I delta 124 zcmX?dnsvc3R^CUm^{p77fO{kFWpUP@zZTq=ovbJ!yO~Mi7UO0asRYK&C!|v(87FMk zHBHWe%70uW%gw+5QhW25kPq8r#{JrpUo1SsQ{>F1!(rQY5G1qt!6FSN7BGuv=@F*Q OQ&zArZ$7i;#|{9nj5Zqp diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index d0f6ab290..98a4d324c 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -934,8 +934,18 @@ td::Status GetOverlaysStatsJsonQuery::receive(td::BufferSlice data) { sb << " \"" << t->key_ << "\": \"" << t->value_ << "\""; } - sb << "\n }\n"; - sb << "}\n"; + sb << "\n }"; + if (!s->extra_.empty()) { + sb << ",\n \"extra\": "; + for (char c : s->extra_) { + if (c == '\n') { + sb << "\n "; + } else { + sb << c; + } + } + } + sb << "\n}\n"; } sb << "]\n"; sb << std::flush; diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 3c7b80638..a99197546 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -39,6 +39,9 @@ #include "td/utils/Random.h" #include "common/delay.h" +#include "td/utils/JsonBuilder.h" +#include "tl/tl_json.h" +#include "auto/tl/ton_api_json.h" namespace ton { @@ -99,6 +102,9 @@ void FullNodeShardImpl::create_overlay() { td::Promise promise) override { td::actor::send_closure(node_, &FullNodeShardImpl::check_broadcast, src, std::move(data), std::move(promise)); } + void get_stats_extra(td::Promise promise) override { + td::actor::send_closure(node_, &FullNodeShardImpl::get_stats_extra, std::move(promise)); + } Callback(td::actor::ActorId node) : node_(node) { } @@ -1290,6 +1296,34 @@ void FullNodeShardImpl::ping_neighbours() { } } +void FullNodeShardImpl::get_stats_extra(td::Promise promise) { + auto res = create_tl_object(); + res->shard_ = shard_.to_str(); + switch (mode_) { + case active: + res->mode_ = "active"; + break; + case active_temp: + res->mode_ = "active_temp"; + break; + case inactive: + res->mode_ = "inactive"; + break; + } + for (const auto &p : neighbours_) { + const auto &n = p.second; + auto f = create_tl_object(); + f->id_ = n.adnl_id.bits256_value().to_hex(); + f->proto_verison_ = n.proto_version; + f->capabilities_ = n.capabilities; + f->roundtrip_ = n.roundtrip; + f->unreliability_ = n.unreliability; + f->has_state_ = (n.has_state_known ? (n.has_state ? "true" : "false") : "undefined"); + res->neighbours_.push_back(std::move(f)); + } + promise.set_result(td::json_encode(td::ToJson(*res), true)); +} + FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index b2be91081..af16ccc91 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -164,6 +164,7 @@ class FullNodeShardImpl : public FullNodeShard { void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); void receive_broadcast(PublicKeyHash src, td::BufferSlice query); void check_broadcast(PublicKeyHash src, td::BufferSlice query, td::Promise promise); + void get_stats_extra(td::Promise promise); void remove_neighbour(adnl::AdnlNodeIdShort id); void send_ihr_message(td::BufferSlice data) override; From 783c75fc8569e567c7028c5c929be62f2c7684d6 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 26 Jul 2023 16:29:37 +0300 Subject: [PATCH 066/388] Decrease verbosity in out-msg-queue-proof.cpp --- validator/impl/out-msg-queue-proof.cpp | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 52c59afa0..365913760 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -198,13 +198,8 @@ void WaitOutMsgQueueProof::alarm() { void WaitOutMsgQueueProof::abort_query(td::Status reason) { if (promise_) { - if (priority_ > 0 || (reason.code() != ErrorCode::timeout && reason.code() != ErrorCode::notready)) { - LOG(WARNING) << "aborting wait msg queue query for " << block_id_.to_str() << " priority=" << priority_ << ": " - << reason; - } else { - LOG(DEBUG) << "aborting wait msg queue query for " << block_id_.to_str() << " priority=" << priority_ << ": " - << reason; - } + LOG(DEBUG) << "aborting wait msg queue query for " << block_id_.to_str() << " priority=" << priority_ << ": " + << reason; promise_.set_error( reason.move_as_error_prefix(PSTRING() << "failed to get msg queue for " << block_id_.to_str() << ": ")); } @@ -283,11 +278,7 @@ void WaitOutMsgQueueProof::run_net() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id = block_id_, retry_after = td::Timestamp::in(0.5)](td::Result> R) { if (R.is_error()) { - if (R.error().code() == ErrorCode::notready) { - LOG(DEBUG) << "failed to get msg queue for " << block_id.to_str() << " from net: " << R.move_as_error(); - } else { - LOG(WARNING) << "failed to get msg queue for " << block_id.to_str() << " from net: " << R.move_as_error(); - } + LOG(DEBUG) << "failed to get msg queue for " << block_id.to_str() << " from net: " << R.move_as_error(); delay_action([SelfId]() mutable { td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::run_net); }, retry_after); } else { From 503e10c05acd4c1d346f42d5ad4fd324cdf5b1e6 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 26 Jul 2023 17:28:14 +0300 Subject: [PATCH 067/388] Fix choose_neighbour in full-node-shard --- validator/full-node-shard.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index a99197546..7938111a3 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -1194,6 +1194,10 @@ const Neighbour &FullNodeShardImpl::choose_neighbour(bool require_state) const { return Neighbour::zero; } + double min_unreliability = 1e9; + for (auto &x : neighbours_) { + min_unreliability = std::min(min_unreliability, x.second.unreliability); + } for (int attempt = 0; attempt < (require_state ? 2 : 1); ++attempt) { const Neighbour *best = nullptr; td::uint32 sum = 0; @@ -1207,7 +1211,7 @@ const Neighbour &FullNodeShardImpl::choose_neighbour(bool require_state) const { continue; } } - auto unr = static_cast(x.second.unreliability); + auto unr = static_cast(x.second.unreliability - min_unreliability); if (x.second.proto_version < proto_version()) { unr += 4; From 44ba04093475d6514903854751811a68acdaddd2 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 27 Jul 2023 17:15:11 +0300 Subject: [PATCH 068/388] Don't store candidates on collator nodes --- create-hardfork/create-hardfork.cpp | 3 ++- test/test-ton-collator.cpp | 3 ++- validator/collator-node.cpp | 3 ++- validator/fabric.h | 5 +++-- validator/impl/collator-impl.h | 4 +++- validator/impl/collator.cpp | 21 +++++++++++++-------- validator/impl/fabric.cpp | 11 ++++++----- 7 files changed, 31 insertions(+), 19 deletions(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index 3fb57f41b..b5c151a75 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -271,7 +271,8 @@ class HardforkCreator : public td::actor::Actor { td::Promise promise) override { } - void download_out_msg_queue_proof(ton::BlockIdExt block_id, ton::ShardIdFull dst_shard, td::Timestamp timeout, + void download_out_msg_queue_proof(ton::BlockIdExt block_id, ton::ShardIdFull dst_shard, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, td::Promise> promise) override { } diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index 8ef469b5f..1bd1cadae 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -372,7 +372,8 @@ class TestNode : public td::actor::Actor { td::Promise promise) override { } - void download_out_msg_queue_proof(ton::BlockIdExt block_id, ton::ShardIdFull dst_shard, td::Timestamp timeout, + void download_out_msg_queue_proof(ton::BlockIdExt block_id, ton::ShardIdFull dst_shard, + block::ImportedMsgQueueLimits, td::Timestamp timeout, td::Promise> promise) override { } diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index f5900e0a9..b9992d400 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -272,7 +272,8 @@ void CollatorNode::receive_query_cont(ShardIdFull shard, td::Ref prev_blocks, Ed25519_PublicKey creator, td::Promise promise) { run_collate_query(shard, min_mc_state->get_block_id(), std::move(prev_blocks), creator, - min_mc_state->get_validator_set(shard), manager_, td::Timestamp::in(10.0), std::move(promise)); + min_mc_state->get_validator_set(shard), manager_, td::Timestamp::in(10.0), std::move(promise), + CollateMode::skip_store_candidate); } void CollatorNode::process_result(std::shared_ptr cache_entry, td::Result R) { diff --git a/validator/fabric.h b/validator/fabric.h index d1611bea3..bd26b0e39 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -26,6 +26,7 @@ namespace ton { namespace validator { enum ValidateMode { fake = 1, full_collated_data = 2 }; +enum CollateMode { skip_store_candidate = 1 }; td::actor::ActorOwn create_db_actor(td::actor::ActorId manager, std::string db_root_); td::actor::ActorOwn create_liteserver_cache_actor(td::actor::ActorId manager, @@ -83,9 +84,9 @@ void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, unsigned mode = 0); void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, - Ed25519_PublicKey local_id, td::Ref validator_set, + Ed25519_PublicKey creator, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise); + td::Promise promise, unsigned mode = 0); void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 2839a0e5b..0a0d59ca6 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -75,6 +75,7 @@ class Collator final : public td::actor::Actor { td::Timestamp timeout; td::Timestamp queue_cleanup_timeout_, soft_timeout_, medium_timeout_; td::Promise main_promise; + unsigned mode_ = 0; ton::BlockSeqno last_block_seqno{0}; ton::BlockSeqno prev_mc_block_seqno{0}; ton::BlockSeqno new_block_seqno{0}; @@ -89,7 +90,8 @@ class Collator final : public td::actor::Actor { public: Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector prev, Ref validator_set, Ed25519_PublicKey collator_id, - td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); + td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, + unsigned mode); ~Collator() override = default; bool is_busy() const { return busy_; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index f11bd1169..b254b70c9 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -57,7 +57,7 @@ static inline bool dbg(int c) { Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector prev, td::Ref validator_set, Ed25519_PublicKey collator_id, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise) + td::Promise promise, unsigned mode) : shard_(shard) , is_hardfork_(is_hardfork) , min_mc_block_id{min_masterchain_block_id} @@ -71,6 +71,7 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_mastercha , soft_timeout_(td::Timestamp::at(timeout.at() - 3.0)) , medium_timeout_(td::Timestamp::at(timeout.at() - 1.5)) , main_promise(std::move(promise)) + , mode_(mode) , perf_timer_("collate", 0.1, [manager](double duration) { send_closure(manager, &ValidatorManager::add_perf_timer_stat, "collate", duration); }) { @@ -4200,13 +4201,17 @@ bool Collator::create_block_candidate() { << consensus_config.max_collated_data_size << ")"); } // 4. save block candidate - LOG(INFO) << "saving new BlockCandidate"; - td::actor::send_closure_later(manager, &ValidatorManager::set_block_candidate, block_candidate->id, - block_candidate->clone(), [self = get_self()](td::Result saved) -> void { - LOG(DEBUG) << "got answer to set_block_candidate"; - td::actor::send_closure_later(std::move(self), &Collator::return_block_candidate, - std::move(saved)); - }); + if (mode_ & CollateMode::skip_store_candidate) { + td::actor::send_closure_later(actor_id(this), &Collator::return_block_candidate, td::Unit()); + } else { + LOG(INFO) << "saving new BlockCandidate"; + td::actor::send_closure_later(manager, &ValidatorManager::set_block_candidate, block_candidate->id, + block_candidate->clone(), [self = get_self()](td::Result saved) -> void { + LOG(DEBUG) << "got answer to set_block_candidate"; + td::actor::send_closure_later(std::move(self), &Collator::return_block_candidate, + std::move(saved)); + }); + } // 5. communicate about bad and delayed external messages if (!bad_ext_msgs_.empty() || !delay_ext_msgs_.empty()) { LOG(INFO) << "sending complete_external_messages() to Manager"; diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index 5d8b64400..a5c52f7cc 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -210,9 +210,9 @@ void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, } void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, - Ed25519_PublicKey collator_id, td::Ref validator_set, + Ed25519_PublicKey creator, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise) { + td::Promise promise, unsigned mode) { BlockSeqno seqno = 0; for (auto& p : prev) { if (p.seqno() > seqno) { @@ -220,8 +220,8 @@ void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_bloc } } td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, false, - min_masterchain_block_id, std::move(prev), std::move(validator_set), collator_id, - std::move(manager), timeout, std::move(promise)) + min_masterchain_block_id, std::move(prev), std::move(validator_set), creator, + std::move(manager), timeout, std::move(promise), mode) .release(); } @@ -236,7 +236,8 @@ void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_b } td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, min_masterchain_block_id, std::move(prev), td::Ref{}, - Ed25519_PublicKey{Bits256::zero()}, std::move(manager), timeout, std::move(promise)) + Ed25519_PublicKey{Bits256::zero()}, std::move(manager), timeout, std::move(promise), + 0) .release(); } From 5c02459fd8a1cc07f03633c7d095982bc1ab9730 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 31 Jul 2023 18:12:09 +0300 Subject: [PATCH 069/388] Optimize importing out queues --- create-hardfork/create-hardfork.cpp | 7 +- crypto/block/block.h | 4 +- test/test-ton-collator.cpp | 6 +- tl/generate/scheme/ton_api.tl | 4 +- tl/generate/scheme/ton_api.tlo | Bin 91588 -> 91588 bytes validator/collator-node.cpp | 33 +- validator/full-node-shard.cpp | 82 ++-- validator/full-node-shard.h | 4 +- validator/full-node-shard.hpp | 5 +- validator/full-node.cpp | 24 +- validator/full-node.hpp | 5 +- validator/impl/collator-impl.h | 3 +- validator/impl/collator.cpp | 38 +- validator/impl/out-msg-queue-proof.cpp | 499 ++++++++++++++------- validator/impl/out-msg-queue-proof.hpp | 94 ++-- validator/interfaces/out-msg-queue-proof.h | 33 +- validator/interfaces/validator-manager.h | 6 +- validator/manager-disk.hpp | 10 +- validator/manager-hardfork.hpp | 10 +- validator/manager.cpp | 123 ++--- validator/manager.hpp | 16 +- validator/validator.h | 10 +- 22 files changed, 577 insertions(+), 439 deletions(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index b5c151a75..165748f98 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -268,12 +268,11 @@ class HardforkCreator : public td::actor::Actor { td::Promise> promise) override { } void download_archive(ton::BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) override { } - void download_out_msg_queue_proof(ton::BlockIdExt block_id, ton::ShardIdFull dst_shard, - block::ImportedMsgQueueLimits limits, td::Timestamp timeout, - td::Promise> promise) override { + void download_out_msg_queue_proof( + ton::ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, + td::Timestamp timeout, td::Promise>> promise) override { } void new_key_block(ton::validator::BlockHandle handle) override { diff --git a/crypto/block/block.h b/crypto/block/block.h index 5f3f31630..6a94f0aad 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -218,8 +218,8 @@ static inline std::ostream& operator<<(std::ostream& os, const MsgProcessedUptoC struct ImportedMsgQueueLimits { // Default values - td::uint32 max_bytes = 1 << 18; - td::uint32 max_msgs = 40; + td::uint32 max_bytes = 1 << 19; + td::uint32 max_msgs = 500; bool deserialize(vm::CellSlice& cs); }; diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index 1bd1cadae..aa4c256b6 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -372,9 +372,9 @@ class TestNode : public td::actor::Actor { td::Promise promise) override { } - void download_out_msg_queue_proof(ton::BlockIdExt block_id, ton::ShardIdFull dst_shard, - block::ImportedMsgQueueLimits, td::Timestamp timeout, - td::Promise> promise) override { + void download_out_msg_queue_proof( + ton::ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, + td::Timestamp timeout, td::Promise>> promise) override { } void new_key_block(ton::validator::BlockHandle handle) override { diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 0a4134e5f..6d64b692b 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -416,7 +416,7 @@ tonNode.archiveNotFound = tonNode.ArchiveInfo; tonNode.archiveInfo id:long = tonNode.ArchiveInfo; tonNode.importedMsgQueueLimits max_bytes:int max_msgs:int = ImportedMsgQueueLimits; -tonNode.outMsgQueueProof queue_proof:bytes block_state_proof:bytes msg_count:int = tonNode.OutMsgQueueProof; +tonNode.outMsgQueueProof queue_proofs:bytes block_state_proofs:bytes msg_counts:(vector int) = tonNode.OutMsgQueueProof; tonNode.outMsgQueueProofEmpty = tonNode.OutMsgQueueProof; tonNode.forgetPeer = tonNode.ForgetPeer; @@ -452,7 +452,7 @@ tonNode.downloadBlockProofLinks blocks:(vector tonNode.blockIdExt) = tonNode.Dat tonNode.downloadKeyBlockProofLinks blocks:(vector tonNode.blockIdExt) = tonNode.DataList; tonNode.getArchiveInfo masterchain_seqno:int = tonNode.ArchiveInfo; tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data; -tonNode.getOutMsgQueueProof block_id:tonNode.blockIdExt dst_workchain:int dst_shard:long +tonNode.getOutMsgQueueProof dst_shard:tonNode.shardId blocks:(vector tonNode.blockIdExt) limits:tonNode.importedMsgQueueLimits = tonNode.OutMsgQueueProof; tonNode.getCapabilities = tonNode.Capabilities; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 974e5ad6d42377302c711e0636525175d496910f..9ecd272dc182524dcfa66469b1b53029f87bafa0 100644 GIT binary patch delta 193 zcmX?dn)S$O)(vm;SqwyH{Mr0L-{LIGC26LZ$vS^!WI;kVj|urOGcYhnKfABVQ&^f> zni^kFl%JngJb7Y^w5U*0PJVKBd~r!)2~3=UVe-cyac-{M;`I3B{L;LV;>r0XhG63- zU-@ew0W%b6CPR@kn+}I<+d;71GXHWCSQI0DTBi5JG0Jel)G<&08^{ zh9YM+9S+;JgJ8={{^cZSzO{6|DqWJF=a-+7s+XQx;$K?gTbv#UvM2y#4@@`9 last_masterchain_block_ = state->get_block_id(); last_top_blocks_.clear(); last_top_blocks_[ShardIdFull{masterchainId, shardIdAll}] = last_masterchain_block_; - if (state->get_unix_time() > (td::uint32)td::Clocks::system() - 20) { - std::vector next_shards; - if (can_collate_shard(ShardIdFull(masterchainId))) { - next_shards.push_back(ShardIdFull(masterchainId)); - } - for (const auto& desc : state->get_shards()) { - last_top_blocks_[desc->shard()] = desc->top_block_id(); - ShardIdFull shard = desc->shard(); - if (desc->before_split()) { - if (can_collate_shard(shard_child(shard, true))) { - next_shards.push_back(shard_child(shard, true)); - } - if (can_collate_shard(shard_child(shard, false))) { - next_shards.push_back(shard_child(shard, false)); - } - } else if (desc->before_merge()) { - if (is_left_child(shard) && can_collate_shard(shard_parent(shard))) { - next_shards.push_back(shard_parent(shard)); - } - } else if (can_collate_shard(shard)) { - next_shards.push_back(shard); - } - } - for (const ShardIdFull& shard : next_shards) { - for (const auto& neighbor : last_top_blocks_) { - if (neighbor.first != shard && block::ShardConfig::is_neighbor(shard, neighbor.first)) { - td::actor::send_closure(manager_, &ValidatorManager::wait_out_msg_queue_proof, neighbor.second, shard, 0, - td::Timestamp::in(10.0), [](td::Ref) {}); - } - } - } + for (const auto& desc : state->get_shards()) { + last_top_blocks_[desc->shard()] = desc->top_block_id(); } if (validators_.empty() || state->is_key_state()) { validators_.clear(); diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 7938111a3..5fcd62c64 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -670,18 +670,26 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getOutMsgQueueProof &query, td::Promise promise) { - BlockIdExt block_id = create_block_id(query.block_id_); - ShardIdFull dst_shard(query.dst_workchain_, query.dst_shard_); - block::ImportedMsgQueueLimits limits{(td::uint32)query.limits_->max_bytes_, (td::uint32)query.limits_->max_msgs_}; - if (!block_id.is_valid_ext()) { - promise.set_error(td::Status::Error("invalid block_id")); - return; + std::vector blocks; + for (const auto& x : query.blocks_) { + BlockIdExt id = create_block_id(x); + if (!id.is_valid_ext()) { + promise.set_error(td::Status::Error("invalid block_id")); + return; + } + if (!shard_is_ancestor(shard_, id.shard_full())) { + promise.set_error(td::Status::Error("query in wrong overlay")); + return; + } + blocks.push_back(create_block_id(x)); } + ShardIdFull dst_shard = create_shard_id(query.dst_shard_); if (!dst_shard.is_valid_ext()) { promise.set_error(td::Status::Error("invalid shard")); return; } - if (limits.max_bytes > (1 << 21)) { + block::ImportedMsgQueueLimits limits{(td::uint32)query.limits_->max_bytes_, (td::uint32)query.limits_->max_msgs_}; + if (limits.max_bytes > (1 << 24)) { promise.set_error(td::Status::Error("max_bytes is too big")); return; } @@ -693,10 +701,10 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod promise.set_result(serialize_tl_object(R.move_as_ok(), true)); } }); - VLOG(FULL_NODE_DEBUG) << "Got query getOutMsgQueueProof " << block_id.to_str() << " " << dst_shard.to_str() - << " from " << src; - td::actor::create_actor("buildqueueproof", block_id, dst_shard, limits, validator_manager_, - std::move(P)) + VLOG(FULL_NODE_DEBUG) << "Got query getOutMsgQueueProof (" << blocks.size() << " blocks) to shard " + << dst_shard.to_str() << " from " << src; + td::actor::create_actor("buildqueueproof", dst_shard, std::move(blocks), limits, + validator_manager_, std::move(P)) .release(); } @@ -935,33 +943,43 @@ void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, std::stri .release(); } -void FullNodeShardImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, +void FullNodeShardImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Timestamp timeout, - td::Promise> promise) { + td::Promise>> promise) { // TODO: maybe more complex download (like other requests here) auto &b = choose_neighbour(true); if (b.adnl_id == adnl::AdnlNodeIdShort::zero()) { promise.set_error(td::Status::Error(ErrorCode::notready, "no nodes")); return; } - auto P = td::PromiseCreator::lambda( - [=, promise = create_neighbour_promise(b, std::move(promise), true)](td::Result R) mutable { - if (R.is_error()) { - promise.set_result(R.move_as_error()); - return; - } - TRY_RESULT_PROMISE(promise, f, fetch_tl_object(R.move_as_ok(), true)); - ton_api::downcast_call(*f, td::overloaded( - [&](ton_api::tonNode_outMsgQueueProofEmpty &x) { - promise.set_error(td::Status::Error("node doesn't have this block")); - }, - [&](ton_api::tonNode_outMsgQueueProof &x) { - promise.set_result(OutMsgQueueProof::fetch(block_id, dst_shard, limits, x)); - })); - }); + std::vector> blocks_tl; + for (const BlockIdExt &id : blocks) { + blocks_tl.push_back(create_tl_block_id(id)); + } td::BufferSlice query = create_serialize_tl_object( - create_tl_block_id(block_id), dst_shard.workchain, dst_shard.shard, + create_tl_shard_id(dst_shard), std::move(blocks_tl), create_tl_object(limits.max_bytes, limits.max_msgs)); + + auto P = td::PromiseCreator::lambda([=, promise = create_neighbour_promise(b, std::move(promise), true), + blocks = std::move(blocks)](td::Result R) mutable { + if (R.is_error()) { + promise.set_result(R.move_as_error()); + return; + } + TRY_RESULT_PROMISE(promise, f, fetch_tl_object(R.move_as_ok(), true)); + ton_api::downcast_call( + *f, td::overloaded( + [&](ton_api::tonNode_outMsgQueueProofEmpty &x) { + promise.set_error(td::Status::Error("node doesn't have this block")); + }, + [&](ton_api::tonNode_outMsgQueueProof &x) { + delay_action( + [=, promise = std::move(promise), blocks = std::move(blocks), x = std::move(x)]() mutable { + promise.set_result(OutMsgQueueProof::fetch(dst_shard, blocks, limits, x)); + }, + td::Timestamp::now()); + })); + }); td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, b.adnl_id, adnl_id_, overlay_id_, "get_msg_queue", std::move(P), timeout, std::move(query), 1 << 22, rldp_); } @@ -1356,9 +1374,9 @@ td::actor::ActorOwn FullNodeShard::create( td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, FullNodeShardMode mode) { - return td::actor::create_actor("tonnode", shard, local_id, adnl_id, zero_state_file_hash, config, - keyring, adnl, rldp, rldp2, overlays, validator_manager, client, - mode); + return td::actor::create_actor(PSTRING() << "fullnode" << shard.to_str(), shard, local_id, adnl_id, + zero_state_file_hash, config, keyring, adnl, rldp, rldp2, overlays, + validator_manager, client, mode); } } // namespace fullnode diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index fca17bf4b..2131939fe 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -70,9 +70,9 @@ class FullNodeShard : public td::actor::Actor { td::Promise> promise) = 0; virtual void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) = 0; - virtual void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, + virtual void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Timestamp timeout, - td::Promise> promise) = 0; + td::Promise>> promise) = 0; virtual void set_handle(BlockHandle handle, td::Promise promise) = 0; diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index af16ccc91..f2b8bb669 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -187,8 +187,9 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise> promise) override; void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) override; - void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, - td::Timestamp timeout, td::Promise> promise) override; + void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise) override; void set_handle(BlockHandle handle, td::Promise promise) override; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index eebc49f8b..bd4132705 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -378,17 +378,22 @@ void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, std::string tm std::move(promise)); } -void FullNodeImpl::download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, +void FullNodeImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Timestamp timeout, - td::Promise> promise) { - auto shard = get_shard(block_id.shard_full()); + td::Promise>> promise) { + if (blocks.empty()) { + promise.set_value({}); + return; + } + // All blocks are expected to have the same minsplit shard prefix + auto shard = get_shard(blocks[0].shard_full()); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping download msg queue query to unknown shard"; promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); return; } - td::actor::send_closure(shard, &FullNodeShard::download_out_msg_queue_proof, block_id, dst_shard, limits, timeout, - std::move(promise)); + td::actor::send_closure(shard, &FullNodeShard::download_out_msg_queue_proof, dst_shard, std::move(blocks), limits, + timeout, std::move(promise)); } td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { @@ -588,10 +593,11 @@ void FullNodeImpl::start_up() { td::actor::send_closure(id_, &FullNodeImpl::download_archive, masterchain_seqno, std::move(tmp_dir), timeout, std::move(promise)); } - void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, - td::Timestamp timeout, td::Promise> promise) override { - td::actor::send_closure(id_, &FullNodeImpl::download_out_msg_queue_proof, block_id, dst_shard, limits, timeout, - std::move(promise)); + void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise) override { + td::actor::send_closure(id_, &FullNodeImpl::download_out_msg_queue_proof, dst_shard, std::move(blocks), limits, + timeout, std::move(promise)); } void new_key_block(BlockHandle handle) override { diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 99ea49cc3..fd0240cd9 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -74,8 +74,9 @@ class FullNodeImpl : public FullNode { void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise); void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise); - void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, - td::Timestamp timeout, td::Promise> promise); + void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise); void got_key_block_proof(td::Ref proof); void got_zero_block_state(td::Ref state); diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 0a0d59ca6..10953a97f 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -233,7 +233,8 @@ class Collator final : public td::actor::Actor { void after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res); bool fix_one_processed_upto(block::MsgProcessedUpto& proc, const ton::ShardIdFull& owner); bool fix_processed_upto(block::MsgProcessedUptoCollection& upto); - void got_neighbor_msg_queue(unsigned i, td::Result> R); + void got_neighbor_msg_queues(td::Result>> R); + void got_neighbor_msg_queue(unsigned i, Ref res); bool adjust_shard_config(); bool store_shard_fees(ShardIdFull shard, const block::CurrencyCollection& fees, const block::CurrencyCollection& created); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index b254b70c9..d2ace04f6 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -632,27 +632,45 @@ bool Collator::request_neighbor_msg_queues() { } neighbors_.emplace_back(*shard_ptr); } + std::vector top_blocks; unsigned i = 0; for (block::McShardDescr& descr : neighbors_) { LOG(DEBUG) << "neighbor #" << i << " : " << descr.blk_.to_str(); - ++pending; - send_closure_later(manager, &ValidatorManager::wait_out_msg_queue_proof, descr.blk_, shard_, priority(), timeout, - [self = get_self(), i](td::Result> res) { - LOG(DEBUG) << "got msg queue for neighbor #" << i; - send_closure_later(std::move(self), &Collator::got_neighbor_msg_queue, i, std::move(res)); - }); + top_blocks.push_back(descr.blk_); ++i; } + ++pending; + td::actor::send_closure_later( + manager, &ValidatorManager::wait_neighbor_msg_queue_proofs, shard_, std::move(top_blocks), timeout, + [self = get_self()](td::Result>> res) { + td::actor::send_closure_later(std::move(self), &Collator::got_neighbor_msg_queues, std::move(res)); + }); return true; } -void Collator::got_neighbor_msg_queue(unsigned i, td::Result> R) { +void Collator::got_neighbor_msg_queues(td::Result>> R) { --pending; if (R.is_error()) { - fatal_error(R.move_as_error()); + fatal_error(R.move_as_error_prefix("failed to get neighbor msg queues: ")); return; } + LOG(INFO) << "neighbor output queues fetched"; auto res = R.move_as_ok(); + unsigned i = 0; + for (block::McShardDescr& descr : neighbors_) { + LOG(DEBUG) << "neighbor #" << i << " : " << descr.blk_.to_str(); + auto it = res.find(descr.blk_); + if (it == res.end()) { + fatal_error(PSTRING() << "no msg queue from neighbor #" << i); + return; + } + got_neighbor_msg_queue(i, it->second); + ++i; + } + check_pending(); +} + +void Collator::got_neighbor_msg_queue(unsigned i, Ref res) { BlockIdExt block_id = neighbors_.at(i).blk_; if (res->block_state_proof_.not_null() && !block_id.is_masterchain()) { block_state_proofs_.emplace(block_id.root_hash, res->block_state_proof_); @@ -732,10 +750,6 @@ void Collator::got_neighbor_msg_queue(unsigned i, td::Result process_queue(BlockIdExt block_id, ShardIdFull dst_shard, - block::ImportedMsgQueueLimits limits, - const block::gen::OutMsgQueueInfo::Record& qinfo) { +static td::Result> process_queue( + ShardIdFull dst_shard, std::vector> blocks, + block::ImportedMsgQueueLimits limits) { td::uint64 estimated_proof_size = 0; td::HashSet visited; @@ -66,14 +66,18 @@ static td::Result process_queue(BlockIdExt block_id, ShardIdFull dst_ dfs(cs.prefetch_ref(i)); } }; - TRY_STATUS_PREFIX(check_no_prunned(*qinfo.proc_info), "invalid proc_info proof: ") - TRY_STATUS_PREFIX(check_no_prunned(*qinfo.ihr_pending), "invalid ihr_pending proof: ") - dfs_cs(*qinfo.proc_info); - dfs_cs(*qinfo.ihr_pending); - - block::OutputQueueMerger queue_merger{ - dst_shard, {block::OutputQueueMerger::Neighbor{block_id, qinfo.out_queue->prefetch_ref()}}}; - td::int32 msg_count = 0; + std::vector neighbors; + for (auto& b : blocks) { + TRY_STATUS_PREFIX(check_no_prunned(*b.second.proc_info), "invalid proc_info proof: ") + TRY_STATUS_PREFIX(check_no_prunned(*b.second.ihr_pending), "invalid ihr_pending proof: ") + dfs_cs(*b.second.proc_info); + dfs_cs(*b.second.ihr_pending); + neighbors.emplace_back(b.first, b.second.out_queue->prefetch_ref()); + } + + block::OutputQueueMerger queue_merger{dst_shard, std::move(neighbors)}; + std::vector msg_count(blocks.size()); + td::int32 msg_count_total = 0; bool limit_reached = false; while (!queue_merger.is_eof()) { @@ -87,7 +91,8 @@ static td::Result process_queue(BlockIdExt block_id, ShardIdFull dst_ if (limit_reached) { break; } - ++msg_count; + ++msg_count[kv->source]; + ++msg_count_total; // TODO: Get processed_upto from destination shard (in request?) /* @@ -114,234 +119,410 @@ static td::Result process_queue(BlockIdExt block_id, ShardIdFull dst_ dfs_cs(*kv->msg); TRY_STATUS_PREFIX(check_no_prunned(*kv->msg), "invalid message proof: ") - if (estimated_proof_size >= limits.max_bytes || msg_count >= (long long)limits.max_msgs) { + if (estimated_proof_size >= limits.max_bytes || msg_count_total >= (long long)limits.max_msgs) { limit_reached = true; } } - return limit_reached ? msg_count : -1; + if (!limit_reached) { + std::fill(msg_count.begin(), msg_count.end(), -1); + } + return msg_count; } td::Result> OutMsgQueueProof::build( - BlockIdExt block_id, ShardIdFull dst_shard, - block::ImportedMsgQueueLimits limits, Ref state_root, Ref block_root) { + ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits) { if (!dst_shard.is_valid_ext()) { return td::Status::Error("invalid shard"); } + if (blocks.empty()) { + return create_tl_object(td::BufferSlice{}, td::BufferSlice{}, + std::vector{}); + } + + std::vector> block_state_proofs; + for (auto& block : blocks) { + if (block.id.seqno() != 0) { + if (block.block_root.is_null()) { + return td::Status::Error("block is null"); + } + TRY_RESULT(proof, create_block_state_proof(block.block_root)); + block_state_proofs.push_back(std::move(proof)); + } + if (!block::ShardConfig::is_neighbor(dst_shard, block.id.shard_full())) { + return td::Status::Error("shards are not neighbors"); + } + } + TRY_RESULT(block_state_proof, vm::std_boc_serialize_multi(block_state_proofs)); - vm::MerkleProofBuilder mpb{std::move(state_root)}; - TRY_RESULT(state, ShardStateQ::fetch(block_id, {}, mpb.root())); - TRY_RESULT(outq_descr, state->message_queue()); - block::gen::OutMsgQueueInfo::Record qinfo; - if (!tlb::unpack_cell(outq_descr->root_cell(), qinfo)) { - return td::Status::Error("invalid message queue"); + vm::Dictionary states_dict_pure{32}; + for (size_t i = 0; i < blocks.size(); ++i) { + if (blocks[i].state_root.is_null()) { + return td::Status::Error("state is null"); + } + states_dict_pure.set_ref(td::BitArray<32>{(long long)i}, blocks[i].state_root); } - TRY_RESULT(cnt, process_queue(block_id, dst_shard, limits, qinfo)); - TRY_RESULT(queue_proof, mpb.extract_proof_boc()); - td::BufferSlice block_state_proof; - if (block_id.seqno() != 0) { - TRY_RESULT(proof, create_block_state_proof(std::move(block_root))); - TRY_RESULT_ASSIGN(block_state_proof, vm::std_boc_serialize(std::move(proof), 31)); + vm::MerkleProofBuilder mpb{states_dict_pure.get_root_cell()}; + vm::Dictionary states_dict{mpb.root(), 32}; + std::vector> data(blocks.size()); + for (size_t i = 0; i < blocks.size(); ++i) { + data[i].first = blocks[i].id; + TRY_RESULT(state, ShardStateQ::fetch(blocks[i].id, {}, states_dict.lookup_ref(td::BitArray<32>{(long long)i}))); + TRY_RESULT(outq_descr, state->message_queue()); + block::gen::OutMsgQueueInfo::Record qinfo; + if (!tlb::unpack_cell(outq_descr->root_cell(), data[i].second)) { + return td::Status::Error("invalid message queue"); + } } + TRY_RESULT(msg_count, process_queue(dst_shard, std::move(data), limits)); + + TRY_RESULT(proof, mpb.extract_proof()); + vm::Dictionary states_dict_proof{vm::CellSlice{vm::NoVm(), proof}.prefetch_ref(), 32}; + std::vector> state_proofs; + for (size_t i = 0; i < blocks.size(); ++i) { + td::Ref proof_raw = states_dict_proof.lookup_ref(td::BitArray<32>{(long long)i}); + CHECK(proof_raw.not_null()); + state_proofs.push_back(vm::CellBuilder::create_merkle_proof(proof_raw)); + } + TRY_RESULT(queue_proof, vm::std_boc_serialize_multi(state_proofs)); return create_tl_object(std::move(queue_proof), std::move(block_state_proof), - cnt); + std::move(msg_count)); } -td::Result> OutMsgQueueProof::fetch(BlockIdExt block_id, ShardIdFull dst_shard, - block::ImportedMsgQueueLimits limits, - const ton_api::tonNode_outMsgQueueProof& f) { +td::Result>> OutMsgQueueProof::fetch(ShardIdFull dst_shard, + std::vector blocks, + block::ImportedMsgQueueLimits limits, + const ton_api::tonNode_outMsgQueueProof& f) { try { - Ref block_state_proof; - td::Bits256 state_root_hash; - if (block_id.seqno() == 0) { - if (!f.block_state_proof_.empty()) { - return td::Status::Error("expected empty block state proof"); - } - state_root_hash = block_id.root_hash; - } else { - TRY_RESULT_ASSIGN(block_state_proof, vm::std_boc_deserialize(f.block_state_proof_.as_slice())); - TRY_RESULT_ASSIGN(state_root_hash, unpack_block_state_proof(block_id, block_state_proof)); + std::vector> res; + TRY_RESULT(queue_proofs, vm::std_boc_deserialize_multi(f.queue_proofs_, (int)blocks.size())); + TRY_RESULT(block_state_proofs, vm::std_boc_deserialize_multi(f.block_state_proofs_, (int)blocks.size())); + if (queue_proofs.size() != blocks.size()) { + return td::Status::Error("invalid size of queue_proofs"); } - - TRY_RESULT(queue_proof, vm::std_boc_deserialize(f.queue_proof_.as_slice())); - auto virtual_root = vm::MerkleProof::virtualize(queue_proof, 1); - if (virtual_root.is_null()) { - return td::Status::Error("invalid queue proof"); + if (f.msg_counts_.size() != blocks.size()) { + return td::Status::Error("invalid size of msg_counts"); } - if (virtual_root->get_hash().as_slice() != state_root_hash.as_slice()) { - return td::Status::Error("state root hash mismatch"); + size_t j = 0; + std::vector> data(blocks.size()); + for (size_t i = 0; i < blocks.size(); ++i) { + td::Bits256 state_root_hash; + Ref block_state_proof = {}; + if (blocks[i].seqno() == 0) { + state_root_hash = blocks[i].root_hash; + } else { + if (j == block_state_proofs.size()) { + return td::Status::Error("invalid size of block_state_proofs"); + } + block_state_proof = block_state_proofs[j++]; + TRY_RESULT_ASSIGN(state_root_hash, unpack_block_state_proof(blocks[i], block_state_proof)); + } + auto state_root = vm::MerkleProof::virtualize(queue_proofs[i], 1); + if (state_root->get_hash().as_slice() != state_root_hash.as_slice()) { + return td::Status::Error("state root hash mismatch"); + } + res.emplace_back(true, blocks[i], state_root, block_state_proof, f.msg_counts_[i]); + + data[i].first = blocks[i]; + TRY_RESULT(state, ShardStateQ::fetch(blocks[i], {}, state_root)); + TRY_RESULT(outq_descr, state->message_queue()); + block::gen::OutMsgQueueInfo::Record qinfo; + if (!tlb::unpack_cell(outq_descr->root_cell(), data[i].second)) { + return td::Status::Error("invalid message queue"); + } } - - // Validate proof - TRY_RESULT_PREFIX(state, ShardStateQ::fetch(block_id, {}, virtual_root), "invalid proof: "); - TRY_RESULT_PREFIX(outq_descr, state->message_queue(), "invalid proof: "); - - block::gen::OutMsgQueueInfo::Record qinfo; - if (!tlb::unpack_cell(outq_descr->root_cell(), qinfo)) { - return td::Status::Error("invalid proof: invalid message queue"); + if (j != block_state_proofs.size()) { + return td::Status::Error("invalid size of block_state_proofs"); } - TRY_STATUS_PREFIX(check_no_prunned(qinfo.proc_info->prefetch_ref(0)), "invalid proc_info: ") - TRY_STATUS_PREFIX(check_no_prunned(qinfo.ihr_pending->prefetch_ref(0)), "invalid ihr_pending: ") - TRY_RESULT(cnt, process_queue(block_id, dst_shard, limits, qinfo)); - if (cnt != f.msg_count_) { - return td::Status::Error(PSTRING() << "invalid msg_count: expected=" << f.msg_count_ << ", found=" << cnt); + TRY_RESULT(msg_count, process_queue(dst_shard, std::move(data), limits)); + if (msg_count != f.msg_counts_) { + return td::Status::Error("incorrect msg_count"); } - return Ref(true, std::move(virtual_root), std::move(block_state_proof), cnt); + return res; } catch (vm::VmVirtError& err) { return td::Status::Error(PSTRING() << "invalid proof: " << err.get_msg()); } } -void WaitOutMsgQueueProof::alarm() { - abort_query(td::Status::Error(ErrorCode::timeout, "timeout")); +void OutMsgQueueImporter::new_masterchain_block_notification(td::Ref state, + std::set collating_shards) { + last_masterchain_state_ = state; + if (collating_shards.empty() || state->get_unix_time() < (td::uint32)td::Clocks::system() - 20) { + return; + } + auto can_collate_shard = [&](const ShardIdFull& shard) -> bool { + return std::any_of(collating_shards.begin(), collating_shards.end(), + [&](ShardIdFull our_shard) { return shard_intersects(shard, our_shard); }); + }; + auto shards = state->get_shards(); + auto process_dst_shard = [&](const ShardIdFull& dst_shard) { + if (!can_collate_shard(dst_shard)) { + return; + } + std::vector top_blocks; + for (const auto& shard : shards) { + if (block::ShardConfig::is_neighbor(dst_shard, shard->shard())) { + top_blocks.push_back(shard->top_block_id()); + } + } + get_neighbor_msg_queue_proofs(dst_shard, std::move(top_blocks), td::Timestamp::in(15.0), + [](td::Result>>) {}); + }; + for (const auto& shard : shards) { + if (shard->before_merge()) { + if (is_left_child(shard->shard())) { + process_dst_shard(shard_parent(shard->shard())); + } + } else if (shard->before_split()) { + process_dst_shard(shard_child(shard->shard(), true)); + process_dst_shard(shard_child(shard->shard(), false)); + } else { + process_dst_shard(shard->shard()); + } + } } -void WaitOutMsgQueueProof::abort_query(td::Status reason) { - if (promise_) { - LOG(DEBUG) << "aborting wait msg queue query for " << block_id_.to_str() << " priority=" << priority_ << ": " - << reason; - promise_.set_error( - reason.move_as_error_prefix(PSTRING() << "failed to get msg queue for " << block_id_.to_str() << ": ")); +void OutMsgQueueImporter::get_neighbor_msg_queue_proofs( + ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, + td::Promise>> promise) { + std::sort(blocks.begin(), blocks.end()); + auto entry = cache_[{dst_shard, blocks}]; + if (entry) { + if (entry->done) { + promise.set_result(entry->result); + alarm_timestamp().relax(entry->timeout = td::Timestamp::in(CACHE_TTL)); + } else { + entry->timeout = std::max(entry->timeout, timeout); + entry->promises.emplace_back(std::move(promise), timeout); + alarm_timestamp().relax(timeout); + } + return; } - stop(); -} -void WaitOutMsgQueueProof::finish_query(Ref result) { - promise_.set_result(std::move(result)); - stop(); + LOG(DEBUG) << "Importing neighbor msg queues for shard " << dst_shard.to_str() << ", " << blocks.size() << " blocks"; + + cache_[{dst_shard, blocks}] = entry = std::make_shared(); + entry->dst_shard = dst_shard; + entry->blocks = blocks; + entry->promises.emplace_back(std::move(promise), timeout); + alarm_timestamp().relax(entry->timeout = timeout); + + std::map> new_queries; + for (const BlockIdExt& block : blocks) { + if (opts_->need_monitor(block.shard_full(), last_masterchain_state_)) { + ++entry->pending; + get_proof_local(entry, block); + } else { + ShardIdFull prefix = block.shard_full(); + int min_split = last_masterchain_state_->monitor_min_split_depth(prefix.workchain); + if (prefix.pfx_len() > min_split) { + prefix = shard_prefix(prefix, min_split); + } + new_queries[prefix].push_back(block); + } + }; + auto limits = last_masterchain_state_->get_imported_msg_queue_limits(dst_shard.workchain); + for (auto& p : new_queries) { + ++entry->pending; + get_proof_import(entry, std::move(p.second), limits); + } + if (entry->pending == 0) { + finish_query(entry); + } } -void WaitOutMsgQueueProof::start_up() { - alarm_timestamp() = timeout_; - if (local_) { - run_local(); - } else { - run_net(); +void OutMsgQueueImporter::get_proof_local(std::shared_ptr entry, BlockIdExt block) { + if (!check_timeout(entry)) { + return; } + td::actor::send_closure( + manager_, &ValidatorManager::wait_block_state_short, block, 0, entry->timeout, + [=, SelfId = actor_id(this), manager = manager_, timeout = entry->timeout, + retry_after = td::Timestamp::in(0.5)](td::Result> R) mutable { + if (R.is_error()) { + LOG(DEBUG) << "Failed to get block state for " << block.to_str() << ": " << R.move_as_error(); + delay_action([=]() { td::actor::send_closure(SelfId, &OutMsgQueueImporter::get_proof_local, entry, block); }, + retry_after); + return; + } + auto state = R.move_as_ok(); + if (block.seqno() == 0) { + std::vector> proof = { + td::Ref(true, block, state->root_cell(), td::Ref{})}; + td::actor::send_closure(SelfId, &OutMsgQueueImporter::got_proof, entry, std::move(proof)); + return; + } + td::actor::send_closure( + manager, &ValidatorManager::wait_block_data_short, block, 0, timeout, + [=](td::Result> R) mutable { + if (R.is_error()) { + LOG(DEBUG) << "Failed to get block data for " << block.to_str() << ": " << R.move_as_error(); + delay_action( + [=]() { td::actor::send_closure(SelfId, &OutMsgQueueImporter::get_proof_local, entry, block); }, + retry_after); + return; + } + Ref block_state_proof = create_block_state_proof(R.ok()->root_cell()).move_as_ok(); + std::vector> proof = { + td::Ref(true, block, state->root_cell(), std::move(block_state_proof))}; + td::actor::send_closure(SelfId, &OutMsgQueueImporter::got_proof, entry, std::move(proof)); + }); + }); } -void WaitOutMsgQueueProof::run_local() { - ++pending; - td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, block_id_, priority_, timeout_, - [SelfId = actor_id(this)](td::Result> R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::abort_query, - R.move_as_error_prefix("failed to get shard state: ")); - } else { - td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::got_state_root, - R.move_as_ok()->root_cell()); - } - }); - if (block_id_.seqno() != 0) { - ++pending; - td::actor::send_closure(manager_, &ValidatorManager::wait_block_data_short, block_id_, priority_, timeout_, - [SelfId = actor_id(this)](td::Result> R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::abort_query, - R.move_as_error_prefix("failed to get block data: ")); - } else { - td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::got_block_root, - R.move_as_ok()->root_cell()); - } - }); +void OutMsgQueueImporter::get_proof_import(std::shared_ptr entry, std::vector blocks, + block::ImportedMsgQueueLimits limits) { + if (!check_timeout(entry)) { + return; } + td::actor::send_closure( + manager_, &ValidatorManager::send_get_out_msg_queue_proof_request, entry->dst_shard, blocks, limits, + [=, SelfId = actor_id(this), retry_after = td::Timestamp::in(0.5), + dst_shard = entry->dst_shard](td::Result>> R) { + if (R.is_error()) { + LOG(DEBUG) << "Failed to get out msg queue for " << dst_shard.to_str() << ": " << R.move_as_error(); + delay_action( + [=]() { + td::actor::send_closure(SelfId, &OutMsgQueueImporter::get_proof_import, entry, std::move(blocks), + limits); + }, + retry_after); + return; + } + td::actor::send_closure(SelfId, &OutMsgQueueImporter::got_proof, entry, R.move_as_ok()); + }); } -void WaitOutMsgQueueProof::got_state_root(Ref root) { - state_root_ = std::move(root); - if (--pending == 0) { - run_local_cont(); +void OutMsgQueueImporter::got_proof(std::shared_ptr entry, std::vector> proofs) { + if (!check_timeout(entry)) { + return; + } + for (auto& p : proofs) { + entry->result[p->block_id_] = std::move(p); + } + CHECK(entry->pending > 0); + if (--entry->pending == 0) { + finish_query(entry); } } -void WaitOutMsgQueueProof::got_block_root(Ref root) { - block_root_ = std::move(root); - if (--pending == 0) { - run_local_cont(); +void OutMsgQueueImporter::finish_query(std::shared_ptr entry) { + LOG(DEBUG) << "Done importing neighbor msg queues for shard " << entry->dst_shard.to_str() << ", " + << entry->blocks.size() << " blocks in " << entry->timer.elapsed() << "s"; + entry->done = true; + alarm_timestamp().relax(entry->timeout = td::Timestamp::in(CACHE_TTL)); + for (auto& p : entry->promises) { + p.first.set_result(entry->result); } + entry->promises.clear(); } -void WaitOutMsgQueueProof::run_local_cont() { - Ref block_state_proof; - if (block_id_.seqno() != 0) { - auto R = create_block_state_proof(std::move(block_root_)); - if (R.is_error()) { - abort_query(R.move_as_error_prefix("failed to create block state proof")); - return; +bool OutMsgQueueImporter::check_timeout(std::shared_ptr entry) { + if (entry->timeout.is_in_past()) { + LOG(DEBUG) << "Aborting importing neighbor msg queues for shard " << entry->dst_shard.to_str() << ", " + << entry->blocks.size() << " blocks: timeout"; + for (auto& p : entry->promises) { + p.first.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); } - block_state_proof = R.move_as_ok(); + entry->promises.clear(); + auto it = cache_.find({entry->dst_shard, entry->blocks}); + if (it != cache_.end() && it->second == entry) { + cache_.erase(it); + } + return false; } - finish_query(td::Ref(true, std::move(state_root_), std::move(block_state_proof))); + return true; } -void WaitOutMsgQueueProof::run_net() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id = block_id_, - retry_after = td::Timestamp::in(0.5)](td::Result> R) { - if (R.is_error()) { - LOG(DEBUG) << "failed to get msg queue for " << block_id.to_str() << " from net: " << R.move_as_error(); - delay_action([SelfId]() mutable { td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::run_net); }, - retry_after); - } else { - td::actor::send_closure(SelfId, &WaitOutMsgQueueProof::finish_query, R.move_as_ok()); +void OutMsgQueueImporter::alarm() { + auto it = cache_.begin(); + while (it != cache_.end()) { + auto& promises = it->second->promises; + if (it->second->timeout.is_in_past()) { + if (!it->second->done) { + LOG(DEBUG) << "Aborting importing neighbor msg queues for shard " << it->second->dst_shard.to_str() << ", " + << it->second->blocks.size() << " blocks: timeout"; + for (auto& p : promises) { + p.first.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); + } + promises.clear(); + } + it = cache_.erase(it); + continue; } - }); - - td::actor::send_closure(manager_, &ValidatorManager::send_get_out_msg_queue_proof_request, block_id_, dst_shard_, - limits_, priority_, std::move(P)); + alarm_timestamp().relax(it->second->timeout); + size_t j = 0; + for (auto& p : promises) { + if (p.second.is_in_past()) { + p.first.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); + } else { + alarm_timestamp().relax(p.second); + promises[j++] = std::move(p); + } + } + promises.resize(j); + ++it; + } } void BuildOutMsgQueueProof::abort_query(td::Status reason) { if (promise_) { - LOG(DEBUG) << "failed to build msg queue proof for " << block_id_.to_str() << ": " << reason; + LOG(DEBUG) << "failed to build msg queue proof to " << dst_shard_.to_str() << ": " << reason; promise_.set_error( - reason.move_as_error_prefix(PSTRING() << "failed to build msg queue proof for " << block_id_.to_str() << ": ")); + reason.move_as_error_prefix(PSTRING() << "failed to build msg queue proof to " << dst_shard_.to_str() << ": ")); } stop(); } void BuildOutMsgQueueProof::start_up() { - ++pending; - td::actor::send_closure(manager_, &ValidatorManagerInterface::get_shard_state_from_db_short, block_id_, - [SelfId = actor_id(this)](td::Result> R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::abort_query, - R.move_as_error_prefix("failed to get shard state: ")); - } else { - td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_state_root, - R.move_as_ok()->root_cell()); - } - }); - if (block_id_.seqno() != 0) { + for (size_t i = 0; i < blocks_.size(); ++i) { + BlockIdExt id = blocks_[i].id; ++pending; - td::actor::send_closure(manager_, &ValidatorManagerInterface::get_block_data_from_db_short, block_id_, - [SelfId = actor_id(this)](td::Result> R) { + td::actor::send_closure(manager_, &ValidatorManagerInterface::get_shard_state_from_db_short, id, + [SelfId = actor_id(this), i](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::abort_query, - R.move_as_error_prefix("failed to get block data: ")); + R.move_as_error_prefix("failed to get shard state: ")); } else { - td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_block_root, + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_state_root, i, R.move_as_ok()->root_cell()); } }); + if (id.seqno() != 0) { + ++pending; + td::actor::send_closure(manager_, &ValidatorManagerInterface::get_block_data_from_db_short, id, + [SelfId = actor_id(this), i](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::abort_query, + R.move_as_error_prefix("failed to get block data: ")); + } else { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_block_root, i, + R.move_as_ok()->root_cell()); + } + }); + } + } + if (pending == 0) { + build_proof(); } } -void BuildOutMsgQueueProof::got_state_root(Ref root) { - state_root_ = std::move(root); +void BuildOutMsgQueueProof::got_state_root(size_t i, Ref root) { + blocks_[i].state_root = std::move(root); if (--pending == 0) { build_proof(); } } -void BuildOutMsgQueueProof::got_block_root(Ref root) { - block_root_ = std::move(root); +void BuildOutMsgQueueProof::got_block_root(size_t i, Ref root) { + blocks_[i].block_root = std::move(root); if (--pending == 0) { build_proof(); } } void BuildOutMsgQueueProof::build_proof() { - auto result = OutMsgQueueProof::build(block_id_, dst_shard_, limits_, std::move(state_root_), std::move(block_root_)); + auto result = OutMsgQueueProof::build(dst_shard_, std::move(blocks_), limits_); if (result.is_error()) { LOG(ERROR) << "Failed to build msg queue proof: " << result.error(); } diff --git a/validator/impl/out-msg-queue-proof.hpp b/validator/impl/out-msg-queue-proof.hpp index 9c5acfef9..33f6c399e 100644 --- a/validator/impl/out-msg-queue-proof.hpp +++ b/validator/impl/out-msg-queue-proof.hpp @@ -20,6 +20,8 @@ #include "auto/tl/ton_api.h" #include "interfaces/out-msg-queue-proof.h" #include "td/actor/actor.h" +#include "interfaces/shard.h" +#include "validator.h" namespace ton { @@ -29,83 +31,77 @@ using td::Ref; class ValidatorManager; class ValidatorManagerInterface; -class WaitOutMsgQueueProof : public td::actor::Actor { +class OutMsgQueueImporter : public td::actor::Actor { public: - WaitOutMsgQueueProof(BlockIdExt block_id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, bool local, - td::uint32 priority, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise> promise) - : block_id_(std::move(block_id)) - , dst_shard_(dst_shard) - , limits_(limits) - , local_(local) - , priority_(priority) - , manager_(manager) - , timeout_(timeout) - , promise_(std::move(promise)) { + OutMsgQueueImporter(td::actor::ActorId manager, td::Ref opts, + td::Ref last_masterchain_state) + : manager_(manager), opts_(opts), last_masterchain_state_(last_masterchain_state) { } - void update_timeout(td::Timestamp timeout, td::uint32 priority) { - timeout_ = timeout; - alarm_timestamp() = timeout_; - priority_ = priority; + void new_masterchain_block_notification(td::Ref state, std::set collating_shards); + void get_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, + td::Promise>> promise); + + void update_options(td::Ref opts) { + opts_ = std::move(opts); } - void abort_query(td::Status reason); - void finish_query(Ref result); void alarm() override; - void start_up() override; - - void run_local(); - void got_state_root(Ref root); - void got_block_root(Ref root); - void run_local_cont(); - - void run_net(); - private: - BlockIdExt block_id_; - ShardIdFull dst_shard_; - block::ImportedMsgQueueLimits limits_; - bool local_; - td::uint32 priority_; - td::actor::ActorId manager_; - td::Timestamp timeout_; - td::Promise> promise_; - - Ref state_root_, block_root_; - unsigned pending = 0; + td::Ref opts_; + td::Ref last_masterchain_state_; + + struct CacheEntry { + ShardIdFull dst_shard; + std::vector blocks; + bool done = false; + std::map> result; + std::vector>>, td::Timestamp>> promises; + td::Timestamp timeout = td::Timestamp::never(); + td::Timer timer; + size_t pending = 0; + }; + std::map>, std::shared_ptr> cache_; + + void get_proof_local(std::shared_ptr entry, BlockIdExt block); + void get_proof_import(std::shared_ptr entry, std::vector blocks, + block::ImportedMsgQueueLimits limits); + void got_proof(std::shared_ptr entry, std::vector> proofs); + void finish_query(std::shared_ptr entry); + bool check_timeout(std::shared_ptr entry); + + constexpr static const double CACHE_TTL = 30.0; }; class BuildOutMsgQueueProof : public td::actor::Actor { public: - BuildOutMsgQueueProof(BlockIdExt block_id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, + BuildOutMsgQueueProof(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::actor::ActorId manager, td::Promise> promise) - : block_id_(std::move(block_id)) - , dst_shard_(dst_shard) - , limits_(limits) - , manager_(manager) - , promise_(std::move(promise)) { + : dst_shard_(dst_shard), limits_(limits), manager_(manager), promise_(std::move(promise)) { + blocks_.resize(blocks.size()); + for (size_t i = 0; i < blocks_.size(); ++i) { + blocks_[i].id = blocks[i]; + } } void abort_query(td::Status reason); void start_up() override; - void got_state_root(Ref root); - void got_block_root(Ref root); + void got_state_root(size_t i, Ref root); + void got_block_root(size_t i, Ref root); void build_proof(); private: - BlockIdExt block_id_; ShardIdFull dst_shard_; + std::vector blocks_; block::ImportedMsgQueueLimits limits_; td::actor::ActorId manager_; td::Promise> promise_; - Ref state_root_, block_root_; - unsigned pending = 0; + size_t pending = 0; }; } // namespace validator diff --git a/validator/interfaces/out-msg-queue-proof.h b/validator/interfaces/out-msg-queue-proof.h index 524168d0d..c0aa56106 100644 --- a/validator/interfaces/out-msg-queue-proof.h +++ b/validator/interfaces/out-msg-queue-proof.h @@ -26,22 +26,31 @@ namespace validator { using td::Ref; struct OutMsgQueueProof : public td::CntObject { - OutMsgQueueProof(Ref state_root, Ref block_state_proof, td::int32 msg_count = -1) - : state_root_(std::move(state_root)), block_state_proof_(std::move(block_state_proof)), msg_count_(msg_count) { + OutMsgQueueProof(BlockIdExt block_id, Ref state_root, Ref block_state_proof, + td::int32 msg_count = -1) + : block_id_(block_id) + , state_root_(std::move(state_root)) + , block_state_proof_(std::move(block_state_proof)) + , msg_count_(msg_count) { } + BlockIdExt block_id_; Ref state_root_; Ref block_state_proof_; - td::int32 msg_count_; // -1 - up to end of queue - - static td::Result> fetch(BlockIdExt block_id, ShardIdFull dst_shard, - block::ImportedMsgQueueLimits limits, - const ton_api::tonNode_outMsgQueueProof &f); - static td::Result> build(BlockIdExt block_id, ShardIdFull dst_shard, - block::ImportedMsgQueueLimits limits, - Ref state_root, - Ref block_root); - + td::int32 msg_count_; // -1 - no limit + + static td::Result>> fetch(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, + const ton_api::tonNode_outMsgQueueProof &f); + + struct OneBlock { + BlockIdExt id; + Ref state_root; + Ref block_root; + }; + static td::Result> build(ShardIdFull dst_shard, + std::vector blocks, + block::ImportedMsgQueueLimits limits); }; } // namespace validator diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index a4a1dd167..c09d42937 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -129,9 +129,9 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void send_ihr_message(td::Ref message) = 0; virtual void send_top_shard_block_description(td::Ref desc) = 0; virtual void send_block_broadcast(BlockBroadcast broadcast) = 0; - virtual void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, - block::ImportedMsgQueueLimits limits, td::uint32 priority, - td::Promise> promise) = 0; + virtual void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, + td::Promise>> promise) = 0; virtual void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_shard_client_state(bool from_db, td::Promise promise) = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 17a1bffae..bad3fc35f 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -152,8 +152,8 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; - void wait_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) override { + void wait_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, + td::Promise>> promise) override { UNREACHABLE(); } @@ -255,9 +255,9 @@ class ValidatorManagerImpl : public ValidatorManager { void send_top_shard_block_description(td::Ref desc) override; void send_block_broadcast(BlockBroadcast broadcast) override { } - void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, - td::uint32 priority, - td::Promise> promise) override { + void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, + td::Promise>> promise) override { UNREACHABLE(); } diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 65d2b3c0d..33b6a1a71 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -182,8 +182,8 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; - void wait_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) override { + void wait_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, + td::Promise>> promise) override { UNREACHABLE(); } @@ -321,9 +321,9 @@ class ValidatorManagerImpl : public ValidatorManager { } void send_block_broadcast(BlockBroadcast broadcast) override { } - void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, - td::uint32 priority, - td::Promise> promise) override { + void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, + td::Promise>> promise) override { UNREACHABLE(); } diff --git a/validator/manager.cpp b/validator/manager.cpp index d419f80ee..955334dbc 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -618,33 +618,15 @@ void ValidatorManagerImpl::wait_block_state_short(BlockIdExt block_id, td::uint3 get_block_handle(block_id, true, std::move(P)); } -void ValidatorManagerImpl::wait_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::uint32 priority, - td::Timestamp timeout, - td::Promise> promise) { - auto key = std::make_pair(block_id, dst_shard); - auto it = wait_out_msg_queue_proof_.find(key); - if (it == wait_out_msg_queue_proof_.end()) { - auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), block_id, dst_shard](td::Result> R) { - td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_msg_queue, block_id, dst_shard, - std::move(R)); - }); - auto id = td::actor::create_actor( - "waitmsgqueue", block_id, dst_shard, - last_masterchain_state_->get_imported_msg_queue_limits(block_id.is_masterchain()), - need_monitor(block_id.shard_full()), priority, actor_id(this), td::Timestamp::at(timeout.at() + 10.0), - std::move(P)) - .release(); - wait_out_msg_queue_proof_[key].actor_ = id; - it = wait_out_msg_queue_proof_.find(key); - } else if (it->second.done_) { - promise.set_result(it->second.result_); - it->second.remove_at_ = td::Timestamp::in(30.0); - return; +void ValidatorManagerImpl::wait_neighbor_msg_queue_proofs( + ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, + td::Promise>> promise) { + if (out_msg_queue_importer_.empty()) { + out_msg_queue_importer_ = td::actor::create_actor("outmsgqueueimporter", actor_id(this), opts_, + last_masterchain_state_); } - it->second.waiting_.emplace_back(timeout, priority, std::move(promise)); - auto X = it->second.get_timeout(); - td::actor::send_closure(it->second.actor_, &WaitOutMsgQueueProof::update_timeout, X.first, X.second); + td::actor::send_closure(out_msg_queue_importer_, &OutMsgQueueImporter::get_neighbor_msg_queue_proofs, dst_shard, + std::move(blocks), timeout, std::move(promise)); } void ValidatorManagerImpl::wait_block_data(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, @@ -1082,44 +1064,6 @@ void ValidatorManagerImpl::finished_wait_data(BlockHandle handle, td::Result> R) { - auto it = wait_out_msg_queue_proof_.find({block_id, dst_shard}); - if (it != wait_out_msg_queue_proof_.end()) { - if (R.is_error()) { - auto S = R.move_as_error(); - if (S.code() != ErrorCode::timeout) { - for (auto &X : it->second.waiting_) { - X.promise.set_error(S.clone()); - } - } else { - auto X = it->second.get_timeout(); - auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), block_id, dst_shard](td::Result> R) { - td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_msg_queue, block_id, dst_shard, - std::move(R)); - }); - auto id = td::actor::create_actor( - "waitmsgqueue", block_id, dst_shard, - last_masterchain_state_->get_imported_msg_queue_limits(block_id.is_masterchain()), - need_monitor(block_id.shard_full()), X.second, actor_id(this), X.first, std::move(P)) - .release(); - it->second.actor_ = id; - return; - } - wait_out_msg_queue_proof_.erase(it); - } else { - auto r = R.move_as_ok(); - for (auto &X : it->second.waiting_) { - X.promise.set_result(r); - } - it->second.done_ = true; - it->second.result_ = std::move(r); - it->second.remove_at_ = td::Timestamp::in(30.0); - } - } -} - void ValidatorManagerImpl::set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) { auto P = td::PromiseCreator::lambda( @@ -1513,11 +1457,11 @@ void ValidatorManagerImpl::send_block_broadcast(BlockBroadcast broadcast) { callback_->send_broadcast(std::move(broadcast)); } -void ValidatorManagerImpl::send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, - block::ImportedMsgQueueLimits limits, - td::uint32 priority, - td::Promise> promise) { - callback_->download_out_msg_queue_proof(id, dst_shard, limits, td::Timestamp::in(10.0), std::move(promise)); +void ValidatorManagerImpl::send_get_out_msg_queue_proof_request( + ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, + td::Promise>> promise) { + callback_->download_out_msg_queue_proof(dst_shard, std::move(blocks), limits, td::Timestamp::in(10.0), + std::move(promise)); } void ValidatorManagerImpl::start_up() { @@ -1836,18 +1780,26 @@ void ValidatorManagerImpl::new_masterchain_block() { td::actor::send_closure(shard_client_, &ShardClient::new_masterchain_block_notification, last_masterchain_block_handle_, last_masterchain_state_); } + if (is_collator()) { + std::set collating_shards; + if (validating_masterchain()) { + collating_shards.emplace(masterchainId); + } + for (const auto &collator : collator_nodes_) { + collating_shards.insert(collator.second.shards.begin(), collator.second.shards.end()); + } + if (!collating_shards.empty()) { + if (out_msg_queue_importer_.empty()) { + out_msg_queue_importer_ = td::actor::create_actor("outmsgqueueimporter", actor_id(this), + opts_, last_masterchain_state_); + } + td::actor::send_closure(out_msg_queue_importer_, &OutMsgQueueImporter::new_masterchain_block_notification, + last_masterchain_state_, std::move(collating_shards)); + } + } for (auto &c : collator_nodes_) { td::actor::send_closure(c.second.actor, &CollatorNode::new_masterchain_block_notification, last_masterchain_state_); } - if (opts_->validator_mode() == ValidatorManagerOptions::validator_lite_shards && validating_masterchain() && - last_masterchain_state_->get_unix_time() > (td::uint32)td::Clocks::system() - 20) { - // Prepare neighboours' queues for collating masterchain - for (const auto &desc : last_masterchain_state_->get_shards()) { - wait_out_msg_queue_proof(desc->top_block_id(), ShardIdFull(masterchainId), 0, td::Timestamp::in(10.0), - [](td::Result>) {}); - } - } - if (last_masterchain_seqno_ % 1024 == 0) { LOG(WARNING) << "applied masterchain block " << last_masterchain_block_id_; } @@ -2514,18 +2466,6 @@ void ValidatorManagerImpl::alarm() { } } alarm_timestamp().relax(check_shard_clients_); - if (cleanup_wait_caches_at_.is_in_past()) { - auto it = wait_out_msg_queue_proof_.begin(); - while (it != wait_out_msg_queue_proof_.end()) { - if (it->second.done_ && it->second.remove_at_.is_in_past()) { - it = wait_out_msg_queue_proof_.erase(it); - } else { - ++it; - } - } - cleanup_wait_caches_at_ = td::Timestamp::in(10.0); - } - alarm_timestamp().relax(cleanup_wait_caches_at_); } void ValidatorManagerImpl::update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) { @@ -2809,6 +2749,9 @@ void ValidatorManagerImpl::update_options(td::Ref opts) if (!serializer_.empty()) { td::actor::send_closure(serializer_, &AsyncStateSerializer::update_options, opts); } + if (!out_msg_queue_importer_.empty()) { + td::actor::send_closure(out_msg_queue_importer_, &OutMsgQueueImporter::update_options, opts); + } opts_ = std::move(opts); } diff --git a/validator/manager.hpp b/validator/manager.hpp index a37fcb4e0..ce5254acf 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -189,15 +189,14 @@ class ValidatorManagerImpl : public ValidatorManager { }; std::map>> wait_state_; std::map>> wait_block_data_; - std::map, WaitListCaching>> - wait_out_msg_queue_proof_; - td::Timestamp cleanup_wait_caches_at_ = td::Timestamp::now(); struct WaitBlockHandle { std::vector> waiting_; }; std::map wait_block_handle_; + td::actor::ActorOwn out_msg_queue_importer_; + private: // HANDLES CACHE std::map> handles_; @@ -372,8 +371,8 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; - void wait_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) override; + void wait_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, + td::Promise>> promise) override; void set_block_data(BlockHandle handle, td::Ref data, td::Promise promise) override; void wait_block_data(BlockHandle handle, td::uint32 priority, td::Timestamp, @@ -462,9 +461,9 @@ class ValidatorManagerImpl : public ValidatorManager { void send_ihr_message(td::Ref message) override; void send_top_shard_block_description(td::Ref desc) override; void send_block_broadcast(BlockBroadcast broadcast) override; - void send_get_out_msg_queue_proof_request(BlockIdExt id, ShardIdFull dst_shard, block::ImportedMsgQueueLimits limits, - td::uint32 priority, - td::Promise> promise) override; + void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, + td::Promise>> promise) override; void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; @@ -499,7 +498,6 @@ class ValidatorManagerImpl : public ValidatorManager { void finished_wait_state(BlockHandle handle, td::Result> R); void finished_wait_data(BlockHandle handle, td::Result> R); - void finished_wait_msg_queue(BlockIdExt block_id, ShardIdFull dst_shard, td::Result> R); void start_up() override; void started(ValidatorManagerInitResult result); diff --git a/validator/validator.h b/validator/validator.h index 5fa23467f..506950e7f 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -140,9 +140,9 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Promise> promise) = 0; virtual void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) = 0; - virtual void download_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, + virtual void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Timestamp timeout, - td::Promise> promise) = 0; + td::Promise>> promise) = 0; virtual void new_key_block(BlockHandle handle) = 0; }; @@ -226,9 +226,9 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) = 0; - virtual void wait_out_msg_queue_proof(BlockIdExt block_id, ShardIdFull dst_shard, td::uint32 priority, - td::Timestamp timeout, - td::Promise> promise) = 0; + virtual void wait_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, + td::Timestamp timeout, + td::Promise>> promise) = 0; virtual void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) = 0; virtual void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, From e814973749d97025e06aa61c8b2fa3cd501be6bf Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 31 Jul 2023 22:56:25 +0300 Subject: [PATCH 070/388] Limit query size in importing out queues --- validator-engine/validator-engine.cpp | 2 ++ validator/impl/out-msg-queue-proof.cpp | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index d43b9b708..93bd23bee 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1850,6 +1850,8 @@ void ValidatorEngine::started_dht() { void ValidatorEngine::start_rldp() { rldp_ = ton::rldp::Rldp::create(adnl_.get()); rldp2_ = ton::rldp2::Rldp::create(adnl_.get()); + td::actor::send_closure(rldp_, &ton::rldp::Rldp::set_default_mtu, 2048); + td::actor::send_closure(rldp2_, &ton::rldp2::Rldp::set_default_mtu, 2048); started_rldp(); } diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 601f083aa..d0c3be957 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -324,7 +324,10 @@ void OutMsgQueueImporter::get_neighbor_msg_queue_proofs( auto limits = last_masterchain_state_->get_imported_msg_queue_limits(dst_shard.workchain); for (auto& p : new_queries) { ++entry->pending; - get_proof_import(entry, std::move(p.second), limits); + for (size_t i = 0; i < p.second.size(); i += 16) { + size_t j = std::min(i + 16, p.second.size()); + get_proof_import(entry, std::vector(p.second.begin() + i, p.second.begin() + j), limits); + } } if (entry->pending == 0) { finish_query(entry); From 9e02853cbb9283f50fe34a1c0e161eff0e8d862f Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 3 Aug 2023 12:06:41 +0300 Subject: [PATCH 071/388] Improve importing msg queues --- crypto/block/block.h | 7 +++- validator/full-node-shard.cpp | 40 +++++++++---------- validator/impl/out-msg-queue-proof.cpp | 55 ++++++++++++++++++++++++-- validator/manager.cpp | 31 +++++++++++++-- validator/manager.hpp | 2 + 5 files changed, 107 insertions(+), 28 deletions(-) diff --git a/crypto/block/block.h b/crypto/block/block.h index 6a94f0aad..f2c89286f 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -218,9 +218,12 @@ static inline std::ostream& operator<<(std::ostream& os, const MsgProcessedUptoC struct ImportedMsgQueueLimits { // Default values - td::uint32 max_bytes = 1 << 19; - td::uint32 max_msgs = 500; + td::uint32 max_bytes = 1 << 16; + td::uint32 max_msgs = 30; bool deserialize(vm::CellSlice& cs); + ImportedMsgQueueLimits operator*(td::uint32 x) const { + return {max_bytes * x, max_msgs * x}; + } }; struct ParamLimits { diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 5fcd62c64..3d9d19235 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -960,26 +960,26 @@ void FullNodeShardImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std: create_tl_shard_id(dst_shard), std::move(blocks_tl), create_tl_object(limits.max_bytes, limits.max_msgs)); - auto P = td::PromiseCreator::lambda([=, promise = create_neighbour_promise(b, std::move(promise), true), - blocks = std::move(blocks)](td::Result R) mutable { - if (R.is_error()) { - promise.set_result(R.move_as_error()); - return; - } - TRY_RESULT_PROMISE(promise, f, fetch_tl_object(R.move_as_ok(), true)); - ton_api::downcast_call( - *f, td::overloaded( - [&](ton_api::tonNode_outMsgQueueProofEmpty &x) { - promise.set_error(td::Status::Error("node doesn't have this block")); - }, - [&](ton_api::tonNode_outMsgQueueProof &x) { - delay_action( - [=, promise = std::move(promise), blocks = std::move(blocks), x = std::move(x)]() mutable { - promise.set_result(OutMsgQueueProof::fetch(dst_shard, blocks, limits, x)); - }, - td::Timestamp::now()); - })); - }); + auto P = td::PromiseCreator::lambda( + [=, promise = std::move(promise), blocks = std::move(blocks)](td::Result R) mutable { + if (R.is_error()) { + promise.set_result(R.move_as_error()); + return; + } + TRY_RESULT_PROMISE(promise, f, fetch_tl_object(R.move_as_ok(), true)); + ton_api::downcast_call( + *f, td::overloaded( + [&](ton_api::tonNode_outMsgQueueProofEmpty &x) { + promise.set_error(td::Status::Error("node doesn't have this block")); + }, + [&](ton_api::tonNode_outMsgQueueProof &x) { + delay_action( + [=, promise = std::move(promise), blocks = std::move(blocks), x = std::move(x)]() mutable { + promise.set_result(OutMsgQueueProof::fetch(dst_shard, blocks, limits, x)); + }, + td::Timestamp::now()); + })); + }); td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, b.adnl_id, adnl_id_, overlay_id_, "get_msg_queue", std::move(P), timeout, std::move(query), 1 << 22, rldp_); } diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index d0c3be957..3512bf0d6 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -285,6 +285,54 @@ void OutMsgQueueImporter::new_masterchain_block_notification(td::Ref blocks, td::Timestamp timeout, td::Promise>> promise) { + if (blocks.empty()) { + promise.set_value({}); + return; + } + if (dst_shard.is_masterchain() && blocks.size() != 1) { + // We spit queries for masterchain {dst_shard, {block_1, ..., block_n}} into separate queries + // {dst_shard, {block_1}}, ..., {dst_shard, {block_n}} + class Worker : public td::actor::Actor { + public: + Worker(size_t pending, td::Promise>> promise) + : pending_(pending), promise_(std::move(promise)) { + CHECK(pending_ > 0); + } + + void on_result(td::Ref res) { + result_[res->block_id_] = res; + if (--pending_ == 0) { + promise_.set_result(std::move(result_)); + stop(); + } + } + + void on_error(td::Status error) { + promise_.set_error(std::move(error)); + stop(); + } + + private: + size_t pending_; + td::Promise>> promise_; + std::map> result_; + }; + auto worker = td::actor::create_actor("queueworker", blocks.size(), std::move(promise)).release(); + for (const BlockIdExt& block : blocks) { + get_neighbor_msg_queue_proofs(dst_shard, {block}, timeout, + [=](td::Result>> R) { + if (R.is_error()) { + td::actor::send_closure(worker, &Worker::on_error, R.move_as_error()); + } else { + auto res = R.move_as_ok(); + CHECK(res.size() == 1); + td::actor::send_closure(worker, &Worker::on_result, res.begin()->second); + } + }); + } + return; + } + std::sort(blocks.begin(), blocks.end()); auto entry = cache_[{dst_shard, blocks}]; if (entry) { @@ -326,7 +374,8 @@ void OutMsgQueueImporter::get_neighbor_msg_queue_proofs( ++entry->pending; for (size_t i = 0; i < p.second.size(); i += 16) { size_t j = std::min(i + 16, p.second.size()); - get_proof_import(entry, std::vector(p.second.begin() + i, p.second.begin() + j), limits); + get_proof_import(entry, std::vector(p.second.begin() + i, p.second.begin() + j), + limits * (td::uint32)(j - i)); } } if (entry->pending == 0) { @@ -341,7 +390,7 @@ void OutMsgQueueImporter::get_proof_local(std::shared_ptr entry, Blo td::actor::send_closure( manager_, &ValidatorManager::wait_block_state_short, block, 0, entry->timeout, [=, SelfId = actor_id(this), manager = manager_, timeout = entry->timeout, - retry_after = td::Timestamp::in(0.5)](td::Result> R) mutable { + retry_after = td::Timestamp::in(0.1)](td::Result> R) mutable { if (R.is_error()) { LOG(DEBUG) << "Failed to get block state for " << block.to_str() << ": " << R.move_as_error(); delay_action([=]() { td::actor::send_closure(SelfId, &OutMsgQueueImporter::get_proof_local, entry, block); }, @@ -380,7 +429,7 @@ void OutMsgQueueImporter::get_proof_import(std::shared_ptr entry, st } td::actor::send_closure( manager_, &ValidatorManager::send_get_out_msg_queue_proof_request, entry->dst_shard, blocks, limits, - [=, SelfId = actor_id(this), retry_after = td::Timestamp::in(0.5), + [=, SelfId = actor_id(this), retry_after = td::Timestamp::in(0.1), dst_shard = entry->dst_shard](td::Result>> R) { if (R.is_error()) { LOG(DEBUG) << "Failed to get out msg queue for " << dst_shard.to_str() << ": " << R.move_as_error(); diff --git a/validator/manager.cpp b/validator/manager.cpp index 955334dbc..af5a7c37b 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -478,6 +478,10 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refblock_id(), 0, td::Timestamp::in(60.0), std::move(P)); } + if (collating_masterchain() && desc->generated_at() > td::Clocks::system() - 20) { + wait_neighbor_msg_queue_proofs(ShardIdFull{masterchainId}, {desc->block_id()}, td::Timestamp::in(15.0), + [](td::Result>>) {}); + } } } @@ -2525,6 +2529,16 @@ bool ValidatorManagerImpl::validating_masterchain() { .is_zero(); } +bool ValidatorManagerImpl::collating_masterchain() { + if (masterchain_collators_) { + return true; + } + if (opts_->validator_mode() == ValidatorManagerOptions::validator_lite_all) { + return false; + } + return validating_masterchain(); +} + PublicKeyHash ValidatorManagerImpl::get_validator(ShardIdFull shard, td::Ref val_set) { for (auto &key : temp_keys_) { if (val_set->is_validator(key.bits256_value())) { @@ -2726,7 +2740,12 @@ void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh it = collator_nodes_.emplace(id, Collator()).first; it->second.actor = td::actor::create_actor("collatornode", id, actor_id(this), adnl_, rldp_); } - it->second.shards.insert(shard); + if (!it->second.shards.insert(shard).second) { + return; + } + if (shard.is_masterchain()) { + ++masterchain_collators_; + } td::actor::send_closure(it->second.actor, &CollatorNode::add_shard, shard); } @@ -2735,10 +2754,16 @@ void ValidatorManagerImpl::del_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh if (it == collator_nodes_.end()) { return; } - td::actor::send_closure(it->second.actor, &CollatorNode::del_shard, shard); - it->second.shards.erase(shard); + if (!it->second.shards.erase(shard)) { + return; + } + if (shard.is_masterchain()) { + --masterchain_collators_; + } if (it->second.shards.empty()) { collator_nodes_.erase(it); + } else { + td::actor::send_closure(it->second.actor, &CollatorNode::del_shard, shard); } } diff --git a/validator/manager.hpp b/validator/manager.hpp index ce5254acf..f8ca08f35 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -506,6 +506,7 @@ class ValidatorManagerImpl : public ValidatorManager { bool is_validator(); bool is_collator(); bool validating_masterchain(); + bool collating_masterchain(); PublicKeyHash get_validator(ShardIdFull shard, td::Ref val_set); ValidatorManagerImpl(td::Ref opts, std::string db_root, @@ -650,6 +651,7 @@ class ValidatorManagerImpl : public ValidatorManager { std::set shards; }; std::map collator_nodes_; + size_t masterchain_collators_ = 0; std::set extra_active_shards_; std::map last_validated_blocks_; From 7155bf5ecad50700bc4d50a6eba98ebd879fca34 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Sun, 13 Aug 2023 20:37:59 +0300 Subject: [PATCH 072/388] Fix processing message queue in collator and validator --- validator/impl/collator.cpp | 3 --- validator/impl/validate-query.cpp | 35 +++++++++++++++++++++++-------- validator/impl/validate-query.hpp | 3 ++- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index d2ace04f6..4eada9fd9 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -3820,9 +3820,6 @@ bool Collator::store_master_ref(vm::CellBuilder& cb) { bool Collator::update_processed_upto() { auto ref_mc_seqno = is_masterchain() ? new_block_seqno : prev_mc_block_seqno; update_min_mc_seqno(ref_mc_seqno); - if (in_msg_dict->is_empty()) { - return true; - } if (last_proc_int_msg_.first) { if (!processed_upto_->insert(ref_mc_seqno, last_proc_int_msg_.first, last_proc_int_msg_.second.cbits())) { return fatal_error("cannot update our ProcessedUpto to reflect processed inbound message"); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 8d0b51731..ecce370d3 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -3974,7 +3974,7 @@ bool ValidateQuery::check_processed_upto() { // similar to Collator::process_inbound_message bool ValidateQuery::check_neighbor_outbound_message(Ref enq_msg, ton::LogicalTime lt, td::ConstBitPtr key, const block::McShardDescr& nb, - bool& unprocessed) { + bool& unprocessed, bool& processed_here, td::Bits256& msg_hash) { unprocessed = false; block::EnqueuedMsgDescr enq; if (!enq.unpack(enq_msg.write())) { // unpack EnqueuedMsg @@ -3995,6 +3995,8 @@ bool ValidateQuery::check_neighbor_outbound_message(Ref enq_msg, auto out_entry = out_msg_dict_->lookup(key + 96, 256); bool f0 = ps_.processed_upto_->already_processed(enq); bool f1 = ns_.processed_upto_->already_processed(enq); + processed_here = f1 && !f0; + msg_hash = enq.hash_; if (f0 && !f1) { return fatal_error( "a previously processed message has been un-processed (impossible situation after the validation of " @@ -4074,6 +4076,18 @@ bool ValidateQuery::check_neighbor_outbound_message(Ref enq_msg, } bool ValidateQuery::check_in_queue() { + int imported_messages_count = 0; + in_msg_dict_->check_for_each_extra([&](Ref value, Ref, td::ConstBitPtr, int) { + int tag = block::gen::t_InMsg.get_tag(*value); + if (tag == block::gen::InMsg::msg_import_fin || tag == block::gen::InMsg::msg_import_tr) { + ++imported_messages_count; + } + return true; + }); + if (imported_messages_count == 0 && claimed_proc_lt_ == 0) { + return true; + } + std::vector neighbor_queues; for (const auto& descr : neighbors_) { td::BitArray<96> key; @@ -4082,13 +4096,6 @@ bool ValidateQuery::check_in_queue() { neighbor_queues.emplace_back(descr.top_block_id(), descr.outmsg_root, descr.disabled_); } block::OutputQueueMerger nb_out_msgs(shard_, std::move(neighbor_queues)); - if (in_msg_dict_->is_empty()) { - LOG(DEBUG) << "in_msg_dict is empty, skip checking neighbors' message queues"; - if (processed_upto_updated_) { - return reject_query("processed_upto was updated, but no messages were processed"); - } - return true; - } while (!nb_out_msgs.is_eof()) { auto kv = nb_out_msgs.extract_cur(); CHECK(kv && kv->msg.not_null()); @@ -4099,7 +4106,10 @@ bool ValidateQuery::check_in_queue() { block::gen::t_EnqueuedMsg.print(std::cerr, *(kv->msg)); } bool unprocessed = false; - if (!check_neighbor_outbound_message(kv->msg, kv->lt, kv->key.cbits(), neighbors_.at(kv->source), unprocessed)) { + bool processed_here = false; + td::Bits256 msg_hash; + if (!check_neighbor_outbound_message(kv->msg, kv->lt, kv->key.cbits(), neighbors_.at(kv->source), unprocessed, + processed_here, msg_hash)) { if (verbosity > 1) { std::cerr << "invalid neighbor outbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() << " msg="; @@ -4108,6 +4118,13 @@ bool ValidateQuery::check_in_queue() { return reject_query("error processing outbound internal message "s + kv->key.to_hex() + " of neighbor " + neighbors_.at(kv->source).blk_.to_str()); } + if (processed_here) { + --imported_messages_count; + } + auto msg_lt = kv->lt; + if (imported_messages_count == 0 && msg_lt == claimed_proc_lt_ && msg_hash == claimed_proc_hash_) { + return true; + } if (unprocessed) { return true; } diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index b1d528197..eb052886d 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -340,7 +340,8 @@ class ValidateQuery : public td::actor::Actor { bool check_out_msg_descr(); bool check_processed_upto(); bool check_neighbor_outbound_message(Ref enq_msg, ton::LogicalTime lt, td::ConstBitPtr key, - const block::McShardDescr& src_nb, bool& unprocessed); + const block::McShardDescr& src_nb, bool& unprocessed, bool& processed_here, + td::Bits256& msg_hash); bool check_in_queue(); bool check_delivered_dequeued(); std::unique_ptr make_account_from(td::ConstBitPtr addr, Ref account, From 47c60d8bf01983089eeaa604ef26df70d9f581c5 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 21 Aug 2023 14:33:58 +0300 Subject: [PATCH 073/388] Bugfix in OutMsgQueueImporter --- validator/impl/out-msg-queue-proof.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 3512bf0d6..2ffc52de8 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -371,8 +371,8 @@ void OutMsgQueueImporter::get_neighbor_msg_queue_proofs( }; auto limits = last_masterchain_state_->get_imported_msg_queue_limits(dst_shard.workchain); for (auto& p : new_queries) { - ++entry->pending; for (size_t i = 0; i < p.second.size(); i += 16) { + ++entry->pending; size_t j = std::min(i + 16, p.second.size()); get_proof_import(entry, std::vector(p.second.begin() + i, p.second.begin() + j), limits * (td::uint32)(j - i)); From 1e3a12259b41f51260cb9c3f131aeb18f3a283bc Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 30 Aug 2023 20:14:19 +0300 Subject: [PATCH 074/388] Optimize masterchain collation Use only shard blocks with ready msg queues --- validator/impl/collator.cpp | 23 ++-- validator/impl/out-msg-queue-proof.cpp | 44 -------- validator/impl/out-msg-queue-proof.hpp | 2 +- validator/interfaces/validator-manager.h | 4 +- validator/manager-disk.cpp | 4 +- validator/manager-disk.hpp | 4 +- validator/manager-hardfork.cpp | 4 +- validator/manager-hardfork.hpp | 2 +- validator/manager.cpp | 130 ++++++++++++++++++++--- validator/manager.hpp | 16 ++- 10 files changed, 151 insertions(+), 82 deletions(-) diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 4eada9fd9..d9def6906 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -236,12 +236,12 @@ void Collator::start_up() { } if (is_masterchain() && !is_hardfork_) { // 5. load shard block info messages - LOG(DEBUG) << "sending get_shard_blocks() query to Manager"; + LOG(DEBUG) << "sending get_shard_blocks_for_collator() query to Manager"; ++pending; td::actor::send_closure_later( - manager, &ValidatorManager::get_shard_blocks, prev_blocks[0], + manager, &ValidatorManager::get_shard_blocks_for_collator, prev_blocks[0], [self = get_self()](td::Result>> res) -> void { - LOG(DEBUG) << "got answer to get_shard_blocks() query"; + LOG(DEBUG) << "got answer to get_shard_blocks_for_collator() query"; td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_blocks, std::move(res)); }); } @@ -1405,8 +1405,8 @@ bool Collator::import_new_shard_top_blocks() { prev_descr.clear(); descr.clear(); } else { - LOG(INFO) << "updated top shard block information with " << sh_bd->block_id().to_str() << " and " - << prev_bd->block_id().to_str(); + LOG(DEBUG) << "updated top shard block information with " << sh_bd->block_id().to_str() << " and " + << prev_bd->block_id().to_str(); CHECK(ures.move_as_ok()); store_shard_fees(std::move(prev_descr)); store_shard_fees(std::move(descr)); @@ -1448,7 +1448,7 @@ bool Collator::import_new_shard_top_blocks() { store_shard_fees(std::move(descr)); register_shard_block_creators(sh_bd->get_creator_list(chain_len)); shards_max_end_lt_ = std::max(shards_max_end_lt_, end_lt); - LOG(INFO) << "updated top shard block information with " << sh_bd->block_id().to_str(); + LOG(DEBUG) << "updated top shard block information with " << sh_bd->block_id().to_str(); CHECK(ures.move_as_ok()); ++tb_act; used_shard_block_descr_.emplace_back(sh_bd); @@ -1456,10 +1456,13 @@ bool Collator::import_new_shard_top_blocks() { if (tb_act) { shard_conf_adjusted_ = true; } - if (tb_act && verbosity >= 0) { // DEBUG - LOG(INFO) << "updated shard block configuration to "; - auto csr = shard_conf_->get_root_csr(); - block::gen::t_ShardHashes.print(std::cerr, csr.write()); + if (tb_act) { + LOG(INFO) << "updated shard block configuration: " << tb_act << " new top shard blocks"; + if (verbosity >= 1) { + LOG(INFO) << "updated shard block configuration to "; + auto csr = shard_conf_->get_root_csr(); + block::gen::t_ShardHashes.print(std::cerr, csr.write()); + } } block::gen::ShardFeeCreated::Record fc; if (!(tlb::csr_unpack(fees_import_dict_->get_root_extra(), diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 2ffc52de8..d8e02653f 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -289,50 +289,6 @@ void OutMsgQueueImporter::get_neighbor_msg_queue_proofs( promise.set_value({}); return; } - if (dst_shard.is_masterchain() && blocks.size() != 1) { - // We spit queries for masterchain {dst_shard, {block_1, ..., block_n}} into separate queries - // {dst_shard, {block_1}}, ..., {dst_shard, {block_n}} - class Worker : public td::actor::Actor { - public: - Worker(size_t pending, td::Promise>> promise) - : pending_(pending), promise_(std::move(promise)) { - CHECK(pending_ > 0); - } - - void on_result(td::Ref res) { - result_[res->block_id_] = res; - if (--pending_ == 0) { - promise_.set_result(std::move(result_)); - stop(); - } - } - - void on_error(td::Status error) { - promise_.set_error(std::move(error)); - stop(); - } - - private: - size_t pending_; - td::Promise>> promise_; - std::map> result_; - }; - auto worker = td::actor::create_actor("queueworker", blocks.size(), std::move(promise)).release(); - for (const BlockIdExt& block : blocks) { - get_neighbor_msg_queue_proofs(dst_shard, {block}, timeout, - [=](td::Result>> R) { - if (R.is_error()) { - td::actor::send_closure(worker, &Worker::on_error, R.move_as_error()); - } else { - auto res = R.move_as_ok(); - CHECK(res.size() == 1); - td::actor::send_closure(worker, &Worker::on_result, res.begin()->second); - } - }); - } - return; - } - std::sort(blocks.begin(), blocks.end()); auto entry = cache_[{dst_shard, blocks}]; if (entry) { diff --git a/validator/impl/out-msg-queue-proof.hpp b/validator/impl/out-msg-queue-proof.hpp index 33f6c399e..c115a2763 100644 --- a/validator/impl/out-msg-queue-proof.hpp +++ b/validator/impl/out-msg-queue-proof.hpp @@ -72,7 +72,7 @@ class OutMsgQueueImporter : public td::actor::Actor { void finish_query(std::shared_ptr entry); bool check_timeout(std::shared_ptr entry); - constexpr static const double CACHE_TTL = 30.0; + constexpr static const double CACHE_TTL = 60.0; }; class BuildOutMsgQueueProof : public td::actor::Actor { diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index c09d42937..f63e453dc 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -102,8 +102,8 @@ class ValidatorManager : public ValidatorManagerInterface { td::Promise> promise) = 0; virtual void get_external_messages(ShardIdFull shard, td::Promise>> promise) = 0; virtual void get_ihr_messages(ShardIdFull shard, td::Promise>> promise) = 0; - virtual void get_shard_blocks(BlockIdExt masterchain_block_id, - td::Promise>> promise) = 0; + virtual void get_shard_blocks_for_collator(BlockIdExt masterchain_block_id, + td::Promise>> promise) = 0; virtual void complete_external_messages(std::vector to_delay, std::vector to_delete) = 0; virtual void complete_ihr_messages(std::vector to_delay, diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 5a314e2f2..15954e472 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -516,8 +516,8 @@ void ValidatorManagerImpl::get_ihr_messages(ShardIdFull shard, td::Promise>> promise) { +void ValidatorManagerImpl::get_shard_blocks_for_collator( + BlockIdExt masterchain_block_id, td::Promise>> promise) { if (!last_masterchain_block_handle_) { promise.set_result(std::vector>{}); return; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index bad3fc35f..53c399ed0 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -193,8 +193,8 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void get_external_messages(ShardIdFull shard, td::Promise>> promise) override; void get_ihr_messages(ShardIdFull shard, td::Promise>> promise) override; - void get_shard_blocks(BlockIdExt masterchain_block_id, - td::Promise>> promise) override; + void get_shard_blocks_for_collator(BlockIdExt masterchain_block_id, + td::Promise>> promise) override; void complete_external_messages(std::vector to_delay, std::vector to_delete) override; void complete_ihr_messages(std::vector to_delay, std::vector to_delete) override; diff --git a/validator/manager-hardfork.cpp b/validator/manager-hardfork.cpp index 80a64d25f..d1be057f0 100644 --- a/validator/manager-hardfork.cpp +++ b/validator/manager-hardfork.cpp @@ -366,8 +366,8 @@ void ValidatorManagerImpl::get_ihr_messages(ShardIdFull shard, td::Promise>> promise) { +void ValidatorManagerImpl::get_shard_blocks_for_collator( + BlockIdExt masterchain_block_id, td::Promise>> promise) { } void ValidatorManagerImpl::get_block_data_from_db(ConstBlockHandle handle, td::Promise> promise) { diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 33b6a1a71..be8e08249 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -233,7 +233,7 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void get_external_messages(ShardIdFull shard, td::Promise>> promise) override; void get_ihr_messages(ShardIdFull shard, td::Promise>> promise) override; - void get_shard_blocks(BlockIdExt masterchain_block_id, + void get_shard_blocks_for_collator(BlockIdExt masterchain_block_id, td::Promise>> promise) override; void complete_external_messages(std::vector to_delay, std::vector to_delete) override { diff --git a/validator/manager.cpp b/validator/manager.cpp index af5a7c37b..5b5ad43ec 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -441,7 +441,7 @@ void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc return; } auto it = shard_blocks_.find(ShardTopBlockDescriptionId{block_id.shard_full(), cc_seqno}); - if (it != shard_blocks_.end() && block_id.id.seqno <= it->second->block_id().id.seqno) { + if (it != shard_blocks_.end() && block_id.id.seqno <= it->second.latest_desc->block_id().id.seqno) { VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block broadcast"; return; } @@ -459,11 +459,11 @@ void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc void ValidatorManagerImpl::add_shard_block_description(td::Ref desc) { if (desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { auto it = shard_blocks_.find(ShardTopBlockDescriptionId{desc->shard(), desc->catchain_seqno()}); - if (it != shard_blocks_.end() && desc->block_id().id.seqno <= it->second->block_id().id.seqno) { + if (it != shard_blocks_.end() && desc->block_id().id.seqno <= it->second.latest_desc->block_id().id.seqno) { VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block broadcast"; return; } - shard_blocks_[ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}] = desc; + shard_blocks_[ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}].latest_desc = desc; VLOG(VALIDATOR_DEBUG) << "new shard block descr for " << desc->block_id(); if (need_monitor(desc->block_id().shard_full())) { auto P = td::PromiseCreator::lambda([](td::Result> R) { @@ -478,13 +478,53 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refblock_id(), 0, td::Timestamp::in(60.0), std::move(P)); } - if (collating_masterchain() && desc->generated_at() > td::Clocks::system() - 20) { - wait_neighbor_msg_queue_proofs(ShardIdFull{masterchainId}, {desc->block_id()}, td::Timestamp::in(15.0), - [](td::Result>>) {}); + if (collating_masterchain()) { + preload_msg_queue_to_masterchain(desc); } } } +void ValidatorManagerImpl::preload_msg_queue_to_masterchain(td::Ref desc) { + auto id = ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}; + auto it = shard_blocks_.find(id); + if (!collating_masterchain() || it == shard_blocks_.end() || it->second.latest_desc->block_id() != desc->block_id()) { + return; + } + wait_neighbor_msg_queue_proofs( + ShardIdFull{masterchainId}, {desc->block_id()}, td::Timestamp::in(10.0), + [=, SelfId = actor_id(this), + retry_at = td::Timestamp::in(1.0)](td::Result>> R) { + if (R.is_error()) { + delay_action( + [=]() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::preload_msg_queue_to_masterchain, desc); }, + retry_at); + return; + } + auto res = R.move_as_ok(); + auto &queue = res[desc->block_id()]; + CHECK(queue.not_null()); + td::actor::send_closure(SelfId, &ValidatorManagerImpl::loaded_msg_queue_to_masterchain, desc, std::move(queue)); + }); +} + +void ValidatorManagerImpl::loaded_msg_queue_to_masterchain(td::Ref desc, + td::Ref res) { + auto id = ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}; + auto it = shard_blocks_.find(id); + if (it == shard_blocks_.end()) { + return; + } + auto &info = it->second; + if (info.ready_desc.is_null() || info.ready_desc->block_id().seqno() < desc->block_id().seqno()) { + VLOG(VALIDATOR_DEBUG) << "loaded out msg queue to masterchain from " << desc->block_id(); + if (info.ready_desc.not_null()) { + cached_msg_queue_to_masterchain_.erase(info.ready_desc->block_id()); + } + info.ready_desc = desc; + cached_msg_queue_to_masterchain_[desc->block_id()] = std::move(res); + } +} + void ValidatorManagerImpl::add_ext_server_id(adnl::AdnlNodeIdShort id) { class Cb : public adnl::Adnl::Callback { private: @@ -629,6 +669,56 @@ void ValidatorManagerImpl::wait_neighbor_msg_queue_proofs( out_msg_queue_importer_ = td::actor::create_actor("outmsgqueueimporter", actor_id(this), opts_, last_masterchain_state_); } + if (dst_shard.is_masterchain()) { + // We spit queries for masterchain {dst_shard, {block_1, ..., block_n}} into separate queries + // {dst_shard, {block_1}}, ..., {dst_shard, {block_n}} + // Also, use cache + class Worker : public td::actor::Actor { + public: + Worker(size_t pending, td::Promise>> promise) + : pending_(pending), promise_(std::move(promise)) { + CHECK(pending_ > 0); + } + + void on_result(td::Ref res) { + result_[res->block_id_] = res; + if (--pending_ == 0) { + promise_.set_result(std::move(result_)); + stop(); + } + } + + void on_error(td::Status error) { + promise_.set_error(std::move(error)); + stop(); + } + + private: + size_t pending_; + td::Promise>> promise_; + std::map> result_; + }; + auto worker = td::actor::create_actor("queueworker", blocks.size(), std::move(promise)).release(); + for (const BlockIdExt &block : blocks) { + auto it = cached_msg_queue_to_masterchain_.find(block); + if (it != cached_msg_queue_to_masterchain_.end()) { + td::actor::send_closure(worker, &Worker::on_result, it->second); + continue; + } + td::actor::send_closure(out_msg_queue_importer_, &OutMsgQueueImporter::get_neighbor_msg_queue_proofs, dst_shard, + std::vector{1, block}, timeout, + [=](td::Result>> R) { + if (R.is_error()) { + td::actor::send_closure(worker, &Worker::on_error, R.move_as_error()); + } else { + auto res = R.move_as_ok(); + CHECK(res.size() == 1); + td::actor::send_closure(worker, &Worker::on_result, res.begin()->second); + } + }); + } + return; + } td::actor::send_closure(out_msg_queue_importer_, &OutMsgQueueImporter::get_neighbor_msg_queue_proofs, dst_shard, std::move(blocks), timeout, std::move(promise)); } @@ -843,11 +933,14 @@ void ValidatorManagerImpl::get_ihr_messages(ShardIdFull shard, td::Promise>> promise) { +void ValidatorManagerImpl::get_shard_blocks_for_collator( + BlockIdExt masterchain_block_id, td::Promise>> promise) { std::vector> v; for (auto &b : shard_blocks_) { - v.push_back(b.second); + if (b.second.ready_desc.is_null()) { + continue; + } + v.push_back(b.second.ready_desc); } promise.set_value(std::move(v)); } @@ -1786,7 +1879,7 @@ void ValidatorManagerImpl::new_masterchain_block() { } if (is_collator()) { std::set collating_shards; - if (validating_masterchain()) { + if (collating_masterchain()) { collating_shards.emplace(masterchainId); } for (const auto &collator : collator_nodes_) { @@ -2080,12 +2173,19 @@ void ValidatorManagerImpl::update_shard_blocks() { auto it = shard_blocks_.begin(); while (it != shard_blocks_.end()) { auto &B = it->second; - if (!B->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { - auto it2 = it++; - shard_blocks_.erase(it2); - } else { - ++it; + if (!B.latest_desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { + if (B.ready_desc.not_null()) { + cached_msg_queue_to_masterchain_.erase(B.ready_desc->block_id()); + } + it = shard_blocks_.erase(it); + continue; } + if (B.ready_desc.not_null() && + !B.ready_desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { + cached_msg_queue_to_masterchain_.erase(B.ready_desc->block_id()); + B.ready_desc = {}; + } + ++it; } } diff --git a/validator/manager.hpp b/validator/manager.hpp index f8ca08f35..45a0c1e63 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -220,7 +220,15 @@ class ValidatorManagerImpl : public ValidatorManager { } }; // DATA FOR COLLATOR - std::map> shard_blocks_; + // Shard block will not be used until queue is ready (to avoid too long masterchain collation) + // latest_desc - latest known block + // ready_desc - block with ready msg queue (may be null) + struct ShardTopBlock { + td::Ref latest_desc; + td::Ref ready_desc; + }; + std::map shard_blocks_; + std::map> cached_msg_queue_to_masterchain_; std::map, std::unique_ptr>> ext_messages_; std::map, std::map>> ext_addr_messages_; std::map> ext_messages_hashes_; @@ -410,8 +418,8 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void get_external_messages(ShardIdFull shard, td::Promise>> promise) override; void get_ihr_messages(ShardIdFull shard, td::Promise>> promise) override; - void get_shard_blocks(BlockIdExt masterchain_block_id, - td::Promise>> promise) override; + void get_shard_blocks_for_collator(BlockIdExt masterchain_block_id, + td::Promise>> promise) override; void complete_external_messages(std::vector to_delay, std::vector to_delete) override; void complete_ihr_messages(std::vector to_delay, std::vector to_delete) override; @@ -493,6 +501,8 @@ class ValidatorManagerImpl : public ValidatorManager { } void add_shard_block_description(td::Ref desc); + void preload_msg_queue_to_masterchain(td::Ref desc); + void loaded_msg_queue_to_masterchain(td::Ref desc, td::Ref res); void register_block_handle(BlockHandle handle); From 98fe1f886d1fdd6780ddb5a8b9f2d7061b275af5 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 31 Aug 2023 11:57:02 +0300 Subject: [PATCH 075/388] Remove excessive logs in collator --- validator/impl/collator.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index d9def6906..7cf14eec9 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -1458,7 +1458,7 @@ bool Collator::import_new_shard_top_blocks() { } if (tb_act) { LOG(INFO) << "updated shard block configuration: " << tb_act << " new top shard blocks"; - if (verbosity >= 1) { + if (verbosity >= 3) { LOG(INFO) << "updated shard block configuration to "; auto csr = shard_conf_->get_root_csr(); block::gen::t_ShardHashes.print(std::cerr, csr.write()); @@ -3336,7 +3336,7 @@ bool Collator::create_mc_state_extra() { } // 3. save new shard_hashes state_extra.shard_hashes = shard_conf_->get_root_csr(); - if (verbosity >= 3 * 0) { // DEBUG + if (verbosity >= 3) { std::cerr << "updated shard configuration to "; block::gen::t_ShardHashes.print(std::cerr, *state_extra.shard_hashes); } @@ -3734,7 +3734,7 @@ bool Collator::update_public_libraries() { } } } - if (libraries_changed_ && verbosity >= 2 * 0) { + if (libraries_changed_ && verbosity >= 2) { std::cerr << "New public libraries: "; block::gen::t_HashmapE_256_LibDescr.print(std::cerr, shard_libraries_->get_root()); shard_libraries_->get_root()->print_rec(std::cerr); From 4d5b79224dd6f616923cf747373f19751009d8a2 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 21 Sep 2023 15:14:34 +0300 Subject: [PATCH 076/388] Proxy liteserver --- tl/generate/scheme/ton_api.tl | 3 + tl/generate/scheme/ton_api.tlo | Bin 91588 -> 91764 bytes utils/CMakeLists.txt | 3 + utils/proxy-liteserver.cpp | 323 +++++++++++++++++++++++++++++++++ 4 files changed, 329 insertions(+) create mode 100644 utils/proxy-liteserver.cpp diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 6d64b692b..50e0aabbc 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -931,3 +931,6 @@ storage.daemon.withdraw contract:string = storage.daemon.Success; storage.daemon.sendCoins address:string amount:string message:string = storage.daemon.Success; storage.daemon.closeStorageContract address:string = storage.daemon.Success; storage.daemon.removeStorageProvider = storage.daemon.Success; + +---types--- +proxyLiteserver.config port:int id:PublicKey = proxyLiteserver.Config; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 9ecd272dc182524dcfa66469b1b53029f87bafa0..4ff97e05b6900f7621443a2f1d5ccd95166714a1 100644 GIT binary patch delta 125 zcmX?dn)S;WR^CUm^{p77fNvx3Wd)JGR6_%?f};G2N}tS<)Z)~lvecr<2|Nm$A1L^U zFizgAYwA2f6`LZx|XoB(pi?Xu<*j DjteYF delta 41 scmexzhV{s4R^CUm^{p77fOjMBWrfWmiasKYlQt)sI#1ZFa4cj202#Xtj{pDw diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 3d029c938..67c48851f 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -25,4 +25,7 @@ target_link_libraries(opcode-timing ton_crypto) target_include_directories(pack-viewer PUBLIC $/..) +add_executable(proxy-liteserver proxy-liteserver.cpp) +target_link_libraries(proxy-liteserver tdutils tdactor adnl dht tl_api ton_crypto git lite-client-common) + install(TARGETS generate-random-id RUNTIME DESTINATION bin) diff --git a/utils/proxy-liteserver.cpp b/utils/proxy-liteserver.cpp new file mode 100644 index 000000000..f7da39640 --- /dev/null +++ b/utils/proxy-liteserver.cpp @@ -0,0 +1,323 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#include "td/utils/filesystem.h" +#include "td/actor/actor.h" +#include "td/actor/MultiPromise.h" +#include "td/utils/OptionParser.h" +#include "td/utils/port/path.h" +#include "td/utils/port/signals.h" +#include "td/utils/port/user.h" +#include "td/utils/port/IPAddress.h" +#include "td/utils/Random.h" +#include "td/utils/FileLog.h" +#include "git.h" +#include "auto/tl/ton_api.h" +#include "auto/tl/lite_api.h" +#include "auto/tl/lite_api.hpp" +#include "tl-utils/lite-utils.hpp" +#include "ton/lite-tl.hpp" +#include "auto/tl/ton_api_json.h" +#include "adnl/adnl.h" +#include "lite-client/QueryTraits.h" +#include "lite-client/ext-client.h" + +#if TD_DARWIN || TD_LINUX +#include +#endif +#include + +using namespace ton; + +class ProxyLiteserver : public td::actor::Actor { + public: + ProxyLiteserver(std::string global_config, std::string db_root, td::uint16 port) + : global_config_(std::move(global_config)), db_root_(std::move(db_root)), port_(port) { + } + + void start_up() override { + LOG_CHECK(db_root_ != "") << "db root is not set"; + td::mkdir(db_root_).ensure(); + db_root_ = td::realpath(db_root_).move_as_ok(); + keyring_ = keyring::Keyring::create(db_root_ + "/keyring"); + + td::Status S = prepare_local_config(); + if (S.is_error()) { + LOG(FATAL) << "Local config error: " << S; + } + + S = create_ext_client(); + if (S.is_error()) { + LOG(FATAL) << S; + } + + create_ext_server(); + } + + td::Status prepare_local_config() { + auto r_conf_data = td::read_file(config_file()); + if (r_conf_data.is_ok()) { + auto conf_data = r_conf_data.move_as_ok(); + TRY_RESULT_PREFIX(conf_json, td::json_decode(conf_data.as_slice()), "failed to parse json: "); + TRY_STATUS_PREFIX(ton_api::from_json(*config_, conf_json.get_object()), "json does not fit TL scheme: "); + TRY_RESULT_PREFIX_ASSIGN(port_, td::narrow_cast_safe(config_->port_), "invalid port: "); + TRY_RESULT_PREFIX_ASSIGN(id_, adnl::AdnlNodeIdFull::create(config_->id_), "invalid id: "); + } else { + LOG(WARNING) << "First launch, creating local config"; + if (port_ == 0) { + return td::Status::Error("port is not set"); + } + config_->port_ = port_; + auto pk = PrivateKey{privkeys::Ed25519::random()}; + id_ = adnl::AdnlNodeIdFull{pk.compute_public_key()}; + config_->id_ = id_.tl(); + td::actor::send_closure(keyring_, &keyring::Keyring::add_key, std::move(pk), false, [](td::Result R) { + if (R.is_error()) { + LOG(FATAL) << "Failed to store private key"; + } + }); + + auto s = td::json_encode(td::ToJson(*config_), true); + TRY_STATUS_PREFIX(td::write_file(config_file(), s), "failed to write file: "); + } + return td::Status::OK(); + } + + td::Status create_ext_client() { + std::vector servers; + TRY_RESULT_PREFIX(global_config_data, td::read_file(global_config_), "Failed to read global config: "); + TRY_RESULT_PREFIX(global_config_json, td::json_decode(global_config_data.as_slice()), + "Failed to parse global config: "); + ton::ton_api::liteclient_config_global gc; + ton::ton_api::from_json(gc, global_config_json.get_object()).ensure(); + + size_t size = gc.liteservers_.size() + gc.liteservers_v2_.size(); + if (size == 0) { + return td::Status::Error("No liteservers in global config"); + } + + for (auto& s : gc.liteservers_) { + td::IPAddress addr; + addr.init_host_port(td::IPAddress::ipv4_to_str(s->ip_), s->port_).ensure(); + liteclient::ExtClient::LiteServer serv; + serv.address = addr; + serv.adnl_id = ton::adnl::AdnlNodeIdFull::create(s->id_).move_as_ok(); + servers.push_back(std::move(serv)); + } + for (auto& s : gc.liteservers_v2_) { + td::IPAddress addr; + addr.init_host_port(td::IPAddress::ipv4_to_str(s->ip_), s->port_).ensure(); + liteclient::ExtClient::LiteServer serv; + serv.address = addr; + serv.adnl_id = ton::adnl::AdnlNodeIdFull::create(s->id_).move_as_ok(); + serv.is_full = false; + for (auto& shard : s->shards_) { + serv.shards.emplace_back(shard->workchain_, (ton::ShardId)shard->shard_); + CHECK(serv.shards.back().is_valid_ext()); + } + servers.push_back(std::move(serv)); + } + class Callback : public liteclient::ExtClient::Callback {}; + ext_client_ = liteclient::ExtClient::create(std::move(servers), td::make_unique()); + return td::Status::OK(); + } + + void create_ext_server() { + adnl_ = adnl::Adnl::create("", keyring_.get()); + td::actor::send_closure(adnl_, &adnl::Adnl::add_id, id_, ton::adnl::AdnlAddressList{}, (td::uint8)255); + td::actor::send_closure(adnl_, &adnl::Adnl::create_ext_server, + std::vector{id_.compute_short_id()}, std::vector{port_}, + [SelfId = actor_id(this)](td::Result> R) { + R.ensure(); + td::actor::send_closure(SelfId, &ProxyLiteserver::created_ext_server, R.move_as_ok()); + }); + } + + void created_ext_server(td::actor::ActorOwn s) { + ext_server_ = std::move(s); + LOG(WARNING) << "Started proxy liteserver on port " << port_; + + class AdnlCallback : public adnl::Adnl::Callback { + public: + AdnlCallback(td::actor::ActorId client) : client_(client) { + } + + void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { + } + void receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + td::actor::create_actor("worker", client_, std::move(data), std::move(promise)).release(); + } + + private: + td::actor::ActorId client_; + }; + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, id_.compute_short_id(), + adnl::Adnl::int_to_bytestring(lite_api::liteServer_query::ID), + std::make_unique(ext_client_.get())); + } + + class QueryWorker : public td::actor::Actor { + public: + QueryWorker(td::actor::ActorId client, td::BufferSlice data, + td::Promise promise) + : client_(std::move(client)), data_(std::move(data)), promise_(std::move(promise)) { + } + + void start_up() override { + auto data = data_.clone(); + auto F = fetch_tl_object(data, true); + if (F.is_ok()) { + data = std::move(F.move_as_ok()->data_); + } else { + auto G = fetch_tl_prefix(data, true); + if (G.is_error()) { + fatal_error(G.move_as_error()); + } + } + fetch_tl_prefix(data, true).ignore(); + auto F2 = fetch_tl_object(std::move(data), true); + if (F2.is_error()) { + fatal_error(F2.move_as_error()); + } + auto query = F2.move_as_ok(); + lite_api::downcast_call(*query, [&](auto& obj) { shard_ = liteclient::get_query_shard(obj); }); + + LOG(INFO) << "Got query: shard=" << shard_.to_str() << " size=" << data_.size(); + td::actor::send_closure(client_, &liteclient::ExtClient::send_query, "q", std::move(data_), shard_, + td::Timestamp::in(8.0), [SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(SelfId, &QueryWorker::got_result, std::move(R)); + }); + } + + void got_result(td::Result R) { + if (R.is_error()) { + LOG(INFO) << "Query to shard=" << shard_.to_str() << ": " << R.error(); + promise_.set_value(create_serialize_tl_object( + R.error().code(), "gateway error: " + R.error().message().str())); + } else { + td::BufferSlice response = R.move_as_ok(); + LOG(INFO) << "Query to shard=" << shard_.to_str() << ": OK, size=" << response.size() + << " time=" << timer_.elapsed(); + promise_.set_value(std::move(response)); + } + stop(); + } + + void fatal_error(td::Status S) { + promise_.set_error(std::move(S)); + stop(); + } + + private: + td::actor::ActorId client_; + td::BufferSlice data_; + td::Promise promise_; + td::Timer timer_ = {}; + ShardIdFull shard_; + }; + + private: + std::string global_config_; + std::string db_root_; + td::uint16 port_; + + tl_object_ptr config_ = create_tl_object(); + adnl::AdnlNodeIdFull id_; + + td::actor::ActorOwn keyring_; + td::actor::ActorOwn adnl_; + td::actor::ActorOwn ext_server_; + td::actor::ActorOwn ext_client_; + + std::string config_file() const { + return db_root_ + "/config.json"; + } +}; + +int main(int argc, char* argv[]) { + SET_VERBOSITY_LEVEL(verbosity_WARNING); + td::set_default_failure_signal_handler().ensure(); + + td::unique_ptr logger_; + SCOPE_EXIT { + td::log_interface = td::default_log_interface; + }; + + std::string global_config, db_root; + td::uint16 port = 0; + td::uint32 threads = 4; + + td::OptionParser p; + p.set_description("Proxy liteserver: distributes incoming queries to servers in global config\n"); + p.add_option('v', "verbosity", "set verbosity level", [&](td::Slice arg) { + int v = VERBOSITY_NAME(FATAL) + (td::to_integer(arg)); + SET_VERBOSITY_LEVEL(v); + }); + p.add_option('V', "version", "show build information", [&]() { + std::cout << "proxy-liteserver build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::exit(0); + }); + p.add_option('h', "help", "print help", [&]() { + char b[10240]; + td::StringBuilder sb(td::MutableSlice{b, 10000}); + sb << p; + std::cout << sb.as_cslice().c_str(); + std::exit(2); + }); + p.add_checked_option('p', "port", "liteserver port (use only on first launch)", [&](td::Slice arg) -> td::Status { + TRY_RESULT_ASSIGN(port, td::to_integer_safe(arg)); + return td::Status::OK(); + }); + p.add_option('C', "global-config", "global TON configuration file", + [&](td::Slice arg) { global_config = arg.str(); }); + p.add_option('D', "db", "db root", [&](td::Slice arg) { db_root = arg.str(); }); + p.add_option('d', "daemonize", "set SIGHUP", [&]() { + td::set_signal_handler(td::SignalType::HangUp, [](int sig) { +#if TD_DARWIN || TD_LINUX + close(0); + setsid(); +#endif + }).ensure(); + }); + p.add_option('l', "logname", "log to file", [&](td::Slice fname) { + logger_ = td::FileLog::create(fname.str()).move_as_ok(); + td::log_interface = logger_.get(); + }); + p.add_checked_option('t', "threads", PSTRING() << "number of threads (default=" << 4 << ")", + [&](td::Slice arg) -> td::Status { + TRY_RESULT_ASSIGN(threads, td::to_integer_safe(arg)); + return td::Status::OK(); + }); + + p.run(argc, argv).ensure(); + td::actor::Scheduler scheduler({threads}); + + scheduler.run_in_context( + [&] { td::actor::create_actor("proxy-liteserver", global_config, db_root, port).release(); }); + while (scheduler.run(1)) { + } +} From a198ca7fe3567596a0b8d8b0baffe779d9690994 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 10 Oct 2023 23:52:54 +0300 Subject: [PATCH 077/388] Bugfix in proxy-liteserver --- utils/proxy-liteserver.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/proxy-liteserver.cpp b/utils/proxy-liteserver.cpp index f7da39640..7abaa1c7a 100644 --- a/utils/proxy-liteserver.cpp +++ b/utils/proxy-liteserver.cpp @@ -195,12 +195,14 @@ class ProxyLiteserver : public td::actor::Actor { auto G = fetch_tl_prefix(data, true); if (G.is_error()) { fatal_error(G.move_as_error()); + return; } } fetch_tl_prefix(data, true).ignore(); auto F2 = fetch_tl_object(std::move(data), true); if (F2.is_error()) { fatal_error(F2.move_as_error()); + return; } auto query = F2.move_as_ok(); lite_api::downcast_call(*query, [&](auto& obj) { shard_ = liteclient::get_query_shard(obj); }); From 1eef6ed55ef812da1ee7828373f8c062f1fc7b4a Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 16 Oct 2023 11:22:41 +0300 Subject: [PATCH 078/388] Bugfix in check_neighbor_outbound_message --- validator/impl/validate-query.cpp | 33 +++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index ecce370d3..49885a5f8 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -4010,8 +4010,37 @@ bool ValidateQuery::check_neighbor_outbound_message(Ref enq_msg, key.to_hex(352) + " of neighbor " + nb.blk_.to_str()); } if (shard_contains(shard_, enq.cur_prefix_)) { - // this message couldn't come from our own outbound queue because processed messages from our queue don't stay here - return fatal_error("have an already processed EnqueuedMsg from our shard: "s + key.to_hex(352)); + // if this message comes from our own outbound queue, we must have dequeued it + if (out_entry.is_null()) { + return reject_query("our old outbound queue contains EnqueuedMsg with key "s + key.to_hex(352) + + " already processed by this shard, but there is no ext_message_deq OutMsg record for this " + "message in this block"); + } + int tag = block::gen::t_OutMsg.get_tag(*out_entry); + if (tag == block::gen::OutMsg::msg_export_deq_short) { + block::gen::OutMsg::Record_msg_export_deq_short deq; + if (!tlb::csr_unpack(std::move(out_entry), deq)) { + return reject_query( + "cannot unpack msg_export_deq_short OutMsg record for already processed EnqueuedMsg with key "s + + key.to_hex(352) + " of old outbound queue"); + } + if (deq.msg_env_hash != enq.msg_env_->get_hash().bits()) { + return reject_query("unpack ext_message_deq OutMsg record for already processed EnqueuedMsg with key "s + + key.to_hex(352) + " of old outbound queue refers to MsgEnvelope with different hash " + + deq.msg_env_hash.to_hex()); + } + } else { + block::gen::OutMsg::Record_msg_export_deq deq; + if (!tlb::csr_unpack(std::move(out_entry), deq)) { + return reject_query( + "cannot unpack msg_export_deq OutMsg record for already processed EnqueuedMsg with key "s + + key.to_hex(352) + " of old outbound queue"); + } + if (deq.out_msg->get_hash() != enq.msg_env_->get_hash()) { + return reject_query("unpack ext_message_deq OutMsg record for already processed EnqueuedMsg with key "s + + key.to_hex(352) + " of old outbound queue contains a different MsgEnvelope"); + } + } } // next check is incorrect after a merge, when ns_.processed_upto has > 1 entries // we effectively comment it out From 3c55abf2d21cc8cd67021bcca4920948862d68f1 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 17 Oct 2023 16:07:44 +0300 Subject: [PATCH 079/388] Add proxy-liteserver to github builds --- .github/workflows/macos-11.7-compile.yml | 3 ++- .github/workflows/macos-12.6-compile.yml | 3 ++- .github/workflows/ubuntu-22.04-compile.yml | 4 ++-- .github/workflows/ubuntu-compile.yml | 4 ++-- .github/workflows/win-2019-compile.yml | 4 ++-- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/macos-11.7-compile.yml b/.github/workflows/macos-11.7-compile.yml index 910b16fc1..d96cdb7b9 100644 --- a/.github/workflows/macos-11.7-compile.yml +++ b/.github/workflows/macos-11.7-compile.yml @@ -28,7 +28,7 @@ jobs: mkdir build cd build cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=11.7 -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_BUILD_TYPE=Release .. - ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator + ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver - name: Find & copy binaries run: | @@ -52,6 +52,7 @@ jobs: cp build/utils/json2tlo artifacts/ cp build/adnl/adnl-proxy artifacts/ cp build/emulator/*emulator.* artifacts/ + cp build/utils/proxy-liteserver artifacts/ chmod +x artifacts/* rsync -r crypto/smartcont artifacts/ rsync -r crypto/fift/lib artifacts/ diff --git a/.github/workflows/macos-12.6-compile.yml b/.github/workflows/macos-12.6-compile.yml index 368abf02c..75f355595 100644 --- a/.github/workflows/macos-12.6-compile.yml +++ b/.github/workflows/macos-12.6-compile.yml @@ -39,7 +39,7 @@ jobs: mkdir build cd build cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=12.6 -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_BUILD_TYPE=Release .. - ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator + ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver - name: Find & copy binaries run: | @@ -63,6 +63,7 @@ jobs: cp build/utils/json2tlo artifacts/ cp build/adnl/adnl-proxy artifacts/ cp build/emulator/*emulator.* artifacts/ + cp build/utils/proxy-liteserver artifacts/ chmod +x artifacts/* rsync -r crypto/smartcont artifacts/ rsync -r crypto/fift/lib artifacts/ diff --git a/.github/workflows/ubuntu-22.04-compile.yml b/.github/workflows/ubuntu-22.04-compile.yml index 397116ff6..fc13701e5 100644 --- a/.github/workflows/ubuntu-22.04-compile.yml +++ b/.github/workflows/ubuntu-22.04-compile.yml @@ -40,12 +40,12 @@ jobs: cd build cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" .. - ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state emulator + ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state emulator proxy-liteserver - name: Find & copy binaries run: | mkdir artifacts - cp build/storage/storage-daemon/storage-daemon build/storage/storage-daemon/storage-daemon-cli build/crypto/fift build/crypto/tlbc build/crypto/func build/crypto/create-state build/validator-engine-console/validator-engine-console build/tonlib/tonlib-cli build/tonlib/libtonlibjson.so.0.5 build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.* artifacts + cp build/storage/storage-daemon/storage-daemon build/storage/storage-daemon/storage-daemon-cli build/crypto/fift build/crypto/tlbc build/crypto/func build/crypto/create-state build/validator-engine-console/validator-engine-console build/tonlib/tonlib-cli build/tonlib/libtonlibjson.so.0.5 build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.* build/utils/proxy-liteserver artifacts chmod +x artifacts/* cp -R crypto/smartcont artifacts/ cp -R crypto/fift/lib artifacts/ diff --git a/.github/workflows/ubuntu-compile.yml b/.github/workflows/ubuntu-compile.yml index 171ba46d8..6907606d4 100644 --- a/.github/workflows/ubuntu-compile.yml +++ b/.github/workflows/ubuntu-compile.yml @@ -44,12 +44,12 @@ jobs: buildPath=`pwd` cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$buildPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$buildPath/openssl_1_1_1/libcrypto.a -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" .. - ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator + ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator proxy-liteserver - name: Find & copy binaries run: | mkdir artifacts-${{ matrix.os }} - cp build-${{ matrix.os }}/storage/storage-daemon/storage-daemon build-${{ matrix.os }}/storage/storage-daemon/storage-daemon-cli build-${{ matrix.os }}/crypto/fift build-${{ matrix.os }}/crypto/tlbc build-${{ matrix.os }}/crypto/func build-${{ matrix.os }}/crypto/create-state build-${{ matrix.os }}/validator-engine-console/validator-engine-console build-${{ matrix.os }}/tonlib/tonlib-cli build-${{ matrix.os }}/tonlib/libtonlibjson.so.0.5 build-${{ matrix.os }}/http/http-proxy build-${{ matrix.os }}/rldp-http-proxy/rldp-http-proxy build-${{ matrix.os }}/dht-server/dht-server build-${{ matrix.os }}/lite-client/lite-client build-${{ matrix.os }}/validator-engine/validator-engine build-${{ matrix.os }}/utils/generate-random-id build-${{ matrix.os }}/utils/json2tlo build-${{ matrix.os }}/adnl/adnl-proxy build-${{ matrix.os }}/emulator/libemulator.* artifacts-${{ matrix.os }} + cp build-${{ matrix.os }}/storage/storage-daemon/storage-daemon build-${{ matrix.os }}/storage/storage-daemon/storage-daemon-cli build-${{ matrix.os }}/crypto/fift build-${{ matrix.os }}/crypto/tlbc build-${{ matrix.os }}/crypto/func build-${{ matrix.os }}/crypto/create-state build-${{ matrix.os }}/validator-engine-console/validator-engine-console build-${{ matrix.os }}/tonlib/tonlib-cli build-${{ matrix.os }}/tonlib/libtonlibjson.so.0.5 build-${{ matrix.os }}/http/http-proxy build-${{ matrix.os }}/rldp-http-proxy/rldp-http-proxy build-${{ matrix.os }}/dht-server/dht-server build-${{ matrix.os }}/lite-client/lite-client build-${{ matrix.os }}/validator-engine/validator-engine build-${{ matrix.os }}/utils/generate-random-id build-${{ matrix.os }}/utils/json2tlo build-${{ matrix.os }}/adnl/adnl-proxy build-${{ matrix.os }}/emulator/libemulator.* build-${{ matrix.os }}/utils/proxy-liteserver artifacts-${{ matrix.os }} chmod +x artifacts-${{ matrix.os }}/* cp -R crypto/smartcont artifacts-${{ matrix.os }} cp -R crypto/fift/lib artifacts-${{ matrix.os }} diff --git a/.github/workflows/win-2019-compile.yml b/.github/workflows/win-2019-compile.yml index 1edcb0e16..62a560d2a 100644 --- a/.github/workflows/win-2019-compile.yml +++ b/.github/workflows/win-2019-compile.yml @@ -73,7 +73,7 @@ jobs: mkdir build cd build cmake -DCMAKE_BUILD_TYPE=Release -DSODIUM_USE_STATIC_LIBS=1 -DSECP256K1_INCLUDE_DIR=%root%\secp256k1\include -DSECP256K1_LIBRARY=%root%\secp256k1\bin\x64\Release\v142\static\secp256k1.lib -DREADLINE_INCLUDE_DIR=%root%\readline-5.0-1-lib\include\readline -DREADLINE_LIBRARY=%root%\readline-5.0-1-lib\lib\readline.lib -DPORTABLE=1 -DZLIB_FOUND=1 -DMHD_FOUND=1 -DMHD_LIBRARY=%root%\libmicrohttpd-0.9.77-w32-bin\x86_64\VS2019\Release-static\libmicrohttpd.lib -DMHD_INCLUDE_DIR=%root%\libmicrohttpd-0.9.77-w32-bin\x86_64\VS2019\Release-static -DZLIB_INCLUDE_DIR=%root%\zlib -DZLIB_LIBRARY=%root%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=%root%/openssl-1.1.1j/include -DOPENSSL_CRYPTO_LIBRARY=%root%/openssl-1.1.1j/lib/libcrypto_static.lib -DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj /W0" .. - cmake --build . --target storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator --config Release + cmake --build . --target storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator proxy-liteserver --config Release - name: Show executables run: | @@ -92,7 +92,7 @@ jobs: mkdir artifacts\smartcont mkdir artifacts\lib - for %%I in (build\storage\storage-daemon\Release\storage-daemon.exe build\storage\storage-daemon\Release\storage-daemon-cli.exe build\blockchain-explorer\blockchain-explorer.exe build\crypto\Release\fift.exe build\crypto\Release\tlbc.exe build\crypto\Release\func.exe build\crypto\Release\create-state.exe build\validator-engine-console\Release\validator-engine-console.exe build\tonlib\Release\tonlib-cli.exe build\tonlib\Release\tonlibjson.dll build\http\Release\http-proxy.exe build\rldp-http-proxy\Release\rldp-http-proxy.exe build\dht-server\Release\dht-server.exe build\lite-client\Release\lite-client.exe build\validator-engine\Release\validator-engine.exe build\utils\Release\generate-random-id.exe build\utils\Release\json2tlo.exe build\adnl\Release\adnl-proxy.exe build\emulator\Release\emulator.dll) do copy %%I artifacts\ + for %%I in (build\storage\storage-daemon\Release\storage-daemon.exe build\storage\storage-daemon\Release\storage-daemon-cli.exe build\blockchain-explorer\blockchain-explorer.exe build\crypto\Release\fift.exe build\crypto\Release\tlbc.exe build\crypto\Release\func.exe build\crypto\Release\create-state.exe build\validator-engine-console\Release\validator-engine-console.exe build\tonlib\Release\tonlib-cli.exe build\tonlib\Release\tonlibjson.dll build\http\Release\http-proxy.exe build\rldp-http-proxy\Release\rldp-http-proxy.exe build\dht-server\Release\dht-server.exe build\lite-client\Release\lite-client.exe build\validator-engine\Release\validator-engine.exe build\utils\Release\generate-random-id.exe build\utils\Release\json2tlo.exe build\adnl\Release\adnl-proxy.exe build\emulator\Release\emulator.dll build\utils\proxy-liteserver.exe) do copy %%I artifacts\ xcopy /e /k /h /i crypto\smartcont artifacts\smartcont xcopy /e /k /h /i crypto\fift\lib artifacts\lib From 747b24aa18c3214a4072fe0e09a19917946745ce Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 19 Feb 2024 13:10:47 +0300 Subject: [PATCH 080/388] Fix QueueSizeCounter and ValidatorGroup after merge --- validator/full-node.cpp | 4 ++-- validator/manager-disk.hpp | 4 ++-- validator/manager-hardfork.hpp | 4 ++-- validator/manager.cpp | 3 +++ validator/manager.hpp | 16 ++++++++++------ validator/queue-size-counter.cpp | 4 +++- validator/queue-size-counter.hpp | 10 ++++++++-- validator/validator-group.cpp | 2 +- 8 files changed, 31 insertions(+), 16 deletions(-) diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 7ee8f1c58..d67963378 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -654,7 +654,7 @@ void FullNodeImpl::update_private_block_overlays() { } void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { - CHECK(local_keys_.count(key)); + /*CHECK(local_keys_.count(key)); if (current_validators_.count(key)) { std::vector nodes; for (const auto &p : current_validators_) { @@ -663,7 +663,7 @@ void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { private_block_overlays_[key] = td::actor::create_actor( "BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, config_, keyring_, adnl_, rldp_, rldp2_, overlays_, validator_manager_); - } + }*/ } FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index add71ca5c..df0c1453b 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -388,8 +388,8 @@ class ValidatorManagerImpl : public ValidatorManager { } void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { - queue_size_counter_ = - td::actor::create_actor("queuesizecounter", td::Ref{}, actor_id(this)); + queue_size_counter_ = td::actor::create_actor("queuesizecounter", td::Ref{}, + opts_, actor_id(this)); } td::actor::send_closure(queue_size_counter_, &QueueSizeCounter::get_queue_size, block_id, std::move(promise)); } diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 43c979eb9..065cad62f 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -450,8 +450,8 @@ class ValidatorManagerImpl : public ValidatorManager { } void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { - queue_size_counter_ = - td::actor::create_actor("queuesizecounter", td::Ref{}, actor_id(this)); + queue_size_counter_ = td::actor::create_actor("queuesizecounter", td::Ref{}, + opts_, actor_id(this)); } td::actor::send_closure(queue_size_counter_, &QueueSizeCounter::get_queue_size, block_id, std::move(promise)); } diff --git a/validator/manager.cpp b/validator/manager.cpp index f1d65127a..1dceeee95 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -3115,6 +3115,9 @@ void ValidatorManagerImpl::update_options(td::Ref opts) if (!out_msg_queue_importer_.empty()) { td::actor::send_closure(out_msg_queue_importer_, &OutMsgQueueImporter::update_options, opts); } + if (!queue_size_counter_.empty()) { + td::actor::send_closure(queue_size_counter_, &QueueSizeCounter::update_options, opts); + } opts_ = std::move(opts); } diff --git a/validator/manager.hpp b/validator/manager.hpp index 045f69a25..df1cc77c6 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -591,13 +591,17 @@ class ValidatorManagerImpl : public ValidatorManager { void update_options(td::Ref opts) override; void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { + if (last_masterchain_state_.is_null()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "not ready")); + return; + } if (queue_size_counter_.empty()) { - if (last_masterchain_state_.is_null()) { - promise.set_error(td::Status::Error(ErrorCode::notready, "not ready")); - return; - } - queue_size_counter_ = td::actor::create_actor("queuesizecounter", - last_masterchain_state_, actor_id(this)); + queue_size_counter_ = + td::actor::create_actor("queuesizecounter", last_masterchain_state_, opts_, actor_id(this)); + } + if (!opts_->need_monitor(block_id.shard_full(), last_masterchain_state_)) { + return promise.set_error( + td::Status::Error(PSTRING() << "not monitoring shard " << block_id.shard_full().to_str())); } td::actor::send_closure(queue_size_counter_, &QueueSizeCounter::get_queue_size, block_id, std::move(promise)); } diff --git a/validator/queue-size-counter.cpp b/validator/queue-size-counter.cpp index 4780f202c..fd70cf2e7 100644 --- a/validator/queue-size-counter.cpp +++ b/validator/queue-size-counter.cpp @@ -234,7 +234,9 @@ void QueueSizeCounter::process_top_shard_blocks_cont(td::Ref s last_top_blocks_.clear(); last_top_blocks_.push_back(state->get_block_id()); for (auto &shard : state->get_shards()) { - last_top_blocks_.push_back(shard->top_block_id()); + if (opts_->need_monitor(shard->shard(), state)) { + last_top_blocks_.push_back(shard->top_block_id()); + } } for (const BlockIdExt &block_id : last_top_blocks_) { get_queue_size_ex_retry(block_id, init, ig.get_promise()); diff --git a/validator/queue-size-counter.hpp b/validator/queue-size-counter.hpp index fabb0cec3..4c962fc17 100644 --- a/validator/queue-size-counter.hpp +++ b/validator/queue-size-counter.hpp @@ -21,16 +21,22 @@ namespace ton::validator { class QueueSizeCounter : public td::actor::Actor { public: - QueueSizeCounter(td::Ref last_masterchain_state, td::actor::ActorId manager) - : init_masterchain_state_(last_masterchain_state), manager_(std::move(manager)) { + QueueSizeCounter(td::Ref last_masterchain_state, td::Ref opts, + td::actor::ActorId manager) + : init_masterchain_state_(last_masterchain_state), opts_(std::move(opts)), manager_(std::move(manager)) { } void start_up() override; void get_queue_size(BlockIdExt block_id, td::Promise promise); void alarm() override; + void update_options(td::Ref opts) { + opts_ = std::move(opts); + } + private: td::Ref init_masterchain_state_; + td::Ref opts_; td::actor::ActorId manager_; bool simple_mode_ = false; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index dbbefa7dc..79c1a2da4 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -195,7 +195,7 @@ void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref } }); - run_accept_block_query(block_id, std::move(block), prev_block_ids_, validator_set_, std::move(sig_set), + run_accept_block_query(block_id, std::move(block), std::move(prev), validator_set_, std::move(sig_set), std::move(approve_sig_set), send_broadcast, shard_.is_masterchain() || mode_ == ValidatorManagerOptions::validator_normal, manager_, std::move(P)); From e575d273025add5c2a5232704c0ef85c3f3c15a9 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 19 Feb 2024 14:21:15 +0300 Subject: [PATCH 081/388] Remove unused priority_broadcast_receivers --- crypto/block/block.tlb | 2 +- crypto/block/mc-config.cpp | 7 +------ crypto/block/mc-config.h | 1 - overlay/overlay-broadcast.cpp | 8 ++------ overlay/overlay-broadcast.hpp | 4 +--- overlay/overlay-fec-broadcast.cpp | 20 +++++++------------- overlay/overlay-fec-broadcast.hpp | 13 ++++--------- overlay/overlay-manager.cpp | 13 ------------- overlay/overlay-manager.h | 2 -- overlay/overlay.h | 1 - overlay/overlay.hpp | 17 ----------------- overlay/overlays.h | 2 -- validator/full-node-shard.cpp | 13 ------------- validator/full-node-shard.h | 1 - validator/full-node-shard.hpp | 2 -- validator/full-node.cpp | 30 ------------------------------ validator/full-node.hpp | 4 ---- 17 files changed, 16 insertions(+), 124 deletions(-) diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 9988f2d1c..475276184 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -782,7 +782,7 @@ misbehaviour_punishment_config_v1#01 _ MisbehaviourPunishmentConfig = ConfigParam 40; // collator_nodes: each collator is (workchain:int32 shard:uint64 adnl_id:uint256) -collator_info#0 full_node_id:(Maybe uint256) = CollatorInfo; +collator_info#00 = CollatorInfo; colator_config#a0 full_collated_data:Bool collator_nodes:(HashmapE 352 CollatorInfo) = CollatorConfig; _ CollatorConfig = ConfigParam 41; diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 547069365..628093b0b 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2333,12 +2333,7 @@ CollatorConfig Config::get_collator_config(bool need_collator_nodes) const { td::uint64 shard = key.get_uint(64); key.advance(64); td::Bits256 adnl_id(key); - td::Bits256 full_node_id = td::Bits256::zero(); - gen::CollatorInfo::Record info; - if (tlb::csr_unpack(std::move(value), info) && info.full_node_id->size() == 257) { - full_node_id = td::Bits256(info.full_node_id->data_bits() + 1); - } - collator_config.collator_nodes.push_back({ton::ShardIdFull(workchain, shard), adnl_id, full_node_id}); + collator_config.collator_nodes.push_back({ton::ShardIdFull(workchain, shard), adnl_id}); return true; }); } diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 1262a1d8e..ef8f4dfa7 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -529,7 +529,6 @@ struct BurningConfig { struct CollatorNodeDescr { ton::ShardIdFull shard; ton::NodeIdShort adnl_id; - ton::NodeIdShort full_node_id; }; struct CollatorConfig { diff --git a/overlay/overlay-broadcast.cpp b/overlay/overlay-broadcast.cpp index bcb7c2858..03991b76b 100644 --- a/overlay/overlay-broadcast.cpp +++ b/overlay/overlay-broadcast.cpp @@ -68,11 +68,7 @@ td::Status BroadcastSimple::run_checks() { td::Status BroadcastSimple::distribute() { auto B = serialize(); - auto nodes = overlay_->get_neighbours(5); - if (is_ours_) { - auto priority_nodes = overlay_->get_priority_broadcast_receivers(3); - nodes.insert(nodes.end(), priority_nodes.begin(), priority_nodes.end()); - } + auto nodes = overlay_->get_neighbours(3); auto manager = overlay_->overlay_manager(); for (auto &n : nodes) { @@ -144,7 +140,7 @@ td::Status BroadcastSimple::create_new(td::actor::ActorId overlay, auto date = static_cast(td::Clocks::system()); auto B = std::make_unique(broadcast_hash, PublicKey{}, nullptr, flags, std::move(data), date, - td::BufferSlice{}, false, nullptr, adnl::AdnlNodeIdShort::zero(), true); + td::BufferSlice{}, false, nullptr, adnl::AdnlNodeIdShort::zero()); auto to_sign = B->to_sign(); auto P = td::PromiseCreator::lambda( diff --git a/overlay/overlay-broadcast.hpp b/overlay/overlay-broadcast.hpp index 8a5d13249..e7b39eecc 100644 --- a/overlay/overlay-broadcast.hpp +++ b/overlay/overlay-broadcast.hpp @@ -46,7 +46,6 @@ class BroadcastSimple : public td::ListNode { td::uint32 date_; td::BufferSlice signature_; bool is_valid_{false}; - bool is_ours_{false}; OverlayImpl *overlay_; @@ -64,7 +63,7 @@ class BroadcastSimple : public td::ListNode { public: BroadcastSimple(Overlay::BroadcastHash broadcast_hash, PublicKey source, std::shared_ptr cert, td::uint32 flags, td::BufferSlice data, td::uint32 date, td::BufferSlice signature, bool is_valid, - OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, bool is_ours = false) + OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id) : broadcast_hash_(broadcast_hash) , source_(std::move(source)) , cert_(std::move(cert)) @@ -73,7 +72,6 @@ class BroadcastSimple : public td::ListNode { , date_(date) , signature_(std::move(signature)) , is_valid_(is_valid) - , is_ours_(is_ours) , overlay_(overlay) , src_peer_id_(src_peer_id) { } diff --git a/overlay/overlay-fec-broadcast.cpp b/overlay/overlay-fec-broadcast.cpp index e38ff78d7..aed5248b8 100644 --- a/overlay/overlay-fec-broadcast.cpp +++ b/overlay/overlay-fec-broadcast.cpp @@ -26,8 +26,8 @@ namespace overlay { td::Result> BroadcastFec::create(Overlay::BroadcastHash hash, PublicKey src, Overlay::BroadcastDataHash data_hash, td::uint32 flags, - td::uint32 date, fec::FecType fec_type, bool is_ours) { - auto F = std::make_unique(hash, std::move(src), data_hash, flags, date, std::move(fec_type), is_ours); + td::uint32 date, fec::FecType fec_type) { + auto F = std::make_unique(hash, std::move(src), data_hash, flags, date, std::move(fec_type)); TRY_STATUS(F->run_checks()); TRY_STATUS(F->init_fec_type()); return std::move(F); @@ -94,12 +94,12 @@ void BroadcastFec::broadcast_checked(td::Result R) { overlay_->deliver_broadcast(get_source().compute_short_id(), data_.clone()); auto manager = overlay_->overlay_manager(); while (!parts_.empty()) { - distribute_part(parts_.begin()->first); + distribute_part(parts_.begin()->first); } } // Do we need status here?? -td::Status BroadcastFec::distribute_part(td::uint32 seqno) { +td::Status BroadcastFec::distribute_part(td::uint32 seqno) { auto i = parts_.find(seqno); if (i == parts_.end()) { // should not get here @@ -111,12 +111,8 @@ td::Status BroadcastFec::distribute_part(td::uint32 seqno) { td::BufferSlice data = std::move(tls.second); auto nodes = overlay_->get_neighbours(5); - if (is_ours_) { - auto priority_nodes = overlay_->get_priority_broadcast_receivers(3); - nodes.insert(nodes.end(), priority_nodes.begin(), priority_nodes.end()); - } - auto manager = overlay_->overlay_manager(); + for (auto &n : nodes) { if (neighbour_completed(n)) { continue; @@ -144,8 +140,7 @@ td::Status OverlayFecBroadcastPart::apply() { if (is_short_) { return td::Status::Error(ErrorCode::protoviolation, "short broadcast part for incomplete broadcast"); } - TRY_RESULT( - B, BroadcastFec::create(broadcast_hash_, source_, broadcast_data_hash_, flags_, date_, fec_type_, is_ours_)); + TRY_RESULT(B, BroadcastFec::create(broadcast_hash_, source_, broadcast_data_hash_, flags_, date_, fec_type_)); bcast_ = B.get(); overlay_->register_fec_broadcast(std::move(B)); } @@ -309,8 +304,7 @@ td::Status OverlayFecBroadcastPart::create_new(OverlayImpl *overlay, td::actor:: auto B = std::make_unique( broadcast_hash, part_hash, PublicKey{}, overlay->get_certificate(local_id), data_hash, size, flags, - part_data_hash, std::move(part), seqno, std::move(fec_type), date, td::BufferSlice{}, false, nullptr, overlay, - adnl::AdnlNodeIdShort::zero(), true); + part_data_hash, std::move(part), seqno, std::move(fec_type), date, td::BufferSlice{}, false, nullptr, overlay, adnl::AdnlNodeIdShort::zero()); auto to_sign = B->to_sign(); auto P = td::PromiseCreator::lambda( diff --git a/overlay/overlay-fec-broadcast.hpp b/overlay/overlay-fec-broadcast.hpp index c97ad8858..199574c31 100644 --- a/overlay/overlay-fec-broadcast.hpp +++ b/overlay/overlay-fec-broadcast.hpp @@ -131,19 +131,18 @@ class BroadcastFec : public td::ListNode { td::Status run_checks(); BroadcastFec(Overlay::BroadcastHash hash, PublicKey src, Overlay::BroadcastDataHash data_hash, td::uint32 flags, - td::uint32 date, fec::FecType fec_type, bool is_ours = false) + td::uint32 date, fec::FecType fec_type) : hash_(hash) , data_hash_(data_hash) , flags_(flags) , date_(date) , src_(std::move(src)) - , fec_type_(std::move(fec_type)) - , is_ours_(is_ours) { + , fec_type_(std::move(fec_type)) { } static td::Result> create(Overlay::BroadcastHash hash, PublicKey src, Overlay::BroadcastDataHash data_hash, td::uint32 flags, - td::uint32 date, fec::FecType fec_type, bool is_ours); + td::uint32 date, fec::FecType fec_type); bool neighbour_received(adnl::AdnlNodeIdShort id) const { return received_neighbours_.find(id) != received_neighbours_.end(); @@ -226,7 +225,6 @@ class BroadcastFec : public td::ListNode { OverlayImpl *overlay_; adnl::AdnlNodeIdShort src_peer_id_ = adnl::AdnlNodeIdShort::zero(); td::BufferSlice data_; - bool is_ours_ = false; }; class OverlayFecBroadcastPart : public td::ListNode { @@ -248,7 +246,6 @@ class OverlayFecBroadcastPart : public td::ListNode { bool is_short_; bool untrusted_{false}; - bool is_ours_; BroadcastFec *bcast_; OverlayImpl *overlay_; @@ -268,8 +265,7 @@ class OverlayFecBroadcastPart : public td::ListNode { std::shared_ptr cert, Overlay::BroadcastDataHash data_hash, td::uint32 data_size, td::uint32 flags, Overlay::BroadcastDataHash part_data_hash, td::BufferSlice data, td::uint32 seqno, fec::FecType fec_type, td::uint32 date, td::BufferSlice signature, - bool is_short, BroadcastFec *bcast, OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, - bool is_ours = false) + bool is_short, BroadcastFec *bcast, OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id) : broadcast_hash_(broadcast_hash) , part_hash_(part_hash) , source_(std::move(source)) @@ -284,7 +280,6 @@ class OverlayFecBroadcastPart : public td::ListNode { , date_(date) , signature_(std::move(signature)) , is_short_(is_short) - , is_ours_(is_ours) , bcast_(bcast) , overlay_(overlay) , src_peer_id_(src_peer_id) { diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index 17e94aae7..0591e7320 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -350,19 +350,6 @@ void OverlayManager::get_stats(td::Promise nodes) { - auto it = overlays_.find(local_id); - if (it == overlays_.end()) { - return; - } - auto it2 = it->second.find(overlay); - if (it2 == it->second.end()) { - return; - } - td::actor::send_closure(it2->second, &Overlay::set_priority_broadcast_receivers, std::move(nodes)); -} - void OverlayManager::forget_peer(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, adnl::AdnlNodeIdShort peer_id) { auto it = overlays_.find(local_id); diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h index aeca69da7..659069bd8 100644 --- a/overlay/overlay-manager.h +++ b/overlay/overlay-manager.h @@ -96,8 +96,6 @@ class OverlayManager : public Overlays { td::actor::ActorOwn overlay); void get_stats(td::Promise> promise) override; - void set_priority_broadcast_receivers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, - std::vector nodes) override; void forget_peer(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, adnl::AdnlNodeIdShort peer_id) override; struct PrintId {}; diff --git a/overlay/overlay.h b/overlay/overlay.h index 1b5264a85..b21de3628 100644 --- a/overlay/overlay.h +++ b/overlay/overlay.h @@ -69,7 +69,6 @@ class Overlay : public td::actor::Actor { virtual void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) = 0; //virtual void receive_broadcast(td::BufferSlice data) = 0; //virtual void subscribe(std::unique_ptr callback) = 0; - virtual void set_priority_broadcast_receivers(std::vector nodes) = 0; virtual void forget_peer(adnl::AdnlNodeIdShort peer_id) = 0; }; diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 1137aa8dd..f3ef3ab41 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -216,18 +216,6 @@ class OverlayImpl : public Overlay { return vec; } } - std::vector get_priority_broadcast_receivers(td::uint32 max_size = 0) const { - if (max_size == 0 || max_size >= priority_broadcast_receivers_.size()) { - return priority_broadcast_receivers_; - } else { - std::vector vec; - for (td::uint32 i = 0; i < max_size; i++) { - vec.push_back(priority_broadcast_receivers_[td::Random::fast( - 0, static_cast(priority_broadcast_receivers_.size()) - 1)]); - } - return vec; - } - } td::actor::ActorId overlay_manager() const { return manager_; } @@ -281,10 +269,6 @@ class OverlayImpl : public Overlay { } } - void set_priority_broadcast_receivers(std::vector nodes) override { - priority_broadcast_receivers_ = std::move(nodes); - } - void forget_peer(adnl::AdnlNodeIdShort peer_id) override { del_peer(peer_id); } @@ -364,7 +348,6 @@ class OverlayImpl : public Overlay { std::queue bcast_lru_; std::map> out_fec_bcasts_; - std::vector priority_broadcast_receivers_; void bcast_gc(); diff --git a/overlay/overlays.h b/overlay/overlays.h index cc93fae33..62daa0d93 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -250,8 +250,6 @@ class Overlays : public td::actor::Actor { td::Promise> promise) = 0; virtual void get_stats(td::Promise> promise) = 0; - virtual void set_priority_broadcast_receivers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, - std::vector nodes) = 0; virtual void forget_peer(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, adnl::AdnlNodeIdShort peer_id) = 0; }; diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index fc2ae3a48..026e8190a 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -124,10 +124,6 @@ void FullNodeShardImpl::create_overlay() { if (cert_) { td::actor::send_closure(overlays_, &overlay::Overlays::update_certificate, adnl_id_, overlay_id_, local_id_, cert_); } - if (!collator_nodes_.empty()) { - td::actor::send_closure(overlays_, &overlay::Overlays::set_priority_broadcast_receivers, adnl_id_, overlay_id_, - collator_nodes_); - } } void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broadcast, td::Promise promise) { @@ -1141,15 +1137,6 @@ void FullNodeShardImpl::update_validators(std::vector public_key_ } } -void FullNodeShardImpl::update_collators(std::vector nodes) { - if (!client_.empty()) { - return; - } - collator_nodes_ = std::move(nodes); - td::actor::send_closure(overlays_, &overlay::Overlays::set_priority_broadcast_receivers, adnl_id_, overlay_id_, - collator_nodes_); -} - void FullNodeShardImpl::reload_neighbours() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 2131939fe..d984aeadf 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -77,7 +77,6 @@ class FullNodeShard : public td::actor::Actor { virtual void set_handle(BlockHandle handle, td::Promise promise) = 0; virtual void update_validators(std::vector public_key_hashes, PublicKeyHash local_hash) = 0; - virtual void update_collators(std::vector nodes) = 0; static td::actor::ActorOwn create( ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index cf27cfeef..102fc6212 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -199,7 +199,6 @@ class FullNodeShardImpl : public FullNodeShard { void alarm() override; void update_validators(std::vector public_key_hashes, PublicKeyHash local_hash) override; - void update_collators(std::vector nodes) override; void sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) override; void import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) override; @@ -278,7 +277,6 @@ class FullNodeShardImpl : public FullNodeShard { adnl::AdnlNodeIdShort last_pinged_neighbour_ = adnl::AdnlNodeIdShort::zero(); FullNodeShardMode mode_; - std::vector collator_nodes_; FullNodeConfig config_; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index d67963378..b8842a61e 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -218,33 +218,6 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s ++it; } } - - if (!collators_inited_ || state->is_key_state()) { - update_collators(state); - } -} - -void FullNodeImpl::update_collators(td::Ref state) { - collators_inited_ = true; - collator_config_ = state->get_collator_config(true); - for (auto& s : shards_) { - update_shard_collators(s.first, s.second); - } -} - -void FullNodeImpl::update_shard_collators(ShardIdFull shard, ShardInfo& info) { - if (info.actor.empty()) { - return; - } - std::vector nodes; - for (const block::CollatorNodeDescr& desc : collator_config_.collator_nodes) { - if (!desc.full_node_id.is_zero() && shard_intersects(shard, desc.shard)) { - nodes.emplace_back(desc.full_node_id); - } - } - std::sort(nodes.begin(), nodes.end()); - nodes.erase(std::unique(nodes.begin(), nodes.end()), nodes.end()); - td::actor::send_closure(info.actor, &FullNodeShard::update_collators, std::move(nodes)); } void FullNodeImpl::add_shard_actor(ShardIdFull shard, FullNodeShardMode mode) { @@ -259,9 +232,6 @@ void FullNodeImpl::add_shard_actor(ShardIdFull shard, FullNodeShardMode mode) { if (all_validators_.size() > 0) { td::actor::send_closure(info.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); } - if (collators_inited_) { - update_shard_collators(shard, info); - } } void FullNodeImpl::sync_completed() { diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 5bb95186b..5ed8101fb 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -102,8 +102,6 @@ class FullNodeImpl : public FullNode { }; void add_shard_actor(ShardIdFull shard, FullNodeShardMode mode); - void update_collators(td::Ref state); - void update_shard_collators(ShardIdFull shard, ShardInfo& info); PublicKeyHash local_id_; adnl::AdnlNodeIdShort adnl_id_; @@ -131,8 +129,6 @@ class FullNodeImpl : public FullNode { std::set local_keys_; td::Promise started_promise_; - bool collators_inited_ = false; - block::CollatorConfig collator_config_; FullNodeConfig config_; std::map> private_block_overlays_; From 28699654fb35ad4f11204d0578832ba030850ed3 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 21 Feb 2024 18:38:02 +0300 Subject: [PATCH 082/388] New private block overlays --- tl/generate/scheme/ton_api.tl | 2 + tl/generate/scheme/ton_api.tlo | Bin 92172 -> 92852 bytes validator-engine/validator-engine.cpp | 12 + validator/CMakeLists.txt | 2 + validator/full-node-private-overlay-v2.cpp | 330 +++++++++++++++++++++ validator/full-node-private-overlay-v2.hpp | 109 +++++++ validator/full-node.cpp | 53 +++- validator/full-node.h | 2 + validator/full-node.hpp | 9 + 9 files changed, 505 insertions(+), 14 deletions(-) create mode 100644 validator/full-node-private-overlay-v2.cpp create mode 100644 validator/full-node-private-overlay-v2.hpp diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index e911eacfc..aa761ad26 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -398,6 +398,7 @@ tonNode.newShardBlockBroadcast block:tonNode.newShardBlock = tonNode.Broadcast; tonNode.shardPublicOverlayId workchain:int shard:long zero_state_file_hash:int256 = tonNode.ShardPublicOverlayId; tonNode.privateBlockOverlayId zero_state_file_hash:int256 nodes:(vector int256) = tonNode.PrivateBlockOverlayId; +tonNode.privateBlockOverlayIdV2 zero_state_file_hash:int256 workchain:int shard:long nodes:(vector int256) senders:(vector int256) = tonNode.PrivateBlockOverlayIdV2; tonNode.keyBlocks blocks:(vector tonNode.blockIdExt) incomplete:Bool error:Bool = tonNode.KeyBlocks; @@ -665,6 +666,7 @@ engine.validator.shardOverlayStats.neighbour id:string proto_verison:int capabil roundtrip:double unreliability:double has_state:string = engine.validator.shardOverlayStats.Neighbour; engine.validator.shardOverlayStats shard:string mode:string neighbours:(vector engine.validator.shardOverlayStats.neighbour) = engine.validator.ShardOverlayStats; +engine.validator.privateBlockOverlayV2Stats shard:string nodes:(vector int256) senders:(vector int256) created_at:int = engine.validator.PrivateBlockOverlayV2Stats; engine.validator.onePerfTimerStat time:int min:double avg:double max:double = engine.validator.OnePerfTimerStat; engine.validator.perfTimerStatsByName name:string stats:(vector engine.validator.OnePerfTimerStat) = engine.validator.PerfTimerStatsByName; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index afd8e0fb16b8b5e5cfd48d072aa4604134a85ceb..3afe3f33e8e089d6d7691e781d0441ed40885811 100644 GIT binary patch delta 304 zcmeCVz`ErsEAOM(`c@23Ah?m2Thid_(t|zPsd?#{d8vA3i8+}mi6!|(dI3e5Wr-!J zPC5C>+5TmzMLCI;VMf6vi6zCGBy}WIjuy;dlP}57^UF_3#irIXCCq5DfT`qWUBi49 z#%Y`9n#vclf^C|7aG9_oF0(-@ZXOfzVVPXGLVfaqMJkhHR`X2Wu$YylI5jV2^1?KE zkic{W1x8k`nu02Z-!bN~PV delta 52 zcmdmTm9^&rEAOM(`c@23z`v1~TXJ)Pq=>}k4Tc3Qj8ir%n#mV#c35#IYP$mqV@VnS Dn7a@P diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 5abf913aa..c3d374248 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1937,6 +1937,10 @@ void ValidatorEngine::start_full_node() { td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_permanent_key, v.first, [](td::Unit) {}); } + for (auto &c : config_.collators) { + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_collator_adnl_id, + ton::adnl::AdnlNodeIdShort(c.adnl_id)); + } } else { started_full_node(); } @@ -3647,6 +3651,10 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollat td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_collator, ton::adnl::AdnlNodeIdShort(id), shard); } + if (!full_node_.empty()) { + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_collator_adnl_id, + ton::adnl::AdnlNodeIdShort(id)); + } write_config([promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); @@ -3722,6 +3730,10 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCollat td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::del_collator, ton::adnl::AdnlNodeIdShort(id), shard); } + if (!full_node_.empty()) { + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::del_collator_adnl_id, + ton::adnl::AdnlNodeIdShort(id)); + } write_config([promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index afeb77934..0cde874b4 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -149,6 +149,8 @@ set(FULL_NODE_SOURCE full-node-master.cpp full-node-private-overlay.hpp full-node-private-overlay.cpp + full-node-private-overlay-v2.hpp + full-node-private-overlay-v2.cpp net/download-block.hpp net/download-block.cpp diff --git a/validator/full-node-private-overlay-v2.cpp b/validator/full-node-private-overlay-v2.cpp new file mode 100644 index 000000000..fedb8a788 --- /dev/null +++ b/validator/full-node-private-overlay-v2.cpp @@ -0,0 +1,330 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "full-node-private-overlay-v2.hpp" +#include "ton/ton-tl.hpp" +#include "common/delay.h" +#include "td/utils/JsonBuilder.h" +#include "tl/tl_json.h" +#include "auto/tl/ton_api_json.h" + +namespace ton::validator::fullnode { + +void FullNodePrivateOverlayV2::process_broadcast(PublicKeyHash, ton_api::tonNode_blockBroadcast &query) { + std::vector signatures; + for (auto &sig : query.signatures_) { + signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)}); + } + + BlockIdExt block_id = create_block_id(query.id_); + BlockBroadcast B{block_id, + std::move(signatures), + static_cast(query.catchain_seqno_), + static_cast(query.validator_set_hash_), + std::move(query.data_), + std::move(query.proof_)}; + + auto P = td::PromiseCreator::lambda([](td::Result R) { + if (R.is_error()) { + if (R.error().code() == ErrorCode::notready) { + LOG(DEBUG) << "dropped broadcast: " << R.move_as_error(); + } else { + LOG(INFO) << "dropped broadcast: " << R.move_as_error(); + } + } + }); + LOG(FULL_NODE_DEBUG) << "Got block broadcast in private overlay: " << B.block_id.to_str(); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, std::move(B), + std::move(P)); +} + +void FullNodePrivateOverlayV2::process_broadcast(PublicKeyHash, ton_api::tonNode_newShardBlockBroadcast &query) { + BlockIdExt block_id = create_block_id(query.block_->block_); + LOG(FULL_NODE_DEBUG) << "Got block description broadcast in private overlay: " << block_id.to_str(); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block, block_id, + query.block_->cc_seqno_, std::move(query.block_->data_)); +} + +void FullNodePrivateOverlayV2::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { + auto B = fetch_tl_object(std::move(broadcast), true); + if (B.is_error()) { + return; + } + + ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto &obj) { Self->process_broadcast(src, obj); }); +} + +void FullNodePrivateOverlayV2::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) { + if (!inited_) { + return; + } + auto B = create_serialize_tl_object( + create_tl_object(create_tl_block_id(block_id), cc_seqno, std::move(data))); + if (B.size() <= overlay::Overlays::max_simple_broadcast_size()) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), 0, std::move(B)); + } else { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); + } +} + +void FullNodePrivateOverlayV2::send_broadcast(BlockBroadcast broadcast) { + if (!inited_) { + return; + } + std::vector> sigs; + for (auto &sig : broadcast.signatures) { + sigs.emplace_back(create_tl_object(sig.node, sig.signature.clone())); + } + auto B = create_serialize_tl_object( + create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, std::move(sigs), + broadcast.proof.clone(), broadcast.data.clone()); + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); +} + +void FullNodePrivateOverlayV2::start_up() { + std::sort(nodes_.begin(), nodes_.end()); + nodes_.erase(std::unique(nodes_.begin(), nodes_.end()), nodes_.end()); + + std::vector nodes, senders; + for (const adnl::AdnlNodeIdShort &id : nodes_) { + nodes.push_back(id.bits256_value()); + } + for (const adnl::AdnlNodeIdShort &id : senders_) { + senders.push_back(id.bits256_value()); + } + auto X = create_hash_tl_object( + zero_state_file_hash_, shard_.workchain, shard_.shard, std::move(nodes), std::move(senders)); + td::BufferSlice b{32}; + b.as_slice().copy_from(as_slice(X)); + overlay_id_full_ = overlay::OverlayIdFull{std::move(b)}; + overlay_id_ = overlay_id_full_.compute_short_id(); + + try_init(); +} + +void FullNodePrivateOverlayV2::try_init() { + // Sometimes adnl id is added to validator engine later (or not at all) + td::actor::send_closure( + adnl_, &adnl::Adnl::check_id_exists, local_id_, [SelfId = actor_id(this)](td::Result R) { + if (R.is_ok() && R.ok()) { + td::actor::send_closure(SelfId, &FullNodePrivateOverlayV2::init); + } else { + delay_action([SelfId]() { td::actor::send_closure(SelfId, &FullNodePrivateOverlayV2::try_init); }, + td::Timestamp::in(30.0)); + } + }); +} + +void FullNodePrivateOverlayV2::init() { + LOG(FULL_NODE_INFO) << "Creating private block overlay for shard " << shard_.to_str() << ", adnl_id=" << local_id_ + << " : " << nodes_.size() << " nodes"; + class Callback : public overlay::Overlays::Callback { + public: + void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { + } + void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + } + void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { + td::actor::send_closure(node_, &FullNodePrivateOverlayV2::receive_broadcast, src, std::move(data)); + } + void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + } + void get_stats_extra(td::Promise promise) override { + td::actor::send_closure(node_, &FullNodePrivateOverlayV2::get_stats_extra, std::move(promise)); + } + Callback(td::actor::ActorId node) : node_(node) { + } + + private: + td::actor::ActorId node_; + }; + + std::map authorized_keys; + for (const adnl::AdnlNodeIdShort &sender : senders_) { + authorized_keys[sender.pubkey_hash()] = overlay::Overlays::max_fec_broadcast_size(); + } + overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(), 0, std::move(authorized_keys)}; + td::actor::send_closure(overlays_, &overlay::Overlays::create_private_overlay, local_id_, overlay_id_full_.clone(), + nodes_, std::make_unique(actor_id(this)), rules); + + td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); + td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_id_); + inited_ = true; +} + +void FullNodePrivateOverlayV2::tear_down() { + if (inited_) { + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, local_id_, overlay_id_); + } +} + +void FullNodePrivateOverlayV2::get_stats_extra(td::Promise promise) { + auto res = create_tl_object(); + res->shard_ = shard_.to_str(); + for (const auto &x : nodes_) { + res->nodes_.push_back(x.bits256_value()); + } + for (const auto &x : senders_) { + res->senders_.push_back(x.bits256_value()); + } + res->created_at_ = created_at_; + promise.set_result(td::json_encode(td::ToJson(*res), true)); +} + +td::actor::ActorId FullNodePrivateBlockOverlays::choose_overlay(ShardIdFull shard) { + for (auto &p : id_to_overlays_) { + auto &overlays = p.second.overlays_; + ShardIdFull cur_shard = shard; + while (true) { + auto it = overlays.find(cur_shard); + if (it != overlays.end() && it->second.is_sender_) { + return it->second.overlay_.get(); + } + if (cur_shard.pfx_len() == 0) { + break; + } + cur_shard = shard_parent(cur_shard); + } + } + return {}; +} + +void FullNodePrivateBlockOverlays::update_overlays( + td::Ref state, std::set my_adnl_ids, const FileHash &zero_state_file_hash, + const td::actor::ActorId &keyring, const td::actor::ActorId &adnl, + const td::actor::ActorId &rldp, const td::actor::ActorId &rldp2, + const td::actor::ActorId &overlays, + const td::actor::ActorId &validator_manager) { + if (my_adnl_ids.empty()) { + id_to_overlays_.clear(); + return; + } + auto collators = state->get_collator_config(true); + auto all_validators = state->get_total_validator_set(0); + + struct OverlayInfo { + std::vector nodes, senders; + }; + std::map pverlay_infos; + + // Masterchain overlay: all validators + collators + OverlayInfo &mc_overlay = pverlay_infos[ShardIdFull(masterchainId)]; + for (const auto &x : all_validators->export_vector()) { + td::Bits256 addr = x.addr.is_zero() ? ValidatorFullId(x.key).compute_short_id().bits256_value() : x.addr; + mc_overlay.nodes.emplace_back(addr); + mc_overlay.senders.emplace_back(addr); + } + for (const auto &x : collators.collator_nodes) { + mc_overlay.nodes.emplace_back(x.adnl_id); + } + + // Shard overlays: validators of the shard + collators of the shard + // See ValidatorManagerImpl::update_shards + std::set new_shards; + for (auto &v : state->get_shards()) { + ShardIdFull shard = v->shard(); + if (shard.is_masterchain()) { + continue; + } + if (v->before_split()) { + ShardIdFull l_shard{shard.workchain, shard_child(shard.shard, true)}; + ShardIdFull r_shard{shard.workchain, shard_child(shard.shard, false)}; + new_shards.insert(l_shard); + new_shards.insert(r_shard); + } else if (v->before_merge()) { + ShardIdFull p_shard{shard.workchain, shard_parent(shard.shard)}; + new_shards.insert(p_shard); + } else { + new_shards.insert(shard); + } + } + for (ShardIdFull shard : new_shards) { + auto val_set = state->get_validator_set(shard); + td::uint32 min_split = state->monitor_min_split_depth(shard.workchain); + OverlayInfo &overlay = + pverlay_infos[shard_prefix_length(shard) <= min_split ? shard : shard_prefix(shard, min_split)]; + for (const auto &x : val_set->export_vector()) { + td::Bits256 addr = x.addr.is_zero() ? ValidatorFullId(x.key).compute_short_id().bits256_value() : x.addr; + overlay.nodes.emplace_back(addr); + overlay.senders.emplace_back(addr); + } + } + for (auto &p : pverlay_infos) { + ShardIdFull shard = p.first; + OverlayInfo &overlay = p.second; + if (!shard.is_masterchain()) { + for (const auto &collator : collators.collator_nodes) { + if (shard_intersects(collator.shard, shard)) { + overlay.nodes.emplace_back(collator.adnl_id); + } + } + } + + std::sort(overlay.nodes.begin(), overlay.nodes.end()); + overlay.nodes.erase(std::unique(overlay.nodes.begin(), overlay.nodes.end()), overlay.nodes.end()); + std::sort(overlay.senders.begin(), overlay.senders.end()); + overlay.senders.erase(std::unique(overlay.senders.begin(), overlay.senders.end()), overlay.senders.end()); + } + + std::map old_private_block_overlays = std::move(id_to_overlays_); + id_to_overlays_.clear(); + + for (const auto &p : pverlay_infos) { + ShardIdFull shard = p.first; + const OverlayInfo &new_overlay_info = p.second; + for (adnl::AdnlNodeIdShort local_id : new_overlay_info.nodes) { + if (!my_adnl_ids.count(local_id)) { + continue; + } + Overlays::ShardOverlay &new_overlay = id_to_overlays_[local_id].overlays_[shard]; + Overlays::ShardOverlay &old_overlay = old_private_block_overlays[local_id].overlays_[shard]; + if (!old_overlay.overlay_.empty() && old_overlay.nodes_ == new_overlay_info.nodes && + old_overlay.senders_ == new_overlay_info.senders) { + new_overlay = std::move(old_overlay); + old_overlay = {}; + } else { + new_overlay.nodes_ = new_overlay_info.nodes; + new_overlay.senders_ = new_overlay_info.senders; + new_overlay.is_sender_ = std::binary_search(new_overlay.senders_.begin(), new_overlay.senders_.end(), local_id); + new_overlay.overlay_ = td::actor::create_actor( + "BlocksPrivateOverlay", local_id, shard, new_overlay.nodes_, new_overlay.senders_, zero_state_file_hash, + keyring, adnl, rldp, rldp2, overlays, validator_manager); + } + } + } + + // Delete old overlays, but not immediately + for (auto &p : old_private_block_overlays) { + for (auto &x : p.second.overlays_) { + if (x.second.overlay_.empty()) { + continue; + } + td::actor::ActorId id = x.second.overlay_.release(); + delay_action([id = std::move(id)]() { td::actor::send_closure(id, &FullNodePrivateOverlayV2::destroy); }, + td::Timestamp::in(60.0)); + } + } +} + +} // namespace ton::validator::fullnode diff --git a/validator/full-node-private-overlay-v2.hpp b/validator/full-node-private-overlay-v2.hpp new file mode 100644 index 000000000..6fbcbbf71 --- /dev/null +++ b/validator/full-node-private-overlay-v2.hpp @@ -0,0 +1,109 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "full-node.h" + +namespace ton::validator::fullnode { + +class FullNodePrivateOverlayV2 : public td::actor::Actor { + public: + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); + template + void process_broadcast(PublicKeyHash, T &) { + VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast"; + } + void receive_broadcast(PublicKeyHash src, td::BufferSlice query); + + void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data); + void send_broadcast(BlockBroadcast broadcast); + + void start_up() override; + void tear_down() override; + + void destroy() { + stop(); + } + + FullNodePrivateOverlayV2(adnl::AdnlNodeIdShort local_id, ShardIdFull shard, std::vector nodes, + std::vector senders, FileHash zero_state_file_hash, + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId rldp2, + td::actor::ActorId overlays, + td::actor::ActorId validator_manager) + : local_id_(local_id) + , shard_(shard) + , nodes_(std::move(nodes)) + , senders_(std::move(senders)) + , zero_state_file_hash_(zero_state_file_hash) + , keyring_(keyring) + , adnl_(adnl) + , rldp_(rldp) + , rldp2_(rldp2) + , overlays_(overlays) + , validator_manager_(validator_manager) { + } + + private: + adnl::AdnlNodeIdShort local_id_; + ShardIdFull shard_; + std::vector nodes_; + std::vector senders_; + FileHash zero_state_file_hash_; + + td::actor::ActorId keyring_; + td::actor::ActorId adnl_; + td::actor::ActorId rldp_; + td::actor::ActorId rldp2_; + td::actor::ActorId overlays_; + td::actor::ActorId validator_manager_; + + bool inited_ = false; + overlay::OverlayIdFull overlay_id_full_; + overlay::OverlayIdShort overlay_id_; + UnixTime created_at_ = (UnixTime)td::Clocks::system(); + + void try_init(); + void init(); + void get_stats_extra(td::Promise promise); +}; + +class FullNodePrivateBlockOverlays { + public: + td::actor::ActorId choose_overlay(ShardIdFull shard); + void update_overlays(td::Ref state, std::set my_adnl_ids, + const FileHash& zero_state_file_hash, const td::actor::ActorId& keyring, + const td::actor::ActorId& adnl, const td::actor::ActorId& rldp, + const td::actor::ActorId& rldp2, + const td::actor::ActorId& overlays, + const td::actor::ActorId& validator_manager); + + private: + struct Overlays { + struct ShardOverlay { + td::actor::ActorOwn overlay_; + std::vector nodes_, senders_; + bool is_sender_ = false; + }; + std::map overlays_; + }; + + std::map id_to_overlays_; // local_id -> overlays +}; + +} // namespace ton::validator::fullnode diff --git a/validator/full-node.cpp b/validator/full-node.cpp index b8842a61e..57323c4b1 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -20,6 +20,7 @@ #include "ton/ton-io.hpp" #include "td/actor/MultiPromise.h" #include "full-node.h" +#include "common/delay.h" namespace ton { @@ -53,7 +54,7 @@ void FullNodeImpl::add_permanent_key(PublicKeyHash key, td::Promise pr td::actor::send_closure(shard.second.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); } } - create_private_block_overlay(key); + // create_private_block_overlay(key); promise.set_value(td::Unit()); } @@ -80,10 +81,20 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise pr td::actor::send_closure(shard.second.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); } } - private_block_overlays_.erase(key); + // private_block_overlays_.erase(key); promise.set_value(td::Unit()); } +void FullNodeImpl::add_collator_adnl_id(adnl::AdnlNodeIdShort id) { + ++local_collator_nodes_[id]; +} + +void FullNodeImpl::del_collator_adnl_id(adnl::AdnlNodeIdShort id) { + if (--local_collator_nodes_[id] == 0) { + local_collator_nodes_.erase(id); + } +} + void FullNodeImpl::sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) { auto it = shards_.find(shard_id); @@ -218,6 +229,19 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s ++it; } } + + std::set my_adnl_ids; + for (const auto &p : local_collator_nodes_) { + my_adnl_ids.insert(p.first); + } + for (auto key : local_keys_) { + auto it = current_validators_.find(key); + if (it != current_validators_.end()) { + my_adnl_ids.insert(it->second); + } + } + private_block_overlays_.update_overlays(state, std::move(my_adnl_ids), zero_state_file_hash_, keyring_, adnl_, rldp_, + rldp2_, overlays_, validator_manager_); } void FullNodeImpl::add_shard_actor(ShardIdFull shard, FullNodeShardMode mode) { @@ -262,9 +286,10 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; return; } - if (!private_block_overlays_.empty()) { - td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateOverlay::send_shard_block_info, - block_id, cc_seqno, data.clone()); + auto private_overlay = private_block_overlays_.choose_overlay(ShardIdFull(masterchainId)); + if (!private_overlay.empty()) { + td::actor::send_closure(private_overlay, &FullNodePrivateOverlayV2::send_shard_block_info, block_id, cc_seqno, + data.clone()); } td::actor::send_closure(shard, &FullNodeShard::send_shard_block_info, block_id, cc_seqno, std::move(data)); } @@ -275,9 +300,9 @@ void FullNodeImpl::send_broadcast(BlockBroadcast broadcast) { VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; return; } - if (!private_block_overlays_.empty()) { - td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateOverlay::send_broadcast, - broadcast.clone()); + auto private_overlay = private_block_overlays_.choose_overlay(broadcast.block_id.shard_full()); + if (!private_overlay.empty()) { + td::actor::send_closure(private_overlay, &FullNodePrivateOverlayV2::send_broadcast, broadcast.clone()); } td::actor::send_closure(shard, &FullNodeShard::send_broadcast, std::move(broadcast)); } @@ -440,7 +465,7 @@ void FullNodeImpl::got_key_block_proof(td::Ref proof) { if (current_validators != current_validators_) { current_validators_ = std::move(current_validators); - update_private_block_overlays(); + // update_private_block_overlays(); } if (keys == all_validators_) { @@ -483,7 +508,7 @@ void FullNodeImpl::got_zero_block_state(td::Ref state) { if (current_validators != current_validators_) { current_validators_ = std::move(current_validators); - update_private_block_overlays(); + // update_private_block_overlays(); } if (keys == all_validators_) { @@ -613,7 +638,7 @@ void FullNodeImpl::start_up() { std::make_unique(actor_id(this)), std::move(started_promise_)); } -void FullNodeImpl::update_private_block_overlays() { +/* void FullNodeImpl::update_private_block_overlays() { private_block_overlays_.clear(); if (local_keys_.empty()) { return; @@ -624,7 +649,7 @@ void FullNodeImpl::update_private_block_overlays() { } void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { - /*CHECK(local_keys_.count(key)); + CHECK(local_keys_.count(key)); if (current_validators_.count(key)) { std::vector nodes; for (const auto &p : current_validators_) { @@ -633,8 +658,8 @@ void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { private_block_overlays_[key] = td::actor::create_actor( "BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, config_, keyring_, adnl_, rldp_, rldp2_, overlays_, validator_manager_); - }*/ -} + } +} */ FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, diff --git a/validator/full-node.h b/validator/full-node.h index b10baeda7..65e6fae0b 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -63,6 +63,8 @@ class FullNode : public td::actor::Actor { virtual void add_permanent_key(PublicKeyHash key, td::Promise promise) = 0; virtual void del_permanent_key(PublicKeyHash key, td::Promise promise) = 0; + virtual void add_collator_adnl_id(adnl::AdnlNodeIdShort id) = 0; + virtual void del_collator_adnl_id(adnl::AdnlNodeIdShort id) = 0; virtual void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) = 0; diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 5ed8101fb..ff998f320 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -24,6 +24,7 @@ #include "interfaces/proof.h" #include "interfaces/shard.h" #include "full-node-private-overlay.hpp" +#include "full-node-private-overlay-v2.hpp" #include #include @@ -42,6 +43,8 @@ class FullNodeImpl : public FullNode { void add_permanent_key(PublicKeyHash key, td::Promise promise) override; void del_permanent_key(PublicKeyHash key, td::Promise promise) override; + void add_collator_adnl_id(adnl::AdnlNodeIdShort id) override; + void del_collator_adnl_id(adnl::AdnlNodeIdShort id) override; void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, @@ -127,14 +130,20 @@ class FullNodeImpl : public FullNode { std::map current_validators_; std::set local_keys_; + std::map local_collator_nodes_; td::Promise started_promise_; FullNodeConfig config_; + // TODO: Decide what to do with old private overlays. Maybe use old or new depending on some flag in config. + /* std::map> private_block_overlays_; void update_private_block_overlays(); void create_private_block_overlay(PublicKeyHash key); + */ + + FullNodePrivateBlockOverlays private_block_overlays_; }; } // namespace fullnode From 0fb781b4c3c7cabb675167f003cb1a3a6b81e8dd Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 22 Feb 2024 14:55:11 +0300 Subject: [PATCH 083/388] Improve queue cleanup (after merge) --- validator/impl/collator.cpp | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 50e01fd7f..c0c3cf16f 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -2323,8 +2323,22 @@ bool Collator::out_msg_queue_cleanup() { << nb.blk_.to_str()); } } + auto queue_root = out_msg_queue_->get_root_cell(); + if (queue_root.is_null()) { + LOG(DEBUG) << "out_msg_queue is empty"; + return true; + } + // Unwrap UsageCell: don't build proof for visiting output queue (unless something is deleted) + auto r_cell = queue_root->load_cell(); + if (r_cell.is_error()) { + return fatal_error(r_cell.move_as_error()); + } + auto pure_out_msg_queue = + std::make_unique(r_cell.move_as_ok().data_cell, 352, block::tlb::aug_OutMsgQueue); td::uint32 deleted = 0; - auto res = out_msg_queue_->filter([&](vm::CellSlice& cs, td::ConstBitPtr key, int n) -> int { + bool fail = false; + pure_out_msg_queue->check_for_each([&](Ref value, td::ConstBitPtr key, int n) -> bool { + vm::CellSlice& cs = value.write(); assert(n == 352); block::EnqueuedMsgDescr enq_msg_descr; unsigned long long created_lt; @@ -2333,7 +2347,8 @@ bool Collator::out_msg_queue_cleanup() { && enq_msg_descr.check_key(key) // check key && enq_msg_descr.lt_ == created_lt)) { LOG(ERROR) << "cannot unpack EnqueuedMsg with key " << key.to_hex(n); - return -1; + fail = true; + return false; } LOG(DEBUG) << "scanning outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," << enq_msg_descr.hash_.to_hex() << ") enqueued_lt=" << enq_msg_descr.enqueued_lt_; @@ -2354,10 +2369,20 @@ bool Collator::out_msg_queue_cleanup() { --out_msg_queue_size_; LOG(DEBUG) << "outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," << enq_msg_descr.hash_.to_hex() << ") enqueued_lt=" << enq_msg_descr.enqueued_lt_ << " has been already delivered, dequeueing"; + // Get value from out_msg_queue_ instead of pure_out_msg_queue (for proof) + auto value2 = out_msg_queue_->lookup_delete_with_extra(key, n); + CHECK(value2.not_null()); + vm::CellSlice& cs2 = value2.write(); + CHECK(cs2.fetch_ulong_bool(64, created_lt) // augmentation + && enq_msg_descr.unpack(cs2) // unpack EnqueuedMsg + && enq_msg_descr.check_key(key) // check key + && enq_msg_descr.lt_ == created_lt); + if (!dequeue_message(std::move(enq_msg_descr.msg_env_), deliver_lt)) { fatal_error(PSTRING() << "cannot dequeue outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," << enq_msg_descr.hash_.to_hex() << ") by inserting a msg_export_deq record"); - return -1; + fail = true; + return false; } register_out_msg_queue_op(); if (!block_limit_status_->fits(block::ParamLimits::cl_normal)) { @@ -2368,7 +2393,7 @@ bool Collator::out_msg_queue_cleanup() { }); LOG(WARNING) << "deleted " << deleted << " messages from out_msg_queue after merge, remaining queue size is " << out_msg_queue_size_; - if (res < 0) { + if (fail) { return fatal_error("error scanning/updating OutMsgQueue"); } } else { From 6c680dbc4e9b106b7fed3a151919715bedd82d07 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 22 Feb 2024 15:00:41 +0300 Subject: [PATCH 084/388] Cleanup code in validator-group.cpp (after merge) --- validator/validator-group.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 79c1a2da4..ed1359933 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -130,7 +130,6 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat return; } VLOG(VALIDATOR_DEBUG) << "validating block candidate " << next_block_id; - block.id = next_block_id; run_validate_query(shard_, min_masterchain_block_id_, prev_block_ids_, std::move(block), validator_set_, manager_, td::Timestamp::in(15.0), std::move(P), collator_config_.full_collated_data ? ValidateMode::full_collated_data : 0); @@ -170,7 +169,6 @@ void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash s prev_block_ids_ = std::vector{next_block_id}; cached_collated_block_ = nullptr; approved_candidates_cache_.clear(); - approved_candidates_cache_.clear(); } void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref block, std::vector prev, From fc6e2ead29f5d141b22b6dca9c0f4ec9bd051a20 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 15 Mar 2024 14:13:07 +0300 Subject: [PATCH 085/388] Revert "Process adnl query errors" This reverts commit 1869a2506287dbc66e04ac52ab7b620e9960371e. --- adnl/adnl-message.cpp | 3 --- adnl/adnl-message.h | 21 +-------------------- adnl/adnl-peer.cpp | 16 ---------------- adnl/adnl-peer.hpp | 1 - rldp/rldp-in.hpp | 4 ---- rldp/rldp.cpp | 22 ---------------------- rldp2/rldp-in.hpp | 4 ---- rldp2/rldp.cpp | 22 ---------------------- tl/generate/scheme/ton_api.tl | 2 -- tl/generate/scheme/ton_api.tlo | Bin 92852 -> 92660 bytes 10 files changed, 1 insertion(+), 94 deletions(-) diff --git a/adnl/adnl-message.cpp b/adnl/adnl-message.cpp index 1d3ba5f8a..0d7129783 100644 --- a/adnl/adnl-message.cpp +++ b/adnl/adnl-message.cpp @@ -46,9 +46,6 @@ AdnlMessage::AdnlMessage(tl_object_ptr message) { [&](ton_api::adnl_message_part &msg) { message_ = adnlmessage::AdnlMessagePart{msg.hash_, static_cast(msg.total_size_), static_cast(msg.offset_), std::move(msg.data_)}; - }, - [&](ton_api::adnl_message_queryError &msg) { - message_ = adnlmessage::AdnlMessageQueryError{msg.query_id_}; })); } diff --git a/adnl/adnl-message.h b/adnl/adnl-message.h index d557c3195..43849e982 100644 --- a/adnl/adnl-message.h +++ b/adnl/adnl-message.h @@ -170,24 +170,6 @@ class AdnlMessageAnswer { td::BufferSlice data_; }; -class AdnlMessageQueryError { - public: - explicit AdnlMessageQueryError(AdnlQueryId query_id) : query_id_(query_id) { - } - const auto &query_id() const { - return query_id_; - } - td::uint32 size() const { - return 36; - } - tl_object_ptr tl() const { - return create_tl_object(query_id_); - } - - private: - AdnlQueryId query_id_; -}; - class AdnlMessagePart { public: AdnlMessagePart(td::Bits256 hash, td::uint32 total_size, td::uint32 offset, td::BufferSlice data) @@ -238,8 +220,7 @@ class AdnlMessage { private: td::Variant + adnlmessage::AdnlMessageQuery, adnlmessage::AdnlMessageAnswer, adnlmessage::AdnlMessagePart> message_{Empty{}}; public: diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp index 85632f1de..3e21a7f53 100644 --- a/adnl/adnl-peer.cpp +++ b/adnl/adnl-peer.cpp @@ -536,14 +536,10 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageQuery &mess flags = static_cast(0)](td::Result R) { if (R.is_error()) { LOG(WARNING) << "failed to answer query: " << R.move_as_error(); - td::actor::send_closure(SelfId, &AdnlPeerPairImpl::send_message, - OutboundAdnlMessage{adnlmessage::AdnlMessageQueryError{query_id}, flags}); } else { auto data = R.move_as_ok(); if (data.size() > Adnl::huge_packet_max_size()) { LOG(WARNING) << "dropping too big answer query: size=" << data.size(); - td::actor::send_closure(SelfId, &AdnlPeerPairImpl::send_message, - OutboundAdnlMessage{adnlmessage::AdnlMessageQueryError{query_id}, flags}); } else { td::actor::send_closure(SelfId, &AdnlPeerPairImpl::send_message, OutboundAdnlMessage{adnlmessage::AdnlMessageAnswer{query_id, std::move(data)}, flags}); @@ -625,18 +621,6 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessagePart &messa } } -void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageQueryError &message) { - auto Q = out_queries_.find(message.query_id()); - - if (Q == out_queries_.end()) { - VLOG(ADNL_NOTICE) << this << ": dropping IN query error: unknown query id " << message.query_id(); - return; - } - - td::actor::send_closure_later(Q->second, &AdnlQuery::set_error, td::Status::Error("adnl query rejected")); - out_queries_.erase(Q); -} - void AdnlPeerPairImpl::delete_query(AdnlQueryId id) { auto Q = out_queries_.find(id); diff --git a/adnl/adnl-peer.hpp b/adnl/adnl-peer.hpp index 041de23c4..12ee01c6c 100644 --- a/adnl/adnl-peer.hpp +++ b/adnl/adnl-peer.hpp @@ -104,7 +104,6 @@ class AdnlPeerPairImpl : public AdnlPeerPair { void process_message(const adnlmessage::AdnlMessageQuery &message); void process_message(const adnlmessage::AdnlMessageAnswer &message); void process_message(const adnlmessage::AdnlMessagePart &message); - void process_message(const adnlmessage::AdnlMessageQueryError &message); void process_message(const AdnlMessage::Empty &message) { UNREACHABLE(); } diff --git a/rldp/rldp-in.hpp b/rldp/rldp-in.hpp index 2073b59ca..266f128a5 100644 --- a/rldp/rldp-in.hpp +++ b/rldp/rldp-in.hpp @@ -78,8 +78,6 @@ class RldpIn : public RldpImpl { td::uint64 max_answer_size) override; void answer_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, adnl::AdnlQueryId query_id, TransferId transfer_id, td::BufferSlice data); - void reject_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, - adnl::AdnlQueryId query_id, TransferId transfer_id); void alarm_query(adnl::AdnlQueryId query_id, TransferId transfer_id); @@ -95,8 +93,6 @@ class RldpIn : public RldpImpl { ton_api::rldp_query &message); void process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, ton_api::rldp_answer &message); - void process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, - ton_api::rldp_queryError &message); void receive_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, td::BufferSlice data); diff --git a/rldp/rldp.cpp b/rldp/rldp.cpp index 3c9e7b27f..402740e19 100644 --- a/rldp/rldp.cpp +++ b/rldp/rldp.cpp @@ -87,13 +87,6 @@ void RldpIn::answer_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, transfer(src, dst, timeout, std::move(B), transfer_id); } -void RldpIn::reject_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, - adnl::AdnlQueryId query_id, TransferId transfer_id) { - auto B = serialize_tl_object(create_tl_object(query_id), true); - - transfer(src, dst, timeout, std::move(B), transfer_id); -} - void RldpIn::alarm_query(adnl::AdnlQueryId query_id, TransferId transfer_id) { queries_.erase(query_id); max_size_.erase(transfer_id); @@ -206,16 +199,12 @@ void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort auto data = R.move_as_ok(); if (data.size() > max_answer_size) { VLOG(RLDP_NOTICE) << "rldp query failed: answer too big"; - td::actor::send_closure(SelfId, &RldpIn::reject_query, local_id, source, timeout, query_id, - transfer_id ^ TransferId::ones()); } else { td::actor::send_closure(SelfId, &RldpIn::answer_query, local_id, source, timeout, query_id, transfer_id ^ TransferId::ones(), std::move(data)); } } else { VLOG(RLDP_NOTICE) << "rldp query failed: " << R.move_as_error(); - td::actor::send_closure(SelfId, &RldpIn::reject_query, local_id, source, timeout, query_id, - transfer_id ^ TransferId::ones()); } }); VLOG(RLDP_DEBUG) << "delivering rldp query"; @@ -234,17 +223,6 @@ void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort } } -void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, - ton_api::rldp_queryError &message) { - auto it = queries_.find(message.query_id_); - if (it != queries_.end()) { - td::actor::send_closure(it->second, &adnl::AdnlQuery::set_error, td::Status::Error("adnl query rejected")); - queries_.erase(it); - } else { - VLOG(RLDP_INFO) << "received reject to unknown query " << message.query_id_; - } -} - void RldpIn::transfer_completed(TransferId transfer_id) { senders_.erase(transfer_id); VLOG(RLDP_DEBUG) << "rldp: completed transfer " << transfer_id << "; " << senders_.size() << " out transfer pending "; diff --git a/rldp2/rldp-in.hpp b/rldp2/rldp-in.hpp index 0092dae4f..095c14796 100644 --- a/rldp2/rldp-in.hpp +++ b/rldp2/rldp-in.hpp @@ -76,8 +76,6 @@ class RldpIn : public RldpImpl { td::uint64 max_answer_size) override; void answer_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, adnl::AdnlQueryId query_id, TransferId transfer_id, td::BufferSlice data); - void reject_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, - adnl::AdnlQueryId query_id, TransferId transfer_id); void receive_message_part(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, td::BufferSlice data); @@ -87,8 +85,6 @@ class RldpIn : public RldpImpl { ton_api::rldp_query &message); void process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, ton_api::rldp_answer &message); - void process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, - ton_api::rldp_queryError &message); void receive_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, td::Result data); diff --git a/rldp2/rldp.cpp b/rldp2/rldp.cpp index 8a3f77309..95780b237 100644 --- a/rldp2/rldp.cpp +++ b/rldp2/rldp.cpp @@ -121,13 +121,6 @@ void RldpIn::answer_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, send_closure(create_connection(src, dst), &RldpConnectionActor::send, transfer_id, std::move(B), timeout); } -void RldpIn::reject_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, - adnl::AdnlQueryId query_id, TransferId transfer_id) { - auto B = serialize_tl_object(create_tl_object(query_id), true); - - send_closure(create_connection(src, dst), &RldpConnectionActor::send, transfer_id, std::move(B), timeout); -} - void RldpIn::receive_message_part(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, td::BufferSlice data) { send_closure(create_connection(local_id, source), &RldpConnectionActor::receive_raw, std::move(data)); } @@ -187,16 +180,12 @@ void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort auto data = R.move_as_ok(); if (data.size() > max_answer_size) { VLOG(RLDP_NOTICE) << "rldp query failed: answer too big"; - td::actor::send_closure(SelfId, &RldpIn::reject_query, local_id, source, timeout, query_id, - transfer_id ^ TransferId::ones()); } else { td::actor::send_closure(SelfId, &RldpIn::answer_query, local_id, source, timeout, query_id, transfer_id ^ TransferId::ones(), std::move(data)); } } else { VLOG(RLDP_NOTICE) << "rldp query failed: " << R.move_as_error(); - td::actor::send_closure(SelfId, &RldpIn::reject_query, local_id, source, timeout, query_id, - transfer_id ^ TransferId::ones()); } }); VLOG(RLDP_DEBUG) << "delivering rldp query"; @@ -215,17 +204,6 @@ void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort } } -void RldpIn::process_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id, - ton_api::rldp_queryError &message) { - auto it = queries_.find(transfer_id); - if (it != queries_.end()) { - it->second.set_error(td::Status::Error("rejected")); - queries_.erase(it); - } else { - VLOG(RLDP_INFO) << "received reject to unknown query " << message.query_id_; - } -} - void RldpIn::on_sent(TransferId transfer_id, td::Result state) { //TODO: completed transfer } diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index aa761ad26..ef8679d65 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -137,7 +137,6 @@ adnl.message.reinit date:int = adnl.Message; adnl.message.query query_id:int256 query:bytes = adnl.Message; adnl.message.answer query_id:int256 answer:bytes = adnl.Message; -adnl.message.queryError query_id:int256 = adnl.Message; adnl.message.part hash:int256 total_size:int offset:int data:bytes = adnl.Message; @@ -163,7 +162,6 @@ rldp.complete transfer_id:int256 part:int = rldp.MessagePart; rldp.message id:int256 data:bytes = rldp.Message; rldp.query query_id:int256 max_answer_size:long timeout:int data:bytes = rldp.Message; rldp.answer query_id:int256 data:bytes = rldp.Message; -rldp.queryError query_id:int256 = rldp.Message; ---functions--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 3afe3f33e8e089d6d7691e781d0441ed40885811..9680b01e9bd9d374e8925c5a05da6a11d883635f 100644 GIT binary patch delta 331 zcmdmTmG#SI)(sKNq8&c_p7SK8zvcd_L$qA7XlN-V~nL%PTJ~|M|H9iFphJvpO zgyG;~0AbYlsz`tgxOq&-2WT%tku#ePhi%(Iu%SC(${+Ybm5ca64Y2VGffz6aCb9=6 zQsEC(v%r4}Sj}dOfCG#W)qetQAVM}lJ4#tVew+Lu1mtoR2*=}u3@1nw0RVl0jE(>R delta 489 zcmexznRUxm)(sKNf@}X=ILwonl9!|Bn_66)n4UUWkxzPa3-b+G(f5Bcs(6ZWQVL*F z3=9k`n+p}!i!e^xtZ2qA#sbnjS;xl*!l?1lfiTwi6hIgXzA6xggNp%FjjxIXNY~9{ zLOwv_7>beej%nHT_96H;Hn%~jOvCw z6>+d5oiMu zx)Hdelm!$VlNBavOy9u6C^NahmqQaI0m!Va+4VD=!q1`^pUbMnRl01cVa A*#H0l From 3190c67f1817aa988675cd312db1116021aae413 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 26 Mar 2024 11:42:30 +0300 Subject: [PATCH 086/388] Simplify getCapabilities in fullnode --- tl/generate/scheme/ton_api.tl | 6 +-- tl/generate/scheme/ton_api.tlo | Bin 92660 -> 92500 bytes validator/full-node-shard.cpp | 66 +++++++++++++-------------------- validator/full-node-shard.hpp | 29 ++++++++------- 4 files changed, 42 insertions(+), 59 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index ef8679d65..81f7192ed 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -408,8 +408,7 @@ tonNode.dataList data:(vector bytes) = tonNode.DataList; tonNode.dataFull id:tonNode.blockIdExt proof:bytes block:bytes is_link:Bool = tonNode.DataFull; tonNode.dataFullEmpty = tonNode.DataFull; -tonNode.capabilities version:int capabilities:long = tonNode.Capabilities; -tonNode.capabilitiesV2 version:int capabilities:long have_state:Bool = tonNode.Capabilities; +tonNode.capabilities#f5bf60c0 version_major:int version_minor:int flags:# = tonNode.Capabilities; tonNode.success = tonNode.Success; @@ -457,7 +456,6 @@ tonNode.getOutMsgQueueProof dst_shard:tonNode.shardId blocks:(vector tonNode.blo limits:tonNode.importedMsgQueueLimits = tonNode.OutMsgQueueProof; tonNode.getCapabilities = tonNode.Capabilities; -tonNode.getCapabilitiesV2 = tonNode.Capabilities; tonNode.slave.sendExtMessage message:tonNode.externalMessage = tonNode.Success; @@ -660,7 +658,7 @@ engine.validator.overlayStatsNode adnl_id:int256 ip_addr:string bdcst_errors:int engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 scope:string nodes:(vector engine.validator.overlayStatsNode) stats:(vector engine.validator.oneStat) extra:string = engine.validator.OverlayStats; engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.OverlaysStats; -engine.validator.shardOverlayStats.neighbour id:string proto_verison:int capabilities:long +engine.validator.shardOverlayStats.neighbour id:string verison_major:int version_minor:int flags:# roundtrip:double unreliability:double has_state:string = engine.validator.shardOverlayStats.Neighbour; engine.validator.shardOverlayStats shard:string mode:string neighbours:(vector engine.validator.shardOverlayStats.neighbour) = engine.validator.ShardOverlayStats; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 9680b01e9bd9d374e8925c5a05da6a11d883635f..c44ee331648d597810cdcbf77ecae5532dc399d3 100644 GIT binary patch delta 269 zcmexznf1yg)(uCbSwxObc|Tcat;*&%(no|O4fY&0|77?2{*k2v4q9E67`xT9jFw zpBJB-n3Z2N+0I&u4GD|X3 ziznJjZ+@a@!_GKmv!dDkVz9Q&EUQ$QVFnm*f=s%3Ovs0A^289~$rWn_c?yd1OY-B( zQj0Q+^QTW_Wt5&S;l!vic|$47-Efj;fEE&mVVMYuv?aVOkDw6{qv2tZ3mZioQmn4>?PJaJVmIb70bHc9{b;kDX xPZb##+iAk|fvhQVX4B!YZ99l=Z=wg=W69}-35?=0K%3xJp=;T`Ac1iU2LQPvem(#I diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index d64dff252..d70cea8fe 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -51,21 +51,10 @@ namespace fullnode { Neighbour Neighbour::zero = Neighbour{adnl::AdnlNodeIdShort::zero()}; -void Neighbour::update_proto_version(ton_api::tonNode_Capabilities &q) { - ton_api::downcast_call(q, td::overloaded( - [&](ton_api::tonNode_capabilities &x) { - proto_version = x.version_; - capabilities = x.capabilities_; - if (!supports_v2()) { - has_state_known = has_state = true; - } - }, - [&](ton_api::tonNode_capabilitiesV2 &x) { - proto_version = x.version_; - capabilities = x.capabilities_; - has_state_known = true; - has_state = x.have_state_; - })); +void Neighbour::update_proto_version(ton_api::tonNode_capabilities &q) { + version_major = q.version_major_; + version_minor = q.version_minor_; + flags = q.flags_; } void Neighbour::query_success(double t) { @@ -188,7 +177,7 @@ void FullNodeShardImpl::try_get_next_block(td::Timestamp timeout, td::Promise= 1) { + if (!b.adnl_id.is_zero() && b.version_major >= 1) { VLOG(FULL_NODE_DEBUG) << "using new download method with adnlid=" << b.adnl_id; td::actor::create_actor("downloadnext", adnl_id_, overlay_id_, handle_->id(), b.adnl_id, download_next_priority(), timeout, validator_manager_, rldp_, overlays_, @@ -631,14 +620,12 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, td::Promise promise) { VLOG(FULL_NODE_DEBUG) << "Got query getCapabilities from " << src; - promise.set_value(create_serialize_tl_object(proto_version(), proto_capabilities())); -} - -void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilitiesV2 &query, - td::Promise promise) { - VLOG(FULL_NODE_DEBUG) << "Got query getCapabilitiesV2 from " << src; - promise.set_value(create_serialize_tl_object(proto_version(), proto_capabilities(), - mode_ == FullNodeShardMode::active)); + td::uint32 flags = 0; + if (mode_ != FullNodeShardMode::active) { + flags |= Neighbour::FLAG_NO_STATE; + } + promise.set_value( + create_serialize_tl_object(proto_version_major(), proto_version_minor(), flags)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, @@ -878,7 +865,7 @@ void FullNodeShardImpl::send_broadcast(BlockBroadcast broadcast) { void FullNodeShardImpl::download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) { auto &b = choose_neighbour(); - if (!b.adnl_id.is_zero() && b.proto_version >= 1) { + if (!b.adnl_id.is_zero() && b.version_major >= 1) { VLOG(FULL_NODE_DEBUG) << "new block download"; td::actor::create_actor("downloadreq", id, adnl_id_, overlay_id_, b.adnl_id, priority, timeout, validator_manager_, rldp_, overlays_, adnl_, client_, @@ -1173,7 +1160,7 @@ void FullNodeShardImpl::got_neighbours(std::vector vec) { if (neighbours_.size() == max_neighbours()) { td::uint32 neighbours_with_state = 0; for (const auto &n : neighbours_) { - if (n.second.has_state) { + if (n.second.has_state_known() && n.second.has_state()) { ++neighbours_with_state; } } @@ -1183,7 +1170,8 @@ void FullNodeShardImpl::got_neighbours(std::vector vec) { td::uint32 cnt = 0; double u = 0; for (auto &n : neighbours_) { - if (neighbours_with_state <= min_neighbours_with_state() && n.second.has_state) { + if (neighbours_with_state <= min_neighbours_with_state() && n.second.has_state_known() && + n.second.has_state()) { continue; } if (n.second.unreliability > u) { @@ -1224,18 +1212,18 @@ const Neighbour &FullNodeShardImpl::choose_neighbour(bool require_state) const { for (auto &x : neighbours_) { if (require_state) { - if (attempt == 0 && !x.second.has_state) { + if (attempt == 0 && !(x.second.has_state_known() && x.second.has_state())) { continue; } - if (attempt == 1 && x.second.has_state_known) { + if (attempt == 1 && x.second.has_state_known()) { continue; } } auto unr = static_cast(x.second.unreliability - min_unreliability); - if (x.second.proto_version < proto_version()) { + if (x.second.version_major < proto_version_major()) { unr += 4; - } else if (x.second.proto_version == proto_version() && x.second.capabilities < proto_capabilities()) { + } else if (x.second.version_major == proto_version_major() && x.second.version_minor < proto_version_minor()) { unr += 2; } @@ -1272,7 +1260,7 @@ void FullNodeShardImpl::got_neighbour_capabilities(adnl::AdnlNodeIdShort adnl_id if (it == neighbours_.end()) { return; } - auto F = fetch_tl_object(std::move(data), true); + auto F = fetch_tl_object(std::move(data), true); if (F.is_error()) { it->second.query_failed(); } else { @@ -1305,12 +1293,7 @@ void FullNodeShardImpl::ping_neighbours() { td::Time::now() - start_time, R.move_as_ok()); } }); - td::BufferSlice q; - if (it->second.supports_v2()) { - q = create_serialize_tl_object(); - } else { - q = create_serialize_tl_object(); - } + td::BufferSlice q = create_serialize_tl_object(); td::actor::send_closure(overlays_, &overlay::Overlays::send_query, it->first, adnl_id_, overlay_id_, "get_prepare_block", std::move(P), td::Timestamp::in(1.0), std::move(q)); @@ -1338,11 +1321,12 @@ void FullNodeShardImpl::get_stats_extra(td::Promise promise) { const auto &n = p.second; auto f = create_tl_object(); f->id_ = n.adnl_id.bits256_value().to_hex(); - f->proto_verison_ = n.proto_version; - f->capabilities_ = n.capabilities; + f->verison_major_ = n.version_major; + f->version_minor_ = n.version_minor; + f->flags_ = n.flags; f->roundtrip_ = n.roundtrip; f->unreliability_ = n.unreliability; - f->has_state_ = (n.has_state_known ? (n.has_state ? "true" : "false") : "undefined"); + f->has_state_ = (n.has_state_known() ? (n.has_state() ? "true" : "false") : "undefined"); res->neighbours_.push_back(std::move(f)); } promise.set_result(td::json_encode(td::ToJson(*res), true)); diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index b1417e223..834fee857 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -31,30 +31,33 @@ namespace fullnode { struct Neighbour { adnl::AdnlNodeIdShort adnl_id; - td::uint32 proto_version = 0; - td::uint64 capabilities = 0; + td::uint32 version_major = 0; + td::uint32 version_minor = 0; + td::uint32 flags = 0; double roundtrip = 0; double roundtrip_relax_at = 0; double roundtrip_weight = 0; double unreliability = 0; - bool has_state_known = false; - bool has_state = false; Neighbour(adnl::AdnlNodeIdShort adnl_id) : adnl_id(std::move(adnl_id)) { } - void update_proto_version(ton_api::tonNode_Capabilities &q); + void update_proto_version(ton_api::tonNode_capabilities &q); void query_success(double t); void query_failed(); void update_roundtrip(double t); bool use_rldp2() const { - return std::make_pair(proto_version, capabilities) >= std::make_pair(2, 2); + return std::make_pair(version_major, version_minor) >= std::make_pair(2, 2); } - bool supports_v2() const { - return proto_version >= 3; + bool has_state() const { + return !(flags & FLAG_NO_STATE); + } + bool has_state_known() const { + return version_major != 0; } static Neighbour zero; + static constexpr td::uint32 FLAG_NO_STATE = 1; }; class FullNodeShardImpl : public FullNodeShard { @@ -72,11 +75,11 @@ class FullNodeShardImpl : public FullNodeShard { static constexpr td::uint32 download_next_priority() { return 1; } - static constexpr td::uint32 proto_version() { + static constexpr td::uint32 proto_version_major() { return 3; } - static constexpr td::uint64 proto_capabilities() { - return 2; + static constexpr td::uint32 proto_version_minor() { + return 0; } static constexpr td::uint32 max_neighbours() { return 16; @@ -146,8 +149,6 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, td::Promise promise); - void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilitiesV2 &query, - td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, @@ -216,7 +217,7 @@ class FullNodeShardImpl : public FullNodeShard { template td::Promise create_neighbour_promise(const Neighbour &x, td::Promise p, bool require_state = false) { return td::PromiseCreator::lambda([id = x.adnl_id, SelfId = actor_id(this), p = std::move(p), ts = td::Time::now(), - ignore_error = require_state && !x.has_state_known](td::Result R) mutable { + ignore_error = require_state && !x.has_state_known()](td::Result R) mutable { if (R.is_error() && R.error().code() != ErrorCode::notready && R.error().code() != ErrorCode::cancelled) { if (!ignore_error) { td::actor::send_closure(SelfId, &FullNodeShardImpl::update_neighbour_stats, id, td::Time::now() - ts, false); From 103eb9aad5d2ea72c3b2066e50b4f38968a7dd0d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 26 Mar 2024 12:50:01 +0300 Subject: [PATCH 087/388] Separate limits for collated data --- .github/workflows/macos-11.7-compile.yml | 0 crypto/block/block.cpp | 25 ++++++++++++++++-------- crypto/block/block.h | 2 +- crypto/block/block.tlb | 2 +- crypto/vm/boc.h | 2 +- 5 files changed, 20 insertions(+), 11 deletions(-) delete mode 100644 .github/workflows/macos-11.7-compile.yml diff --git a/.github/workflows/macos-11.7-compile.yml b/.github/workflows/macos-11.7-compile.yml deleted file mode 100644 index e69de29bb..000000000 diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp index cb9344173..d17f25976 100644 --- a/crypto/block/block.cpp +++ b/crypto/block/block.cpp @@ -678,10 +678,19 @@ bool BlockLimits::deserialize(vm::CellSlice& cs) { } // block_limits#5d // block_limits_v2#5e - return bytes.deserialize(cs) // bytes:ParamLimits - && gas.deserialize(cs) // gas:ParamLimits - && lt_delta.deserialize(cs) // lt_delta:ParamLimits - && (tag == 0x5d || imported_msg_queue.deserialize(cs)); // imported_msg_queue:ImportedMsgQueueLimits + bool ok = bytes.deserialize(cs) // bytes:ParamLimits + && gas.deserialize(cs) // gas:ParamLimits + && lt_delta.deserialize(cs); // lt_delta:ParamLimits + if (!ok) { + return false; + } + if (tag == 0x5d) { + return collated_data.deserialize(cs) && // collated_data:ParamLimits + imported_msg_queue.deserialize(cs); // imported_msg_queue:ImportedMsgQueueLimits + } else { + collated_data = bytes; + return true; + } } int ParamLimits::classify(td::uint64 value) const { @@ -714,7 +723,7 @@ int BlockLimits::classify_lt(ton::LogicalTime lt) const { } int BlockLimits::classify_collated_data_size(td::uint64 size) const { - return bytes.classify(size); // TODO: Maybe separate limits in config + return collated_data.classify(size); } int BlockLimits::classify(td::uint64 size, td::uint64 gas, ton::LogicalTime lt, td::uint64 collated_size) const { @@ -725,7 +734,7 @@ int BlockLimits::classify(td::uint64 size, td::uint64 gas, ton::LogicalTime lt, bool BlockLimits::fits(unsigned cls, td::uint64 size, td::uint64 gas_value, ton::LogicalTime lt, td::uint64 collated_size) const { return bytes.fits(cls, size) && gas.fits(cls, gas_value) && lt_delta.fits(cls, lt - start_lt) && - bytes.fits(cls, collated_size); + collated_data.fits(cls, collated_size); } td::uint64 BlockLimitStatus::estimate_block_size(const vm::NewCellStorageStat::Stat* extra) const { @@ -745,7 +754,7 @@ bool BlockLimitStatus::fits(unsigned cls) const { return cls >= ParamLimits::limits_cnt || (limits.gas.fits(cls, gas_used) && limits.lt_delta.fits(cls, cur_lt - limits.start_lt) && limits.bytes.fits(cls, estimate_block_size()) && - limits.bytes.fits(cls, collated_data_stat.estimate_proof_size())); + limits.collated_data.fits(cls, collated_data_stat.estimate_proof_size())); } bool BlockLimitStatus::would_fit(unsigned cls, ton::LogicalTime end_lt, td::uint64 more_gas, @@ -753,7 +762,7 @@ bool BlockLimitStatus::would_fit(unsigned cls, ton::LogicalTime end_lt, td::uint return cls >= ParamLimits::limits_cnt || (limits.gas.fits(cls, gas_used + more_gas) && limits.lt_delta.fits(cls, std::max(cur_lt, end_lt) - limits.start_lt) && limits.bytes.fits(cls, estimate_block_size(extra)) && - limits.bytes.fits(cls, collated_data_stat.estimate_proof_size())); + limits.collated_data.fits(cls, collated_data_stat.estimate_proof_size())); } // SETS: account_dict, shard_libraries_, mc_state_extra diff --git a/crypto/block/block.h b/crypto/block/block.h index 61b057c44..603265436 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -255,7 +255,7 @@ struct ParamLimits { }; struct BlockLimits { - ParamLimits bytes, gas, lt_delta; + ParamLimits bytes, gas, lt_delta, collated_data; ton::LogicalTime start_lt{0}; ImportedMsgQueueLimits imported_msg_queue; const vm::CellUsageTree* usage_tree{nullptr}; diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 6aababa1a..e05b50180 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -708,7 +708,7 @@ imported_msg_queue_limits#d3 max_bytes:# max_msgs:# = ImportedMsgQueueLimits; block_limits#5d bytes:ParamLimits gas:ParamLimits lt_delta:ParamLimits = BlockLimits; block_limits_v2#5e bytes:ParamLimits gas:ParamLimits lt_delta:ParamLimits - imported_msg_queue:ImportedMsgQueueLimits + collated_data:ParamLimits imported_msg_queue:ImportedMsgQueueLimits = BlockLimits; config_mc_block_limits#_ BlockLimits = ConfigParam 22; diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index e9feb71e9..f81928cd1 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -41,7 +41,6 @@ class NewCellStorageStat { Stat(td::uint64 cells_, td::uint64 bits_, td::uint64 internal_refs_ = 0, td::uint64 external_refs_ = 0) : cells(cells_), bits(bits_), internal_refs(internal_refs_), external_refs(external_refs_) { } - Stat(const Stat&) = default; td::uint64 cells{0}; td::uint64 bits{0}; td::uint64 internal_refs{0}; @@ -53,6 +52,7 @@ class NewCellStorageStat { bool operator==(const Stat& other) const { return key() == other.key(); } + Stat(const Stat& other) = default; Stat& operator=(const Stat& other) = default; Stat& operator+=(const Stat& other) { cells += other.cells; From 036ce697869459e65ad4cfe19058048cb8f4853a Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 26 Mar 2024 16:22:52 +0300 Subject: [PATCH 088/388] Use compression in new private overlays --- validator/full-node-private-overlay-v2.cpp | 41 +++++++++++----------- validator/full-node-private-overlay-v2.hpp | 3 ++ 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/validator/full-node-private-overlay-v2.cpp b/validator/full-node-private-overlay-v2.cpp index fedb8a788..a356825e2 100644 --- a/validator/full-node-private-overlay-v2.cpp +++ b/validator/full-node-private-overlay-v2.cpp @@ -22,23 +22,24 @@ #include "td/utils/JsonBuilder.h" #include "tl/tl_json.h" #include "auto/tl/ton_api_json.h" +#include "full-node-serializer.hpp" namespace ton::validator::fullnode { -void FullNodePrivateOverlayV2::process_broadcast(PublicKeyHash, ton_api::tonNode_blockBroadcast &query) { - std::vector signatures; - for (auto &sig : query.signatures_) { - signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)}); - } +void FullNodePrivateOverlayV2::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { + process_block_broadcast(src, query); +} - BlockIdExt block_id = create_block_id(query.id_); - BlockBroadcast B{block_id, - std::move(signatures), - static_cast(query.catchain_seqno_), - static_cast(query.validator_set_hash_), - std::move(query.data_), - std::move(query.proof_)}; +void FullNodePrivateOverlayV2::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query) { + process_block_broadcast(src, query); +} +void FullNodePrivateOverlayV2::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { + auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); + if (B.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << B.move_as_error(); + return; + } auto P = td::PromiseCreator::lambda([](td::Result R) { if (R.is_error()) { if (R.error().code() == ErrorCode::notready) { @@ -48,8 +49,8 @@ void FullNodePrivateOverlayV2::process_broadcast(PublicKeyHash, ton_api::tonNode } } }); - LOG(FULL_NODE_DEBUG) << "Got block broadcast in private overlay: " << B.block_id.to_str(); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, std::move(B), + LOG(FULL_NODE_DEBUG) << "Got block broadcast in private overlay: " << B.ok().block_id.to_str(); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, B.move_as_ok(), std::move(P)); } @@ -89,15 +90,13 @@ void FullNodePrivateOverlayV2::send_broadcast(BlockBroadcast broadcast) { if (!inited_) { return; } - std::vector> sigs; - for (auto &sig : broadcast.signatures) { - sigs.emplace_back(create_tl_object(sig.node, sig.signature.clone())); + auto B = serialize_block_broadcast(broadcast, false); // compression_enabled = false + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error(); + return; } - auto B = create_serialize_tl_object( - create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, std::move(sigs), - broadcast.proof.clone(), broadcast.data.clone()); td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, - local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); } void FullNodePrivateOverlayV2::start_up() { diff --git a/validator/full-node-private-overlay-v2.hpp b/validator/full-node-private-overlay-v2.hpp index 6fbcbbf71..a48dc4ccf 100644 --- a/validator/full-node-private-overlay-v2.hpp +++ b/validator/full-node-private-overlay-v2.hpp @@ -23,6 +23,9 @@ namespace ton::validator::fullnode { class FullNodePrivateOverlayV2 : public td::actor::Actor { public: void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query); + void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); template void process_broadcast(PublicKeyHash, T &) { From f5cedc3b6edadd00795cc9d198e85b72b130784e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 29 Mar 2024 15:04:07 +0300 Subject: [PATCH 089/388] Enable compression in private overlays v2 and in collator node --- tl/generate/scheme/ton_api.tl | 4 +- tl/generate/scheme/ton_api.tlo | Bin 92716 -> 93212 bytes validator-session/candidate-serializer.cpp | 57 +++++++++------ validator-session/candidate-serializer.h | 7 +- validator/collator-node.cpp | 78 ++++++++++++++++----- validator/collator-node.hpp | 13 ++-- validator/full-node-private-overlay-v2.cpp | 2 +- validator/validator-group.cpp | 32 ++++----- 8 files changed, 128 insertions(+), 65 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 4b038f6fb..cafc09260 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -789,7 +789,9 @@ validatorSession.stats success:Bool id:tonNode.blockIdExt timestamp:long self:in signatures:int signatures_weight:long approve_signatures:int approve_signatures_weight:long first_round:int rounds:(vector validatorSession.statsRound) = validatorSession.Stats; -collatorNode.generateBlockSuccess candidate:db.Candidate = collatorNode.GenerateBlockResult; +collatorNode.candidate source:PublicKey id:tonNode.blockIdExt data:bytes collated_data:bytes = collatorNode.Candidate; +collatorNode.compressedCandidate flags:# source:PublicKey id:tonNode.blockIdExt decompressed_size:int data:bytes = collatorNode.Candidate; +collatorNode.generateBlockSuccess candidate:collatorNode.Candidate = collatorNode.GenerateBlockResult; collatorNode.generateBlockError code:int message:string = collatorNode.GenerateBlockResult; ---functions--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index de65fb4086c32897f552c854bb1b9e970a2e498d..b3c6bf820a2fc331f453baff383bdcdd29dece7b 100644 GIT binary patch delta 270 zcmZ2;g>}viR^CUm^{p77KxiW`H;14P_olNTB*v;6CfGu|x2 zA)&`OW3#E5vz`L`4_gHkWqQepc`2DGi6yBFAR{JaOi={UFvG9Pam9Fl{=QQI+34vH z`547HLCS6(6Y^o5EEuRgc|oqt^d5f301nOs+{$;RLw_q@Hp5LPbVt31oFe t&TKjywrvMNN+#b)F#x-JvOsD8!~-d*EC<1?={ahQ7r?CT4(g0=oB;Q;TeAQF delta 133 zcmbPpgLTanR^CUm^{p77KyV{3HwViRgS_jT12`n~7^iK{G;`KtiMx7MXL3%O3MWYB z<}o23#_1E48Kos|W!+4LX)SVQ)8VjfI|x!T`AnJt*px}>0T9zd(pe6IS<_=Q7%zZX L+YK}s-#7sPVva8I diff --git a/validator-session/candidate-serializer.cpp b/validator-session/candidate-serializer.cpp index c220b4a95..c2c0a81af 100644 --- a/validator-session/candidate-serializer.cpp +++ b/validator-session/candidate-serializer.cpp @@ -23,27 +23,15 @@ namespace ton::validatorsession { -td::Result serialize_candidate(const tl_object_ptr &block, +td::Result serialize_candidate(const tl_object_ptr& block, bool compression_enabled) { if (!compression_enabled) { return serialize_tl_object(block, true); } - vm::BagOfCells boc1, boc2; - TRY_STATUS(boc1.deserialize(block->data_)); - if (boc1.get_root_count() != 1) { - return td::Status::Error("block candidate should have exactly one root"); - } - std::vector> roots = {boc1.get_root_cell()}; - TRY_STATUS(boc2.deserialize(block->collated_data_)); - for (int i = 0; i < boc2.get_root_count(); ++i) { - roots.push_back(boc2.get_root_cell(i)); - } - TRY_RESULT(data, vm::std_boc_serialize_multi(std::move(roots), 2)); - td::BufferSlice compressed = td::lz4_compress(data); - LOG(VALIDATOR_SESSION_DEBUG) << "Compressing block candidate: " << block->data_.size() + block->collated_data_.size() - << " -> " << compressed.size(); + size_t decompressed_size; + TRY_RESULT(compressed, compress_candidate_data(block->data_, block->collated_data_, decompressed_size)) return create_serialize_tl_object( - 0, block->src_, block->round_, block->root_hash_, (int)data.size(), std::move(compressed)); + 0, block->src_, block->round_, block->root_hash_, (int)decompressed_size, std::move(compressed)); } td::Result> deserialize_candidate(td::Slice data, @@ -56,8 +44,34 @@ td::Result> deserialize_candi if (f->decompressed_size_ > max_decompressed_data_size) { return td::Status::Error("decompressed size is too big"); } - TRY_RESULT(decompressed, td::lz4_decompress(f->data_, f->decompressed_size_)); - if (decompressed.size() != (size_t)f->decompressed_size_) { + TRY_RESULT(p, decompress_candidate_data(f->data_, f->decompressed_size_)); + return create_tl_object(f->src_, f->round_, f->root_hash_, std::move(p.first), + std::move(p.second)); +} + +td::Result compress_candidate_data(td::Slice block, td::Slice collated_data, + size_t& decompressed_size) { + vm::BagOfCells boc1, boc2; + TRY_STATUS(boc1.deserialize(block)); + if (boc1.get_root_count() != 1) { + return td::Status::Error("block candidate should have exactly one root"); + } + std::vector> roots = {boc1.get_root_cell()}; + TRY_STATUS(boc2.deserialize(collated_data)); + for (int i = 0; i < boc2.get_root_count(); ++i) { + roots.push_back(boc2.get_root_cell(i)); + } + TRY_RESULT(data, vm::std_boc_serialize_multi(std::move(roots), 2)); + decompressed_size = data.size(); + td::BufferSlice compressed = td::lz4_compress(data); + LOG(DEBUG) << "Compressing block candidate: " << block.size() + collated_data.size() << " -> " << compressed.size(); + return compressed; +} + +td::Result> decompress_candidate_data(td::Slice compressed, + int decompressed_size) { + TRY_RESULT(decompressed, td::lz4_decompress(compressed, decompressed_size)); + if (decompressed.size() != (size_t)decompressed_size) { return td::Status::Error("decompressed size mismatch"); } TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed)); @@ -67,10 +81,9 @@ td::Result> deserialize_candi TRY_RESULT(block_data, vm::std_boc_serialize(roots[0], 31)); roots.erase(roots.begin()); TRY_RESULT(collated_data, vm::std_boc_serialize_multi(std::move(roots), 31)); - LOG(VALIDATOR_SESSION_DEBUG) << "Decompressing block candidate: " << f->data_.size() << " -> " - << block_data.size() + collated_data.size(); - return create_tl_object(f->src_, f->round_, f->root_hash_, std::move(block_data), - std::move(collated_data)); + LOG(DEBUG) << "Decompressing block candidate: " << compressed.size() << " -> " + << block_data.size() + collated_data.size(); + return std::make_pair(std::move(block_data), std::move(collated_data)); } } // namespace ton::validatorsession diff --git a/validator-session/candidate-serializer.h b/validator-session/candidate-serializer.h index 030a412c0..e88376cd7 100644 --- a/validator-session/candidate-serializer.h +++ b/validator-session/candidate-serializer.h @@ -20,10 +20,15 @@ namespace ton::validatorsession { -td::Result serialize_candidate(const tl_object_ptr &block, +td::Result serialize_candidate(const tl_object_ptr& block, bool compression_enabled); td::Result> deserialize_candidate(td::Slice data, bool compression_enabled, int max_decompressed_data_size); +td::Result compress_candidate_data(td::Slice block, td::Slice collated_data, + size_t& decompressed_size); +td::Result> decompress_candidate_data(td::Slice compressed, + int decompressed_size); + } // namespace ton::validatorsession diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 9bfd857ad..ce3c4be4b 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -19,10 +19,11 @@ #include "fabric.h" #include "block-auto.h" #include "block-db.h" +#include "td/utils/lz4.h" +#include "checksum.h" +#include "validator-session/candidate-serializer.h" -namespace ton { - -namespace validator { +namespace ton::validator { CollatorNode::CollatorNode(adnl::AdnlNodeIdShort local_id, td::actor::ActorId manager, td::actor::ActorId adnl, td::actor::ActorId rldp) @@ -110,12 +111,6 @@ static td::BufferSlice serialize_error(td::Status error) { return create_serialize_tl_object(error.code(), error.message().c_str()); } -static td::BufferSlice serialize_response(BlockCandidate block) { - return create_serialize_tl_object(create_tl_object( - PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), std::move(block.data), - std::move(block.collated_data))); -} - static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey creator) { CHECK(!block.id.is_masterchain()); if (block.pubkey == creator) { @@ -145,7 +140,8 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data promise.set_result(serialize_error(R.move_as_error())); } else { LOG(INFO) << "Query from " << src << ", success"; - promise.set_result(serialize_response(R.move_as_ok())); + promise.set_result(create_serialize_tl_object( + serialize_candidate(R.move_as_ok(), true))); } }; if (!last_masterchain_block_.is_valid()) { @@ -263,14 +259,62 @@ void CollatorNode::process_result(std::shared_ptr cache_entry, td::R } bool CollatorNode::can_collate_shard(ShardIdFull shard) const { - for (ShardIdFull our_shard : shards_) { - if (shard_intersects(shard, our_shard)) { - return true; - } + return std::any_of(shards_.begin(), shards_.end(), + [&](const ShardIdFull& our_shard) { return shard_intersects(shard, our_shard); }); +} + +tl_object_ptr CollatorNode::serialize_candidate(const BlockCandidate& block, + bool compress) { + if (!compress) { + return create_tl_object( + PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), block.data.clone(), + block.collated_data.clone()); } - return false; + size_t decompressed_size; + td::BufferSlice compressed = + validatorsession::compress_candidate_data(block.data, block.collated_data, decompressed_size).move_as_ok(); + return create_tl_object( + 0, PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), + (int)decompressed_size, std::move(compressed)); } -} // namespace validator +td::Result CollatorNode::deserialize_candidate(tl_object_ptr f, + int max_decompressed_data_size) { + td::Result res; + ton_api::downcast_call(*f, td::overloaded( + [&](ton_api::collatorNode_candidate& c) { + res = [&]() -> td::Result { + auto hash = td::sha256_bits256(c.collated_data_); + auto key = ton::PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), hash, std::move(c.data_), + std::move(c.collated_data_)}; + }(); + }, + [&](ton_api::collatorNode_compressedCandidate& c) { + res = [&]() -> td::Result { + if (c.decompressed_size_ <= 0) { + return td::Status::Error("invalid decompressed size"); + } + if (c.decompressed_size_ > max_decompressed_data_size) { + return td::Status::Error("decompressed size is too big"); + } + TRY_RESULT( + p, validatorsession::decompress_candidate_data(c.data_, c.decompressed_size_)); + auto collated_data_hash = td::sha256_bits256(p.second); + auto key = ton::PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, + std::move(p.first), std::move(p.second)}; + }(); + })); + return res; +} -} // namespace ton +} // namespace ton::validator diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp index d640244a7..ca2fd73d4 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node.hpp @@ -20,9 +20,7 @@ #include "rldp/rldp.h" #include -namespace ton { - -namespace validator { +namespace ton::validator { class ValidatorManager; @@ -77,8 +75,11 @@ class CollatorNode : public td::actor::Actor { } void process_result(std::shared_ptr cache_entry, td::Result R); -}; -} // namespace validator + public: + static tl_object_ptr serialize_candidate(const BlockCandidate& block, bool compress); + static td::Result deserialize_candidate(tl_object_ptr f, + int max_decompressed_data_size); +}; -} // namespace ton +} // namespace ton::validator diff --git a/validator/full-node-private-overlay-v2.cpp b/validator/full-node-private-overlay-v2.cpp index a356825e2..402c8d093 100644 --- a/validator/full-node-private-overlay-v2.cpp +++ b/validator/full-node-private-overlay-v2.cpp @@ -90,7 +90,7 @@ void FullNodePrivateOverlayV2::send_broadcast(BlockBroadcast broadcast) { if (!inited_) { return; } - auto B = serialize_block_broadcast(broadcast, false); // compression_enabled = false + auto B = serialize_block_broadcast(broadcast, true); // compression_enabled = true if (B.is_error()) { VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error(); return; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 2ceffae55..68ac6d570 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -24,6 +24,7 @@ #include "ton/lite-tl.hpp" #include "ton/ton-tl.hpp" #include "td/utils/Random.h" +#include "collator-node.hpp" namespace ton { @@ -506,7 +507,7 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo std::move(promise)); }); LOG(INFO) << "sending collate query for " << next_block_id.to_str() << ": send to " << collator; - size_t max_answer_size = config_.max_block_size + config_.max_collated_data_size + 256; + size_t max_answer_size = config_.max_block_size + config_.max_collated_data_size + 1024; td::Timestamp query_timeout = td::Timestamp::in(10.0); query_timeout.relax(timeout); td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_adnl_id_, collator, "collatequery", std::move(P), @@ -520,29 +521,26 @@ void ValidatorGroup::receive_collate_query_response(td::uint32 round_id, td::Buf return; } TRY_RESULT_PROMISE(promise, f, fetch_tl_object(data, true)); - tl_object_ptr b; + td::Result res; ton_api::downcast_call(*f, td::overloaded( - [&](ton_api::collatorNode_generateBlockError &r) { - td::Status error = td::Status::Error(r.code_, r.message_); - promise.set_error(error.move_as_error_prefix("collate query: ")); - }, - [&](ton_api::collatorNode_generateBlockSuccess &r) { b = std::move(r.candidate_); })); - if (!b) { - return; - } - auto key = PublicKey{b->source_}; - if (key != local_id_full_) { + [&](ton_api::collatorNode_generateBlockError &r) { + td::Status error = td::Status::Error(r.code_, r.message_); + res = error.move_as_error_prefix("collate query: "); + }, + [&](ton_api::collatorNode_generateBlockSuccess &r) { + res = CollatorNode::deserialize_candidate( + std::move(r.candidate_), + config_.max_block_size + config_.max_collated_data_size + 1024); + })); + TRY_RESULT_PROMISE(promise, candidate, std::move(res)); + if (candidate.pubkey.as_bits256() != local_id_full_.ed25519_value().raw()) { promise.set_error(td::Status::Error("collate query: block candidate source mismatch")); return; } - auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; - auto block_id = ton::create_block_id(b->id_); - if (block_id.shard_full() != shard_) { + if (candidate.id.shard_full() != shard_) { promise.set_error(td::Status::Error("collate query: shard mismatch")); return; } - auto collated_data_hash = td::sha256_bits256(b->collated_data_); - BlockCandidate candidate(e_key, block_id, collated_data_hash, std::move(b->data_), std::move(b->collated_data_)); auto P = td::PromiseCreator::lambda( [candidate = candidate.clone(), promise = std::move(promise)](td::Result R) mutable { From 86d3ed97c1dd06db97d3cbd8b4fd801fb1794b16 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 30 May 2024 18:19:38 +0300 Subject: [PATCH 090/388] Fix parsing block limits config --- crypto/block/block.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp index d17f25976..fdbe8e20e 100644 --- a/crypto/block/block.cpp +++ b/crypto/block/block.cpp @@ -684,7 +684,7 @@ bool BlockLimits::deserialize(vm::CellSlice& cs) { if (!ok) { return false; } - if (tag == 0x5d) { + if (tag == 0x5e) { return collated_data.deserialize(cs) && // collated_data:ParamLimits imported_msg_queue.deserialize(cs); // imported_msg_queue:ImportedMsgQueueLimits } else { From c129a784c6b3930f40bbaa213fb57c35fbb95a70 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 30 May 2024 18:57:50 +0300 Subject: [PATCH 091/388] Prepare transition between old and new private overlays --- validator/full-node-private-overlay-v2.cpp | 26 +++++++----- validator/full-node-private-overlay-v2.hpp | 3 +- validator/full-node.cpp | 48 +++++++++++++--------- validator/full-node.h | 5 ++- validator/full-node.hpp | 17 ++++---- validator/manager.cpp | 4 ++ 6 files changed, 61 insertions(+), 42 deletions(-) diff --git a/validator/full-node-private-overlay-v2.cpp b/validator/full-node-private-overlay-v2.cpp index 8dd7f6dd7..20e9cc55f 100644 --- a/validator/full-node-private-overlay-v2.cpp +++ b/validator/full-node-private-overlay-v2.cpp @@ -239,7 +239,7 @@ void FullNodePrivateOverlayV2::get_stats_extra(td::Promise promise) promise.set_result(td::json_encode(td::ToJson(*res), true)); } -td::actor::ActorId FullNodePrivateBlockOverlays::choose_overlay(ShardIdFull shard) { +td::actor::ActorId FullNodePrivateBlockOverlaysV2::choose_overlay(ShardIdFull shard) { for (auto &p : id_to_overlays_) { auto &overlays = p.second.overlays_; ShardIdFull cur_shard = shard; @@ -257,7 +257,7 @@ td::actor::ActorId FullNodePrivateBlockOverlays::choos return {}; } -void FullNodePrivateBlockOverlays::update_overlays( +void FullNodePrivateBlockOverlaysV2::update_overlays( td::Ref state, std::set my_adnl_ids, const FileHash &zero_state_file_hash, const td::actor::ActorId &keyring, const td::actor::ActorId &adnl, const td::actor::ActorId &rldp, const td::actor::ActorId &rldp2, @@ -274,10 +274,10 @@ void FullNodePrivateBlockOverlays::update_overlays( struct OverlayInfo { std::vector nodes, senders; }; - std::map pverlay_infos; + std::map overlay_infos; // Masterchain overlay: all validators + collators - OverlayInfo &mc_overlay = pverlay_infos[ShardIdFull(masterchainId)]; + OverlayInfo &mc_overlay = overlay_infos[ShardIdFull(masterchainId)]; for (const auto &x : all_validators->export_vector()) { td::Bits256 addr = x.addr.is_zero() ? ValidatorFullId(x.key).compute_short_id().bits256_value() : x.addr; mc_overlay.nodes.emplace_back(addr); @@ -311,14 +311,14 @@ void FullNodePrivateBlockOverlays::update_overlays( auto val_set = state->get_validator_set(shard); td::uint32 min_split = state->monitor_min_split_depth(shard.workchain); OverlayInfo &overlay = - pverlay_infos[shard_prefix_length(shard) <= min_split ? shard : shard_prefix(shard, min_split)]; + overlay_infos[shard_prefix_length(shard) <= min_split ? shard : shard_prefix(shard, min_split)]; for (const auto &x : val_set->export_vector()) { td::Bits256 addr = x.addr.is_zero() ? ValidatorFullId(x.key).compute_short_id().bits256_value() : x.addr; overlay.nodes.emplace_back(addr); overlay.senders.emplace_back(addr); } } - for (auto &p : pverlay_infos) { + for (auto &p : overlay_infos) { ShardIdFull shard = p.first; OverlayInfo &overlay = p.second; if (!shard.is_masterchain()) { @@ -338,7 +338,7 @@ void FullNodePrivateBlockOverlays::update_overlays( std::map old_private_block_overlays = std::move(id_to_overlays_); id_to_overlays_.clear(); - for (const auto &p : pverlay_infos) { + for (const auto &p : overlay_infos) { ShardIdFull shard = p.first; const OverlayInfo &new_overlay_info = p.second; for (adnl::AdnlNodeIdShort local_id : new_overlay_info.nodes) { @@ -356,8 +356,9 @@ void FullNodePrivateBlockOverlays::update_overlays( new_overlay.senders_ = new_overlay_info.senders; new_overlay.is_sender_ = std::binary_search(new_overlay.senders_.begin(), new_overlay.senders_.end(), local_id); new_overlay.overlay_ = td::actor::create_actor( - "BlocksPrivateOverlay", local_id, shard, new_overlay.nodes_, new_overlay.senders_, zero_state_file_hash, - keyring, adnl, rldp, rldp2, overlays, validator_manager, full_node); + PSTRING() << "BlocksPrivateOverlay" << shard.to_str(), local_id, shard, new_overlay.nodes_, + new_overlay.senders_, zero_state_file_hash, keyring, adnl, rldp, rldp2, overlays, validator_manager, + full_node); } } } @@ -370,9 +371,14 @@ void FullNodePrivateBlockOverlays::update_overlays( } td::actor::ActorId id = x.second.overlay_.release(); delay_action([id = std::move(id)]() { td::actor::send_closure(id, &FullNodePrivateOverlayV2::destroy); }, - td::Timestamp::in(60.0)); + td::Timestamp::in(30.0)); } } } +void FullNodePrivateBlockOverlaysV2::destroy_overlays() { + id_to_overlays_.clear(); +} + + } // namespace ton::validator::fullnode diff --git a/validator/full-node-private-overlay-v2.hpp b/validator/full-node-private-overlay-v2.hpp index 9a6ec932d..abb88fb43 100644 --- a/validator/full-node-private-overlay-v2.hpp +++ b/validator/full-node-private-overlay-v2.hpp @@ -96,7 +96,7 @@ class FullNodePrivateOverlayV2 : public td::actor::Actor { void get_stats_extra(td::Promise promise); }; -class FullNodePrivateBlockOverlays { +class FullNodePrivateBlockOverlaysV2 { public: td::actor::ActorId choose_overlay(ShardIdFull shard); void update_overlays(td::Ref state, std::set my_adnl_ids, @@ -106,6 +106,7 @@ class FullNodePrivateBlockOverlays { const td::actor::ActorId& overlays, const td::actor::ActorId& validator_manager, const td::actor::ActorId& full_node); + void destroy_overlays(); private: struct Overlays { diff --git a/validator/full-node.cpp b/validator/full-node.cpp index e5ffa9c9d..a6eb121fb 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -37,7 +37,7 @@ void FullNodeImpl::add_permanent_key(PublicKeyHash key, td::Promise pr } local_keys_.insert(key); - // create_private_block_overlay(key); + create_private_block_overlay(key); for (auto &p : custom_overlays_) { update_custom_overlay(p.second); } @@ -67,7 +67,7 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise pr return; } local_keys_.erase(key); - // private_block_overlays_.erase(key); + private_block_overlays_.erase(key); for (auto &p : custom_overlays_) { update_custom_overlay(p.second); } @@ -151,9 +151,9 @@ void FullNodeImpl::set_config(FullNodeConfig config) { td::actor::send_closure(s.second.actor, &FullNodeShard::set_config, config); } } - /*for (auto& overlay : private_block_overlays_) { + for (auto& overlay : private_block_overlays_) { td::actor::send_closure(overlay.second, &FullNodePrivateBlockOverlay::set_config, config); - }*/ + } for (auto& overlay : custom_overlays_) { for (auto &actor : overlay.second.actors_) { td::actor::send_closure(actor.second, &FullNodeCustomOverlay::set_config, config); @@ -286,8 +286,12 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s my_adnl_ids.insert(it->second); } } - private_block_overlays_.update_overlays(state, std::move(my_adnl_ids), zero_state_file_hash_, keyring_, adnl_, rldp_, - rldp2_, overlays_, validator_manager_, actor_id(this)); + if (use_old_private_overlays_) { + private_block_overlays_v2_.destroy_overlays(); + } else { + private_block_overlays_v2_.update_overlays(state, std::move(my_adnl_ids), zero_state_file_hash_, keyring_, adnl_, + rldp_, rldp2_, overlays_, validator_manager_, actor_id(this)); + } } void FullNodeImpl::add_shard_actor(ShardIdFull shard, FullNodeShardMode mode) { @@ -340,11 +344,11 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; return; } - /*if (!private_block_overlays_.empty()) { + if (!private_block_overlays_.empty()) { td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_shard_block_info, block_id, cc_seqno, data.clone()); - }*/ - auto private_overlay = private_block_overlays_.choose_overlay(ShardIdFull(masterchainId)); + } + auto private_overlay = private_block_overlays_v2_.choose_overlay(ShardIdFull(masterchainId)); if (!private_overlay.empty()) { td::actor::send_closure(private_overlay, &FullNodePrivateOverlayV2::send_shard_block_info, block_id, cc_seqno, data.clone()); @@ -360,11 +364,11 @@ void FullNodeImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_se VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; return; } - /*if (!private_block_overlays_.empty()) { + if (!private_block_overlays_.empty()) { td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_block_candidate, block_id, cc_seqno, validator_set_hash, data.clone()); - }*/ - auto private_overlay = private_block_overlays_.choose_overlay(block_id.shard_full()); + } + auto private_overlay = private_block_overlays_v2_.choose_overlay(block_id.shard_full()); if (!private_overlay.empty()) { td::actor::send_closure(private_overlay, &FullNodePrivateOverlayV2::send_block_candidate, block_id, cc_seqno, validator_set_hash, data.clone()); @@ -385,11 +389,11 @@ void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, bool custom_overlays VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; return; } - /*if (broadcast.block_id.is_masterchain() && !private_block_overlays_.empty()) { + if (broadcast.block_id.is_masterchain() && !private_block_overlays_.empty()) { td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_broadcast, broadcast.clone()); - }*/ - auto private_overlay = private_block_overlays_.choose_overlay(broadcast.block_id.shard_full()); + } + auto private_overlay = private_block_overlays_v2_.choose_overlay(broadcast.block_id.shard_full()); if (!private_overlay.empty()) { td::actor::send_closure(private_overlay, &FullNodePrivateOverlayV2::send_broadcast, broadcast.clone()); } @@ -550,7 +554,7 @@ void FullNodeImpl::got_key_block_state(td::Ref state) { } } - // set_private_block_overlays_enable_compression(m->get_consensus_config().proto_version >= 3); + set_private_block_overlays_enable_compression(m->get_consensus_config().proto_version >= 3); if (current_validators != current_validators_) { current_validators_ = std::move(current_validators); @@ -703,16 +707,17 @@ void FullNodeImpl::update_private_overlays() { for (auto &p : custom_overlays_) { update_custom_overlay(p.second); } - /*private_block_overlays_.clear(); + + private_block_overlays_.clear(); if (local_keys_.empty()) { return; } for (const auto &key : local_keys_) { create_private_block_overlay(key); - }*/ + } } -/*void FullNodeImpl::set_private_block_overlays_enable_compression(bool value) { +void FullNodeImpl::set_private_block_overlays_enable_compression(bool value) { if (private_block_overlays_enable_compression_ == value) { return; } @@ -723,6 +728,9 @@ void FullNodeImpl::update_private_overlays() { } void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { + if (!use_old_private_overlays_) { + return; + } CHECK(local_keys_.count(key)); if (current_validators_.count(key)) { std::vector nodes; @@ -734,7 +742,7 @@ void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { private_block_overlays_enable_compression_, keyring_, adnl_, rldp_, rldp2_, overlays_, validator_manager_, actor_id(this)); } -}*/ +} void FullNodeImpl::update_custom_overlay(CustomOverlayInfo &overlay) { auto old_actors = std::move(overlay.actors_); diff --git a/validator/full-node.h b/validator/full-node.h index 477d961e5..0545dfb98 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -75,8 +75,9 @@ class FullNode : public td::actor::Actor { virtual void add_collator_adnl_id(adnl::AdnlNodeIdShort id) = 0; virtual void del_collator_adnl_id(adnl::AdnlNodeIdShort id) = 0; - virtual void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, td::uint32 expiry_at, - td::uint32 max_size, td::Promise promise) = 0; + virtual void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, + td::uint32 expiry_at, td::uint32 max_size, + td::Promise promise) = 0; virtual void import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) = 0; diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 1a172993a..16f6fffbe 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -144,14 +144,14 @@ class FullNodeImpl : public FullNode { td::Promise started_promise_; FullNodeConfig config_; - // TODO: Decide what to do with old private overlays. Maybe use old or new depending on some flag in config. - /* - std::map> private_block_overlays_; + // Private overlays: + // Old overlays - one overlay for all validators + // New overlays (v2) - overlay per shard (monitor_min_split depth). + bool use_old_private_overlays_ = false; // TODO: set from config + std::map> private_block_overlays_; bool private_block_overlays_enable_compression_ = false; - void set_private_block_overlays_enable_compression(bool value); - void create_private_block_overlay(PublicKeyHash key); - */ bool broadcast_block_candidates_in_public_overlay_ = false; + FullNodePrivateBlockOverlaysV2 private_block_overlays_v2_; struct CustomOverlayInfo { CustomOverlayParams params_; @@ -162,13 +162,12 @@ class FullNodeImpl : public FullNode { std::queue custom_overlays_sent_broadcasts_lru_; void update_private_overlays(); - // void set_private_block_overlays_enable_compression(bool value); - // void create_private_block_overlay(PublicKeyHash key); + void set_private_block_overlays_enable_compression(bool value); + void create_private_block_overlay(PublicKeyHash key); void update_custom_overlay(CustomOverlayInfo& overlay); void send_block_broadcast_to_custom_overlays(const BlockBroadcast& broadcast); void send_block_candidate_broadcast_to_custom_overlays(const BlockIdExt& block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, const td::BufferSlice& data); - FullNodePrivateBlockOverlays private_block_overlays_; }; } // namespace fullnode diff --git a/validator/manager.cpp b/validator/manager.cpp index 663cc7a11..c2233e388 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -504,6 +504,10 @@ void ValidatorManagerImpl::new_block_candidate(BlockIdExt block_id, td::BufferSl if (!started_) { return; } + if (!need_monitor(block_id.shard_full())) { + VLOG(VALIDATOR_DEBUG) << "dropping block candidate broadcast: not monitoring shard"; + return; + } add_cached_block_candidate(ReceivedBlock{block_id, std::move(data)}); } From b8999be2c0d8c36e339d28c977bb27800a4cfd58 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 5 Jun 2024 18:05:08 +0300 Subject: [PATCH 092/388] New archive format and importing archive slices --- create-hardfork/create-hardfork.cpp | 4 +- test/test-ton-collator.cpp | 5 +- tl/generate/scheme/ton_api.tl | 1 + tl/generate/scheme/ton_api.tlo | Bin 96776 -> 96928 bytes validator/db/archive-manager.cpp | 15 +- validator/db/archive-manager.hpp | 7 +- validator/db/archive-slice.cpp | 228 +++++++++++++----- validator/db/archive-slice.hpp | 24 +- validator/db/archiver.cpp | 20 +- validator/db/archiver.hpp | 5 +- validator/db/rootdb.cpp | 11 +- validator/db/rootdb.hpp | 2 +- validator/full-node-master.cpp | 2 +- validator/full-node-shard.cpp | 29 ++- validator/full-node-shard.h | 4 +- validator/full-node-shard.hpp | 6 +- validator/full-node.cpp | 23 +- validator/full-node.hpp | 4 +- validator/import-db-slice.cpp | 290 +++++++++++++++++------ validator/import-db-slice.hpp | 44 +++- validator/interfaces/db.h | 3 +- validator/interfaces/validator-manager.h | 2 + validator/manager-disk.hpp | 7 +- validator/manager-hardfork.hpp | 8 +- validator/manager.cpp | 86 +++---- validator/manager.hpp | 9 +- validator/net/download-archive-slice.cpp | 22 +- validator/net/download-archive-slice.hpp | 6 +- validator/validator.h | 7 +- 29 files changed, 607 insertions(+), 267 deletions(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index 77fe7155e..bb590960c 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -270,8 +270,8 @@ class HardforkCreator : public td::actor::Actor { void get_next_key_blocks(ton::BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) override { } - void download_archive(ton::BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) override { + void download_archive(ton::BlockSeqno masterchain_seqno, ton::ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override { } void download_out_msg_queue_proof( ton::ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index e3bac4e1c..6deb369ad 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -371,9 +371,8 @@ class TestNode : public td::actor::Actor { void get_next_key_blocks(ton::BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) override { } - void download_archive(ton::BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - - td::Promise promise) override { + void download_archive(ton::BlockSeqno masterchain_seqno, ton::ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override { } void download_out_msg_queue_proof( ton::ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index f7dfb93fa..a4956780c 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -456,6 +456,7 @@ tonNode.downloadKeyBlockProof block:tonNode.blockIdExt = tonNode.Data; tonNode.downloadBlockProofLink block:tonNode.blockIdExt = tonNode.Data; tonNode.downloadKeyBlockProofLink block:tonNode.blockIdExt = tonNode.Data; tonNode.getArchiveInfo masterchain_seqno:int = tonNode.ArchiveInfo; +tonNode.getShardArchiveInfo masterchain_seqno:int shard_prefix:tonNode.shardId = tonNode.ArchiveInfo; tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data; tonNode.getOutMsgQueueProof dst_shard:tonNode.shardId blocks:(vector tonNode.blockIdExt) limits:tonNode.importedMsgQueueLimits = tonNode.OutMsgQueueProof; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 83f7d32f9dd86c38cda1b789ece45c7586c918a8..9ffd9fd82766d1736b79d0a245f2275660325268 100644 GIT binary patch delta 97 zcmeD9!n)uqYr_`C6l2D=?Wx9$pB*$KtbYqhm*nU9<)@_TrKgqzXCxM-I2I*mWR|6R z=B4HHzdY7(^O%qi({#fEM*T?(S$T^=D&h-@QqwXk7$)~km784fOJI9M3F8J<0Aorf AUH||9 delta 25 hcmZ4Rm9^swYr_`C6l2EL?Wx9$pB=V;C}G^d3ILJ)3c>&Y diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index b87d04f78..88da8dc8c 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -600,8 +600,8 @@ void ArchiveManager::load_package(PackageId id) { } } - desc.file = - td::actor::create_actor("slice", id.id, id.key, id.temp, false, db_root_, archive_lru_.get(), statistics_); + desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, false, 0, db_root_, + archive_lru_.get(), statistics_); m.emplace(id, std::move(desc)); update_permanent_slices(); @@ -635,8 +635,9 @@ const ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull FileDescription new_desc{id, false}; td::mkdir(db_root_ + id.path()).ensure(); std::string prefix = PSTRING() << db_root_ << id.path() << id.name(); - new_desc.file = - td::actor::create_actor("slice", id.id, id.key, id.temp, false, db_root_, archive_lru_.get(), statistics_); + new_desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, false, + id.key || id.temp ? 0 : cur_shard_split_depth_, db_root_, + archive_lru_.get(), statistics_); const FileDescription &desc = f.emplace(id, std::move(new_desc)); if (!id.temp) { update_desc(f, desc, shard, seqno, ts, lt); @@ -1091,14 +1092,16 @@ PackageId ArchiveManager::get_package_id_force(BlockSeqno masterchain_seqno, Sha return it->first; } -void ArchiveManager::get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) { +void ArchiveManager::get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) { auto F = get_file_desc_by_seqno(ShardIdFull{masterchainId}, masterchain_seqno, false); if (!F) { promise.set_error(td::Status::Error(ErrorCode::notready, "archive not found")); return; } - td::actor::send_closure(F->file_actor_id(), &ArchiveSlice::get_archive_id, masterchain_seqno, std::move(promise)); + td::actor::send_closure(F->file_actor_id(), &ArchiveSlice::get_archive_id, masterchain_seqno, shard_prefix, + std::move(promise)); } void ArchiveManager::get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index a1ed97022..e260b8dc7 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -65,7 +65,7 @@ class ArchiveManager : public td::actor::Actor { void get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise promise); void get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno seqno, td::Promise promise); - void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise); + void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::Promise promise); void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise); @@ -75,6 +75,10 @@ class ArchiveManager : public td::actor::Actor { void commit_transaction(); void set_async_mode(bool mode, td::Promise promise); + void set_current_shard_split_depth(td::uint32 value) { + cur_shard_split_depth_ = value; + } + static constexpr td::uint32 archive_size() { return 20000; } @@ -173,6 +177,7 @@ class ArchiveManager : public td::actor::Actor { bool async_mode_ = false; bool huge_transaction_started_ = false; td::uint32 huge_transaction_size_ = 0; + td::uint32 cur_shard_split_depth_ = 0; DbStatistics statistics_; diff --git a/validator/db/archive-slice.cpp b/validator/db/archive-slice.cpp index d392431a9..0fcdf7924 100644 --- a/validator/db/archive-slice.cpp +++ b/validator/db/archive-slice.cpp @@ -39,7 +39,7 @@ class PackageStatistics { void record_close(uint64_t count = 1) { close_count.fetch_add(count, std::memory_order_relaxed); } - + void record_read(double time, uint64_t bytes) { read_bytes.fetch_add(bytes, std::memory_order_relaxed); std::lock_guard guard(read_mutex); @@ -56,10 +56,10 @@ class PackageStatistics { std::stringstream ss; ss.setf(std::ios::fixed); ss.precision(6); - + ss << "ton.pack.open COUNT : " << open_count.exchange(0, std::memory_order_relaxed) << "\n"; ss << "ton.pack.close COUNT : " << close_count.exchange(0, std::memory_order_relaxed) << "\n"; - + ss << "ton.pack.read.bytes COUNT : " << read_bytes.exchange(0, std::memory_order_relaxed) << "\n"; ss << "ton.pack.write.bytes COUNT : " << write_bytes.exchange(0, std::memory_order_relaxed) << "\n"; @@ -118,7 +118,7 @@ void PackageWriter::append(std::string filename, td::BufferSlice data, return; } start = td::Timestamp::now(); - offset = p->append(std::move(filename), std::move(data), !async_mode_); + offset = p->append(std::move(filename), std::move(data), !async_mode_); end = td::Timestamp::now(); size = p->size(); } @@ -152,6 +152,21 @@ class PackageReader : public td::actor::Actor { std::shared_ptr statistics_; }; +static std::string get_package_file_name(PackageId p_id, ShardIdFull shard_prefix) { + td::StringBuilder sb; + sb << p_id.name(); + if (!shard_prefix.is_masterchain()) { + sb << "."; + sb << shard_prefix.workchain << ":" << shard_to_str(shard_prefix.shard); + } + sb << ".pack"; + return sb.as_cslice().str(); +} + +static std::string package_info_to_str(BlockSeqno seqno, ShardIdFull shard_prefix) { + return PSTRING() << seqno << "." << shard_prefix.workchain << ":" << shard_to_str(shard_prefix.shard); +} + void ArchiveSlice::add_handle(BlockHandle handle, td::Promise promise) { if (destroyed_) { promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd")); @@ -271,7 +286,8 @@ void ArchiveSlice::add_file(BlockHandle handle, FileReference ref_id, td::Buffer TRY_RESULT_PROMISE( promise, p, choose_package( - handle ? handle->id().is_masterchain() ? handle->id().seqno() : handle->masterchain_ref_block() : 0, true)); + handle ? handle->id().is_masterchain() ? handle->id().seqno() : handle->masterchain_ref_block() : 0, + handle ? handle->id().shard_full() : ShardIdFull{masterchainId}, true)); std::string value; auto R = kv_->get(ref_id.hash().to_hex(), value); R.ensure(); @@ -376,7 +392,8 @@ void ArchiveSlice::get_file(ConstBlockHandle handle, FileReference ref_id, td::P TRY_RESULT_PROMISE( promise, p, choose_package( - handle ? handle->id().is_masterchain() ? handle->id().seqno() : handle->masterchain_ref_block() : 0, false)); + handle ? handle->id().is_masterchain() ? handle->id().seqno() : handle->masterchain_ref_block() : 0, + handle ? handle->id().shard_full() : ShardIdFull{masterchainId}, false)); promise = begin_async_query(std::move(promise)); auto P = td::PromiseCreator::lambda( [promise = std::move(promise)](td::Result> R) mutable { @@ -536,18 +553,32 @@ void ArchiveSlice::get_slice(td::uint64 archive_id, td::uint64 offset, td::uint3 } before_query(); auto value = static_cast(archive_id >> 32); - TRY_RESULT_PROMISE(promise, p, choose_package(value, false)); + PackageInfo *p; + if (shard_split_depth_ == 0) { + TRY_RESULT_PROMISE_ASSIGN(promise, p, choose_package(value, ShardIdFull{masterchainId}, false)); + } else { + if (value >= packages_.size()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "no such package")); + return; + } + p = &packages_[value]; + } promise = begin_async_query(std::move(promise)); td::actor::create_actor("readfile", p->path, offset, limit, 0, std::move(promise)).release(); } -void ArchiveSlice::get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) { +void ArchiveSlice::get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) { before_query(); if (!sliced_mode_) { promise.set_result(archive_id_); } else { - TRY_RESULT_PROMISE(promise, p, choose_package(masterchain_seqno, false)); - promise.set_result(p->id * (1ull << 32) + archive_id_); + TRY_RESULT_PROMISE(promise, p, choose_package(masterchain_seqno, shard_prefix, false)); + if (shard_split_depth_ == 0) { + promise.set_result(p->seqno * (1ull << 32) + archive_id_); + } else { + promise.set_result(p->idx * (1ull << 32) + archive_id_); + } } } @@ -573,9 +604,18 @@ void ArchiveSlice::before_query() { R2.ensure(); slice_size_ = td::to_integer(value); CHECK(slice_size_ > 0); + R2 = kv_->get("shard_split_depth", value); + R2.ensure(); + if (R2.move_as_ok() == td::KeyValue::GetStatus::Ok) { + shard_split_depth_ = td::to_integer(value); + CHECK(shard_split_depth_ <= 60); + } else { + shard_split_depth_ = 0; + } for (td::uint32 i = 0; i < tot; i++) { R2 = kv_->get(PSTRING() << "status." << i, value); R2.ensure(); + CHECK(R2.move_as_ok() == td::KeyValue::GetStatus::Ok); auto len = td::to_integer(value); R2 = kv_->get(PSTRING() << "version." << i, value); R2.ensure(); @@ -583,12 +623,24 @@ void ArchiveSlice::before_query() { if (R2.move_as_ok() == td::KeyValue::GetStatus::Ok) { ver = td::to_integer(value); } - auto v = archive_id_ + slice_size_ * i; - add_package(v, len, ver); + td::uint32 seqno; + ShardIdFull shard_prefix; + if (shard_split_depth_ == 0) { + seqno = archive_id_ + slice_size_ * i; + shard_prefix = ShardIdFull{masterchainId}; + } else { + R2 = kv_->get(PSTRING() << "info." << i, value); + R2.ensure(); + CHECK(R2.move_as_ok() == td::KeyValue::GetStatus::Ok); + unsigned long long shard; + CHECK(sscanf(value.c_str(), "%u.%d:%016llx", &seqno, &shard_prefix.workchain, &shard) == 3); + shard_prefix.shard = shard; + } + add_package(seqno, shard_prefix, len, ver); } } else { auto len = td::to_integer(value); - add_package(archive_id_, len, 0); + add_package(archive_id_, ShardIdFull{masterchainId}, len, 0); } } else { if (!temp_ && !key_blocks_only_) { @@ -599,13 +651,17 @@ void ArchiveSlice::before_query() { kv_->set("slice_size", td::to_string(slice_size_)).ensure(); kv_->set("status.0", "0").ensure(); kv_->set("version.0", td::to_string(default_package_version())).ensure(); + if (shard_split_depth_ > 0) { + kv_->set("info.0", package_info_to_str(archive_id_, ShardIdFull{masterchainId})).ensure(); + kv_->set("shard_split_depth", td::to_string(shard_split_depth_)).ensure(); + } kv_->commit_transaction().ensure(); - add_package(archive_id_, 0, default_package_version()); + add_package(archive_id_, ShardIdFull{masterchainId}, 0, default_package_version()); } else { kv_->begin_transaction().ensure(); kv_->set("status", "0").ensure(); kv_->commit_transaction().ensure(); - add_package(archive_id_, 0, 0); + add_package(archive_id_, ShardIdFull{masterchainId}, 0, 0); } } } @@ -642,6 +698,7 @@ void ArchiveSlice::do_close() { statistics_.pack_statistics->record_close(packages_.size()); } packages_.clear(); + id_to_package_.clear(); } template @@ -697,48 +754,61 @@ void ArchiveSlice::set_async_mode(bool mode, td::Promise promise) { } } -ArchiveSlice::ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, std::string db_root, +ArchiveSlice::ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, + td::uint32 shard_split_depth, std::string db_root, td::actor::ActorId archive_lru, DbStatistics statistics) : archive_id_(archive_id) , key_blocks_only_(key_blocks_only) , temp_(temp) , finalized_(finalized) , p_id_(archive_id_, key_blocks_only_, temp_) + , shard_split_depth_(temp || key_blocks_only ? 0 : shard_split_depth) , db_root_(std::move(db_root)) , archive_lru_(std::move(archive_lru)) , statistics_(statistics) { db_path_ = PSTRING() << db_root_ << p_id_.path() << p_id_.name() << ".index"; } -td::Result ArchiveSlice::choose_package(BlockSeqno masterchain_seqno, bool force) { +td::Result ArchiveSlice::choose_package(BlockSeqno masterchain_seqno, + ShardIdFull shard_prefix, bool force) { if (temp_ || key_blocks_only_ || !sliced_mode_) { return &packages_[0]; } if (masterchain_seqno < archive_id_) { return td::Status::Error(ErrorCode::notready, "too small masterchain seqno"); } - auto v = (masterchain_seqno - archive_id_) / slice_size_; - if (v >= packages_.size()) { + masterchain_seqno -= (masterchain_seqno - archive_id_) % slice_size_; + CHECK((masterchain_seqno - archive_id_) % slice_size_ == 0); + if (shard_split_depth_ == 0) { + shard_prefix = ShardIdFull{masterchainId}; + } else if (!shard_prefix.is_masterchain()) { + shard_prefix.shard |= 1; // In case length is < split depth + shard_prefix = ton::shard_prefix(shard_prefix, shard_split_depth_); + } + auto it = id_to_package_.find({masterchain_seqno, shard_prefix}); + if (it == id_to_package_.end()) { if (!force) { - return td::Status::Error(ErrorCode::notready, "too big masterchain seqno"); + return td::Status::Error(ErrorCode::notready, "no such package"); } - CHECK(v == packages_.size()); begin_transaction(); + size_t v = packages_.size(); kv_->set("slices", td::to_string(v + 1)).ensure(); kv_->set(PSTRING() << "status." << v, "0").ensure(); kv_->set(PSTRING() << "version." << v, td::to_string(default_package_version())).ensure(); + if (shard_split_depth_ > 0) { + kv_->set(PSTRING() << "info." << v, package_info_to_str(masterchain_seqno, shard_prefix)).ensure(); + } commit_transaction(); - CHECK((masterchain_seqno - archive_id_) % slice_size_ == 0); - add_package(masterchain_seqno, 0, default_package_version()); + add_package(masterchain_seqno, shard_prefix, 0, default_package_version()); return &packages_[v]; } else { - return &packages_[v]; + return &packages_[it->second]; } } -void ArchiveSlice::add_package(td::uint32 seqno, td::uint64 size, td::uint32 version) { +void ArchiveSlice::add_package(td::uint32 seqno, ShardIdFull shard_prefix, td::uint64 size, td::uint32 version) { PackageId p_id{seqno, key_blocks_only_, temp_}; - std::string path = PSTRING() << db_root_ << p_id.path() << p_id.name() << ".pack"; + std::string path = PSTRING() << db_root_ << p_id.path() << get_package_file_name(p_id, shard_prefix); auto R = Package::open(path, false, true); if (R.is_error()) { LOG(FATAL) << "failed to open/create archive '" << path << "': " << R.move_as_error(); @@ -748,8 +818,9 @@ void ArchiveSlice::add_package(td::uint32 seqno, td::uint64 size, td::uint32 ver statistics_.pack_statistics->record_open(); } auto idx = td::narrow_cast(packages_.size()); + id_to_package_[{seqno, shard_prefix}] = idx; if (finalized_) { - packages_.emplace_back(nullptr, td::actor::ActorOwn(), seqno, path, idx, version); + packages_.emplace_back(nullptr, td::actor::ActorOwn(), seqno, shard_prefix, path, idx, version); return; } auto pack = std::make_shared(R.move_as_ok()); @@ -757,7 +828,7 @@ void ArchiveSlice::add_package(td::uint32 seqno, td::uint64 size, td::uint32 ver pack->truncate(size).ensure(); } auto writer = td::actor::create_actor("writer", pack, async_mode_, statistics_.pack_statistics); - packages_.emplace_back(std::move(pack), std::move(writer), seqno, path, idx, version); + packages_.emplace_back(std::move(pack), std::move(writer), seqno, shard_prefix, path, idx, version); } namespace { @@ -793,6 +864,7 @@ void ArchiveSlice::destroy(td::Promise promise) { statistics_.pack_statistics->record_close(packages_.size()); } packages_.clear(); + id_to_package_.clear(); kv_ = nullptr; PackageId p_id{archive_id_, key_blocks_only_, temp_}; @@ -866,7 +938,7 @@ void ArchiveSlice::move_handle(ConstBlockHandle handle, Package *old_pack, Packa move_file(fileref::Block{handle->id()}, old_pack, pack); } -bool ArchiveSlice::truncate_block(BlockSeqno masterchain_seqno, BlockIdExt block_id, td::uint32 cutoff_idx, +bool ArchiveSlice::truncate_block(BlockSeqno masterchain_seqno, BlockIdExt block_id, td::uint32 cutoff_seqno, Package *pack) { std::string value; auto R = kv_->get(get_db_key_block_info(block_id), value); @@ -881,18 +953,18 @@ bool ArchiveSlice::truncate_block(BlockSeqno masterchain_seqno, BlockIdExt block return false; } - auto S = choose_package(seqno, false); + auto S = choose_package(seqno, block_id.shard_full(), false); S.ensure(); auto p = S.move_as_ok(); - CHECK(p->idx <= cutoff_idx); - if (p->idx == cutoff_idx) { + CHECK(p->seqno <= cutoff_seqno); + if (p->seqno == cutoff_seqno) { move_handle(std::move(handle), p->package.get(), pack); } return true; } -void ArchiveSlice::truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shard, td::uint32 cutoff_idx, +void ArchiveSlice::truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shard, td::uint32 cutoff_seqno, Package *pack) { auto key = get_db_key_lt_desc(shard); std::string value; @@ -918,7 +990,7 @@ void ArchiveSlice::truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shar E.ensure(); auto e = E.move_as_ok(); - if (truncate_block(masterchain_seqno, create_block_id(e->id_), cutoff_idx, pack)) { + if (truncate_block(masterchain_seqno, create_block_id(e->id_), cutoff_seqno, pack)) { CHECK(new_last_idx == i); new_last_idx = i + 1; } @@ -930,7 +1002,7 @@ void ArchiveSlice::truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shar } } -void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise) { +void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle, td::Promise promise) { if (temp_ || archive_id_ > masterchain_seqno) { destroy(std::move(promise)); return; @@ -943,15 +1015,8 @@ void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handl return; } - auto cutoff = choose_package(masterchain_seqno, false); - cutoff.ensure(); - auto pack = cutoff.move_as_ok(); - CHECK(pack); - - auto pack_r = Package::open(pack->path + ".new", false, true); - pack_r.ensure(); - auto new_package = std::make_shared(pack_r.move_as_ok()); - new_package->truncate(0).ensure(); + std::map old_packages; + std::map> new_packages; std::string value; auto status_key = create_serialize_tl_object(); @@ -972,38 +1037,71 @@ void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handl auto G = fetch_tl_object(value, true); G.ensure(); auto g = G.move_as_ok(); + ShardIdFull shard{g->workchain_, static_cast(g->shard_)}; + + auto package_r = choose_package(masterchain_seqno, shard, false); + if (package_r.is_error()) { + continue; + } + auto package = package_r.move_as_ok(); + CHECK(package); + if (!old_packages.count(package->shard_prefix)) { + old_packages[package->shard_prefix] = package; + auto new_package_r = Package::open(package->path + ".new", false, true); + new_package_r.ensure(); + auto new_package = std::make_shared(new_package_r.move_as_ok()); + new_package->truncate(0).ensure(); + new_packages[package->shard_prefix] = std::move(new_package); + } + truncate_shard(masterchain_seqno, shard, package->seqno, new_packages[package->shard_prefix].get()); + } - truncate_shard(masterchain_seqno, ShardIdFull{g->workchain_, static_cast(g->shard_)}, pack->idx, - new_package.get()); + for (auto& [shard_prefix, package] : old_packages) { + auto new_package = new_packages[shard_prefix]; + CHECK(new_package); + package->package = new_package; + package->writer.reset(); + td::unlink(package->path).ensure(); + td::rename(package->path + ".new", package->path).ensure(); + package->writer = td::actor::create_actor("writer", new_package, async_mode_); } + std::vector new_packages_info; + if (!sliced_mode_) { - kv_->set("status", td::to_string(new_package->size())).ensure(); + kv_->set("status", td::to_string(packages_.at(0).package->size())).ensure(); } else { - kv_->set(PSTRING() << "status." << pack->idx, td::to_string(new_package->size())).ensure(); - for (size_t i = pack->idx + 1; i < packages_.size(); i++) { + for (PackageInfo &package : packages_) { + if (package.seqno <= masterchain_seqno) { + new_packages_info.push_back(std::move(package)); + } else { + td::unlink(package.path).ensure(); + } + } + id_to_package_.clear(); + for (td::uint32 i = 0; i < new_packages_info.size(); ++i) { + PackageInfo &package = new_packages_info[i]; + package.idx = i; + kv_->set(PSTRING() << "status." << i, td::to_string(package.package->size())).ensure(); + kv_->set(PSTRING() << "version." << i, td::to_string(package.version)).ensure(); + if (shard_split_depth_ > 0) { + kv_->set(PSTRING() << "info." << i, package_info_to_str(package.seqno, package.shard_prefix)).ensure(); + } + id_to_package_[{package.seqno, package.shard_prefix}] = i; + } + for (size_t i = new_packages_info.size(); i < packages_.size(); i++) { kv_->erase(PSTRING() << "status." << i); kv_->erase(PSTRING() << "version." << i); + kv_->erase(PSTRING() << "info." << i); } - kv_->set("slices", td::to_string(pack->idx + 1)); - } - - pack->package = new_package; - pack->writer.reset(); - td::unlink(pack->path).ensure(); - td::rename(pack->path + ".new", pack->path).ensure(); - pack->writer = td::actor::create_actor("writer", new_package, async_mode_); - - for (auto idx = pack->idx + 1; idx < packages_.size(); idx++) { - td::unlink(packages_[idx].path).ensure(); - } - if (statistics_.pack_statistics) { - statistics_.pack_statistics->record_close(packages_.size() - pack->idx - 1); + kv_->set("slices", td::to_string(new_packages_info.size())); + if (statistics_.pack_statistics) { + statistics_.pack_statistics->record_close(packages_.size() - new_packages_info.size()); + } + packages_ = std::move(new_packages_info); } - packages_.erase(packages_.begin() + pack->idx + 1, packages_.end()); kv_->commit_transaction().ensure(); - promise.set_value(td::Unit()); } diff --git a/validator/db/archive-slice.hpp b/validator/db/archive-slice.hpp index faec2fb83..a027ec0ff 100644 --- a/validator/db/archive-slice.hpp +++ b/validator/db/archive-slice.hpp @@ -96,10 +96,10 @@ class ArchiveLru; class ArchiveSlice : public td::actor::Actor { public: - ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, std::string db_root, - td::actor::ActorId archive_lru, DbStatistics statistics = {}); + ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, td::uint32 shard_split_depth, + std::string db_root, td::actor::ActorId archive_lru, DbStatistics statistics = {}); - void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise); + void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::Promise promise); void add_handle(BlockHandle handle, td::Promise promise); void update_handle(BlockHandle handle, td::Promise promise); @@ -159,6 +159,7 @@ class ArchiveSlice : public td::actor::Actor { bool sliced_mode_{false}; td::uint32 huge_transaction_size_ = 0; td::uint32 slice_size_{100}; + td::uint32 shard_split_depth_ = 0; enum Status { st_closed, st_open, st_want_close @@ -171,28 +172,31 @@ class ArchiveSlice : public td::actor::Actor { std::unique_ptr kv_; struct PackageInfo { - PackageInfo(std::shared_ptr package, td::actor::ActorOwn writer, BlockSeqno id, + PackageInfo(std::shared_ptr package, td::actor::ActorOwn writer, BlockSeqno seqno, ShardIdFull shard_prefix, std::string path, td::uint32 idx, td::uint32 version) : package(std::move(package)) , writer(std ::move(writer)) - , id(id) + , seqno(seqno) + , shard_prefix(shard_prefix) , path(std::move(path)) , idx(idx) , version(version) { } std::shared_ptr package; td::actor::ActorOwn writer; - BlockSeqno id; + BlockSeqno seqno; + ShardIdFull shard_prefix; std::string path; td::uint32 idx; td::uint32 version; }; std::vector packages_; + std::map, td::uint32> id_to_package_; - td::Result choose_package(BlockSeqno masterchain_seqno, bool force); - void add_package(BlockSeqno masterchain_seqno, td::uint64 size, td::uint32 version); - void truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shard, td::uint32 cutoff_idx, Package *pack); - bool truncate_block(BlockSeqno masterchain_seqno, BlockIdExt block_id, td::uint32 cutoff_idx, Package *pack); + td::Result choose_package(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, bool force); + void add_package(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::uint64 size, td::uint32 version); + void truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shard, td::uint32 cutoff_seqno, Package *pack); + bool truncate_block(BlockSeqno masterchain_seqno, BlockIdExt block_id, td::uint32 cutoff_seqno, Package *pack); void delete_handle(ConstBlockHandle handle); void delete_file(FileReference ref_id); diff --git a/validator/db/archiver.cpp b/validator/db/archiver.cpp index c8cbd7b97..93ba18ca2 100644 --- a/validator/db/archiver.cpp +++ b/validator/db/archiver.cpp @@ -25,11 +25,27 @@ namespace ton { namespace validator { BlockArchiver::BlockArchiver(BlockHandle handle, td::actor::ActorId archive_db, - td::Promise promise) - : handle_(std::move(handle)), archive_(archive_db), promise_(std::move(promise)) { + td::actor::ActorId db, td::Promise promise) + : handle_(std::move(handle)), archive_(archive_db), db_(std::move(db)), promise_(std::move(promise)) { } void BlockArchiver::start_up() { + if (handle_->id().is_masterchain()) { + td::actor::send_closure(db_, &Db::get_block_state, handle_, + [SelfId = actor_id(this), archive = archive_](td::Result> R) { + R.ensure(); + td::Ref state{R.move_as_ok()}; + td::uint32 monitor_min_split = state->monitor_min_split_depth(basechainId); + td::actor::send_closure(archive, &ArchiveManager::set_current_shard_split_depth, + monitor_min_split); + td::actor::send_closure(SelfId, &BlockArchiver::move_handle); + }); + } else { + move_handle(); + } +} + +void BlockArchiver::move_handle() { if (handle_->handle_moved_to_archive()) { moved_handle(); } else { diff --git a/validator/db/archiver.hpp b/validator/db/archiver.hpp index 859f269cd..9498977fd 100644 --- a/validator/db/archiver.hpp +++ b/validator/db/archiver.hpp @@ -33,11 +33,13 @@ class FileDb; class BlockArchiver : public td::actor::Actor { public: - BlockArchiver(BlockHandle handle, td::actor::ActorId archive_db, td::Promise promise); + BlockArchiver(BlockHandle handle, td::actor::ActorId archive_db, td::actor::ActorId db, + td::Promise promise); void abort_query(td::Status error); void start_up() override; + void move_handle(); void moved_handle(); void got_proof(td::BufferSlice data); void written_proof(); @@ -50,6 +52,7 @@ class BlockArchiver : public td::actor::Actor { private: BlockHandle handle_; td::actor::ActorId archive_; + td::actor::ActorId db_; td::Promise promise_; }; diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 89868f723..d1c1c2433 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -330,7 +330,8 @@ void RootDb::try_get_static_file(FileHash file_hash, td::Promise promise) { - td::actor::create_actor("archiver", std::move(handle), archive_db_.get(), std::move(promise)) + td::actor::create_actor("archiver", std::move(handle), archive_db_.get(), actor_id(this), + std::move(promise)) .release(); } @@ -404,7 +405,8 @@ void RootDb::start_up() { } void RootDb::archive(BlockHandle handle, td::Promise promise) { - td::actor::create_actor("archiveblock", std::move(handle), archive_db_.get(), std::move(promise)) + td::actor::create_actor("archiveblock", std::move(handle), archive_db_.get(), actor_id(this), + std::move(promise)) .release(); } @@ -483,8 +485,9 @@ void RootDb::check_key_block_proof_link_exists(BlockIdExt block_id, td::Promise< std::move(P)); } -void RootDb::get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) { - td::actor::send_closure(archive_db_, &ArchiveManager::get_archive_id, masterchain_seqno, std::move(promise)); +void RootDb::get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::Promise promise) { + td::actor::send_closure(archive_db_, &ArchiveManager::get_archive_id, masterchain_seqno, shard_prefix, + std::move(promise)); } void RootDb::get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 63509371a..dc94c600d 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -129,7 +129,7 @@ class RootDb : public Db { void check_key_block_proof_exists(BlockIdExt block_id, td::Promise promise) override; void check_key_block_proof_link_exists(BlockIdExt block_id, td::Promise promise) override; - void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) override; + void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::Promise promise) override; void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise) override; void set_async_mode(bool mode, td::Promise promise) override; diff --git a/validator/full-node-master.cpp b/validator/full-node-master.cpp index 8dca60d45..da49f0e2e 100644 --- a/validator/full-node-master.cpp +++ b/validator/full-node-master.cpp @@ -386,7 +386,7 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo } }); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_id, query.masterchain_seqno_, - std::move(P)); + ShardIdFull{masterchainId}, std::move(P)); } void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 1284cf979..d0dbf1a64 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -643,7 +643,24 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod }); VLOG(FULL_NODE_DEBUG) << "Got query getArchiveInfo " << query.masterchain_seqno_ << " from " << src; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_id, query.masterchain_seqno_, - std::move(P)); + ShardIdFull{masterchainId}, std::move(P)); +} + +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getShardArchiveInfo &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_serialize_tl_object()); + } else { + promise.set_value(create_serialize_tl_object(R.move_as_ok())); + } + }); + ShardIdFull shard_prefix = create_shard_id(query.shard_prefix_); + VLOG(FULL_NODE_DEBUG) << "Got query getShardArchiveInfo " << query.masterchain_seqno_ << " " << shard_prefix.to_str() + << " from " << src; + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_id, query.masterchain_seqno_, + shard_prefix, std::move(P)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, @@ -965,13 +982,13 @@ void FullNodeShardImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp t .release(); } -void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) { +void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) { auto &b = choose_neighbour(true); td::actor::create_actor( - "archive", masterchain_seqno, std::move(tmp_dir), adnl_id_, overlay_id_, b.adnl_id, timeout, validator_manager_, - b.use_rldp2() ? (td::actor::ActorId)rldp2_ : rldp_, overlays_, adnl_, client_, - create_neighbour_promise(b, std::move(promise))) + "archive", masterchain_seqno, shard_prefix, std::move(tmp_dir), adnl_id_, overlay_id_, b.adnl_id, timeout, + validator_manager_, b.use_rldp2() ? (td::actor::ActorId)rldp2_ : rldp_, overlays_, + adnl_, client_, create_neighbour_promise(b, std::move(promise))) .release(); } diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 4856989a0..235e30514 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -70,8 +70,8 @@ class FullNodeShard : public td::actor::Actor { td::Promise promise) = 0; virtual void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) = 0; - virtual void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) = 0; + virtual void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) = 0; virtual void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Timestamp timeout, td::Promise>> promise) = 0; diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 393ee269e..472abb194 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -152,6 +152,8 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getShardArchiveInfo &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getOutMsgQueueProof &query, @@ -198,8 +200,8 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise) override; void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) override; - void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) override; + void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override; void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Timestamp timeout, td::Promise>> promise) override; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 9f5fbf698..335578cfb 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -468,12 +468,17 @@ void FullNodeImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeou td::actor::send_closure(shard, &FullNodeShard::get_next_key_blocks, block_id, timeout, std::move(promise)); } -void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) { - auto shard = get_shard(ShardIdFull{masterchainId}); +void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) { + auto shard = get_shard(shard_prefix); + if (shard.empty()) { + VLOG(FULL_NODE_WARNING) << "dropping download archive query to unknown shard"; + promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); + return; + } CHECK(!shard.empty()); - td::actor::send_closure(shard, &FullNodeShard::download_archive, masterchain_seqno, std::move(tmp_dir), timeout, - std::move(promise)); + td::actor::send_closure(shard, &FullNodeShard::download_archive, masterchain_seqno, shard_prefix, std::move(tmp_dir), + timeout, std::move(promise)); } void FullNodeImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, @@ -689,10 +694,10 @@ void FullNodeImpl::start_up() { td::Promise> promise) override { td::actor::send_closure(id_, &FullNodeImpl::get_next_key_blocks, block_id, timeout, std::move(promise)); } - void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) override { - td::actor::send_closure(id_, &FullNodeImpl::download_archive, masterchain_seqno, std::move(tmp_dir), timeout, - std::move(promise)); + void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override { + td::actor::send_closure(id_, &FullNodeImpl::download_archive, masterchain_seqno, shard_prefix, std::move(tmp_dir), + timeout, std::move(promise)); } void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Timestamp timeout, diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 76daf1159..33ccf2cd1 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -82,8 +82,8 @@ class FullNodeImpl : public FullNode { void download_block_proof_link(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise); - void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise); + void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise); void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Timestamp timeout, td::Promise>> promise); diff --git a/validator/import-db-slice.cpp b/validator/import-db-slice.cpp index a93fb05be..06573d347 100644 --- a/validator/import-db-slice.cpp +++ b/validator/import-db-slice.cpp @@ -17,6 +17,7 @@ Copyright 2019-2020 Telegram Systems LLP */ #include "import-db-slice.hpp" + #include "validator/db/fileref.hpp" #include "td/utils/overloaded.h" #include "validator/fabric.h" @@ -26,35 +27,91 @@ #include "ton/ton-io.hpp" #include "downloaders/download-state.hpp" +#include + namespace ton { namespace validator { -ArchiveImporter::ArchiveImporter(std::string path, td::Ref state, BlockSeqno shard_client_seqno, +ArchiveImporter::ArchiveImporter(std::string db_root, td::Ref state, BlockSeqno shard_client_seqno, td::Ref opts, td::actor::ActorId manager, - td::Promise> promise) - : path_(std::move(path)) - , state_(std::move(state)) + std::vector to_import_files, + td::Promise> promise) + : db_root_(std::move(db_root)) + , last_masterchain_state_(std::move(state)) , shard_client_seqno_(shard_client_seqno) + , start_import_seqno_(shard_client_seqno + 1) , opts_(std::move(opts)) , manager_(manager) + , to_import_files_(std::move(to_import_files)) + , use_imported_files_(!to_import_files_.empty()) , promise_(std::move(promise)) { } void ArchiveImporter::start_up() { - auto R = Package::open(path_, false, false); - if (R.is_error()) { - abort_query(R.move_as_error()); + if (use_imported_files_) { + LOG(INFO) << "Importing archive for masterchain seqno #" << start_import_seqno_ << " from disk"; + for (const std::string& path : to_import_files_) { + LOG(INFO) << "Importing file from disk " << path; + td::Status S = process_package(path, true); + if (S.is_error()) { + LOG(INFO) << "Error processing package " << path << ": " << S; + } + } + files_to_cleanup_.clear(); + processed_mc_archive(); + return; + } + LOG(INFO) << "Importing archive for masterchain seqno #" << start_import_seqno_ << " from net"; + td::actor::send_closure(manager_, &ValidatorManager::send_download_archive_request, start_import_seqno_, + ShardIdFull{masterchainId}, db_root_ + "/tmp/", td::Timestamp::in(3600.0), + [SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporter::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ArchiveImporter::downloaded_mc_archive, R.move_as_ok()); + } + }); +} + +void ArchiveImporter::downloaded_mc_archive(std::string path) { + td::Status S = process_package(path, true); + if (S.is_error()) { + abort_query(std::move(S)); + return; + } + processed_mc_archive(); +} + +void ArchiveImporter::processed_mc_archive() { + if (masterchain_blocks_.empty()) { + LOG(DEBUG) << "No masterhchain blocks in archive"; + last_masterchain_seqno_ = last_masterchain_state_->get_seqno(); + checked_all_masterchain_blocks(); + return; + } + + auto seqno = masterchain_blocks_.begin()->first; + LOG(DEBUG) << "First mc seqno in archive = " << seqno; + if (seqno > last_masterchain_state_->get_seqno() + 1) { + abort_query(td::Status::Error(ErrorCode::notready, "too big first masterchain seqno")); return; } - package_ = std::make_shared(R.move_as_ok()); - bool fail = false; - package_->iterate([&](std::string filename, td::BufferSlice data, td::uint64 offset) -> bool { + check_masterchain_block(seqno); +} + +td::Status ArchiveImporter::process_package(std::string path, bool with_masterchain) { + LOG(DEBUG) << "Processing package " << path << " (with_masterchain=" << with_masterchain << ")"; + files_to_cleanup_.push_back(path); + TRY_RESULT(p, Package::open(path, false, false)); + auto package = std::make_shared(std::move(p)); + + td::Status S = td::Status::OK(); + package->iterate([&](std::string filename, td::BufferSlice, td::uint64 offset) -> bool { auto F = FileReference::create(filename); if (F.is_error()) { - abort_query(F.move_as_error()); - fail = true; + S = F.move_as_error(); return false; } auto f = F.move_as_ok(); @@ -79,33 +136,26 @@ void ArchiveImporter::start_up() { ignore = false; is_proof = false; }, - [&](const auto &p) { ignore = true; })); - - if (!ignore) { - blocks_[b][is_proof ? 0 : 1] = offset; + [&](const auto &) { ignore = true; })); + + if (!ignore && (with_masterchain || !b.is_masterchain())) { + if (is_proof) { + blocks_[b].proof_pkg = package; + blocks_[b].proof_offset = offset; + } else { + blocks_[b].data_pkg = package; + blocks_[b].data_offset = offset; + } if (b.is_masterchain()) { masterchain_blocks_[b.seqno()] = b; + last_masterchain_seqno_ = std::max(last_masterchain_seqno_, b.seqno()); + } else { + have_shard_blocks_ = true; } } return true; }); - - if (fail) { - return; - } - - if (masterchain_blocks_.size() == 0) { - abort_query(td::Status::Error(ErrorCode::notready, "archive does not contain any masterchain blocks")); - return; - } - - auto seqno = masterchain_blocks_.begin()->first; - if (seqno > state_->get_seqno() + 1) { - abort_query(td::Status::Error(ErrorCode::notready, "too big first masterchain seqno")); - return; - } - - check_masterchain_block(seqno); + return S; } void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) { @@ -115,17 +165,17 @@ void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) { abort_query(td::Status::Error(ErrorCode::notready, "no new blocks")); return; } - checked_all_masterchain_blocks(seqno - 1); + checked_all_masterchain_blocks(); return; } - while (seqno <= state_->get_block_id().seqno()) { - if (seqno < state_->get_block_id().seqno()) { - if (!state_->check_old_mc_block_id(it->second)) { + while (seqno <= last_masterchain_state_->get_block_id().seqno()) { + if (seqno < last_masterchain_state_->get_block_id().seqno()) { + if (!last_masterchain_state_->check_old_mc_block_id(it->second)) { abort_query(td::Status::Error(ErrorCode::protoviolation, "bad old masterchain block id")); return; } } else { - if (state_->get_block_id() != it->second) { + if (last_masterchain_state_->get_block_id() != it->second) { abort_query(td::Status::Error(ErrorCode::protoviolation, "bad old masterchain block id")); return; } @@ -133,18 +183,27 @@ void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) { seqno++; it = masterchain_blocks_.find(seqno); if (it == masterchain_blocks_.end()) { - checked_all_masterchain_blocks(seqno - 1); + checked_all_masterchain_blocks(); return; } } - if (seqno != state_->get_block_id().seqno() + 1) { + LOG(DEBUG) << "Checking masterchain block #" << seqno; + if (seqno != last_masterchain_state_->get_block_id().seqno() + 1) { abort_query(td::Status::Error(ErrorCode::protoviolation, "hole in masterchain seqno")); return; } auto it2 = blocks_.find(it->second); CHECK(it2 != blocks_.end()); + if (!it2->second.proof_pkg) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block proof")); + return; + } + if (!it2->second.data_pkg) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block data")); + return; + } - auto R1 = package_->read(it2->second[0]); + auto R1 = it2->second.proof_pkg->read(it2->second.proof_offset); if (R1.is_error()) { abort_query(R1.move_as_error()); return; @@ -156,7 +215,7 @@ void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) { return; } - auto R2 = package_->read(it2->second[1]); + auto R2 = it2->second.data_pkg->read(it2->second.data_offset); if (R2.is_error()) { abort_query(R2.move_as_error()); return; @@ -175,7 +234,7 @@ void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) { auto proof = proofR.move_as_ok(); auto data = dataR.move_as_ok(); - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), id = state_->get_block_id(), + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), id = last_masterchain_state_->get_block_id(), data](td::Result R) mutable { if (R.is_error()) { td::actor::send_closure(SelfId, &ArchiveImporter::abort_query, R.move_as_error()); @@ -191,11 +250,12 @@ void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) { td::actor::send_closure(SelfId, &ArchiveImporter::checked_masterchain_proof, std::move(handle), std::move(data)); }); - run_check_proof_query(it->second, std::move(proof), manager_, td::Timestamp::in(2.0), std::move(P), state_, - opts_->is_hardfork(it->second)); + run_check_proof_query(it->second, std::move(proof), manager_, td::Timestamp::in(2.0), std::move(P), + last_masterchain_state_, opts_->is_hardfork(it->second)); } void ArchiveImporter::checked_masterchain_proof(BlockHandle handle, td::Ref data) { + LOG(DEBUG) << "Checked proof for masterchain block #" << handle->id().seqno(); CHECK(data.not_null()); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result R) { R.ensure(); @@ -205,6 +265,7 @@ void ArchiveImporter::checked_masterchain_proof(BlockHandle handle, td::Refid().seqno(); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); td::actor::send_closure(SelfId, &ArchiveImporter::got_new_materchain_state, @@ -214,22 +275,87 @@ void ArchiveImporter::applied_masterchain_block(BlockHandle handle) { } void ArchiveImporter::got_new_materchain_state(td::Ref state) { - state_ = std::move(state); - check_masterchain_block(state_->get_block_id().seqno() + 1); + last_masterchain_state_ = std::move(state); + imported_any_ = true; + check_masterchain_block(last_masterchain_state_->get_block_id().seqno() + 1); } -void ArchiveImporter::checked_all_masterchain_blocks(BlockSeqno seqno) { - check_next_shard_client_seqno(shard_client_seqno_ + 1); +void ArchiveImporter::checked_all_masterchain_blocks() { + LOG(DEBUG) << "Done importing masterchain blocks. Last block seqno = " << last_masterchain_seqno_; + if (start_import_seqno_ > last_masterchain_state_->get_seqno()) { + abort_query(td::Status::Error("no new masterchain blocks were imported")); + return; + } + BlockIdExt block_id; + CHECK(last_masterchain_state_->get_old_mc_block_id(start_import_seqno_, block_id)); + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db_short, block_id, + [SelfId = actor_id(this)](td::Result> R) { + R.ensure(); + td::Ref state{R.move_as_ok()}; + td::actor::send_closure(SelfId, &ArchiveImporter::download_shard_archives, + std::move(state)); + }); +} + +void ArchiveImporter::download_shard_archives(td::Ref start_state) { + start_state_ = start_state; + td::uint32 monitor_min_split = start_state->monitor_min_split_depth(basechainId); + LOG(DEBUG) << "Monitor min split = " << monitor_min_split; + // If monitor_min_split == 0, we use the old archive format (packages are not separated by shard) + // If masterchain package has shard blocks then it's old archive format, don't need to download shards + if (monitor_min_split > 0 && !have_shard_blocks_ && !use_imported_files_) { + for (td::uint64 i = 0; i < (1ULL << monitor_min_split); ++i) { + ShardIdFull shard_prefix{basechainId, (i * 2 + 1) << (64 - monitor_min_split - 1)}; + if (opts_->need_monitor(shard_prefix, start_state)) { + ++pending_shard_archives_; + LOG(DEBUG) << "Downloading shard archive #" << start_import_seqno_ << " " << shard_prefix.to_str(); + download_shard_archive(shard_prefix); + } + } + } else { + LOG(DEBUG) << "Skip downloading shard archives"; + } + if (pending_shard_archives_ == 0) { + check_next_shard_client_seqno(shard_client_seqno_ + 1); + } +} + +void ArchiveImporter::download_shard_archive(ShardIdFull shard_prefix) { + td::actor::send_closure( + manager_, &ValidatorManager::send_download_archive_request, start_import_seqno_, shard_prefix, db_root_ + "/tmp/", + td::Timestamp::in(3600.0), + [SelfId = actor_id(this), seqno = start_import_seqno_, shard_prefix](td::Result R) { + if (R.is_error()) { + LOG(WARNING) << "Failed to download archive slice #" << seqno << " for shard " << shard_prefix.to_str(); + delay_action( + [=]() { td::actor::send_closure(SelfId, &ArchiveImporter::download_shard_archive, shard_prefix); }, + td::Timestamp::in(2.0)); + } else { + LOG(DEBUG) << "Downloaded shard archive #" << seqno << " " << shard_prefix.to_str(); + td::actor::send_closure(SelfId, &ArchiveImporter::downloaded_shard_archive, R.move_as_ok()); + } + }); +} + +void ArchiveImporter::downloaded_shard_archive(std::string path) { + td::Status S = process_package(path, false); + if (S.is_error()) { + LOG(INFO) << "Error processing package: " << S; + } + --pending_shard_archives_; + if (pending_shard_archives_ == 0) { + check_next_shard_client_seqno(shard_client_seqno_ + 1); + } } void ArchiveImporter::check_next_shard_client_seqno(BlockSeqno seqno) { - if (seqno > state_->get_seqno()) { + if (seqno > last_masterchain_state_->get_seqno() || seqno > last_masterchain_seqno_) { finish_query(); - } else if (seqno == state_->get_seqno()) { - got_masterchain_state(state_); + } else if (seqno == last_masterchain_state_->get_seqno()) { + got_masterchain_state(last_masterchain_state_); } else { BlockIdExt b; - bool f = state_->get_old_mc_block_id(seqno, b); + bool f = last_masterchain_state_->get_old_mc_block_id(seqno, b); CHECK(f); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); @@ -241,33 +367,38 @@ void ArchiveImporter::check_next_shard_client_seqno(BlockSeqno seqno) { } void ArchiveImporter::got_masterchain_state(td::Ref state) { + if (state->get_seqno() != start_import_seqno_ && state->is_key_state()) { + finish_query(); + return; + } + LOG(DEBUG) << "Applying shard client seqno " << state->get_seqno(); auto s = state->get_shards(); - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), seqno = state->get_seqno()](td::Result R) { + td::MultiPromise mp; + auto ig = mp.init_guard(); + for (auto &shard : s) { + if (opts_->need_monitor(shard->shard(), state)) { + apply_shard_block(shard->top_block_id(), state->get_block_id(), ig.get_promise()); + } + } + ig.add_promise([SelfId = actor_id(this), seqno = state->get_seqno()](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &ArchiveImporter::abort_query, R.move_as_error()); } else { td::actor::send_closure(SelfId, &ArchiveImporter::checked_shard_client_seqno, seqno); } }); - - td::MultiPromise mp; - auto ig = mp.init_guard(); - ig.add_promise(std::move(P)); - - for (auto &shard : s) { - apply_shard_block(shard->top_block_id(), state->get_block_id(), ig.get_promise()); - } } void ArchiveImporter::checked_shard_client_seqno(BlockSeqno seqno) { CHECK(shard_client_seqno_ + 1 == seqno); shard_client_seqno_++; + imported_any_ = true; check_next_shard_client_seqno(seqno + 1); } void ArchiveImporter::apply_shard_block(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) { + LOG(DEBUG) << "Applying shard block " << block_id.id.to_str(); auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), masterchain_block_id, promise = std::move(promise)](td::Result R) mutable { R.ensure(); @@ -286,7 +417,7 @@ void ArchiveImporter::apply_shard_block_cont1(BlockHandle handle, BlockIdExt mas if (handle->id().seqno() == 0) { auto P = td::PromiseCreator::lambda( - [promise = std::move(promise)](td::Result> R) mutable { promise.set_value(td::Unit()); }); + [promise = std::move(promise)](td::Result>) mutable { promise.set_value(td::Unit()); }); td::actor::create_actor("downloadstate", handle->id(), masterchain_block_id, 2, manager_, td::Timestamp::in(3600), std::move(P)) .release(); @@ -294,12 +425,13 @@ void ArchiveImporter::apply_shard_block_cont1(BlockHandle handle, BlockIdExt mas } auto it = blocks_.find(handle->id()); - if (it == blocks_.end()) { - promise.set_error(td::Status::Error(ErrorCode::notready, PSTRING() << "no proof for shard block " << handle->id())); + if (it == blocks_.end() || !it->second.proof_pkg || !it->second.data_pkg) { + promise.set_error( + td::Status::Error(ErrorCode::notready, PSTRING() << "no data/proof for shard block " << handle->id())); return; } - TRY_RESULT_PROMISE(promise, data, package_->read(it->second[0])); - TRY_RESULT_PROMISE(promise, proof, create_proof_link(handle->id(), std::move(data.second))); + TRY_RESULT_PROMISE(promise, proof_data, it->second.proof_pkg->read(it->second.proof_offset)); + TRY_RESULT_PROMISE(promise, proof, create_proof_link(handle->id(), std::move(proof_data.second))); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle, masterchain_block_id, promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { @@ -345,8 +477,8 @@ void ArchiveImporter::apply_shard_block_cont2(BlockHandle handle, BlockIdExt mas void ArchiveImporter::apply_shard_block_cont3(BlockHandle handle, BlockIdExt masterchain_block_id, td::Promise promise) { auto it = blocks_.find(handle->id()); - CHECK(it != blocks_.end()); - TRY_RESULT_PROMISE(promise, data, package_->read(it->second[1])); + CHECK(it != blocks_.end() && it->second.data_pkg); + TRY_RESULT_PROMISE(promise, data, it->second.data_pkg->read(it->second.data_offset)); if (sha256_bits256(data.second.as_slice()) != handle->id().file_hash) { promise.set_error(td::Status::Error(ErrorCode::protoviolation, "bad block file hash")); return; @@ -367,6 +499,7 @@ void ArchiveImporter::check_shard_block_applied(BlockIdExt block_id, td::Promise if (!handle->is_applied()) { promise.set_error(td::Status::Error(ErrorCode::notready, "not applied")); } else { + LOG(DEBUG) << "Applied shard block " << handle->id().id.to_str(); promise.set_value(td::Unit()); } } @@ -375,13 +508,24 @@ void ArchiveImporter::check_shard_block_applied(BlockIdExt block_id, td::Promise } void ArchiveImporter::abort_query(td::Status error) { - LOG(INFO) << error; + if (!imported_any_) { + for (const std::string &f : files_to_cleanup_) { + td::unlink(f).ignore(); + } + promise_.set_error(std::move(error)); + return; + } + LOG(INFO) << "Archive import: " << error; finish_query(); } + void ArchiveImporter::finish_query() { + for (const std::string &f : files_to_cleanup_) { + td::unlink(f).ignore(); + } if (promise_) { - promise_.set_value( - std::vector{state_->get_seqno(), std::min(state_->get_seqno(), shard_client_seqno_)}); + promise_.set_value({last_masterchain_state_->get_seqno(), + std::min(last_masterchain_state_->get_seqno(), shard_client_seqno_)}); } stop(); } diff --git a/validator/import-db-slice.hpp b/validator/import-db-slice.hpp index 0993d4bba..04f22642d 100644 --- a/validator/import-db-slice.hpp +++ b/validator/import-db-slice.hpp @@ -19,6 +19,7 @@ #pragma once #include "td/actor/actor.h" +#include "td/utils/port/path.h" #include "validator/interfaces/validator-manager.h" #include "validator/db/package.hpp" @@ -28,19 +29,27 @@ namespace validator { class ArchiveImporter : public td::actor::Actor { public: - ArchiveImporter(std::string path, td::Ref state, BlockSeqno shard_client_seqno, + ArchiveImporter(std::string db_root, td::Ref state, BlockSeqno shard_client_seqno, td::Ref opts, td::actor::ActorId manager, - td::Promise> promise); + std::vector to_import_files, td::Promise> promise); void start_up() override; void abort_query(td::Status error); void finish_query(); + void downloaded_mc_archive(std::string path); + td::Status process_package(std::string path, bool with_masterchain); + + void processed_mc_archive(); void check_masterchain_block(BlockSeqno seqno); void checked_masterchain_proof(BlockHandle handle, td::Ref data); void applied_masterchain_block(BlockHandle handle); void got_new_materchain_state(td::Ref state); - void checked_all_masterchain_blocks(BlockSeqno seqno); + + void checked_all_masterchain_blocks(); + void download_shard_archives(td::Ref start_state); + void download_shard_archive(ShardIdFull shard_prefix); + void downloaded_shard_archive(std::string path); void check_next_shard_client_seqno(BlockSeqno seqno); void checked_shard_client_seqno(BlockSeqno seqno); @@ -52,19 +61,36 @@ class ArchiveImporter : public td::actor::Actor { void check_shard_block_applied(BlockIdExt block_id, td::Promise promise); private: - std::string path_; - td::Ref state_; + std::string db_root_; + td::Ref last_masterchain_state_; BlockSeqno shard_client_seqno_; + BlockSeqno start_import_seqno_; td::Ref opts_; - std::shared_ptr package_; - td::actor::ActorId manager_; - td::Promise> promise_; + + std::vector to_import_files_; + bool use_imported_files_; + td::Promise> promise_; std::map masterchain_blocks_; - std::map> blocks_; + BlockSeqno last_masterchain_seqno_ = 0; + + struct BlockInfo { + std::shared_ptr data_pkg; + td::uint64 data_offset = 0; + std::shared_ptr proof_pkg; + td::uint64 proof_offset = 0; + }; + std::map blocks_; + + td::Ref start_state_; + size_t pending_shard_archives_ = 0; + + bool imported_any_ = false; + bool have_shard_blocks_ = false; + std::vector files_to_cleanup_; }; } // namespace validator diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index 66c1f0761..2d8566462 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -114,7 +114,8 @@ class Db : public td::actor::Actor { virtual void check_key_block_proof_exists(BlockIdExt block_id, td::Promise promise) = 0; virtual void check_key_block_proof_link_exists(BlockIdExt block_id, td::Promise promise) = 0; - virtual void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) = 0; + virtual void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) = 0; virtual void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise) = 0; virtual void set_async_mode(bool mode, td::Promise promise) = 0; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 5e567eeeb..3b44e894f 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -135,6 +135,8 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) = 0; + virtual void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) = 0; virtual void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_shard_client_state(bool from_db, td::Promise promise) = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 695ac566f..509e743d1 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -265,6 +265,10 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise>> promise) override { UNREACHABLE(); } + void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override { + UNREACHABLE(); + } void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; @@ -285,7 +289,8 @@ class ValidatorManagerImpl : public ValidatorManager { promise.set_error(td::Status::Error(ErrorCode::error, "download disabled")); } - void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) override { + void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) override { UNREACHABLE(); } void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index f85b2f641..c97ad1c8a 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -20,6 +20,7 @@ #include "interfaces/validator-manager.h" #include "interfaces/db.h" +#include "ton/ton-types.h" #include "validator-group.hpp" #include "manager-init.h" #include "manager-hardfork.h" @@ -332,6 +333,10 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise>> promise) override { UNREACHABLE(); } + void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override { + UNREACHABLE(); + } void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override { UNREACHABLE(); @@ -357,7 +362,8 @@ class ValidatorManagerImpl : public ValidatorManager { promise.set_error(td::Status::Error(ErrorCode::error, "download disabled")); } - void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) override { + void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) override { UNREACHABLE(); } void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, diff --git a/validator/manager.cpp b/validator/manager.cpp index d79267eaf..aa46e9e80 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1729,6 +1729,12 @@ void ValidatorManagerImpl::send_get_out_msg_queue_proof_request( std::move(promise)); } +void ValidatorManagerImpl::send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, + std::string tmp_dir, td::Timestamp timeout, + td::Promise promise) { + callback_->download_archive(mc_seqno, shard_prefix, std::move(tmp_dir), timeout, std::move(promise)); +} + void ValidatorManagerImpl::start_up() { db_ = create_db_actor(actor_id(this), db_root_, opts_); lite_server_cache_ = create_liteserver_cache_actor(actor_id(this), db_root_); @@ -1763,7 +1769,6 @@ void ValidatorManagerImpl::start_up() { if (fname.substr(fname.size() - 5) != ".pack") { return; } - fname = fname.substr(0, fname.size() - 5); if (fname.substr(0, 8) != "archive.") { return; } @@ -1772,13 +1777,18 @@ void ValidatorManagerImpl::start_up() { while (fname.size() > 1 && fname[0] == '0') { fname.remove_prefix(1); } + auto i = fname.find('.'); + if (i == td::Slice::npos) { + return; + } + fname = fname.substr(0, i); auto v = td::to_integer_safe(fname); if (v.is_error()) { return; } - auto pos = v.move_as_ok(); - LOG(INFO) << "found archive slice '" << cfname << "' for position " << pos; - to_import_[pos] = std::make_pair(cfname.str(), true); + auto seqno = v.move_as_ok(); + LOG(INFO) << "found archive slice '" << cfname << "' for seqno " << seqno; + to_import_[seqno].push_back(cfname.str()); } }); if (S.is_error()) { @@ -1926,8 +1936,7 @@ bool ValidatorManagerImpl::out_of_sync() { void ValidatorManagerImpl::prestart_sync() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); - // Don't download archives - td::actor::send_closure(SelfId, &ValidatorManagerImpl::finish_prestart_sync); + td::actor::send_closure(SelfId, &ValidatorManagerImpl::download_next_archive); }); td::actor::send_closure(db_, &Db::set_async_mode, false, std::move(P)); } @@ -1939,61 +1948,35 @@ void ValidatorManagerImpl::download_next_archive() { } auto seqno = std::min(last_masterchain_seqno_, shard_client_handle_->id().seqno()); + std::vector to_import_files; auto it = to_import_.upper_bound(seqno + 1); if (it != to_import_.begin()) { - it--; - if (it->second.second) { - it->second.second = false; - downloaded_archive_slice(it->second.first, false); - return; - } + --it; + to_import_files = std::move(it->second); + it->second.clear(); } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - LOG(INFO) << "failed to download archive slice: " << R.error(); - delay_action([SelfId]() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::download_next_archive); }, - td::Timestamp::in(2.0)); - } else { - td::actor::send_closure(SelfId, &ValidatorManagerImpl::downloaded_archive_slice, R.move_as_ok(), true); - } - }); - callback_->download_archive(seqno + 1, db_root_ + "/tmp/", td::Timestamp::in(36000.0), std::move(P)); -} - -void ValidatorManagerImpl::downloaded_archive_slice(std::string name, bool is_tmp) { - LOG(INFO) << "downloaded archive slice: " << name; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), name, is_tmp](td::Result> R) { - if (is_tmp) { - td::unlink(name).ensure(); - } + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { - LOG(INFO) << "failed to check downloaded archive slice: " << R.error(); + LOG(INFO) << "failed to download and import archive slice: " << R.error(); delay_action([SelfId]() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::download_next_archive); }, td::Timestamp::in(2.0)); } else { - td::actor::send_closure(SelfId, &ValidatorManagerImpl::checked_archive_slice, R.move_as_ok()); + td::actor::send_closure(SelfId, &ValidatorManagerImpl::checked_archive_slice, R.ok().first, R.ok().second); } }); - - auto seqno = std::min(last_masterchain_seqno_, shard_client_handle_->id().seqno()); - - td::actor::create_actor("archiveimport", name, last_masterchain_state_, seqno, opts_, actor_id(this), - std::move(P)) + td::actor::create_actor("archiveimport", db_root_, last_masterchain_state_, seqno, opts_, + actor_id(this), std::move(to_import_files), std::move(P)) .release(); } -void ValidatorManagerImpl::checked_archive_slice(std::vector seqno) { - CHECK(seqno.size() == 2); - LOG(INFO) << "checked downloaded archive slice: mc_top_seqno=" << seqno[0] << " shard_top_seqno_=" << seqno[1]; - CHECK(seqno[0] <= last_masterchain_seqno_); - CHECK(seqno[1] <= last_masterchain_seqno_); +void ValidatorManagerImpl::checked_archive_slice(BlockSeqno new_last_mc_seqno, BlockSeqno new_shard_client_seqno) { + LOG(INFO) << "checked downloaded archive slice: mc_top_seqno=" << new_last_mc_seqno + << " shard_top_seqno_=" << new_shard_client_seqno; + CHECK(new_last_mc_seqno <= last_masterchain_seqno_); + CHECK(new_shard_client_seqno <= last_masterchain_seqno_); - BlockIdExt b; - if (seqno[1] < last_masterchain_seqno_) { - CHECK(last_masterchain_state_->get_old_mc_block_id(seqno[1], b)); - } else { - b = last_masterchain_block_id_; - } + BlockIdExt shard_client_block_id; + CHECK(last_masterchain_state_->get_old_mc_block_id(new_shard_client_seqno, shard_client_block_id)); auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), db = db_.get(), client = shard_client_.get()](td::Result R) { @@ -2009,7 +1992,7 @@ void ValidatorManagerImpl::checked_archive_slice(std::vector seqno) }); td::actor::send_closure(db, &Db::get_block_state, std::move(handle), std::move(P)); }); - get_block_handle(b, true, std::move(P)); + get_block_handle(shard_client_block_id, true, std::move(P)); } void ValidatorManagerImpl::finish_prestart_sync() { @@ -2844,12 +2827,13 @@ void ValidatorManagerImpl::try_get_static_file(FileHash file_hash, td::Promise promise) { +void ValidatorManagerImpl::get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) { if (masterchain_seqno > last_masterchain_seqno_) { promise.set_error(td::Status::Error(ErrorCode::notready, "masterchain seqno too big")); return; } - td::actor::send_closure(db_, &Db::get_archive_id, masterchain_seqno, std::move(promise)); + td::actor::send_closure(db_, &Db::get_archive_id, masterchain_seqno, shard_prefix, std::move(promise)); } void ValidatorManagerImpl::get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, diff --git a/validator/manager.hpp b/validator/manager.hpp index 37b447cc7..dc18e6278 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -337,8 +337,7 @@ class ValidatorManagerImpl : public ValidatorManager { void applied_hardfork(); void prestart_sync(); void download_next_archive(); - void downloaded_archive_slice(std::string name, bool is_tmp); - void checked_archive_slice(std::vector seqno); + void checked_archive_slice(BlockSeqno new_last_mc_seqno, BlockSeqno new_shard_client_seqno); void finish_prestart_sync(); void completed_prestart_sync(); @@ -516,6 +515,8 @@ class ValidatorManagerImpl : public ValidatorManager { void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) override; + void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override; void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; @@ -532,7 +533,7 @@ class ValidatorManagerImpl : public ValidatorManager { std::move(promise)); } - void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) override; + void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::Promise promise) override; void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise) override; @@ -712,7 +713,7 @@ class ValidatorManagerImpl : public ValidatorManager { td::actor::ActorOwn serializer_; - std::map> to_import_; + std::map> to_import_; private: std::unique_ptr callback_; diff --git a/validator/net/download-archive-slice.cpp b/validator/net/download-archive-slice.cpp index 6235b8b08..c2f8eceac 100644 --- a/validator/net/download-archive-slice.cpp +++ b/validator/net/download-archive-slice.cpp @@ -20,6 +20,8 @@ #include "td/utils/port/path.h" #include "td/utils/overloaded.h" +#include + namespace ton { namespace validator { @@ -27,12 +29,13 @@ namespace validator { namespace fullnode { DownloadArchiveSlice::DownloadArchiveSlice( - BlockSeqno masterchain_seqno, std::string tmp_dir, adnl::AdnlNodeIdShort local_id, + BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, adnl::AdnlNodeIdShort local_id, overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, td::Timestamp timeout, td::actor::ActorId validator_manager, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId adnl, td::actor::ActorId client, td::Promise promise) : masterchain_seqno_(masterchain_seqno) + , shard_prefix_(shard_prefix) , tmp_dir_(std::move(tmp_dir)) , local_id_(local_id) , overlay_id_(overlay_id) @@ -114,7 +117,13 @@ void DownloadArchiveSlice::got_node_to_download(adnl::AdnlNodeIdShort download_f } }); - auto q = create_serialize_tl_object(masterchain_seqno_); + td::BufferSlice q; + if (shard_prefix_.is_masterchain()) { + q = create_serialize_tl_object(masterchain_seqno_); + } else { + q = create_serialize_tl_object(masterchain_seqno_, + create_tl_shard_id(shard_prefix_)); + } if (client_.empty()) { td::actor::send_closure(overlays_, &overlay::Overlays::send_query, download_from_, local_id_, overlay_id_, "get_archive_info", std::move(P), td::Timestamp::in(3.0), std::move(q)); @@ -145,7 +154,8 @@ void DownloadArchiveSlice::got_archive_info(td::BufferSlice data) { } prev_logged_timer_ = td::Timer(); - LOG(INFO) << "downloading archive slice #" << masterchain_seqno_ << " from " << download_from_; + LOG(INFO) << "downloading archive slice #" << masterchain_seqno_ << " " << shard_prefix_.to_str() << " from " + << download_from_; get_archive_slice(); } @@ -186,13 +196,15 @@ void DownloadArchiveSlice::got_archive_slice(td::BufferSlice data) { double elapsed = prev_logged_timer_.elapsed(); if (elapsed > 10.0) { prev_logged_timer_ = td::Timer(); - LOG(INFO) << "downloading archive slice #" << masterchain_seqno_ << ": total=" << offset_ << " (" + LOG(INFO) << "downloading archive slice #" << masterchain_seqno_ << " " << shard_prefix_.to_str() + << ": total=" << offset_ << " (" << td::format::as_size((td::uint64)(double(offset_ - prev_logged_sum_) / elapsed)) << "/s)"; prev_logged_sum_ = offset_; } if (data.size() < slice_size()) { - LOG(INFO) << "finished downloading arcrive slice #" << masterchain_seqno_ << ": total=" << offset_; + LOG(INFO) << "finished downloading arcrive slice #" << masterchain_seqno_ << " " << shard_prefix_.to_str() + << ": total=" << offset_; finish_query(); } else { get_archive_slice(); diff --git a/validator/net/download-archive-slice.hpp b/validator/net/download-archive-slice.hpp index 0384ac8c9..42fd715f7 100644 --- a/validator/net/download-archive-slice.hpp +++ b/validator/net/download-archive-slice.hpp @@ -32,8 +32,9 @@ namespace fullnode { class DownloadArchiveSlice : public td::actor::Actor { public: - DownloadArchiveSlice(BlockSeqno masterchain_seqno, std::string tmp_dir, adnl::AdnlNodeIdShort local_id, - overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, td::Timestamp timeout, + DownloadArchiveSlice(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + adnl::AdnlNodeIdShort local_id, overlay::OverlayIdShort overlay_id, + adnl::AdnlNodeIdShort download_from, td::Timestamp timeout, td::actor::ActorId validator_manager, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId adnl, @@ -55,6 +56,7 @@ class DownloadArchiveSlice : public td::actor::Actor { private: BlockSeqno masterchain_seqno_; + ShardIdFull shard_prefix_; std::string tmp_dir_; std::string tmp_name_; td::FileFd fd_; diff --git a/validator/validator.h b/validator/validator.h index afd884acc..c7ef2ce87 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -160,8 +160,8 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Promise promise) = 0; virtual void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) = 0; - virtual void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) = 0; + virtual void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) = 0; virtual void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Timestamp timeout, td::Promise>> promise) = 0; @@ -253,7 +253,8 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Timestamp timeout, td::Promise>> promise) = 0; - virtual void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) = 0; + virtual void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) = 0; virtual void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise) = 0; From 38ab70c037a49faf2db8b5a874c26163ce90cbc5 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 11 Jun 2024 11:29:52 +0300 Subject: [PATCH 093/388] Changes in validation * Configure collator list in validator-console * Remove "lite-validator" flags * Better compatibility in validate-query.cpp --- tl/generate/scheme/ton_api.tl | 9 ++ tl/generate/scheme/ton_api.tlo | Bin 96928 -> 97828 bytes .../validator-engine-console-query.cpp | 77 +++++++++++++++ .../validator-engine-console-query.h | 60 ++++++++++++ .../validator-engine-console.cpp | 3 + validator-engine/validator-engine.cpp | 87 ++++++++++++++--- validator-engine/validator-engine.hpp | 14 ++- validator/fabric.h | 2 +- validator/impl/validate-query.cpp | 20 ++-- validator/manager.cpp | 48 ++++------ validator/manager.hpp | 3 - validator/validator-group.cpp | 88 +++++++++++------- validator/validator-group.hpp | 16 +++- validator/validator-options.cpp | 16 ++++ validator/validator-options.hpp | 10 +- validator/validator.h | 22 ++++- 16 files changed, 368 insertions(+), 107 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index a4956780c..0d7f1249e 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -626,6 +626,12 @@ engine.validator.customOverlayNode adnl_id:int256 msg_sender:Bool msg_sender_pri engine.validator.customOverlay name:string nodes:(vector engine.validator.customOverlayNode) = engine.validator.CustomOverlay; engine.validator.customOverlaysConfig overlays:(vector engine.validator.customOverlay) = engine.validator.CustomOverlaysConfig; +engine.validator.collatorsList.collator adnl_id:int256 trusted:Bool = engine.validator.collatorsList.Collator; +engine.validator.collatorsList.shard shard_id:tonNode.shardId collators:(vector engine.validator.collatorsList.collator) + = engine.validator.collatorsList.Shard; +engine.validator.collatorsList self_collate:Bool use_config_41:Bool shards:(vector engine.validator.collatorsList.shard) + = engine.validator.CollatorsList; + ---functions--- ---types--- @@ -752,6 +758,9 @@ engine.validator.addShard shard:tonNode.shardId = engine.validator.Success; engine.validator.delCollator adnl_id:int256 shard:tonNode.shardId = engine.validator.Success; engine.validator.delShard shard:tonNode.shardId = engine.validator.Success; +engine.validator.setCollatorsList list:engine.validator.collatorsList = engine.validator.Success; +engine.validator.showCollatorsList = engine.validator.CollatorsList; + ---types--- storage.pong = storage.Pong; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 9ffd9fd82766d1736b79d0a245f2275660325268..0d286bdc091a5b25e2a8deab3019aa84e0e73fe5 100644 GIT binary patch delta 587 zcmZ4Rm37G0TI4~`xT^i`UOiy9S-)A zqSE4$)Rf5umNKCrVOWS>mE(#bCLoGI0RhwmQVa4H6VzKAV2Sw5l<5ac7{w>eXPIm< zS3?411G*PLIyZN0zEjWGu|3t8v0aw6=S6X(;`EOJj3UC$$g#t~(7o)*=IM9B7-d*M t9-l5?#i+3TO#ov8Bdci80ez+EfkBL7swi4u!42{ka-eN@2x3go005-h!+`(* delta 71 zcmZ4ThjqbMR^CUm^{p77KyoARf04}|qA%Dt?~pZ-W?Z=Wt;PG~%`4{Ch;L5VcA=iJ XZM&fvW4r8jl@P`R#_bnE8ACJxv3MGm diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 660915bb7..3d22ff814 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1316,3 +1316,80 @@ td::Status DelShardQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << "successfully removed shard\n"; return td::Status::OK(); } + +td::Status SetCollatorsListQuery::run() { + TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status SetCollatorsListQuery::send() { + TRY_RESULT(data, td::read_file(file_name_)); + TRY_RESULT(json, td::json_decode(data.as_slice())); + auto list = ton::create_tl_object(); + TRY_STATUS(ton::ton_api::from_json(*list, json.get_object())); + auto b = ton::create_serialize_tl_object(std::move(list)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status SetCollatorsListQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status ClearCollatorsListQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status ClearCollatorsListQuery::send() { + auto list = ton::create_tl_object(); + list->self_collate_ = true; + auto b = ton::create_serialize_tl_object(std::move(list)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ClearCollatorsListQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status ShowCollatorsListQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status ShowCollatorsListQuery::send() { + auto b = ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ShowCollatorsListQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(list, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "Collators list:\n"; + if (list->self_collate_) { + td::TerminalIO::out() << "self_collate = true\n"; + } + if (list->use_config_41_) { + td::TerminalIO::out() << "use_config_41 = true\n"; + } + if (list->shards_.empty()) { + td::TerminalIO::out() << "Shard list is empty\n"; + return td::Status::OK(); + } + for (const auto &shard : list->shards_) { + td::TerminalIO::out() << "Shard " << create_shard_id(shard->shard_id_).to_str() << "\n"; + for (const auto &collator : shard->collators_) { + td::TerminalIO::out() << " Collator " << collator->adnl_id_ << (collator->trusted_ ? " (trusted)" : "") << "\n"; + } + } + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index ecdfa5e2b..4d2a50a30 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1342,3 +1342,63 @@ class DelShardQuery : public Query { td::int32 wc_; td::int64 shard_; }; + +class SetCollatorsListQuery : public Query { + public: + SetCollatorsListQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "setcollatorslist"; + } + static std::string get_help() { + return "setcollatorslist \tset list of collators from file "; + } + std::string name() const override { + return get_name(); + } + + private: + std::string file_name_; +}; + +class ClearCollatorsListQuery : public Query { + public: + ClearCollatorsListQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "clearcollatorslist"; + } + static std::string get_help() { + return "clearcollatorslist\tclear list of collators"; + } + std::string name() const override { + return get_name(); + } +}; + +class ShowCollatorsListQuery : public Query { + public: + ShowCollatorsListQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "showcollatorslist"; + } + static std::string get_help() { + return "showcollatorslist\tshow list of collators"; + } + std::string name() const override { + return get_name(); + } +}; diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index cd4c7c36b..e9c26484d 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -152,6 +152,9 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 585c4c74e..4dd25030a 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1472,7 +1472,6 @@ td::Status ValidatorEngine::load_global_config() { } validator_options_.write().set_hardforks(std::move(h)); validator_options_.write().set_state_serializer_enabled(config_.state_serializer_enabled); - validator_options_.write().set_validator_mode(validator_mode_); return td::Status::OK(); } @@ -1502,6 +1501,31 @@ void ValidatorEngine::set_shard_check_function() { } } +void ValidatorEngine::load_collators_list() { + collators_list_ = {}; + auto data_R = td::read_file(collators_list_file()); + if (data_R.is_error()) { + return; + } + auto data = data_R.move_as_ok(); + auto json_R = td::json_decode(data.as_slice()); + if (json_R.is_error()) { + LOG(ERROR) << "Failed to parse collators list: " << json_R.move_as_error(); + return; + } + auto json = json_R.move_as_ok(); + collators_list_ = ton::create_tl_object(); + auto S = ton::ton_api::from_json(*collators_list_, json.get_object()); + if (S.is_error()) { + LOG(ERROR) << "Failed to parse collators list: " << S; + collators_list_ = {}; + return; + } + td::Ref list{true}; + list.write().unpack(*collators_list_); + validator_options_.write().set_collators_list(std::move(list)); +} + void ValidatorEngine::load_empty_local_config(td::Promise promise) { auto ret_promise = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { @@ -1794,6 +1818,7 @@ void ValidatorEngine::got_key(ton::PublicKey key) { void ValidatorEngine::start() { set_shard_check_function(); + load_collators_list(); read_config_ = true; start_adnl(); } @@ -3792,6 +3817,52 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setStateS }); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setCollatorsList &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + auto s = td::json_encode(td::ToJson(*query.list_), true); + auto S = td::write_file(collators_list_file(), s); + if (S.is_error()) { + promise.set_value(create_control_query_error(std::move(S))); + return; + } + + collators_list_ = std::move(query.list_); + td::Ref list{true}; + list.write().unpack(*collators_list_); + validator_options_.write().set_collators_list(std::move(list)); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCollatorsList &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + if (collators_list_) { + promise.set_value(ton::serialize_tl_object(collators_list_, true)); + } else { + auto list = ton::create_tl_object(); + list->self_collate_ = true; + promise.set_value(ton::serialize_tl_object(list, true)); + } +} + void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getValidatorSessionsInfo &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { @@ -4260,20 +4331,6 @@ int main(int argc, char *argv[]) { p.add_option('M', "not-all-shards", "monitor only a necessary set of shards instead of all", [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_not_all_shards); }); }); - p.add_option('\0', "lite-validator-shards", - "lite-mode validator for shard blocks (don't collate blocks, use collator nodes instead)", [&]() { - acts.push_back([&x]() { - td::actor::send_closure(x, &ValidatorEngine::set_validator_mode, - ton::validator::ValidatorManagerOptions::validator_lite_shards); - }); - }); - p.add_option('\0', "lite-validator-all", - "lite-mode validator for all blocks (don't collate blocks, use collator nodes instead)", [&]() { - acts.push_back([&x]() { - td::actor::send_closure(x, &ValidatorEngine::set_validator_mode, - ton::validator::ValidatorManagerOptions::validator_lite_all); - }); - }); td::uint32 threads = 7; p.add_checked_option( 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice fname) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index ebf8fb581..64154718a 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -184,6 +184,7 @@ class ValidatorEngine : public td::actor::Actor { td::Ref validator_options_; Config config_; ton::tl_object_ptr custom_overlays_config_; + ton::tl_object_ptr collators_list_; std::set running_gc_; @@ -235,8 +236,6 @@ class ValidatorEngine : public td::actor::Actor { ton::BlockSeqno truncate_seqno_{0}; std::string session_logs_file_; bool not_all_shards_ = false; - ton::validator::ValidatorManagerOptions::ValidatorMode validator_mode_ = - ton::validator::ValidatorManagerOptions::validator_normal; std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; @@ -319,9 +318,6 @@ class ValidatorEngine : public td::actor::Actor { void set_not_all_shards() { not_all_shards_ = true; } - void set_validator_mode(ton::validator::ValidatorManagerOptions::ValidatorMode mode) { - validator_mode_ = mode; - } void start_up() override; ValidatorEngine() { @@ -333,6 +329,7 @@ class ValidatorEngine : public td::actor::Actor { void load_local_config(td::Promise promise); void load_config(td::Promise promise); void set_shard_check_function(); + void load_collators_list(); void start(); @@ -411,6 +408,9 @@ class ValidatorEngine : public td::actor::Actor { std::string custom_overlays_config_file() const { return db_root_ + "/custom-overlays.json"; } + std::string collators_list_file() const { + return db_root_ + "/collators-list.json"; + } void load_custom_overlays_config(); td::Status write_custom_overlays_config(); @@ -514,6 +514,10 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_setStateSerializerEnabled &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_setCollatorsList &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_showCollatorsList &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator/fabric.h b/validator/fabric.h index e809be43f..a38b40568 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -26,7 +26,7 @@ namespace ton { namespace validator { -enum ValidateMode { fake = 1, full_collated_data = 2 }; +enum ValidateMode { fake = 1 }; enum CollateMode { skip_store_candidate = 1 }; td::actor::ActorOwn create_db_actor(td::actor::ActorId manager, std::string db_root_, diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index c0c4a2e83..85ced27d7 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -82,7 +82,6 @@ ValidateQuery::ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block , timeout(timeout) , main_promise(std::move(promise)) , is_fake_(mode & ValidateMode::fake) - , full_collated_data_(mode & ValidateMode::full_collated_data) , shard_pfx_(shard_.shard) , shard_pfx_len_(ton::shard_prefix_length(shard_)) , perf_timer_("validateblock", 0.1, [manager](double duration) { @@ -258,7 +257,6 @@ void ValidateQuery::finish_query() { */ void ValidateQuery::start_up() { LOG(WARNING) << "validate query for " << block_candidate.id.to_str() << " started"; - LOG(DEBUG) << "full_collated_data is " << full_collated_data_; alarm_timestamp() = timeout; rand_seed_.set_zero(); created_by_ = block_candidate.pubkey; @@ -359,11 +357,16 @@ void ValidateQuery::start_up() { td::actor::send_closure_later( std::move(self), &ValidateQuery::after_get_latest_mc_state, std::move(res)); }); - // 3. load state(s) corresponding to previous block(s) (not full-collaoted-data or masterchain) + // 3. unpack block candidate (while necessary data is being loaded) + if (!unpack_block_candidate()) { + reject_query("error unpacking block candidate"); + return; + } + // 4. load state(s) corresponding to previous block(s) (not full-collated-data or masterchain) prev_states.resize(prev_blocks.size()); if (is_masterchain() || !full_collated_data_) { for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { - // 3.1. load state + // 4.1. load state LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; ++pending; td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), @@ -374,11 +377,6 @@ void ValidateQuery::start_up() { }); } } - // 4. unpack block candidate (while necessary data is being loaded) - if (!unpack_block_candidate()) { - reject_query("error unpacking block candidate"); - return; - } // 5. request masterchain state referred to in the block if (!is_masterchain()) { ++pending; @@ -629,6 +627,7 @@ bool ValidateQuery::extract_collated_data_from(Ref croot, int idx) { if (!ins.second) { return reject_query("Merkle proof with duplicate virtual root hash "s + virt_hash.to_hex()); } + full_collated_data_ = true; return true; } if (block::gen::t_TopBlockDescrSet.has_valid_tag(cs)) { @@ -666,6 +665,9 @@ bool ValidateQuery::extract_collated_data() { return reject_query(PSTRING() << "virtualization error " << err.get_msg()); } } + if (full_collated_data_) { + LOG(INFO) << "full_collated_data = true"; + } return true; } diff --git a/validator/manager.cpp b/validator/manager.cpp index aa46e9e80..bde7a8056 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -373,7 +373,8 @@ void ValidatorManagerImpl::get_key_block_proof_link(BlockIdExt block_id, td::Pro } void ValidatorManagerImpl::new_external_message(td::BufferSlice data, int priority) { - if (!is_collator()) { + if (!validating_masterchain() && collator_nodes_.empty() && + (!is_validator() || !opts_->get_collators_list()->self_collate)) { return; } if (last_masterchain_state_.is_null()) { @@ -453,7 +454,7 @@ void ValidatorManagerImpl::check_external_message(td::BufferSlice data, td::Prom } void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { - if (!is_collator()) { + if (collator_nodes_.empty() && (!is_validator() || !opts_->get_collators_list()->self_collate)) { return; } auto R = create_ihr_message(std::move(data)); @@ -470,7 +471,7 @@ void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { } void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { - if (!is_collator() && !is_validator()) { + if (!is_validator()) { return; } if (!last_masterchain_block_handle_) { @@ -533,7 +534,7 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refblock_id(), 0, td::Timestamp::in(60.0), std::move(P)); } - if (collating_masterchain()) { + if (validating_masterchain()) { preload_msg_queue_to_masterchain(desc); } } @@ -542,7 +543,8 @@ void ValidatorManagerImpl::add_shard_block_description(td::Ref desc) { auto id = ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}; auto it = shard_blocks_.find(id); - if (!collating_masterchain() || it == shard_blocks_.end() || it->second.latest_desc->block_id() != desc->block_id()) { + if (!validating_masterchain() || it == shard_blocks_.end() || + it->second.latest_desc->block_id() != desc->block_id()) { return; } wait_neighbor_msg_queue_proofs( @@ -2031,9 +2033,9 @@ void ValidatorManagerImpl::new_masterchain_block() { td::actor::send_closure(shard_client_, &ShardClient::new_masterchain_block_notification, last_masterchain_block_handle_, last_masterchain_state_); } - if (is_collator()) { + if (validating_masterchain() || !collator_nodes_.empty()) { std::set collating_shards; - if (collating_masterchain()) { + if (validating_masterchain()) { collating_shards.emplace(masterchainId); } for (const auto &collator : collator_nodes_) { @@ -2398,7 +2400,8 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group auto G = td::actor::create_actor( "validatorgroup", shard, validator_id, session_id, validator_set, last_masterchain_state_->get_collator_config(true), opts, keyring_, adnl_, rldp_, overlays_, db_root_, - actor_id(this), init_session, opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_); + actor_id(this), init_session, opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_, + opts_->need_monitor(shard, last_masterchain_state_)); return G; } } @@ -2845,27 +2848,12 @@ bool ValidatorManagerImpl::is_validator() { return temp_keys_.size() > 0 || permanent_keys_.size() > 0; } -bool ValidatorManagerImpl::is_collator() { - return !collator_nodes_.empty() || - (opts_->validator_mode() != ValidatorManagerOptions::validator_lite_all && is_validator()); -} - bool ValidatorManagerImpl::validating_masterchain() { return !get_validator(ShardIdFull(masterchainId), last_masterchain_state_->get_validator_set(ShardIdFull(masterchainId))) .is_zero(); } -bool ValidatorManagerImpl::collating_masterchain() { - if (masterchain_collators_) { - return true; - } - if (opts_->validator_mode() == ValidatorManagerOptions::validator_lite_all) { - return false; - } - return validating_masterchain(); -} - PublicKeyHash ValidatorManagerImpl::get_validator(ShardIdFull shard, td::Ref val_set) { for (auto &key : temp_keys_) { if (val_set->is_validator(key.bits256_value())) { @@ -3331,6 +3319,14 @@ void ValidatorManagerImpl::update_options(td::Ref opts) if (!queue_size_counter_.empty()) { td::actor::send_closure(queue_size_counter_, &QueueSizeCounter::update_options, opts); } + for (auto &group : validator_groups_) { + td::actor::send_closure(group.second.actor, &ValidatorGroup::update_options, opts, + opts->need_monitor(group.second.shard, last_masterchain_state_)); + } + for (auto &group : next_validator_groups_) { + td::actor::send_closure(group.second.actor, &ValidatorGroup::update_options, opts, + opts->need_monitor(group.second.shard, last_masterchain_state_)); + } opts_ = std::move(opts); } @@ -3376,9 +3372,6 @@ void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh if (!it->second.shards.insert(shard).second) { return; } - if (shard.is_masterchain()) { - ++masterchain_collators_; - } td::actor::send_closure(it->second.actor, &CollatorNode::add_shard, shard); } @@ -3390,9 +3383,6 @@ void ValidatorManagerImpl::del_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh if (!it->second.shards.erase(shard)) { return; } - if (shard.is_masterchain()) { - --masterchain_collators_; - } if (it->second.shards.empty()) { collator_nodes_.erase(it); } else { diff --git a/validator/manager.hpp b/validator/manager.hpp index dc18e6278..4e5d1d3e7 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -560,9 +560,7 @@ class ValidatorManagerImpl : public ValidatorManager { void read_gc_list(std::vector list); bool is_validator(); - bool is_collator(); bool validating_masterchain(); - bool collating_masterchain(); PublicKeyHash get_validator(ShardIdFull shard, td::Ref val_set); ValidatorManagerImpl(td::Ref opts, std::string db_root, @@ -767,7 +765,6 @@ class ValidatorManagerImpl : public ValidatorManager { std::set shards; }; std::map collator_nodes_; - size_t masterchain_collators_ = 0; std::set extra_active_shards_; std::map last_validated_blocks_; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index fc1fb0090..e4ae88dd4 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -57,14 +57,7 @@ void ValidatorGroup::generate_block_candidate( cache = cached_collated_block_](td::Result R) { td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_candidate, std::move(cache), std::move(R)); }; - if (opts_->validator_mode() == ValidatorManagerOptions::validator_lite_all || - (opts_->validator_mode() == ValidatorManagerOptions::validator_lite_shards && !shard_.is_masterchain())) { - send_collate_query(round_id, td::Timestamp::in(10.0), std::move(P)); - return; - } - run_collate_query(shard_, min_masterchain_block_id_, prev_block_ids_, - Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, validator_set_, manager_, - td::Timestamp::in(10.0), std::move(P)); + collate_block(round_id, td::Timestamp::in(10.0), std::move(P)); } void ValidatorGroup::generated_block_candidate(std::shared_ptr cache, @@ -142,8 +135,7 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat } VLOG(VALIDATOR_DEBUG) << "validating block candidate " << next_block_id; run_validate_query(shard_, min_masterchain_block_id_, prev_block_ids_, std::move(block), validator_set_, manager_, - td::Timestamp::in(15.0), std::move(P), - collator_config_.full_collated_data ? ValidateMode::full_collated_data : 0); + td::Timestamp::in(15.0), std::move(P)); } void ValidatorGroup::update_approve_cache(CacheKey key, UnixTime value) { @@ -204,10 +196,8 @@ void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref } }); - run_accept_block_query( - block_id, std::move(block), std::move(prev), validator_set_, std::move(sig_set), std::move(approve_sig_set), - send_broadcast, shard_.is_masterchain() || opts_->validator_mode() == ValidatorManagerOptions::validator_normal, - manager_, std::move(P)); + run_accept_block_query(block_id, std::move(block), std::move(prev), validator_set_, std::move(sig_set), + std::move(approve_sig_set), send_broadcast, apply_blocks_, manager_, std::move(P)); } void ValidatorGroup::skip_round(td::uint32 round_id) { @@ -480,25 +470,55 @@ void ValidatorGroup::get_session_info( td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_session_info, std::move(P)); } -void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeout, td::Promise promise, - unsigned max_retries) { +void ValidatorGroup::collate_block(td::uint32 round_id, td::Timestamp timeout, td::Promise promise, + unsigned max_retries) { if (round_id < last_known_round_id_) { promise.set_error(td::Status::Error("too old")); return; } BlockId next_block_id = create_next_block_id_simple(); - adnl::AdnlNodeIdShort collator = adnl::AdnlNodeIdShort::zero(); - // TODO: some way to choose node (similar to "unreliability" in full-node) - int cnt = 0; - for (const block::CollatorNodeDescr &c : collator_config_.collator_nodes) { - if (shard_intersects(shard_, c.shard)) { - if (td::Random::fast(0, cnt) == 0) { - collator = adnl::AdnlNodeIdShort(c.adnl_id); + adnl::AdnlNodeIdShort collator_adnl_id = adnl::AdnlNodeIdShort::zero(); + bool self_collate = false; + bool trusted_collator = false; + + if (shard_.is_masterchain()) { + self_collate = true; + } else { + for (const auto &s : opts_->get_collators_list()->shards) { + if (!shard_intersects(s.shard_id, shard_)) { + continue; + } + if (!s.collators.empty()) { + const CollatorsList::Collator &col = s.collators[td::Random::fast(0, s.collators.size() - 1)]; + collator_adnl_id = col.adnl_id; + trusted_collator = col.trusted; + break; } - ++cnt; } + if (collator_adnl_id.is_zero()) { + if (opts_->get_collators_list()->self_collate) { + self_collate = true; + } else if (opts_->get_collators_list()->use_config_41) { + // TODO: some way to choose node (similar to "unreliability" in full-node) + int cnt = 0; + for (const block::CollatorNodeDescr &c : collator_config_.collator_nodes) { + if (shard_intersects(shard_, c.shard)) { + if (td::Random::fast(0, cnt) == 0) { + collator_adnl_id = adnl::AdnlNodeIdShort(c.adnl_id); + } + ++cnt; + } + } + } + } + } + if (self_collate) { + run_collate_query(shard_, min_masterchain_block_id_, prev_block_ids_, + Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, validator_set_, manager_, + td::Timestamp::in(10.0), std::move(promise)); + return; } - if (collator.is_zero()) { + if (collator_adnl_id.is_zero()) { promise.set_error(td::Status::Error(PSTRING() << "no collator for shard " << shard_.to_str())); return; } @@ -514,7 +534,7 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo LOG(WARNING) << "collate query for " << next_block_id.to_str() << ": " << R.error() << ", time=" << timer.elapsed() << "s, " << (retry ? "retrying" : "giving up"); if (retry) { - td::actor::send_closure(SelfId, &ValidatorGroup::send_collate_query, round_id, timeout, std::move(promise), + td::actor::send_closure(SelfId, &ValidatorGroup::collate_block, round_id, timeout, std::move(promise), max_retries - 1); } else { promise.set_result(td::Status::Error(ErrorCode::timeout, "timeout")); @@ -530,23 +550,23 @@ void ValidatorGroup::send_collate_query(td::uint32 round_id, td::Timestamp timeo local_id_full_.ed25519_value().raw()); auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), round_id, promise = std::move(promise)](td::Result R) mutable { + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error_prefix("rldp query failed: ")); return; } td::actor::send_closure(SelfId, &ValidatorGroup::receive_collate_query_response, round_id, R.move_as_ok(), - std::move(promise)); + trusted_collator, std::move(promise)); }); - LOG(INFO) << "sending collate query for " << next_block_id.to_str() << ": send to " << collator; + LOG(INFO) << "sending collate query for " << next_block_id.to_str() << ": send to " << collator_adnl_id; size_t max_answer_size = config_.max_block_size + config_.max_collated_data_size + 1024; td::Timestamp query_timeout = td::Timestamp::in(10.0); query_timeout.relax(timeout); - td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_adnl_id_, collator, "collatequery", std::move(P), - timeout, std::move(query), max_answer_size); + td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_adnl_id_, collator_adnl_id, "collatequery", + std::move(P), timeout, std::move(query), max_answer_size); } -void ValidatorGroup::receive_collate_query_response(td::uint32 round_id, td::BufferSlice data, +void ValidatorGroup::receive_collate_query_response(td::uint32 round_id, td::BufferSlice data, bool trusted_collator, td::Promise promise) { if (round_id < last_known_round_id_) { promise.set_error(td::Status::Error("too old")); @@ -574,6 +594,10 @@ void ValidatorGroup::receive_collate_query_response(td::uint32 round_id, td::Buf return; } + if (trusted_collator) { + promise.set_result(std::move(candidate)); + return; + } auto P = td::PromiseCreator::lambda( [candidate = candidate.clone(), promise = std::move(promise)](td::Result> R) mutable { if (R.is_error()) { diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index d4080f6a1..627573427 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -66,13 +66,18 @@ class ValidatorGroup : public td::actor::Actor { void get_session_info(td::Promise> promise); + void update_options(td::Ref opts, bool apply_blocks) { + opts_ = std::move(opts); + apply_blocks_ = apply_blocks; + } + ValidatorGroup(ShardIdFull shard, PublicKeyHash local_id, ValidatorSessionId session_id, td::Ref validator_set, block::CollatorConfig collator_config, validatorsession::ValidatorSessionOptions config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, td::actor::ActorId validator_manager, bool create_session, - bool allow_unsafe_self_blocks_resync, td::Ref opts) + bool allow_unsafe_self_blocks_resync, td::Ref opts, bool apply_blocks) : shard_(shard) , local_id_(std::move(local_id)) , session_id_(session_id) @@ -87,14 +92,16 @@ class ValidatorGroup : public td::actor::Actor { , manager_(validator_manager) , init_(create_session) , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) - , opts_(std::move(opts)) { + , opts_(std::move(opts)) + , apply_blocks_(apply_blocks) { } private: std::unique_ptr make_validator_session_callback(); - void send_collate_query(td::uint32 round_id, td::Timestamp timeout, td::Promise promise, + void collate_block(td::uint32 round_id, td::Timestamp timeout, td::Promise promise, unsigned max_retries = 4); - void receive_collate_query_response(td::uint32 round_id, td::BufferSlice data, td::Promise promise); + void receive_collate_query_response(td::uint32 round_id, td::BufferSlice data, bool trusted_collator, + td::Promise promise); struct PostponedAccept { RootHash root_hash; @@ -134,6 +141,7 @@ class ValidatorGroup : public td::actor::Actor { bool allow_unsafe_self_blocks_resync_; td::Ref opts_; td::uint32 last_known_round_id_ = 0; + bool apply_blocks_ = true; struct CachedCollatedBlock { td::optional result; diff --git a/validator/validator-options.cpp b/validator/validator-options.cpp index cb26fe44d..4f9b8c538 100644 --- a/validator/validator-options.cpp +++ b/validator/validator-options.cpp @@ -20,10 +20,26 @@ #include "ton/ton-shard.h" +#include + namespace ton { namespace validator { +void CollatorsList::unpack(const ton_api::engine_validator_collatorsList& obj) { + shards.clear(); + self_collate = obj.self_collate_; + use_config_41 = obj.use_config_41_; + for (const auto& shard_obj : obj.shards_) { + shards.emplace_back(); + Shard& shard = shards.back(); + shard.shard_id = create_shard_id(shard_obj->shard_id_); + for (const auto& collator : shard_obj->collators_) { + shard.collators.push_back({adnl::AdnlNodeIdShort{collator->adnl_id_}, collator->trusted_}); + } + } +} + td::Ref ValidatorManagerOptions::create( BlockIdExt zero_block_id, BlockIdExt init_block_id, std::function check_shard, bool allow_blockchain_init, diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 55b40a339..e67c4e2d0 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -142,8 +142,8 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool get_state_serializer_enabled() const override { return state_serializer_enabled_; } - ValidatorMode validator_mode() const override { - return validator_mode_; + td::Ref get_collators_list() const override { + return collators_list_; } void set_zero_block_id(BlockIdExt block_id) override { @@ -228,8 +228,8 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_state_serializer_enabled(bool value) override { state_serializer_enabled_ = value; } - void set_validator_mode(ValidatorMode value) override { - validator_mode_ = value; + void set_collators_list(td::Ref list) override { + collators_list_ = std::move(list); } ValidatorManagerOptionsImpl *make_copy() const override { @@ -281,7 +281,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool celldb_preload_all_ = false; td::optional catchain_max_block_delay_; bool state_serializer_enabled_ = true; - ValidatorMode validator_mode_ = validator_normal; + td::Ref collators_list_{true, CollatorsList{}}; }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index c7ef2ce87..77b39607d 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -52,10 +52,24 @@ struct PerfTimerStats { std::deque> stats; // }; +struct CollatorsList : public td::CntObject { + struct Collator { + adnl::AdnlNodeIdShort adnl_id; + bool trusted; + }; + struct Shard { + ShardIdFull shard_id; + std::vector collators; + }; + bool self_collate = true; + bool use_config_41 = false; + std::vector shards; + + void unpack(const ton_api::engine_validator_collatorsList& obj); +}; + struct ValidatorManagerOptions : public td::CntObject { public: - enum ValidatorMode { validator_normal, validator_lite_shards, validator_lite_all }; - virtual BlockIdExt zero_block_id() const = 0; virtual BlockIdExt init_block_id() const = 0; virtual bool need_monitor(ShardIdFull shard, const td::Ref& state) const = 0; @@ -91,7 +105,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual bool get_celldb_preload_all() const = 0; virtual td::optional get_catchain_max_block_delay() const = 0; virtual bool get_state_serializer_enabled() const = 0; - virtual ValidatorMode validator_mode() const = 0; + virtual td::Ref get_collators_list() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; @@ -120,7 +134,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_celldb_preload_all(bool value) = 0; virtual void set_catchain_max_block_delay(double value) = 0; virtual void set_state_serializer_enabled(bool value) = 0; - virtual void set_validator_mode(ValidatorMode value) = 0; + virtual void set_collators_list(td::Ref list) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, From 007f1fb1d71e221dc2a5a680bf997f0031a6bc94 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 12 Jun 2024 18:12:45 +0300 Subject: [PATCH 094/388] New liteserver config format * Specify shards and seqno/utime/lt limits for liteservers in global config * Support in lite-client, tonlib, blockchain-explorer * Rework proxy-liteserver --- .../blockchain-explorer-query.cpp | 146 ++-- blockchain-explorer/blockchain-explorer.cpp | 54 +- blockchain-explorer/blockchain-explorer.hpp | 2 +- lite-client/CMakeLists.txt | 6 +- lite-client/QueryTraits.h | 229 ------ lite-client/ext-client.cpp | 151 ++-- lite-client/ext-client.h | 19 +- lite-client/lite-client.cpp | 689 +++++++----------- lite-client/lite-client.h | 34 +- lite-client/query-utils.cpp | 394 ++++++++++ lite-client/query-utils.hpp | 89 +++ tl-utils/common-utils.hpp | 33 + tl/generate/scheme/ton_api.tl | 6 +- tl/generate/scheme/ton_api.tlo | Bin 97828 -> 98440 bytes tl/generate/scheme/tonlib_api.tl | 2 +- tl/generate/scheme/tonlib_api.tlo | Bin 33292 -> 33224 bytes tonlib/tonlib/Config.cpp | 63 +- tonlib/tonlib/Config.h | 3 +- tonlib/tonlib/ExtClient.cpp | 4 +- tonlib/tonlib/ExtClient.h | 8 +- tonlib/tonlib/ExtClientOutbound.cpp | 7 +- tonlib/tonlib/ExtClientOutbound.h | 3 +- tonlib/tonlib/TonlibClient.cpp | 26 +- tonlib/tonlib/TonlibClient.h | 2 +- tonlib/tonlib/tonlib-cli.cpp | 2 +- utils/proxy-liteserver.cpp | 321 ++++---- 26 files changed, 1175 insertions(+), 1118 deletions(-) delete mode 100644 lite-client/QueryTraits.h create mode 100644 lite-client/query-utils.cpp create mode 100644 lite-client/query-utils.hpp diff --git a/blockchain-explorer/blockchain-explorer-query.cpp b/blockchain-explorer/blockchain-explorer-query.cpp index 920ae2c7f..1808a3c46 100644 --- a/blockchain-explorer/blockchain-explorer-query.cpp +++ b/blockchain-explorer/blockchain-explorer-query.cpp @@ -43,7 +43,6 @@ #include "block/block-auto.h" #include "crypto/vm/utils.h" #include "td/utils/crypto.h" -#include "lite-client/QueryTraits.h" #include "vm/boc.h" #include "vm/cellops.h" @@ -212,7 +211,8 @@ void HttpQueryBlockData::finish_query() { } void HttpQueryBlockData::start_up() { - auto query = ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)); + auto query = ton::serialize_tl_object( + ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)), true); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { @@ -223,7 +223,7 @@ void HttpQueryBlockData::start_up() { }); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + std::move(query), std::move(P)); } void HttpQueryBlockData::got_block_data(td::BufferSlice data) { @@ -274,7 +274,8 @@ void HttpQueryBlockView::finish_query() { } void HttpQueryBlockView::start_up_query() { - auto query = ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)); + auto query = ton::serialize_tl_object( + ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)), true); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { @@ -285,7 +286,7 @@ void HttpQueryBlockView::start_up_query() { }); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + std::move(query), std::move(P)); } void HttpQueryBlockView::got_block_data(td::BufferSlice data) { @@ -321,10 +322,11 @@ void HttpQueryBlockInfo::start_up_query() { td::actor::send_closure(SelfId, &HttpQueryBlockInfo::got_block_header, R.move_as_ok()); } }); - auto query = - ton::create_tl_object(ton::create_tl_lite_block_id(block_id_), 0); + auto query = ton::serialize_tl_object( + ton::create_tl_object(ton::create_tl_lite_block_id(block_id_), 0), + true); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + std::move(query), std::move(P)); pending_queries_ = 1; if (block_id_.is_masterchain()) { @@ -336,15 +338,16 @@ void HttpQueryBlockInfo::start_up_query() { td::actor::send_closure(SelfId, &HttpQueryBlockInfo::got_shard_info, R.move_as_ok()); } }); - auto query_2 = - ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)); + auto query_2 = ton::serialize_tl_object( + ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)), + true); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query_2, true), liteclient::get_query_shard(*query_2), - std::move(P_2)); + std::move(query_2), std::move(P_2)); pending_queries_++; } - auto query_3 = ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), 7, 1024, nullptr, false, false); + auto query_3 = ton::serialize_tl_object(ton::create_tl_object( + ton::create_tl_lite_block_id(block_id_), 7, 1024, nullptr, false, false), + true); auto P_3 = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &HttpQueryBlockInfo::abort_query, R.move_as_error_prefix("litequery failed: ")); @@ -353,8 +356,7 @@ void HttpQueryBlockInfo::start_up_query() { } }); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query_3, true), liteclient::get_query_shard(*query_3), - std::move(P_3)); + std::move(query_3), std::move(P_3)); pending_queries_++; } @@ -407,9 +409,11 @@ void HttpQueryBlockInfo::got_transactions(td::BufferSlice data) { if (f->incomplete_ && transactions_.size() > 0) { const auto &T = *transactions_.rbegin(); - auto query_3 = ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), 7 + 128, 1024, - ton::create_tl_object(T.addr.addr, T.lt), false, false); + auto query_3 = ton::serialize_tl_object( + ton::create_tl_object( + ton::create_tl_lite_block_id(block_id_), 7 + 128, 1024, + ton::create_tl_object(T.addr.addr, T.lt), false, false), + true); auto P_3 = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &HttpQueryBlockInfo::abort_query, R.move_as_error_prefix("litequery failed: ")); @@ -418,8 +422,7 @@ void HttpQueryBlockInfo::got_transactions(td::BufferSlice data) { } }); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query_3, true), liteclient::get_query_shard(*query_3), - std::move(P_3)); + std::move(query_3), std::move(P_3)); } else { if (!--pending_queries_) { finish_query(); @@ -539,13 +542,14 @@ void HttpQueryBlockSearch::start_up_query() { td::actor::send_closure(SelfId, &HttpQueryBlockSearch::got_block_header, R.move_as_ok()); } }); - auto query = ton::create_tl_object( - mode_, - ton::create_tl_lite_block_id_simple( - ton::BlockId{account_prefix_.workchain, account_prefix_.account_id_prefix, seqno_}), - lt_, utime_); + auto query = ton::serialize_tl_object(ton::create_tl_object( + mode_, + ton::create_tl_lite_block_id_simple(ton::BlockId{ + account_prefix_.workchain, account_prefix_.account_id_prefix, seqno_}), + lt_, utime_), + true); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + std::move(query), std::move(P)); } void HttpQueryBlockSearch::got_block_header(td::BufferSlice data) { @@ -567,16 +571,17 @@ void HttpQueryBlockSearch::got_block_header(td::BufferSlice data) { td::actor::send_closure(SelfId, &HttpQueryBlockSearch::got_shard_info, R.move_as_ok()); } }); - auto query_2 = - ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)); + auto query_2 = ton::serialize_tl_object( + ton::create_tl_object(ton::create_tl_lite_block_id(block_id_)), + true); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query_2, true), liteclient::get_query_shard(*query_2), - std::move(P_2)); + std::move(query_2), std::move(P_2)); pending_queries_++; } - auto query_3 = ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), 7, 1024, nullptr, false, false); + auto query_3 = ton::serialize_tl_object(ton::create_tl_object( + ton::create_tl_lite_block_id(block_id_), 7, 1024, nullptr, false, false), + true); auto P_3 = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &HttpQueryBlockSearch::abort_query, R.move_as_error_prefix("litequery failed: ")); @@ -585,8 +590,7 @@ void HttpQueryBlockSearch::got_block_header(td::BufferSlice data) { } }); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query_3, true), liteclient::get_query_shard(*query_3), - std::move(P_3)); + std::move(query_3), std::move(P_3)); pending_queries_++; } @@ -626,9 +630,11 @@ void HttpQueryBlockSearch::got_transactions(td::BufferSlice data) { if (f->incomplete_ && transactions_.size() > 0) { const auto &T = *transactions_.rbegin(); - auto query_3 = ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), 7 + 128, 1024, - ton::create_tl_object(T.addr.addr, T.lt), false, false); + auto query_3 = ton::serialize_tl_object( + ton::create_tl_object( + ton::create_tl_lite_block_id(block_id_), 7 + 128, 1024, + ton::create_tl_object(T.addr.addr, T.lt), false, false), + true); auto P_3 = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &HttpQueryBlockSearch::abort_query, @@ -638,8 +644,7 @@ void HttpQueryBlockSearch::got_transactions(td::BufferSlice data) { } }); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query_3, true), liteclient::get_query_shard(*query_3), - std::move(P_3)); + std::move(query_3), std::move(P_3)); } else { if (!--pending_queries_) { finish_query(); @@ -727,10 +732,11 @@ void HttpQueryViewAccount::start_up_query() { } }); auto a = ton::create_tl_object(addr_.workchain, addr_.addr); - auto query = ton::create_tl_object(ton::create_tl_lite_block_id(block_id_), - std::move(a)); + auto query = ton::serialize_tl_object(ton::create_tl_object( + ton::create_tl_lite_block_id(block_id_), std::move(a)), + true); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + std::move(query), std::move(P)); } void HttpQueryViewAccount::got_account(td::BufferSlice data) { @@ -823,9 +829,10 @@ void HttpQueryViewTransaction::start_up_query() { } }); auto a = ton::create_tl_object(addr_.workchain, addr_.addr); - auto query = ton::create_tl_object(1, std::move(a), lt_, hash_); + auto query = ton::serialize_tl_object( + ton::create_tl_object(1, std::move(a), lt_, hash_), true); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + std::move(query), std::move(P)); } void HttpQueryViewTransaction::got_transaction(td::BufferSlice data) { @@ -913,10 +920,11 @@ void HttpQueryViewTransaction2::start_up_query() { } }); auto a = ton::create_tl_object(addr_.workchain, addr_.addr); - auto query = ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), std::move(a), lt_); + auto query = ton::serialize_tl_object(ton::create_tl_object( + ton::create_tl_lite_block_id(block_id_), std::move(a), lt_), + true); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + std::move(query), std::move(P)); } void HttpQueryViewTransaction2::got_transaction(td::BufferSlice data) { @@ -975,9 +983,9 @@ void HttpQueryViewLastBlock::start_up() { } }); - auto query = ton::create_tl_object(); + auto query = ton::serialize_tl_object(ton::create_tl_object(), true); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + std::move(query), std::move(P)); } void HttpQueryViewLastBlock::got_result(td::BufferSlice data) { @@ -1041,9 +1049,9 @@ void HttpQueryConfig::start_up() { } }); - auto query = ton::create_tl_object(); + auto query = ton::serialize_tl_object(ton::create_tl_object(), true); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + std::move(query), std::move(P)); } } @@ -1067,17 +1075,16 @@ void HttpQueryConfig::send_main_query() { td::actor::send_closure(SelfId, &HttpQueryConfig::got_result, R.move_as_ok()); } }); - if (params_.size() > 0) { - auto query = ton::create_tl_object( - 0, ton::create_tl_lite_block_id(block_id_), std::vector(params_)); - td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); - } else { - auto query = - ton::create_tl_object(0, ton::create_tl_lite_block_id(block_id_)); - td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); - } + auto query = + params_.size() > 0 + ? ton::serialize_tl_object(ton::create_tl_object( + 0, ton::create_tl_lite_block_id(block_id_), std::vector(params_)), + true) + : ton::serialize_tl_object(ton::create_tl_object( + 0, ton::create_tl_lite_block_id(block_id_)), + true); + td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, + std::move(query), std::move(P)); } void HttpQueryConfig::got_result(td::BufferSlice data) { @@ -1219,9 +1226,10 @@ void HttpQuerySend::start_up() { td::actor::send_closure(SelfId, &HttpQuerySend::got_result, R.move_as_ok()); } }); - auto query = ton::create_tl_object(std::move(data_)); + auto query = + ton::serialize_tl_object(ton::create_tl_object(std::move(data_)), true); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + std::move(query), std::move(P)); } void HttpQuerySend::got_result(td::BufferSlice data) { @@ -1327,10 +1335,12 @@ void HttpQueryRunMethod::start_up_query() { return abort_query(params_serialized.move_as_error_prefix("cannot serialize stack with get-method parameters : ")); } - auto query = ton::create_tl_object( - 0x17, ton::create_tl_lite_block_id(block_id_), std::move(a), method_id, params_serialized.move_as_ok()); + auto query = ton::serialize_tl_object( + ton::create_tl_object( + 0x17, ton::create_tl_lite_block_id(block_id_), std::move(a), method_id, params_serialized.move_as_ok()), + true); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, - ton::serialize_tl_object(query, true), liteclient::get_query_shard(*query), std::move(P)); + std::move(query), std::move(P)); } void HttpQueryRunMethod::got_result(td::BufferSlice data) { diff --git a/blockchain-explorer/blockchain-explorer.cpp b/blockchain-explorer/blockchain-explorer.cpp index c22ce1b3c..029c718d6 100644 --- a/blockchain-explorer/blockchain-explorer.cpp +++ b/blockchain-explorer/blockchain-explorer.cpp @@ -155,8 +155,6 @@ class CoreActor : public CoreActorInterface { td::int32 attempt_ = 0; td::int32 waiting_ = 0; - std::vector ready_; - //void run_queries(); void got_result(td::uint32 idx, td::int32 attempt, td::Result data); @@ -189,12 +187,6 @@ class CoreActor : public CoreActorInterface { static CoreActor* instance_; td::actor::ActorId self_id_; - void conn_ready(td::uint32 idx) { - ready_.at(idx) = true; - } - void conn_closed(td::uint32 idx) { - ready_.at(idx) = false; - } void set_global_config(std::string str) { global_config_ = str; } @@ -219,7 +211,7 @@ class CoreActor : public CoreActorInterface { hide_ips_ = value; } - void send_lite_query(td::BufferSlice query, ton::ShardIdFull shard, td::Promise promise) override; + void send_lite_query(td::BufferSlice query, td::Promise promise) override; void get_last_result(td::Promise> promise) override { } void get_results(td::uint32 max, td::Promise promise) override { @@ -439,50 +431,24 @@ class CoreActor : public CoreActorInterface { } void run() { - std::vector servers; + std::vector servers; if (remote_public_key_.empty()) { auto G = td::read_file(global_config_).move_as_ok(); auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); ton::ton_api::liteclient_config_global gc; ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); - - size_t size = gc.liteservers_.size() + gc.liteservers_v2_.size(); - CHECK(size > 0); - ready_.resize(size, false); - - for (auto& s : gc.liteservers_) { - td::IPAddress addr; - addr.init_host_port(td::IPAddress::ipv4_to_str(s->ip_), s->port_).ensure(); - addrs_.push_back(addr); - liteclient::ExtClient::LiteServer serv; - serv.address = addr; - serv.adnl_id = ton::adnl::AdnlNodeIdFull::create(s->id_).move_as_ok(); - servers.push_back(std::move(serv)); - } - for (auto& s : gc.liteservers_v2_) { - td::IPAddress addr; - addr.init_host_port(td::IPAddress::ipv4_to_str(s->ip_), s->port_).ensure(); - addrs_.push_back(addr); - liteclient::ExtClient::LiteServer serv; - serv.address = addr; - serv.adnl_id = ton::adnl::AdnlNodeIdFull::create(s->id_).move_as_ok(); - serv.is_full = false; - for (auto& shard : s->shards_) { - serv.shards.emplace_back(shard->workchain_, (ton::ShardId)shard->shard_); - CHECK(serv.shards.back().is_valid_ext()); - } - servers.push_back(std::move(serv)); + auto r_servers = liteclient::LiteServerConfig::parse_global_config(gc); + r_servers.ensure(); + servers = r_servers.move_as_ok(); + for (const auto& serv : servers) { + addrs_.push_back(serv.addr); } } else { if (!remote_addr_.is_valid()) { LOG(FATAL) << "remote addr not set"; } - ready_.resize(1, false); addrs_.push_back(remote_addr_); - liteclient::ExtClient::LiteServer serv; - serv.address = remote_addr_; - serv.adnl_id = ton::adnl::AdnlNodeIdFull{remote_public_key_}; - servers.push_back(std::move(serv)); + servers.push_back(liteclient::LiteServerConfig{ton::adnl::AdnlNodeIdFull{remote_public_key_}, remote_addr_}); } client_ = liteclient::ExtClient::create(std::move(servers), make_callback()); daemon_ = MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, static_cast(http_port_), nullptr, nullptr, @@ -545,7 +511,7 @@ void CoreActor::got_result(td::uint32 idx, td::int32 attempt, td::Result promise) { +void CoreActor::send_lite_query(td::BufferSlice query, td::Promise promise) { auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); @@ -563,7 +529,7 @@ void CoreActor::send_lite_query(td::BufferSlice query, ton::ShardIdFull shard, t promise.set_value(std::move(B)); }); auto q = ton::create_tl_object(std::move(query)); - td::actor::send_closure(client_, &liteclient::ExtClient::send_query, "query", serialize_tl_object(q, true), shard, + td::actor::send_closure(client_, &liteclient::ExtClient::send_query, "query", serialize_tl_object(q, true), td::Timestamp::in(10.0), std::move(P)); } diff --git a/blockchain-explorer/blockchain-explorer.hpp b/blockchain-explorer/blockchain-explorer.hpp index d3c53e451..1bae362de 100644 --- a/blockchain-explorer/blockchain-explorer.hpp +++ b/blockchain-explorer/blockchain-explorer.hpp @@ -64,7 +64,7 @@ class CoreActorInterface : public td::actor::Actor { }; virtual ~CoreActorInterface() = default; - virtual void send_lite_query(td::BufferSlice data, ton::ShardIdFull shard, td::Promise promise) = 0; + virtual void send_lite_query(td::BufferSlice data, td::Promise promise) = 0; virtual void get_last_result(td::Promise> promise) = 0; virtual void get_results(td::uint32 max, td::Promise promise) = 0; diff --git a/lite-client/CMakeLists.txt b/lite-client/CMakeLists.txt index 5774c050b..5b0c0fa91 100644 --- a/lite-client/CMakeLists.txt +++ b/lite-client/CMakeLists.txt @@ -1,7 +1,9 @@ cmake_minimum_required(VERSION 3.5 FATAL_ERROR) -add_library(lite-client-common STATIC lite-client-common.cpp lite-client-common.h ext-client.cpp ext-client.h QueryTraits.h) -target_link_libraries(lite-client-common PUBLIC tdutils tdactor adnllite tl_api tl_lite_api tl-lite-utils ton_crypto ton_block) +add_library(lite-client-common STATIC lite-client-common.cpp lite-client-common.h ext-client.cpp ext-client.h + query-utils.hpp query-utils.cpp) +target_link_libraries(lite-client-common PUBLIC tdutils tdactor adnllite tl_api tl_lite_api tl-lite-utils ton_crypto + ton_block) add_executable(lite-client lite-client.cpp lite-client.h ext-client.h ext-client.cpp) target_link_libraries(lite-client tdutils tdactor adnllite tl_api tl_lite_api tl-lite-utils ton_crypto ton_block diff --git a/lite-client/QueryTraits.h b/lite-client/QueryTraits.h deleted file mode 100644 index 54190244e..000000000 --- a/lite-client/QueryTraits.h +++ /dev/null @@ -1,229 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . -*/ -#pragma once -#include "ton/ton-types.h" -#include "auto/tl/lite_api.h" -#include "auto/tl/lite_api.hpp" -#include "vm/boc.h" -#include "vm/cellslice.h" -#include "block/block-auto.h" -#include "block/block-parse.h" -#include "auto/tl/lite_api.hpp" - -namespace liteclient { - -template -struct QueryTraits { - static ton::ShardIdFull get_shard(const Query& q) { - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getMasterchainInfo& q) { - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getMasterchainInfoExt& q) { - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getTime& q) { - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getVersion& q) { - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getBlock& q) { - return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getState& q) { - return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getBlockHeader& q) { - return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_sendMessage& q) { - auto shard = [&]() -> td::Result { - vm::BagOfCells boc; - TRY_STATUS(boc.deserialize(q.body_.as_slice())); - if (boc.get_root_count() != 1) { - return td::Status::Error("external message is not a valid bag of cells"); - } - block::gen::CommonMsgInfo::Record_ext_in_msg_info info; - if (!tlb::unpack_cell_inexact(boc.get_root_cell(), info)) { - return td::Status::Error("cannot unpack external message header"); - } - auto dest_prefix = block::tlb::t_MsgAddressInt.get_prefix(info.dest); - if (!dest_prefix.is_valid()) { - return td::Status::Error("destination of an inbound external message is an invalid blockchain address"); - } - return dest_prefix.as_leaf_shard(); - }(); - if (shard.is_error()) { - LOG(DEBUG) << "Failed to get shard from query liteServer.sendMessage: " << shard.move_as_error(); - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } - return shard.move_as_ok(); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getAccountState& q) { - return ton::AccountIdPrefixFull(q.account_->workchain_, q.account_->id_.bits().get_uint(64)).as_leaf_shard(); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getAccountStatePrunned& q) { - return ton::AccountIdPrefixFull(q.account_->workchain_, q.account_->id_.bits().get_uint(64)).as_leaf_shard(); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_runSmcMethod& q) { - return ton::AccountIdPrefixFull(q.account_->workchain_, q.account_->id_.bits().get_uint(64)).as_leaf_shard(); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getShardInfo& q) { - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getAllShardsInfo& q) { - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getOneTransaction& q) { - return ton::AccountIdPrefixFull(q.account_->workchain_, q.account_->id_.bits().get_uint(64)).as_leaf_shard(); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getTransactions& q) { - return ton::AccountIdPrefixFull(q.account_->workchain_, q.account_->id_.bits().get_uint(64)).as_leaf_shard(); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_lookupBlock& q) { - return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_listBlockTransactions& q) { - return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_listBlockTransactionsExt& q) { - return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getBlockProof& q) { - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getConfigAll& q) { - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getConfigParams& q) { - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getValidatorStats& q) { - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getLibraries& q) { - return ton::ShardIdFull(ton::masterchainId, ton::shardIdAll); - } -}; - -template<> -struct QueryTraits { - static ton::ShardIdFull get_shard(const ton::lite_api::liteServer_getShardBlockProof& q) { - return ton::ShardIdFull(q.id_->workchain_, q.id_->shard_); - } -}; - -template -inline ton::ShardIdFull get_query_shard(const Query& q) { - return QueryTraits::get_shard(q); -} - -} // namespace tonlib diff --git a/lite-client/ext-client.cpp b/lite-client/ext-client.cpp index e79ae9a4f..0232c28f4 100644 --- a/lite-client/ext-client.cpp +++ b/lite-client/ext-client.cpp @@ -17,92 +17,75 @@ #include "ext-client.h" #include "td/utils/Random.h" #include "ton/ton-shard.h" -#include namespace liteclient { class ExtClientImpl : public ExtClient { public: - ExtClientImpl(std::vector servers, td::unique_ptr callback) + ExtClientImpl(std::vector liteservers, td::unique_ptr callback) : callback_(std::move(callback)) { - CHECK(!servers.empty()); - servers_.resize(servers.size()); + CHECK(!liteservers.empty()); + servers_.resize(liteservers.size()); for (size_t i = 0; i < servers_.size(); ++i) { - servers_[i].s = std::move(servers[i]); - if (!servers_[i].s.is_full) { - for (auto shard : servers_[i].s.shards) { - CHECK(shard.is_valid_ext()); - max_server_shard_depth_ = std::max(max_server_shard_depth_, shard.pfx_len()); - } - } + servers_[i].config = std::move(liteservers[i]); + servers_[i].idx = i; } } void start_up() override { + LOG(INFO) << "Started ext client, " << servers_.size() << " liteservers"; td::Random::Fast rnd; td::random_shuffle(td::as_mutable_span(servers_), rnd); } - void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, + void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, td::Promise promise) override { - TRY_RESULT_PROMISE(promise, server_idx, before_query(shard)); + QueryInfo query_info = get_query_info(data); + TRY_RESULT_PROMISE(promise, server_idx, select_server(query_info)); auto& server = servers_[server_idx]; CHECK(!server.client.empty()); alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT)); td::Promise P = [SelfId = actor_id(this), server_idx, - promise = std::move(promise)](td::Result R) mutable { + promise = std::move(promise)](td::Result R) mutable { if (R.is_error() && (R.error().code() == ton::ErrorCode::timeout || R.error().code() == ton::ErrorCode::cancelled)) { - td::actor::send_closure(SelfId, &ExtClientImpl::set_server_bad, server_idx); + td::actor::send_closure(SelfId, &ExtClientImpl::on_server_error, server_idx); } promise.set_result(std::move(R)); }; + LOG(DEBUG) << "Sending query " << query_info.to_str() << " to server #" << server.idx << " (" + << server.config.addr.get_ip_str() << ":" << server.config.addr.get_port() << ")"; send_closure(server.client, &ton::adnl::AdnlExtClient::send_query, std::move(name), std::move(data), timeout, std::move(P)); } - void force_change_liteserver() override { - if (servers_.size() == 1) { - return; - } - auto it = shard_to_server_.find(ton::ShardIdFull(ton::masterchainId)); - if (it != shard_to_server_.end()) { - set_server_bad(it->second); + void reset_servers() override { + LOG(INFO) << "Force resetting all liteservers"; + for (Server& server : servers_) { + server.alive = false; + server.timeout = {}; + server.ignore_until = {}; + server.client.reset(); } } private: - td::Result before_query(ton::ShardIdFull shard) { - if (!shard.is_valid_ext()) { - return td::Status::Error("Invalid shard"); - } - if (is_closing_) { - return td::Status::Error("Client is closing"); - } - if (shard.pfx_len() > max_server_shard_depth_) { - shard = shard_prefix(shard, max_server_shard_depth_); - } - auto it = shard_to_server_.find(shard); - if (it != shard_to_server_.end()) { - size_t server_idx = it->second; - if (!servers_[server_idx].client.empty()) { - return server_idx; + td::Result select_server(const QueryInfo& query_info) { + for (size_t i = 0; i < servers_.size(); ++i) { + if (servers_[i].alive && servers_[i].config.accepts_query(query_info)) { + return i; } - shard_to_server_.erase(it); } - size_t server_idx = servers_.size(); int cnt = 0; int best_priority = -1; for (size_t i = 0; i < servers_.size(); ++i) { Server& server = servers_[i]; - if (!server.supports(shard)) { + if (!server.config.accepts_query(query_info)) { continue; } int priority = 0; - priority += (server.client.empty() ? 0 : 100); priority += (server.ignore_until && !server.ignore_until.is_in_past() ? 0 : 10); - priority += (server.s.is_full ? 1 : 0); if (priority < best_priority) { continue; } @@ -116,100 +99,76 @@ class ExtClientImpl : public ExtClient { ++cnt; } if (server_idx == servers_.size()) { - return td::Status::Error(PSTRING() << "No liteserver for shard " << shard.to_str()); + return td::Status::Error(PSTRING() << "no liteserver for query " << query_info.to_str()); } Server& server = servers_[server_idx]; + server.alive = true; + server.ignore_until = {}; + alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT)); if (!server.client.empty()) { return server_idx; } class Callback : public ton::adnl::AdnlExtClient::Callback { public: - explicit Callback(td::actor::ActorShared parent, size_t idx) - : parent_(std::move(parent)), idx_(idx) { + explicit Callback(td::actor::ActorId parent, size_t idx) : parent_(std::move(parent)), idx_(idx) { } void on_ready() override { } void on_stop_ready() override { - td::actor::send_closure(parent_, &ExtClientImpl::set_server_bad, idx_); + td::actor::send_closure(parent_, &ExtClientImpl::on_server_error, idx_); } private: - td::actor::ActorShared parent_; + td::actor::ActorId parent_; size_t idx_; }; - ref_cnt_++; - if (shard.is_masterchain()) { - LOG(INFO) << "Connecting to liteserver " << server.s.address << " for masterchain"; - } else { - LOG(INFO) << "Connecting to liteserver " << server.s.address << " for shard " << shard.to_str(); - } - server.client = ton::adnl::AdnlExtClient::create( - server.s.adnl_id, server.s.address, std::make_unique(td::actor::actor_shared(this), server_idx)); - alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT)); + LOG(INFO) << "Connecting to liteserver #" << server.idx << " (" << server.config.addr.get_ip_str() << ":" + << server.config.addr.get_port() << ") for query " << query_info.to_str(); + server.client = ton::adnl::AdnlExtClient::create(server.config.adnl_id, server.config.addr, + std::make_unique(actor_id(this), server_idx)); return server_idx; } struct Server { - LiteServer s; + LiteServerConfig config; + size_t idx = 0; td::actor::ActorOwn client; + bool alive = false; td::Timestamp timeout = td::Timestamp::never(); td::Timestamp ignore_until = td::Timestamp::never(); - - bool supports(const ton::ShardIdFull& shard) const { - return s.is_full || shard.is_masterchain() || - std::any_of(s.shards.begin(), s.shards.end(), - [&](const ton::ShardIdFull s_shard) { return ton::shard_intersects(shard, s_shard); }); - } }; std::vector servers_; - std::map shard_to_server_; - int max_server_shard_depth_ = 0; - - td::unique_ptr callback_; - static constexpr double MAX_NO_QUERIES_TIMEOUT = 100; - bool is_closing_{false}; - td::uint32 ref_cnt_{1}; + td::unique_ptr callback_; + static constexpr double MAX_NO_QUERIES_TIMEOUT = 100.0; + static constexpr double BAD_SERVER_TIMEOUT = 30.0; void alarm() override { for (Server& server : servers_) { if (server.timeout && server.timeout.is_in_past()) { + LOG(INFO) << "Closing connection to liteserver #" << server.idx << " (" << server.config.addr.get_ip_str() + << ":" << server.config.addr.get_port() << ")"; server.client.reset(); + server.alive = false; + server.ignore_until = {}; } } } - void set_server_bad(size_t idx) { - servers_[idx].client.reset(); - servers_[idx].timeout = td::Timestamp::never(); - servers_[idx].ignore_until = td::Timestamp::in(60.0); - } - void hangup_shared() override { - ref_cnt_--; - try_stop(); - } - void hangup() override { - is_closing_ = true; - ref_cnt_--; - for (Server& server : servers_) { - server.client.reset(); - } - try_stop(); - } - void try_stop() { - if (is_closing_ && ref_cnt_ == 0) { - stop(); - } + + void on_server_error(size_t idx) { + servers_[idx].alive = false; + servers_[idx].ignore_until = td::Timestamp::in(BAD_SERVER_TIMEOUT); } }; td::actor::ActorOwn ExtClient::create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, - td::unique_ptr callback) { - return create({LiteServer{dst, dst_addr, true, {}}}, std::move(callback)); + td::unique_ptr callback) { + return create({LiteServerConfig{dst, dst_addr}}, std::move(callback)); } -td::actor::ActorOwn ExtClient::create(std::vector servers, - td::unique_ptr callback) { - return td::actor::create_actor("ExtClient", std::move(servers), std::move(callback)); +td::actor::ActorOwn ExtClient::create(std::vector liteservers, + td::unique_ptr callback) { + return td::actor::create_actor("ExtClient", std::move(liteservers), std::move(callback)); } } // namespace liteclient diff --git a/lite-client/ext-client.h b/lite-client/ext-client.h index c8569964f..adcff9bed 100644 --- a/lite-client/ext-client.h +++ b/lite-client/ext-client.h @@ -18,29 +18,24 @@ #include "td/actor/actor.h" #include "ton/ton-types.h" #include "adnl/adnl-ext-client.h" +#include "query-utils.hpp" namespace liteclient { class ExtClient : public td::actor::Actor { public: - struct LiteServer { - ton::adnl::AdnlNodeIdFull adnl_id; - td::IPAddress address; - bool is_full = true; - std::vector shards; - }; - class Callback { public: - virtual ~Callback() { - } + virtual ~Callback() = default; }; - virtual void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, + virtual void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, td::Promise promise) = 0; - virtual void force_change_liteserver() = 0; + virtual void reset_servers() { + } static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, td::unique_ptr callback); - static td::actor::ActorOwn create(std::vector servers, td::unique_ptr callback); + static td::actor::ActorOwn create(std::vector liteservers, + td::unique_ptr callback); }; } // namespace liteclient diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index 923ea617b..2e2770a47 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -67,12 +67,6 @@ using td::Ref; int verbosity; -bool TestNode::LiteServer::supports(ton::ShardIdFull shard) const { - return is_full || shard.is_masterchain() || - std::any_of(shards.begin(), shards.end(), - [&](const ton::ShardIdFull& our_shard) { return ton::shard_intersects(shard, our_shard); }); -} - void TestNode::run() { class Cb : public td::TerminalIO::Callback { public: @@ -88,55 +82,39 @@ void TestNode::run() { io_ = td::TerminalIO::create("> ", readline_enabled_, ex_mode_, std::make_unique(actor_id(this))); td::actor::send_closure(io_, &td::TerminalIO::set_log_interface); + std::vector servers; if (!single_remote_public_key_.empty()) { // Use single provided liteserver - LiteServer s; - s.addr = single_remote_addr_; - s.public_key = single_remote_public_key_; - single_liteserver_idx_ = 0; - td::TerminalIO::out() << "using liteserver " << s.addr << "\n"; - servers_.push_back(std::move(s)); - run_init_queries(); - return; - } - - auto G = td::read_file(global_config_).move_as_ok(); - auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); - ton::ton_api::liteclient_config_global gc; - ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); - CHECK(gc.liteservers_.size() + gc.liteservers_v2_.size() > 0); - - if (gc.validator_ && gc.validator_->zero_state_) { - zstate_id_.workchain = gc.validator_->zero_state_->workchain_; - if (zstate_id_.workchain != ton::workchainInvalid) { - zstate_id_.root_hash = gc.validator_->zero_state_->root_hash_; - zstate_id_.file_hash = gc.validator_->zero_state_->file_hash_; - td::TerminalIO::out() << "zerostate set to " << zstate_id_.to_str() << "\n"; + servers.push_back( + liteclient::LiteServerConfig{ton::adnl::AdnlNodeIdFull{single_remote_public_key_}, single_remote_addr_}); + td::TerminalIO::out() << "using liteserver " << single_remote_addr_ << "\n"; + } else { + auto G = td::read_file(global_config_).move_as_ok(); + auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); + ton::ton_api::liteclient_config_global gc; + ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); + auto r_servers = liteclient::LiteServerConfig::parse_global_config(gc); + r_servers.ensure(); + servers = r_servers.move_as_ok(); + + if (gc.validator_ && gc.validator_->zero_state_) { + zstate_id_.workchain = gc.validator_->zero_state_->workchain_; + if (zstate_id_.workchain != ton::workchainInvalid) { + zstate_id_.root_hash = gc.validator_->zero_state_->root_hash_; + zstate_id_.file_hash = gc.validator_->zero_state_->file_hash_; + td::TerminalIO::out() << "zerostate set to " << zstate_id_.to_str() << "\n"; + } } - } - for (auto& server : gc.liteservers_) { - LiteServer s; - s.addr.init_host_port(td::IPAddress::ipv4_to_str(server->ip_), server->port_).ensure(); - s.public_key = ton::PublicKey{server->id_}; - servers_.push_back(std::move(s)); - } - for (auto& server : gc.liteservers_v2_) { - LiteServer s; - s.addr.init_host_port(td::IPAddress::ipv4_to_str(server->ip_), server->port_).ensure(); - s.public_key = ton::PublicKey{server->id_}; - s.is_full = false; - for (const auto& shard : server->shards_) { - s.shards.emplace_back(shard->workchain_, shard->shard_); - CHECK(s.shards.back().is_valid_ext()); + if (single_liteserver_idx_ != -1) { // Use single liteserver from config + CHECK(single_liteserver_idx_ >= 0 && (size_t)single_liteserver_idx_ < servers.size()); + td::TerminalIO::out() << "using liteserver #" << single_liteserver_idx_ << " with addr " + << servers[single_liteserver_idx_].addr << "\n"; + servers = {servers[single_liteserver_idx_]}; } - servers_.push_back(std::move(s)); - } - - if (single_liteserver_idx_ != -1) { // Use single liteserver from config - CHECK(single_liteserver_idx_ >= 0 && (size_t)single_liteserver_idx_ < gc.liteservers_.size()); - td::TerminalIO::out() << "using liteserver " << single_liteserver_idx_ << " with addr " - << servers_[single_liteserver_idx_].addr << "\n"; } + CHECK(!servers.empty()); + client_ = liteclient::ExtClient::create(std::move(servers), nullptr); + ready_ = true; run_init_queries(); } @@ -183,142 +161,23 @@ void TestNode::after_got_result(bool ok) { } } -bool TestNode::envelope_send_query_to_any(td::BufferSlice query, td::Promise promise) { - return envelope_send_query_to_shard(ton::ShardIdFull(ton::masterchainId), std::move(query), std::move(promise)); -} - -bool TestNode::envelope_send_query_to_account(ton::AccountIdPrefixFull prefix, td::BufferSlice query, - td::Promise promise) { - return envelope_send_query_to_shard(prefix.as_leaf_shard(), std::move(query), std::move(promise)); -} - -bool TestNode::envelope_send_query_to_shard(ton::ShardIdFull shard, td::BufferSlice query, - td::Promise promise) { - if (single_liteserver_idx_ >= 0) { - return envelope_send_query_to_server(single_liteserver_idx_, std::move(query), std::move(promise)); - } - if (shard.is_masterchain() && mc_server_idx_ != -1) { - return envelope_send_query_to_server(mc_server_idx_, std::move(query), std::move(promise)); - } - auto it = shard_server_idx_cached_.find(shard); - if (it != shard_server_idx_cached_.end()) { - return envelope_send_query_to_server(it->second, std::move(query), std::move(promise)); - } - int server_idx = -1; - int random_idx = -1; - int cnt = 0; - bool selected_full = false; - for (int i = 0; i < (int)servers_.size(); ++i) { - const LiteServer& server = servers_[i]; - if (!server.supports(shard)) { - continue; - } - if (server.is_full && !selected_full) { - selected_full = true; - server_idx = -1; - cnt = 0; - } - if (!server.is_full && selected_full) { - continue; - } - if (!server.client.empty()) { - server_idx = i; - } - if (td::Random::fast(0, cnt) == 0) { - random_idx = i; - } - ++cnt; - } - if (server_idx == -1) { - server_idx = random_idx; - } - if (server_idx == -1) { - running_queries_++; - got_result(td::Status::Error(PSTRING() << "failed to select a suitable server for " << shard.to_str()), - std::move(promise)); - return false; - } - shard_server_idx_cached_[shard] = server_idx; - if (shard.is_masterchain()) { - mc_server_idx_ = server_idx; - } - return envelope_send_query_to_server(server_idx, std::move(query), std::move(promise)); -} - -bool TestNode::envelope_send_query_to_server(td::int32 server_idx, td::BufferSlice query, - td::Promise promise) { +bool TestNode::envelope_send_query(td::BufferSlice query, td::Promise promise) { running_queries_++; - LiteServer& server = servers_.at(server_idx); - if (server.client.empty()) { - start_client(server_idx); + if (!ready_ || client_.empty()) { + got_result(td::Status::Error("failed to send query to server: not ready"), std::move(promise)); + return false; } - CHECK(!server.client.empty()); - auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { td::actor::send_closure(SelfId, &TestNode::got_result, std::move(R), std::move(promise)); }); td::BufferSlice b = ton::serialize_tl_object(ton::create_tl_object(std::move(query)), true); - if (server.client_ready) { - td::actor::send_closure(server.client, &ton::adnl::AdnlExtClient::send_query, "query", std::move(b), - td::Timestamp::in(10.0), std::move(P)); - } else { - server.wait_client_ready.push_back( - [client = server.client.get(), b = std::move(b), P = std::move(P)](td::Result R) mutable { - if (R.is_ok()) { - td::actor::send_closure(client, &ton::adnl::AdnlExtClient::send_query, "query", std::move(b), - td::Timestamp::in(10.0), std::move(P)); - } else { - P.set_error(R.move_as_error_prefix("failed to connect: ")); - } - }); - } + td::actor::send_closure(client_, &liteclient::ExtClient::send_query, "query", std::move(b), td::Timestamp::in(10.0), + std::move(P)); return true; } -void TestNode::start_client(int server_idx) { - LiteServer& server = servers_[server_idx]; - CHECK(server.client.empty()); - class Callback : public ton::adnl::AdnlExtClient::Callback { - public: - void on_ready() override { - td::actor::send_closure(id_, &TestNode::conn_ready, server_idx_); - } - void on_stop_ready() override { - td::actor::send_closure(id_, &TestNode::conn_closed, server_idx_); - } - Callback(td::actor::ActorId id, int server_idx) : id_(std::move(id)), server_idx_(server_idx) { - } - - private: - td::actor::ActorId id_; - int server_idx_; - }; - server.client_ready = false; - server.wait_client_ready.clear(); - LOG(INFO) << "Connecting to " << server.addr << " (liteserver #" << server_idx << ")"; - server.client = ton::adnl::AdnlExtClient::create(ton::adnl::AdnlNodeIdFull{server.public_key}, server.addr, - std::make_unique(actor_id(this), server_idx)); -} - -void TestNode::conn_ready(int server_idx) { - LiteServer& server = servers_[server_idx]; - LOG(INFO) << "Connection to " << server.addr << " (liteserver #" << server_idx << ") is ready"; - server.client_ready = true; - for (auto& p : server.wait_client_ready) { - p.set_result(td::Unit()); - } - server.wait_client_ready.clear(); -} - -void TestNode::conn_closed(int server_idx) { - LiteServer& server = servers_[server_idx]; - LOG(INFO) << "Connection to " << server.addr << " (liteserver #" << server_idx << ") closed"; - server.client_ready = false; - server.wait_client_ready.clear(); -} - td::Promise TestNode::trivial_promise() { return td::PromiseCreator::lambda([Self = actor_id(this)](td::Result res) { if (res.is_error()) { @@ -433,7 +292,7 @@ bool TestNode::dump_cached_cell(td::Slice hash_pfx, td::Slice type_name) { bool TestNode::get_server_time() { auto b = ton::serialize_tl_object(ton::create_tl_object(), true); - return envelope_send_query_to_any(std::move(b), [&, Self = actor_id(this)](td::Result res) -> void { + return envelope_send_query(std::move(b), [&, Self = actor_id(this)](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "cannot get server time"; return; @@ -453,7 +312,7 @@ bool TestNode::get_server_time() { bool TestNode::get_server_version(int mode) { auto b = ton::serialize_tl_object(ton::create_tl_object(), true); - return envelope_send_query_to_any(std::move(b), [Self = actor_id(this), mode](td::Result res) { + return envelope_send_query(std::move(b), [Self = actor_id(this), mode](td::Result res) { td::actor::send_closure_later(Self, &TestNode::got_server_version, std::move(res), mode); }); }; @@ -501,7 +360,7 @@ bool TestNode::get_server_mc_block_id() { int mode = (mc_server_capabilities_ & 2) ? 0 : -1; if (mode < 0) { auto b = ton::serialize_tl_object(ton::create_tl_object(), true); - return envelope_send_query_to_any(std::move(b), [Self = actor_id(this)](td::Result res) -> void { + return envelope_send_query(std::move(b), [Self = actor_id(this)](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "cannot get masterchain info from server"; return; @@ -521,25 +380,24 @@ bool TestNode::get_server_mc_block_id() { } else { auto b = ton::serialize_tl_object(ton::create_tl_object(mode), true); - return envelope_send_query_to_any( - std::move(b), [Self = actor_id(this), mode](td::Result res) -> void { - if (res.is_error()) { - LOG(ERROR) << "cannot get extended masterchain info from server"; - return; - } else { - auto F = ton::fetch_tl_object(res.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getMasterchainInfoExt"; - } else { - auto f = F.move_as_ok(); - auto blk_id = create_block_id(f->last_); - auto zstate_id = create_zero_state_id(f->init_); - LOG(INFO) << "last masterchain block is " << blk_id.to_str(); - td::actor::send_closure_later(Self, &TestNode::got_server_mc_block_id_ext, blk_id, zstate_id, mode, - f->version_, f->capabilities_, f->last_utime_, f->now_); - } - } - }); + return envelope_send_query(std::move(b), [Self = actor_id(this), mode](td::Result res) -> void { + if (res.is_error()) { + LOG(ERROR) << "cannot get extended masterchain info from server"; + return; + } else { + auto F = ton::fetch_tl_object(res.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getMasterchainInfoExt"; + } else { + auto f = F.move_as_ok(); + auto blk_id = create_block_id(f->last_); + auto zstate_id = create_zero_state_id(f->init_); + LOG(INFO) << "last masterchain block is " << blk_id.to_str(); + td::actor::send_closure_later(Self, &TestNode::got_server_mc_block_id_ext, blk_id, zstate_id, mode, + f->version_, f->capabilities_, f->last_utime_, f->now_); + } + } + }); } } @@ -594,54 +452,52 @@ void TestNode::got_server_mc_block_id_ext(ton::BlockIdExt blkid, ton::ZeroStateI bool TestNode::request_block(ton::BlockIdExt blkid) { auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); - return envelope_send_query_to_shard( - blkid.shard_full(), std::move(b), [Self = actor_id(this), blkid](td::Result res) -> void { - if (res.is_error()) { - LOG(ERROR) << "cannot obtain block " << blkid.to_str() << " from server"; - return; - } else { - auto F = ton::fetch_tl_object(res.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getBlock"; - } else { - auto f = F.move_as_ok(); - auto blk_id = ton::create_block_id(f->id_); - LOG(INFO) << "obtained block " << blk_id.to_str() << " from server"; - if (blk_id != blkid) { - LOG(ERROR) << "block id mismatch: expected data for block " << blkid.to_str() << ", obtained for " - << blk_id.to_str(); - } - td::actor::send_closure_later(Self, &TestNode::got_mc_block, blk_id, std::move(f->data_)); - } + return envelope_send_query(std::move(b), [Self = actor_id(this), blkid](td::Result res) -> void { + if (res.is_error()) { + LOG(ERROR) << "cannot obtain block " << blkid.to_str() << " from server"; + return; + } else { + auto F = ton::fetch_tl_object(res.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getBlock"; + } else { + auto f = F.move_as_ok(); + auto blk_id = ton::create_block_id(f->id_); + LOG(INFO) << "obtained block " << blk_id.to_str() << " from server"; + if (blk_id != blkid) { + LOG(ERROR) << "block id mismatch: expected data for block " << blkid.to_str() << ", obtained for " + << blk_id.to_str(); } - }); + td::actor::send_closure_later(Self, &TestNode::got_mc_block, blk_id, std::move(f->data_)); + } + } + }); } bool TestNode::request_state(ton::BlockIdExt blkid) { auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); - return envelope_send_query_to_shard( - blkid.shard_full(), std::move(b), [Self = actor_id(this), blkid](td::Result res) -> void { - if (res.is_error()) { - LOG(ERROR) << "cannot obtain state " << blkid.to_str() << " from server"; - return; - } else { - auto F = ton::fetch_tl_object(res.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getState"; - } else { - auto f = F.move_as_ok(); - auto blk_id = ton::create_block_id(f->id_); - LOG(INFO) << "obtained state " << blk_id.to_str() << " from server"; - if (blk_id != blkid) { - LOG(ERROR) << "block id mismatch: expected state for block " << blkid.to_str() << ", obtained for " - << blk_id.to_str(); - } - td::actor::send_closure_later(Self, &TestNode::got_mc_state, blk_id, f->root_hash_, f->file_hash_, - std::move(f->data_)); - } + return envelope_send_query(std::move(b), [Self = actor_id(this), blkid](td::Result res) -> void { + if (res.is_error()) { + LOG(ERROR) << "cannot obtain state " << blkid.to_str() << " from server"; + return; + } else { + auto F = ton::fetch_tl_object(res.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getState"; + } else { + auto f = F.move_as_ok(); + auto blk_id = ton::create_block_id(f->id_); + LOG(INFO) << "obtained state " << blk_id.to_str() << " from server"; + if (blk_id != blkid) { + LOG(ERROR) << "block id mismatch: expected state for block " << blkid.to_str() << ", obtained for " + << blk_id.to_str(); } - }); + td::actor::send_closure_later(Self, &TestNode::got_mc_state, blk_id, f->root_hash_, f->file_hash_, + std::move(f->data_)); + } + } + }); } void TestNode::got_mc_block(ton::BlockIdExt blkid, td::BufferSlice data) { @@ -1280,34 +1136,27 @@ td::Status TestNode::send_ext_msg_from_filename(std::string filename) { LOG(ERROR) << "failed to read file `" << filename << "`: " << err.to_string(); return err; } - LOG(ERROR) << "sending query from file " << filename; - - TRY_RESULT_PREFIX(root, vm::std_boc_deserialize(F.ok().as_slice()), "invalid boc: "); - block::gen::CommonMsgInfo::Record_ext_in_msg_info info; - if (!tlb::unpack_cell_inexact(root, info)) { - return td::Status::Error("failed to unpack external message header"); - } - auto dest_prefix = block::tlb::MsgAddressInt::get_prefix(info.dest); - if (!dest_prefix.is_valid()) { - return td::Status::Error("destination of the message is invalid"); + if (ready_ && !client_.empty()) { + LOG(ERROR) << "sending query from file " << filename; + auto P = td::PromiseCreator::lambda([](td::Result R) { + if (R.is_error()) { + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.sendMessage"; + } else { + int status = F.move_as_ok()->status_; + LOG(INFO) << "external message status is " << status; + } + }); + auto b = + ton::serialize_tl_object(ton::create_tl_object(F.move_as_ok()), true); + return envelope_send_query(std::move(b), std::move(P)) ? td::Status::OK() + : td::Status::Error("cannot send query to server"); + } else { + return td::Status::Error("server connection not ready"); } - - auto P = td::PromiseCreator::lambda([](td::Result R) { - if (R.is_error()) { - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.sendMessage"; - } else { - int status = F.move_as_ok()->status_; - LOG(INFO) << "external message status is " << status; - } - }); - auto b = ton::serialize_tl_object(ton::create_tl_object(F.move_as_ok()), true); - return envelope_send_query_to_account(dest_prefix, std::move(b), std::move(P)) - ? td::Status::OK() - : td::Status::Error("cannot send query to server"); } bool TestNode::get_account_state(ton::WorkchainId workchain, ton::StdSmcAddress addr, ton::BlockIdExt ref_blkid, @@ -1315,6 +1164,9 @@ bool TestNode::get_account_state(ton::WorkchainId workchain, ton::StdSmcAddress if (!ref_blkid.is_valid()) { return set_error("must obtain last block information before making other queries"); } + if (!(ready_ && !client_.empty())) { + return set_error("server connection not ready"); + } if (addr_ext) { return get_special_smc_addr( addr_ext, [this, ref_blkid, filename, mode, prunned](td::Result res) { @@ -1336,26 +1188,24 @@ bool TestNode::get_account_state(ton::WorkchainId workchain, ton::StdSmcAddress ton::create_tl_lite_block_id(ref_blkid), std::move(a)), true); } - ton::AccountIdPrefixFull account_prefix(workchain, addr.bits().get_uint(64)); LOG(INFO) << "requesting " << (prunned ? "prunned " : "") << "account state for " << workchain << ":" << addr.to_hex() << " with respect to " << ref_blkid.to_str() << " with savefile `" << filename << "` and mode " << mode; - return envelope_send_query_to_account( - account_prefix, std::move(b), - [Self = actor_id(this), workchain, addr, ref_blkid, filename, mode, prunned](td::Result R) { - if (R.is_error()) { - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getAccountState"; - } else { - auto f = F.move_as_ok(); - td::actor::send_closure_later(Self, &TestNode::got_account_state, ref_blkid, ton::create_block_id(f->id_), - ton::create_block_id(f->shardblk_), std::move(f->shard_proof_), - std::move(f->proof_), std::move(f->state_), workchain, addr, filename, mode, - prunned); - } - }); + return envelope_send_query(std::move(b), [Self = actor_id(this), workchain, addr, ref_blkid, filename, mode, + prunned](td::Result R) { + if (R.is_error()) { + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getAccountState"; + } else { + auto f = F.move_as_ok(); + td::actor::send_closure_later(Self, &TestNode::got_account_state, ref_blkid, ton::create_block_id(f->id_), + ton::create_block_id(f->shardblk_), std::move(f->shard_proof_), + std::move(f->proof_), std::move(f->state_), workchain, addr, filename, mode, + prunned); + } + }); } td::int64 TestNode::compute_method_id(std::string method) { @@ -1421,18 +1271,19 @@ bool TestNode::start_run_method(ton::WorkchainId workchain, ton::StdSmcAddress a if (!ref_blkid.is_valid()) { return set_error("must obtain last block information before making other queries"); } + if (!(ready_ && !client_.empty())) { + return set_error("server connection not ready"); + } auto a = ton::create_tl_object(workchain, addr); - ton::AccountIdPrefixFull account_prefix(workchain, addr.bits().get_uint(64)); if (!mode) { auto b = ton::serialize_tl_object(ton::create_tl_object( ton::create_tl_lite_block_id(ref_blkid), std::move(a)), true); LOG(INFO) << "requesting account state for " << workchain << ":" << addr.to_hex() << " with respect to " << ref_blkid.to_str() << " to run method " << method_name << " with " << params.size() << " parameters"; - return envelope_send_query_to_account( - account_prefix, std::move(b), - [Self = actor_id(this), workchain, addr, ref_blkid, method_name, params = std::move(params), - promise = std::move(promise)](td::Result R) mutable { + return envelope_send_query( + std::move(b), [Self = actor_id(this), workchain, addr, ref_blkid, method_name, params = std::move(params), + promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); return; @@ -1472,27 +1323,26 @@ bool TestNode::start_run_method(ton::WorkchainId workchain, ton::StdSmcAddress a LOG(INFO) << "requesting remote get-method execution for " << workchain << ":" << addr.to_hex() << " with respect to " << ref_blkid.to_str() << " to run method " << method_name << " with " << params.size() << " parameters"; - return envelope_send_query_to_account( - account_prefix, std::move(b), - [Self = actor_id(this), workchain, addr, ref_blkid, method_name, mode, params = std::move(params), - promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.runSmcMethod"; - promise.set_error(td::Status::Error("cannot parse answer to liteServer.runSmcMethod")); - } else { - auto f = F.move_as_ok(); - td::actor::send_closure_later( - Self, &TestNode::run_smc_method, mode, ref_blkid, ton::create_block_id(f->id_), - ton::create_block_id(f->shardblk_), std::move(f->shard_proof_), std::move(f->proof_), - std::move(f->state_proof_), workchain, addr, method_name, std::move(params), std::move(f->init_c7_), - std::move(f->lib_extras_), std::move(f->result_), f->exit_code_, std::move(promise)); - } - }); + return envelope_send_query(std::move(b), [Self = actor_id(this), workchain, addr, ref_blkid, method_name, mode, + params = std::move(params), + promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.runSmcMethod"; + promise.set_error(td::Status::Error("cannot parse answer to liteServer.runSmcMethod")); + } else { + auto f = F.move_as_ok(); + td::actor::send_closure_later(Self, &TestNode::run_smc_method, mode, ref_blkid, ton::create_block_id(f->id_), + ton::create_block_id(f->shardblk_), std::move(f->shard_proof_), + std::move(f->proof_), std::move(f->state_proof_), workchain, addr, method_name, + std::move(params), std::move(f->init_c7_), std::move(f->lib_extras_), + std::move(f->result_), f->exit_code_, std::move(promise)); + } + }); } } @@ -1722,8 +1572,8 @@ void TestNode::send_compute_complaint_price_query(ton::StdSmcAddress elector_add params.emplace_back(td::make_refint(bits)); params.emplace_back(td::make_refint(refs)); params.emplace_back(td::make_refint(expires_in)); - auto P = - td::PromiseCreator::lambda([expires_in, bits, refs, chash, filename](td::Result> R) { + auto P = td::PromiseCreator::lambda( + [this, expires_in, bits, refs, chash, filename](td::Result> R) { if (R.is_error()) { LOG(ERROR) << R.move_as_error(); return; @@ -1748,9 +1598,8 @@ void TestNode::send_compute_complaint_price_query(ton::StdSmcAddress elector_add } bool TestNode::get_msg_queue_sizes() { - // TODO: rework for separated liteservers auto q = ton::serialize_tl_object(ton::create_tl_object(0, 0, 0), true); - return envelope_send_query_to_any(std::move(q), [Self = actor_id(this)](td::Result res) -> void { + return envelope_send_query(std::move(q), [Self = actor_id(this)](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "liteServer.getOutMsgQueueSizes error: " << res.move_as_error(); return; @@ -1810,6 +1659,10 @@ bool TestNode::dns_resolve_start(ton::WorkchainId workchain, ton::StdSmcAddress return set_error("domain name too long"); } + if (!(ready_ && !client_.empty())) { + return set_error("server connection not ready"); + } + if (workchain == ton::workchainInvalid) { if (dns_root_queried_) { workchain = ton::masterchainId; @@ -2018,16 +1871,17 @@ bool TestNode::get_one_transaction(ton::BlockIdExt blkid, ton::WorkchainId workc if (!ton::shard_contains(blkid.shard_full(), ton::extract_addr_prefix(workchain, addr))) { return set_error("the shard of this block cannot contain this account"); } + if (!(ready_ && !client_.empty())) { + return set_error("server connection not ready"); + } auto a = ton::create_tl_object(workchain, addr); auto b = ton::serialize_tl_object(ton::create_tl_object( ton::create_tl_lite_block_id(blkid), std::move(a), lt), true); - ton::AccountIdPrefixFull account_prefix(workchain, addr.bits().get_uint(64)); LOG(INFO) << "requesting transaction " << lt << " of " << workchain << ":" << addr.to_hex() << " from block " << blkid.to_str(); - return envelope_send_query_to_account( - account_prefix, std::move(b), - [Self = actor_id(this), workchain, addr, lt, blkid, dump](td::Result R) -> void { + return envelope_send_query( + std::move(b), [Self = actor_id(this), workchain, addr, lt, blkid, dump](td::Result R) -> void { if (R.is_error()) { return; } @@ -2044,15 +1898,16 @@ bool TestNode::get_one_transaction(ton::BlockIdExt blkid, ton::WorkchainId workc bool TestNode::get_last_transactions(ton::WorkchainId workchain, ton::StdSmcAddress addr, ton::LogicalTime lt, ton::Bits256 hash, unsigned count, bool dump) { + if (!(ready_ && !client_.empty())) { + return set_error("server connection not ready"); + } auto a = ton::create_tl_object(workchain, addr); auto b = ton::serialize_tl_object( ton::create_tl_object(count, std::move(a), lt, hash), true); - ton::AccountIdPrefixFull account_prefix(workchain, addr.bits().get_uint(64)); LOG(INFO) << "requesting " << count << " last transactions from " << lt << ":" << hash.to_hex() << " of " << workchain << ":" << addr.to_hex(); - return envelope_send_query_to_account( - account_prefix, std::move(b), - [Self = actor_id(this), workchain, addr, lt, hash, count, dump](td::Result R) { + return envelope_send_query( + std::move(b), [Self = actor_id(this), workchain, addr, lt, hash, count, dump](td::Result R) { if (R.is_error()) { return; } @@ -2420,10 +2275,10 @@ void TestNode::got_one_transaction(ton::BlockIdExt req_blkid, ton::BlockIdExt bl << " but received data has " << root->get_hash().bits().to_hex(256); return; } - } catch (vm::VmError& err) { + } catch (vm::VmError err) { LOG(ERROR) << "error while traversing block transaction proof : " << err.get_msg(); return; - } catch (vm::VmVirtError& err) { + } catch (vm::VmVirtError err) { LOG(ERROR) << "virtualization error while traversing block transaction proof : " << err.get_msg(); return; } @@ -2602,30 +2457,32 @@ void TestNode::got_last_transactions(std::vector blkids, td::Bu bool TestNode::get_block_transactions(ton::BlockIdExt blkid, int mode, unsigned count, ton::Bits256 acc_addr, ton::LogicalTime lt) { + if (!(ready_ && !client_.empty())) { + return set_error("server connection not ready"); + } auto a = ton::create_tl_object(acc_addr, lt); auto b = ton::serialize_tl_object(ton::create_tl_object( ton::create_tl_lite_block_id(blkid), mode, count, std::move(a), false, false), true); LOG(INFO) << "requesting " << count << " transactions from block " << blkid.to_str() << " starting from account " << acc_addr.to_hex() << " lt " << lt; - return envelope_send_query_to_shard( - blkid.shard_full(), std::move(b), [Self = actor_id(this), mode](td::Result R) { - if (R.is_error()) { - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.listBlockTransactions"; - } else { - auto f = F.move_as_ok(); - std::vector transactions; - for (auto& id : f->ids_) { - transactions.emplace_back(id->account_, id->lt_, id->hash_); - } - td::actor::send_closure_later(Self, &TestNode::got_block_transactions, ton::create_block_id(f->id_), mode, - f->req_count_, f->incomplete_, std::move(transactions), std::move(f->proof_)); - } - }); + return envelope_send_query(std::move(b), [Self = actor_id(this), mode](td::Result R) { + if (R.is_error()) { + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.listBlockTransactions"; + } else { + auto f = F.move_as_ok(); + std::vector transactions; + for (auto& id : f->ids_) { + transactions.emplace_back(id->account_, id->lt_, id->hash_); + } + td::actor::send_closure_later(Self, &TestNode::got_block_transactions, ton::create_block_id(f->id_), mode, + f->req_count_, f->incomplete_, std::move(transactions), std::move(f->proof_)); + } + }); } void TestNode::got_block_transactions(ton::BlockIdExt blkid, int mode, unsigned req_count, bool incomplete, @@ -2651,23 +2508,25 @@ bool TestNode::get_all_shards(std::string filename, bool use_last, ton::BlockIdE if (!blkid.is_masterchain()) { return set_error("only masterchain blocks contain shard configuration"); } + if (!(ready_ && !client_.empty())) { + return set_error("server connection not ready"); + } auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); LOG(INFO) << "requesting recent shard configuration"; - return envelope_send_query_to_any( - std::move(b), [Self = actor_id(this), filename](td::Result R) -> void { - if (R.is_error()) { - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getAllShardsInfo"; - } else { - auto f = F.move_as_ok(); - td::actor::send_closure_later(Self, &TestNode::got_all_shards, ton::create_block_id(f->id_), - std::move(f->proof_), std::move(f->data_), filename); - } - }); + return envelope_send_query(std::move(b), [Self = actor_id(this), filename](td::Result R) -> void { + if (R.is_error()) { + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getAllShardsInfo"; + } else { + auto f = F.move_as_ok(); + td::actor::send_closure_later(Self, &TestNode::got_all_shards, ton::create_block_id(f->id_), std::move(f->proof_), + std::move(f->data_), filename); + } + }); } void TestNode::got_all_shards(ton::BlockIdExt blk, td::BufferSlice proof, td::BufferSlice data, std::string filename) { @@ -2731,6 +2590,9 @@ bool TestNode::parse_get_config_params(ton::BlockIdExt blkid, int mode, std::str params.push_back(x); } } + if (!(ready_ && !client_.empty())) { + return set_error("server connection not ready"); + } if (!blkid.is_masterchain_ext()) { return set_error("only masterchain blocks contain configuration"); } @@ -2749,6 +2611,10 @@ bool TestNode::get_config_params(ton::BlockIdExt blkid, td::Promise promise, int mode, std::string filename, std::vector params) { + if (!(ready_ && !client_.empty())) { + promise.set_error(td::Status::Error("server connection not ready")); + return false; + } if (!blkid.is_masterchain_ext()) { promise.set_error(td::Status::Error("masterchain reference block expected")); return false; @@ -2766,12 +2632,11 @@ bool TestNode::get_config_params_ext(ton::BlockIdExt blkid, td::Promise R) mutable { - td::actor::send_closure_later(Self, &TestNode::got_config_params, blkid, mode, filename, std::move(params), - std::move(R), std::move(promise)); - }); + return envelope_send_query(std::move(b), [Self = actor_id(this), mode, filename, blkid, params = std::move(params), + promise = std::move(promise)](td::Result R) mutable { + td::actor::send_closure_later(Self, &TestNode::got_config_params, blkid, mode, filename, std::move(params), + std::move(R), std::move(promise)); + }); } void TestNode::got_config_params(ton::BlockIdExt req_blkid, int mode, std::string filename, std::vector params, @@ -2959,8 +2824,8 @@ bool TestNode::get_block(ton::BlockIdExt blkid, bool dump) { LOG(INFO) << "got block download request for " << blkid.to_str(); auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); - return envelope_send_query_to_shard( - blkid.shard_full(), std::move(b), [Self = actor_id(this), blkid, dump](td::Result res) -> void { + return envelope_send_query( + std::move(b), [Self = actor_id(this), blkid, dump](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "cannot obtain block " << blkid.to_str() << " from server : " << res.move_as_error().to_string(); @@ -2988,8 +2853,8 @@ bool TestNode::get_state(ton::BlockIdExt blkid, bool dump) { LOG(INFO) << "got state download request for " << blkid.to_str(); auto b = ton::serialize_tl_object( ton::create_tl_object(ton::create_tl_lite_block_id(blkid)), true); - return envelope_send_query_to_shard( - blkid.shard_full(), std::move(b), [Self = actor_id(this), blkid, dump](td::Result res) -> void { + return envelope_send_query( + std::move(b), [Self = actor_id(this), blkid, dump](td::Result res) -> void { if (res.is_error()) { LOG(ERROR) << "cannot obtain state " << blkid.to_str() << " from server : " << res.move_as_error().to_string(); @@ -3126,7 +2991,7 @@ void TestNode::got_state(ton::BlockIdExt blkid, ton::RootHash root_hash, ton::Fi } bool TestNode::get_show_block_header(ton::BlockIdExt blkid, int mode) { - return get_block_header(blkid, mode, [this](td::Result R) { + return get_block_header(blkid, mode, [this, blkid](td::Result R) { if (R.is_error()) { LOG(ERROR) << "unable to fetch block header: " << R.move_as_error(); } else { @@ -3141,9 +3006,8 @@ bool TestNode::get_block_header(ton::BlockIdExt blkid, int mode, td::Promise(ton::create_tl_lite_block_id(blkid), mode), true); - return envelope_send_query_to_shard( - blkid.shard_full(), std::move(b), - [this, blkid, promise = std::move(promise)](td::Result R) mutable -> void { + return envelope_send_query( + std::move(b), [this, blkid, promise = std::move(promise)](td::Result R) mutable -> void { TRY_RESULT_PROMISE_PREFIX(promise, res, std::move(R), PSLICE() << "cannot obtain block header for " << blkid.to_str() << " from server :"); got_block_header_raw(std::move(res), std::move(promise), blkid); @@ -3169,9 +3033,8 @@ bool TestNode::lookup_block(ton::ShardIdFull shard, int mode, td::uint64 arg, auto b = ton::serialize_tl_object(ton::create_tl_object( mode, ton::create_tl_lite_block_id_simple(id), arg, (td::uint32)arg), true); - return envelope_send_query_to_shard( - shard, std::move(b), - [this, id, mode, arg, promise = std::move(promise)](td::Result R) mutable -> void { + return envelope_send_query( + std::move(b), [this, id, mode, arg, promise = std::move(promise)](td::Result R) mutable -> void { TRY_RESULT_PROMISE_PREFIX(promise, res, std::move(R), PSLICE() << "cannot look up block header for " << id.to_str() << " with mode " << mode << " and argument " << arg << " from server :"); @@ -3285,9 +3148,9 @@ void TestNode::got_block_header(ton::BlockIdExt blkid, td::BufferSlice data, int return; } show_block_header(blkid, std::move(virt_root), mode); - } catch (vm::VmError& err) { + } catch (vm::VmError err) { LOG(ERROR) << "error processing header for " << blkid.to_str() << " : " << err.get_msg(); - } catch (vm::VmVirtError& err) { + } catch (vm::VmVirtError err) { LOG(ERROR) << "error processing header for " << blkid.to_str() << " : " << err.get_msg(); } show_new_blkids(); @@ -3316,15 +3179,14 @@ bool TestNode::get_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mod ton::serialize_tl_object(ton::create_tl_object( mode & 0xfff, ton::create_tl_lite_block_id(from), ton::create_tl_lite_block_id(to)), true); - return envelope_send_query_to_any( - std::move(b), [Self = actor_id(this), from, to, mode](td::Result res) { - if (res.is_error()) { - LOG(ERROR) << "cannot obtain block proof for " << ((mode & 1) ? to.to_str() : "last masterchain block") - << " starting from " << from.to_str() << " from server : " << res.move_as_error().to_string(); - } else { - td::actor::send_closure_later(Self, &TestNode::got_block_proof, from, to, mode, res.move_as_ok()); - } - }); + return envelope_send_query(std::move(b), [Self = actor_id(this), from, to, mode](td::Result res) { + if (res.is_error()) { + LOG(ERROR) << "cannot obtain block proof for " << ((mode & 1) ? to.to_str() : "last masterchain block") + << " starting from " << from.to_str() << " from server : " << res.move_as_error().to_string(); + } else { + td::actor::send_closure_later(Self, &TestNode::got_block_proof, from, to, mode, res.move_as_ok()); + } + }); } void TestNode::got_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mode, td::BufferSlice pchain) { @@ -3378,6 +3240,9 @@ void TestNode::got_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mod bool TestNode::get_creator_stats(ton::BlockIdExt blkid, int mode, unsigned req_count, ton::Bits256 start_after, ton::UnixTime min_utime) { + if (!(ready_ && !client_.empty())) { + return set_error("server connection not ready"); + } if (!blkid.is_masterchain_ext()) { return set_error("only masterchain blocks contain block creator statistics"); } @@ -3388,8 +3253,8 @@ bool TestNode::get_creator_stats(ton::BlockIdExt blkid, int mode, unsigned req_c auto& os = *osp; return get_creator_stats( blkid, mode, req_count, start_after, min_utime, - [&os](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, - const block::DiscountedCounter& shard_cnt) -> bool { + [min_utime, &os](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, + const block::DiscountedCounter& shard_cnt) -> bool { os << key.to_hex() << " mc_cnt:" << mc_cnt << " shard_cnt:" << shard_cnt << std::endl; return true; }, @@ -3418,6 +3283,10 @@ bool TestNode::get_creator_stats(ton::BlockIdExt blkid, int mode, unsigned req_c bool TestNode::get_creator_stats(ton::BlockIdExt blkid, unsigned req_count, ton::UnixTime min_utime, TestNode::creator_stats_func_t func, std::unique_ptr state, td::Promise> promise) { + if (!(ready_ && !client_.empty())) { + promise.set_error(td::Status::Error("server connection not ready")); + return false; + } if (!state) { promise.set_error(td::Status::Error("null CreatorStatsRes")); return false; @@ -3436,7 +3305,7 @@ bool TestNode::get_creator_stats(ton::BlockIdExt blkid, unsigned req_count, ton: LOG(INFO) << "requesting up to " << req_count << " block creator stats records with respect to masterchain block " << blkid.to_str() << " starting from validator public key " << state->last_key.to_hex() << " created after " << min_utime << " (mode=" << state->mode << ")"; - return envelope_send_query_to_any( + return envelope_send_query( std::move(b), [this, blkid, req_count, state = std::move(state), min_utime, func = std::move(func), promise = std::move(promise)](td::Result R) mutable { TRY_RESULT_PROMISE(promise, res, std::move(R)); @@ -3653,8 +3522,8 @@ bool TestNode::load_creator_stats(std::unique_ptr l ton::UnixTime min_utime = info.valid_since - 1000; return get_creator_stats( info.blk_id, 1000, min_utime, - [&info](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, - const block::DiscountedCounter& shard_cnt) -> bool { + [min_utime, &info](const td::Bits256& key, const block::DiscountedCounter& mc_cnt, + const block::DiscountedCounter& shard_cnt) -> bool { info.store_record(key, mc_cnt, shard_cnt); return true; }, @@ -3857,7 +3726,7 @@ bool compute_punishment_default(int interval, bool severe, td::RefInt256& fine, fine = td::make_refint(101 * 1000000000LL); // 101 fine_part = 0; - return true; // todo: (tolya-yanot) temporary reduction of fine + return true; // todo: (tolya-yanot) temporary reduction of fine if (severe) { fine = td::make_refint(2500 * 1000000000LL); // GR$2500 @@ -3879,49 +3748,41 @@ bool compute_punishment_default(int interval, bool severe, td::RefInt256& fine, return true; } -bool compute_punishment(int interval, bool severe, td::RefInt256& fine, unsigned& fine_part, - Ref punishment_params) { - if (punishment_params.is_null()) { +bool compute_punishment(int interval, bool severe, td::RefInt256& fine, unsigned& fine_part, Ref punishment_params) { + if(punishment_params.is_null()) { return compute_punishment_default(interval, severe, fine, fine_part); } block::gen::MisbehaviourPunishmentConfig::Record rec; if (!tlb::unpack_cell(punishment_params, rec)) { - return false; + return false; } - if (interval <= rec.unpunishable_interval) { - return false; + if(interval <= rec.unpunishable_interval) { + return false; } fine = block::tlb::t_Grams.as_integer(rec.default_flat_fine); fine_part = rec.default_proportional_fine; if (severe) { - fine = fine * rec.severity_flat_mult; - fine >>= 8; - fine_part = fine_part * rec.severity_proportional_mult; - fine_part >>= 8; + fine = fine * rec.severity_flat_mult; fine >>= 8; + fine_part = fine_part * rec.severity_proportional_mult; fine_part >>= 8; } if (interval >= rec.long_interval) { - fine = fine * rec.long_flat_mult; - fine >>= 8; - fine_part = fine_part * rec.long_proportional_mult; - fine_part >>= 8; + fine = fine * rec.long_flat_mult; fine >>= 8; + fine_part = fine_part * rec.long_proportional_mult; fine_part >>= 8; return true; } if (interval >= rec.medium_interval) { - fine = fine * rec.medium_flat_mult; - fine >>= 8; - fine_part = fine_part * rec.medium_proportional_mult; - fine_part >>= 8; + fine = fine * rec.medium_flat_mult; fine >>= 8; + fine_part = fine_part * rec.medium_proportional_mult; fine_part >>= 8; return true; } return true; } -bool check_punishment(int interval, bool severe, td::RefInt256 fine, unsigned fine_part, - Ref punishment_params) { +bool check_punishment(int interval, bool severe, td::RefInt256 fine, unsigned fine_part, Ref punishment_params) { td::RefInt256 computed_fine; unsigned computed_fine_part; return compute_punishment(interval, severe, computed_fine, computed_fine_part, punishment_params) && @@ -3957,7 +3818,7 @@ td::Status TestNode::write_val_create_proof(TestNode::ValidatorLoadInfo& info1, int severity = (severe ? 2 : 1); td::RefInt256 fine = td::make_refint(101000000000); - unsigned fine_part = 0; // todo: (tolya-yanot) temporary reduction of fine // 0xffffffff / 16; // 1/16 + unsigned fine_part = 0; // todo: (tolya-yanot) temporary reduction of fine // 0xffffffff / 16; // 1/16 if (!compute_punishment(interval, severe, fine, fine_part, punishment_params)) { return td::Status::Error("cannot compute adequate punishment"); } @@ -4056,7 +3917,7 @@ td::Result> TestNode::ValidatorLoadInfo::build_proof(int idx, td:: block::gen::ValidatorDescr::Record_validator_addr rec2; if (tlb::csr_unpack(entry, rec1)) { pk = std::move(rec1.public_key); - } else if (tlb::csr_unpack(entry, rec2)) { + } else if (tlb::csr_unpack(std::move(entry), rec2)) { pk = std::move(rec2.public_key); } else { return td::Status::Error("cannot unpack ValidatorDescr"); @@ -4272,8 +4133,7 @@ td::Status TestNode::continue_check_validator_load_proof(std::unique_ptrconfig->get_config_param(40))) { + if (!check_punishment(interval, severe, suggested_fine, rec.suggested_fine_part, info2->config->get_config_param(40))) { LOG(ERROR) << "proposed punishment (fine " << td::dec_string(suggested_fine) << ", fine_part=" << (double)rec.suggested_fine_part / (1LL << 32) << " is too harsh"; show_vote(root->get_hash().bits(), false); @@ -4422,8 +4282,7 @@ int main(int argc, char* argv[]) { return (verbosity >= 0 && verbosity <= 9) ? td::Status::OK() : td::Status::Error("verbosity must be 0..9"); }); p.add_option('V', "version", "shows lite-client build information", [&]() { - std::cout << "lite-client build information: [ Commit: " << GitMetadata::CommitSHA1() - << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "lite-client build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); }); diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index da45ec92e..59fc1165c 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -26,6 +26,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "ext-client.h" #include "adnl/adnl-ext-client.h" #include "tl-utils/tl-utils.hpp" #include "ton/ton-types.h" @@ -46,35 +47,19 @@ class TestNode : public td::actor::Actor { min_ls_version = 0x101, min_ls_capabilities = 1 }; // server version >= 1.1, capabilities at least +1 = build proof chains + td::actor::ActorOwn client_; td::actor::ActorOwn io_; - - struct LiteServer { - td::IPAddress addr; - ton::PublicKey public_key; - bool is_full = true; - std::vector shards; - - td::actor::ActorOwn client; - bool client_ready = false; - std::vector> wait_client_ready; - - bool supports(ton::ShardIdFull shard) const; - }; - std::vector servers_; + bool ready_ = false; td::int32 single_liteserver_idx_ = -1; td::IPAddress single_remote_addr_; ton::PublicKey single_remote_public_key_; - std::map shard_server_idx_cached_; - bool readline_enabled_ = true; int print_limit_ = 1024; std::string db_root_; - // mc_server is the server for queries to masterchain - int mc_server_idx_ = -1; int mc_server_time_ = 0; int mc_server_time_got_at_ = 0; int mc_server_version_ = 0; @@ -443,18 +428,7 @@ class TestNode : public td::actor::Actor { void got_result(td::Result R, td::Promise promise); void after_got_result(bool ok); - bool envelope_send_query_to_any(td::BufferSlice query, td::Promise promise); - bool envelope_send_query_to_shard(ton::ShardIdFull shard, td::BufferSlice query, - td::Promise promise); - bool envelope_send_query_to_account(ton::AccountIdPrefixFull prefix, td::BufferSlice query, - td::Promise promise); - - bool envelope_send_query_to_server(td::int32 server_idx, td::BufferSlice query, td::Promise promise); - - void start_client(int server_idx); - void conn_ready(int server_idx); - void conn_closed(int server_idx); - + bool envelope_send_query(td::BufferSlice query, td::Promise promise); void parse_line(td::BufferSlice data); TestNode() = default; diff --git a/lite-client/query-utils.cpp b/lite-client/query-utils.cpp new file mode 100644 index 000000000..eddaa7276 --- /dev/null +++ b/lite-client/query-utils.cpp @@ -0,0 +1,394 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "query-utils.hpp" + +#include "block-parse.h" +#include "td/utils/overloaded.h" +#include "tl-utils/common-utils.hpp" + +#include "block/block-auto.h" +#include "auto/tl/lite_api.hpp" +#include "overlay/overlay-broadcast.hpp" +#include "tl-utils/lite-utils.hpp" +#include "ton/lite-tl.hpp" +#include "ton/ton-shard.h" + +#include + +namespace liteclient { + +using namespace ton; + +std::string QueryInfo::to_str() const { + td::StringBuilder sb; + sb << "[ " << lite_query_name_by_id(query_id) << " " << shard_id.to_str(); + switch (type) { + case t_simple: + break; + case t_seqno: + sb << " seqno=" << value; + break; + case t_utime: + sb << " utime=" << value; + break; + case t_lt: + sb << " lt=" << value; + break; + case t_mc_seqno: + sb << " mc_seqno=" << value; + break; + } + sb << " ]"; + return sb.as_cslice().str(); +} + +QueryInfo get_query_info(td::Slice data) { + auto F = fetch_tl_object(data, true); + if (F.is_ok()) { + data = F.ok()->data_; + } else { + fetch_tl_prefix(data, true).ignore(); + } + fetch_tl_prefix(data, true).ignore(); + auto Q = fetch_tl_object(data, true); + if (Q.is_error()) { + return {}; + } + return get_query_info(*Q.ok()); +} + +QueryInfo get_query_info(const lite_api::Function& f) { + QueryInfo info; + info.query_id = f.get_id(); + auto from_block_id = [&](const tl_object_ptr& id) { + BlockIdExt block_id = create_block_id(id); + info.shard_id = block_id.shard_full(); + info.type = QueryInfo::t_seqno; + info.value = block_id.seqno(); + }; + downcast_call( + const_cast(f), + td::overloaded([&](const lite_api::liteServer_getTime& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getVersion& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getMasterchainInfo& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getMasterchainInfoExt& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getBlock& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getBlockHeader& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getState& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getAccountState& q) { + BlockIdExt block_id = create_block_id(q.id_); + AccountIdPrefixFull acc_id_prefix = extract_addr_prefix(q.account_->workchain_, q.account_->id_); + info.shard_id = acc_id_prefix.as_leaf_shard(); + // See LiteQuery::perform_getAccountState + if (block_id.id.workchain != masterchainId) { + info.type = QueryInfo::t_seqno; + info.value = block_id.seqno(); + } else if (block_id.id.seqno != ~0U) { + info.type = QueryInfo::t_mc_seqno; + info.value = block_id.seqno(); + } else { + info.type = QueryInfo::t_simple; + } + }, + [&](const lite_api::liteServer_getAccountStatePrunned& q) { + BlockIdExt block_id = create_block_id(q.id_); + AccountIdPrefixFull acc_id_prefix = extract_addr_prefix(q.account_->workchain_, q.account_->id_); + info.shard_id = acc_id_prefix.as_leaf_shard(); + // See LiteQuery::perform_getAccountState + if (block_id.id.workchain != masterchainId) { + info.type = QueryInfo::t_seqno; + info.value = block_id.seqno(); + } else if (block_id.id.seqno != ~0U) { + info.type = QueryInfo::t_mc_seqno; + info.value = block_id.seqno(); + } else { + info.type = QueryInfo::t_simple; + } + }, + [&](const lite_api::liteServer_getOneTransaction& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getTransactions& q) { + AccountIdPrefixFull acc_id_prefix = extract_addr_prefix(q.account_->workchain_, q.account_->id_); + info.shard_id = acc_id_prefix.as_leaf_shard(); + info.type = QueryInfo::t_lt; + info.value = q.lt_; + }, + [&](const lite_api::liteServer_sendMessage& q) { + info.type = QueryInfo::t_simple; + auto r_root = vm::std_boc_deserialize(q.body_); + if (r_root.is_error()) { + return; + } + block::gen::CommonMsgInfo::Record_ext_in_msg_info msg_info; + if (!tlb::unpack_cell_inexact(r_root.ok(), msg_info)) { + return; + } + auto dest_prefix = block::tlb::MsgAddressInt::get_prefix(msg_info.dest); + if (!dest_prefix.is_valid()) { + return; + } + info.shard_id = dest_prefix.as_leaf_shard(); + }, + [&](const lite_api::liteServer_getShardInfo& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getAllShardsInfo& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_lookupBlock& q) { + BlockId block_id = create_block_id_simple(q.id_); + info.shard_id = block_id.shard_full(); + // See LiteQuery::perform_lookupBlock + if (q.mode_ & 1) { + info.type = QueryInfo::t_seqno; + info.value = block_id.seqno; + } else if (q.mode_ == 2) { + info.type = QueryInfo::t_lt; + info.value = q.lt_; + } else if (q.mode_ == 4) { + info.type = QueryInfo::t_utime; + info.value = q.utime_; + } + }, + [&](const lite_api::liteServer_lookupBlockWithProof& q) { + BlockId block_id = create_block_id_simple(q.id_); + info.shard_id = block_id.shard_full(); + // See LiteQuery::perform_lookupBlockWithProof + if (q.mode_ & 1) { + info.type = QueryInfo::t_seqno; + info.value = block_id.seqno; + } else if (q.mode_ == 2) { + info.type = QueryInfo::t_lt; + info.value = q.lt_; + } else if (q.mode_ == 4) { + info.type = QueryInfo::t_utime; + info.value = q.utime_; + } + }, + [&](const lite_api::liteServer_listBlockTransactions& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_listBlockTransactionsExt& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getConfigParams& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getConfigAll& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getBlockProof& q) { + info.shard_id = ShardIdFull{masterchainId}; + BlockIdExt from = create_block_id(q.known_block_); + BlockIdExt to = create_block_id(q.target_block_); + // See LiteQuery::perform_getBlockProof + if ((q.mode_ & 1) && (q.mode_ & 0x1000)) { + info.type = QueryInfo::t_seqno; + info.value = std::max(from.seqno(), to.seqno()); + } else { + info.type = QueryInfo::t_simple; + } + }, + [&](const lite_api::liteServer_getValidatorStats& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_runSmcMethod& q) { + BlockIdExt block_id = create_block_id(q.id_); + AccountIdPrefixFull acc_id_prefix = extract_addr_prefix(q.account_->workchain_, q.account_->id_); + info.shard_id = acc_id_prefix.as_leaf_shard(); + // See LiteQuery::perform_getAccountState + if (block_id.id.workchain != masterchainId) { + info.type = QueryInfo::t_seqno; + info.value = block_id.seqno(); + } else if (block_id.id.seqno != ~0U) { + info.type = QueryInfo::t_mc_seqno; + info.value = block_id.seqno(); + } else { + info.type = QueryInfo::t_simple; + } + }, + [&](const lite_api::liteServer_getLibraries& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getLibrariesWithProof& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getShardBlockProof& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_nonfinal_getCandidate& q) { /* t_simple */ }, + [&](const lite_api::liteServer_nonfinal_getValidatorGroups& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getOutMsgQueueSizes& q) { /* t_simple */ }, + [&](const auto&) { /* t_simple */ })); + if (info.shard_id.workchain == masterchainId) { + info.shard_id.shard = shardIdAll; + } + if (!info.shard_id.is_valid_ext()) { + info.shard_id = ShardIdFull{masterchainId}; + info.type = QueryInfo::t_simple; + info.value = 0; + } + return info; +} + +bool LiteServerConfig::accepts_query(const QueryInfo& query_info) const { + if (is_full) { + return true; + } + for (const Slice& s : slices) { + if (s.accepts_query(query_info)) { + return true; + } + } + return false; +} + +bool LiteServerConfig::Slice::accepts_query(const QueryInfo& query_info) const { + if (unlimited) { + for (const ShardInfo& shard : shards_from) { + if (shard_intersects(shard.shard_id, query_info.shard_id)) { + return true; + } + } + return false; + } + if (!shards_from.empty()) { + bool from_ok = false; + DCHECK(shards_from[0].shard_id.is_masterchain()); + for (const ShardInfo& shard : shards_from) { + if (shard_intersects(shard.shard_id, query_info.shard_id)) { + switch (query_info.type) { + case QueryInfo::t_simple: + from_ok = true; + break; + case QueryInfo::t_seqno: + from_ok = shard.seqno <= query_info.value; + break; + case QueryInfo::t_utime: + from_ok = shard.utime <= query_info.value; + break; + case QueryInfo::t_lt: + from_ok = shard.lt <= query_info.value; + break; + case QueryInfo::t_mc_seqno: + from_ok = shards_from[0].seqno <= query_info.value; + break; + } + if (from_ok) { + break; + } + } + } + if (!from_ok) { + return false; + } + } + if (!shards_to.empty()) { + bool to_ok = false; + DCHECK(shards_to[0].shard_id.is_masterchain()); + for (const ShardInfo& shard : shards_to) { + if (shard_intersects(shard.shard_id, query_info.shard_id)) { + switch (query_info.type) { + case QueryInfo::t_simple: + break; + case QueryInfo::t_seqno: + to_ok = shard.seqno >= query_info.value; + break; + case QueryInfo::t_utime: + to_ok = shard.utime >= query_info.value; + break; + case QueryInfo::t_lt: + to_ok = shard.lt >= query_info.value; + break; + case QueryInfo::t_mc_seqno: + to_ok = shards_from[0].seqno >= query_info.value; + break; + } + if (to_ok) { + break; + } + } + } + if (!to_ok) { + return false; + } + } + return true; +} + +td::Result> LiteServerConfig::parse_global_config( + const ton_api::liteclient_config_global& config) { + std::vector servers; + for (const auto& f : config.liteservers_) { + LiteServerConfig server; + TRY_STATUS(server.addr.init_host_port(td::IPAddress::ipv4_to_str(f->ip_), f->port_)); + server.adnl_id = adnl::AdnlNodeIdFull{PublicKey{f->id_}}; + server.is_full = true; + servers.push_back(std::move(server)); + } + for (const auto& f : config.liteservers_v2_) { + LiteServerConfig server; + TRY_STATUS(server.addr.init_host_port(td::IPAddress::ipv4_to_str(f->ip_), f->port_)); + server.adnl_id = adnl::AdnlNodeIdFull{PublicKey{f->id_}}; + server.is_full = false; + for (const auto& slice_obj : f->slices_) { + Slice slice; + td::Status S = td::Status::OK(); + downcast_call(*slice_obj, + td::overloaded( + [&](const ton_api::liteserver_descV2_sliceSimple& s) { + slice.unlimited = true; + slice.shards_from.push_back({ShardIdFull{masterchainId}, 0, 0, 0}); + for (const auto& shard_obj : s.shards_) { + ShardIdFull shard_id = create_shard_id(shard_obj); + if (!shard_id.is_valid_ext()) { + S = td::Status::Error(PSTRING() << "invalid shard id " << shard_id.to_str()); + break; + } + if (!shard_id.is_masterchain()) { + slice.shards_from.push_back({shard_id, 0, 0, 0}); + } + } + }, + [&](const ton_api::liteserver_descV2_sliceTimed& s) { + auto parse_shards = + [](const std::vector>& shard_objs, + std::vector& shards) -> td::Status { + if (shard_objs.empty()) { + return td::Status::OK(); + } + size_t i = 0; + int mc_idx = -1; + for (const auto& shard_obj : shard_objs) { + ShardIdFull shard_id = create_shard_id(shard_obj->shard_id_); + if (!shard_id.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard id " << shard_id.to_str()); + } + if (shard_id.is_masterchain()) { + shard_id = ShardIdFull{masterchainId}; + if (mc_idx != -1) { + return td::Status::Error("duplicate masterchain shard in sliceTimed"); + } + mc_idx = (int)i; + } + shards.push_back({shard_id, (BlockSeqno)shard_obj->seqno_, (UnixTime)shard_obj->utime_, + (LogicalTime)shard_obj->lt_}); + ++i; + } + if (mc_idx == -1) { + return td::Status::Error("no masterchain shard in sliceTimed"); + } + std::swap(shards[0], shards[mc_idx]); + return td::Status::OK(); + }; + S = parse_shards(s.shards_from_, slice.shards_from); + if (S.is_ok()) { + S = parse_shards(s.shards_to_, slice.shards_to); + } + if (S.is_ok() && slice.shards_from.empty() && slice.shards_to.empty()) { + S = td::Status::Error("shards_from and shards_to are both empty"); + } + })); + TRY_STATUS(std::move(S)); + server.slices.push_back(slice); + } + + servers.push_back(std::move(server)); + } + return servers; +} + +} // namespace liteclient \ No newline at end of file diff --git a/lite-client/query-utils.hpp b/lite-client/query-utils.hpp new file mode 100644 index 000000000..28500e266 --- /dev/null +++ b/lite-client/query-utils.hpp @@ -0,0 +1,89 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "ton/ton-types.h" +#include "auto/tl/lite_api.h" +#include "td/utils/port/IPAddress.h" +#include "adnl/adnl-node-id.hpp" + +namespace liteclient { + +struct QueryInfo { + enum Type { t_simple, t_seqno, t_utime, t_lt, t_mc_seqno }; + int query_id = 0; + ton::ShardIdFull shard_id{ton::masterchainId}; + Type type = t_simple; + td::uint64 value = 0; + /* Query types and examples: + * t_simple - query to the recent blocks in a shard, or general info. value = 0. + * getTime, getMasterchainInfo (shard_id = masterchain) + * sendMessage + * getAccountState, runSmcMethod - when no block is given + * t_seqno - query to block with seqno in a shard. value = seqno. + * lookupBlock by seqno + * getBlock, getBlockHeader + * getAccountState, runSmcMethod - when shard block is given + * t_utime - query to a block with given unixtime in a shard. value = utime. + * lookupBlock by utime + * t_lt - query to a block with given lt in a shard. value = lt. + * lookupBlock by lt + * getTransactions + * t_mc_seqno - query to a block in a shard, masterchain seqno is given. value = mc_seqno. + * getAccountState, runSmcMethod - when mc block is given + */ + + std::string to_str() const; +}; + +QueryInfo get_query_info(td::Slice data); +QueryInfo get_query_info(const ton::lite_api::Function& f); + +struct LiteServerConfig { + private: + struct ShardInfo { + ton::ShardIdFull shard_id; + ton::BlockSeqno seqno; + ton::UnixTime utime; + ton::LogicalTime lt; + }; + + struct Slice { + std::vector shards_from, shards_to; + bool unlimited = false; + + bool accepts_query(const QueryInfo& query_info) const; + }; + + bool is_full = false; + std::vector slices; + + public: + ton::adnl::AdnlNodeIdFull adnl_id; + td::IPAddress addr; + + LiteServerConfig() = default; + LiteServerConfig(ton::adnl::AdnlNodeIdFull adnl_id, td::IPAddress addr) + : is_full(true), adnl_id(adnl_id), addr(addr) { + } + + bool accepts_query(const QueryInfo& query_info) const; + + static td::Result> parse_global_config( + const ton::ton_api::liteclient_config_global& config); +}; + +} // namespace liteclient diff --git a/tl-utils/common-utils.hpp b/tl-utils/common-utils.hpp index d61bd79c0..05f8a825f 100644 --- a/tl-utils/common-utils.hpp +++ b/tl-utils/common-utils.hpp @@ -148,6 +148,39 @@ td::Result::value, T>>> } } +template +td::Result::value, T>>> fetch_tl_prefix(td::Slice &data, + bool boxed) { + td::TlParser p(data); + tl_object_ptr R; + if (boxed) { + R = TlFetchBoxed, T::ID>::parse(p); + } else { + R = move_tl_object_as(T::fetch(p)); + } + if (p.get_status().is_ok()) { + data.remove_prefix(data.size() - p.get_left_len()); + return std::move(R); + } else { + return p.get_status(); + } +} + +template +td::Result::value, T>>> fetch_tl_prefix(td::Slice &data, + bool boxed) { + CHECK(boxed); + td::TlParser p(data); + tl_object_ptr R; + R = move_tl_object_as(T::fetch(p)); + if (p.get_status().is_ok()) { + data.remove_prefix(data.size() - p.get_left_len()); + return std::move(R); + } else { + return p.get_status(); + } +} + template [[deprecated]] tl_object_ptr clone_tl_object(const tl_object_ptr &obj) { auto B = serialize_tl_object(obj, true); diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 0d7f1249e..3a46406fa 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -588,8 +588,12 @@ dummyworkchain0.config.global zero_state_hash:int256 = dummyworkchain0.config.Gl validator.config.global zero_state:tonNode.blockIdExt init_block:tonNode.blockIdExt hardforks:(vector tonNode.blockIdExt) = validator.config.Global; config.global adnl:adnl.config.global dht:dht.config.Global validator:validator.config.global = config.Global; +liteserver.descV2.sliceSimple shards:(vector tonNode.shardId) = liteserver.descV2.Slice; +liteserver.descV2.shardInfo shard_id:tonNode.shardId seqno:int utime:int lt:long = liteserver.descV2.ShardInfo; +liteserver.descV2.sliceTimed shards_from:(vector liteserver.descV2.shardInfo) shards_to:(vector liteserver.descV2.shardInfo) = liteserver.descV2.Slice; + liteserver.desc id:PublicKey ip:int port:int = liteserver.Desc; -liteserver.descV2 id:PublicKey ip:int port:int shards:(vector tonNode.shardId) = liteserver.DescV2; +liteserver.descV2 id:PublicKey ip:int port:int slices:(vector liteserver.descV2.Slice) = liteserver.DescV2; liteclient.config.global liteservers:(vector liteserver.desc) liteservers_v2:(vector liteserver.descV2) validator:validator.config.global = liteclient.config.Global; engine.adnl id:int256 category:int = engine.Adnl; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 0d286bdc091a5b25e2a8deab3019aa84e0e73fe5..77bcff0c5f4e7a7914a3bc653e7cc4f5587d3720 100644 GIT binary patch delta 463 zcmZ4Thqa@TjrY-PeJchiklV=nTb`v`hTnTKk3xdXlI@yI(m9zWsl};9WvNAaDXGQD zVMcnv8Hq(HlO0rLWPVn(d5dFFkdv95I?+L5bB;oT1mp6}Z!M(Tpr-D>A_FpB5JM9K z1K8R>`_q_#x+e=xm6rgS4s%(NGn)>FZQDVxqB#dXXyR~NG01J6d1?6|`J2atd{`#G z+^Rja8wX`HNck)F~>B$XM3br75nBiCDxMH|NPj<*+xJs`WTR56j>%AtW<5H3euL%mk3t($DT|a)aGl9G_N{pF4R&qKpK{ eS&UG3ffR5;6--`}sDL5~3lNZrn;Q-;*#iK}$DR!U delta 107 zcmeBZWLxrwmG{wXeJchiklD!lTb@PP!Crqdk3z!c8ww2)j7v60T1vM;1S|GmfiMCN uh;V|G-8?4b!!p@0Sb6e<19p?o%~hKGVFAx%g~J>WWgG`TZ2oX~#vTA1R3_5c6? diff --git a/tonlib/tonlib/Config.cpp b/tonlib/tonlib/Config.cpp index 73d4e033c..0f317f94a 100644 --- a/tonlib/tonlib/Config.cpp +++ b/tonlib/tonlib/Config.cpp @@ -19,6 +19,7 @@ #include "Config.h" #include "adnl/adnl-node-id.hpp" #include "td/utils/JsonBuilder.h" +#include "auto/tl/ton_api_json.h" namespace tonlib { td::Result parse_block_id_ext(td::JsonObject &obj) { @@ -65,67 +66,11 @@ td::Result Config::parse(std::string str) { if (json.type() != td::JsonValue::Type::Object) { return td::Status::Error("Invalid config (1)"); } - td::JsonArray empty_array; - TRY_RESULT(lite_servers_obj, - td::get_json_object_field(json.get_object(), "liteservers", td::JsonValue::Type::Array, true)); - auto &lite_servers = - lite_servers_obj.type() == td::JsonValue::Type::Array ? lite_servers_obj.get_array() : empty_array; - TRY_RESULT(lite_servers_v2_obj, - td::get_json_object_field(json.get_object(), "liteservers_v2", td::JsonValue::Type::Array, true)); - auto &lite_servers_v2 = - lite_servers_v2_obj.type() == td::JsonValue::Type::Array ? lite_servers_v2_obj.get_array() : empty_array; - - auto parse_desc = [&](td::JsonValue &value) -> td::Result { - if (value.type() != td::JsonValue::Type::Object) { - return td::Status::Error("Invalid config (2)"); - } - auto &object = value.get_object(); - - TRY_RESULT(ip, td::get_json_object_long_field(object, "ip", false)); - TRY_RESULT(port, td::get_json_object_int_field(object, "port", false)); - Config::LiteServer server; - TRY_STATUS(server.address.init_host_port(td::IPAddress::ipv4_to_str(static_cast(ip)), port)); - - TRY_RESULT(id_obj, td::get_json_object_field(object, "id", td::JsonValue::Type::Object, false)); - auto &id = id_obj.get_object(); - TRY_RESULT(id_type, td::get_json_object_string_field(id, "@type", false)); - if (id_type != "pub.ed25519") { - return td::Status::Error("Invalid config (3)"); - } - TRY_RESULT(key_base64, td::get_json_object_string_field(id, "key", false)); - TRY_RESULT(key, td::base64_decode(key_base64)); - if (key.size() != 32) { - return td::Status::Error("Invalid config (4)"); - } - - server.adnl_id = ton::adnl::AdnlNodeIdFull(ton::pubkeys::Ed25519(td::Bits256(td::Slice(key).ubegin()))); - return server; - }; Config res; - for (auto &value : lite_servers) { - TRY_RESULT(server, parse_desc(value)); - res.lite_servers.push_back(std::move(server)); - } - for (auto &value : lite_servers_v2) { - TRY_RESULT(server, parse_desc(value)); - server.is_full = false; - TRY_RESULT(shards_obj, td::get_json_object_field(value.get_object(), "shards", td::JsonValue::Type::Array, false)); - for (auto &shard : shards_obj.get_array()) { - if (shard.type() != td::JsonValue::Type::Object) { - return td::Status::Error("Invalid config (5)"); - } - auto &shard_obj = shard.get_object(); - TRY_RESULT(workchain, td::get_json_object_int_field(shard_obj, "workchain", false)); - TRY_RESULT(shard_id, td::get_json_object_long_field(shard_obj, "shard", false)); - if (shard_id == 0) { - return td::Status::Error("Invalid config (6)"); - } - server.shards.emplace_back(workchain, shard_id); - } - - res.lite_servers.push_back(std::move(server)); - } + ton::ton_api::liteclient_config_global conf; + TRY_STATUS(ton::ton_api::from_json(conf, json.get_object())); + TRY_RESULT_ASSIGN(res.lite_servers, liteclient::LiteServerConfig::parse_global_config(conf)); TRY_RESULT(validator_obj, td::get_json_object_field(json.get_object(), "validator", td::JsonValue::Type::Object, false)); diff --git a/tonlib/tonlib/Config.h b/tonlib/tonlib/Config.h index 5ae304ea8..28f23881b 100644 --- a/tonlib/tonlib/Config.h +++ b/tonlib/tonlib/Config.h @@ -24,11 +24,10 @@ namespace tonlib { struct Config { - using LiteServer = liteclient::ExtClient::LiteServer; ton::BlockIdExt zero_state_id; ton::BlockIdExt init_block_id; std::vector hardforks; - std::vector lite_servers; + std::vector lite_servers; std::string name; static td::Result parse(std::string str); }; diff --git a/tonlib/tonlib/ExtClient.cpp b/tonlib/tonlib/ExtClient.cpp index f0c84d616..b66ca25c1 100644 --- a/tonlib/tonlib/ExtClient.cpp +++ b/tonlib/tonlib/ExtClient.cpp @@ -54,7 +54,7 @@ void ExtClient::with_last_block(td::Promise promise) { td::actor::send_closure(client_.last_block_actor_, &LastBlock::get_last_block, std::move(P)); } -void ExtClient::send_raw_query(td::BufferSlice query, ton::ShardIdFull shard, td::Promise promise) { +void ExtClient::send_raw_query(td::BufferSlice query, td::Promise promise) { auto query_id = queries_.create(std::move(promise)); td::Promise P = [query_id, self = this, actor_id = td::actor::actor_id()](td::Result result) { @@ -66,6 +66,6 @@ void ExtClient::send_raw_query(td::BufferSlice query, ton::ShardIdFull shard, td return P.set_error(TonlibError::NoLiteServers()); } td::actor::send_closure(client_.adnl_ext_client_, &liteclient::ExtClient::send_query, "query", std::move(query), - shard, td::Timestamp::in(10.0), std::move(P)); + td::Timestamp::in(10.0), std::move(P)); } } // namespace tonlib diff --git a/tonlib/tonlib/ExtClient.h b/tonlib/tonlib/ExtClient.h index 9db040bb1..b0194c18f 100644 --- a/tonlib/tonlib/ExtClient.h +++ b/tonlib/tonlib/ExtClient.h @@ -31,7 +31,6 @@ #include "lite-client/ext-client.h" #include "TonlibError.h" #include "utils.h" -#include "lite-client/QueryTraits.h" namespace tonlib { class LastBlock; @@ -65,7 +64,6 @@ class ExtClient { template void send_query(QueryT query, td::Promise promise, td::int32 seq_no = -1) { - ton::ShardIdFull shard = liteclient::QueryTraits::get_shard(query); auto raw_query = ton::serialize_tl_object(&query, true); td::uint32 tag = td::Random::fast_uint32(); VLOG(lite_server) << "send query to liteserver: " << tag << " " << to_string(query); @@ -79,7 +77,7 @@ class ExtClient { ton::serialize_tl_object(ton::create_tl_object(std::move(raw_query)), true); send_raw_query( - std::move(liteserver_query), shard, [promise = std::move(promise), tag](td::Result R) mutable { + std::move(liteserver_query), [promise = std::move(promise), tag](td::Result R) mutable { auto res = [&]() -> td::Result { TRY_RESULT_PREFIX(data, std::move(R), TonlibError::LiteServerNetwork()); auto r_error = ton::fetch_tl_object(data.clone(), true); @@ -99,7 +97,7 @@ class ExtClient { void force_change_liteserver() { if (!client_.adnl_ext_client_.empty()) { - td::actor::send_closure(client_.adnl_ext_client_, &liteclient::ExtClient::force_change_liteserver); + td::actor::send_closure(client_.adnl_ext_client_, &liteclient::ExtClient::reset_servers); } } @@ -109,6 +107,6 @@ class ExtClient { td::Container> last_block_queries_; td::Container> last_config_queries_; - void send_raw_query(td::BufferSlice query, ton::ShardIdFull shard, td::Promise promise); + void send_raw_query(td::BufferSlice query, td::Promise promise); }; } // namespace tonlib diff --git a/tonlib/tonlib/ExtClientOutbound.cpp b/tonlib/tonlib/ExtClientOutbound.cpp index eb923734e..e5fac8b47 100644 --- a/tonlib/tonlib/ExtClientOutbound.cpp +++ b/tonlib/tonlib/ExtClientOutbound.cpp @@ -28,14 +28,11 @@ class ExtClientOutboundImpl : public ExtClientOutbound { ExtClientOutboundImpl(td::unique_ptr callback) : callback_(std::move(callback)) { } - void send_query(std::string name, td::BufferSlice data, ton::ShardIdFull shard, td::Timestamp timeout, + void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, td::Promise promise) override { auto query_id = next_query_id_++; queries_[query_id] = std::move(promise); - callback_->request(query_id, data.as_slice().str(), shard); - } - - void force_change_liteserver() override { + callback_->request(query_id, data.as_slice().str()); } void on_query_result(td::int64 id, td::Result r_data, td::Promise promise) override { diff --git a/tonlib/tonlib/ExtClientOutbound.h b/tonlib/tonlib/ExtClientOutbound.h index 6eb6aa983..bf52c8c26 100644 --- a/tonlib/tonlib/ExtClientOutbound.h +++ b/tonlib/tonlib/ExtClientOutbound.h @@ -27,9 +27,8 @@ class ExtClientOutbound : public liteclient::ExtClient { public: virtual ~Callback() { } - virtual void request(td::int64 id, std::string data, ton::ShardIdFull shard) = 0; + virtual void request(td::int64 id, std::string data) = 0; }; - virtual void on_query_result(td::int64 id, td::Result r_data, td::Promise promise) = 0; static td::actor::ActorOwn create(td::unique_ptr callback); }; diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 3dda7feaf..13a69594c 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -2078,9 +2078,8 @@ ExtClientRef TonlibClient::get_client_ref() { return ref; } -void TonlibClient::proxy_request(td::int64 query_id, std::string data, ton::ShardIdFull shard) { - on_update( - tonlib_api::make_object(query_id, data, shard.workchain, shard.shard)); +void TonlibClient::proxy_request(td::int64 query_id, std::string data) { + on_update(tonlib_api::make_object(query_id, data)); } void TonlibClient::init_ext_client() { @@ -2091,9 +2090,9 @@ void TonlibClient::init_ext_client() { : parent_(std::move(parent)), config_generation_(config_generation) { } - void request(td::int64 id, std::string data, ton::ShardIdFull shard) override { - send_closure(parent_, &TonlibClient::proxy_request, (id << 16) | (config_generation_ & 0xffff), std::move(data), - shard); + void request(td::int64 id, std::string data) override { + send_closure(parent_, &TonlibClient::proxy_request, (id << 16) | (config_generation_ & 0xffff), + std::move(data)); } private: @@ -2106,17 +2105,8 @@ void TonlibClient::init_ext_client() { ext_client_outbound_ = client.get(); raw_client_ = std::move(client); } else { - class Callback : public liteclient::ExtClient::Callback { - public: - explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { - } - - private: - td::actor::ActorShared<> parent_; - }; ext_client_outbound_ = {}; - ref_cnt_++; - raw_client_ = liteclient::ExtClient::create(config_.lite_servers, td::make_unique(td::actor::actor_shared())); + raw_client_ = liteclient::ExtClient::create(config_.lite_servers, nullptr); } } @@ -4491,8 +4481,8 @@ void deep_library_search(std::set& set, std::set& v } return; } - for (unsigned int i = 0; i < loaded_cell.data_cell->get_refs_cnt(); i++) { - deep_library_search(set, visited, libs, loaded_cell.data_cell->get_ref(i), depth - 1); + for (unsigned int i=0; iget_refs_cnt(); i++) { + deep_library_search(set, visited, libs, loaded_cell.data_cell->get_ref(i), depth - 1, max_libs); } } diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index 32731e4e9..973ede94a 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -401,7 +401,7 @@ class TonlibClient : public td::actor::Actor { td::Status do_request(const tonlib_api::getConfigAll& request, td::Promise>&& promise); - void proxy_request(td::int64 query_id, std::string data, ton::ShardIdFull shard); + void proxy_request(td::int64 query_id, std::string data); void load_libs_from_disk(); void store_libs_to_disk(); diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index ba44481f5..eb0f5af4f 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -1541,7 +1541,7 @@ class TonlibCli : public td::actor::Actor { CHECK(!raw_client_.empty()); snd_bytes_ += update->data_.size(); send_closure(raw_client_, &liteclient::ExtClient::send_query, "query", td::BufferSlice(update->data_), - ton::ShardIdFull(update->workchain_, update->shard_), td::Timestamp::in(5), + td::Timestamp::in(5), [actor_id = actor_id(this), id = update->id_](td::Result res) { send_closure(actor_id, &TonlibCli::on_adnl_result, id, std::move(res)); }); diff --git a/utils/proxy-liteserver.cpp b/utils/proxy-liteserver.cpp index 7abaa1c7a..66161fc45 100644 --- a/utils/proxy-liteserver.cpp +++ b/utils/proxy-liteserver.cpp @@ -29,50 +29,71 @@ #include "td/utils/OptionParser.h" #include "td/utils/port/path.h" #include "td/utils/port/signals.h" -#include "td/utils/port/user.h" #include "td/utils/port/IPAddress.h" #include "td/utils/Random.h" #include "td/utils/FileLog.h" #include "git.h" #include "auto/tl/ton_api.h" #include "auto/tl/lite_api.h" -#include "auto/tl/lite_api.hpp" #include "tl-utils/lite-utils.hpp" -#include "ton/lite-tl.hpp" #include "auto/tl/ton_api_json.h" #include "adnl/adnl.h" -#include "lite-client/QueryTraits.h" #include "lite-client/ext-client.h" #if TD_DARWIN || TD_LINUX #include #endif #include +#include using namespace ton; class ProxyLiteserver : public td::actor::Actor { public: - ProxyLiteserver(std::string global_config, std::string db_root, td::uint16 port) - : global_config_(std::move(global_config)), db_root_(std::move(db_root)), port_(port) { + ProxyLiteserver(std::string global_config, std::string db_root, td::uint16 port, PublicKeyHash public_key_hash) + : global_config_(std::move(global_config)) + , db_root_(std::move(db_root)) + , port_(port) + , public_key_hash_(public_key_hash) { } void start_up() override { - LOG_CHECK(db_root_ != "") << "db root is not set"; + LOG_CHECK(!db_root_.empty()) << "db root is not set"; td::mkdir(db_root_).ensure(); db_root_ = td::realpath(db_root_).move_as_ok(); keyring_ = keyring::Keyring::create(db_root_ + "/keyring"); + if (public_key_hash_.is_zero()) { + id_ = {}; + run(); + } else { + td::actor::send_closure(keyring_, &keyring::Keyring::get_public_key, public_key_hash_, + [SelfId = actor_id(this)](td::Result R) mutable { + if (R.is_error()) { + LOG(FATAL) << "Failed to load public key: " << R.move_as_error(); + } + td::actor::send_closure(SelfId, &ProxyLiteserver::got_public_key, R.move_as_ok()); + }); + } + } + + void got_public_key(PublicKey pub) { + id_ = adnl::AdnlNodeIdFull{pub}; + run(); + } + + void run() { td::Status S = prepare_local_config(); if (S.is_error()) { LOG(FATAL) << "Local config error: " << S; } - S = create_ext_client(); + S = parse_global_config(); if (S.is_error()) { LOG(FATAL) << S; } + run_clients(); create_ext_server(); } @@ -82,169 +103,198 @@ class ProxyLiteserver : public td::actor::Actor { auto conf_data = r_conf_data.move_as_ok(); TRY_RESULT_PREFIX(conf_json, td::json_decode(conf_data.as_slice()), "failed to parse json: "); TRY_STATUS_PREFIX(ton_api::from_json(*config_, conf_json.get_object()), "json does not fit TL scheme: "); - TRY_RESULT_PREFIX_ASSIGN(port_, td::narrow_cast_safe(config_->port_), "invalid port: "); - TRY_RESULT_PREFIX_ASSIGN(id_, adnl::AdnlNodeIdFull::create(config_->id_), "invalid id: "); - } else { - LOG(WARNING) << "First launch, creating local config"; + TRY_RESULT_PREFIX(cfg_port, td::narrow_cast_safe(config_->port_), "invalid port: "); + TRY_RESULT_PREFIX(cfg_id, adnl::AdnlNodeIdFull::create(config_->id_), "invalid id: "); + bool rewrite_config = false; if (port_ == 0) { - return td::Status::Error("port is not set"); + port_ = cfg_port; + } else { + rewrite_config |= (port_ != cfg_port); } - config_->port_ = port_; + if (id_.empty()) { + id_ = std::move(cfg_id); + } else { + rewrite_config |= (id_ != cfg_id); + } + if (!rewrite_config) { + return td::Status::OK(); + } + } else { + LOG(WARNING) << "First launch, creating local config"; + } + if (port_ == 0) { + return td::Status::Error("port is not set"); + } + config_->port_ = port_; + if (id_.empty()) { auto pk = PrivateKey{privkeys::Ed25519::random()}; id_ = adnl::AdnlNodeIdFull{pk.compute_public_key()}; - config_->id_ = id_.tl(); td::actor::send_closure(keyring_, &keyring::Keyring::add_key, std::move(pk), false, [](td::Result R) { if (R.is_error()) { LOG(FATAL) << "Failed to store private key"; } }); - - auto s = td::json_encode(td::ToJson(*config_), true); - TRY_STATUS_PREFIX(td::write_file(config_file(), s), "failed to write file: "); } + config_->id_ = id_.tl(); + + auto s = td::json_encode(td::ToJson(*config_), true); + TRY_STATUS_PREFIX(td::write_file(config_file(), s), "failed to write file: "); + LOG(WARNING) << "Writing config.json"; return td::Status::OK(); } - td::Status create_ext_client() { - std::vector servers; + td::Status parse_global_config() { TRY_RESULT_PREFIX(global_config_data, td::read_file(global_config_), "Failed to read global config: "); TRY_RESULT_PREFIX(global_config_json, td::json_decode(global_config_data.as_slice()), "Failed to parse global config: "); - ton::ton_api::liteclient_config_global gc; - ton::ton_api::from_json(gc, global_config_json.get_object()).ensure(); - - size_t size = gc.liteservers_.size() + gc.liteservers_v2_.size(); - if (size == 0) { + ton_api::liteclient_config_global gc; + TRY_STATUS_PREFIX(ton_api::from_json(gc, global_config_json.get_object()), "Failed to parse global config: "); + TRY_RESULT_PREFIX(servers, liteclient::LiteServerConfig::parse_global_config(gc), + "Falied to parse liteservers in global config: "); + if (servers.empty()) { return td::Status::Error("No liteservers in global config"); } - - for (auto& s : gc.liteservers_) { - td::IPAddress addr; - addr.init_host_port(td::IPAddress::ipv4_to_str(s->ip_), s->port_).ensure(); - liteclient::ExtClient::LiteServer serv; - serv.address = addr; - serv.adnl_id = ton::adnl::AdnlNodeIdFull::create(s->id_).move_as_ok(); - servers.push_back(std::move(serv)); + for (auto& s : servers) { + servers_.emplace_back(); + servers_.back().config = std::move(s); } - for (auto& s : gc.liteservers_v2_) { - td::IPAddress addr; - addr.init_host_port(td::IPAddress::ipv4_to_str(s->ip_), s->port_).ensure(); - liteclient::ExtClient::LiteServer serv; - serv.address = addr; - serv.adnl_id = ton::adnl::AdnlNodeIdFull::create(s->id_).move_as_ok(); - serv.is_full = false; - for (auto& shard : s->shards_) { - serv.shards.emplace_back(shard->workchain_, (ton::ShardId)shard->shard_); - CHECK(serv.shards.back().is_valid_ext()); + return td::Status::OK(); + } + + void run_clients() { + class Callback : public adnl::AdnlExtClient::Callback { + public: + explicit Callback(td::actor::ActorId id, size_t idx) : id_(std::move(id)), idx_(idx) { + } + void on_ready() override { + td::actor::send_closure(id_, &ProxyLiteserver::on_client_status, idx_, true); } - servers.push_back(std::move(serv)); + void on_stop_ready() override { + td::actor::send_closure(id_, &ProxyLiteserver::on_client_status, idx_, false); + } + + private: + td::actor::ActorId id_; + size_t idx_; + }; + + for (size_t i = 0; i < servers_.size(); ++i) { + Server& server = servers_[i]; + server.client = adnl::AdnlExtClient::create(server.config.adnl_id, server.config.addr, + std::make_unique(actor_id(this), i)); + server.alive = false; } - class Callback : public liteclient::ExtClient::Callback {}; - ext_client_ = liteclient::ExtClient::create(std::move(servers), td::make_unique()); - return td::Status::OK(); } - void create_ext_server() { - adnl_ = adnl::Adnl::create("", keyring_.get()); - td::actor::send_closure(adnl_, &adnl::Adnl::add_id, id_, ton::adnl::AdnlAddressList{}, (td::uint8)255); - td::actor::send_closure(adnl_, &adnl::Adnl::create_ext_server, - std::vector{id_.compute_short_id()}, std::vector{port_}, - [SelfId = actor_id(this)](td::Result> R) { - R.ensure(); - td::actor::send_closure(SelfId, &ProxyLiteserver::created_ext_server, R.move_as_ok()); - }); + void on_client_status(size_t idx, bool ready) { + Server& server = servers_[idx]; + if (server.alive == ready) { + return; + } + server.alive = ready; + LOG(WARNING) << (ready ? "Connected to" : "Disconnected from") << " server #" << idx << " (" + << server.config.addr.get_ip_str() << ":" << server.config.addr.get_port() << ")"; } - void created_ext_server(td::actor::ActorOwn s) { - ext_server_ = std::move(s); - LOG(WARNING) << "Started proxy liteserver on port " << port_; + void create_ext_server() { + adnl_ = adnl::Adnl::create("", keyring_.get()); + td::actor::send_closure(adnl_, &adnl::Adnl::add_id, id_, adnl::AdnlAddressList{}, (td::uint8)255); class AdnlCallback : public adnl::Adnl::Callback { public: - AdnlCallback(td::actor::ActorId client) : client_(client) { + explicit AdnlCallback(td::actor::ActorId id) : id_(id) { } void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { } void receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) override { - td::actor::create_actor("worker", client_, std::move(data), std::move(promise)).release(); + td::actor::send_closure(id_, &ProxyLiteserver::receive_query, std::move(data), std::move(promise)); } private: - td::actor::ActorId client_; + td::actor::ActorId id_; }; td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, id_.compute_short_id(), adnl::Adnl::int_to_bytestring(lite_api::liteServer_query::ID), - std::make_unique(ext_client_.get())); + std::make_unique(actor_id(this))); + td::actor::send_closure(adnl_, &adnl::Adnl::create_ext_server, std::vector{id_.compute_short_id()}, + std::vector{port_}, + [SelfId = actor_id(this)](td::Result> R) { + R.ensure(); + td::actor::send_closure(SelfId, &ProxyLiteserver::created_ext_server, R.move_as_ok()); + }); } - class QueryWorker : public td::actor::Actor { - public: - QueryWorker(td::actor::ActorId client, td::BufferSlice data, - td::Promise promise) - : client_(std::move(client)), data_(std::move(data)), promise_(std::move(promise)) { - } + void created_ext_server(td::actor::ActorOwn s) { + ext_server_ = std::move(s); + LOG(WARNING) << "Started proxy liteserver on port " << port_; + alarm(); + } - void start_up() override { - auto data = data_.clone(); - auto F = fetch_tl_object(data, true); - if (F.is_ok()) { - data = std::move(F.move_as_ok()->data_); - } else { - auto G = fetch_tl_prefix(data, true); - if (G.is_error()) { - fatal_error(G.move_as_error()); - return; - } + td::Result select_server(const liteclient::QueryInfo& query_info) { + size_t best_idx = servers_.size(); + int cnt = 0; + for (size_t i = 0; i < servers_.size(); ++i) { + Server& server = servers_[i]; + if (!server.alive || !server.config.accepts_query(query_info)) { + continue; } - fetch_tl_prefix(data, true).ignore(); - auto F2 = fetch_tl_object(std::move(data), true); - if (F2.is_error()) { - fatal_error(F2.move_as_error()); - return; + ++cnt; + if (td::Random::fast(1, cnt) == 1) { + best_idx = i; } - auto query = F2.move_as_ok(); - lite_api::downcast_call(*query, [&](auto& obj) { shard_ = liteclient::get_query_shard(obj); }); - - LOG(INFO) << "Got query: shard=" << shard_.to_str() << " size=" << data_.size(); - td::actor::send_closure(client_, &liteclient::ExtClient::send_query, "q", std::move(data_), shard_, - td::Timestamp::in(8.0), [SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &QueryWorker::got_result, std::move(R)); - }); } + if (best_idx == servers_.size()) { + return td::Status::Error(PSTRING() << "no liteserver for query " << query_info.to_str()); + } + return best_idx; + } - void got_result(td::Result R) { - if (R.is_error()) { - LOG(INFO) << "Query to shard=" << shard_.to_str() << ": " << R.error(); - promise_.set_value(create_serialize_tl_object( - R.error().code(), "gateway error: " + R.error().message().str())); - } else { - td::BufferSlice response = R.move_as_ok(); - LOG(INFO) << "Query to shard=" << shard_.to_str() << ": OK, size=" << response.size() - << " time=" << timer_.elapsed(); - promise_.set_value(std::move(response)); + void receive_query(td::BufferSlice data, td::Promise promise) { + liteclient::QueryInfo query_info = liteclient::get_query_info(data); + ++ls_stats_[query_info.query_id]; + promise = [promise = std::move(promise), query_info, timer = td::Timer()](td::Result R) mutable { + if (R.is_ok()) { + LOG(INFO) << "Query " << query_info.to_str() << ": OK, time=" << timer.elapsed() + << ", response_size=" << R.ok().size(); + promise.set_value(R.move_as_ok()); + return; } - stop(); - } + LOG(INFO) << "Query " << query_info.to_str() << ": " << R.error(); + promise.set_value(create_serialize_tl_object( + R.error().code(), "Gateway error: " + R.error().message().str())); + }; + TRY_RESULT_PROMISE(promise, server_idx, select_server(query_info)); - void fatal_error(td::Status S) { - promise_.set_error(std::move(S)); - stop(); - } + Server& server = servers_[server_idx]; + LOG(INFO) << "Sending query " << query_info.to_str() << ", size=" << data.size() << ", to server #" << server_idx + << " (" << server.config.addr.get_ip_str() << ":" << server.config.addr.get_port() << ")"; + td::actor::send_closure(server.client, &adnl::AdnlExtClient::send_query, "q", std::move(data), + td::Timestamp::in(8.0), std::move(promise)); + } - private: - td::actor::ActorId client_; - td::BufferSlice data_; - td::Promise promise_; - td::Timer timer_ = {}; - ShardIdFull shard_; - }; + void alarm() override { + alarm_timestamp() = td::Timestamp::in(60.0); + if (!ls_stats_.empty()) { + td::StringBuilder sb; + sb << "Liteserver stats (1 minute):"; + td::uint32 total = 0; + for (const auto& p : ls_stats_) { + sb << " " << lite_query_name_by_id(p.first) << ":" << p.second; + total += p.second; + } + sb << " TOTAL:" << total; + LOG(WARNING) << sb.as_cslice(); + ls_stats_.clear(); + } + } private: std::string global_config_; std::string db_root_; td::uint16 port_; + PublicKeyHash public_key_hash_; tl_object_ptr config_ = create_tl_object(); adnl::AdnlNodeIdFull id_; @@ -252,7 +302,15 @@ class ProxyLiteserver : public td::actor::Actor { td::actor::ActorOwn keyring_; td::actor::ActorOwn adnl_; td::actor::ActorOwn ext_server_; - td::actor::ActorOwn ext_client_; + + struct Server { + liteclient::LiteServerConfig config; + td::actor::ActorOwn client; + bool alive = false; + }; + std::vector servers_; + + std::map ls_stats_; // lite_api ID -> count, 0 for unknown std::string config_file() const { return db_root_ + "/config.json"; @@ -270,6 +328,7 @@ int main(int argc, char* argv[]) { std::string global_config, db_root; td::uint16 port = 0; + PublicKeyHash public_key_hash = PublicKeyHash::zero(); td::uint32 threads = 4; td::OptionParser p; @@ -290,15 +349,27 @@ int main(int argc, char* argv[]) { std::cout << sb.as_cslice().c_str(); std::exit(2); }); - p.add_checked_option('p', "port", "liteserver port (use only on first launch)", [&](td::Slice arg) -> td::Status { - TRY_RESULT_ASSIGN(port, td::to_integer_safe(arg)); - return td::Status::OK(); - }); + p.add_checked_option('p', "port", "liteserver port (required only on first launch)", + [&](td::Slice arg) -> td::Status { + TRY_RESULT_ASSIGN(port, td::to_integer_safe(arg)); + return td::Status::OK(); + }); + p.add_checked_option( + 'A', "adnl-id", + "liteserver public key hash in hex (optional). The corresponding private key is required in /keyring/", + [&](td::Slice arg) -> td::Status { + td::Bits256 value; + if (value.from_hex(arg) != 256) { + return td::Status::Error("invalid adnl-id"); + } + public_key_hash = PublicKeyHash{value}; + return td::Status::OK(); + }); p.add_option('C', "global-config", "global TON configuration file", [&](td::Slice arg) { global_config = arg.str(); }); p.add_option('D', "db", "db root", [&](td::Slice arg) { db_root = arg.str(); }); p.add_option('d', "daemonize", "set SIGHUP", [&]() { - td::set_signal_handler(td::SignalType::HangUp, [](int sig) { + td::set_signal_handler(td::SignalType::HangUp, [](int) { #if TD_DARWIN || TD_LINUX close(0); setsid(); @@ -318,8 +389,10 @@ int main(int argc, char* argv[]) { p.run(argc, argv).ensure(); td::actor::Scheduler scheduler({threads}); - scheduler.run_in_context( - [&] { td::actor::create_actor("proxy-liteserver", global_config, db_root, port).release(); }); + scheduler.run_in_context([&] { + td::actor::create_actor("proxy-liteserver", global_config, db_root, port, public_key_hash) + .release(); + }); while (scheduler.run(1)) { } } From 90d2edf5354c3f3664e10580e8815cf18565b859 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 25 Jun 2024 14:06:15 +0300 Subject: [PATCH 095/388] Improve CollatorNode * Keep track of validator groups * Pre-generate shard blocks --- tl/generate/scheme/ton_api.tl | 2 +- tl/generate/scheme/ton_api.tlo | Bin 98440 -> 98404 bytes validator-engine/validator-engine.cpp | 4 +- validator/collator-node.cpp | 362 ++++++++++++++++++-------- validator/collator-node.hpp | 50 ++-- validator/fabric.h | 3 +- validator/impl/collator-impl.h | 9 +- validator/impl/collator.cpp | 37 ++- validator/impl/fabric.cpp | 7 +- validator/manager.cpp | 88 +++++-- validator/manager.hpp | 4 + validator/validator-group.cpp | 2 +- 12 files changed, 402 insertions(+), 166 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 3a46406fa..9c94e3ee2 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -837,7 +837,7 @@ validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:lo self_idx:int nodes:(vector validatorSession.newValidatorGroupStats.node) = validatorSession.NewValidatorGroupStats; ---functions--- -collatorNode.generateBlock workchain:int shard:long min_mc_id:tonNode.blockIdExt prev_blocks:(vector tonNode.blockIdExt) +collatorNode.generateBlock shard:tonNode.shardId cc_seqno:int prev_blocks:(vector tonNode.blockIdExt) creator:int256 = collatorNode.GenerateBlockResult; ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 77bcff0c5f4e7a7914a3bc653e7cc4f5587d3720..e4ec14379ddd72c2f9543763412a6b3791c194b5 100644 GIT binary patch delta 38 ucmeBZWP8%U*06_7E#Z6E^@Q<_+}# delta 48 zcmaFTz}C^o*06 Config::config_add_control_process(ton::PublicKeyHash key, td:: } td::Result Config::config_add_shard(ton::ShardIdFull shard) { - if (!shard.is_valid_ext()) { - return td::Status::Error(PSTRING() << "invalid shard " << shard.to_str()); + if (!shard.is_valid_ext() || shard.is_masterchain()) { + return td::Status::Error(PSTRING() << "invalid shard to collate " << shard.to_str()); } if (std::find(shards_to_monitor.begin(), shards_to_monitor.end(), shard) != shards_to_monitor.end()) { return false; diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index ce3c4be4b..79455de44 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -21,6 +21,7 @@ #include "block-db.h" #include "td/utils/lz4.h" #include "checksum.h" +#include "impl/shard.hpp" #include "validator-session/candidate-serializer.h" namespace ton::validator { @@ -33,7 +34,7 @@ CollatorNode::CollatorNode(adnl::AdnlNodeIdShort local_id, td::actor::ActorId id) : id_(std::move(id)) { + explicit Cb(td::actor::ActorId id) : id_(std::move(id)) { } void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { } @@ -57,56 +58,197 @@ void CollatorNode::tear_down() { } void CollatorNode::add_shard(ShardIdFull shard) { - if (std::find(shards_.begin(), shards_.end(), shard) != shards_.end()) { + CHECK(shard.is_valid_ext() && !shard.is_masterchain()); + if (std::find(collating_shards_.begin(), collating_shards_.end(), shard) != collating_shards_.end()) { return; } LOG(INFO) << "Collator node: local_id=" << local_id_ << " , shard=" << shard.to_str(); - shards_.push_back(shard); + collating_shards_.push_back(shard); } void CollatorNode::del_shard(ShardIdFull shard) { - auto it = std::find(shards_.begin(), shards_.end(), shard); - if (it != shards_.end()) { - shards_.erase(it); + auto it = std::find(collating_shards_.begin(), collating_shards_.end(), shard); + if (it != collating_shards_.end()) { + collating_shards_.erase(it); } } void CollatorNode::new_masterchain_block_notification(td::Ref state) { - last_masterchain_block_ = state->get_block_id(); - last_top_blocks_.clear(); - last_top_blocks_[ShardIdFull{masterchainId, shardIdAll}] = last_masterchain_block_; - for (const auto& desc : state->get_shards()) { - last_top_blocks_[desc->shard()] = desc->top_block_id(); - } - if (validators_.empty() || state->is_key_state()) { - validators_.clear(); + last_masterchain_state_ = state; + if (validator_adnl_ids_.empty() || state->is_key_state()) { + validator_adnl_ids_.clear(); for (int next : {-1, 0, 1}) { td::Ref vals = state->get_total_validator_set(next); if (vals.not_null()) { for (const ValidatorDescr& descr : vals->export_vector()) { if (descr.addr.is_zero()) { - validators_.insert( + validator_adnl_ids_.insert( adnl::AdnlNodeIdShort(PublicKey(pubkeys::Ed25519{descr.key.as_bits256()}).compute_short_id())); } else { - validators_.insert(adnl::AdnlNodeIdShort(descr.addr)); + validator_adnl_ids_.insert(adnl::AdnlNodeIdShort(descr.addr)); } } } } } - // Remove old cache entries - auto it = cache_.begin(); - while (it != cache_.end()) { - auto prev_block_id = std::get<2>(it->first)[0]; - auto top_block = get_shard_top_block(prev_block_id.shard_full()); - if (top_block && top_block.value().seqno() > prev_block_id.seqno()) { - it = cache_.erase(it); + + std::map> new_shards; + for (auto& v : state->get_shards()) { + auto shard = v->shard(); + if (v->before_split()) { + CHECK(!v->before_merge()); + new_shards.emplace(shard_child(shard, true), std::vector{v->top_block_id()}); + new_shards.emplace(shard_child(shard, false), std::vector{v->top_block_id()}); + } else if (v->before_merge()) { + ShardIdFull p_shard = shard_parent(shard); + auto it = new_shards.find(p_shard); + if (it == new_shards.end()) { + new_shards.emplace(p_shard, std::vector(2)); + } + bool left = shard_child(p_shard.shard, true) == shard.shard; + new_shards[p_shard][left ? 0 : 1] = v->top_block_id(); + } else { + new_shards.emplace(shard, std::vector{v->top_block_id()}); + } + } + + for (auto& [shard, prev] : new_shards) { + CatchainSeqno cc_seqno = state->get_validator_set(shard)->get_catchain_seqno(); + auto it = validator_groups_.emplace(shard, ValidatorGroupInfo{}); + ValidatorGroupInfo& info = it.first->second; + if (it.second || info.cc_seqno != cc_seqno) { + info.cleanup(); + info.cc_seqno = cc_seqno; + } + } + for (auto it = validator_groups_.begin(); it != validator_groups_.end();) { + if (new_shards.count(it->first)) { + ++it; } else { + it->second.cleanup(); + it = validator_groups_.erase(it); + } + } + for (auto& [shard, prev] : new_shards) { + ValidatorGroupInfo& info = validator_groups_[shard]; + update_validator_group_info(shard, std::move(prev), info.cc_seqno); + auto it = future_validator_groups_.find({shard, info.cc_seqno}); + if (it != future_validator_groups_.end()) { + for (auto& new_prev : it->second.pending_blocks) { + update_validator_group_info(shard, std::move(new_prev), info.cc_seqno); + } + for (auto& promise : it->second.promises) { + promise.set_value(td::Unit()); + } + future_validator_groups_.erase(it); + } + } + + for (auto it = future_validator_groups_.begin(); it != future_validator_groups_.end(); ++it) { + if (get_future_validator_group(it->first.first, it->first.second).is_ok()) { ++it; + } else { + auto& future_group = it->second; + for (auto& promise : future_group.promises) { + promise.set_error(td::Status::Error("validator group is outdated")); + } + it = future_validator_groups_.erase(it); } } } +void CollatorNode::update_validator_group_info(ShardIdFull shard, std::vector prev, + CatchainSeqno cc_seqno) { + if (!can_collate_shard(shard)) { + return; + } + CHECK(prev.size() == 1 || prev.size() == 2); + BlockSeqno next_block_seqno = prev[0].seqno() + 1; + if (prev.size() == 2) { + next_block_seqno = std::max(next_block_seqno, prev[1].seqno() + 1); + } + auto it = validator_groups_.find(shard); + if (it != validator_groups_.end()) { + ValidatorGroupInfo& info = it->second; + if (info.cc_seqno == cc_seqno) { // block from currently known validator group + if (info.next_block_seqno < next_block_seqno) { + LOG(DEBUG) << "updated validator group info: shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno + << ", next_block_seqno=" << next_block_seqno; + info.next_block_seqno = next_block_seqno; + info.prev = std::move(prev); + for (auto cache_it = info.cache.begin(); cache_it != info.cache.end();) { + auto& [cached_prev, cache_entry] = *cache_it; + if (cache_entry->block_seqno < info.next_block_seqno) { + cache_entry->cancel(td::Status::Error(PSTRING() << "next block seqno " << cache_entry->block_seqno + << " is too small, expected " << info.next_block_seqno)); + cache_it = info.cache.erase(cache_it); + continue; + } + if (cache_entry->block_seqno == info.next_block_seqno && cached_prev != info.prev) { + cache_entry->cancel(td::Status::Error("invalid prev blocks")); + cache_it = info.cache.erase(cache_it); + continue; + } + ++cache_it; + } + if (do_pregen_) { + generate_block(shard, cc_seqno, info.prev, td::Timestamp::in(10.0), [](td::Result) {}); + } + } + return; + } + } + auto future_validator_group = get_future_validator_group(shard, cc_seqno); + if (future_validator_group.is_ok()) { + // future validator group, remember for later + future_validator_group.ok()->pending_blocks.push_back(std::move(prev)); + } +} + +td::Result CollatorNode::get_future_validator_group(ShardIdFull shard, + CatchainSeqno cc_seqno) { + auto it = validator_groups_.find(shard); + if (it == validator_groups_.end() && shard.pfx_len() != 0) { + it = validator_groups_.find(shard_parent(shard)); + } + if (it == validator_groups_.end() && shard.pfx_len() < max_shard_pfx_len) { + it = validator_groups_.find(shard_child(shard, true)); + } + if (it == validator_groups_.end() && shard.pfx_len() < max_shard_pfx_len) { + it = validator_groups_.find(shard_child(shard, false)); + } + if (it == validator_groups_.end()) { + return td::Status::Error("no such shard"); + } + if (cc_seqno < it->second.cc_seqno) { // past validator group + return td::Status::Error(PSTRING() << "cc_seqno " << cc_seqno << " is outdated (current is" << it->second.cc_seqno + << ")"); + } + if (cc_seqno - it->second.cc_seqno > 1) { // future validator group, cc_seqno too big + return td::Status::Error(PSTRING() << "cc_seqno " << cc_seqno << " is too big (currently known is" + << it->second.cc_seqno << ")"); + } + // future validator group + return &future_validator_groups_[{shard, cc_seqno}]; +} + +void CollatorNode::ValidatorGroupInfo::cleanup() { + prev.clear(); + next_block_seqno = 0; + for (auto& [_, cache_entry] : cache) { + cache_entry->cancel(td::Status::Error("validator group is outdated")); + } + cache.clear(); +} + +void CollatorNode::CacheEntry::cancel(td::Status reason) { + for (auto& promise : promises) { + promise.set_error(reason.clone()); + } + promises.clear(); + cancellation_token_source.cancel(); +} + static td::BufferSlice serialize_error(td::Status error) { return create_serialize_tl_object(error.code(), error.message().c_str()); } @@ -136,111 +278,121 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data td::Promise promise) { td::Promise new_promise = [promise = std::move(promise), src](td::Result R) mutable { if (R.is_error()) { - LOG(WARNING) << "Query from " << src << ", error: " << R.error(); - promise.set_result(serialize_error(R.move_as_error())); + LOG(INFO) << "adnl query from " << src << ", error: " << R.error(); + if (R.error().code() == ErrorCode::timeout) { + promise.set_error(R.move_as_error()); + } else { + promise.set_result(serialize_error(R.move_as_error())); + } } else { - LOG(INFO) << "Query from " << src << ", success"; + LOG(INFO) << "adnl query from " << src << ", success"; promise.set_result(create_serialize_tl_object( serialize_candidate(R.move_as_ok(), true))); } }; - if (!last_masterchain_block_.is_valid()) { - new_promise.set_error(td::Status::Error("not ready")); - return; - } - if (!validators_.count(src)) { + if (!validator_adnl_ids_.count(src)) { new_promise.set_error(td::Status::Error("src is not a validator")); } TRY_RESULT_PROMISE(new_promise, f, fetch_tl_object(std::move(data), true)); - ShardIdFull shard(f->workchain_, f->shard_); - BlockIdExt min_mc_id = create_block_id(f->min_mc_id_); - LOG(INFO) << "Got query from " << src << ": shard=" << shard.to_str() << ", min_mc_seqno=" << min_mc_id.seqno(); + ShardIdFull shard = create_shard_id(f->shard_); + CatchainSeqno cc_seqno = f->cc_seqno_; + std::vector prev_blocks; + for (const auto& b : f->prev_blocks_) { + prev_blocks.push_back(create_block_id(b)); + } + Ed25519_PublicKey creator(f->creator_); + new_promise = [new_promise = std::move(new_promise), creator](td::Result R) mutable { + TRY_RESULT_PROMISE(new_promise, block, std::move(R)); + new_promise.set_value(change_creator(std::move(block), creator)); + }; if (!shard.is_valid_ext()) { new_promise.set_error(td::Status::Error(PSTRING() << "invalid shard " << shard.to_str())); return; } - if (!can_collate_shard(shard)) { - new_promise.set_error(td::Status::Error(PSTRING() << "this node doesn't collate shard " << shard.to_str())); + if (prev_blocks.size() != 1 && prev_blocks.size() != 2) { + new_promise.set_error(td::Status::Error(PSTRING() << "invalid size of prev_blocks: " << prev_blocks.size())); return; } - if (f->prev_blocks_.size() != 1 && f->prev_blocks_.size() != 2) { - new_promise.set_error(td::Status::Error(PSTRING() << "invalid size of prev_blocks: " << f->prev_blocks_.size())); + LOG(INFO) << "got adnl query from " << src << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno; + generate_block(shard, cc_seqno, std::move(prev_blocks), td::Timestamp::in(10.0), std::move(new_promise)); +} + +void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std::vector prev_blocks, + td::Timestamp timeout, td::Promise promise) { + if (last_masterchain_state_.is_null()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "not ready")); return; } - if (!min_mc_id.is_masterchain_ext()) { - new_promise.set_error(td::Status::Error("min_mc_id is not form masterchain")); + if (!can_collate_shard(shard)) { + promise.set_error(td::Status::Error(PSTRING() << "this node can't collate shard " << shard.to_str())); return; } - std::vector prev_blocks; - for (const auto& b : f->prev_blocks_) { - auto id = create_block_id(b); - if (!id.is_valid_full()) { - new_promise.set_error(td::Status::Error("invalid prev_block")); - return; - } - auto top_block = get_shard_top_block(id.shard_full()); - if (top_block && top_block.value().seqno() > id.seqno()) { - new_promise.set_error(td::Status::Error("cannot collate block: already exists in blockchain")); - return; - } - prev_blocks.push_back(id); - } - Ed25519_PublicKey creator(f->creator_); - if (!shard.is_masterchain()) { - // Collation of masterchain cannot be cached because changing "created_by" in masterchain is hard - // It does not really matter because validators can collate masterchain themselves - new_promise = [promise = std::move(new_promise), creator](td::Result R) mutable { + auto it = validator_groups_.find(shard); + if (it == validator_groups_.end() || it->second.cc_seqno != cc_seqno) { + TRY_RESULT_PROMISE(promise, future_validator_group, get_future_validator_group(shard, cc_seqno)); + future_validator_group->promises.push_back([=, SelfId = actor_id(this), prev_blocks = std::move(prev_blocks), + promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); - } else { - promise.set_result(change_creator(R.move_as_ok(), creator)); + return; } - }; - auto cache_key = std::make_tuple(min_mc_id.seqno(), shard, prev_blocks); - auto cache_entry = cache_[cache_key]; - if (cache_entry == nullptr) { - cache_entry = cache_[cache_key] = std::make_shared(); - } - if (cache_entry->result) { - LOG(INFO) << "Using cached result"; - new_promise.set_result(cache_entry->result.value().clone()); - return; - } - cache_entry->promises.push_back(std::move(new_promise)); - if (cache_entry->started) { - LOG(INFO) << "Collating of this block is already in progress, waiting"; - return; - } - cache_entry->started = true; - new_promise = [SelfId = actor_id(this), cache_entry](td::Result R) mutable { - td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R)); - }; - } - - auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this), prev_blocks = std::move(prev_blocks), - promise = std::move(new_promise)](td::Result> R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error_prefix("failed to get masterchain state: ")); - } else { - td::Ref state(R.move_as_ok()); - if (state.is_null()) { - promise.set_error(R.move_as_error_prefix("failed to get masterchain state: ")); + if (timeout.is_in_past()) { + promise.set_error(td::Status::Error(ErrorCode::timeout)); return; } - td::actor::send_closure(SelfId, &CollatorNode::receive_query_cont, shard, std::move(state), - std::move(prev_blocks), creator, std::move(promise)); - } - }); - td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, min_mc_id, 1, td::Timestamp::in(5.0), - std::move(P)); -} + td::actor::send_closure(SelfId, &CollatorNode::generate_block, shard, cc_seqno, std::move(prev_blocks), timeout, + std::move(promise)); + }); + return; + } + ValidatorGroupInfo& validator_group_info = it->second; + BlockSeqno block_seqno = prev_blocks.at(0).seqno() + 1; + if (prev_blocks.size() == 2) { + block_seqno = std::max(block_seqno, prev_blocks.at(1).seqno() + 1); + } + if (validator_group_info.next_block_seqno > block_seqno) { + promise.set_error(td::Status::Error(PSTRING() << "next block seqno " << block_seqno << " is too small, expected " + << validator_group_info.next_block_seqno)); + return; + } + if (validator_group_info.next_block_seqno == block_seqno && validator_group_info.prev != prev_blocks) { + promise.set_error(td::Status::Error("invalid prev_blocks")); + return; + } -void CollatorNode::receive_query_cont(ShardIdFull shard, td::Ref min_mc_state, - std::vector prev_blocks, Ed25519_PublicKey creator, - td::Promise promise) { - run_collate_query(shard, min_mc_state->get_block_id(), std::move(prev_blocks), creator, - min_mc_state->get_validator_set(shard), manager_, td::Timestamp::in(10.0), std::move(promise), - CollateMode::skip_store_candidate); + auto cache_entry = validator_group_info.cache[prev_blocks]; + if (cache_entry == nullptr) { + cache_entry = validator_group_info.cache[prev_blocks] = std::make_shared(); + } + if (cache_entry->result) { + LOG(INFO) << "generate block query" + << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno + << ": using cached result"; + promise.set_result(cache_entry->result.value().clone()); + return; + } + cache_entry->promises.push_back(std::move(promise)); + if (cache_entry->started) { + LOG(INFO) << "generate block query" + << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno + << ": collation in progress, waiting"; + return; + } + LOG(INFO) << "generate block query" + << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno + << ": starting collation"; + cache_entry->started = true; + cache_entry->block_seqno = block_seqno; + run_collate_query( + shard, last_masterchain_state_->get_block_id(), std::move(prev_blocks), Ed25519_PublicKey{td::Bits256::zero()}, + last_masterchain_state_->get_validator_set(shard), manager_, timeout, + [=, SelfId = actor_id(this), timer = td::Timer{}](td::Result R) { + LOG(INFO) << "generate block result" + << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno + << ", time=" << timer.elapsed() << ": " << (R.is_ok() ? "OK" : R.error().to_string()); + td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R)); + }, + cache_entry->cancellation_token_source.get_cancellation_token(), CollateMode::skip_store_candidate); } void CollatorNode::process_result(std::shared_ptr cache_entry, td::Result R) { @@ -259,7 +411,7 @@ void CollatorNode::process_result(std::shared_ptr cache_entry, td::R } bool CollatorNode::can_collate_shard(ShardIdFull shard) const { - return std::any_of(shards_.begin(), shards_.end(), + return std::any_of(collating_shards_.begin(), collating_shards_.end(), [&](const ShardIdFull& our_shard) { return shard_intersects(shard, our_shard); }); } diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp index ca2fd73d4..fe7ff43ac 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node.hpp @@ -34,12 +34,10 @@ class CollatorNode : public td::actor::Actor { void del_shard(ShardIdFull shard); void new_masterchain_block_notification(td::Ref state); + void update_validator_group_info(ShardIdFull shard, std::vector prev, CatchainSeqno cc_seqno); private: void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); - void receive_query_cont(ShardIdFull shard, td::Ref min_mc_state, - std::vector prev_blocks, Ed25519_PublicKey creator, - td::Promise promise); bool can_collate_shard(ShardIdFull shard) const; @@ -47,33 +45,39 @@ class CollatorNode : public td::actor::Actor { td::actor::ActorId manager_; td::actor::ActorId adnl_; td::actor::ActorId rldp_; - std::vector shards_; - std::set validators_; - - BlockIdExt last_masterchain_block_{}; - std::map last_top_blocks_; + std::vector collating_shards_; + std::set validator_adnl_ids_; struct CacheEntry { bool started = false; + BlockSeqno block_seqno = 0; td::optional result; + td::CancellationTokenSource cancellation_token_source; std::vector> promises; + + void cancel(td::Status reason); + }; + struct ValidatorGroupInfo { + CatchainSeqno cc_seqno{0}; + std::vector prev; + BlockSeqno next_block_seqno{0}; + std::map, std::shared_ptr> cache; + + void cleanup(); }; - std::map>, std::shared_ptr> cache_; - - td::optional get_shard_top_block(ShardIdFull shard) const { - auto it = last_top_blocks_.lower_bound(shard); - if (it != last_top_blocks_.end() && shard_intersects(it->first, shard)) { - return it->second; - } - if (it != last_top_blocks_.begin()) { - --it; - if (shard_intersects(it->first, shard)) { - return it->second; - } - } - return {}; - } + struct FutureValidatorGroup { + std::vector> pending_blocks; + std::vector> promises; + }; + std::map validator_groups_; + std::map, FutureValidatorGroup> future_validator_groups_; + + td::Ref last_masterchain_state_; + + td::Result get_future_validator_group(ShardIdFull shard, CatchainSeqno cc_seqno); + void generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std::vector prev_blocks, + td::Timestamp timeout, td::Promise promise); void process_result(std::shared_ptr cache_entry, td::Result R); public: diff --git a/validator/fabric.h b/validator/fabric.h index a38b40568..1aa5a1670 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -88,7 +88,8 @@ void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, Ed25519_PublicKey creator, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, unsigned mode = 0); + td::Promise promise, td::CancellationToken cancellation_token = {}, + unsigned mode = 0); void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 1775d657c..b8003277b 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -87,9 +87,9 @@ class Collator final : public td::actor::Actor { static constexpr bool shard_splitting_enabled = true; public: - Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, - std::vector prev, Ref validator_set, Ed25519_PublicKey collator_id, - td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, + Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector prev, + Ref validator_set, Ed25519_PublicKey collator_id, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise promise, td::CancellationToken cancellation_token, unsigned mode); ~Collator() override = default; bool is_busy() const { @@ -343,6 +343,9 @@ class Collator final : public td::actor::Actor { void return_block_candidate(td::Result saved); bool update_last_proc_int_msg(const std::pair& new_lt_hash); + td::CancellationToken cancellation_token_; + bool check_cancelled(); + public: static td::uint32 get_skip_externals_queue_size(); }; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index fc031351c..2f8d7bbd7 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -77,7 +77,7 @@ static inline bool dbg(int c) { Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector prev, td::Ref validator_set, Ed25519_PublicKey collator_id, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, unsigned mode) + td::Promise promise, td::CancellationToken cancellation_token, unsigned mode) : shard_(shard) , is_hardfork_(is_hardfork) , min_mc_block_id{min_masterchain_block_id} @@ -92,9 +92,11 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_mastercha , medium_timeout_(td::Timestamp::at(timeout.at() - 1.5)) , main_promise(std::move(promise)) , mode_(mode) - , perf_timer_("collate", 0.1, [manager](double duration) { - send_closure(manager, &ValidatorManager::add_perf_timer_stat, "collate", duration); - }) { + , perf_timer_("collate", 0.1, + [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "collate", duration); + }) + , cancellation_token_(std::move(cancellation_token)) { } /** @@ -382,6 +384,9 @@ bool Collator::fatal_error(std::string err_msg, int err_code) { */ void Collator::check_pending() { // LOG(DEBUG) << "pending = " << pending; + if (!check_cancelled()) { + return; + } if (!pending) { step = 2; try { @@ -2423,6 +2428,9 @@ bool Collator::out_msg_queue_cleanup() { LOG(WARNING) << "cleaning up outbound queue takes too long, ending"; break; } + if (!check_cancelled()) { + return false; + } if (i == queue_parts.size()) { i = 0; } @@ -3516,6 +3524,9 @@ bool Collator::process_inbound_internal_messages() { LOG(WARNING) << "soft timeout reached, stop processing inbound internal messages"; break; } + if (!check_cancelled()) { + return false; + } LOG(DEBUG) << "processing inbound message with (lt,hash)=(" << kv->lt << "," << kv->key.to_hex() << ") from neighbor #" << kv->source; if (verbosity > 2) { @@ -3565,6 +3576,9 @@ bool Collator::process_inbound_external_messages() { LOG(WARNING) << "medium timeout reached, stop processing inbound external messages"; break; } + if (!check_cancelled()) { + return false; + } auto ext_msg = ext_msg_struct.cell; ton::Bits256 hash{ext_msg->get_hash().bits()}; int r = process_external_message(std::move(ext_msg)); @@ -3819,6 +3833,9 @@ bool Collator::process_new_messages(bool enqueue_only) { LOG(INFO) << "BLOCK FULL, enqueue all remaining new messages"; enqueue_only = true; } + if (!check_cancelled()) { + return false; + } LOG(DEBUG) << "have message with lt=" << msg.lt; int res = process_one_new_message(std::move(msg), enqueue_only); if (res < 0) { @@ -5360,6 +5377,18 @@ void Collator::after_get_external_messages(td::Result prev, Ed25519_PublicKey creator, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, unsigned mode) { + td::Promise promise, td::CancellationToken cancellation_token, unsigned mode) { BlockSeqno seqno = 0; for (auto& p : prev) { if (p.seqno() > seqno) { @@ -224,7 +224,8 @@ void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_bloc } td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, false, min_masterchain_block_id, std::move(prev), std::move(validator_set), creator, - std::move(manager), timeout, std::move(promise), mode) + std::move(manager), timeout, std::move(promise), std::move(cancellation_token), + mode) .release(); } @@ -240,7 +241,7 @@ void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_b td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, min_masterchain_block_id, std::move(prev), td::Ref{}, Ed25519_PublicKey{Bits256::zero()}, std::move(manager), timeout, std::move(promise), - 0) + td::CancellationToken{}, 0) .release(); } diff --git a/validator/manager.cpp b/validator/manager.cpp index bde7a8056..d3ce388e7 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -218,12 +218,28 @@ void ValidatorManagerImpl::prevalidate_block(BlockBroadcast broadcast, td::Promi promise.set_error(td::Status::Error("not monitoring shard")); return; } + promise = [SelfId = actor_id(this), promise = std::move(promise), block_id = broadcast.block_id, + cc_seqno = broadcast.catchain_seqno](td::Result R) mutable { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::validated_block_broadcast, block_id, cc_seqno); + } + promise.set_result(std::move(R)); + }; td::actor::create_actor("broadcast", std::move(broadcast), last_masterchain_block_handle_, last_masterchain_state_, last_known_key_block_handle_, actor_id(this), td::Timestamp::in(2.0), std::move(promise)) .release(); } +void ValidatorManagerImpl::validated_block_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno) { + for (auto &[_, collator_node] : collator_nodes_) { + if (collator_node.can_collate_shard(block_id.shard_full())) { + td::actor::send_closure(collator_node.actor, &CollatorNode::update_validator_group_info, block_id.shard_full(), + std::vector{block_id}, cc_seqno); + } + } +} + void ValidatorManagerImpl::sync_complete(td::Promise promise) { started_ = true; @@ -471,7 +487,7 @@ void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { } void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { - if (!is_validator()) { + if (!is_validator() && !is_shard_collator(block_id.shard_full())) { return; } if (!last_masterchain_block_handle_) { @@ -513,29 +529,36 @@ void ValidatorManagerImpl::new_block_candidate(BlockIdExt block_id, td::BufferSl } void ValidatorManagerImpl::add_shard_block_description(td::Ref desc) { - if (desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { - auto it = shard_blocks_.find(ShardTopBlockDescriptionId{desc->shard(), desc->catchain_seqno()}); - if (it != shard_blocks_.end() && desc->block_id().id.seqno <= it->second.latest_desc->block_id().id.seqno) { - VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block broadcast"; - return; - } - shard_blocks_[ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}].latest_desc = desc; - VLOG(VALIDATOR_DEBUG) << "new shard block descr for " << desc->block_id(); - if (need_monitor(desc->block_id().shard_full())) { - auto P = td::PromiseCreator::lambda([](td::Result> R) { - if (R.is_error()) { - auto S = R.move_as_error(); - if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) { - VLOG(VALIDATOR_NOTICE) << "failed to get shard state: " << S; - } else { - VLOG(VALIDATOR_DEBUG) << "failed to get shard state: " << S; - } + if (!desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { + return; + } + auto it = shard_blocks_.find(ShardTopBlockDescriptionId{desc->shard(), desc->catchain_seqno()}); + if (it != shard_blocks_.end() && desc->block_id().id.seqno <= it->second.latest_desc->block_id().id.seqno) { + VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block broadcast"; + return; + } + shard_blocks_[ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}].latest_desc = desc; + VLOG(VALIDATOR_DEBUG) << "new shard block descr for " << desc->block_id(); + if (need_monitor(desc->block_id().shard_full())) { + auto P = td::PromiseCreator::lambda([](td::Result> R) { + if (R.is_error()) { + auto S = R.move_as_error(); + if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) { + VLOG(VALIDATOR_NOTICE) << "failed to get shard state: " << S; + } else { + VLOG(VALIDATOR_DEBUG) << "failed to get shard state: " << S; } - }); - wait_block_state_short(desc->block_id(), 0, td::Timestamp::in(60.0), std::move(P)); - } - if (validating_masterchain()) { - preload_msg_queue_to_masterchain(desc); + } + }); + wait_block_state_short(desc->block_id(), 0, td::Timestamp::in(60.0), std::move(P)); + } + if (validating_masterchain()) { + preload_msg_queue_to_masterchain(desc); + } + for (auto& [_, collator_node] : collator_nodes_) { + if (collator_node.can_collate_shard(desc->shard())) { + td::actor::send_closure(collator_node.actor, &CollatorNode::update_validator_group_info, desc->shard(), + std::vector{desc->block_id()}, desc->catchain_seqno()); } } } @@ -2863,6 +2886,21 @@ PublicKeyHash ValidatorManagerImpl::get_validator(ShardIdFull shard, td::Ref r) { if (r.size() == 0) { delay_action([SelfId = actor_id( @@ -3364,6 +3402,10 @@ void ValidatorManagerImpl::get_validator_sessions_info( } void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) { + if (shard.is_masterchain() || !shard.is_valid_ext()) { + LOG(WARNING) << "cannot collate shard " << shard.to_str(); + return; + } auto it = collator_nodes_.find(id); if (it == collator_nodes_.end()) { it = collator_nodes_.emplace(id, Collator()).first; diff --git a/validator/manager.hpp b/validator/manager.hpp index 4e5d1d3e7..b85808d3c 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -372,6 +372,7 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise promise) override; void validate_block(ReceivedBlock block, td::Promise promise) override; void prevalidate_block(BlockBroadcast broadcast, td::Promise promise) override; + void validated_block_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno); //void create_validate_block(BlockId block, td::BufferSlice data, td::Promise promise) = 0; void sync_complete(td::Promise promise) override; @@ -562,6 +563,7 @@ class ValidatorManagerImpl : public ValidatorManager { bool is_validator(); bool validating_masterchain(); PublicKeyHash get_validator(ShardIdFull shard, td::Ref val_set); + bool is_shard_collator(ShardIdFull shard); ValidatorManagerImpl(td::Ref opts, std::string db_root, td::actor::ActorId keyring, td::actor::ActorId adnl, @@ -763,6 +765,8 @@ class ValidatorManagerImpl : public ValidatorManager { struct Collator { td::actor::ActorOwn actor; std::set shards; + + bool can_collate_shard(ShardIdFull shard) const; }; std::map collator_nodes_; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index e4ae88dd4..94490e5a2 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -546,7 +546,7 @@ void ValidatorGroup::collate_block(td::uint32 round_id, td::Timestamp timeout, t prev_blocks.push_back(create_tl_block_id(p)); } td::BufferSlice query = create_serialize_tl_object( - shard_.workchain, shard_.shard, create_tl_block_id(min_masterchain_block_id_), std::move(prev_blocks), + create_tl_shard_id(shard_), validator_set_->get_catchain_seqno(), std::move(prev_blocks), local_id_full_.ed25519_value().raw()); auto P = td::PromiseCreator::lambda( From 81b7993d661c0a208048e6e173b1b022ab19867e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 28 Jun 2024 11:57:14 +0300 Subject: [PATCH 096/388] Update query-utils.cpp --- lite-client/query-utils.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lite-client/query-utils.cpp b/lite-client/query-utils.cpp index eddaa7276..f48e3e8dd 100644 --- a/lite-client/query-utils.cpp +++ b/lite-client/query-utils.cpp @@ -211,7 +211,11 @@ QueryInfo get_query_info(const lite_api::Function& f) { [&](const lite_api::liteServer_getShardBlockProof& q) { from_block_id(q.id_); }, [&](const lite_api::liteServer_nonfinal_getCandidate& q) { /* t_simple */ }, [&](const lite_api::liteServer_nonfinal_getValidatorGroups& q) { /* t_simple */ }, - [&](const lite_api::liteServer_getOutMsgQueueSizes& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getOutMsgQueueSizes& q) { + // This query is expected to be removed, as it is not fully compatible with separated liteservers + /* t_simple */ + }, + [&](const lite_api::liteServer_getBlockOutMsgQueueSize& q) { from_block_id(q.id_); }, [&](const auto&) { /* t_simple */ })); if (info.shard_id.workchain == masterchainId) { info.shard_id.shard = shardIdAll; From 7f263b8b11eb71eb60df5f16900a1b7edbeb5329 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 28 Jun 2024 12:46:08 +0300 Subject: [PATCH 097/388] Don't request queue size if not needed in validate-query --- validator/impl/validate-query.cpp | 9 ++++++++- validator/impl/validate-query.hpp | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index e3a499a84..616fe31c7 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -2297,9 +2297,14 @@ bool ValidateQuery::prepare_out_msg_queue_size() { if (ps_.out_msg_queue_size_) { // if after_split then out_msg_queue_size is always present, since it is calculated during split old_out_msg_queue_size_ = ps_.out_msg_queue_size_.value(); + out_msg_queue_size_known_ = true; + return true; + } + if (!store_out_msg_queue_size_) { // Don't need it return true; } old_out_msg_queue_size_ = 0; + out_msg_queue_size_known_ = true; for (size_t i = 0; i < prev_blocks.size(); ++i) { ++pending; send_closure_later(manager, &ValidatorManager::get_out_msg_queue_size, prev_blocks[i], @@ -3363,7 +3368,9 @@ bool ValidateQuery::precheck_message_queue_update() { return reject_query("invalid OutMsgQueue dictionary difference between the old and the new state: "s + err.get_msg()); } - LOG(INFO) << "outbound message queue size: " << old_out_msg_queue_size_ << " -> " << new_out_msg_queue_size_; + if (out_msg_queue_size_known_) { + LOG(INFO) << "outbound message queue size: " << old_out_msg_queue_size_ << " -> " << new_out_msg_queue_size_; + } if (store_out_msg_queue_size_) { if (!ns_.out_msg_queue_size_) { return reject_query(PSTRING() << "outbound message queue size in the new state is not correct (expected: " diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 09ed7984e..c039fec44 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -236,6 +236,7 @@ class ValidateQuery : public td::actor::Actor { std::map, Ref> new_dispatch_queue_messages_; std::set account_expected_defer_all_messages_; td::uint64 old_out_msg_queue_size_ = 0, new_out_msg_queue_size_ = 0; + bool out_msg_queue_size_known_ = false; bool msg_metadata_enabled_ = false; bool deferring_messages_enabled_ = false; From f4b23a257b2ba3200389d45ec6ba55bdff4d990e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 28 Jun 2024 15:05:54 +0300 Subject: [PATCH 098/388] Fix parsing config in tonlib --- tonlib/test/offline.cpp | 3 ++- tonlib/tonlib/Config.cpp | 38 ++++++++++++-------------------------- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/tonlib/test/offline.cpp b/tonlib/test/offline.cpp index b7423853c..47d5d6a26 100644 --- a/tonlib/test/offline.cpp +++ b/tonlib/test/offline.cpp @@ -659,11 +659,12 @@ TEST(Tonlib, ConfigCache) { ], "validator": { "@type": "validator.config.global", - "zero_state": { + "init_block": { "workchain": -1, "shard": -9223372036854775808, "seqno": 0, "file_hash": "eh9yveSz1qMdJ7mOsO+I+H77jkLr9NpAuEkoJuseXBo=" + "root_hash": "ZXSXxDHhTALFxReyTZRd8E4Ya3ySOmpOWAS4rBX9XBY=", } } })abc"; diff --git a/tonlib/tonlib/Config.cpp b/tonlib/tonlib/Config.cpp index 0f317f94a..063b34bbd 100644 --- a/tonlib/tonlib/Config.cpp +++ b/tonlib/tonlib/Config.cpp @@ -20,6 +20,7 @@ #include "adnl/adnl-node-id.hpp" #include "td/utils/JsonBuilder.h" #include "auto/tl/ton_api_json.h" +#include "ton/ton-tl.hpp" namespace tonlib { td::Result parse_block_id_ext(td::JsonObject &obj) { @@ -64,41 +65,26 @@ td::Result parse_block_id_ext(td::JsonObject &obj) { td::Result Config::parse(std::string str) { TRY_RESULT(json, td::json_decode(str)); if (json.type() != td::JsonValue::Type::Object) { - return td::Status::Error("Invalid config (1)"); + return td::Status::Error("Invalid config: json is not an object"); } - Config res; ton::ton_api::liteclient_config_global conf; TRY_STATUS(ton::ton_api::from_json(conf, json.get_object())); TRY_RESULT_ASSIGN(res.lite_servers, liteclient::LiteServerConfig::parse_global_config(conf)); - TRY_RESULT(validator_obj, - td::get_json_object_field(json.get_object(), "validator", td::JsonValue::Type::Object, false)); - auto &validator = validator_obj.get_object(); - TRY_RESULT(validator_type, td::get_json_object_string_field(validator, "@type", false)); - if (validator_type != "validator.config.global") { - return td::Status::Error("Invalid config (7)"); + if (!conf.validator_) { + return td::Status::Error("Invalid config: no 'validator' section"); + } + if (!conf.validator_->zero_state_) { + return td::Status::Error("Invalid config: no zerostate"); } - TRY_RESULT(zero_state_obj, td::get_json_object_field(validator, "zero_state", td::JsonValue::Type::Object, false)); - TRY_RESULT(zero_state_id, parse_block_id_ext(zero_state_obj.get_object())); - res.zero_state_id = zero_state_id; - auto r_init_block_obj = td::get_json_object_field(validator, "init_block", td::JsonValue::Type::Object, false); - if (r_init_block_obj.is_ok()) { - TRY_RESULT(init_block_id, parse_block_id_ext(r_init_block_obj.move_as_ok().get_object())); - res.init_block_id = init_block_id; + res.zero_state_id = ton::create_block_id(conf.validator_->zero_state_); + if (conf.validator_->init_block_) { + res.init_block_id = ton::create_block_id(conf.validator_->init_block_); } - auto r_hardforks = td::get_json_object_field(validator, "hardforks", td::JsonValue::Type::Array, false); - if (r_hardforks.is_ok()) { - auto hardforks_obj = r_hardforks.move_as_ok(); - auto &hardforks = hardforks_obj.get_array(); - for (auto &fork : hardforks) { - if (fork.type() != td::JsonValue::Type::Object) { - return td::Status::Error("Invalid config (8)"); - } - TRY_RESULT(fork_block, parse_block_id_ext(fork.get_object())); - res.hardforks.push_back(std::move(fork_block)); - } + for (auto &fork : conf.validator_->hardforks_) { + res.hardforks.push_back(ton::create_block_id(fork)); } for (auto hardfork : res.hardforks) { From 3a8ef60bbb4c4dbf725eee9a2a8d3bba286a08ca Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Sat, 13 Jul 2024 15:40:26 +0300 Subject: [PATCH 099/388] Fast-sync overlays * Semiprivate overlays * Fast sync overlays in full-node * Adjust shard overlays --- CMakeLists.txt | 2 + create-hardfork/create-hardfork.cpp | 4 - overlay/overlay-broadcast.cpp | 5 +- overlay/overlay-fec-broadcast.cpp | 2 +- overlay/overlay-id.hpp | 100 ++- overlay/overlay-manager.cpp | 277 ++++++-- overlay/overlay-manager.h | 25 +- overlay/overlay-peers.cpp | 590 +++++++++++++++--- overlay/overlay.cpp | 315 ++++++---- overlay/overlay.h | 40 +- overlay/overlay.hpp | 237 ++++--- overlay/overlays.h | 127 +++- test/test-overlay.cpp | 450 +++++++++++++ test/test-ton-collator.cpp | 4 - tl/generate/scheme/ton_api.tl | 23 +- tl/generate/scheme/ton_api.tlo | Bin 99024 -> 101484 bytes .../validator-engine-console-query.cpp | 48 ++ .../validator-engine-console-query.h | 52 ++ .../validator-engine-console.cpp | 2 + validator-engine/validator-engine.cpp | 110 +++- validator-engine/validator-engine.hpp | 6 + validator/CMakeLists.txt | 4 +- validator/full-node-fast-sync-overlays.cpp | 416 ++++++++++++ ...2.hpp => full-node-fast-sync-overlays.hpp} | 70 +-- validator/full-node-private-overlay-v2.cpp | 384 ------------ validator/full-node-shard.cpp | 6 +- validator/full-node.cpp | 60 +- validator/full-node.h | 3 + validator/full-node.hpp | 23 +- validator/interfaces/validator-manager.h | 3 +- validator/manager-disk.hpp | 2 - validator/manager-hardfork.hpp | 3 - validator/manager-init.cpp | 1 + validator/manager.cpp | 36 +- validator/manager.hpp | 3 +- validator/shard-client.cpp | 64 +- validator/shard-client.hpp | 4 - validator/validator.h | 6 +- 38 files changed, 2544 insertions(+), 963 deletions(-) create mode 100644 test/test-overlay.cpp create mode 100644 validator/full-node-fast-sync-overlays.cpp rename validator/{full-node-private-overlay-v2.hpp => full-node-fast-sync-overlays.hpp} (59%) delete mode 100644 validator/full-node-private-overlay-v2.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b92ff6f1b..70cb8b8d8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -529,6 +529,8 @@ target_link_libraries(test-rldp2 adnl adnltest dht rldp2 tl_api) add_executable(test-validator-session-state test/test-validator-session-state.cpp) target_link_libraries(test-validator-session-state adnl dht rldp validatorsession tl_api) +add_executable(test-overlay test/test-overlay.cpp) +target_link_libraries(test-overlay overlay tdutils tdactor adnl adnltest tl_api dht ) add_executable(test-catchain test/test-catchain.cpp) target_link_libraries(test-catchain overlay tdutils tdactor adnl adnltest rldp tl_api dht catchain ) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index bb590960c..32aa01c32 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -236,10 +236,6 @@ class HardforkCreator : public td::actor::Actor { td::actor::send_closure(id_, &ton::validator::ValidatorManager::sync_complete, td::PromiseCreator::lambda([](td::Unit) {})); } - void update_shard_configuration(td::Ref state, - std::set shards_to_monitor, - std::set temporary_shards) override { - } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } void send_ext_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { diff --git a/overlay/overlay-broadcast.cpp b/overlay/overlay-broadcast.cpp index 03991b76b..615b3e7c8 100644 --- a/overlay/overlay-broadcast.cpp +++ b/overlay/overlay-broadcast.cpp @@ -68,7 +68,7 @@ td::Status BroadcastSimple::run_checks() { td::Status BroadcastSimple::distribute() { auto B = serialize(); - auto nodes = overlay_->get_neighbours(3); + auto nodes = overlay_->get_neighbours(overlay_->propagate_broadcast_to()); auto manager = overlay_->overlay_manager(); for (auto &n : nodes) { @@ -115,7 +115,8 @@ td::Status BroadcastSimple::run() { return run_continue(); } -td::Status BroadcastSimple::create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, tl_object_ptr broadcast) { +td::Status BroadcastSimple::create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, + tl_object_ptr broadcast) { auto src = PublicKey{broadcast->src_}; auto data_hash = sha256_bits256(broadcast->data_.as_slice()); auto broadcast_hash = compute_broadcast_id(src, data_hash, broadcast->flags_); diff --git a/overlay/overlay-fec-broadcast.cpp b/overlay/overlay-fec-broadcast.cpp index cd030742a..5a0ad10dd 100644 --- a/overlay/overlay-fec-broadcast.cpp +++ b/overlay/overlay-fec-broadcast.cpp @@ -112,7 +112,7 @@ td::Status BroadcastFec::distribute_part(td::uint32 seqno) { td::BufferSlice data_short = std::move(tls.first); td::BufferSlice data = std::move(tls.second); - auto nodes = overlay_->get_neighbours(5); + auto nodes = overlay_->get_neighbours(overlay_->propagate_broadcast_to()); auto manager = overlay_->overlay_manager(); for (auto &n : nodes) { diff --git a/overlay/overlay-id.hpp b/overlay/overlay-id.hpp index c22b0faa7..2625773bd 100644 --- a/overlay/overlay-id.hpp +++ b/overlay/overlay-id.hpp @@ -21,8 +21,14 @@ #include "auto/tl/ton_api.h" #include "adnl/adnl-node-id.hpp" #include "overlay/overlays.h" +#include "td/utils/SharedSlice.h" +#include "td/utils/buffer.h" #include "td/utils/overloaded.h" #include "keys/encryptor.h" +#include "td/utils/port/StdStreams.h" +#include "td/utils/unique_ptr.h" +#include +#include namespace ton { @@ -30,18 +36,30 @@ namespace overlay { class OverlayNode { public: - explicit OverlayNode(adnl::AdnlNodeIdShort self_id, OverlayIdShort overlay) { + explicit OverlayNode(adnl::AdnlNodeIdShort self_id, OverlayIdShort overlay, td::uint32 flags) { source_ = self_id; overlay_ = overlay; + flags_ = flags; version_ = static_cast(td::Clocks::system()); } static td::Result create(const tl_object_ptr &node) { TRY_RESULT(source, adnl::AdnlNodeIdFull::create(node->id_)); - return OverlayNode{source, OverlayIdShort{node->overlay_}, node->version_, node->signature_.as_slice()}; + return OverlayNode{source, OverlayIdShort{node->overlay_}, 0, node->version_, node->signature_.as_slice()}; } - OverlayNode(td::Variant source, OverlayIdShort overlay, + static td::Result create(const tl_object_ptr &node) { + TRY_RESULT(source, adnl::AdnlNodeIdFull::create(node->id_)); + auto res = OverlayNode{source, OverlayIdShort{node->overlay_}, (td::uint32)node->flags_, node->version_, + node->signature_.as_slice()}; + res.update_certificate(OverlayMemberCertificate(node->certificate_.get())); + return res; + } + OverlayNode(td::Variant source, OverlayIdShort overlay, td::uint32 flags, td::int32 version, td::Slice signature) - : source_(std::move(source)), overlay_(overlay), version_(version), signature_(td::SharedSlice(signature)) { + : source_(std::move(source)) + , overlay_(overlay) + , flags_(flags) + , version_(version) + , signature_(td::SharedSlice(signature)) { } OverlayNode(td::Variant source, OverlayIdShort overlay, td::int32 version, td::SharedSlice signature) @@ -64,10 +82,17 @@ class OverlayNode { } td::BufferSlice to_sign() const { - auto obj = create_tl_object(nullptr, overlay_.tl(), version_); - source_.visit(td::overloaded([&](const adnl::AdnlNodeIdShort &id) { obj->id_ = id.tl(); }, - [&](const adnl::AdnlNodeIdFull &id) { obj->id_ = id.compute_short_id().tl(); })); - return serialize_tl_object(obj, true); + if (flags_ == 0) { + auto obj = create_tl_object(nullptr, overlay_.tl(), version_); + source_.visit(td::overloaded([&](const adnl::AdnlNodeIdShort &id) { obj->id_ = id.tl(); }, + [&](const adnl::AdnlNodeIdFull &id) { obj->id_ = id.compute_short_id().tl(); })); + return serialize_tl_object(obj, true); + } else { + auto obj = create_tl_object(nullptr, overlay_.tl(), flags_, version_); + source_.visit(td::overloaded([&](const adnl::AdnlNodeIdShort &id) { obj->id_ = id.tl(); }, + [&](const adnl::AdnlNodeIdFull &id) { obj->id_ = id.compute_short_id().tl(); })); + return serialize_tl_object(obj, true); + } } void update_adnl_id(adnl::AdnlNodeIdFull node_id) { source_ = node_id; @@ -81,6 +106,9 @@ class OverlayNode { td::int32 version() const { return version_; } + td::uint32 flags() const { + return flags_; + } td::BufferSlice signature() const { return signature_.clone_as_buffer_slice(); } @@ -103,15 +131,69 @@ class OverlayNode { [&](const adnl::AdnlNodeIdFull &id) { obj->id_ = id.tl(); })); return obj; } + tl_object_ptr tl_v2() const { + tl_object_ptr cert; + if (cert_ && !cert_->empty()) { + cert = cert_->tl(); + } else { + cert = create_tl_object(); + } + auto obj = create_tl_object(nullptr, overlay_.tl(), flags_, version_, + signature_.clone_as_buffer_slice(), std::move(cert)); + source_.visit(td::overloaded([&](const adnl::AdnlNodeIdShort &id) { UNREACHABLE(); }, + [&](const adnl::AdnlNodeIdFull &id) { obj->id_ = id.tl(); })); + return obj; + } OverlayNode clone() const { - return OverlayNode{source_, overlay_, version_, signature_.clone()}; + auto res = OverlayNode{source_, overlay_, version_, signature_.clone()}; + if (cert_) { + res.cert_ = td::make_unique(*cert_); + } + return res; + } + + const OverlayMemberCertificate *certificate() const { + if (cert_) { + return cert_.get(); + } + return &empty_certificate_; + } + + void update_certificate(OverlayMemberCertificate cert) { + if (!cert_ || cert_->empty() || cert_->is_expired() || cert.is_newer(*cert_)) { + cert_ = td::make_unique(std::move(cert)); + } + } + + void update(OverlayNode from) { + if (version_ < from.version_) { + source_ = from.source_; + overlay_ = from.overlay_; + flags_ = from.flags_; + version_ = from.version_; + signature_ = from.signature_.clone(); + } + if (from.cert_ && !from.cert_->empty()) { + update_certificate(std::move(*from.cert_)); + } + } + + void clear_certificate() { + cert_ = nullptr; + } + + bool has_full_id() const { + return source_.get_offset() == source_.offset(); } private: td::Variant source_; OverlayIdShort overlay_; + td::uint32 flags_; td::int32 version_; + td::unique_ptr cert_; td::SharedSlice signature_; + static const OverlayMemberCertificate empty_certificate_; }; } // namespace overlay diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index e10557615..878f6637a 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -18,6 +18,7 @@ */ #include "overlay-manager.h" #include "auto/tl/ton_api.h" +#include "auto/tl/ton_api.hpp" #include "overlay.h" #include "adnl/utils.hpp" @@ -28,6 +29,7 @@ #include "td/db/RocksDb.h" #include "td/utils/Status.h" +#include "td/utils/buffer.h" #include "td/utils/overloaded.h" #include "td/utils/port/Poll.h" @@ -41,13 +43,13 @@ void OverlayManager::update_dht_node(td::actor::ActorId dht) { dht_node_ = dht; for (auto &X : overlays_) { for (auto &Y : X.second) { - td::actor::send_closure(Y.second, &Overlay::update_dht_node, dht); + td::actor::send_closure(Y.second.overlay, &Overlay::update_dht_node, dht); } } } void OverlayManager::register_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, - td::actor::ActorOwn overlay) { + OverlayMemberCertificate cert, td::actor::ActorOwn overlay) { auto it = overlays_.find(local_id); VLOG(OVERLAY_INFO) << this << ": registering overlay " << overlay_id << "@" << local_id; if (it == overlays_.end()) { @@ -57,19 +59,34 @@ void OverlayManager::register_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdS td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id, adnl::Adnl::int_to_bytestring(ton_api::overlay_query::ID), std::make_unique(actor_id(this))); + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id, + adnl::Adnl::int_to_bytestring(ton_api::overlay_messageWithExtra::ID), + std::make_unique(actor_id(this))); + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id, + adnl::Adnl::int_to_bytestring(ton_api::overlay_queryWithExtra::ID), + std::make_unique(actor_id(this))); } - overlays_[local_id][overlay_id] = std::move(overlay); - - auto P = td::PromiseCreator::lambda([id = overlays_[local_id][overlay_id].get()](td::Result R) { - R.ensure(); - auto value = R.move_as_ok(); - if (value.status == td::KeyValue::GetStatus::Ok) { - auto F = fetch_tl_object(std::move(value.value), true); - F.ensure(); - auto nodes = std::move(F.move_as_ok()->nodes_); - td::actor::send_closure(id, &Overlay::receive_nodes_from_db, std::move(nodes)); - } - }); + overlays_[local_id][overlay_id] = OverlayDescription{std::move(overlay), std::move(cert)}; + + auto P = + td::PromiseCreator::lambda([id = overlays_[local_id][overlay_id].overlay.get()](td::Result R) { + R.ensure(); + auto value = R.move_as_ok(); + if (value.status == td::KeyValue::GetStatus::Ok) { + auto F = fetch_tl_object(std::move(value.value), true); + F.ensure(); + ton_api::downcast_call( + *F.move_as_ok(), td::overloaded( + [&](ton_api::overlay_db_nodes &V) { + auto nodes = std::move(V.nodes_); + td::actor::send_closure(id, &Overlay::receive_nodes_from_db, std::move(nodes)); + }, + [&](ton_api::overlay_db_nodesV2 &V) { + auto nodes = std::move(V.nodes_); + td::actor::send_closure(id, &Overlay::receive_nodes_from_db_v2, std::move(nodes)); + })); + } + }); auto key = create_hash_tl_object(local_id.bits256_value(), overlay_id.bits256_value()); db_.get(key, std::move(P)); } @@ -83,6 +100,10 @@ void OverlayManager::delete_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdSho adnl::Adnl::int_to_bytestring(ton_api::overlay_message::ID)); td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id, adnl::Adnl::int_to_bytestring(ton_api::overlay_query::ID)); + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id, + adnl::Adnl::int_to_bytestring(ton_api::overlay_messageWithExtra::ID)); + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id, + adnl::Adnl::int_to_bytestring(ton_api::overlay_queryWithExtra::ID)); overlays_.erase(it); } } @@ -100,74 +121,113 @@ void OverlayManager::create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, Ov td::string scope, OverlayOptions opts) { CHECK(!dht_node_.empty()); auto id = overlay_id.compute_short_id(); - register_overlay(local_id, id, - Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), - std::move(callback), std::move(rules), scope, std::move(opts))); + register_overlay(local_id, id, OverlayMemberCertificate{}, + Overlay::create_public(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), + std::move(callback), std::move(rules), scope, std::move(opts))); } void OverlayManager::create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules, std::string scope) { + create_private_overlay_ex(local_id, std::move(overlay_id), std::move(nodes), std::move(callback), std::move(rules), + std::move(scope), {}); +} + +void OverlayManager::create_private_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::vector nodes, + std::unique_ptr callback, OverlayPrivacyRules rules, + std::string scope, OverlayOptions opts) { auto id = overlay_id.compute_short_id(); - register_overlay(local_id, id, - Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), - std::move(nodes), std::move(callback), std::move(rules), std::move(scope))); + register_overlay(local_id, id, OverlayMemberCertificate{}, + Overlay::create_private(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), + std::move(nodes), std::move(callback), std::move(rules), std::move(scope), + std::move(opts))); +} + +void OverlayManager::create_semiprivate_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::vector nodes, + std::vector root_public_keys, + OverlayMemberCertificate certificate, + std::unique_ptr callback, OverlayPrivacyRules rules, + td::string scope, OverlayOptions opts) { + auto id = overlay_id.compute_short_id(); + register_overlay( + local_id, id, certificate, + Overlay::create_semiprivate(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), + std::move(nodes), std::move(root_public_keys), certificate, std::move(callback), + std::move(rules), std::move(scope), std::move(opts))); } void OverlayManager::receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) { - auto R = fetch_tl_prefix(data, true); - - if (R.is_error()) { - VLOG(OVERLAY_WARNING) << this << ": can not parse overlay message: " << R.move_as_error(); - return; + OverlayIdShort overlay_id; + tl_object_ptr extra; + auto R = fetch_tl_prefix(data, true); + if (R.is_ok()) { + overlay_id = OverlayIdShort{R.ok()->overlay_}; + extra = std::move(R.ok()->extra_); + } else { + auto R2 = fetch_tl_prefix(data, true); + if (R2.is_ok()) { + overlay_id = OverlayIdShort{R2.ok()->overlay_}; + } else { + VLOG(OVERLAY_WARNING) << this << ": can not parse overlay message [" << src << "->" << dst + << "]: " << R2.move_as_error(); + return; + } } - auto M = R.move_as_ok(); - auto it = overlays_.find(dst); if (it == overlays_.end()) { - VLOG(OVERLAY_NOTICE) << this << ": message to unknown overlay " << M->overlay_ << "@" << dst; + VLOG(OVERLAY_NOTICE) << this << ": message to unknown overlay " << overlay_id << "@" << dst; return; } - auto it2 = it->second.find(OverlayIdShort{M->overlay_}); + auto it2 = it->second.find(overlay_id); if (it2 == it->second.end()) { - VLOG(OVERLAY_NOTICE) << this << ": message to localid is not in overlay " << M->overlay_ << "@" << dst; + VLOG(OVERLAY_NOTICE) << this << ": message to localid is not in overlay " << overlay_id << "@" << dst; return; } - td::actor::send_closure(it2->second, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), false); - td::actor::send_closure(it2->second, &Overlay::receive_message, src, std::move(data)); + td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), false); + td::actor::send_closure(it2->second.overlay, &Overlay::receive_message, src, std::move(extra), std::move(data)); } void OverlayManager::receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { - auto R = fetch_tl_prefix(data, true); - - if (R.is_error()) { - VLOG(OVERLAY_WARNING) << this << ": can not parse overlay query [" << src << "->" << dst - << "]: " << R.move_as_error(); - promise.set_error(td::Status::Error(ErrorCode::protoviolation, "bad overlay query header")); - return; + OverlayIdShort overlay_id; + tl_object_ptr extra; + auto R = fetch_tl_prefix(data, true); + if (R.is_ok()) { + overlay_id = OverlayIdShort{R.ok()->overlay_}; + extra = std::move(R.ok()->extra_); + } else { + auto R2 = fetch_tl_prefix(data, true); + if (R2.is_ok()) { + overlay_id = OverlayIdShort{R2.ok()->overlay_}; + } else { + VLOG(OVERLAY_WARNING) << this << ": can not parse overlay query [" << src << "->" << dst + << "]: " << R2.move_as_error(); + promise.set_error(td::Status::Error(ErrorCode::protoviolation, "bad overlay query header")); + return; + } } - auto M = R.move_as_ok(); - auto it = overlays_.find(dst); if (it == overlays_.end()) { - VLOG(OVERLAY_NOTICE) << this << ": query to unknown overlay " << M->overlay_ << "@" << dst << " from " << src; + VLOG(OVERLAY_NOTICE) << this << ": query to unknown overlay " << overlay_id << "@" << dst << " from " << src; promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad local_id " << dst)); return; } - auto it2 = it->second.find(OverlayIdShort{M->overlay_}); + auto it2 = it->second.find(overlay_id); if (it2 == it->second.end()) { - VLOG(OVERLAY_NOTICE) << this << ": query to localid not in overlay " << M->overlay_ << "@" << dst << " from " << src; - promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad overlay_id " << M->overlay_)); + VLOG(OVERLAY_NOTICE) << this << ": query to localid not in overlay " << overlay_id << "@" << dst << " from " << src; + promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad overlay_id " << overlay_id)); return; } - td::actor::send_closure(it2->second, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), true); - td::actor::send_closure(it2->second, &Overlay::receive_query, src, std::move(data), std::move(promise)); + td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), true); + td::actor::send_closure(it2->second.overlay, &Overlay::receive_query, src, std::move(extra), std::move(data), + std::move(promise)); } void OverlayManager::send_query_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id, @@ -175,35 +235,64 @@ void OverlayManager::send_query_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdS td::BufferSlice query, td::uint64 max_answer_size, td::actor::ActorId via) { CHECK(query.size() <= adnl::Adnl::huge_packet_max_size()); - + + auto extra = create_tl_object(); + extra->flags_ = 0; + auto it = overlays_.find(src); if (it != overlays_.end()) { auto it2 = it->second.find(overlay_id); if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::update_throughput_out_ctr, dst, (td::uint32)query.size(), true); + td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_out_ctr, dst, (td::uint32)query.size(), + true); + if (!it2->second.member_certificate.empty()) { + extra->flags_ |= 1; + extra->certificate_ = it2->second.member_certificate.tl(); + } } } - - td::actor::send_closure( - via, &adnl::AdnlSenderInterface::send_query_ex, src, dst, std::move(name), std::move(promise), timeout, - create_serialize_tl_object_suffix(query.as_slice(), overlay_id.tl()), max_answer_size); + + auto extra_flags = extra->flags_; + td::BufferSlice serialized_query = + (extra_flags ? create_serialize_tl_object_suffix( + query.as_slice(), overlay_id.tl(), std::move(extra)) + : create_serialize_tl_object_suffix(query.as_slice(), overlay_id.tl())); + + td::actor::send_closure(via, &adnl::AdnlSenderInterface::send_query_ex, src, dst, std::move(name), std::move(promise), + timeout, std::move(serialized_query), max_answer_size); } void OverlayManager::send_message_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id, td::BufferSlice object, td::actor::ActorId via) { CHECK(object.size() <= adnl::Adnl::huge_packet_max_size()); - + + auto extra = create_tl_object(); + extra->flags_ = 0; + auto it = overlays_.find(src); if (it != overlays_.end()) { auto it2 = it->second.find(overlay_id); if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::update_throughput_out_ctr, dst, (td::uint32)object.size(), false); + td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_out_ctr, dst, (td::uint32)object.size(), + false); + if (!it2->second.member_certificate.empty()) { + // do not send certificate here, we hope that all our neighbours already know of out certificate + // we send it every second to some random nodes. Here we don't want to increase the size of the message + if (false) { + extra->flags_ |= 1; + extra->certificate_ = it2->second.member_certificate.tl(); + } + } } } - - td::actor::send_closure( - via, &adnl::AdnlSenderInterface::send_message, src, dst, - create_serialize_tl_object_suffix(object.as_slice(), overlay_id.tl())); + + auto extra_flags = extra->flags_; + td::BufferSlice serialized_message = + (extra_flags ? create_serialize_tl_object_suffix( + object.as_slice(), overlay_id.tl(), std::move(extra)) + : create_serialize_tl_object_suffix(object.as_slice(), overlay_id.tl())); + + td::actor::send_closure(via, &adnl::AdnlSenderInterface::send_message, src, dst, std::move(serialized_message)); } void OverlayManager::send_broadcast(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, td::BufferSlice object) { @@ -217,7 +306,7 @@ void OverlayManager::send_broadcast_ex(adnl::AdnlNodeIdShort local_id, OverlayId if (it != overlays_.end()) { auto it2 = it->second.find(overlay_id); if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::send_broadcast, send_as, flags, std::move(object)); + td::actor::send_closure(it2->second.overlay, &Overlay::send_broadcast, send_as, flags, std::move(object)); } } } @@ -234,7 +323,7 @@ void OverlayManager::send_broadcast_fec_ex(adnl::AdnlNodeIdShort local_id, Overl if (it != overlays_.end()) { auto it2 = it->second.find(overlay_id); if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::send_broadcast_fec, send_as, flags, std::move(object)); + td::actor::send_closure(it2->second.overlay, &Overlay::send_broadcast_fec, send_as, flags, std::move(object)); } } } @@ -245,7 +334,7 @@ void OverlayManager::set_privacy_rules(adnl::AdnlNodeIdShort local_id, OverlayId if (it != overlays_.end()) { auto it2 = it->second.find(overlay_id); if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::set_privacy_rules, std::move(rules)); + td::actor::send_closure(it2->second.overlay, &Overlay::set_privacy_rules, std::move(rules)); } } } @@ -256,7 +345,34 @@ void OverlayManager::update_certificate(adnl::AdnlNodeIdShort local_id, OverlayI if (it != overlays_.end()) { auto it2 = it->second.find(overlay_id); if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::add_certificate, key, std::move(cert)); + td::actor::send_closure(it2->second.overlay, &Overlay::add_certificate, key, std::move(cert)); + } + } +} + +void OverlayManager::update_member_certificate(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + OverlayMemberCertificate certificate) { + auto it = overlays_.find(local_id); + if (it != overlays_.end()) { + auto it2 = it->second.find(overlay_id); + if (it2 != it->second.end()) { + it2->second.member_certificate = certificate; + td::actor::send_closure(it2->second.overlay, &Overlay::update_member_certificate, certificate); + } + } +} + +void OverlayManager::update_root_member_list(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + std::vector nodes, + std::vector root_public_keys, + OverlayMemberCertificate certificate) { + auto it = overlays_.find(local_id); + if (it != overlays_.end()) { + auto it2 = it->second.find(overlay_id); + if (it2 != it->second.end()) { + it2->second.member_certificate = certificate; + td::actor::send_closure(it2->second.overlay, &Overlay::update_root_member_list, std::move(nodes), + std::move(root_public_keys), std::move(certificate)); } } } @@ -274,7 +390,7 @@ void OverlayManager::get_overlay_random_peers(adnl::AdnlNodeIdShort local_id, Ov promise.set_error(td::Status::Error(PSTRING() << "no such overlay " << overlay_id)); return; } - td::actor::send_closure(it2->second, &Overlay::get_overlay_random_peers, max_peers, std::move(promise)); + td::actor::send_closure(it2->second.overlay, &Overlay::get_overlay_random_peers, max_peers, std::move(promise)); } td::actor::ActorOwn Overlays::create(std::string db_root, td::actor::ActorId keyring, @@ -337,7 +453,7 @@ void OverlayManager::get_stats(td::Promise> R) { if (R.is_ok()) { td::actor::send_closure(act, &Cb::receive_answer, R.move_as_ok()); @@ -361,7 +477,7 @@ void OverlayManager::forget_peer(adnl::AdnlNodeIdShort local_id, OverlayIdShort if (it2 == it->second.end()) { return; } - td::actor::send_closure(it2->second, &Overlay::forget_peer, peer_id); + td::actor::send_closure(it2->second.overlay, &Overlay::forget_peer, peer_id); } Certificate::Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, @@ -470,6 +586,35 @@ tl_object_ptr Certificate::empty_tl() { return create_tl_object(); } +OverlayMemberCertificate::OverlayMemberCertificate(const ton_api::overlay_MemberCertificate *cert) { + if (!cert) { + expire_at_ = std::numeric_limits::max(); + return; + } + if (cert->get_id() == ton_api::overlay_emptyMemberCertificate::ID) { + expire_at_ = std::numeric_limits::max(); + return; + } + CHECK(cert->get_id() == ton_api::overlay_memberCertificate::ID); + const auto *real_cert = static_cast(cert); + signed_by_ = PublicKey(real_cert->issued_by_); + flags_ = real_cert->flags_; + slot_ = real_cert->slot_; + expire_at_ = real_cert->expire_at_; + signature_ = td::SharedSlice(real_cert->signature_.as_slice()); +} + +td::Status OverlayMemberCertificate::check_signature(const adnl::AdnlNodeIdShort &node) { + if (is_expired()) { + return td::Status::Error(ErrorCode::notready, "certificate is expired"); + } + td::BufferSlice data_to_sign = to_sign_data(node); + + TRY_RESULT(encryptor, signed_by_.create_encryptor()); + TRY_STATUS(encryptor->check_signature(data_to_sign.as_slice(), signature_.as_slice())); + return td::Status::OK(); +} + } // namespace overlay } // namespace ton diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h index 2e25c944e..12206e048 100644 --- a/overlay/overlay-manager.h +++ b/overlay/overlay-manager.h @@ -53,11 +53,19 @@ class OverlayManager : public Overlays { void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) override; void create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, - std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope, - OverlayOptions opts) override; + std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope, + OverlayOptions opts) override; + void create_semiprivate_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate certificate, + std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope, + OverlayOptions opts) override; void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules, std::string scope) override; + void create_private_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::vector nodes, std::unique_ptr callback, + OverlayPrivacyRules rules, std::string scope, OverlayOptions opts) override; void delete_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id) override; void send_query(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id, std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice query) override { @@ -84,6 +92,11 @@ class OverlayManager : public Overlays { void set_privacy_rules(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, OverlayPrivacyRules rules) override; void update_certificate(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, PublicKeyHash key, std::shared_ptr cert) override; + void update_member_certificate(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + OverlayMemberCertificate certificate) override; + void update_root_member_list(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate certificate) override; void get_overlay_random_peers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, td::uint32 max_peers, td::Promise> promise) override; @@ -92,7 +105,7 @@ class OverlayManager : public Overlays { td::Promise promise); void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data); - void register_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + void register_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, OverlayMemberCertificate cert, td::actor::ActorOwn overlay); void get_stats(td::Promise> promise) override; @@ -105,7 +118,11 @@ class OverlayManager : public Overlays { } private: - std::map>> overlays_; + struct OverlayDescription { + td::actor::ActorOwn overlay; + OverlayMemberCertificate member_certificate; + }; + std::map> overlays_; std::string db_root_; diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index 77fd1bbe3..c61e49652 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -16,85 +16,159 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "adnl/adnl-node-id.hpp" +#include "adnl/adnl-node.h" +#include "auto/tl/ton_api.h" #include "overlay.hpp" +#include "td/utils/Status.h" +#include "td/utils/Time.h" +#include "td/utils/port/signals.h" +#include +#include namespace ton { namespace overlay { -void OverlayImpl::del_peer(adnl::AdnlNodeIdShort id) { - auto P = peers_.get(id); +void OverlayImpl::del_peer(const adnl::AdnlNodeIdShort &id) { + auto P = peer_list_.peers_.get(id); if (P == nullptr) { return; } + if (P->is_permanent_member()) { + VLOG(OVERLAY_DEBUG) << this << ": not deleting peer " << id << ": a permanent member"; + return; + } VLOG(OVERLAY_DEBUG) << this << ": deleting peer " << id; if (P->is_neighbour()) { - VLOG(OVERLAY_INFO) << this << ": deleting neighbour " << id; - bool deleted = false; - for (auto &n : neighbours_) { - if (n == id) { - n = neighbours_[neighbours_.size() - 1]; - neighbours_.resize(neighbours_.size() - 1); - deleted = true; - break; - } + del_from_neighbour_list(P); + } + peer_list_.peers_.remove(id); + peer_list_.bad_peers_.erase(id); +} + +void OverlayImpl::del_from_neighbour_list(OverlayPeer *P) { + CHECK(P); + if (!P->is_neighbour()) { + return; + } + auto id = P->get_id(); + bool deleted = false; + auto &neighbours = peer_list_.neighbours_; + for (auto &n : neighbours) { + if (n == id) { + n = neighbours[neighbours.size() - 1]; + neighbours.resize(neighbours.size() - 1); + deleted = true; + break; } - CHECK(deleted); - P->set_neighbour(false); } - peers_.remove(id); - bad_peers_.erase(id); - update_neighbours(0); + CHECK(deleted); + P->set_neighbour(false); +} + +void OverlayImpl::del_from_neighbour_list(const adnl::AdnlNodeIdShort &id) { + auto P = peer_list_.peers_.get(id); + CHECK(P != nullptr); + return del_from_neighbour_list(P); } void OverlayImpl::del_some_peers() { - if (!public_) { + if (overlay_type_ == OverlayType::FixedMemberList) { return; } - while (peers_.size() > max_peers()) { + const size_t max_iterations = 10; + size_t iteration_seqno = 0; + while (peer_list_.peers_.size() > max_peers() && iteration_seqno++ < max_iterations) { OverlayPeer *P; - if (bad_peers_.empty()) { + if (peer_list_.bad_peers_.empty()) { P = get_random_peer(); } else { - auto it = bad_peers_.upper_bound(next_bad_peer_); - if (it == bad_peers_.end()) { - it = bad_peers_.begin(); + auto it = peer_list_.bad_peers_.upper_bound(peer_list_.next_bad_peer_); + if (it == peer_list_.bad_peers_.end()) { + it = peer_list_.bad_peers_.begin(); } - P = peers_.get(next_bad_peer_ = *it); + P = peer_list_.peers_.get(peer_list_.next_bad_peer_ = *it); } - if (P) { + if (P && !P->is_permanent_member()) { auto id = P->get_id(); del_peer(id); } } + update_neighbours(0); } -void OverlayImpl::do_add_peer(OverlayNode node) { - auto id = node.adnl_id_short(); - - auto V = peers_.get(id); - if (V) { - VLOG(OVERLAY_DEBUG) << this << ": updating peer " << id << " up to version " << node.version(); - V->update(std::move(node)); - } else { - VLOG(OVERLAY_DEBUG) << this << ": adding peer " << id << " of version " << node.version(); - peers_.insert(id, OverlayPeer(std::move(node))); - - del_some_peers(); - update_neighbours(0); +td::Status OverlayImpl::validate_peer_certificate(const adnl::AdnlNodeIdShort &node, + const OverlayMemberCertificate &cert) { + if (cert.empty()) { + if (is_persistent_node(node) || overlay_type_ == OverlayType::Public) { + return td::Status::OK(); + } + return td::Status::Error(ErrorCode::protoviolation, "no member certificate found"); + } + if (cert.is_expired()) { + return td::Status::Error(ErrorCode::timeout, "member certificate is expired"); + } + if (cert.slot() < 0 || cert.slot() >= opts_.max_slaves_in_semiprivate_overlay_) { + return td::Status::Error(ErrorCode::timeout, "member certificate has invalid slot"); } + const auto &issued_by = cert.issued_by(); + auto it = peer_list_.root_public_keys_.find(issued_by.compute_short_id()); + if (it == peer_list_.root_public_keys_.end()) { + return td::Status::Error(ErrorCode::protoviolation, "member certificate is signed by unknown public key"); + } + if (it->second.size() > (size_t)cert.slot()) { + auto &el = it->second[cert.slot()]; + if (cert.expire_at() < el.expire_at) { + return td::Status::Error(ErrorCode::protoviolation, + "member certificate rejected, because we know of newer certificate at the same slot"); + } else if (cert.expire_at() == el.expire_at) { + if (node < el.node) { + return td::Status::Error(ErrorCode::protoviolation, + "member certificate rejected, because we know of newer certificate at the same slot"); + } else if (el.node == node) { + // we could return OK here, but we must make sure, that the unchecked signature will not be used for updating PeerNode. + } + } + } + auto R = get_encryptor(issued_by); + if (R.is_error()) { + return R.move_as_error_prefix("failed to check member certificate: failed to create encryptor: "); + } + auto enc = R.move_as_ok(); + auto S = enc->check_signature(cert.to_sign_data(node).as_slice(), cert.signature()); + if (S.is_error()) { + return S.move_as_error_prefix("failed to check member certificate: bad signature: "); + } + if (it->second.size() <= (size_t)cert.slot()) { + it->second.resize((size_t)cert.slot() + 1); + } + it->second[cert.slot()].expire_at = cert.expire_at(); + it->second[cert.slot()].node = node; + return td::Status::OK(); } -void OverlayImpl::add_peer_in_cont(OverlayNode node) { - CHECK(public_); +td::Status OverlayImpl::validate_peer_certificate(const adnl::AdnlNodeIdShort &node, + ton_api::overlay_MemberCertificate *cert) { + OverlayMemberCertificate ncert(cert); + return validate_peer_certificate(node, ncert); +} - do_add_peer(std::move(node)); +td::Status OverlayImpl::validate_peer_certificate(const adnl::AdnlNodeIdShort &node, + const OverlayMemberCertificate *cert) { + if (!cert) { + if (is_persistent_node(node) || overlay_type_ == OverlayType::Public) { + return td::Status::OK(); + } + return td::Status::Error(ErrorCode::protoviolation, "no member certificate found"); + } + return validate_peer_certificate(node, *cert); } -void OverlayImpl::add_peer_in(OverlayNode node) { - CHECK(public_); +void OverlayImpl::add_peer(OverlayNode node) { + CHECK(overlay_type_ != OverlayType::FixedMemberList); if (node.overlay_id() != overlay_id_) { VLOG(OVERLAY_WARNING) << this << ": received node with bad overlay"; return; @@ -117,35 +191,82 @@ void OverlayImpl::add_peer_in(OverlayNode node) { return; } - add_peer_in_cont(std::move(node)); + if (overlay_type_ == OverlayType::CertificatedMembers) { + auto R = validate_peer_certificate(node.adnl_id_short(), *node.certificate()); + if (R.is_error()) { + VLOG(OVERLAY_WARNING) << this << ": bad peer certificate node=" << node.adnl_id_short() << ": " + << R.move_as_error(); + UNREACHABLE(); + return; + } + } + + auto id = node.adnl_id_short(); + + auto V = peer_list_.peers_.get(id); + if (V) { + VLOG(OVERLAY_DEBUG) << this << ": updating peer " << id << " up to version " << node.version(); + V->update(std::move(node)); + } else { + VLOG(OVERLAY_DEBUG) << this << ": adding peer " << id << " of version " << node.version(); + CHECK(overlay_type_ != OverlayType::CertificatedMembers || (node.certificate() && !node.certificate()->empty())); + peer_list_.peers_.insert(id, OverlayPeer(std::move(node))); + del_some_peers(); + auto X = peer_list_.peers_.get(id); + CHECK(X); + + if (peer_list_.neighbours_.size() < max_neighbours() && + !(X->get_node()->flags() & OverlayMemberFlags::DoNotReceiveBroadcasts)) { + peer_list_.neighbours_.push_back(X->get_id()); + X->set_neighbour(true); + } + + update_neighbours(0); + } } void OverlayImpl::add_peers(std::vector peers) { for (auto &node : peers) { - add_peer_in(std::move(node)); + add_peer(std::move(node)); } } -void OverlayImpl::add_peer(OverlayNode P) { - add_peer_in(std::move(P)); +void OverlayImpl::add_peers(const tl_object_ptr &nodes) { + for (auto &n : nodes->nodes_) { + auto N = OverlayNode::create(n); + if (N.is_ok()) { + add_peer(N.move_as_ok()); + } + } +} + +void OverlayImpl::add_peers(const tl_object_ptr &nodes) { + for (auto &n : nodes->nodes_) { + auto N = OverlayNode::create(n); + if (N.is_ok()) { + add_peer(N.move_as_ok()); + } + } } void OverlayImpl::on_ping_result(adnl::AdnlNodeIdShort peer, bool success) { - if (!public_) { + if (overlay_type_ == OverlayType::FixedMemberList) { return; } - if (OverlayPeer *p = peers_.get(peer)) { + if (OverlayPeer *p = peer_list_.peers_.get(peer)) { p->on_ping_result(success); - if (p->is_alive()) { - bad_peers_.erase(peer); - } else { - bad_peers_.insert(peer); + if (!p->is_permanent_member()) { + if (p->is_alive()) { + peer_list_.bad_peers_.erase(peer); + } else { + peer_list_.bad_peers_.insert(peer); + } } } } void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::Result R) { - CHECK(public_); + CHECK(overlay_type_ != OverlayType::FixedMemberList); on_ping_result(src, R.is_ok()); if (R.is_error()) { VLOG(OVERLAY_NOTICE) << this << ": failed getRandomPeers query: " << R.move_as_error(); @@ -158,16 +279,24 @@ void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::Result nodes; - for (auto &n : res->nodes_) { - auto N = OverlayNode::create(n); - if (N.is_ok()) { - nodes.emplace_back(N.move_as_ok()); - } +void OverlayImpl::receive_random_peers_v2(adnl::AdnlNodeIdShort src, td::Result R) { + CHECK(overlay_type_ != OverlayType::FixedMemberList); + on_ping_result(src, R.is_ok()); + if (R.is_error()) { + VLOG(OVERLAY_NOTICE) << this << ": failed getRandomPeersV2 query: " << R.move_as_error(); + return; + } + auto R2 = fetch_tl_object(R.move_as_ok(), true); + if (R2.is_error()) { + VLOG(OVERLAY_WARNING) << this << ": dropping incorrect answer to overlay.getRandomPeers query from " << src << ": " + << R2.move_as_error(); + return; } - add_peers(std::move(nodes)); + + add_peers(R2.move_as_ok()); } void OverlayImpl::send_random_peers_cont(adnl::AdnlNodeIdShort src, OverlayNode node, @@ -177,10 +306,13 @@ void OverlayImpl::send_random_peers_cont(adnl::AdnlNodeIdShort src, OverlayNode vec.emplace_back(node.tl()); } - for (td::uint32 i = 0; i < nodes_to_send(); i++) { + td::uint32 max_iterations = nodes_to_send() + 16; + for (td::uint32 i = 0; i < max_iterations && vec.size() < nodes_to_send(); i++) { auto P = get_random_peer(true); if (P) { - vec.emplace_back(P->get().tl()); + if (P->has_full_id()) { + vec.emplace_back(P->get_node()->tl()); + } } else { break; } @@ -215,58 +347,110 @@ void OverlayImpl::send_random_peers(adnl::AdnlNodeIdShort src, td::Promise promise) { + std::vector> vec; + if (announce_self_) { + CHECK(is_persistent_node(node.adnl_id_short()) || !node.certificate()->empty()); + vec.emplace_back(node.tl_v2()); + } + + td::uint32 max_iterations = nodes_to_send() + 16; + for (td::uint32 i = 0; i < max_iterations && vec.size() < nodes_to_send(); i++) { + auto P = get_random_peer(true); + if (P) { + if (P->has_full_id() && !P->is_permanent_member()) { + vec.emplace_back(P->get_node()->tl_v2()); + } + } else { + break; + } + } + + if (promise) { + auto Q = create_tl_object(std::move(vec)); + promise.set_value(serialize_tl_object(Q, true)); + } else { + auto P = + td::PromiseCreator::lambda([SelfId = actor_id(this), src, oid = print_id()](td::Result res) { + td::actor::send_closure(SelfId, &OverlayImpl::receive_random_peers_v2, src, std::move(res)); + }); + auto Q = + create_tl_object(create_tl_object(std::move(vec))); + td::actor::send_closure(manager_, &OverlayManager::send_query, src, local_id_, overlay_id_, + "overlay getRandomPeers", std::move(P), + td::Timestamp::in(5.0 + td::Random::fast(0, 50) * 0.1), serialize_tl_object(Q, true)); + } +} + +void OverlayImpl::send_random_peers_v2(adnl::AdnlNodeIdShort src, td::Promise promise) { + auto P = td::PromiseCreator::lambda([src, promise = std::move(promise), + SelfId = actor_id(this)](td::Result res) mutable { + if (res.is_error()) { + promise.set_error(td::Status::Error(ErrorCode::error, "cannot get self node")); + return; + } + td::actor::send_closure(SelfId, &OverlayImpl::send_random_peers_v2_cont, src, res.move_as_ok(), std::move(promise)); + }); + + get_self_node(std::move(P)); +} + void OverlayImpl::update_neighbours(td::uint32 nodes_to_change) { - if (peers_.size() == 0) { + if (peer_list_.peers_.size() == 0) { return; } td::uint32 iter = 0; - while (iter < 10 && (nodes_to_change > 0 || neighbours_.size() < max_neighbours())) { - auto X = peers_.get_random(); + while (iter++ < 10 && (nodes_to_change > 0 || peer_list_.neighbours_.size() < max_neighbours())) { + auto X = peer_list_.peers_.get_random(); if (!X) { break; } if (X->get_id() == local_id_) { - iter++; continue; } - if (public_ && X->get_version() <= td::Clocks::system() - Overlays::overlay_peer_ttl()) { + if (X->get_version() <= td::Clocks::system() - Overlays::overlay_peer_ttl()) { + if (X->is_permanent_member()) { + del_from_neighbour_list(X); + } else { + auto id = X->get_id(); + del_peer(id); + } + continue; + } + + if (overlay_type_ == OverlayType::CertificatedMembers && !X->is_permanent_member() && + X->certificate()->is_expired()) { + auto id = X->get_id(); + del_peer(id); + continue; + } + + if (X->get_node()->flags() & OverlayMemberFlags::DoNotReceiveBroadcasts) { if (X->is_neighbour()) { - bool found = false; - for (auto &n : neighbours_) { - if (n == X->get_id()) { - n = *neighbours_.rbegin(); - found = true; - break; - } - } - CHECK(found); - neighbours_.pop_back(); - X->set_neighbour(false); + del_from_neighbour_list(X); } - bad_peers_.erase(X->get_id()); - peers_.remove(X->get_id()); continue; } if (X->is_neighbour()) { - iter++; continue; } - if (neighbours_.size() < max_neighbours()) { + if (peer_list_.neighbours_.size() < max_neighbours()) { VLOG(OVERLAY_INFO) << this << ": adding new neighbour " << X->get_id(); - neighbours_.push_back(X->get_id()); + peer_list_.neighbours_.push_back(X->get_id()); X->set_neighbour(true); } else { CHECK(nodes_to_change > 0); - auto i = td::Random::fast(0, static_cast(neighbours_.size()) - 1); - auto Y = peers_.get(neighbours_[i]); + auto i = td::Random::fast(0, static_cast(peer_list_.neighbours_.size()) - 1); + auto Y = peer_list_.peers_.get(peer_list_.neighbours_[i]); CHECK(Y != nullptr); CHECK(Y->is_neighbour()); Y->set_neighbour(false); - neighbours_[i] = X->get_id(); + peer_list_.neighbours_[i] = X->get_id(); X->set_neighbour(true); nodes_to_change--; VLOG(OVERLAY_INFO) << this << ": changing neighbour " << Y->get_id() << " -> " << X->get_id(); @@ -276,9 +460,11 @@ void OverlayImpl::update_neighbours(td::uint32 nodes_to_change) { OverlayPeer *OverlayImpl::get_random_peer(bool only_alive) { size_t skip_bad = 3; - while (peers_.size() > (only_alive ? bad_peers_.size() : 0)) { - auto P = peers_.get_random(); - if (public_ && P->get_version() + 3600 < td::Clocks::system()) { + OverlayPeer *res = nullptr; + while (!res && peer_list_.peers_.size() > (only_alive ? peer_list_.bad_peers_.size() : 0)) { + auto P = peer_list_.peers_.get_random(); + if (!P->is_permanent_member() && + (P->get_version() + 3600 < td::Clocks::system() || P->certificate()->is_expired())) { VLOG(OVERLAY_INFO) << this << ": deleting outdated peer " << P->get_id(); del_peer(P->get_id()); continue; @@ -292,18 +478,19 @@ OverlayPeer *OverlayImpl::get_random_peer(bool only_alive) { continue; } } - return P; + res = P; } - return nullptr; + update_neighbours(0); + return res; } void OverlayImpl::get_overlay_random_peers(td::uint32 max_peers, td::Promise> promise) { std::vector v; auto t = td::Clocks::system(); - while (v.size() < max_peers && v.size() < peers_.size() - bad_peers_.size()) { - auto P = peers_.get_random(); - if (public_ && P->get_version() + 3600 < t) { + while (v.size() < max_peers && v.size() < peer_list_.peers_.size() - peer_list_.bad_peers_.size()) { + auto P = peer_list_.peers_.get_random(); + if (!P->is_permanent_member() && (P->get_version() + 3600 < t || P->certificate()->is_expired(t))) { VLOG(OVERLAY_INFO) << this << ": deleting outdated peer " << P->get_id(); del_peer(P->get_id()); } else if (P->is_alive()) { @@ -319,22 +506,221 @@ void OverlayImpl::get_overlay_random_peers(td::uint32 max_peers, } } } + update_neighbours(0); promise.set_result(std::move(v)); } void OverlayImpl::receive_nodes_from_db(tl_object_ptr tl_nodes) { - if (public_) { - std::vector nodes; - for (auto &n : tl_nodes->nodes_) { - auto N = OverlayNode::create(n); - if (N.is_ok()) { - nodes.emplace_back(N.move_as_ok()); + if (overlay_type_ != OverlayType::FixedMemberList) { + add_peers(tl_nodes); + } +} + +void OverlayImpl::receive_nodes_from_db_v2(tl_object_ptr tl_nodes) { + if (overlay_type_ != OverlayType::FixedMemberList) { + add_peers(tl_nodes); + } +} + +bool OverlayImpl::is_persistent_node(const adnl::AdnlNodeIdShort &id) { + auto P = peer_list_.peers_.get(id); + if (!P) { + return false; + } + return P->is_permanent_member(); +} + +bool OverlayImpl::is_valid_peer(const adnl::AdnlNodeIdShort &src, + const ton_api::overlay_MemberCertificate *certificate) { + if (overlay_type_ == OverlayType::Public) { + on_ping_result(src, true); + return true; + } else if (overlay_type_ == OverlayType::FixedMemberList) { + return peer_list_.peers_.get(src); + } else { + OverlayMemberCertificate cert(certificate); + if (cert.empty()) { + auto P = peer_list_.peers_.get(src); + if (P && !P->is_permanent_member()) { + auto C = P->certificate(); + if (C) { + cert = *C; + } } } - add_peers(std::move(nodes)); + + auto S = validate_peer_certificate(src, cert); + if (S.is_error()) { + VLOG(OVERLAY_WARNING) << "adnl=" << src << ": certificate is invalid: " << S; + return false; + } + auto P = peer_list_.peers_.get(src); + if (P) { + CHECK(P->is_permanent_member() || !cert.empty()); + P->update_certificate(std::move(cert)); + } + return true; + } +} + +void OverlayImpl::iterate_all_peers(std::function cb) { + peer_list_.peers_.iterate([&](const adnl::AdnlNodeIdShort &key, OverlayPeer &peer) { cb(key, peer); }); +} + +void OverlayImpl::update_peer_err_ctr(adnl::AdnlNodeIdShort peer_id, bool is_fec) { + auto src_peer = peer_list_.peers_.get(peer_id); + if (src_peer) { + if (is_fec) { + src_peer->fec_broadcast_errors++; + } else { + src_peer->broadcast_errors++; + } } } +void OverlayImpl::update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) { + auto out_peer = peer_list_.peers_.get(peer_id); + if (out_peer) { + out_peer->throughput_out_bytes_ctr += msg_size; + out_peer->throughput_out_packets_ctr++; + + if (is_query) { + out_peer->last_out_query_at = td::Timestamp::now(); + } + } +} + +void OverlayImpl::update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) { + auto in_peer = peer_list_.peers_.get(peer_id); + if (in_peer) { + in_peer->throughput_in_bytes_ctr += msg_size; + in_peer->throughput_in_packets_ctr++; + + if (is_query) { + in_peer->last_in_query_at = td::Timestamp::now(); + } + } +} + +void OverlayImpl::update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) { + auto fpeer = peer_list_.peers_.get(peer_id); + if (fpeer) { + fpeer->ip_addr_str = ip_str; + } +} + +bool OverlayImpl::has_good_peers() const { + return peer_list_.peers_.size() > peer_list_.bad_peers_.size(); +} + +bool OverlayImpl::is_root_public_key(const PublicKeyHash &key) const { + return peer_list_.root_public_keys_.count(key) > 0; +} + +std::vector OverlayImpl::get_neighbours(td::uint32 max_size) const { + if (max_size == 0 || max_size >= peer_list_.neighbours_.size()) { + return peer_list_.neighbours_; + } else { + std::vector vec; + for (td::uint32 i = 0; i < max_size; i++) { + vec.push_back( + peer_list_.neighbours_[td::Random::fast(0, static_cast(peer_list_.neighbours_.size()) - 1)]); + } + return vec; + } +} + +void OverlayImpl::send_message_to_neighbours(td::BufferSlice data) { + wait_neighbours_not_empty([this, data = std::move(data)](td::Result R) { + if (R.is_error()) { + return; + } + for (auto &n : peer_list_.neighbours_) { + td::actor::send_closure(manager_, &OverlayManager::send_message, n, local_id_, overlay_id_, data.clone()); + } + }); +} + +size_t OverlayImpl::neighbours_cnt() const { + return peer_list_.neighbours_.size(); +} + +void OverlayImpl::update_root_member_list(std::vector ids, + std::vector root_public_keys, OverlayMemberCertificate cert) { + td::uint32 expectd_size = + (td::uint32)(ids.size() + root_public_keys.size() * opts_.max_slaves_in_semiprivate_overlay_); + if (expectd_size > opts_.max_peers_) { + opts_.max_peers_ = expectd_size; + } + if (expectd_size > opts_.max_neighbours_) { + opts_.max_neighbours_ = expectd_size; + } + std::sort(ids.begin(), ids.end()); + auto old_root_public_keys = std::move(peer_list_.root_public_keys_); + for (const auto &pub_key : root_public_keys) { + auto it = old_root_public_keys.find(pub_key); + if (it != old_root_public_keys.end()) { + peer_list_.root_public_keys_.emplace(it->first, std::move(it->second)); + } else { + peer_list_.root_public_keys_.emplace(pub_key, PeerList::SlaveKeys{}); + } + } + std::vector to_del; + peer_list_.peers_.iterate([&](const adnl::AdnlNodeIdShort &key, OverlayPeer &peer) { + peer.set_permanent(std::binary_search(ids.begin(), ids.end(), key)); + if (peer.is_permanent_member()) { + peer.clear_certificate(); + } else { + auto S = validate_peer_certificate(peer.get_id(), peer.certificate()); + if (S.is_error()) { + to_del.push_back(peer.get_id()); + } + } + }); + for (const auto &id : to_del) { + del_peer(id); + } + for (const auto &id : ids) { + if (!peer_list_.peers_.exists(id)) { + OverlayNode node(id, overlay_id_, opts_.default_permanent_members_flags_); + OverlayPeer peer(std::move(node)); + peer.set_permanent(true); + CHECK(peer.is_permanent_member()); + peer_list_.peers_.insert(std::move(id), std::move(peer)); + } + } + + update_member_certificate(std::move(cert)); + update_neighbours(0); +} + +void OverlayImpl::update_member_certificate(OverlayMemberCertificate cert) { + peer_list_.cert_ = std::move(cert); + + if (is_persistent_node(local_id_)) { + peer_list_.local_cert_is_valid_until_ = td::Timestamp::in(86400.0 * 365 * 100); /* 100 years */ + } else { + auto R = validate_peer_certificate(local_id_, &peer_list_.cert_); + if (R.is_ok()) { + peer_list_.local_cert_is_valid_until_ = td::Timestamp::at_unix(cert.expire_at()); + } else { + peer_list_.local_cert_is_valid_until_ = td::Timestamp::never(); + } + } +} + +bool OverlayImpl::has_valid_membership_certificate() { + if (overlay_type_ != OverlayType::CertificatedMembers) { + return true; + } + + if (!peer_list_.local_cert_is_valid_until_) { + return false; + } + + return !peer_list_.local_cert_is_valid_until_.is_in_past(); +} + } // namespace overlay } // namespace ton diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index 5b391441b..d05f5ae28 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -27,41 +27,61 @@ #include "auto/tl/ton_api.hpp" #include "keys/encryptor.h" +#include "td/utils/Status.h" #include "td/utils/StringBuilder.h" +#include "td/utils/port/signals.h" +#include namespace ton { namespace overlay { -td::actor::ActorOwn Overlay::create(td::actor::ActorId keyring, - td::actor::ActorId adnl, - td::actor::ActorId manager, - td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, - OverlayIdFull overlay_id, std::unique_ptr callback, - OverlayPrivacyRules rules, td::string scope, OverlayOptions opts) { - auto R = td::actor::create_actor("overlay", keyring, adnl, manager, dht_node, local_id, - std::move(overlay_id), true, std::vector(), - std::move(callback), std::move(rules), scope, opts); +const OverlayMemberCertificate OverlayNode::empty_certificate_{}; + +td::actor::ActorOwn Overlay::create_public(td::actor::ActorId keyring, + td::actor::ActorId adnl, + td::actor::ActorId manager, + td::actor::ActorId dht_node, + adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::unique_ptr callback, + OverlayPrivacyRules rules, td::string scope, OverlayOptions opts) { + auto R = td::actor::create_actor( + "overlay", keyring, adnl, manager, dht_node, local_id, std::move(overlay_id), OverlayType::Public, + std::vector(), std::vector(), OverlayMemberCertificate{}, std::move(callback), + std::move(rules), std::move(scope), std::move(opts)); return td::actor::ActorOwn(std::move(R)); } -td::actor::ActorOwn Overlay::create(td::actor::ActorId keyring, - td::actor::ActorId adnl, - td::actor::ActorId manager, - td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, - OverlayIdFull overlay_id, std::vector nodes, - std::unique_ptr callback, OverlayPrivacyRules rules, - std::string scope) { +td::actor::ActorOwn Overlay::create_private( + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, + OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, + OverlayPrivacyRules rules, std::string scope, OverlayOptions opts) { auto R = td::actor::create_actor("overlay", keyring, adnl, manager, dht_node, local_id, - std::move(overlay_id), false, std::move(nodes), std::move(callback), - std::move(rules), std::move(scope)); + std::move(overlay_id), OverlayType::FixedMemberList, std::move(nodes), + std::vector(), OverlayMemberCertificate{}, + std::move(callback), std::move(rules), std::move(scope)); + return td::actor::ActorOwn(std::move(R)); +} + +td::actor::ActorOwn Overlay::create_semiprivate( + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, + OverlayIdFull overlay_id, std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate cert, std::unique_ptr callback, OverlayPrivacyRules rules, + std::string scope, OverlayOptions opts) { + auto R = td::actor::create_actor( + "overlay", keyring, adnl, manager, dht_node, local_id, std::move(overlay_id), OverlayType::CertificatedMembers, + std::move(nodes), std::move(root_public_keys), std::move(cert), std::move(callback), std::move(rules), + std::move(scope), std::move(opts)); return td::actor::ActorOwn(std::move(R)); } OverlayImpl::OverlayImpl(td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId manager, td::actor::ActorId dht_node, - adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, bool pub, - std::vector nodes, std::unique_ptr callback, + adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, OverlayType overlay_type, + std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate cert, std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope, OverlayOptions opts) : keyring_(keyring) , adnl_(adnl) @@ -70,37 +90,28 @@ OverlayImpl::OverlayImpl(td::actor::ActorId keyring, td::actor , local_id_(local_id) , id_full_(std::move(overlay_id)) , callback_(std::move(callback)) - , public_(pub) + , overlay_type_(overlay_type) , rules_(std::move(rules)) , scope_(scope) , announce_self_(opts.announce_self_) - , frequent_dht_lookup_(opts.frequent_dht_lookup_) { + , opts_(std::move(opts)) { overlay_id_ = id_full_.compute_short_id(); + frequent_dht_lookup_ = opts_.frequent_dht_lookup_; + peer_list_.local_member_flags_ = opts_.local_overlay_member_flags_; - VLOG(OVERLAY_INFO) << this << ": creating " << (public_ ? "public" : "private"); + VLOG(OVERLAY_INFO) << this << ": creating"; - for (auto &node : nodes) { - CHECK(!public_); - auto X = OverlayNode{node, overlay_id_}; - do_add_peer(std::move(X)); - } + update_root_member_list(std::move(nodes), std::move(root_public_keys), std::move(cert)); update_neighbours(static_cast(nodes.size())); } void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeers &query, td::Promise promise) { - if (public_) { + if (overlay_type_ != OverlayType::FixedMemberList) { VLOG(OVERLAY_DEBUG) << this << ": received " << query.peers_->nodes_.size() << " nodes from " << src << " in getRandomPeers query"; - std::vector nodes; - for (auto &n : query.peers_->nodes_) { - auto N = OverlayNode::create(n); - if (N.is_ok()) { - nodes.emplace_back(N.move_as_ok()); - } - } - add_peers(std::move(nodes)); + add_peers(query.peers_); send_random_peers(src, std::move(promise)); } else { VLOG(OVERLAY_WARNING) << this << ": DROPPING getRandomPeers query from " << src << " in private overlay"; @@ -108,6 +119,19 @@ void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getR } } +void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeersV2 &query, + td::Promise promise) { + if (overlay_type_ != OverlayType::FixedMemberList) { + VLOG(OVERLAY_DEBUG) << this << ": received " << query.peers_->nodes_.size() << " nodes from " << src + << " in getRandomPeers query"; + add_peers(query.peers_); + send_random_peers_v2(src, std::move(promise)); + } else { + VLOG(OVERLAY_WARNING) << this << ": DROPPING getRandomPeers query from " << src << " in private overlay"; + promise.set_error(td::Status::Error(ErrorCode::protoviolation, "overlay is private")); + } +} + void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcast &query, td::Promise promise) { auto it = broadcasts_.find(query.hash_); @@ -135,17 +159,19 @@ void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getB promise.set_error(td::Status::Error(ErrorCode::protoviolation, "dropping get broadcast list query")); } -void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise) { - if (!public_) { - auto P = peers_.get(src); - if (P == nullptr) { - VLOG(OVERLAY_WARNING) << this << ": received query in private overlay from unknown source " << src; - promise.set_error(td::Status::Error(ErrorCode::protoviolation, "overlay is private")); - return; - } - } else { - on_ping_result(src, true); +/*void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, adnl::AdnlQueryId query_id, ton_api::overlay_customQuery &query) { + callback_->receive_query(src, query_id, id_, std::move(query.data_)); +} +*/ + +void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, tl_object_ptr extra, + td::BufferSlice data, td::Promise promise) { + if (!is_valid_peer(src, extra ? extra->certificate_.get() : nullptr)) { + VLOG(OVERLAY_WARNING) << this << ": received query in private overlay from unknown source " << src; + promise.set_error(td::Status::Error(ErrorCode::protoviolation, "overlay is not public")); + return; } + auto R = fetch_tl_object(data.clone(), true); if (R.is_error()) { @@ -163,16 +189,25 @@ void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr bcast) { + if (peer_list_.local_member_flags_ & OverlayMemberFlags::DoNotReceiveBroadcasts) { + return td::Status::OK(); + } return BroadcastSimple::create(this, message_from, std::move(bcast)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr b) { + if (peer_list_.local_member_flags_ & OverlayMemberFlags::DoNotReceiveBroadcasts) { + return td::Status::OK(); + } return OverlayFecBroadcastPart::create(this, message_from, std::move(b)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr b) { + if (peer_list_.local_member_flags_ & OverlayMemberFlags::DoNotReceiveBroadcasts) { + return td::Status::OK(); + } return OverlayFecBroadcastPart::create(this, message_from, std::move(b)); } @@ -184,6 +219,7 @@ td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg) { + return td::Status::OK(); // disable this logic for now auto it = fec_broadcasts_.find(msg->hash_); if (it != fec_broadcasts_.end()) { VLOG(OVERLAY_DEBUG) << this << ": received fec opt-out message from " << message_from << " for broadcast " @@ -198,6 +234,7 @@ td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg) { + return td::Status::OK(); // disable this logic for now auto it = fec_broadcasts_.find(msg->hash_); if (it != fec_broadcasts_.end()) { VLOG(OVERLAY_DEBUG) << this << ": received fec completed message from " << message_from << " for broadcast " @@ -217,15 +254,13 @@ td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, return td::Status::OK(); } -void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) { - if (!public_) { - if (peers_.get(src) == nullptr) { - VLOG(OVERLAY_WARNING) << this << ": received query in private overlay from unknown source " << src; - return; - } - } else { - on_ping_result(src, true); +void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, tl_object_ptr extra, + td::BufferSlice data) { + if (!is_valid_peer(src, extra ? extra->certificate_.get() : nullptr)) { + VLOG(OVERLAY_WARNING) << this << ": received message in private overlay from unknown source " << src; + return; } + auto X = fetch_tl_object(data.clone(), true); if (X.is_error()) { VLOG(OVERLAY_DEBUG) << this << ": received custom message"; @@ -240,44 +275,51 @@ void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice dat void OverlayImpl::alarm() { bcast_gc(); - - if(update_throughput_at_.is_in_past()) { + + if (update_throughput_at_.is_in_past()) { double t_elapsed = td::Time::now() - last_throughput_update_.at(); auto SelfId = actor_id(this); - peers_.iterate([&](const adnl::AdnlNodeIdShort &key, OverlayPeer &peer) { + iterate_all_peers([&](const adnl::AdnlNodeIdShort &key, OverlayPeer &peer) { peer.throughput_out_bytes = static_cast(peer.throughput_out_bytes_ctr / t_elapsed); peer.throughput_in_bytes = static_cast(peer.throughput_in_bytes_ctr / t_elapsed); - + peer.throughput_out_packets = static_cast(peer.throughput_out_packets_ctr / t_elapsed); peer.throughput_in_packets = static_cast(peer.throughput_in_packets_ctr / t_elapsed); - + peer.throughput_out_bytes_ctr = 0; peer.throughput_in_bytes_ctr = 0; - + peer.throughput_out_packets_ctr = 0; peer.throughput_in_packets_ctr = 0; - + auto P = td::PromiseCreator::lambda([SelfId, peer_id = key](td::Result result) { result.ensure(); td::actor::send_closure(SelfId, &Overlay::update_peer_ip_str, peer_id, result.move_as_ok()); }); - + td::actor::send_closure(adnl_, &adnl::AdnlSenderInterface::get_conn_ip_str, local_id_, key, std::move(P)); }); - + update_throughput_at_ = td::Timestamp::in(50.0); last_throughput_update_ = td::Timestamp::now(); } - - if (public_) { - if (peers_.size() > 0) { + + if (overlay_type_ != OverlayType::FixedMemberList) { + if (has_valid_membership_certificate()) { auto P = get_random_peer(); if (P) { - send_random_peers(P->get_id(), {}); + if (overlay_type_ == OverlayType::Public) { + send_random_peers(P->get_id(), {}); + } else { + send_random_peers_v2(P->get_id(), {}); + } } + } else { + VLOG(OVERLAY_WARNING) << "meber certificate ist invalid, valid_until=" + << peer_list_.local_cert_is_valid_until_.at_unix(); } - if (next_dht_query_ && next_dht_query_.is_in_past()) { + if (next_dht_query_ && next_dht_query_.is_in_past() && overlay_type_ == OverlayType::Public) { next_dht_query_ = td::Timestamp::never(); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result res) { td::actor::send_closure(SelfId, &OverlayImpl::receive_dht_nodes, std::move(res), true); @@ -285,21 +327,22 @@ void OverlayImpl::alarm() { td::actor::send_closure(dht_node_, &dht::Dht::get_value, dht::DhtKey{overlay_id_.pubkey_hash(), "nodes", 0}, std::move(P)); } - if (update_db_at_.is_in_past()) { - if (peers_.size() > 0) { - std::vector vec; - for (td::uint32 i = 0; i < 20; i++) { - auto P = get_random_peer(); - if (!P) { - break; - } - vec.push_back(P->get()); + if (update_db_at_.is_in_past() && overlay_type_ == OverlayType::Public) { + std::vector vec; + for (td::uint32 i = 0; i < 20; i++) { + auto P = get_random_peer(); + if (!P) { + break; } + vec.push_back(P->get_node()->clone()); + } + if (vec.size() > 0) { td::actor::send_closure(manager_, &OverlayManager::save_to_db, local_id_, overlay_id_, std::move(vec)); } update_db_at_ = td::Timestamp::in(60.0); } + update_neighbours(0); alarm_timestamp() = td::Timestamp::in(1.0); } else { update_neighbours(0); @@ -308,7 +351,7 @@ void OverlayImpl::alarm() { } void OverlayImpl::receive_dht_nodes(td::Result res, bool dummy) { - CHECK(public_); + CHECK(overlay_type_ == OverlayType::Public); if (res.is_ok()) { auto v = res.move_as_ok(); auto R = fetch_tl_object(v.value().clone(), true); @@ -354,7 +397,7 @@ void OverlayImpl::receive_dht_nodes(td::Result res, bool dummy) { } void OverlayImpl::update_dht_nodes(OverlayNode node) { - if (!public_) { + if (overlay_type_ != OverlayType::Public) { return; } @@ -412,7 +455,7 @@ void OverlayImpl::bcast_gc() { } void OverlayImpl::wait_neighbours_not_empty(td::Promise promise, int max_retries) { - if (!neighbours_.empty()) { + if (!peer_list_.neighbours_.empty()) { promise.set_result(td::Unit()); } else if (max_retries > 0) { delay_action( @@ -425,18 +468,16 @@ void OverlayImpl::wait_neighbours_not_empty(td::Promise promise, int m } } -void OverlayImpl::send_message_to_neighbours(td::BufferSlice data) { - wait_neighbours_not_empty([this, data = std::move(data)](td::Result R) { - if (R.is_error()) { - return; - } - for (auto &n : neighbours_) { - td::actor::send_closure(manager_, &OverlayManager::send_message, n, local_id_, overlay_id_, data.clone()); - } - }); -} - void OverlayImpl::send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) { + if (!has_valid_membership_certificate()) { + VLOG(OVERLAY_WARNING) << "member certificate is invalid, valid_until=" + << peer_list_.local_cert_is_valid_until_.at_unix(); + return; + } + if (!has_valid_broadcast_certificate(send_as, data.size(), false)) { + VLOG(OVERLAY_WARNING) << "broadcast source certificate is invalid"; + return; + } wait_neighbours_not_empty([this, send_as, flags, data = std::move(data)](td::Result R) mutable { if (R.is_error()) { return; @@ -449,6 +490,15 @@ void OverlayImpl::send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::Bu } void OverlayImpl::send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) { + if (!has_valid_membership_certificate()) { + VLOG(OVERLAY_WARNING) << "meber certificate ist invalid, valid_until=" + << peer_list_.local_cert_is_valid_until_.at_unix(); + return; + } + if (!has_valid_broadcast_certificate(send_as, data.size(), true)) { + VLOG(OVERLAY_WARNING) << "broadcast source certificate is invalid"; + return; + } wait_neighbours_not_empty([this, send_as, flags, data = std::move(data)](td::Result R) mutable { if (R.is_error()) { return; @@ -472,6 +522,22 @@ td::Status OverlayImpl::check_date(td::uint32 date) { return td::Status::OK(); } +BroadcastCheckResult OverlayImpl::check_source_eligible(const PublicKeyHash &source, const Certificate *cert, + td::uint32 size, bool is_fec) { + if (size == 0) { + return BroadcastCheckResult::Forbidden; + } + + auto r = rules_.check_rules(source, size, is_fec); + if (!cert || r == BroadcastCheckResult::Allowed) { + return r; + } + + auto r2 = cert->check(source, overlay_id_, static_cast(td::Clocks::system()), size, is_fec); + r2 = broadcast_check_result_min(r2, rules_.check_rules(cert->issuer_hash(), size, is_fec)); + return broadcast_check_result_max(r, r2); +} + BroadcastCheckResult OverlayImpl::check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size, bool is_fec) { if (size == 0) { @@ -479,7 +545,7 @@ BroadcastCheckResult OverlayImpl::check_source_eligible(PublicKey source, const } auto short_id = source.compute_short_id(); - auto r = rules_.check_rules(source.compute_short_id(), size, is_fec); + auto r = rules_.check_rules(short_id, size, is_fec); if (!cert || r == BroadcastCheckResult::Allowed) { return r; } @@ -514,21 +580,23 @@ void OverlayImpl::register_fec_broadcast(std::unique_ptr bcast) { } void OverlayImpl::get_self_node(td::Promise promise) { - OverlayNode s{local_id_, overlay_id_}; + OverlayNode s{local_id_, overlay_id_, peer_list_.local_member_flags_}; auto to_sign = s.to_sign(); - auto P = td::PromiseCreator::lambda([oid = print_id(), s = std::move(s), promise = std::move(promise)]( - td::Result> R) mutable { - if (R.is_error()) { - auto S = R.move_as_error(); - LOG(ERROR) << oid << ": failed to get self node: " << S; - promise.set_error(std::move(S)); - return; - } - auto V = R.move_as_ok(); - s.update_signature(std::move(V.first)); - s.update_adnl_id(adnl::AdnlNodeIdFull{V.second}); - promise.set_value(std::move(s)); - }); + auto P = td::PromiseCreator::lambda( + [oid = print_id(), s = std::move(s), cert = peer_list_.cert_, + promise = std::move(promise)](td::Result> R) mutable { + if (R.is_error()) { + auto S = R.move_as_error(); + LOG(ERROR) << oid << ": failed to get self node: " << S; + promise.set_error(std::move(S)); + return; + } + auto V = R.move_as_ok(); + s.update_signature(std::move(V.first)); + s.update_adnl_id(adnl::AdnlNodeIdFull{V.second}); + s.update_certificate(std::move(cert)); + promise.set_value(std::move(s)); + }); td::actor::send_closure(keyring_, &keyring::Keyring::sign_add_get_public_key, local_id_.pubkey_hash(), std::move(to_sign), std::move(P)); @@ -620,17 +688,6 @@ void OverlayImpl::check_broadcast(PublicKeyHash src, td::BufferSlice data, td::P callback_->check_broadcast(src, overlay_id_, std::move(data), std::move(promise)); } -void OverlayImpl::update_peer_err_ctr(adnl::AdnlNodeIdShort peer_id, bool is_fec) { - auto src_peer = peers_.get(peer_id); - if(src_peer) { - if(is_fec) { - src_peer->fec_broadcast_errors++; - } else { - src_peer->broadcast_errors++; - } - } -} - void OverlayImpl::broadcast_checked(Overlay::BroadcastHash hash, td::Result R) { { auto it = broadcasts_.find(hash); @@ -652,28 +709,28 @@ void OverlayImpl::get_stats(td::Promiseoverlay_id_ = overlay_id_.bits256_value(); res->overlay_id_full_ = id_full_.pubkey().tl(); res->scope_ = scope_; - peers_.iterate([&](const adnl::AdnlNodeIdShort &key, const OverlayPeer &peer) { + iterate_all_peers([&](const adnl::AdnlNodeIdShort &key, const OverlayPeer &peer) { auto node_obj = create_tl_object(); node_obj->adnl_id_ = key.bits256_value(); node_obj->t_out_bytes_ = peer.throughput_out_bytes; node_obj->t_in_bytes_ = peer.throughput_in_bytes; - + node_obj->t_out_pckts_ = peer.throughput_out_packets; node_obj->t_in_pckts_ = peer.throughput_in_packets; - + node_obj->ip_addr_ = peer.ip_addr_str; - + node_obj->last_in_query_ = static_cast(peer.last_in_query_at.at_unix()); node_obj->last_out_query_ = static_cast(peer.last_out_query_at.at_unix()); - + node_obj->bdcst_errors_ = peer.broadcast_errors; node_obj->fec_bdcst_errors_ = peer.fec_broadcast_errors; - + res->nodes_.push_back(std::move(node_obj)); }); res->stats_.push_back( - create_tl_object("neighbours_cnt", PSTRING() << neighbours_.size())); + create_tl_object("neighbours_cnt", PSTRING() << neighbours_cnt())); callback_->get_stats_extra([promise = std::move(promise), res = std::move(res)](td::Result R) mutable { if (R.is_ok()) { @@ -683,6 +740,14 @@ void OverlayImpl::get_stats(td::Promise std::numeric_limits::max()) { + return false; + } + auto it = certs_.find(source); + return check_source_eligible(source, it == certs_.end() ? nullptr : it->second.get(), (td::uint32)size, is_fec); +} + } // namespace overlay } // namespace ton diff --git a/overlay/overlay.h b/overlay/overlay.h index a6d5cd6b5..f25db206a 100644 --- a/overlay/overlay.h +++ b/overlay/overlay.h @@ -18,6 +18,7 @@ */ #pragma once +#include "auto/tl/ton_api.h" #include "td/utils/buffer.h" #include "td/utils/int_types.h" @@ -37,24 +38,29 @@ class Overlay : public td::actor::Actor { using BroadcastDataHash = td::Bits256; using BroadcastPartHash = td::Bits256; - static td::actor::ActorOwn create(td::actor::ActorId keyring, - td::actor::ActorId adnl, - td::actor::ActorId manager, - td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, - OverlayIdFull overlay_id, std::unique_ptr callback, - OverlayPrivacyRules rules, td::string scope, OverlayOptions opts = {}); - static td::actor::ActorOwn create(td::actor::ActorId keyring, - td::actor::ActorId adnl, - td::actor::ActorId manager, - td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, - OverlayIdFull overlay_id, std::vector nodes, - std::unique_ptr callback, OverlayPrivacyRules rules, - std::string scope); + static td::actor::ActorOwn create_public( + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, + OverlayIdFull overlay_id, std::unique_ptr callback, OverlayPrivacyRules rules, + td::string scope, OverlayOptions opts = {}); + static td::actor::ActorOwn create_private( + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, + OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, + OverlayPrivacyRules rules, std::string scope, OverlayOptions opts = {}); + static td::actor::ActorOwn create_semiprivate( + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, + OverlayIdFull overlay_id, std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate cert, std::unique_ptr callback, OverlayPrivacyRules rules, + std::string scope, OverlayOptions opts = {}); virtual void update_dht_node(td::actor::ActorId dht) = 0; - virtual void receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) = 0; - virtual void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise) = 0; + virtual void receive_message(adnl::AdnlNodeIdShort src, tl_object_ptr extra, + td::BufferSlice data) = 0; + virtual void receive_query(adnl::AdnlNodeIdShort src, tl_object_ptr extra, + td::BufferSlice data, td::Promise promise) = 0; virtual void send_message_to_neighbours(td::BufferSlice data) = 0; virtual void send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) = 0; virtual void send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) = 0; @@ -64,10 +70,14 @@ class Overlay : public td::actor::Actor { virtual void add_certificate(PublicKeyHash key, std::shared_ptr) = 0; virtual void set_privacy_rules(OverlayPrivacyRules rules) = 0; virtual void receive_nodes_from_db(tl_object_ptr nodes) = 0; + virtual void receive_nodes_from_db_v2(tl_object_ptr nodes) = 0; virtual void get_stats(td::Promise> promise) = 0; virtual void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) = 0; virtual void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) = 0; virtual void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) = 0; + virtual void update_member_certificate(OverlayMemberCertificate cert) = 0; + virtual void update_root_member_list(std::vector nodes, + std::vector root_public_keys, OverlayMemberCertificate cert) = 0; //virtual void receive_broadcast(td::BufferSlice data) = 0; //virtual void subscribe(std::unique_ptr callback) = 0; virtual void forget_peer(adnl::AdnlNodeIdShort peer_id) = 0; diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index f3ef3ab41..8cf9e9b0b 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -18,11 +18,15 @@ */ #pragma once +#include +#include #include #include #include +#include #include +#include "adnl/adnl-node-id.hpp" #include "overlay.h" #include "overlay-manager.h" #include "overlay-fec.hpp" @@ -32,6 +36,9 @@ #include "td/utils/DecTree.h" #include "td/utils/List.h" +#include "td/utils/Status.h" +#include "td/utils/Time.h" +#include "td/utils/buffer.h" #include "td/utils/overloaded.h" #include "fec/fec.h" @@ -40,6 +47,8 @@ #include "auto/tl/ton_api.h" #include "auto/tl/ton_api.hpp" +#include "td/utils/port/signals.h" +#include "tl-utils/common-utils.hpp" namespace ton { @@ -58,15 +67,17 @@ class OverlayPeer { adnl::AdnlNodeIdFull get_full_id() const { return node_.adnl_id_full(); } - OverlayNode get() const { - return node_.clone(); + const OverlayNode *get_node() const { + return &node_; } void update(OverlayNode node) { CHECK(get_id() == node.adnl_id_short()); - if (node.version() > node_.version()) { - node_ = std::move(node); - } + node_.update(std::move(node)); + } + void update_certificate(OverlayMemberCertificate cert) { + node_.update_certificate(std::move(cert)); } + OverlayPeer(OverlayNode node) : node_(std::move(node)) { id_ = node_.adnl_id_short(); } @@ -95,24 +106,44 @@ class OverlayPeer { return is_alive_; } + bool is_permanent_member() const { + return is_permanent_member_; + } + + void set_permanent(bool value) { + is_permanent_member_ = value; + } + + void clear_certificate() { + node_.clear_certificate(); + } + + auto certificate() const { + return node_.certificate(); + } + + bool has_full_id() const { + return node_.has_full_id(); + } + td::uint32 throughput_out_bytes = 0; td::uint32 throughput_in_bytes = 0; - + td::uint32 throughput_out_packets = 0; td::uint32 throughput_in_packets = 0; - + td::uint32 throughput_out_bytes_ctr = 0; td::uint32 throughput_in_bytes_ctr = 0; - + td::uint32 throughput_out_packets_ctr = 0; td::uint32 throughput_in_packets_ctr = 0; - + td::uint32 broadcast_errors = 0; td::uint32 fec_broadcast_errors = 0; - + td::Timestamp last_in_query_at = td::Timestamp::now(); td::Timestamp last_out_query_at = td::Timestamp::now(); - + td::string ip_addr_str = "undefined"; private: @@ -122,6 +153,7 @@ class OverlayPeer { bool is_neighbour_ = false; size_t missed_pings_ = 0; bool is_alive_ = true; + bool is_permanent_member_ = false; td::Timestamp last_ping_at_ = td::Timestamp::now(); }; @@ -129,19 +161,23 @@ class OverlayImpl : public Overlay { public: OverlayImpl(td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId manager, td::actor::ActorId dht_node, - adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, bool pub, - std::vector nodes, std::unique_ptr callback, - OverlayPrivacyRules rules, td::string scope = "{ \"type\": \"undefined\" }", OverlayOptions opts = {}); + adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, OverlayType overlay_type, + std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate cert, std::unique_ptr callback, OverlayPrivacyRules rules, + td::string scope = "{ \"type\": \"undefined\" }", OverlayOptions opts = {}); void update_dht_node(td::actor::ActorId dht) override { dht_node_ = dht; } - void receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) override; - void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise) override; + void receive_message(adnl::AdnlNodeIdShort src, tl_object_ptr extra, + td::BufferSlice data) override; + void receive_query(adnl::AdnlNodeIdShort src, tl_object_ptr extra, + td::BufferSlice data, td::Promise promise) override; void send_message_to_neighbours(td::BufferSlice data) override; void send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) override; void send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) override; void receive_nodes_from_db(tl_object_ptr nodes) override; + void receive_nodes_from_db_v2(tl_object_ptr nodes) override; void get_self_node(td::Promise promise); @@ -149,8 +185,8 @@ class OverlayImpl : public Overlay { void start_up() override { update_throughput_at_ = td::Timestamp::in(50.0); last_throughput_update_ = td::Timestamp::now(); - - if (public_) { + + if (overlay_type_ == OverlayType::Public) { update_db_at_ = td::Timestamp::in(60.0); } alarm_timestamp() = td::Timestamp::in(1); @@ -158,13 +194,17 @@ class OverlayImpl : public Overlay { void on_ping_result(adnl::AdnlNodeIdShort peer, bool success); void receive_random_peers(adnl::AdnlNodeIdShort src, td::Result R); + void receive_random_peers_v2(adnl::AdnlNodeIdShort src, td::Result R); void send_random_peers(adnl::AdnlNodeIdShort dst, td::Promise promise); + void send_random_peers_v2(adnl::AdnlNodeIdShort dst, td::Promise promise); void send_random_peers_cont(adnl::AdnlNodeIdShort dst, OverlayNode node, td::Promise promise); + void send_random_peers_v2_cont(adnl::AdnlNodeIdShort dst, OverlayNode node, td::Promise promise); void get_overlay_random_peers(td::uint32 max_peers, td::Promise> promise) override; void set_privacy_rules(OverlayPrivacyRules rules) override; void add_certificate(PublicKeyHash key, std::shared_ptr cert) override { certs_[key] = std::move(cert); } + void update_member_certificate(OverlayMemberCertificate cert) override; void receive_dht_nodes(td::Result res, bool dummy); void update_dht_nodes(OverlayNode node); @@ -187,6 +227,8 @@ class OverlayImpl : public Overlay { td::Status check_date(td::uint32 date); BroadcastCheckResult check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size, bool is_fec); + BroadcastCheckResult check_source_eligible(const PublicKeyHash &source, const Certificate *cert, td::uint32 size, + bool is_fec); td::Status check_delivered(BroadcastHash hash); void broadcast_checked(Overlay::BroadcastHash hash, td::Result R); @@ -205,17 +247,7 @@ class OverlayImpl : public Overlay { void send_new_fec_broadcast_part(PublicKeyHash local_id, Overlay::BroadcastDataHash data_hash, td::uint32 size, td::uint32 flags, td::BufferSlice part, td::uint32 seqno, fec::FecType fec_type, td::uint32 date); - std::vector get_neighbours(td::uint32 max_size = 0) const { - if (max_size == 0 || max_size >= neighbours_.size()) { - return neighbours_; - } else { - std::vector vec; - for (td::uint32 i = 0; i < max_size; i++) { - vec.push_back(neighbours_[td::Random::fast(0, static_cast(neighbours_.size()) - 1)]); - } - return vec; - } - } + std::vector get_neighbours(td::uint32 max_size = 0) const; td::actor::ActorId overlay_manager() const { return manager_; } @@ -235,40 +267,54 @@ class OverlayImpl : public Overlay { td::Result get_encryptor(PublicKey source); void get_stats(td::Promise> promise) override; - - void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override { - auto out_peer = peers_.get(peer_id); - if(out_peer) { - out_peer->throughput_out_bytes_ctr += msg_size; - out_peer->throughput_out_packets_ctr++; - - if(is_query) - { - out_peer->last_out_query_at = td::Timestamp::now(); - } - } + + void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override; + + void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override; + + void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) override; + + void update_root_member_list(std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate cert) override; + + bool is_valid_peer(const adnl::AdnlNodeIdShort &id, const ton_api::overlay_MemberCertificate *certificate); + bool is_persistent_node(const adnl::AdnlNodeIdShort &id); + + td::uint32 max_data_bcasts() const { + return 100; } - - void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override { - auto in_peer = peers_.get(peer_id); - if(in_peer) { - in_peer->throughput_in_bytes_ctr += msg_size; - in_peer->throughput_in_packets_ctr++; - - if(is_query) - { - in_peer->last_in_query_at = td::Timestamp::now(); - } - } + td::uint32 max_bcasts() const { + return 1000; } - - void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) override { - auto fpeer = peers_.get(peer_id); - if(fpeer) { - fpeer->ip_addr_str = ip_str; - } + td::uint32 max_fec_bcasts() const { + return 20; + } + td::uint32 max_sources() const { + return 10; + } + td::uint32 max_encryptors() const { + return 16; + } + + td::uint32 max_neighbours() const { + return opts_.max_neighbours_; } + td::uint32 max_peers() const { + return opts_.max_peers_; + } + + td::uint32 nodes_to_send() const { + return opts_.nodes_to_send_; + } + + td::uint32 propagate_broadcast_to() const { + return opts_.propagate_broadcast_to_; + } + + bool has_valid_membership_certificate(); + bool has_valid_broadcast_certificate(const PublicKeyHash &source, size_t size, bool is_fec); + void forget_peer(adnl::AdnlNodeIdShort peer_id) override { del_peer(peer_id); } @@ -283,6 +329,8 @@ class OverlayImpl : public Overlay { void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeers &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeersV2 &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcast &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcastList &query, @@ -299,20 +347,28 @@ class OverlayImpl : public Overlay { td::Status process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg); td::Status process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg); - void do_add_peer(OverlayNode node); - void add_peer_in_cont(OverlayNode node); - void add_peer_in(OverlayNode node); + td::Status validate_peer_certificate(const adnl::AdnlNodeIdShort &node, const OverlayMemberCertificate &cert); + td::Status validate_peer_certificate(const adnl::AdnlNodeIdShort &node, const OverlayMemberCertificate *cert); + td::Status validate_peer_certificate(const adnl::AdnlNodeIdShort &node, ton_api::overlay_MemberCertificate *cert); void add_peer(OverlayNode node); void add_peers(std::vector nodes); + void add_peers(const tl_object_ptr &nodes); + void add_peers(const tl_object_ptr &nodes); void del_some_peers(); - void del_peer(adnl::AdnlNodeIdShort id); + void del_peer(const adnl::AdnlNodeIdShort &id); + void del_from_neighbour_list(OverlayPeer *P); + void del_from_neighbour_list(const adnl::AdnlNodeIdShort &id); + void iterate_all_peers(std::function cb); OverlayPeer *get_random_peer(bool only_alive = false); + bool is_root_public_key(const PublicKeyHash &key) const; + bool has_good_peers() const; + size_t neighbours_cnt() const; void finish_dht_query() { if (!next_dht_store_query_) { next_dht_store_query_ = td::Timestamp::in(td::Random::fast(60.0, 100.0)); } - if (frequent_dht_lookup_ && peers_.size() == bad_peers_.size()) { + if (frequent_dht_lookup_ && !has_good_peers()) { next_dht_query_ = td::Timestamp::in(td::Random::fast(6.0, 10.0)); } else { next_dht_query_ = next_dht_store_query_; @@ -327,14 +383,11 @@ class OverlayImpl : public Overlay { OverlayIdFull id_full_; OverlayIdShort overlay_id_; - td::DecTree peers_; td::Timestamp next_dht_query_ = td::Timestamp::in(1.0); td::Timestamp next_dht_store_query_ = td::Timestamp::in(1.0); td::Timestamp update_db_at_; td::Timestamp update_throughput_at_; td::Timestamp last_throughput_update_; - std::set bad_peers_; - adnl::AdnlNodeIdShort next_bad_peer_ = adnl::AdnlNodeIdShort::zero(); std::unique_ptr callback_; @@ -342,7 +395,6 @@ class OverlayImpl : public Overlay { std::map> fec_broadcasts_; std::set delivered_broadcasts_; - std::vector neighbours_; td::ListNode bcast_data_lru_; td::ListNode bcast_fec_lru_; std::queue bcast_lru_; @@ -351,33 +403,6 @@ class OverlayImpl : public Overlay { void bcast_gc(); - static td::uint32 max_data_bcasts() { - return 100; - } - static td::uint32 max_bcasts() { - return 1000; - } - static td::uint32 max_fec_bcasts() { - return 20; - } - static td::uint32 max_sources() { - return 10; - } - static td::uint32 max_neighbours() { - return 5; - } - static td::uint32 max_encryptors() { - return 16; - } - - static td::uint32 max_peers() { - return 20; - } - - static td::uint32 nodes_to_send() { - return 4; - } - static BroadcastHash get_broadcast_hash(adnl::AdnlNodeIdShort &src, td::Bits256 &data_hash) { td::uint8 buf[64]; td::MutableSlice m{buf, 64}; @@ -387,8 +412,7 @@ class OverlayImpl : public Overlay { return td::sha256_bits256(td::Slice(buf, 64)); } - bool public_; - bool semi_public_ = false; + OverlayType overlay_type_; OverlayPrivacyRules rules_; td::string scope_; bool announce_self_ = true; @@ -417,6 +441,25 @@ class OverlayImpl : public Overlay { td::ListNode encryptor_lru_; std::map> encryptor_map_; + + struct PeerList { + struct SlaveKey { + td::int32 expire_at{0}; + adnl::AdnlNodeIdShort node{}; + }; + using SlaveKeys = std::vector; + std::map root_public_keys_; + OverlayMemberCertificate cert_; + std::set bad_peers_; + adnl::AdnlNodeIdShort next_bad_peer_ = adnl::AdnlNodeIdShort::zero(); + td::DecTree peers_; + std::vector neighbours_; + + td::Timestamp local_cert_is_valid_until_; + td::uint32 local_member_flags_{0}; + } peer_list_; + + OverlayOptions opts_; }; } // namespace overlay diff --git a/overlay/overlays.h b/overlay/overlays.h index a7d76fe00..cc569343d 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -18,7 +18,9 @@ */ #pragma once +#include "adnl/adnl-node-id.hpp" #include "adnl/adnl.h" +#include "auto/tl/ton_api.h" #include "dht/dht.h" #include "td/actor/PromiseFuture.h" @@ -33,6 +35,8 @@ namespace ton { namespace overlay { +enum class OverlayType { Public, FixedMemberList, CertificatedMembers }; + class OverlayIdShort { public: OverlayIdShort() { @@ -88,6 +92,10 @@ struct CertificateFlags { enum Values : td::uint32 { AllowFec = 1, Trusted = 2 }; }; +struct OverlayMemberFlags { + enum Values : td::uint32 { DoNotReceiveBroadcasts = 1 }; +}; + enum BroadcastCheckResult { Forbidden = 1, NeedCheck = 2, Allowed = 3 }; inline BroadcastCheckResult broadcast_check_result_max(BroadcastCheckResult l, BroadcastCheckResult r) { @@ -108,7 +116,6 @@ class OverlayPrivacyRules { } BroadcastCheckResult check_rules(PublicKeyHash hash, td::uint32 size, bool is_fec) { - auto it = authorized_keys_.find(hash); if (it == authorized_keys_.end()) { if (size > max_unath_size_) { @@ -158,9 +165,107 @@ class Certificate { td::SharedSlice signature_; }; +class OverlayMemberCertificate { + public: + OverlayMemberCertificate() { + expire_at_ = std::numeric_limits::max(); + } + OverlayMemberCertificate(PublicKey signed_by, td::uint32 flags, td::int32 slot, td::int32 expire_at, + td::BufferSlice signature) + : signed_by_(std::move(signed_by)) + , flags_(flags) + , slot_(slot) + , expire_at_(expire_at) + , signature_(std::move(signature)) { + } + OverlayMemberCertificate(const OverlayMemberCertificate &other) + : signed_by_(other.signed_by_) + , flags_(other.flags_) + , slot_(other.slot_) + , expire_at_(other.expire_at_) + , signature_(other.signature_.clone()) { + } + OverlayMemberCertificate(OverlayMemberCertificate &&) = default; + OverlayMemberCertificate &operator=(OverlayMemberCertificate &&) = default; + OverlayMemberCertificate &operator=(const OverlayMemberCertificate &other) { + signed_by_ = other.signed_by_; + flags_ = other.flags_; + slot_ = other.slot_; + expire_at_ = other.expire_at_; + signature_ = other.signature_.clone(); + return *this; + } + explicit OverlayMemberCertificate(const ton_api::overlay_MemberCertificate *cert); + td::Status check_signature(const adnl::AdnlNodeIdShort &node); + + bool is_expired() const { + return expire_at_ < td::Clocks::system() - 3; + } + + bool is_expired(double cur_time) const { + return expire_at_ < cur_time - 3; + } + + auto tl() const { + return create_tl_object(signed_by_.tl(), flags_, slot_, expire_at_, + signature_.clone_as_buffer_slice()); + } + + const auto &issued_by() const { + return signed_by_; + } + + td::Slice signature() const { + return signature_.as_slice(); + } + + td::BufferSlice to_sign_data(const adnl::AdnlNodeIdShort &node) const { + return ton::create_serialize_tl_object(node.tl(), flags_, slot_, + expire_at_); + } + + bool empty() const { + return signed_by_.empty(); + } + + bool is_newer(const OverlayMemberCertificate &other) const { + return !empty() && expire_at_ > other.expire_at_; + } + + auto slot() const { + return slot_; + } + + auto expire_at() const { + return expire_at_; + } + + void set_signature(td::Slice signature) { + signature_ = td::SharedSlice(signature); + } + void set_signature(td::SharedSlice signature) { + signature_ = std::move(signature); + } + + private: + PublicKey signed_by_; + td::uint32 flags_; + td::int32 slot_; + td::int32 expire_at_ = std::numeric_limits::max(); + td::SharedSlice signature_; +}; + + struct OverlayOptions { bool announce_self_ = true; bool frequent_dht_lookup_ = false; + td::uint32 local_overlay_member_flags_ = 0; + td::int32 max_slaves_in_semiprivate_overlay_ = 5; + td::uint32 max_peers_ = 20; + td::uint32 max_neighbours_ = 5; + td::uint32 nodes_to_send_ = 4; + td::uint32 propagate_broadcast_to_ = 5; + td::uint32 default_permanent_members_flags_ = 0; }; class Overlays : public td::actor::Actor { @@ -208,11 +313,20 @@ class Overlays : public td::actor::Actor { std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) = 0; virtual void create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, - std::unique_ptr callback, OverlayPrivacyRules rules, - td::string scope, OverlayOptions opts) = 0; + std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope, + OverlayOptions opts) = 0; + virtual void create_semiprivate_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::vector nodes, + std::vector root_public_keys, + OverlayMemberCertificate certificate, + std::unique_ptr callback, OverlayPrivacyRules rules, + td::string scope, OverlayOptions opts) = 0; virtual void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules, std::string scope) = 0; + virtual void create_private_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::vector nodes, std::unique_ptr callback, + OverlayPrivacyRules rules, std::string scope, OverlayOptions opts) = 0; virtual void delete_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id) = 0; virtual void send_query(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id, @@ -246,6 +360,13 @@ class Overlays : public td::actor::Actor { virtual void update_certificate(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, PublicKeyHash key, std::shared_ptr cert) = 0; + virtual void update_member_certificate(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + OverlayMemberCertificate certificate) = 0; + virtual void update_root_member_list(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + std::vector nodes, + std::vector root_public_keys, + OverlayMemberCertificate certificate) = 0; + virtual void get_overlay_random_peers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, td::uint32 max_peers, td::Promise> promise) = 0; virtual void get_stats(td::Promise> promise) = 0; diff --git a/test/test-overlay.cpp b/test/test-overlay.cpp new file mode 100644 index 000000000..31db3e780 --- /dev/null +++ b/test/test-overlay.cpp @@ -0,0 +1,450 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. + + Copyright 2017-2020 Telegram Systems LLP +*/ +#include "adnl/adnl-node-id.hpp" +#include "adnl/adnl.h" +#include "adnl/utils.hpp" +#include "adnl/adnl-test-loopback-implementation.h" +#include "auto/tl/ton_api.h" +#include "checksum.h" +#include "common/bitstring.h" +#include "dht/dht.h" +#include "keys/keys.hpp" +#include "overlay-manager.h" +#include "overlay.h" +#include "overlay-id.hpp" +#include "overlay/overlays.h" +#include "td/actor/actor.h" +#include "td/utils/OptionParser.h" +#include "td/utils/Status.h" +#include "td/utils/Time.h" +#include "td/utils/UInt.h" +#include "td/utils/buffer.h" +#include "td/utils/crypto.h" +#include "td/utils/filesystem.h" +#include "td/utils/format.h" +#include "td/utils/port/path.h" +#include "td/utils/Random.h" +#include "td/utils/port/signals.h" +#include "td/utils/port/FileFd.h" +#include "td/utils/overloaded.h" +#include "common/errorlog.h" +#include "tl-utils/common-utils.hpp" +#include "tl/TlObject.h" +#include +#include + +#if TD_DARWIN || TD_LINUX +#include +#endif +#include +#include + +#include + +struct Node { + ton::PrivateKey pk; + ton::PublicKeyHash id; + ton::PublicKey id_full; + ton::adnl::AdnlNodeIdShort adnl_id; + ton::adnl::AdnlNodeIdFull adnl_id_full; + bool can_receive; +}; + +static std::vector root_nodes; +static std::vector slave_nodes; +static std::vector all_nodes; +static td::uint32 total_nodes = 4; +static td::int32 node_slaves_cnt = 3; +static size_t remaining = 0; +static td::Bits256 bcast_hash; + +class Callback : public ton::overlay::Overlays::Callback { + public: + Callback(bool can_receive) : can_receive_(can_receive) { + } + void receive_message(ton::adnl::AdnlNodeIdShort src, ton::overlay::OverlayIdShort overlay_id, + td::BufferSlice data) override { + UNREACHABLE(); + } + void receive_query(ton::adnl::AdnlNodeIdShort src, ton::overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + UNREACHABLE(); + } + void receive_broadcast(ton::PublicKeyHash src, ton::overlay::OverlayIdShort overlay_id, + td::BufferSlice data) override { + CHECK(can_receive_); + CHECK(td::sha256_bits256(data.as_slice()) == bcast_hash); + CHECK(remaining > 0); + remaining--; + } + + private: + bool can_receive_; +}; + +int main(int argc, char *argv[]) { + SET_VERBOSITY_LEVEL(verbosity_INFO); + td::set_default_failure_signal_handler().ensure(); + + std::string db_root_ = "tmp-dir-test-catchain"; + td::rmrf(db_root_).ignore(); + td::mkdir(db_root_).ensure(); + + td::set_default_failure_signal_handler().ensure(); + + td::actor::ActorOwn keyring; + td::actor::ActorOwn network_manager; + td::actor::ActorOwn adnl; + td::actor::ActorOwn overlay_manager; + + td::actor::Scheduler scheduler({7}); + scheduler.run_in_context([&] { + ton::errorlog::ErrorLog::create(db_root_); + keyring = ton::keyring::Keyring::create(db_root_); + network_manager = td::actor::create_actor("test net"); + adnl = ton::adnl::Adnl::create(db_root_, keyring.get()); + overlay_manager = + ton::overlay::Overlays::create(db_root_, keyring.get(), adnl.get(), td::actor::ActorId{}); + td::actor::send_closure(adnl, &ton::adnl::Adnl::register_network_manager, network_manager.get()); + }); + + td::uint32 att = 0; + for (td::uint32 start = att; att < start + 5; att++) { + LOG(WARNING) << "Test #" << att; + root_nodes.resize(total_nodes); + slave_nodes.resize(total_nodes * node_slaves_cnt); + + auto overlay_id_full = + ton::create_serialize_tl_object(td::BufferSlice(PSTRING() << "TEST" << att)); + ton::overlay::OverlayIdFull overlay_id(overlay_id_full.clone()); + auto overlay_id_short = overlay_id.compute_short_id(); + + ton::overlay::OverlayOptions opts; + opts.max_slaves_in_semiprivate_overlay_ = node_slaves_cnt; + opts.default_permanent_members_flags_ = ton::overlay::OverlayMemberFlags::DoNotReceiveBroadcasts; + + ton::overlay::OverlayPrivacyRules rules( + 20 << 20, ton::overlay::CertificateFlags::AllowFec | ton::overlay::CertificateFlags::Trusted, {}); + + std::vector root_keys; + std::vector root_adnl; + + size_t real_members = 0; + + scheduler.run_in_context([&] { + auto addr = ton::adnl::TestLoopbackNetworkManager::generate_dummy_addr_list(); + + for (auto &n : root_nodes) { + bool receive_bcasts = (real_members == 0) ? true : (td::Random::fast_uint32() & 1); + if (receive_bcasts) { + real_members++; + } + n.can_receive = receive_bcasts; + + auto pk1 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub1 = pk1.compute_public_key(); + n.adnl_id_full = ton::adnl::AdnlNodeIdFull{pub1}; + n.adnl_id = ton::adnl::AdnlNodeIdShort{pub1.compute_short_id()}; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk1), true, [](td::Unit) {}); + td::actor::send_closure(adnl, &ton::adnl::Adnl::add_id, ton::adnl::AdnlNodeIdFull{pub1}, addr, + static_cast(0)); + td::actor::send_closure(network_manager, &ton::adnl::TestLoopbackNetworkManager::add_node_id, n.adnl_id, true, + true); + + auto pk2 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub2 = pk2.compute_public_key(); + n.id_full = pub2; + n.id = pub2.compute_short_id(); + n.pk = pk2; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk2), true, [](td::Unit) {}); + + LOG(DEBUG) << "created node " << n.adnl_id << " " << n.id; + + all_nodes.push_back(&n); + root_keys.push_back(n.id); + root_adnl.push_back(n.adnl_id); + } + + for (auto &n : slave_nodes) { + bool receive_bcasts = (real_members == 0) ? true : (td::Random::fast_uint32() & 1); + if (receive_bcasts) { + real_members++; + } + n.can_receive = receive_bcasts; + + auto pk1 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub1 = pk1.compute_public_key(); + n.adnl_id_full = ton::adnl::AdnlNodeIdFull{pub1}; + n.adnl_id = ton::adnl::AdnlNodeIdShort{pub1.compute_short_id()}; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk1), true, [](td::Unit) {}); + td::actor::send_closure(adnl, &ton::adnl::Adnl::add_id, ton::adnl::AdnlNodeIdFull{pub1}, addr, + static_cast(0)); + td::actor::send_closure(network_manager, &ton::adnl::TestLoopbackNetworkManager::add_node_id, n.adnl_id, true, + true); + + auto pk2 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub2 = pk2.compute_public_key(); + n.id_full = pub2; + n.id = pub2.compute_short_id(); + n.pk = pk2; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk2), true, [](td::Unit) {}); + + LOG(DEBUG) << "created node " << n.adnl_id << " " << n.id; + all_nodes.push_back(&n); + } + + for (auto &n1 : all_nodes) { + for (auto &n2 : all_nodes) { + td::actor::send_closure(adnl, &ton::adnl::Adnl::add_peer, n1->adnl_id, n2->adnl_id_full, addr); + } + } + + for (auto &n1 : root_nodes) { + opts.local_overlay_member_flags_ = + (n1.can_receive ? 0 : ton::overlay::OverlayMemberFlags::DoNotReceiveBroadcasts); + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::create_semiprivate_overlay, n1.adnl_id, + ton::overlay::OverlayIdFull(overlay_id_full.clone()), root_adnl, root_keys, + ton::overlay::OverlayMemberCertificate{}, std::make_unique(n1.can_receive), + rules, "", opts); + } + for (size_t i = 0; i < slave_nodes.size(); i++) { + auto &n1 = slave_nodes[i]; + opts.local_overlay_member_flags_ = + (n1.can_receive ? 0 : ton::overlay::OverlayMemberFlags::DoNotReceiveBroadcasts); + + ton::overlay::OverlayMemberCertificate cert(root_nodes[i / node_slaves_cnt].id_full, 0, i % node_slaves_cnt, + 2000000000, td::BufferSlice()); + auto buf = cert.to_sign_data(n1.adnl_id); + auto dec = root_nodes[i / node_slaves_cnt].pk.create_decryptor().move_as_ok(); + auto signature = dec->sign(buf.as_slice()).move_as_ok(); + cert.set_signature(signature.as_slice()); + auto enc = root_nodes[i / node_slaves_cnt].id_full.create_encryptor().move_as_ok(); + enc->check_signature(cert.to_sign_data(n1.adnl_id), cert.signature()).ensure(); + + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::create_semiprivate_overlay, n1.adnl_id, + ton::overlay::OverlayIdFull(overlay_id_full.clone()), root_adnl, root_keys, cert, + std::make_unique(n1.can_receive), rules, "", opts); + } + }); + + td::BufferSlice broadcast(1 << 20); + td::Random::secure_bytes(broadcast.as_slice()); + remaining = real_members; + bcast_hash = td::sha256_bits256(broadcast.as_slice()); + + auto t = td::Timestamp::in(20.0); + while (scheduler.run(1)) { + if (t.is_in_past()) { + break; + } + } + + scheduler.run_in_context([&] { + /*td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::get_stats, + [&](td::Result> R) { + if (R.is_ok()) { + auto res = R.move_as_ok(); + for (auto &o : res->overlays_) { + if (o->overlay_id_ == overlay_id_short.bits256_value()) { + LOG(ERROR) << "NODE " << o->adnl_id_ << " nodes=" << o->nodes_.size(); + for (auto &x : o->stats_) { + LOG(ERROR) << "\t" << x->key_ << " " << x->value_; + } + for (auto &x : o->nodes_) { + LOG(ERROR) << "\t\t" << x->adnl_id_; + } + } + } + } + });*/ + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::send_broadcast_fec_ex, root_nodes[0].adnl_id, + overlay_id_short, root_nodes[0].id, 0, std::move(broadcast)); + }); + + t = td::Timestamp::in(10.0); + while (scheduler.run(1)) { + if (t.is_in_past()) { + break; + } + if (!remaining) { + break; + } + } + + LOG_CHECK(!remaining) << "remaining=" << remaining << " all=" << real_members; + + broadcast = td::BufferSlice(700); + td::Random::secure_bytes(broadcast.as_slice()); + remaining = real_members; + bcast_hash = td::sha256_bits256(broadcast.as_slice()); + scheduler.run_in_context([&] { + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::send_broadcast_ex, root_nodes[0].adnl_id, + overlay_id_short, root_nodes[0].id, 0, std::move(broadcast)); + }); + + t = td::Timestamp::in(10.0); + while (scheduler.run(1)) { + if (t.is_in_past()) { + break; + } + if (!remaining) { + break; + } + } + + LOG_CHECK(!remaining) << "remaining=" << remaining; + + scheduler.run_in_context([&] { + root_nodes.clear(); + slave_nodes.clear(); + all_nodes.clear(); + }); + } + + for (td::uint32 start = att; att < start + 5; att++) { + LOG(WARNING) << "Test #" << att; + root_nodes.resize(total_nodes); + + auto overlay_id_full = + ton::create_serialize_tl_object(td::BufferSlice(PSTRING() << "TEST" << att)); + ton::overlay::OverlayIdFull overlay_id(overlay_id_full.clone()); + auto overlay_id_short = overlay_id.compute_short_id(); + + ton::overlay::OverlayOptions opts; + + ton::overlay::OverlayPrivacyRules rules( + 20 << 20, ton::overlay::CertificateFlags::AllowFec | ton::overlay::CertificateFlags::Trusted, {}); + + std::vector root_keys; + std::vector root_adnl; + + size_t real_members = 0; + + scheduler.run_in_context([&] { + auto addr = ton::adnl::TestLoopbackNetworkManager::generate_dummy_addr_list(); + + for (auto &n : root_nodes) { + real_members++; + auto pk1 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub1 = pk1.compute_public_key(); + n.adnl_id_full = ton::adnl::AdnlNodeIdFull{pub1}; + n.adnl_id = ton::adnl::AdnlNodeIdShort{pub1.compute_short_id()}; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk1), true, [](td::Unit) {}); + td::actor::send_closure(adnl, &ton::adnl::Adnl::add_id, ton::adnl::AdnlNodeIdFull{pub1}, addr, + static_cast(0)); + td::actor::send_closure(network_manager, &ton::adnl::TestLoopbackNetworkManager::add_node_id, n.adnl_id, true, + true); + + auto pk2 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub2 = pk2.compute_public_key(); + n.id_full = pub2; + n.id = pub2.compute_short_id(); + n.pk = pk2; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk2), true, [](td::Unit) {}); + + LOG(DEBUG) << "created node " << n.adnl_id << " " << n.id; + + all_nodes.push_back(&n); + root_keys.push_back(n.id); + root_adnl.push_back(n.adnl_id); + } + + for (auto &n1 : all_nodes) { + for (auto &n2 : all_nodes) { + td::actor::send_closure(adnl, &ton::adnl::Adnl::add_peer, n1->adnl_id, n2->adnl_id_full, addr); + } + } + + for (auto &n1 : root_nodes) { + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::create_private_overlay_ex, n1.adnl_id, + ton::overlay::OverlayIdFull(overlay_id_full.clone()), root_adnl, + std::make_unique(true), rules, "", opts); + } + }); + + auto t = td::Timestamp::in(10.0); + while (scheduler.run(1)) { + if (t.is_in_past()) { + break; + } + } + + td::BufferSlice broadcast(1 << 20); + td::Random::secure_bytes(broadcast.as_slice()); + remaining = real_members; + bcast_hash = td::sha256_bits256(broadcast.as_slice()); + + scheduler.run_in_context([&] { + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::send_broadcast_fec_ex, root_nodes[0].adnl_id, + overlay_id_short, root_nodes[0].id, 0, std::move(broadcast)); + }); + + t = td::Timestamp::in(10.0); + while (scheduler.run(1)) { + if (t.is_in_past()) { + break; + } + if (!remaining) { + break; + } + } + + LOG_CHECK(!remaining) << "remaining=" << remaining; + + broadcast = td::BufferSlice(700); + td::Random::secure_bytes(broadcast.as_slice()); + remaining = real_members; + bcast_hash = td::sha256_bits256(broadcast.as_slice()); + scheduler.run_in_context([&] { + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::send_broadcast_ex, root_nodes[0].adnl_id, + overlay_id_short, root_nodes[0].id, 0, std::move(broadcast)); + }); + + t = td::Timestamp::in(10.0); + while (scheduler.run(1)) { + if (t.is_in_past()) { + break; + } + if (!remaining) { + break; + } + } + + LOG_CHECK(!remaining) << "remaining=" << remaining; + + scheduler.run_in_context([&] { + root_nodes.clear(); + slave_nodes.clear(); + all_nodes.clear(); + }); + } + + td::rmrf(db_root_).ensure(); + std::_Exit(0); + return 0; +} diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index 6deb369ad..d3bdf6924 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -323,10 +323,6 @@ class TestNode : public td::actor::Actor { td::actor::send_closure(id_, &ton::validator::ValidatorManager::sync_complete, td::PromiseCreator::lambda([](td::Unit) {})); } - void update_shard_configuration(td::Ref state, - std::set shards_to_monitor, - std::set temporary_shards) override { - } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } void send_ext_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 89d211a4e..0b7c0173b 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -209,10 +209,15 @@ dht.query node:dht.node = True; ---types--- overlay.node.toSign id:adnl.id.short overlay:int256 version:int = overlay.node.ToSign; +overlay.node.toSignEx id:adnl.id.short overlay:int256 flags:int version:int = overlay.node.ToSign; overlay.node id:PublicKey overlay:int256 version:int signature:bytes = overlay.Node; +overlay.nodeV2 id:PublicKey overlay:int256 flags:int version:int signature:bytes certificate:overlay.MemberCertificate = overlay.NodeV2; overlay.nodes nodes:(vector overlay.node) = overlay.Nodes; +overlay.nodesV2 nodes:(vector overlay.NodeV2) = overlay.NodesV2; +overlay.messageExtra flags:# certificate:flags.0?overlay.MemberCertificate = overlay.MessageExtra; overlay.message overlay:int256 = overlay.Message; +overlay.messageWithExtra overlay:int256 extra:overlay.messageExtra = overlay.Message; //overlay.randomPeers peers:(vector adnl.node) = overlay.RandomPeers; overlay.broadcastList hashes:(vector int256) = overlay.BroadcastList; @@ -225,6 +230,9 @@ overlay.broadcastFec.partId broadcast_hash:int256 data_hash:int256 seqno:int = o overlay.broadcast.toSign hash:int256 date:int = overlay.broadcast.ToSign; +overlay.memberCertificateId node:adnl.id.short flags:int slot:int expire_at:int = overlay.MemberCertificateId; +overlay.memberCertificate issued_by:PublicKey flags:int slot:int expire_at:int signature:bytes = overlay.MemberCertificate; +overlay.emptyMemberCertificate = overlay.MemberCertificate; overlay.certificate issued_by:PublicKey expire_at:int max_size:int signature:bytes = overlay.Certificate; overlay.certificateV2 issued_by:PublicKey expire_at:int max_size:int flags:int signature:bytes = overlay.Certificate; overlay.emptyCertificate = overlay.Certificate; @@ -242,13 +250,16 @@ overlay.broadcastNotFound = overlay.Broadcast; ---functions--- overlay.getRandomPeers peers:overlay.nodes = overlay.Nodes; +overlay.getRandomPeersV2 peers:overlay.NodesV2 = overlay.NodesV2; overlay.query overlay:int256 = True; +overlay.queryWithExtra overlay:int256 extra:overlay.messageExtra = True; overlay.getBroadcast hash:int256 = overlay.Broadcast; overlay.getBroadcastList list:overlay.broadcastList = overlay.BroadcastList; ---types--- +overlay.db.nodesV2 nodes:overlay.nodesV2 = overlay.db.Nodes; overlay.db.nodes nodes:overlay.nodes = overlay.db.Nodes; overlay.db.key.nodes local_id:int256 overlay:int256 = overlay.db.Key; @@ -407,7 +418,7 @@ tonNode.shardPublicOverlayId workchain:int shard:long zero_state_file_hash:int25 tonNode.privateBlockOverlayId zero_state_file_hash:int256 nodes:(vector int256) = tonNode.PrivateBlockOverlayId; tonNode.customOverlayId zero_state_file_hash:int256 name:string nodes:(vector int256) = tonNode.CustomOverlayId; -tonNode.privateBlockOverlayIdV2 zero_state_file_hash:int256 workchain:int shard:long nodes:(vector int256) senders:(vector int256) = tonNode.PrivateBlockOverlayIdV2; +tonNode.fastSyncOverlayId zero_state_file_hash:int256 shard:tonNode.shardId = tonNode.FastSyncOverlayId; tonNode.keyBlocks blocks:(vector tonNode.blockIdExt) incomplete:Bool error:Bool = tonNode.KeyBlocks; @@ -614,7 +625,9 @@ engine.dht.config dht:(vector engine.dht) gc:engine.gc = engine.dht.Config; engine.validator.fullNodeMaster port:int adnl:int256 = engine.validator.FullNodeMaster; engine.validator.fullNodeSlave ip:int port:int adnl:PublicKey = engine.validator.FullNodeSlave; engine.validator.fullNodeConfig ext_messages_broadcast_disabled:Bool = engine.validator.FullNodeConfig; -engine.validator.extraConfig state_serializer_enabled:Bool = engine.validator.ExtraConfig; +engine.validator.fastSyncMemberCertificate adnl_id:int256 certificate:overlay.MemberCertificate = engine.validator.FastSyncMemberCertificate; +engine.validator.extraConfig state_serializer_enabled:Bool fast_sync_member_certificates:(vector engine.validator.fastSyncMemberCertificate) + = engine.validator.ExtraConfig; engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) dht:(vector engine.dht) validators:(vector engine.validator) collators:(vector engine.collator) @@ -688,7 +701,8 @@ engine.validator.shardOverlayStats.neighbour id:string verison_major:int version roundtrip:double unreliability:double has_state:string = engine.validator.shardOverlayStats.Neighbour; engine.validator.shardOverlayStats shard:string mode:string neighbours:(vector engine.validator.shardOverlayStats.neighbour) = engine.validator.ShardOverlayStats; -engine.validator.privateBlockOverlayV2Stats shard:string nodes:(vector int256) senders:(vector int256) created_at:int = engine.validator.PrivateBlockOverlayV2Stats; +engine.validator.fastSyncOverlayStats shard:string validators_adnl:(vector int256) root_public_keys:(vector int256) + member_certificate:overlay.MemberCertificate = engine.validator.FastSyncOverlayStats; engine.validator.onePerfTimerStat time:int min:double avg:double max:double = engine.validator.OnePerfTimerStat; engine.validator.perfTimerStatsByName name:string stats:(vector engine.validator.OnePerfTimerStat) = engine.validator.PerfTimerStatsByName; @@ -772,6 +786,9 @@ engine.validator.delShard shard:tonNode.shardId = engine.validator.Success; engine.validator.setCollatorsList list:engine.validator.collatorsList = engine.validator.Success; engine.validator.showCollatorsList = engine.validator.CollatorsList; +engine.validator.signOverlayMemberCertificate sign_by:int256 adnl_id:int256 slot:int expire_at:int = overlay.MemberCertificate; +engine.validator.importFastSyncMemberCertificate adnl_id:int256 certificate:overlay.MemberCertificate = engine.validator.Success; + ---types--- storage.pong = storage.Pong; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 549cc00f526752b96b46747475e1a7a67f35b12e..f574600b1a30e44839e4e7837f8240ee3de48fa7 100644 GIT binary patch delta 1958 zcmZ`(|4$or7`_j?gQ5&OZ0U6^D_=?khh0;qgC*0>eaT30NFcb0%k7{y)=*k#Zwp;! z2709EJ}D{(PXv2KSxS6muOIuy^`v&glD5-QG$h75WKo7Hn?t=~ zGjCE$9H~Obw+rq9d&CHPa*^R$V?~wkkMj+V_2XLSmH>gd0-L3x$oBg)t68y_*e50Y zy#4dPZ+k3x?+1qEs0jDU9Z*u;V4aw{^j#^rOr<7@m9MW z31dly*EWL%7%g(DhkQ#!?cBMaBQxqJ%@Zd%)P}&l{zi3p+{+nKO-@u6yyYmWO3`Qs%%E@BVf~Gv9ahM|jfpV~=HkC(VeX6#MyJh6tCyHe zk!Va11K|)1_SS*_F=viOBC;@eswWie75b%k48*%GNPk@d>2`K!*;+dEdGPPAH!wC; z`7i2@7zQD5Mh@QoWjlm^s$yfIhzyBtCoC+~fcHY(JK0^3 zg5|yL*==-Bq=7*>p6dW@!!xirDy1?XC&wgfMr#RRTpO)G<8SCU8yrcQ^XGHHsNck5 z9+(y@S@e^#^eS;GX@OXp@n^Pldun{+Je_+$((U#}P(G1JuxhbfEkolm)0yglGggXTVbA8VF%5!W>fMGDM ztECJ*O?D|)TZx;+x8&JUGQuS*tz@v4e2`4+CJrYz{n>S^SDEf2Hv7Si&Wm!xPa07P zPFF%n_=zhy*+s@!ZuyG5_XVZBn^=8q#sOXHp^S-GjS$EEyY)pe?HUU`aTuAg!<~yR zF59by7qC&%-%Ua+XZDU+d8J(-<;B|GFbwLagal&9Lpjb?R~ZvXp!UJ{Pd+NAj1QfX dqVZLeG~fL9l^lz(xU9;dmq}%kJx+eD{vV$q!Gr(+ delta 591 zcmaDegY7~q8}Fmp`c@23Ait3}Tb#vB%ph=bjkv{T2MGaY(W^@j_GqW(rDx`)>Xjwt zWTqsRS&g;>7e+#>oen#3z4I z(wXd|th0HEG7F=q)VYqU0x)GMNqT znM1XKZ}S1;3yiEs3uds%PmVN|Wezif$W1VHVP#yg`Ks0O46xHTH!MzJg?Qq{3I$G( zZ*CqF@?o4TxYB&Gfei;lQNU^qPD`wQ(<>;-EK4j&b;`+4&i03RDac^j7EnM<4!E5IW^AsxbHj4`3o*tLR$Xw&l;r1u!W5eyJX6AqKx#l9V4W@~ z$7nn`A&qsqr4pkvPH%{?fOKpRkYTJ*W$fI3){#+KW_w2*;{ulLEGdi$EYnY9Fh*=& JkjZ#P2>@6x-t+(f diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 450dbe496..7691be0f3 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1432,3 +1432,51 @@ td::Status ShowCollatorsListQuery::receive(td::BufferSlice data) { } return td::Status::OK(); } + +td::Status SignOverlayMemberCertificateQuery::run() { + TRY_RESULT_ASSIGN(key_hash_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(slot_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(expire_at_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status SignOverlayMemberCertificateQuery::send() { + auto b = ton::create_serialize_tl_object( + key_hash_, adnl_id_, slot_, expire_at_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status SignOverlayMemberCertificateQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + TRY_STATUS(td::write_file(file_name_, data)); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status ImportFastSyncMemberCertificateQuery::run() { + TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status ImportFastSyncMemberCertificateQuery::send() { + TRY_RESULT(data, td::read_file(file_name_)); + TRY_RESULT(certificate, ton::fetch_tl_object(data, true)); + auto b = ton::create_serialize_tl_object( + adnl_id_, std::move(certificate)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ImportFastSyncMemberCertificateQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 06514d091..3ca962142 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1443,3 +1443,55 @@ class ShowCollatorsListQuery : public Query { return get_name(); } }; + +class SignOverlayMemberCertificateQuery : public Query { + public: + SignOverlayMemberCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "signoverlaymembercertificate"; + } + static std::string get_help() { + return "signoverlaymembercertificate \tsign overlay member " + "certificate for (hex) with (hex) in slot , valid until , " + "save to "; + } + std::string name() const override { + return get_name(); + } + + private: + td::Bits256 key_hash_; + td::Bits256 adnl_id_; + int slot_; + ton::UnixTime expire_at_; + std::string file_name_; +}; + +class ImportFastSyncMemberCertificateQuery : public Query { + public: + ImportFastSyncMemberCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "importfastsyncmembercertificate"; + } + static std::string get_help() { + return "importfastsyncmembercertificate \timport member certificate for fast sync overlay " + "for (hex) from "; + } + std::string name() const override { + return get_name(); + } + + private: + td::Bits256 adnl_id_; + std::string file_name_; +}; diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index f90be5a58..b13591b2f 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -157,6 +157,8 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 4fe74b944..e9779c2e7 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -163,6 +163,13 @@ Config::Config(const ton::ton_api::engine_validator_config &config) { } if (config.extraconfig_) { state_serializer_enabled = config.extraconfig_->state_serializer_enabled_; + for (auto &f : config.extraconfig_->fast_sync_member_certificates_) { + ton::adnl::AdnlNodeIdShort adnl_id{f->adnl_id_}; + ton::overlay::OverlayMemberCertificate certificate{f->certificate_.get()}; + if (!certificate.empty() && !certificate.is_expired()) { + fast_sync_member_certificates.emplace_back(adnl_id, std::move(certificate)); + } + } } else { state_serializer_enabled = true; } @@ -252,9 +259,15 @@ ton::tl_object_ptr Config::tl() const { } ton::tl_object_ptr extra_config_obj = {}; - if (!state_serializer_enabled) { + if (!state_serializer_enabled || !fast_sync_member_certificates.empty()) { // Non-default values - extra_config_obj = ton::create_tl_object(state_serializer_enabled); + extra_config_obj = ton::create_tl_object(); + extra_config_obj->state_serializer_enabled_ = state_serializer_enabled; + for (const auto &[adnl_id, certificate] : fast_sync_member_certificates) { + extra_config_obj->fast_sync_member_certificates_.push_back( + ton::create_tl_object(adnl_id.bits256_value(), + certificate.tl())); + } } std::vector> liteserver_vec; @@ -1995,6 +2008,10 @@ void ValidatorEngine::start_full_node() { td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_collator_adnl_id, ton::adnl::AdnlNodeIdShort(c.adnl_id)); } + for (auto &x : config_.fast_sync_member_certificates) { + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_fast_sync_member_certificate, + x.first, x.second); + } load_custom_overlays_config(); } else { started_full_node(); @@ -4140,6 +4157,95 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delShard }); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signOverlayMemberCertificate &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + ton::PublicKeyHash public_key_hash{query.sign_by_}; + ton::adnl::AdnlNodeIdShort adnl_id{query.adnl_id_}; + int slot = query.slot_; + int expire_at = query.expire_at_; + td::actor::send_closure( + keyring_, &ton::keyring::Keyring::get_public_key, public_key_hash, + [=, promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + ton::overlay::OverlayMemberCertificate certificate{R.move_as_ok(), 0, slot, expire_at, td::BufferSlice{}}; + if (certificate.is_expired()) { + promise.set_value( + create_control_query_error(td::Status::Error(ton::ErrorCode::error, "certificate is expired"))); + return; + } + td::BufferSlice to_sign = certificate.to_sign_data(adnl_id); + td::actor::send_closure(keyring_, &ton::keyring::Keyring::sign_message, public_key_hash, std::move(to_sign), + [certificate = std::move(certificate), + promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + certificate.set_signature(R.move_as_ok()); + promise.set_value(ton::serialize_tl_object(certificate.tl(), true)); + }); + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importFastSyncMemberCertificate &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + ton::adnl::AdnlNodeIdShort adnl_id{query.adnl_id_}; + ton::overlay::OverlayMemberCertificate certificate{query.certificate_.get()}; + if (certificate.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "certificate is empty"))); + return; + } + td::Status S = certificate.check_signature(adnl_id); + if (S.is_error()) { + promise.set_value(create_control_query_error(std::move(S))); + return; + } + for (auto &old_cert : config_.fast_sync_member_certificates) { + if (old_cert.first == adnl_id && old_cert.second.issued_by() == certificate.issued_by() && + old_cert.second.expire_at() == certificate.expire_at() && old_cert.second.slot() == certificate.slot()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "duplicate certificate"))); + return; + } + } + + config_.fast_sync_member_certificates.emplace_back(adnl_id, certificate); + write_config([=, promise = std::move(promise), full_node = full_node_.get()](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + if (!full_node.empty()) { + td::actor::send_closure(full_node, &ton::validator::fullnode::FullNode::import_fast_sync_member_certificate, + adnl_id, std::move(certificate)); + } + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index c4aeafbbc..08ee2ff6f 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -101,6 +101,8 @@ struct Config { std::vector shards_to_monitor; bool state_serializer_enabled = true; + std::vector> + fast_sync_member_certificates; void decref(ton::PublicKeyHash key); void incref(ton::PublicKeyHash key) { @@ -524,6 +526,10 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_showCollatorsList &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_signOverlayMemberCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_importFastSyncMemberCertificate &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index b9e0024a6..2d950b464 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -153,8 +153,8 @@ set(FULL_NODE_SOURCE full-node-private-overlay.cpp full-node-serializer.hpp full-node-serializer.cpp - full-node-private-overlay-v2.hpp - full-node-private-overlay-v2.cpp + full-node-fast-sync-overlays.hpp + full-node-fast-sync-overlays.cpp net/download-block.hpp net/download-block.cpp diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp new file mode 100644 index 000000000..7a1c4209c --- /dev/null +++ b/validator/full-node-fast-sync-overlays.cpp @@ -0,0 +1,416 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ + +#include "full-node-fast-sync-overlays.hpp" + +#include "checksum.h" +#include "ton/ton-tl.hpp" +#include "common/delay.h" +#include "td/utils/JsonBuilder.h" +#include "tl/tl_json.h" +#include "auto/tl/ton_api_json.h" +#include "full-node-serializer.hpp" + +namespace ton::validator::fullnode { + +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { + process_block_broadcast(src, query); +} + +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query) { + process_block_broadcast(src, query); +} + +void FullNodeFastSyncOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { + auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); + if (B.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << B.move_as_error(); + return; + } + VLOG(FULL_NODE_DEBUG) << "Received block broadcast in fast sync overlay from " << src << ": " + << B.ok().block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok()); +} + +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query) { + BlockIdExt block_id = create_block_id(query.block_->block_); + VLOG(FULL_NODE_DEBUG) << "Received newShardBlockBroadcast in fast sync overlay from " << src << ": " + << block_id.to_str(); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block, block_id, + query.block_->cc_seqno_, std::move(query.block_->data_)); +} + +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressed &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeFastSyncOverlay::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { + BlockIdExt block_id; + CatchainSeqno cc_seqno; + td::uint32 validator_set_hash; + td::BufferSlice data; + auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data, + overlay::Overlays::max_fec_broadcast_size()); + if (S.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << S; + return; + } + if (data.size() > FullNode::max_block_size()) { + VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; + return; + } + if (td::sha256_bits256(data.as_slice()) != block_id.file_hash) { + VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; + return; + } + VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in fast sync overlay from " << src << ": " << block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, cc_seqno, + validator_set_hash, std::move(data)); +} + +void FullNodeFastSyncOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { + auto B = fetch_tl_object(std::move(broadcast), true); + if (B.is_error()) { + return; + } + + ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto &obj) { Self->process_broadcast(src, obj); }); +} + +void FullNodeFastSyncOverlay::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { + if (!inited_) { + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending newShardBlockBroadcast in fast sync overlay: " << block_id.to_str(); + auto B = create_serialize_tl_object( + create_tl_object(create_tl_block_id(block_id), cc_seqno, std::move(data))); + if (B.size() <= overlay::Overlays::max_simple_broadcast_size()) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), 0, std::move(B)); + } else { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); + } +} + +void FullNodeFastSyncOverlay::send_broadcast(BlockBroadcast broadcast) { + if (!inited_) { + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending block broadcast in fast sync overlay (with compression): " + << broadcast.block_id.to_str(); + auto B = serialize_block_broadcast(broadcast, true); // compression_enabled = true + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error(); + return; + } + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); +} + +void FullNodeFastSyncOverlay::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) { + if (!inited_) { + return; + } + auto B = + serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error(); + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate in fast sync overlay (with compression): " << block_id.to_str(); + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); +} + +void FullNodeFastSyncOverlay::start_up() { + auto X = create_hash_tl_object(zero_state_file_hash_, create_tl_shard_id(shard_)); + td::BufferSlice b{32}; + b.as_slice().copy_from(as_slice(X)); + overlay_id_full_ = overlay::OverlayIdFull{std::move(b)}; + overlay_id_ = overlay_id_full_.compute_short_id(); + + try_init(); +} + +void FullNodeFastSyncOverlay::try_init() { + // Sometimes adnl id is added to validator engine later (or not at all) + td::actor::send_closure( + adnl_, &adnl::Adnl::check_id_exists, local_id_, [SelfId = actor_id(this)](td::Result R) { + if (R.is_ok() && R.ok()) { + td::actor::send_closure(SelfId, &FullNodeFastSyncOverlay::init); + } else { + delay_action([SelfId]() { td::actor::send_closure(SelfId, &FullNodeFastSyncOverlay::try_init); }, + td::Timestamp::in(30.0)); + } + }); +} + +void FullNodeFastSyncOverlay::init() { + LOG(INFO) << "Creating fast sync overlay for shard " << shard_.to_str() << ", adnl_id=" << local_id_; + class Callback : public overlay::Overlays::Callback { + public: + void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { + } + void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + } + void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { + td::actor::send_closure(node_, &FullNodeFastSyncOverlay::receive_broadcast, src, std::move(data)); + } + void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + } + void get_stats_extra(td::Promise promise) override { + td::actor::send_closure(node_, &FullNodeFastSyncOverlay::get_stats_extra, std::move(promise)); + } + explicit Callback(td::actor::ActorId node) : node_(node) { + } + + private: + td::actor::ActorId node_; + }; + + overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(), + overlay::CertificateFlags::AllowFec | overlay::CertificateFlags::Trusted, + {}}; + std::string scope = PSTRING() << R"({ "type": "fast-sync", "shard_id": )" << shard_.shard + << ", \"workchain_id\": " << shard_.workchain << " }"; + overlay::OverlayOptions options; + bool is_validator = std::find(current_validators_adnl_.begin(), current_validators_adnl_.end(), local_id_) != + current_validators_adnl_.end(); + if (!shard_.is_masterchain()) { + options.default_permanent_members_flags_ = overlay::OverlayMemberFlags::DoNotReceiveBroadcasts; + options.local_overlay_member_flags_ = is_validator ? overlay::OverlayMemberFlags::DoNotReceiveBroadcasts : 0; + } + options.max_slaves_in_semiprivate_overlay_ = 100000; // TODO: set lower limit (high limit for testing) + td::actor::send_closure(overlays_, &overlay::Overlays::create_semiprivate_overlay, local_id_, + overlay_id_full_.clone(), current_validators_adnl_, root_public_keys_, member_certificate_, + std::make_unique(actor_id(this)), rules, std::move(scope), options); + + inited_ = true; +} + +void FullNodeFastSyncOverlay::tear_down() { + if (inited_) { + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, local_id_, overlay_id_); + } +} + +void FullNodeFastSyncOverlay::set_validators(std::vector root_public_keys, + std::vector current_validators_adnl) { + root_public_keys_ = std::move(root_public_keys); + current_validators_adnl_ = std::move(current_validators_adnl); + if (inited_) { + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, local_id_, overlay_id_); + init(); + } +} + +void FullNodeFastSyncOverlay::set_member_certificate(overlay::OverlayMemberCertificate member_certificate) { + member_certificate_ = std::move(member_certificate); + if (inited_) { + td::actor::send_closure(overlays_, &overlay::Overlays::update_member_certificate, local_id_, overlay_id_, + member_certificate_); + } +} + +void FullNodeFastSyncOverlay::get_stats_extra(td::Promise promise) { + auto res = create_tl_object(); + res->shard_ = shard_.to_str(); + for (const auto &x : current_validators_adnl_) { + res->validators_adnl_.push_back(x.bits256_value()); + } + for (const auto &x : root_public_keys_) { + res->root_public_keys_.push_back(x.bits256_value()); + } + res->member_certificate_ = member_certificate_.tl(); + promise.set_result(td::json_encode(td::ToJson(*res), true)); +} + +td::actor::ActorId FullNodeFastSyncOverlays::choose_overlay(ShardIdFull shard) { + for (auto &p : id_to_overlays_) { + auto &overlays = p.second.overlays_; + ShardIdFull cur_shard = shard; + while (true) { + auto it = overlays.find(cur_shard); + if (it != overlays.end()) { + return it->second.get(); + } + if (cur_shard.pfx_len() == 0) { + break; + } + cur_shard = shard_parent(cur_shard); + } + } + return {}; +} + +void FullNodeFastSyncOverlays::update_overlays(td::Ref state, + std::set my_adnl_ids, + std::set monitoring_shards, + const FileHash &zero_state_file_hash, + const td::actor::ActorId &keyring, + const td::actor::ActorId &adnl, + const td::actor::ActorId &overlays, + const td::actor::ActorId &validator_manager, + const td::actor::ActorId &full_node) { + monitoring_shards.insert(ShardIdFull{masterchainId}); + std::set all_shards; + all_shards.insert(ShardIdFull{masterchainId}); + td::uint32 monitor_min_split = state->monitor_min_split_depth(basechainId); + for (td::uint64 i = 0; i < (1ULL << monitor_min_split); ++i) { + all_shards.insert(ShardIdFull{basechainId, (i * 2 + 1) << (63 - monitor_min_split)}); + } + + // Remove overlays for removed adnl ids and shards + for (auto it = id_to_overlays_.begin(); it != id_to_overlays_.end();) { + if (my_adnl_ids.count(it->first)) { + auto &overlays_info = it->second; + ; + auto ¤t_shards = overlays_info.is_validator_ ? all_shards : monitoring_shards; + for (auto it2 = overlays_info.overlays_.begin(); it2 != overlays_info.overlays_.end();) { + if (current_shards.count(it2->first)) { + ++it2; + } else { + it2 = overlays_info.overlays_.erase(it2); + } + } + ++it; + } else { + it = id_to_overlays_.erase(it); + } + } + + // On new keyblock - update validator set + bool updated_validators = false; + if (!last_key_block_seqno_ || last_key_block_seqno_.value() != state->last_key_block_id().seqno()) { + updated_validators = true; + last_key_block_seqno_ = state->last_key_block_id().seqno(); + root_public_keys_.clear(); + current_validators_adnl_.clear(); + // Previous, current and next validator sets + for (int i = -1; i <= 1; ++i) { + auto val_set = state->get_total_validator_set(i); + if (val_set.is_null()) { + continue; + } + for (const ValidatorDescr &val : val_set->export_vector()) { + PublicKeyHash public_key_hash = ValidatorFullId{val.key}.compute_short_id(); + root_public_keys_.push_back(public_key_hash); + if (i == 0) { + current_validators_adnl_.emplace_back(val.addr.is_zero() ? public_key_hash.bits256_value() : val.addr); + } + } + } + std::sort(root_public_keys_.begin(), root_public_keys_.end()); + root_public_keys_.erase(std::unique(root_public_keys_.begin(), root_public_keys_.end()), root_public_keys_.end()); + std::sort(current_validators_adnl_.begin(), current_validators_adnl_.end()); + + for (auto &[local_id, overlays_info] : id_to_overlays_) { + overlays_info.is_validator_ = + std::binary_search(current_validators_adnl_.begin(), current_validators_adnl_.end(), local_id); + for (auto &[_, overlay] : overlays_info.overlays_) { + td::actor::send_closure(overlay, &FullNodeFastSyncOverlay::set_validators, root_public_keys_, + current_validators_adnl_); + } + } + } + + // Cleanup outdated certificates + double now = td::Clocks::system(); + for (auto &[_, certificates] : member_certificates_) { + certificates.erase(std::remove_if(certificates.begin(), certificates.end(), + [&](const overlay::OverlayMemberCertificate &certificate) { + return certificate.is_expired(now); + }), + certificates.end()); + } + + for (adnl::AdnlNodeIdShort local_id : my_adnl_ids) { + bool is_new = !id_to_overlays_.count(local_id); + auto &overlays_info = id_to_overlays_[local_id]; + // Update is_validator and current_certificate + if (is_new) { + overlays_info.is_validator_ = + std::binary_search(current_validators_adnl_.begin(), current_validators_adnl_.end(), local_id); + } + bool changed_certificate = false; + // Check if certificate is outdated or no longer authorized by current root keys + if (!overlays_info.current_certificate_.empty() && overlays_info.current_certificate_.is_expired(now)) { + changed_certificate = true; + overlays_info.current_certificate_ = {}; + } + if (!overlays_info.current_certificate_.empty() && updated_validators && + !std::binary_search(root_public_keys_.begin(), root_public_keys_.end(), + overlays_info.current_certificate_.issued_by().compute_short_id())) { + changed_certificate = true; + overlays_info.current_certificate_ = {}; + } + if (overlays_info.current_certificate_.empty()) { + auto it = member_certificates_.find(local_id); + if (it != member_certificates_.end()) { + for (const overlay::OverlayMemberCertificate &certificate : it->second) { + if (std::binary_search(root_public_keys_.begin(), root_public_keys_.end(), + certificate.issued_by().compute_short_id())) { + changed_certificate = true; + overlays_info.current_certificate_ = it->second.front(); + break; + } + } + } + } + + // Remove if it is not authorized + if (!overlays_info.is_validator_ && overlays_info.current_certificate_.empty()) { + id_to_overlays_.erase(local_id); + continue; + } + + // Update shard overlays + auto ¤t_shards = overlays_info.is_validator_ ? all_shards : monitoring_shards; + for (ShardIdFull shard_id : current_shards) { + auto &overlay = overlays_info.overlays_[shard_id]; + if (overlay.empty()) { + overlay = td::actor::create_actor( + PSTRING() << "FastSyncOv" << shard_id.to_str(), local_id, shard_id, zero_state_file_hash, + root_public_keys_, current_validators_adnl_, overlays_info.current_certificate_, keyring, adnl, overlays, + validator_manager, full_node); + } else if (changed_certificate) { + td::actor::send_closure(overlay, &FullNodeFastSyncOverlay::set_member_certificate, + overlays_info.current_certificate_); + } + } + } +} + +void FullNodeFastSyncOverlays::add_member_certificate(adnl::AdnlNodeIdShort local_id, + overlay::OverlayMemberCertificate member_certificate) { + if (member_certificate.empty() || member_certificate.is_expired()) { + return; + } + member_certificates_[local_id].push_back(std::move(member_certificate)); + // Overlays will be updated in the next update_overlays +} + +} // namespace ton::validator::fullnode diff --git a/validator/full-node-private-overlay-v2.hpp b/validator/full-node-fast-sync-overlays.hpp similarity index 59% rename from validator/full-node-private-overlay-v2.hpp rename to validator/full-node-fast-sync-overlays.hpp index abb88fb43..30cb5dde8 100644 --- a/validator/full-node-private-overlay-v2.hpp +++ b/validator/full-node-fast-sync-overlays.hpp @@ -20,7 +20,7 @@ namespace ton::validator::fullnode { -class FullNodePrivateOverlayV2 : public td::actor::Actor { +class FullNodeFastSyncOverlay : public td::actor::Actor { public: void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast& query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed& query); @@ -28,9 +28,9 @@ class FullNodePrivateOverlayV2 : public td::actor::Actor { void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast& query); - void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); - void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); - void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast& query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed& query); + void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast& query); template void process_broadcast(PublicKeyHash, T&) { @@ -46,26 +46,26 @@ class FullNodePrivateOverlayV2 : public td::actor::Actor { void start_up() override; void tear_down() override; - void destroy() { - stop(); - } - - FullNodePrivateOverlayV2(adnl::AdnlNodeIdShort local_id, ShardIdFull shard, std::vector nodes, - std::vector senders, FileHash zero_state_file_hash, - td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId rldp2, - td::actor::ActorId overlays, - td::actor::ActorId validator_manager, - td::actor::ActorId full_node) + void set_validators(std::vector root_public_keys, + std::vector current_validators_adnl); + void set_member_certificate(overlay::OverlayMemberCertificate member_certificate); + + FullNodeFastSyncOverlay(adnl::AdnlNodeIdShort local_id, ShardIdFull shard, FileHash zero_state_file_hash, + std::vector root_public_keys, + std::vector current_validators_adnl, + overlay::OverlayMemberCertificate member_certificate, + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId overlays, + td::actor::ActorId validator_manager, + td::actor::ActorId full_node) : local_id_(local_id) , shard_(shard) - , nodes_(std::move(nodes)) - , senders_(std::move(senders)) + , root_public_keys_(std::move(root_public_keys)) + , current_validators_adnl_(std::move(current_validators_adnl)) + , member_certificate_(std::move(member_certificate)) , zero_state_file_hash_(zero_state_file_hash) , keyring_(keyring) , adnl_(adnl) - , rldp_(rldp) - , rldp2_(rldp2) , overlays_(overlays) , validator_manager_(validator_manager) , full_node_(full_node) { @@ -74,14 +74,13 @@ class FullNodePrivateOverlayV2 : public td::actor::Actor { private: adnl::AdnlNodeIdShort local_id_; ShardIdFull shard_; - std::vector nodes_; - std::vector senders_; + std::vector root_public_keys_; + std::vector current_validators_adnl_; + overlay::OverlayMemberCertificate member_certificate_; FileHash zero_state_file_hash_; td::actor::ActorId keyring_; td::actor::ActorId adnl_; - td::actor::ActorId rldp_; - td::actor::ActorId rldp2_; td::actor::ActorId overlays_; td::actor::ActorId validator_manager_; td::actor::ActorId full_node_; @@ -96,29 +95,30 @@ class FullNodePrivateOverlayV2 : public td::actor::Actor { void get_stats_extra(td::Promise promise); }; -class FullNodePrivateBlockOverlaysV2 { +class FullNodeFastSyncOverlays { public: - td::actor::ActorId choose_overlay(ShardIdFull shard); + td::actor::ActorId choose_overlay(ShardIdFull shard); void update_overlays(td::Ref state, std::set my_adnl_ids, - const FileHash& zero_state_file_hash, const td::actor::ActorId& keyring, - const td::actor::ActorId& adnl, const td::actor::ActorId& rldp, - const td::actor::ActorId& rldp2, + std::set monitoring_shards, const FileHash& zero_state_file_hash, + const td::actor::ActorId& keyring, const td::actor::ActorId& adnl, const td::actor::ActorId& overlays, const td::actor::ActorId& validator_manager, const td::actor::ActorId& full_node); - void destroy_overlays(); + void add_member_certificate(adnl::AdnlNodeIdShort local_id, overlay::OverlayMemberCertificate member_certificate); private: struct Overlays { - struct ShardOverlay { - td::actor::ActorOwn overlay_; - std::vector nodes_, senders_; - bool is_sender_ = false; - }; - std::map overlays_; + std::map> overlays_; + overlay::OverlayMemberCertificate current_certificate_; + bool is_validator_{false}; }; std::map id_to_overlays_; // local_id -> overlays + std::map> member_certificates_; + + td::optional last_key_block_seqno_; + std::vector root_public_keys_; + std::vector current_validators_adnl_; }; } // namespace ton::validator::fullnode diff --git a/validator/full-node-private-overlay-v2.cpp b/validator/full-node-private-overlay-v2.cpp deleted file mode 100644 index 20e9cc55f..000000000 --- a/validator/full-node-private-overlay-v2.cpp +++ /dev/null @@ -1,384 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . -*/ - -#include "full-node-private-overlay-v2.hpp" - -#include "checksum.h" -#include "ton/ton-tl.hpp" -#include "common/delay.h" -#include "td/utils/JsonBuilder.h" -#include "tl/tl_json.h" -#include "auto/tl/ton_api_json.h" -#include "full-node-serializer.hpp" - -namespace ton::validator::fullnode { - -void FullNodePrivateOverlayV2::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { - process_block_broadcast(src, query); -} - -void FullNodePrivateOverlayV2::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query) { - process_block_broadcast(src, query); -} - -void FullNodePrivateOverlayV2::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { - auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); - if (B.is_error()) { - LOG(DEBUG) << "dropped broadcast: " << B.move_as_error(); - return; - } - VLOG(FULL_NODE_DEBUG) << "Received block broadcast in private overlay from " << src << ": " - << B.ok().block_id.to_str(); - td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok()); -} - -void FullNodePrivateOverlayV2::process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query) { - BlockIdExt block_id = create_block_id(query.block_->block_); - VLOG(FULL_NODE_DEBUG) << "Received newShardBlockBroadcast in private overlay from " << src << ": " - << block_id.to_str(); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block, block_id, - query.block_->cc_seqno_, std::move(query.block_->data_)); -} - -void FullNodePrivateOverlayV2::process_broadcast(PublicKeyHash src, - ton_api::tonNode_newBlockCandidateBroadcast &query) { - process_block_candidate_broadcast(src, query); -} - -void FullNodePrivateOverlayV2::process_broadcast(PublicKeyHash src, - ton_api::tonNode_newBlockCandidateBroadcastCompressed &query) { - process_block_candidate_broadcast(src, query); -} - -void FullNodePrivateOverlayV2::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { - BlockIdExt block_id; - CatchainSeqno cc_seqno; - td::uint32 validator_set_hash; - td::BufferSlice data; - auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data, - overlay::Overlays::max_fec_broadcast_size()); - if (S.is_error()) { - LOG(DEBUG) << "dropped broadcast: " << S; - return; - } - if (data.size() > FullNode::max_block_size()) { - VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; - return; - } - if (td::sha256_bits256(data.as_slice()) != block_id.file_hash) { - VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; - return; - } - VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in private overlay from " << src << ": " << block_id.to_str(); - td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, cc_seqno, - validator_set_hash, std::move(data)); -} - -void FullNodePrivateOverlayV2::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { - auto B = fetch_tl_object(std::move(broadcast), true); - if (B.is_error()) { - return; - } - - ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto &obj) { Self->process_broadcast(src, obj); }); -} - -void FullNodePrivateOverlayV2::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, - td::BufferSlice data) { - if (!inited_) { - return; - } - VLOG(FULL_NODE_DEBUG) << "Sending newShardBlockBroadcast in private overlay: " << block_id.to_str(); - auto B = create_serialize_tl_object( - create_tl_object(create_tl_block_id(block_id), cc_seqno, std::move(data))); - if (B.size() <= overlay::Overlays::max_simple_broadcast_size()) { - td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_, - local_id_.pubkey_hash(), 0, std::move(B)); - } else { - td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, - local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); - } -} - -void FullNodePrivateOverlayV2::send_broadcast(BlockBroadcast broadcast) { - if (!inited_) { - return; - } - VLOG(FULL_NODE_DEBUG) << "Sending block broadcast in private overlay (with compression): " - << broadcast.block_id.to_str(); - auto B = serialize_block_broadcast(broadcast, true); // compression_enabled = true - if (B.is_error()) { - VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error(); - return; - } - td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, - local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); -} - -void FullNodePrivateOverlayV2::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, - td::uint32 validator_set_hash, td::BufferSlice data) { - if (!inited_) { - return; - } - auto B = - serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled - if (B.is_error()) { - VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error(); - return; - } - VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate in private overlay (with compression): " << block_id.to_str(); - td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, - local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); -} - -void FullNodePrivateOverlayV2::start_up() { - std::sort(nodes_.begin(), nodes_.end()); - nodes_.erase(std::unique(nodes_.begin(), nodes_.end()), nodes_.end()); - - std::vector nodes, senders; - for (const adnl::AdnlNodeIdShort &id : nodes_) { - nodes.push_back(id.bits256_value()); - } - for (const adnl::AdnlNodeIdShort &id : senders_) { - senders.push_back(id.bits256_value()); - } - auto X = create_hash_tl_object( - zero_state_file_hash_, shard_.workchain, shard_.shard, std::move(nodes), std::move(senders)); - td::BufferSlice b{32}; - b.as_slice().copy_from(as_slice(X)); - overlay_id_full_ = overlay::OverlayIdFull{std::move(b)}; - overlay_id_ = overlay_id_full_.compute_short_id(); - - try_init(); -} - -void FullNodePrivateOverlayV2::try_init() { - // Sometimes adnl id is added to validator engine later (or not at all) - td::actor::send_closure( - adnl_, &adnl::Adnl::check_id_exists, local_id_, [SelfId = actor_id(this)](td::Result R) { - if (R.is_ok() && R.ok()) { - td::actor::send_closure(SelfId, &FullNodePrivateOverlayV2::init); - } else { - delay_action([SelfId]() { td::actor::send_closure(SelfId, &FullNodePrivateOverlayV2::try_init); }, - td::Timestamp::in(30.0)); - } - }); -} - -void FullNodePrivateOverlayV2::init() { - LOG(FULL_NODE_INFO) << "Creating private block overlay for shard " << shard_.to_str() << ", adnl_id=" << local_id_ - << " : " << nodes_.size() << " nodes"; - class Callback : public overlay::Overlays::Callback { - public: - void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { - } - void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, - td::Promise promise) override { - } - void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { - td::actor::send_closure(node_, &FullNodePrivateOverlayV2::receive_broadcast, src, std::move(data)); - } - void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, - td::Promise promise) override { - } - void get_stats_extra(td::Promise promise) override { - td::actor::send_closure(node_, &FullNodePrivateOverlayV2::get_stats_extra, std::move(promise)); - } - Callback(td::actor::ActorId node) : node_(node) { - } - - private: - td::actor::ActorId node_; - }; - - std::map authorized_keys; - for (const adnl::AdnlNodeIdShort &sender : senders_) { - authorized_keys[sender.pubkey_hash()] = overlay::Overlays::max_fec_broadcast_size(); - } - overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(), 0, std::move(authorized_keys)}; - std::string scope = PSTRING() << R"({ "type": "private-blocks-v2", "shard_id": )" << shard_.shard - << ", \"workchain_id\": " << shard_.workchain << " }"; - td::actor::send_closure(overlays_, &overlay::Overlays::create_private_overlay, local_id_, overlay_id_full_.clone(), - nodes_, std::make_unique(actor_id(this)), rules, std::move(scope)); - - td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); - td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_id_); - inited_ = true; -} - -void FullNodePrivateOverlayV2::tear_down() { - if (inited_) { - td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, local_id_, overlay_id_); - } -} - -void FullNodePrivateOverlayV2::get_stats_extra(td::Promise promise) { - auto res = create_tl_object(); - res->shard_ = shard_.to_str(); - for (const auto &x : nodes_) { - res->nodes_.push_back(x.bits256_value()); - } - for (const auto &x : senders_) { - res->senders_.push_back(x.bits256_value()); - } - res->created_at_ = created_at_; - promise.set_result(td::json_encode(td::ToJson(*res), true)); -} - -td::actor::ActorId FullNodePrivateBlockOverlaysV2::choose_overlay(ShardIdFull shard) { - for (auto &p : id_to_overlays_) { - auto &overlays = p.second.overlays_; - ShardIdFull cur_shard = shard; - while (true) { - auto it = overlays.find(cur_shard); - if (it != overlays.end() && it->second.is_sender_) { - return it->second.overlay_.get(); - } - if (cur_shard.pfx_len() == 0) { - break; - } - cur_shard = shard_parent(cur_shard); - } - } - return {}; -} - -void FullNodePrivateBlockOverlaysV2::update_overlays( - td::Ref state, std::set my_adnl_ids, const FileHash &zero_state_file_hash, - const td::actor::ActorId &keyring, const td::actor::ActorId &adnl, - const td::actor::ActorId &rldp, const td::actor::ActorId &rldp2, - const td::actor::ActorId &overlays, - const td::actor::ActorId &validator_manager, - const td::actor::ActorId &full_node) { - if (my_adnl_ids.empty()) { - id_to_overlays_.clear(); - return; - } - auto collators = state->get_collator_config(true); - auto all_validators = state->get_total_validator_set(0); - - struct OverlayInfo { - std::vector nodes, senders; - }; - std::map overlay_infos; - - // Masterchain overlay: all validators + collators - OverlayInfo &mc_overlay = overlay_infos[ShardIdFull(masterchainId)]; - for (const auto &x : all_validators->export_vector()) { - td::Bits256 addr = x.addr.is_zero() ? ValidatorFullId(x.key).compute_short_id().bits256_value() : x.addr; - mc_overlay.nodes.emplace_back(addr); - mc_overlay.senders.emplace_back(addr); - } - for (const auto &x : collators.collator_nodes) { - mc_overlay.nodes.emplace_back(x.adnl_id); - } - - // Shard overlays: validators of the shard + collators of the shard - // See ValidatorManagerImpl::update_shards - std::set new_shards; - for (auto &v : state->get_shards()) { - ShardIdFull shard = v->shard(); - if (shard.is_masterchain()) { - continue; - } - if (v->before_split()) { - ShardIdFull l_shard{shard.workchain, shard_child(shard.shard, true)}; - ShardIdFull r_shard{shard.workchain, shard_child(shard.shard, false)}; - new_shards.insert(l_shard); - new_shards.insert(r_shard); - } else if (v->before_merge()) { - ShardIdFull p_shard{shard.workchain, shard_parent(shard.shard)}; - new_shards.insert(p_shard); - } else { - new_shards.insert(shard); - } - } - for (ShardIdFull shard : new_shards) { - auto val_set = state->get_validator_set(shard); - td::uint32 min_split = state->monitor_min_split_depth(shard.workchain); - OverlayInfo &overlay = - overlay_infos[shard_prefix_length(shard) <= min_split ? shard : shard_prefix(shard, min_split)]; - for (const auto &x : val_set->export_vector()) { - td::Bits256 addr = x.addr.is_zero() ? ValidatorFullId(x.key).compute_short_id().bits256_value() : x.addr; - overlay.nodes.emplace_back(addr); - overlay.senders.emplace_back(addr); - } - } - for (auto &p : overlay_infos) { - ShardIdFull shard = p.first; - OverlayInfo &overlay = p.second; - if (!shard.is_masterchain()) { - for (const auto &collator : collators.collator_nodes) { - if (shard_intersects(collator.shard, shard)) { - overlay.nodes.emplace_back(collator.adnl_id); - } - } - } - - std::sort(overlay.nodes.begin(), overlay.nodes.end()); - overlay.nodes.erase(std::unique(overlay.nodes.begin(), overlay.nodes.end()), overlay.nodes.end()); - std::sort(overlay.senders.begin(), overlay.senders.end()); - overlay.senders.erase(std::unique(overlay.senders.begin(), overlay.senders.end()), overlay.senders.end()); - } - - std::map old_private_block_overlays = std::move(id_to_overlays_); - id_to_overlays_.clear(); - - for (const auto &p : overlay_infos) { - ShardIdFull shard = p.first; - const OverlayInfo &new_overlay_info = p.second; - for (adnl::AdnlNodeIdShort local_id : new_overlay_info.nodes) { - if (!my_adnl_ids.count(local_id)) { - continue; - } - Overlays::ShardOverlay &new_overlay = id_to_overlays_[local_id].overlays_[shard]; - Overlays::ShardOverlay &old_overlay = old_private_block_overlays[local_id].overlays_[shard]; - if (!old_overlay.overlay_.empty() && old_overlay.nodes_ == new_overlay_info.nodes && - old_overlay.senders_ == new_overlay_info.senders) { - new_overlay = std::move(old_overlay); - old_overlay = {}; - } else { - new_overlay.nodes_ = new_overlay_info.nodes; - new_overlay.senders_ = new_overlay_info.senders; - new_overlay.is_sender_ = std::binary_search(new_overlay.senders_.begin(), new_overlay.senders_.end(), local_id); - new_overlay.overlay_ = td::actor::create_actor( - PSTRING() << "BlocksPrivateOverlay" << shard.to_str(), local_id, shard, new_overlay.nodes_, - new_overlay.senders_, zero_state_file_hash, keyring, adnl, rldp, rldp2, overlays, validator_manager, - full_node); - } - } - } - - // Delete old overlays, but not immediately - for (auto &p : old_private_block_overlays) { - for (auto &x : p.second.overlays_) { - if (x.second.overlay_.empty()) { - continue; - } - td::actor::ActorId id = x.second.overlay_.release(); - delay_action([id = std::move(id)]() { td::actor::send_closure(id, &FullNodePrivateOverlayV2::destroy); }, - td::Timestamp::in(30.0)); - } - } -} - -void FullNodePrivateBlockOverlaysV2::destroy_overlays() { - id_to_overlays_.clear(); -} - - -} // namespace ton::validator::fullnode diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index d0dbf1a64..5aa69fe99 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -1419,9 +1419,9 @@ td::actor::ActorOwn FullNodeShard::create( td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, td::actor::ActorId full_node, FullNodeShardMode mode) { - return td::actor::create_actor("tonnode", shard, local_id, adnl_id, zero_state_file_hash, config, - keyring, adnl, rldp, rldp2, overlays, validator_manager, client, - full_node, mode); + return td::actor::create_actor(PSTRING() << "tonnode" << shard.to_str(), shard, local_id, adnl_id, + zero_state_file_hash, config, keyring, adnl, rldp, rldp2, overlays, + validator_manager, client, full_node, mode); } } // namespace fullnode diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 335578cfb..8e84a4d45 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -198,8 +198,8 @@ void FullNodeImpl::initial_read_complete(BlockHandle top_handle) { td::actor::send_closure(it->second.actor, &FullNodeShard::set_handle, top_handle, std::move(P)); } -void FullNodeImpl::update_shard_configuration(td::Ref state, std::set shards_to_monitor, - std::set temporary_shards) { +void FullNodeImpl::on_new_masterchain_block(td::Ref state, std::set shards_to_monitor, + std::set temporary_shards) { CHECK(shards_to_monitor.count(ShardIdFull(masterchainId))); std::set new_shards; std::map new_active; @@ -232,7 +232,7 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s set_active(cut_shard(shard), FullNodeShardMode::active_temp); } - auto info_set_mode = [&](ShardIdFull shard, ShardInfo& info, FullNodeShardMode mode) { + auto info_set_mode = [&](ShardIdFull shard, ShardInfo &info, FullNodeShardMode mode) { if (info.mode == mode) { return; } @@ -251,7 +251,7 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s info.exists = true; } - for (auto& p : shards_) { + for (auto &p : shards_) { ShardIdFull shard = p.first; ShardInfo &info = p.second; info.exists = new_shards.count(shard); @@ -259,7 +259,7 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s info_set_mode(shard, info, it == new_active.end() ? FullNodeShardMode::inactive : it->second); } - for (const auto& s : new_active) { + for (const auto &s : new_active) { info_set_mode(s.first, shards_[s.first], s.second); } @@ -276,21 +276,25 @@ void FullNodeImpl::update_shard_configuration(td::Ref state, s } } - std::set my_adnl_ids; - for (const auto &p : local_collator_nodes_) { - my_adnl_ids.insert(p.first); - } - for (auto key : local_keys_) { - auto it = current_validators_.find(key); - if (it != current_validators_.end()) { - my_adnl_ids.insert(it->second); + if (!use_old_private_overlays_) { + std::set my_adnl_ids; + my_adnl_ids.insert(adnl_id_); + for (const auto &p : local_collator_nodes_) { + my_adnl_ids.insert(p.first); + } + for (auto key : local_keys_) { + auto it = current_validators_.find(key); + if (it != current_validators_.end()) { + my_adnl_ids.insert(it->second); + } } - } - if (use_old_private_overlays_) { - private_block_overlays_v2_.destroy_overlays(); - } else { - private_block_overlays_v2_.update_overlays(state, std::move(my_adnl_ids), zero_state_file_hash_, keyring_, adnl_, - rldp_, rldp2_, overlays_, validator_manager_, actor_id(this)); + std::set monitoring_shards; + for (ShardIdFull shard : shards_to_monitor) { + monitoring_shards.insert(cut_shard(shard)); + } + fast_sync_overlays_.update_overlays(state, std::move(my_adnl_ids), std::move(monitoring_shards), + zero_state_file_hash_, keyring_, adnl_, overlays_, validator_manager_, + actor_id(this)); } } @@ -348,9 +352,9 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_shard_block_info, block_id, cc_seqno, data.clone()); } - auto private_overlay = private_block_overlays_v2_.choose_overlay(ShardIdFull(masterchainId)); + auto private_overlay = fast_sync_overlays_.choose_overlay(ShardIdFull(masterchainId)); if (!private_overlay.empty()) { - td::actor::send_closure(private_overlay, &FullNodePrivateOverlayV2::send_shard_block_info, block_id, cc_seqno, + td::actor::send_closure(private_overlay, &FullNodeFastSyncOverlay::send_shard_block_info, block_id, cc_seqno, data.clone()); } td::actor::send_closure(shard, &FullNodeShard::send_shard_block_info, block_id, cc_seqno, std::move(data)); @@ -368,9 +372,9 @@ void FullNodeImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_se td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_block_candidate, block_id, cc_seqno, validator_set_hash, data.clone()); } - auto private_overlay = private_block_overlays_v2_.choose_overlay(block_id.shard_full()); + auto private_overlay = fast_sync_overlays_.choose_overlay(block_id.shard_full()); if (!private_overlay.empty()) { - td::actor::send_closure(private_overlay, &FullNodePrivateOverlayV2::send_block_candidate, block_id, cc_seqno, + td::actor::send_closure(private_overlay, &FullNodeFastSyncOverlay::send_block_candidate, block_id, cc_seqno, validator_set_hash, data.clone()); } if (broadcast_block_candidates_in_public_overlay_) { @@ -393,9 +397,9 @@ void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, bool custom_overlays td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_broadcast, broadcast.clone()); } - auto private_overlay = private_block_overlays_v2_.choose_overlay(broadcast.block_id.shard_full()); + auto private_overlay = fast_sync_overlays_.choose_overlay(broadcast.block_id.shard_full()); if (!private_overlay.empty()) { - td::actor::send_closure(private_overlay, &FullNodePrivateOverlayV2::send_broadcast, broadcast.clone()); + td::actor::send_closure(private_overlay, &FullNodeFastSyncOverlay::send_broadcast, broadcast.clone()); } td::actor::send_closure(shard, &FullNodeShard::send_broadcast, std::move(broadcast)); } @@ -645,9 +649,9 @@ void FullNodeImpl::start_up() { void initial_read_complete(BlockHandle handle) override { td::actor::send_closure(id_, &FullNodeImpl::initial_read_complete, handle); } - void update_shard_configuration(td::Ref state, std::set shards_to_monitor, - std::set temporary_shards) override { - td::actor::send_closure(id_, &FullNodeImpl::update_shard_configuration, std::move(state), + void on_new_masterchain_block(td::Ref state, std::set shards_to_monitor, + std::set temporary_shards) override { + td::actor::send_closure(id_, &FullNodeImpl::on_new_masterchain_block, std::move(state), std::move(shards_to_monitor), std::move(temporary_shards)); } void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) override { diff --git a/validator/full-node.h b/validator/full-node.h index 0545dfb98..fa57e4f1c 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -92,6 +92,9 @@ class FullNode : public td::actor::Actor { virtual void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::BufferSlice data) = 0; + virtual void import_fast_sync_member_certificate(adnl::AdnlNodeIdShort local_id, + overlay::OverlayMemberCertificate cert) = 0; + static constexpr td::uint32 max_block_size() { return 4 << 20; } diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 33ccf2cd1..a1f20b9d2 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -24,7 +24,7 @@ #include "interfaces/proof.h" #include "interfaces/shard.h" #include "full-node-private-overlay.hpp" -#include "full-node-private-overlay-v2.hpp" +#include "full-node-fast-sync-overlays.hpp" #include #include @@ -47,9 +47,8 @@ class FullNodeImpl : public FullNode { void add_collator_adnl_id(adnl::AdnlNodeIdShort id) override; void del_collator_adnl_id(adnl::AdnlNodeIdShort id) override; - void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, - td::uint32 expiry_at, td::uint32 max_size, - td::Promise promise) override; + void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, td::uint32 expiry_at, + td::uint32 max_size, td::Promise promise) override; void import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) override; @@ -60,7 +59,7 @@ class FullNodeImpl : public FullNode { void add_custom_overlay(CustomOverlayParams params, td::Promise promise) override; void del_custom_overlay(std::string name, td::Promise promise) override; - void update_shard_configuration(td::Ref state, std::set shards_to_monitor, + void on_new_masterchain_block(td::Ref state, std::set shards_to_monitor, std::set temporary_shards); void sync_completed(); @@ -95,6 +94,11 @@ class FullNodeImpl : public FullNode { void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::BufferSlice data) override; + void import_fast_sync_member_certificate(adnl::AdnlNodeIdShort local_id, + overlay::OverlayMemberCertificate cert) override { + fast_sync_overlays_.add_member_certificate(local_id, std::move(cert)); + } + void start_up() override; FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, @@ -145,12 +149,13 @@ class FullNodeImpl : public FullNode { FullNodeConfig config_; // Private overlays: - // Old overlays - one overlay for all validators - // New overlays (v2) - overlay per shard (monitor_min_split depth). - bool use_old_private_overlays_ = false; // TODO: set from config + // Old overlays - one private overlay for all validators + // New overlays (fast sync overlays) - semiprivate overlay per shard (monitor_min_split depth) + // for validators and authorized nodes + bool use_old_private_overlays_ = false; // TODO: set from config or something std::map> private_block_overlays_; bool broadcast_block_candidates_in_public_overlay_ = false; - FullNodePrivateBlockOverlaysV2 private_block_overlays_v2_; + FullNodeFastSyncOverlays fast_sync_overlays_; struct CustomOverlayInfo { CustomOverlayParams params_; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 3b44e894f..abe6d5202 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -57,6 +57,8 @@ using ValidateCandidateResult = td::Variant; class ValidatorManager : public ValidatorManagerInterface { public: + virtual void init_last_masterchain_state(td::Ref state) { + } virtual void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) = 0; virtual void get_cell_db_reader(td::Promise> promise) = 0; @@ -140,7 +142,6 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_shard_client_state(bool from_db, td::Promise promise) = 0; - virtual void update_shard_configuration(td::Ref state, std::set shards_to_monitor) = 0; virtual void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) = 0; virtual void get_async_serializer_state(td::Promise promise) = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 5153896c0..dce07eac7 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -272,8 +272,6 @@ class ValidatorManagerImpl : public ValidatorManager { void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; - void update_shard_configuration(td::Ref state, std::set shards_to_monitor) override { - } void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) override { UNREACHABLE(); diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 428e2c1a9..d192494b5 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -344,9 +344,6 @@ class ValidatorManagerImpl : public ValidatorManager { void get_shard_client_state(bool from_db, td::Promise promise) override { UNREACHABLE(); } - void update_shard_configuration(td::Ref state, std::set shards_to_monitor) override { - UNREACHABLE(); - } void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) override { UNREACHABLE(); diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index 3c40f1f67..39205f80c 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -268,6 +268,7 @@ void ValidatorManagerMasterchainReiniter::downloaded_masterchain_state(td::Refreceived_state()); CHECK(handle_->is_applied()); LOG(INFO) << "downloaded masterchain state"; + td::actor::send_closure(manager_, &ValidatorManager::init_last_masterchain_state, state_); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::downloaded_all_shards); diff --git a/validator/manager.cpp b/validator/manager.cpp index 499ad7491..ac24c8ed5 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1740,6 +1740,7 @@ void ValidatorManagerImpl::send_top_shard_block_description(td::Refblock_id().shard_full(), desc->catchain_seqno()}] = desc; callback_->send_shard_block_info(desc->block_id(), desc->catchain_seqno(), desc->serialize()); + add_shard_block_description(desc); } } @@ -1826,6 +1827,14 @@ void ValidatorManagerImpl::start_up() { alarm_timestamp().relax(check_waiters_at_); } +void ValidatorManagerImpl::init_last_masterchain_state(td::Ref state) { + if (last_masterchain_state_.not_null()) { + return; + } + last_masterchain_state_ = std::move(state); + update_shard_overlays(); +} + void ValidatorManagerImpl::started(ValidatorManagerInitResult R) { CHECK(R.handle); CHECK(R.state.not_null()); @@ -1873,6 +1882,7 @@ void ValidatorManagerImpl::started(ValidatorManagerInitResult R) { } }); td::actor::send_closure(db_, &Db::get_persistent_state_descriptions, std::move(Q)); + update_shard_overlays(); } void ValidatorManagerImpl::read_gc_list(std::vector list) { @@ -2049,6 +2059,7 @@ void ValidatorManagerImpl::new_masterchain_block() { } } + update_shard_overlays(); update_shards(); update_shard_blocks(); @@ -2081,6 +2092,26 @@ void ValidatorManagerImpl::new_masterchain_block() { } } +void ValidatorManagerImpl::update_shard_overlays() { + CHECK(last_masterchain_state_.not_null()); + std::set shards_to_monitor; + shards_to_monitor.insert(ShardIdFull{masterchainId}); + std::set workchains; + for (const auto& shard : last_masterchain_state_->get_shards()) { + workchains.insert(shard->shard().workchain); + if (opts_->need_monitor(shard->shard(),last_masterchain_state_)) { + shards_to_monitor.insert(shard->shard()); + } + } + for (const auto &[wc, desc] : last_masterchain_state_->get_workchain_list()) { + if (!workchains.count(wc) && desc->active && + opts_->need_monitor(ShardIdFull{wc, shardIdAll}, last_masterchain_state_)) { + shards_to_monitor.insert(ShardIdFull{wc, shardIdAll}); + } + } + callback_->on_new_masterchain_block(last_masterchain_state_, std::move(shards_to_monitor), extra_active_shards_); +} + void ValidatorManagerImpl::update_shards() { if ((last_masterchain_state_->rotated_all_shards() || last_masterchain_seqno_ == 0) && opts_->get_last_fork_masterchain_seqno() <= last_masterchain_seqno_) { @@ -2836,11 +2867,6 @@ void ValidatorManagerImpl::get_shard_client_state(bool from_db, td::Promise state, - std::set shards_to_monitor) { - callback_->update_shard_configuration(std::move(state), std::move(shards_to_monitor), extra_active_shards_); -} - void ValidatorManagerImpl::update_async_serializer_state(AsyncSerializerState state, td::Promise promise) { td::actor::send_closure(db_, &Db::update_async_serializer_state, std::move(state), std::move(promise)); } diff --git a/validator/manager.hpp b/validator/manager.hpp index 2c15eb112..bb4a1513b 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -319,6 +319,7 @@ class ValidatorManagerImpl : public ValidatorManager { std::vector perf_timer_stats; void new_masterchain_block(); + void update_shard_overlays(); void update_shards(); void update_shard_blocks(); void written_destroyed_validator_sessions(std::vector> groups); @@ -521,7 +522,6 @@ class ValidatorManagerImpl : public ValidatorManager { void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; - void update_shard_configuration(td::Ref state, std::set shards_to_monitor) override; void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) override; void get_async_serializer_state(td::Promise promise) override; @@ -557,6 +557,7 @@ class ValidatorManagerImpl : public ValidatorManager { void finished_wait_data(BlockHandle handle, td::Result> R); void start_up() override; + void init_last_masterchain_state(td::Ref state) override; void started(ValidatorManagerInitResult result); void read_gc_list(std::vector list); diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index 303e6333a..bb9da158f 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -70,19 +70,13 @@ void ShardClient::got_init_handle_from_db(BlockHandle handle) { } void ShardClient::got_init_state_from_db(td::Ref state) { - masterchain_state_ = std::move(state); - build_shard_overlays(); - masterchain_state_.clear(); - saved_to_db(); } void ShardClient::start_up_init_mode() { - build_shard_overlays(); - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); - td::actor::send_closure(SelfId, &ShardClient::applied_all_shards); + td::actor::send_closure(SelfId, &ShardClient::apply_all_shards); }); td::MultiPromise mp; @@ -166,7 +160,6 @@ void ShardClient::download_masterchain_state() { void ShardClient::got_masterchain_block_state(td::Ref state) { masterchain_state_ = std::move(state); - build_shard_overlays(); if (started_) { apply_all_shards(); } @@ -189,7 +182,9 @@ void ShardClient::apply_all_shards() { ig.add_promise(std::move(P)); auto vec = masterchain_state_->get_shards(); + std::set workchains; for (auto &shard : vec) { + workchains.insert(shard->shard().workchain); if (opts_->need_monitor(shard->shard(), masterchain_state_)) { auto Q = td::PromiseCreator::lambda([SelfId = actor_id(this), promise = ig.get_promise(), shard = shard->shard()](td::Result> R) mutable { @@ -203,6 +198,21 @@ void ShardClient::apply_all_shards() { shard_client_priority(), td::Timestamp::in(1500), std::move(Q)); } } + for (const auto &[wc, desc] : masterchain_state_->get_workchain_list()) { + if (!workchains.count(wc) && desc->active && opts_->need_monitor(ShardIdFull{wc, shardIdAll}, masterchain_state_)) { + auto Q = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = ig.get_promise(), wc](td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix(PSTRING() << "workchain " << wc << ": ")); + } else { + td::actor::send_closure(SelfId, &ShardClient::downloaded_shard_state, R.move_as_ok(), std::move(promise)); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, + BlockIdExt{wc, shardIdAll, 0, desc->zerostate_root_hash, desc->zerostate_file_hash}, + shard_client_priority(), td::Timestamp::in(1500), std::move(Q)); + } + } } void ShardClient::downloaded_shard_state(td::Ref state, td::Promise promise) { @@ -223,7 +233,6 @@ void ShardClient::new_masterchain_block_notification(BlockHandle handle, td::Ref masterchain_block_handle_ = std::move(handle); masterchain_state_ = std::move(state); waiting_ = false; - build_shard_overlays(); apply_all_shards(); } @@ -244,42 +253,6 @@ void ShardClient::get_processed_masterchain_block_id(td::Promise pro } } -void ShardClient::build_shard_overlays() { - std::set new_shards_to_monitor; - std::set workchains; - auto cur_time = masterchain_state_->get_unix_time(); - new_shards_to_monitor.insert(ShardIdFull(masterchainId)); - for (const auto &info : masterchain_state_->get_shards()) { - auto shard = info->shard(); - workchains.insert(shard.workchain); - if (opts_->need_monitor(shard, masterchain_state_)) { - new_shards_to_monitor.insert(shard); - } - } - - std::vector new_workchains; - for (const auto &wpair : masterchain_state_->get_workchain_list()) { - ton::WorkchainId wc = wpair.first; - const block::WorkchainInfo *winfo = wpair.second.get(); - auto shard = ShardIdFull(wc); - if (workchains.count(wc) == 0 && winfo->active && winfo->enabled_since <= cur_time && - opts_->need_monitor(shard, masterchain_state_)) { - new_shards_to_monitor.insert(shard); - if (shards_to_monitor_.count(shard) == 0) { - new_workchains.push_back(BlockIdExt(wc, shardIdAll, 0, winfo->zerostate_root_hash, winfo->zerostate_file_hash)); - } - } - } - - td::actor::send_closure(manager_, &ValidatorManager::update_shard_configuration, masterchain_state_, - new_shards_to_monitor); - shards_to_monitor_ = std::move(new_shards_to_monitor); - for (BlockIdExt block_id : new_workchains) { - td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, block_id, shard_client_priority(), - td::Timestamp::in(60.0), [](td::Result>) {}); - } -} - void ShardClient::force_update_shard_client(BlockHandle handle, td::Promise promise) { CHECK(!init_mode_); CHECK(!started_); @@ -310,7 +283,6 @@ void ShardClient::force_update_shard_client_ex(BlockHandle handle, td::Ref promise_; - std::set shards_to_monitor_ = {ShardIdFull(masterchainId)}; - public: ShardClient(td::Ref opts, BlockHandle masterchain_block_handle, td::Ref masterchain_state, td::actor::ActorId manager, @@ -64,8 +62,6 @@ class ShardClient : public td::actor::Actor { return 2; } - void build_shard_overlays(); - void start_up() override; void start_up_init_mode(); void start_up_init_mode_finished(); diff --git a/validator/validator.h b/validator/validator.h index 819a46ea9..8cf9368b3 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -169,9 +169,9 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual ~Callback() = default; virtual void initial_read_complete(BlockHandle top_masterchain_blocks) = 0; - virtual void update_shard_configuration(td::Ref state, - std::set shards_to_monitor, - std::set temporary_shards) = 0; + virtual void on_new_masterchain_block(td::Ref state, + std::set shards_to_monitor, + std::set temporary_shards) = 0; virtual void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; From 5f9efc7adde9498268fda85db39a9543563d08de Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 15 Jul 2024 12:07:46 +0300 Subject: [PATCH 100/388] Fix compilation error --- create-hardfork/create-hardfork.cpp | 4 ++++ test/test-ton-collator.cpp | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index 32aa01c32..e2ec3f626 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -236,6 +236,10 @@ class HardforkCreator : public td::actor::Actor { td::actor::send_closure(id_, &ton::validator::ValidatorManager::sync_complete, td::PromiseCreator::lambda([](td::Unit) {})); } + void on_new_masterchain_block(td::Ref state, + std::set shards_to_monitor, + std::set temporary_shards) override { + } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } void send_ext_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index d3bdf6924..afebe11fe 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -323,6 +323,10 @@ class TestNode : public td::actor::Actor { td::actor::send_closure(id_, &ton::validator::ValidatorManager::sync_complete, td::PromiseCreator::lambda([](td::Unit) {})); } + void on_new_masterchain_block(td::Ref state, + std::set shards_to_monitor, + std::set temporary_shards) override { + } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } void send_ext_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { From f518c104446e4ebc5c77d8ed68e61ed51cc81149 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 15 Jul 2024 16:21:01 +0300 Subject: [PATCH 101/388] Improve selection neighbours in overlay --- overlay/overlay-peers.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index c61e49652..a21768613 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -216,7 +216,7 @@ void OverlayImpl::add_peer(OverlayNode node) { CHECK(X); if (peer_list_.neighbours_.size() < max_neighbours() && - !(X->get_node()->flags() & OverlayMemberFlags::DoNotReceiveBroadcasts)) { + !(X->get_node()->flags() & OverlayMemberFlags::DoNotReceiveBroadcasts) && X->get_id() != local_id_) { peer_list_.neighbours_.push_back(X->get_id()); X->set_neighbour(true); } @@ -622,9 +622,17 @@ std::vector OverlayImpl::get_neighbours(td::uint32 max_si return peer_list_.neighbours_; } else { std::vector vec; + std::vector ul; for (td::uint32 i = 0; i < max_size; i++) { - vec.push_back( - peer_list_.neighbours_[td::Random::fast(0, static_cast(peer_list_.neighbours_.size()) - 1)]); + td::uint32 t = td::Random::fast(0, static_cast(peer_list_.neighbours_.size()) - 1 - i); + td::uint32 j; + for (j = 0; j < i; j++) { + if (ul[j] <= t) { + t++; + } + } + ul.emplace(ul.begin() + j, t); + vec.push_back(peer_list_.neighbours_[t]); } return vec; } From d81b69e38bce8e3c9fd57a663be1f8509a91c1b8 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 17 Jul 2024 15:34:45 +0300 Subject: [PATCH 102/388] Use previous and next validators in fast sync overlays --- validator/full-node-fast-sync-overlays.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index 7a1c4209c..b88a55446 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -318,14 +318,14 @@ void FullNodeFastSyncOverlays::update_overlays(td::Ref state, for (const ValidatorDescr &val : val_set->export_vector()) { PublicKeyHash public_key_hash = ValidatorFullId{val.key}.compute_short_id(); root_public_keys_.push_back(public_key_hash); - if (i == 0) { - current_validators_adnl_.emplace_back(val.addr.is_zero() ? public_key_hash.bits256_value() : val.addr); - } + current_validators_adnl_.emplace_back(val.addr.is_zero() ? public_key_hash.bits256_value() : val.addr); } } std::sort(root_public_keys_.begin(), root_public_keys_.end()); root_public_keys_.erase(std::unique(root_public_keys_.begin(), root_public_keys_.end()), root_public_keys_.end()); std::sort(current_validators_adnl_.begin(), current_validators_adnl_.end()); + current_validators_adnl_.erase(std::unique(current_validators_adnl_.begin(), current_validators_adnl_.end()), + current_validators_adnl_.end()); for (auto &[local_id, overlays_info] : id_to_overlays_) { overlays_info.is_validator_ = From 36b98e8aa022b0b7a3cb59ce169c5264aad338c1 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 17 Jul 2024 16:52:56 +0300 Subject: [PATCH 103/388] Fix compilation error in shard-client.cpp --- validator/shard-client.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index bb9da158f..d9cdcf7e3 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -200,14 +200,14 @@ void ShardClient::apply_all_shards() { } for (const auto &[wc, desc] : masterchain_state_->get_workchain_list()) { if (!workchains.count(wc) && desc->active && opts_->need_monitor(ShardIdFull{wc, shardIdAll}, masterchain_state_)) { - auto Q = td::PromiseCreator::lambda( - [SelfId = actor_id(this), promise = ig.get_promise(), wc](td::Result> R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error_prefix(PSTRING() << "workchain " << wc << ": ")); - } else { - td::actor::send_closure(SelfId, &ShardClient::downloaded_shard_state, R.move_as_ok(), std::move(promise)); - } - }); + auto Q = td::PromiseCreator::lambda([SelfId = actor_id(this), promise = ig.get_promise(), + workchain = wc](td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix(PSTRING() << "workchain " << workchain << ": ")); + } else { + td::actor::send_closure(SelfId, &ShardClient::downloaded_shard_state, R.move_as_ok(), std::move(promise)); + } + }); td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, BlockIdExt{wc, shardIdAll, 0, desc->zerostate_root_hash, desc->zerostate_file_hash}, shard_client_priority(), td::Timestamp::in(1500), std::move(Q)); From 5743dbf335816483cc796623ce267318795a7125 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 18 Jul 2024 12:43:55 +0300 Subject: [PATCH 104/388] Changes in fast sync overlays Join all overlays (not only monitoring shards) in order to be able to send broadcasts to other overlays --- validator/full-node-fast-sync-overlays.cpp | 48 ++++++++++++++-------- validator/full-node-fast-sync-overlays.hpp | 5 ++- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index b88a55446..d641215f2 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -201,8 +201,8 @@ void FullNodeFastSyncOverlay::init() { current_validators_adnl_.end(); if (!shard_.is_masterchain()) { options.default_permanent_members_flags_ = overlay::OverlayMemberFlags::DoNotReceiveBroadcasts; - options.local_overlay_member_flags_ = is_validator ? overlay::OverlayMemberFlags::DoNotReceiveBroadcasts : 0; } + options.local_overlay_member_flags_ = receive_broadcasts_ ? 0 : overlay::OverlayMemberFlags::DoNotReceiveBroadcasts; options.max_slaves_in_semiprivate_overlay_ = 100000; // TODO: set lower limit (high limit for testing) td::actor::send_closure(overlays_, &overlay::Overlays::create_semiprivate_overlay, local_id_, overlay_id_full_.clone(), current_validators_adnl_, root_public_keys_, member_certificate_, @@ -235,6 +235,17 @@ void FullNodeFastSyncOverlay::set_member_certificate(overlay::OverlayMemberCerti } } +void FullNodeFastSyncOverlay::set_receive_broadcasts(bool value) { + if (value == receive_broadcasts_) { + return; + } + receive_broadcasts_ = value; + if (inited_) { + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, local_id_, overlay_id_); + init(); + } +} + void FullNodeFastSyncOverlay::get_stats_extra(td::Promise promise) { auto res = create_tl_object(); res->shard_ = shard_.to_str(); @@ -278,19 +289,21 @@ void FullNodeFastSyncOverlays::update_overlays(td::Ref state, monitoring_shards.insert(ShardIdFull{masterchainId}); std::set all_shards; all_shards.insert(ShardIdFull{masterchainId}); - td::uint32 monitor_min_split = state->monitor_min_split_depth(basechainId); - for (td::uint64 i = 0; i < (1ULL << monitor_min_split); ++i) { - all_shards.insert(ShardIdFull{basechainId, (i * 2 + 1) << (63 - monitor_min_split)}); + for (const auto& desc : state->get_shards()) { + ShardIdFull shard = desc->shard(); + td::uint32 monitor_min_split = state->monitor_min_split_depth(shard.workchain); + if (shard.pfx_len() > monitor_min_split) { + shard = shard_prefix(shard, monitor_min_split); + } + all_shards.insert(shard); } // Remove overlays for removed adnl ids and shards for (auto it = id_to_overlays_.begin(); it != id_to_overlays_.end();) { if (my_adnl_ids.count(it->first)) { auto &overlays_info = it->second; - ; - auto ¤t_shards = overlays_info.is_validator_ ? all_shards : monitoring_shards; for (auto it2 = overlays_info.overlays_.begin(); it2 != overlays_info.overlays_.end();) { - if (current_shards.count(it2->first)) { + if (all_shards.count(it2->first)) { ++it2; } else { it2 = overlays_info.overlays_.erase(it2); @@ -330,7 +343,7 @@ void FullNodeFastSyncOverlays::update_overlays(td::Ref state, for (auto &[local_id, overlays_info] : id_to_overlays_) { overlays_info.is_validator_ = std::binary_search(current_validators_adnl_.begin(), current_validators_adnl_.end(), local_id); - for (auto &[_, overlay] : overlays_info.overlays_) { + for (auto &[shard, overlay] : overlays_info.overlays_) { td::actor::send_closure(overlay, &FullNodeFastSyncOverlay::set_validators, root_public_keys_, current_validators_adnl_); } @@ -388,17 +401,20 @@ void FullNodeFastSyncOverlays::update_overlays(td::Ref state, } // Update shard overlays - auto ¤t_shards = overlays_info.is_validator_ ? all_shards : monitoring_shards; - for (ShardIdFull shard_id : current_shards) { - auto &overlay = overlays_info.overlays_[shard_id]; + for (ShardIdFull shard : all_shards) { + bool receive_broadcasts = overlays_info.is_validator_ ? shard.is_masterchain() : monitoring_shards.count(shard); + auto &overlay = overlays_info.overlays_[shard]; if (overlay.empty()) { overlay = td::actor::create_actor( - PSTRING() << "FastSyncOv" << shard_id.to_str(), local_id, shard_id, zero_state_file_hash, - root_public_keys_, current_validators_adnl_, overlays_info.current_certificate_, keyring, adnl, overlays, + PSTRING() << "FastSyncOv" << shard.to_str(), local_id, shard, zero_state_file_hash, root_public_keys_, + current_validators_adnl_, overlays_info.current_certificate_, receive_broadcasts, keyring, adnl, overlays, validator_manager, full_node); - } else if (changed_certificate) { - td::actor::send_closure(overlay, &FullNodeFastSyncOverlay::set_member_certificate, - overlays_info.current_certificate_); + } else { + td::actor::send_closure(overlay, &FullNodeFastSyncOverlay::set_receive_broadcasts, receive_broadcasts); + if (changed_certificate) { + td::actor::send_closure(overlay, &FullNodeFastSyncOverlay::set_member_certificate, + overlays_info.current_certificate_); + } } } } diff --git a/validator/full-node-fast-sync-overlays.hpp b/validator/full-node-fast-sync-overlays.hpp index 30cb5dde8..b89f88024 100644 --- a/validator/full-node-fast-sync-overlays.hpp +++ b/validator/full-node-fast-sync-overlays.hpp @@ -49,11 +49,12 @@ class FullNodeFastSyncOverlay : public td::actor::Actor { void set_validators(std::vector root_public_keys, std::vector current_validators_adnl); void set_member_certificate(overlay::OverlayMemberCertificate member_certificate); + void set_receive_broadcasts(bool value); FullNodeFastSyncOverlay(adnl::AdnlNodeIdShort local_id, ShardIdFull shard, FileHash zero_state_file_hash, std::vector root_public_keys, std::vector current_validators_adnl, - overlay::OverlayMemberCertificate member_certificate, + overlay::OverlayMemberCertificate member_certificate, bool receive_broadcasts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId overlays, td::actor::ActorId validator_manager, @@ -63,6 +64,7 @@ class FullNodeFastSyncOverlay : public td::actor::Actor { , root_public_keys_(std::move(root_public_keys)) , current_validators_adnl_(std::move(current_validators_adnl)) , member_certificate_(std::move(member_certificate)) + , receive_broadcasts_(receive_broadcasts) , zero_state_file_hash_(zero_state_file_hash) , keyring_(keyring) , adnl_(adnl) @@ -77,6 +79,7 @@ class FullNodeFastSyncOverlay : public td::actor::Actor { std::vector root_public_keys_; std::vector current_validators_adnl_; overlay::OverlayMemberCertificate member_certificate_; + bool receive_broadcasts_; FileHash zero_state_file_hash_; td::actor::ActorId keyring_; From 4c8d25ac1bc0d91d187f98c5c36cb11c55d842c3 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 24 Jul 2024 18:02:59 +0300 Subject: [PATCH 105/388] Save and send block candidates in collator nodes --- validator/collator-node.cpp | 23 ++++++++++++++++++++--- validator/full-node.cpp | 28 +++++++++++++++------------- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 2da56e802..24ecda2a2 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -256,7 +256,8 @@ static td::BufferSlice serialize_error(td::Status error) { return create_serialize_tl_object(error.code(), error.message().c_str()); } -static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey creator) { +static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey creator, CatchainSeqno* cc_seqno = nullptr, + td::uint32* val_set_hash = nullptr) { CHECK(!block.id.is_masterchain()); if (block.pubkey == creator) { return block; @@ -264,8 +265,10 @@ static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey cre auto root = vm::std_boc_deserialize(block.data).move_as_ok(); block::gen::Block::Record blk; block::gen::BlockExtra::Record extra; + block::gen::BlockInfo::Record info; CHECK(tlb::unpack_cell(root, blk)); CHECK(tlb::unpack_cell(blk.extra, extra)); + CHECK(tlb::unpack_cell(blk.info, info)); extra.created_by = creator.as_bits256(); CHECK(tlb::pack_cell(blk.extra, extra)); CHECK(tlb::pack_cell(root, blk)); @@ -274,6 +277,13 @@ static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey cre block.id.root_hash = root->get_hash().bits(); block.id.file_hash = block::compute_file_hash(block.data.as_slice()); block.pubkey = creator; + + if (cc_seqno) { + *cc_seqno = info.gen_catchain_seqno; + } + if (val_set_hash) { + *val_set_hash = info.gen_validator_list_hash_short; + } return block; } @@ -304,9 +314,16 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data prev_blocks.push_back(create_block_id(b)); } Ed25519_PublicKey creator(f->creator_); - new_promise = [new_promise = std::move(new_promise), creator](td::Result R) mutable { + new_promise = [new_promise = std::move(new_promise), creator, + manager = manager_](td::Result R) mutable { TRY_RESULT_PROMISE(new_promise, block, std::move(R)); - new_promise.set_value(change_creator(std::move(block), creator)); + CatchainSeqno cc_seqno; + td::uint32 val_set_hash; + block = change_creator(std::move(block), creator, &cc_seqno, &val_set_hash); + td::Promise P = + new_promise.wrap([block = block.clone()](td::Unit&&) mutable -> BlockCandidate { return std::move(block); }); + td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, block.id, std::move(block), cc_seqno, + val_set_hash, std::move(P)); }; if (!shard.is_valid_ext()) { new_promise.set_error(td::Status::Error(PSTRING() << "invalid shard " << shard.to_str())); diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 8e84a4d45..d447943f6 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -352,9 +352,9 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_shard_block_info, block_id, cc_seqno, data.clone()); } - auto private_overlay = fast_sync_overlays_.choose_overlay(ShardIdFull(masterchainId)); - if (!private_overlay.empty()) { - td::actor::send_closure(private_overlay, &FullNodeFastSyncOverlay::send_shard_block_info, block_id, cc_seqno, + auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(ShardIdFull(masterchainId)); + if (!fast_sync_overlay.empty()) { + td::actor::send_closure(fast_sync_overlay, &FullNodeFastSyncOverlay::send_shard_block_info, block_id, cc_seqno, data.clone()); } td::actor::send_closure(shard, &FullNodeShard::send_shard_block_info, block_id, cc_seqno, std::move(data)); @@ -372,9 +372,9 @@ void FullNodeImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_se td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_block_candidate, block_id, cc_seqno, validator_set_hash, data.clone()); } - auto private_overlay = fast_sync_overlays_.choose_overlay(block_id.shard_full()); - if (!private_overlay.empty()) { - td::actor::send_closure(private_overlay, &FullNodeFastSyncOverlay::send_block_candidate, block_id, cc_seqno, + auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(block_id.shard_full()); + if (!fast_sync_overlay.empty()) { + td::actor::send_closure(fast_sync_overlay, &FullNodeFastSyncOverlay::send_block_candidate, block_id, cc_seqno, validator_set_hash, data.clone()); } if (broadcast_block_candidates_in_public_overlay_) { @@ -393,13 +393,15 @@ void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, bool custom_overlays VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; return; } - if (broadcast.block_id.is_masterchain() && !private_block_overlays_.empty()) { - td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_broadcast, - broadcast.clone()); - } - auto private_overlay = fast_sync_overlays_.choose_overlay(broadcast.block_id.shard_full()); - if (!private_overlay.empty()) { - td::actor::send_closure(private_overlay, &FullNodeFastSyncOverlay::send_broadcast, broadcast.clone()); + if (broadcast.block_id.is_masterchain()) { + if (!private_block_overlays_.empty()) { + td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_broadcast, + broadcast.clone()); + } + auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(broadcast.block_id.shard_full()); + if (!fast_sync_overlay.empty()) { + td::actor::send_closure(fast_sync_overlay, &FullNodeFastSyncOverlay::send_broadcast, broadcast.clone()); + } } td::actor::send_closure(shard, &FullNodeShard::send_broadcast, std::move(broadcast)); } From 0e7374610d454f5671042a58f68ce4cf40b890cb Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 25 Jul 2024 16:33:19 +0300 Subject: [PATCH 106/388] Improve block broadcasts --- create-hardfork/create-hardfork.cpp | 3 +- overlay/overlays.h | 5 +- test/test-ton-collator.cpp | 3 +- tl/generate/scheme/ton_api.tl | 4 +- tl/generate/scheme/ton_api.tlo | Bin 101564 -> 101528 bytes validator/collator-node.cpp | 17 ++- validator/full-node-fast-sync-overlays.cpp | 4 +- validator/full-node-shard.cpp | 111 ++++++------------ validator/full-node-shard.h | 11 +- validator/full-node-shard.hpp | 38 ++---- validator/full-node.cpp | 128 +++++++-------------- validator/full-node.hpp | 8 +- validator/interfaces/validator-manager.h | 2 - validator/manager-disk.hpp | 2 - validator/manager-hardfork.hpp | 2 - validator/manager.cpp | 31 +---- validator/manager.hpp | 10 -- validator/validator-group.cpp | 5 +- validator/validator-group.hpp | 10 +- validator/validator.h | 3 +- 20 files changed, 122 insertions(+), 275 deletions(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index e2ec3f626..8956771d6 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -237,8 +237,7 @@ class HardforkCreator : public td::actor::Actor { td::PromiseCreator::lambda([](td::Unit) {})); } void on_new_masterchain_block(td::Ref state, - std::set shards_to_monitor, - std::set temporary_shards) override { + std::set shards_to_monitor) override { } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } diff --git a/overlay/overlays.h b/overlay/overlays.h index cc569343d..cc112bc43 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -206,7 +206,10 @@ class OverlayMemberCertificate { return expire_at_ < cur_time - 3; } - auto tl() const { + tl_object_ptr tl() const { + if (empty()) { + return create_tl_object(); + } return create_tl_object(signed_by_.tl(), flags_, slot_, expire_at_, signature_.clone_as_buffer_slice()); } diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index afebe11fe..c6aee54b3 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -324,8 +324,7 @@ class TestNode : public td::actor::Actor { td::PromiseCreator::lambda([](td::Unit) {})); } void on_new_masterchain_block(td::Ref state, - std::set shards_to_monitor, - std::set temporary_shards) override { + std::set shards_to_monitor) override { } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 723857441..df9a54fd0 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -698,8 +698,8 @@ engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_i engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.OverlaysStats; engine.validator.shardOverlayStats.neighbour id:string verison_major:int version_minor:int flags:# - roundtrip:double unreliability:double has_state:string = engine.validator.shardOverlayStats.Neighbour; -engine.validator.shardOverlayStats shard:string mode:string + roundtrip:double unreliability:double = engine.validator.shardOverlayStats.Neighbour; +engine.validator.shardOverlayStats shard:string active:Bool neighbours:(vector engine.validator.shardOverlayStats.neighbour) = engine.validator.ShardOverlayStats; engine.validator.fastSyncOverlayStats shard:string validators_adnl:(vector int256) root_public_keys:(vector int256) member_certificate:overlay.MemberCertificate = engine.validator.FastSyncOverlayStats; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 4e080adea6755f3f0146586f56616042b9c7a3b8..532a1d3a334f6cc686ec78a758c2a51a4b536608 100644 GIT binary patch delta 140 zcmdlplWoRKwhbaOEFYvwoi|I!s4%fi`ffXOvxD58>0q%*+f+C~ayO3&`7lquSSvnx z!YYC3GUbdi(*v>@StplnQWwOR@9x!Km Xf*_;B0q%*+f+C~ayO3&`7lquSSvnx z!YTn4mfZZ5$seA`OfK1`Apz3B$N&LF&TKjywrvN&#%C3f(rokR6jES diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 24ecda2a2..a817d92aa 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -256,8 +256,8 @@ static td::BufferSlice serialize_error(td::Status error) { return create_serialize_tl_object(error.code(), error.message().c_str()); } -static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey creator, CatchainSeqno* cc_seqno = nullptr, - td::uint32* val_set_hash = nullptr) { +static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey creator, CatchainSeqno& cc_seqno, + td::uint32& val_set_hash) { CHECK(!block.id.is_masterchain()); if (block.pubkey == creator) { return block; @@ -278,12 +278,8 @@ static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey cre block.id.file_hash = block::compute_file_hash(block.data.as_slice()); block.pubkey = creator; - if (cc_seqno) { - *cc_seqno = info.gen_catchain_seqno; - } - if (val_set_hash) { - *val_set_hash = info.gen_validator_list_hash_short; - } + cc_seqno = info.gen_catchain_seqno; + val_set_hash = info.gen_validator_list_hash_short; return block; } @@ -305,8 +301,9 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data }; if (!validator_adnl_ids_.count(src)) { new_promise.set_error(td::Status::Error("src is not a validator")); + return; } - TRY_RESULT_PROMISE(new_promise, f, fetch_tl_object(std::move(data), true)); + TRY_RESULT_PROMISE(new_promise, f, fetch_tl_object(data, true)); ShardIdFull shard = create_shard_id(f->shard_); CatchainSeqno cc_seqno = f->cc_seqno_; std::vector prev_blocks; @@ -319,7 +316,7 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data TRY_RESULT_PROMISE(new_promise, block, std::move(R)); CatchainSeqno cc_seqno; td::uint32 val_set_hash; - block = change_creator(std::move(block), creator, &cc_seqno, &val_set_hash); + block = change_creator(std::move(block), creator, cc_seqno, val_set_hash); td::Promise P = new_promise.wrap([block = block.clone()](td::Unit&&) mutable -> BlockCandidate { return std::move(block); }); td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, block.id, std::move(block), cc_seqno, diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index d641215f2..a5789771b 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -255,7 +255,9 @@ void FullNodeFastSyncOverlay::get_stats_extra(td::Promise promise) for (const auto &x : root_public_keys_) { res->root_public_keys_.push_back(x.bits256_value()); } - res->member_certificate_ = member_certificate_.tl(); + if (!member_certificate_.empty()) { + res->member_certificate_ = member_certificate_.tl(); + } promise.set_result(td::json_encode(td::ToJson(*res), true)); } diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 00f3c6a2a..29950363d 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -104,7 +104,7 @@ void FullNodeShardImpl::create_overlay() { td::actor::ActorId node_; }; overlay::OverlayOptions opts; - opts.announce_self_ = is_active(); + opts.announce_self_ = active_; td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay_ex, adnl_id_, overlay_id_full_.clone(), std::make_unique(actor_id(this)), rules_, PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() @@ -119,7 +119,7 @@ void FullNodeShardImpl::create_overlay() { } void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broadcast, td::Promise promise) { - if (mode_ != FullNodeShardMode::active) { + if (!active_) { return promise.set_error(td::Status::Error("cannot check broadcast: shard is not active")); } auto B = fetch_tl_object(std::move(broadcast), true); @@ -161,16 +161,16 @@ void FullNodeShardImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promis create_overlay(); } -void FullNodeShardImpl::set_mode(FullNodeShardMode mode) { +void FullNodeShardImpl::set_active(bool active) { if (shard_.is_masterchain()) { return; } - bool was_active = is_active(); - mode_ = mode; - if (was_active != is_active()) { - td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); - create_overlay(); + if (active_ == active) { + return; } + active_ = active; + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); + create_overlay(); } void FullNodeShardImpl::try_get_next_block(td::Timestamp timeout, td::Promise promise) { @@ -219,7 +219,6 @@ void FullNodeShardImpl::got_next_block(td::Result R) { } void FullNodeShardImpl::get_next_block() { - //return; attempt_++; auto P = td::PromiseCreator::lambda([validator_manager = validator_manager_, attempt = attempt_, block_id = handle_->id(), SelfId = actor_id(this)](td::Result R) { @@ -623,12 +622,8 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, td::Promise promise) { VLOG(FULL_NODE_DEBUG) << "Got query getCapabilities from " << src; - td::uint32 flags = 0; - if (mode_ != FullNodeShardMode::active) { - flags |= Neighbour::FLAG_NO_STATE; - } promise.set_value( - create_serialize_tl_object(proto_version_major(), proto_version_minor(), flags)); + create_serialize_tl_object(proto_version_major(), proto_version_minor(), 0)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, @@ -717,7 +712,7 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise) { - if (!is_active()) { + if (!active_) { td::actor::send_closure(overlays_, &overlay::Overlays::send_message, src, adnl_id_, overlay_id_, create_serialize_tl_object()); promise.set_error(td::Status::Error("shard is inactive")); @@ -811,7 +806,7 @@ void FullNodeShardImpl::process_block_broadcast(PublicKeyHash src, ton_api::tonN } void FullNodeShardImpl::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { - if (!is_active()) { + if (!active_) { return; } auto B = fetch_tl_object(std::move(broadcast), true); @@ -983,7 +978,7 @@ void FullNodeShardImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp t void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) { - auto &b = choose_neighbour(true); + auto &b = choose_neighbour(); td::actor::create_actor( "archive", masterchain_seqno, shard_prefix, std::move(tmp_dir), adnl_id_, overlay_id_, b.adnl_id, timeout, validator_manager_, rldp2_, overlays_, adnl_, client_, create_neighbour_promise(b, std::move(promise))) @@ -994,7 +989,7 @@ void FullNodeShardImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std: block::ImportedMsgQueueLimits limits, td::Timestamp timeout, td::Promise>> promise) { // TODO: maybe more complex download (like other requests here) - auto &b = choose_neighbour(true); + auto &b = choose_neighbour(); if (b.adnl_id == adnl::AdnlNodeIdShort::zero()) { promise.set_error(td::Status::Error(ErrorCode::notready, "no nodes")); return; @@ -1213,22 +1208,11 @@ void FullNodeShardImpl::got_neighbours(std::vector vec) { continue; } if (neighbours_.size() == max_neighbours()) { - td::uint32 neighbours_with_state = 0; - for (const auto &n : neighbours_) { - if (n.second.has_state_known() && n.second.has_state()) { - ++neighbours_with_state; - } - } - adnl::AdnlNodeIdShort a = adnl::AdnlNodeIdShort::zero(); adnl::AdnlNodeIdShort b = adnl::AdnlNodeIdShort::zero(); td::uint32 cnt = 0; double u = 0; for (auto &n : neighbours_) { - if (neighbours_with_state <= min_neighbours_with_state() && n.second.has_state_known() && - n.second.has_state()) { - continue; - } if (n.second.unreliability > u) { u = n.second.unreliability; a = n.first; @@ -1252,7 +1236,7 @@ void FullNodeShardImpl::got_neighbours(std::vector vec) { } } -const Neighbour &FullNodeShardImpl::choose_neighbour(bool require_state) const { +const Neighbour &FullNodeShardImpl::choose_neighbour() const { if (neighbours_.size() == 0) { return Neighbour::zero; } @@ -1261,40 +1245,30 @@ const Neighbour &FullNodeShardImpl::choose_neighbour(bool require_state) const { for (auto &x : neighbours_) { min_unreliability = std::min(min_unreliability, x.second.unreliability); } - for (int attempt = 0; attempt < (require_state ? 2 : 1); ++attempt) { - const Neighbour *best = nullptr; - td::uint32 sum = 0; + const Neighbour *best = nullptr; + td::uint32 sum = 0; - for (auto &x : neighbours_) { - if (require_state) { - if (attempt == 0 && !(x.second.has_state_known() && x.second.has_state())) { - continue; - } - if (attempt == 1 && x.second.has_state_known()) { - continue; - } - } - auto unr = static_cast(x.second.unreliability - min_unreliability); + for (auto &x : neighbours_) { + auto unr = static_cast(x.second.unreliability - min_unreliability); - if (x.second.version_major < proto_version_major()) { - unr += 4; - } else if (x.second.version_major == proto_version_major() && x.second.version_minor < proto_version_minor()) { - unr += 2; - } + if (x.second.version_major < proto_version_major()) { + unr += 4; + } else if (x.second.version_major == proto_version_major() && x.second.version_minor < proto_version_minor()) { + unr += 2; + } - auto f = static_cast(fail_unreliability()); + auto f = static_cast(fail_unreliability()); - if (unr <= f) { - auto w = 1 << (f - unr); - sum += w; - if (td::Random::fast(0, sum - 1) <= w - 1) { - best = &x.second; - } + if (unr <= f) { + auto w = 1 << (f - unr); + sum += w; + if (td::Random::fast(0, sum - 1) <= w - 1) { + best = &x.second; } } - if (best) { - return *best; - } + } + if (best) { + return *best; } return Neighbour::zero; } @@ -1361,17 +1335,7 @@ void FullNodeShardImpl::ping_neighbours() { void FullNodeShardImpl::get_stats_extra(td::Promise promise) { auto res = create_tl_object(); res->shard_ = shard_.to_str(); - switch (mode_) { - case active: - res->mode_ = "active"; - break; - case active_temp: - res->mode_ = "active_temp"; - break; - case inactive: - res->mode_ = "inactive"; - break; - } + res->active_ = active_; for (const auto &p : neighbours_) { const auto &n = p.second; auto f = create_tl_object(); @@ -1381,7 +1345,6 @@ void FullNodeShardImpl::get_stats_extra(td::Promise promise) { f->flags_ = n.flags; f->roundtrip_ = n.roundtrip; f->unreliability_ = n.unreliability; - f->has_state_ = (n.has_state_known() ? (n.has_state() ? "true" : "false") : "undefined"); res->neighbours_.push_back(std::move(f)); } promise.set_result(td::json_encode(td::ToJson(*res), true)); @@ -1394,7 +1357,7 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, - td::actor::ActorId full_node, FullNodeShardMode mode) + td::actor::ActorId full_node, bool active) : shard_(shard) , local_id_(local_id) , adnl_id_(adnl_id) @@ -1407,7 +1370,7 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, , validator_manager_(validator_manager) , client_(client) , full_node_(full_node) - , mode_(mode) + , active_(active) , config_(config) { } @@ -1416,10 +1379,10 @@ td::actor::ActorOwn FullNodeShard::create( FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, td::actor::ActorId full_node, FullNodeShardMode mode) { + td::actor::ActorId client, td::actor::ActorId full_node, bool active) { return td::actor::create_actor(PSTRING() << "tonnode" << shard.to_str(), shard, local_id, adnl_id, zero_state_file_hash, config, keyring, adnl, rldp, rldp2, overlays, - validator_manager, client, full_node, mode); + validator_manager, client, full_node, active); } } // namespace fullnode diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 235e30514..169453257 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -28,12 +28,6 @@ namespace validator { namespace fullnode { -enum FullNodeShardMode { - active, // Node can answer queries about the shard - active_temp, // Like 'active', but queries about shard state are not allowed (only blocks) - inactive // Node is not a part of the overlay -}; - class FullNodeShard : public td::actor::Actor { public: virtual ~FullNodeShard() = default; @@ -42,7 +36,7 @@ class FullNodeShard : public td::actor::Actor { virtual ShardIdFull get_shard_full() const = 0; virtual void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) = 0; - virtual void set_mode(FullNodeShardMode mode) = 0; + virtual void set_active(bool active) = 0; virtual void set_config(FullNodeConfig config) = 0; virtual void send_ihr_message(td::BufferSlice data) = 0; @@ -85,8 +79,7 @@ class FullNodeShard : public td::actor::Actor { FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, td::actor::ActorId full_node, - FullNodeShardMode mode = FullNodeShardMode::active); + td::actor::ActorId client, td::actor::ActorId full_node, bool active); }; } // namespace fullnode diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 03e5fc0d7..8ac5185d9 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -40,22 +40,14 @@ struct Neighbour { double roundtrip_weight = 0; double unreliability = 0; - Neighbour(adnl::AdnlNodeIdShort adnl_id) : adnl_id(std::move(adnl_id)) { + explicit Neighbour(adnl::AdnlNodeIdShort adnl_id) : adnl_id(std::move(adnl_id)) { } void update_proto_version(ton_api::tonNode_capabilities &q); void query_success(double t); void query_failed(); void update_roundtrip(double t); - bool has_state() const { - return !(flags & FLAG_NO_STATE); - } - bool has_state_known() const { - return version_major != 0; - } - static Neighbour zero; - static constexpr td::uint32 FLAG_NO_STATE = 1; }; class FullNodeShardImpl : public FullNodeShard { @@ -82,9 +74,6 @@ class FullNodeShardImpl : public FullNodeShard { static constexpr td::uint32 max_neighbours() { return 16; } - static constexpr td::uint32 min_neighbours_with_state() { - return 10; - } static constexpr double stop_unreliability() { return 5.0; } @@ -94,15 +83,12 @@ class FullNodeShardImpl : public FullNodeShard { void create_overlay(); void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; - void set_mode(FullNodeShardMode mode) override; + void set_active(bool active) override; void set_config(FullNodeConfig config) override { config_ = config; } - //td::Result fetch_block(td::BufferSlice data); - void prevalidate_block(BlockIdExt block_id, td::BufferSlice data, td::BufferSlice proof, - td::Promise promise); void try_get_next_block(td::Timestamp timestamp, td::Promise promise); void got_next_block(td::Result block); void get_next_block(); @@ -155,8 +141,6 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getOutMsgQueueProof &query, td::Promise promise); - // void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareNextKeyBlockProof &query, - // td::Promise promise); void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise); void receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data); @@ -222,16 +206,14 @@ class FullNodeShardImpl : public FullNodeShard { void got_neighbours(std::vector res); void update_neighbour_stats(adnl::AdnlNodeIdShort adnl_id, double t, bool success); void got_neighbour_capabilities(adnl::AdnlNodeIdShort adnl_id, double t, td::BufferSlice data); - const Neighbour &choose_neighbour(bool require_state = false) const; + const Neighbour &choose_neighbour() const; template td::Promise create_neighbour_promise(const Neighbour &x, td::Promise p, bool require_state = false) { - return td::PromiseCreator::lambda([id = x.adnl_id, SelfId = actor_id(this), p = std::move(p), ts = td::Time::now(), - ignore_error = require_state && !x.has_state_known()](td::Result R) mutable { + return td::PromiseCreator::lambda([id = x.adnl_id, SelfId = actor_id(this), p = std::move(p), + ts = td::Time::now()](td::Result R) mutable { if (R.is_error() && R.error().code() != ErrorCode::notready && R.error().code() != ErrorCode::cancelled) { - if (!ignore_error) { - td::actor::send_closure(SelfId, &FullNodeShardImpl::update_neighbour_stats, id, td::Time::now() - ts, false); - } + td::actor::send_closure(SelfId, &FullNodeShardImpl::update_neighbour_stats, id, td::Time::now() - ts, false); } else { td::actor::send_closure(SelfId, &FullNodeShardImpl::update_neighbour_stats, id, td::Time::now() - ts, true); } @@ -245,17 +227,13 @@ class FullNodeShardImpl : public FullNodeShard { td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, td::actor::ActorId full_node, - FullNodeShardMode mode = FullNodeShardMode::active); + bool active); private: bool use_new_download() const { return false; } - bool is_active() const { - return mode_ != FullNodeShardMode::inactive; - } - ShardIdFull shard_; BlockHandle handle_; td::Promise promise_; @@ -289,7 +267,7 @@ class FullNodeShardImpl : public FullNodeShard { td::Timestamp ping_neighbours_at_; adnl::AdnlNodeIdShort last_pinged_neighbour_ = adnl::AdnlNodeIdShort::zero(); - FullNodeShardMode mode_; + bool active_; FullNodeConfig config_; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index d447943f6..651c01494 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -198,81 +198,56 @@ void FullNodeImpl::initial_read_complete(BlockHandle top_handle) { td::actor::send_closure(it->second.actor, &FullNodeShard::set_handle, top_handle, std::move(P)); } -void FullNodeImpl::on_new_masterchain_block(td::Ref state, std::set shards_to_monitor, - std::set temporary_shards) { +void FullNodeImpl::on_new_masterchain_block(td::Ref state, std::set shards_to_monitor) { CHECK(shards_to_monitor.count(ShardIdFull(masterchainId))); - std::set new_shards; - std::map new_active; - new_shards.insert(ShardIdFull(masterchainId)); + bool join_all_overlays = !sign_cert_by_.is_zero(); + std::set all_shards; + std::set new_active; + all_shards.insert(ShardIdFull(masterchainId)); std::set workchains; auto cut_shard = [&](ShardIdFull shard) -> ShardIdFull { int min_split = state->monitor_min_split_depth(shard.workchain); return min_split < shard.pfx_len() ? shard_prefix(shard, min_split) : shard; }; - auto set_active = [&](ShardIdFull shard, FullNodeShardMode mode) { - while (new_active.emplace(shard, mode).second && shard.pfx_len() > 0) { + auto set_active = [&](ShardIdFull shard) { + while (new_active.emplace(shard).second && shard.pfx_len() > 0) { shard = shard_parent(shard); } }; for (auto &info : state->get_shards()) { workchains.insert(info->shard().workchain); - new_shards.insert(cut_shard(info->shard())); + all_shards.insert(cut_shard(info->shard())); } for (const auto &wpair : state->get_workchain_list()) { ton::WorkchainId wc = wpair.first; const block::WorkchainInfo *winfo = wpair.second.get(); if (workchains.count(wc) == 0 && winfo->active && winfo->enabled_since <= state->get_unix_time()) { - new_shards.insert(ShardIdFull(wc)); + all_shards.insert(ShardIdFull(wc)); } } for (ShardIdFull shard : shards_to_monitor) { - set_active(cut_shard(shard), FullNodeShardMode::active); - } - for (ShardIdFull shard : temporary_shards) { - set_active(cut_shard(shard), FullNodeShardMode::active_temp); + set_active(cut_shard(shard)); } - auto info_set_mode = [&](ShardIdFull shard, ShardInfo &info, FullNodeShardMode mode) { - if (info.mode == mode) { - return; - } - if (info.actor.empty()) { - add_shard_actor(shard, mode); - return; + for (auto it = shards_.begin(); it != shards_.end(); ) { + if (all_shards.count(it->first)) { + ++it; + } else { + it = shards_.erase(it); } - info.mode = mode; - td::actor::send_closure(info.actor, &FullNodeShard::set_mode, mode); - info.delete_at = - mode != FullNodeShardMode::inactive ? td::Timestamp::never() : td::Timestamp::in(INACTIVE_SHARD_TTL); - }; - - for (auto shard : new_shards) { - auto &info = shards_[shard]; - info.exists = true; - } - - for (auto &p : shards_) { - ShardIdFull shard = p.first; - ShardInfo &info = p.second; - info.exists = new_shards.count(shard); - auto it = new_active.find(shard); - info_set_mode(shard, info, it == new_active.end() ? FullNodeShardMode::inactive : it->second); } - - for (const auto &s : new_active) { - info_set_mode(s.first, shards_[s.first], s.second); + for (ShardIdFull shard : all_shards) { + bool active = new_active.count(shard); + bool overlay_exists = !shards_[shard].actor.empty(); + if (active || join_all_overlays || overlay_exists) { + update_shard_actor(shard, active); + } } - auto it = shards_.begin(); - while (it != shards_.end()) { - if (it->second.mode == FullNodeShardMode::inactive && it->second.delete_at && it->second.delete_at.is_in_past()) { - it->second.actor.reset(); - it->second.delete_at = td::Timestamp::never(); - } - if (!it->second.exists && it->second.actor.empty()) { - it = shards_.erase(it); - } else { - ++it; + for (auto &[_, shard_info] : shards_) { + if (!shard_info.active && shard_info.delete_at && shard_info.delete_at.is_in_past() && !join_all_overlays) { + shard_info.actor = {}; + shard_info.delete_at = td::Timestamp::never(); } } @@ -298,18 +273,19 @@ void FullNodeImpl::on_new_masterchain_block(td::Ref state, std } } -void FullNodeImpl::add_shard_actor(ShardIdFull shard, FullNodeShardMode mode) { +void FullNodeImpl::update_shard_actor(ShardIdFull shard, bool active) { ShardInfo &info = shards_[shard]; - if (!info.actor.empty()) { - return; - } - info.actor = FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, config_, keyring_, adnl_, rldp_, - rldp2_, overlays_, validator_manager_, client_, actor_id(this), mode); - info.mode = mode; - info.delete_at = mode != FullNodeShardMode::inactive ? td::Timestamp::never() : td::Timestamp::in(INACTIVE_SHARD_TTL); - if (all_validators_.size() > 0) { - td::actor::send_closure(info.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + if (info.actor.empty()) { + info.actor = FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, config_, keyring_, adnl_, rldp_, + rldp2_, overlays_, validator_manager_, client_, actor_id(this), active); + if (!all_validators_.empty()) { + td::actor::send_closure(info.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + } + } else if (info.active != active) { + td::actor::send_closure(info.actor, &FullNodeShard::set_active, active); } + info.active = active; + info.delete_at = active ? td::Timestamp::never() : td::Timestamp::in(INACTIVE_SHARD_TTL); } void FullNodeImpl::sync_completed() { @@ -506,15 +482,11 @@ void FullNodeImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std::vect } td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { - ShardIdFull shard0 = shard; while (true) { auto it = shards_.find(shard); - if (it != shards_.end() && it->second.exists) { + if (it != shards_.end()) { if (it->second.actor.empty()) { - add_shard_actor(shard, FullNodeShardMode::inactive); - } - if (it->second.mode == FullNodeShardMode::inactive) { - it->second.delete_at = td::Timestamp::in(INACTIVE_SHARD_TTL); + update_shard_actor(shard, false); } return it->second.actor.get(); } @@ -523,19 +495,8 @@ td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { } shard = shard_parent(shard); } - shard = shard0; - auto it = shards_.find(shard); - if (it == shards_.end()) { - it = shards_.emplace(shard = ShardIdFull(shard.workchain), ShardInfo{}).first; - } - auto &info = it->second; - if (info.actor.empty()) { - add_shard_actor(shard, FullNodeShardMode::inactive); - } - if (info.mode == FullNodeShardMode::inactive) { - info.delete_at = td::Timestamp::in(INACTIVE_SHARD_TTL); - } - return info.actor.get(); + update_shard_actor(shard, false); + return shards_[shard].actor.get(); } td::actor::ActorId FullNodeImpl::get_shard(AccountIdPrefixFull dst) { @@ -635,7 +596,7 @@ void FullNodeImpl::process_block_candidate_broadcast(BlockIdExt block_id, Catcha } void FullNodeImpl::start_up() { - add_shard_actor(ShardIdFull{masterchainId}, FullNodeShardMode::active); + update_shard_actor(ShardIdFull{masterchainId}, true); if (local_id_.is_zero()) { if (adnl_id_.is_zero()) { auto pk = ton::PrivateKey{ton::privkeys::Ed25519::random()}; @@ -651,10 +612,9 @@ void FullNodeImpl::start_up() { void initial_read_complete(BlockHandle handle) override { td::actor::send_closure(id_, &FullNodeImpl::initial_read_complete, handle); } - void on_new_masterchain_block(td::Ref state, std::set shards_to_monitor, - std::set temporary_shards) override { + void on_new_masterchain_block(td::Ref state, std::set shards_to_monitor) override { td::actor::send_closure(id_, &FullNodeImpl::on_new_masterchain_block, std::move(state), - std::move(shards_to_monitor), std::move(temporary_shards)); + std::move(shards_to_monitor)); } void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) override { td::actor::send_closure(id_, &FullNodeImpl::send_ihr_message, dst, std::move(data)); @@ -716,7 +676,7 @@ void FullNodeImpl::start_up() { td::actor::send_closure(id_, &FullNodeImpl::new_key_block, std::move(handle)); } - Callback(td::actor::ActorId id) : id_(id) { + explicit Callback(td::actor::ActorId id) : id_(id) { } private: diff --git a/validator/full-node.hpp b/validator/full-node.hpp index a1f20b9d2..442a4db2d 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -59,8 +59,7 @@ class FullNodeImpl : public FullNode { void add_custom_overlay(CustomOverlayParams params, td::Promise promise) override; void del_custom_overlay(std::string name, td::Promise promise) override; - void on_new_masterchain_block(td::Ref state, std::set shards_to_monitor, - std::set temporary_shards); + void on_new_masterchain_block(td::Ref state, std::set shards_to_monitor); void sync_completed(); @@ -111,13 +110,12 @@ class FullNodeImpl : public FullNode { private: struct ShardInfo { - bool exists = false; td::actor::ActorOwn actor; - FullNodeShardMode mode = FullNodeShardMode::inactive; + bool active = false; td::Timestamp delete_at = td::Timestamp::never(); }; - void add_shard_actor(ShardIdFull shard, FullNodeShardMode mode); + void update_shard_actor(ShardIdFull shard, bool active); PublicKeyHash local_id_; adnl::AdnlNodeIdShort adnl_id_; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index abe6d5202..1ee02c482 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -195,8 +195,6 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void add_lite_query_stats(int lite_query_id) { } - virtual void validated_new_block(BlockIdExt block_id) = 0; - virtual void add_persistent_state_description(td::Ref desc) = 0; static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) { diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 69a0a3687..0dcf43879 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -437,8 +437,6 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { promise.set_result(td::Status::Error("not implemented")); } - void validated_new_block(BlockIdExt block_id) override { - } void add_persistent_state_description(td::Ref desc) override { } diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index edc675c45..48904bdb7 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -503,8 +503,6 @@ class ValidatorManagerImpl : public ValidatorManager { void update_options(td::Ref opts) override { opts_ = std::move(opts); } - void validated_new_block(BlockIdExt block_id) override { - } void add_persistent_state_description(td::Ref desc) override { } void get_validator_sessions_info( diff --git a/validator/manager.cpp b/validator/manager.cpp index 3aa222c43..abcc9e91f 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -492,7 +492,7 @@ void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { } void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { - if (!is_validator() && !is_shard_collator(block_id.shard_full())) { + if (!is_validator() && !is_shard_collator(block_id.shard_full()) && !cached_block_candidates_.count(block_id)) { return; } if (!last_masterchain_block_handle_) { @@ -2120,7 +2120,7 @@ void ValidatorManagerImpl::update_shard_overlays() { shards_to_monitor.insert(ShardIdFull{wc, shardIdAll}); } } - callback_->on_new_masterchain_block(last_masterchain_state_, std::move(shards_to_monitor), extra_active_shards_); + callback_->on_new_masterchain_block(last_masterchain_state_, std::move(shards_to_monitor)); } void ValidatorManagerImpl::update_shards() { @@ -2138,7 +2138,6 @@ void ValidatorManagerImpl::update_shards() { opts.proto_version = std::max(opts.proto_version, 1); } auto opts_hash = opts.get_hash(); - extra_active_shards_.clear(); std::map> new_shards; std::set future_shards; @@ -2193,11 +2192,6 @@ void ValidatorManagerImpl::update_shards() { default: LOG(FATAL) << "state=" << static_cast(v->fsm_state()); } - cleanup_last_validated_blocks(v->top_block_id().id); - } - - for (const auto& s : last_validated_blocks_) { - extra_active_shards_.insert(s.first); } new_shards.emplace(ShardIdFull{masterchainId, shardIdAll}, std::vector{last_masterchain_block_id_}); @@ -2270,7 +2264,6 @@ void ValidatorManagerImpl::update_shards() { auto validator_id = get_validator(shard, val_set); if (!validator_id.is_zero()) { - extra_active_shards_.insert(shard); auto val_group_id = get_validator_set_id(shard, val_set, opts_hash, key_seqno, opts); if (force_recover) { @@ -2366,24 +2359,6 @@ void ValidatorManagerImpl::update_shards() { } } -void ValidatorManagerImpl::cleanup_last_validated_blocks(BlockId new_block) { - auto process_shard = [&, this](ShardIdFull shard) { - auto it = last_validated_blocks_.find(shard); - if (it != last_validated_blocks_.end() && it->second < new_block.seqno) { - last_validated_blocks_.erase(it); - } - }; - ShardIdFull shard = new_block.shard_full(); - process_shard(shard); - if (shard.pfx_len() > 0) { - process_shard(shard_parent(shard)); - } - if (shard.pfx_len() < max_shard_pfx_len) { - process_shard(shard_child(shard, true)); - process_shard(shard_child(shard, false)); - } -} - void ValidatorManagerImpl::written_destroyed_validator_sessions(std::vector> list) { for (auto &v : list) { td::actor::send_closure(v, &ValidatorGroup::destroy); @@ -2463,7 +2438,7 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group auto validator_id = get_validator(shard, validator_set); CHECK(!validator_id.is_zero()); auto G = td::actor::create_actor( - "validatorgroup", shard, validator_id, session_id, validator_set, + PSTRING() << "valgroup" << shard.to_str(), shard, validator_id, session_id, validator_set, last_masterchain_state_->get_collator_config(true), opts, keyring_, adnl_, rldp_, overlays_, db_root_, actor_id(this), init_session, opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_, opts_->need_monitor(shard, last_masterchain_state_)); diff --git a/validator/manager.hpp b/validator/manager.hpp index 2a1638bd2..2654b2143 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -624,11 +624,6 @@ class ValidatorManagerImpl : public ValidatorManager { void get_validator_sessions_info( td::Promise> promise) override; - void validated_new_block(BlockIdExt block_id) override { - BlockSeqno &last = last_validated_blocks_[block_id.shard_full()]; - last = std::max(last, block_id.seqno()); - } - void add_persistent_state_description(td::Ref desc) override; void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; @@ -745,8 +740,6 @@ class ValidatorManagerImpl : public ValidatorManager { return 3 * 10; } - void cleanup_last_validated_blocks(BlockId new_block); - void got_persistent_state_descriptions(std::vector> descs); void add_persistent_state_description_impl(td::Ref desc); td::Ref get_block_persistent_state(BlockIdExt block_id); @@ -773,9 +766,6 @@ class ValidatorManagerImpl : public ValidatorManager { }; std::map collator_nodes_; - std::set extra_active_shards_; - std::map last_validated_blocks_; - std::map> persistent_state_descriptions_; std::map> persistent_state_blocks_; }; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 4e23b6130..b708e6ace 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -177,9 +177,6 @@ void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash s void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref block, std::vector prev, td::Ref sig_set, td::Ref approve_sig_set, bool send_broadcast, td::Promise promise, bool is_retry) { - if (!is_retry) { - td::actor::send_closure(manager_, &ValidatorManager::validated_new_block, block_id); - } auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { @@ -197,7 +194,7 @@ void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref }); run_accept_block_query(block_id, std::move(block), std::move(prev), validator_set_, std::move(sig_set), - std::move(approve_sig_set), send_broadcast, apply_blocks_, manager_, std::move(P)); + std::move(approve_sig_set), send_broadcast, monitoring_shard_, manager_, std::move(P)); } void ValidatorGroup::skip_round(td::uint32 round_id) { diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 627573427..debb7d14a 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -68,7 +68,7 @@ class ValidatorGroup : public td::actor::Actor { void update_options(td::Ref opts, bool apply_blocks) { opts_ = std::move(opts); - apply_blocks_ = apply_blocks; + monitoring_shard_ = apply_blocks; } ValidatorGroup(ShardIdFull shard, PublicKeyHash local_id, ValidatorSessionId session_id, @@ -77,7 +77,7 @@ class ValidatorGroup : public td::actor::Actor { td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, td::actor::ActorId validator_manager, bool create_session, - bool allow_unsafe_self_blocks_resync, td::Ref opts, bool apply_blocks) + bool allow_unsafe_self_blocks_resync, td::Ref opts, bool monitoring_shard) : shard_(shard) , local_id_(std::move(local_id)) , session_id_(session_id) @@ -93,7 +93,7 @@ class ValidatorGroup : public td::actor::Actor { , init_(create_session) , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) , opts_(std::move(opts)) - , apply_blocks_(apply_blocks) { + , monitoring_shard_(monitoring_shard) { } private: @@ -141,7 +141,7 @@ class ValidatorGroup : public td::actor::Actor { bool allow_unsafe_self_blocks_resync_; td::Ref opts_; td::uint32 last_known_round_id_ = 0; - bool apply_blocks_ = true; + bool monitoring_shard_ = true; struct CachedCollatedBlock { td::optional result; @@ -151,7 +151,7 @@ class ValidatorGroup : public td::actor::Actor { void generated_block_candidate(std::shared_ptr cache, td::Result R); - typedef std::tuple CacheKey; + using CacheKey = std::tuple; std::map approved_candidates_cache_; void update_approve_cache(CacheKey key, UnixTime value); diff --git a/validator/validator.h b/validator/validator.h index 9428b844d..a5e34bfea 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -170,8 +170,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void initial_read_complete(BlockHandle top_masterchain_blocks) = 0; virtual void on_new_masterchain_block(td::Ref state, - std::set shards_to_monitor, - std::set temporary_shards) = 0; + std::set shards_to_monitor) = 0; virtual void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; From 70322f58c196e326bd41f6aabe68df74c7fbabb2 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 1 Aug 2024 13:57:32 +0300 Subject: [PATCH 107/388] Use candidates from DB in wait block data --- tl/generate/scheme/ton_api.tl | 1 + tl/generate/scheme/ton_api.tlo | Bin 101528 -> 101624 bytes validator/collator-node.cpp | 2 +- validator/db/fileref.cpp | 25 +++++++++++++++ validator/db/fileref.hpp | 37 ++++++++++++++++++++-- validator/db/rootdb.cpp | 37 ++++++++++++++-------- validator/db/rootdb.hpp | 1 + validator/downloaders/wait-block-data.cpp | 19 ++++++++--- validator/downloaders/wait-block-data.hpp | 12 ++++--- validator/interfaces/db.h | 1 + validator/manager-disk.cpp | 5 +++ validator/manager-disk.hpp | 1 + validator/manager-hardfork.cpp | 5 +++ validator/manager-hardfork.hpp | 1 + validator/manager.cpp | 21 +++++++++--- validator/manager.hpp | 1 + validator/validator.h | 1 + 17 files changed, 140 insertions(+), 30 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index df9a54fd0..f7d0ad249 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -524,6 +524,7 @@ db.filedb.key.proof block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.proofLink block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.signatures block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.candidate id:db.candidate.id = db.filedb.Key; +db.filedb.key.candidateRef id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.blockInfo block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.value key:db.filedb.Key prev:int256 next:int256 file_hash:int256 = db.filedb.Value; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 532a1d3a334f6cc686ec78a758c2a51a4b536608..ad6f2ce30e94ea1ba5020ba0a93c1e27918fb3cd 100644 GIT binary patch delta 276 zcmbO+lkLY$whawjqBFH#DD$Qy>7`}nqyj1L)Jg^hu8EG~o7Zq@7%*<$eAVi|u9 z&0|773_wui%%;O(+jbBnHu*%U1`C*Rpxgw)&?(bF6PZ#5l{!9poYqxGKjLv(L%*nbhRM3UQ90)HDXLvkZ!$rtE=fhC5WD R(gy5^=7h@a36+c%!T>-cXH5VA delta 237 zcmew{lWoRKwhawjqUsq-kMgD@>7`}nqyj1L)Jg^h&WVoVo7Zq@7%*|u9 z&0|773_wui%%;O(+jbBnHu*%U1`C*Rpxgw)&?(bF6PZ#5l{! } } - for (auto it = future_validator_groups_.begin(); it != future_validator_groups_.end(); ++it) { + for (auto it = future_validator_groups_.begin(); it != future_validator_groups_.end();) { if (get_future_validator_group(it->first.first, it->first.second).is_ok()) { ++it; } else { diff --git a/validator/db/fileref.cpp b/validator/db/fileref.cpp index 2e2a3b6b6..378e382c9 100644 --- a/validator/db/fileref.cpp +++ b/validator/db/fileref.cpp @@ -213,6 +213,28 @@ std::string CandidateShort::filename_short() const { return PSTRING() << "candidate_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex(); } +CandidateRefShort CandidateRef::shortref() const { + return CandidateRefShort{block_id.id, hash()}; +} + +std::string CandidateRef::filename() const { + return PSTRING() << "candidateref_" << block_id.to_str(); +} + +std::string CandidateRef::filename_short() const { + char s[33]; + sprintf(s, "%llx", static_cast(block_id.id.shard)); + return PSTRING() << "candidateref_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_" + << hash().to_hex(); +} + +std::string CandidateRefShort::filename_short() const { + char s[33]; + sprintf(s, "%llx", static_cast(block_id.shard)); + return PSTRING() << "candidateref_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" + << hash().to_hex(); +} + BlockInfoShort BlockInfo::shortref() const { return BlockInfoShort{block_id.id, hash()}; } @@ -259,6 +281,9 @@ FileReference::FileReference(tl_object_ptr key) { ref_ = fileref::Candidate{PublicKey{key.id_->source_}, create_block_id(key.id_->id_), key.id_->collated_data_file_hash_}; }, + [&](const ton_api::db_filedb_key_candidateRef& key) { + ref_ = fileref::CandidateRef{create_block_id(key.id_)}; + }, [&](const ton_api::db_filedb_key_blockInfo& key) { ref_ = fileref::BlockInfo{create_block_id(key.block_id_)}; })); diff --git a/validator/db/fileref.hpp b/validator/db/fileref.hpp index 14424a653..6710cc063 100644 --- a/validator/db/fileref.hpp +++ b/validator/db/fileref.hpp @@ -278,6 +278,38 @@ class Candidate { FileHash collated_data_file_hash; }; +class CandidateRefShort { + public: + FileHash hash() const { + return hashv; + } + ShardIdFull shard() const { + return block_id.shard_full(); + } + std::string filename_short() const; + + BlockId block_id; + FileHash hashv; +}; + +class CandidateRef { + public: + tl_object_ptr tl() const { + return create_tl_object(create_tl_block_id(block_id)); + } + FileHash hash() const { + return create_hash_tl_object(create_tl_block_id(block_id)); + } + ShardIdFull shard() const { + return block_id.shard_full(); + } + CandidateRefShort shortref() const; + std::string filename() const; + std::string filename_short() const; + + BlockIdExt block_id; +}; + class BlockInfoShort { public: FileHash hash() const { @@ -316,7 +348,7 @@ class FileReferenceShort { private: td::Variant + fileref::CandidateShort, fileref::CandidateRefShort, fileref::BlockInfoShort> ref_; public: @@ -340,7 +372,8 @@ class FileReferenceShort { class FileReference { private: td::Variant + fileref::Proof, fileref::ProofLink, fileref::Signatures, fileref::Candidate, fileref::CandidateRef, + fileref::BlockInfo> ref_; public: diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 10545e46f..ccd5cb290 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -177,21 +177,21 @@ void RootDb::get_block_proof_link(ConstBlockHandle handle, td::Promise promise) { + auto source = PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}; auto obj = create_serialize_tl_object( - PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}.tl(), create_tl_block_id(candidate.id), - std::move(candidate.data), std::move(candidate.collated_data)); - - auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - } else { - promise.set_value(td::Unit()); - } - }); + source.tl(), create_tl_block_id(candidate.id), std::move(candidate.data), std::move(candidate.collated_data)); + auto P = td::PromiseCreator::lambda( + [archive_db = archive_db_.get(), promise = std::move(promise), block_id = candidate.id, source, + collated_file_hash = candidate.collated_file_hash](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, _, std::move(R)); + td::actor::send_closure(archive_db, &ArchiveManager::add_temp_file_short, fileref::CandidateRef{block_id}, + create_serialize_tl_object( + source.tl(), create_tl_block_id(block_id), collated_file_hash), + std::move(promise)); + }); td::actor::send_closure(archive_db_, &ArchiveManager::add_temp_file_short, - fileref::Candidate{PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}, candidate.id, - candidate.collated_file_hash}, - std::move(obj), std::move(P)); + fileref::Candidate{source, candidate.id, candidate.collated_file_hash}, std::move(obj), + std::move(P)); } void RootDb::get_block_candidate(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, @@ -215,6 +215,17 @@ void RootDb::get_block_candidate(PublicKey source, BlockIdExt id, FileHash colla fileref::Candidate{source, id, collated_data_file_hash}, std::move(P)); } +void RootDb::get_block_candidate_by_block_id(BlockIdExt id, td::Promise promise) { + td::actor::send_closure( + archive_db_, &ArchiveManager::get_temp_file_short, fileref::CandidateRef{id}, + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, data, std::move(R)); + TRY_RESULT_PROMISE(promise, f, fetch_tl_object(data, true)); + td::actor::send_closure(SelfId, &RootDb::get_block_candidate, PublicKey{f->source_}, create_block_id(f->id_), + f->collated_data_file_hash_, std::move(promise)); + }); +} + void RootDb::store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) { if (handle->moved_to_archive()) { diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 74c46e43c..52f6098e4 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -58,6 +58,7 @@ class RootDb : public Db { void store_block_candidate(BlockCandidate candidate, td::Promise promise) override; void get_block_candidate(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) override; + void get_block_candidate_by_block_id(BlockIdExt id, td::Promise promise) override; void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; diff --git a/validator/downloaders/wait-block-data.cpp b/validator/downloaders/wait-block-data.cpp index 220a8a2cf..53a3d351b 100644 --- a/validator/downloaders/wait-block-data.cpp +++ b/validator/downloaders/wait-block-data.cpp @@ -106,13 +106,24 @@ void WaitBlockData::start() { }); td::actor::send_closure(manager_, &ValidatorManager::try_get_static_file, handle_->id().file_hash, std::move(P)); + } else if (try_get_candidate_) { + try_get_candidate_ = false; + td::actor::send_closure( + manager_, &ValidatorManager::get_candidate_data_by_block_id_from_db, handle_->id(), + [SelfId = actor_id(this), id = handle_->id()](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &WaitBlockData::start); + } else { + td::actor::send_closure(SelfId, &WaitBlockData::loaded_data, ReceivedBlock{id, R.move_as_ok()}); + } + }); } else { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &WaitBlockData::failed_to_get_block_data_from_net, R.move_as_error_prefix("net error: ")); } else { - td::actor::send_closure(SelfId, &WaitBlockData::got_data_from_net, R.move_as_ok()); + td::actor::send_closure(SelfId, &WaitBlockData::loaded_data, R.move_as_ok()); } }); @@ -137,16 +148,16 @@ void WaitBlockData::failed_to_get_block_data_from_net(td::Status reason) { td::Timestamp::in(0.1)); } -void WaitBlockData::got_data_from_net(ReceivedBlock block) { +void WaitBlockData::loaded_data(ReceivedBlock block) { auto X = create_block(std::move(block)); if (X.is_error()) { failed_to_get_block_data_from_net(X.move_as_error_prefix("bad block from net: ")); return; } - got_block_data_from_net(X.move_as_ok()); + loaded_block_data(X.move_as_ok()); } -void WaitBlockData::got_block_data_from_net(td::Ref block) { +void WaitBlockData::loaded_block_data(td::Ref block) { if (data_.not_null()) { return; } diff --git a/validator/downloaders/wait-block-data.hpp b/validator/downloaders/wait-block-data.hpp index 229b4bfc8..f3b367d51 100644 --- a/validator/downloaders/wait-block-data.hpp +++ b/validator/downloaders/wait-block-data.hpp @@ -30,15 +30,16 @@ class ValidatorManager; class WaitBlockData : public td::actor::Actor { public: WaitBlockData(BlockHandle handle, td::uint32 priority, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise> promise) + td::Timestamp timeout, bool try_get_candidate, td::Promise> promise) : handle_(std::move(handle)) , priority_(priority) , manager_(manager) , timeout_(timeout) + , try_get_candidate_(try_get_candidate) , promise_(std::move(promise)) , perf_timer_("waitdata", 1.0, [manager](double duration) { - send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitdata", duration); - }) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitdata", duration); + }) { } void update_timeout(td::Timestamp timeout, td::uint32 priority) { @@ -57,8 +58,8 @@ class WaitBlockData : public td::actor::Actor { void set_is_hardfork(bool value); void start(); void got_block_data_from_db(td::Ref data); - void got_data_from_net(ReceivedBlock data); - void got_block_data_from_net(td::Ref block); + void loaded_data(ReceivedBlock data); + void loaded_block_data(td::Ref block); void checked_proof_link(); void failed_to_get_block_data_from_net(td::Status reason); @@ -73,6 +74,7 @@ class WaitBlockData : public td::actor::Actor { td::actor::ActorId manager_; td::Timestamp timeout_; + bool try_get_candidate_; td::Promise> promise_; td::Ref data_; diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index f915b25c7..29ef715b3 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -46,6 +46,7 @@ class Db : public td::actor::Actor { virtual void store_block_candidate(BlockCandidate candidate, td::Promise promise) = 0; virtual void get_block_candidate(ton::PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) = 0; + virtual void get_block_candidate_by_block_id(BlockIdExt id, td::Promise promise) = 0; virtual void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 1795ee34f..363446107 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -589,6 +589,11 @@ void ValidatorManagerImpl::get_block_candidate_from_db(PublicKey source, BlockId td::actor::send_closure(db_, &Db::get_block_candidate, source, id, collated_data_file_hash, std::move(promise)); } +void ValidatorManagerImpl::get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) { + td::actor::send_closure(db_, &Db::get_block_candidate_by_block_id, id, + promise.wrap([](BlockCandidate &&b) { return std::move(b.data); })); +} + void ValidatorManagerImpl::get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) { td::actor::send_closure(db_, &Db::get_block_proof, std::move(handle), std::move(promise)); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 0dcf43879..01a0c60eb 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -217,6 +217,7 @@ class ValidatorManagerImpl : public ValidatorManager { void get_shard_state_from_db_short(BlockIdExt block_id, td::Promise> promise) override; void get_block_candidate_from_db(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) override; + void get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) override; void get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) override; void get_block_proof_from_db_short(BlockIdExt id, td::Promise> promise) override; void get_block_proof_link_from_db(ConstBlockHandle handle, td::Promise> promise) override; diff --git a/validator/manager-hardfork.cpp b/validator/manager-hardfork.cpp index 1ba8c68d1..e091983db 100644 --- a/validator/manager-hardfork.cpp +++ b/validator/manager-hardfork.cpp @@ -415,6 +415,11 @@ void ValidatorManagerImpl::get_block_candidate_from_db(PublicKey source, BlockId td::actor::send_closure(db_, &Db::get_block_candidate, source, id, collated_data_file_hash, std::move(promise)); } +void ValidatorManagerImpl::get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) { + td::actor::send_closure(db_, &Db::get_block_candidate_by_block_id, id, + promise.wrap([](BlockCandidate &&b) { return std::move(b.data); })); +} + void ValidatorManagerImpl::get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) { td::actor::send_closure(db_, &Db::get_block_proof, std::move(handle), std::move(promise)); } diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 48904bdb7..28c521f43 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -263,6 +263,7 @@ class ValidatorManagerImpl : public ValidatorManager { void get_shard_state_from_db_short(BlockIdExt block_id, td::Promise> promise) override; void get_block_candidate_from_db(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) override; + void get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) override; void get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) override; void get_block_proof_from_db_short(BlockIdExt id, td::Promise> promise) override; void get_block_proof_link_from_db(ConstBlockHandle handle, td::Promise> promise) override; diff --git a/validator/manager.cpp b/validator/manager.cpp index abcc9e91f..0c4a1a1bc 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -622,7 +622,7 @@ void ValidatorManagerImpl::add_cached_block_candidate(ReceivedBlock block) { if (it != wait_block_data_.end()) { auto r_block = create_block(cached_block_candidates_[id].clone()); if (r_block.is_ok()) { - td::actor::send_closure(it->second.actor_, &WaitBlockData::got_block_data_from_net, r_block.move_as_ok()); + td::actor::send_closure(it->second.actor_, &WaitBlockData::loaded_block_data, r_block.move_as_ok()); } } } @@ -852,7 +852,8 @@ void ValidatorManagerImpl::wait_block_data(BlockHandle handle, td::uint32 priori td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_data, handle, std::move(R)); }); auto id = td::actor::create_actor("waitdata", handle, priority, actor_id(this), - td::Timestamp::at(timeout.at() + 10.0), std::move(P)) + td::Timestamp::at(timeout.at() + 10.0), + is_shard_collator(handle->id().shard_full()), std::move(P)) .release(); wait_block_data_[handle->id()].actor_ = id; it = wait_block_data_.find(handle->id()); @@ -1177,6 +1178,16 @@ void ValidatorManagerImpl::get_block_candidate_from_db(PublicKey source, BlockId td::actor::send_closure(db_, &Db::get_block_candidate, source, id, collated_data_file_hash, std::move(promise)); } +void ValidatorManagerImpl::get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) { + auto it = cached_block_candidates_.find(id); + if (it != cached_block_candidates_.end()) { + promise.set_result(it->second.data.clone()); + return; + } + td::actor::send_closure(db_, &Db::get_block_candidate_by_block_id, id, + promise.wrap([](BlockCandidate &&b) { return std::move(b.data); })); +} + void ValidatorManagerImpl::get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) { td::actor::send_closure(db_, &Db::get_block_proof, std::move(handle), std::move(promise)); } @@ -1289,9 +1300,9 @@ void ValidatorManagerImpl::finished_wait_data(BlockHandle handle, td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_data, handle, std::move(R)); }); - auto id = - td::actor::create_actor("waitdata", handle, X.second, actor_id(this), X.first, std::move(P)) - .release(); + auto id = td::actor::create_actor("waitdata", handle, X.second, actor_id(this), X.first, + is_shard_collator(handle->id().shard_full()), std::move(P)) + .release(); it->second.actor_ = id; return; } diff --git a/validator/manager.hpp b/validator/manager.hpp index 2654b2143..007bce35f 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -479,6 +479,7 @@ class ValidatorManagerImpl : public ValidatorManager { void get_shard_state_from_db_short(BlockIdExt block_id, td::Promise> promise) override; void get_block_candidate_from_db(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) override; + void get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) override; void get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) override; void get_block_proof_from_db_short(BlockIdExt id, td::Promise> promise) override; void get_block_proof_link_from_db(ConstBlockHandle handle, td::Promise> promise) override; diff --git a/validator/validator.h b/validator/validator.h index a5e34bfea..8bda3e871 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -262,6 +262,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_block_data_from_db_short(BlockIdExt block_id, td::Promise> promise) = 0; virtual void get_block_candidate_from_db(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) = 0; + virtual void get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) = 0; virtual void get_shard_state_from_db(ConstBlockHandle handle, td::Promise> promise) = 0; virtual void get_shard_state_from_db_short(BlockIdExt block_id, td::Promise> promise) = 0; virtual void get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) = 0; From a14490637a861fced73dfd05201ea0b2eab71e59 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 1 Aug 2024 18:39:22 +0300 Subject: [PATCH 108/388] More overlay stats --- overlay/overlay.cpp | 4 +++ tl/generate/scheme/ton_api.tl | 3 +- tl/generate/scheme/ton_api.tlo | Bin 101624 -> 101736 bytes .../validator-engine-console-query.cpp | 26 ++++++++++++++---- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index d05f5ae28..e93d698a6 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -726,6 +726,10 @@ void OverlayImpl::get_stats(td::Promisebdcst_errors_ = peer.broadcast_errors; node_obj->fec_bdcst_errors_ = peer.fec_broadcast_errors; + node_obj->is_neighbour_ = peer.is_neighbour(); + node_obj->is_alive_ = peer.is_alive(); + node_obj->node_flags_ = peer.get_node()->flags(); + res->nodes_.push_back(std::move(node_obj)); }); diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index f7d0ad249..70cb790b7 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -693,7 +693,8 @@ engine.validator.proposalVote perm_key:int256 to_send:bytes = engine.validator.P engine.validator.dhtServerStatus id:int256 status:int = engine.validator.DhtServerStatus; engine.validator.dhtServersStatus servers:(vector engine.validator.dhtServerStatus) = engine.validator.DhtServersStatus; -engine.validator.overlayStatsNode adnl_id:int256 ip_addr:string bdcst_errors:int fec_bdcst_errors:int last_in_query:int last_out_query:int t_out_bytes:int t_in_bytes:int t_out_pckts:int t_in_pckts:int = engine.validator.OverlayStatsNode; +engine.validator.overlayStatsNode adnl_id:int256 ip_addr:string is_neighbour:Bool is_alive:Bool node_flags:int + bdcst_errors:int fec_bdcst_errors:int last_in_query:int last_out_query:int t_out_bytes:int t_in_bytes:int t_out_pckts:int t_in_pckts:int = engine.validator.OverlayStatsNode; engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 scope:string nodes:(vector engine.validator.overlayStatsNode) stats:(vector engine.validator.oneStat) extra:string = engine.validator.OverlayStats; engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.OverlaysStats; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index ad6f2ce30e94ea1ba5020ba0a93c1e27918fb3cd..eae2a5590c1411adc052dbea3dbd6b5832abc004 100644 GIT binary patch delta 138 zcmew{lkLSUwhbXttOAVIHHwpErDQjkNp(%*1aTP|fMD|4?dFpmHd;UxC~lDD_sdU7 zWnciQxOq&-hj+5#GWE$9HY)OD7RTqMW~OH(<(C#sR$M5}!2uLW%*ia9tgujqhbs@L eEj}$LF};{!a>FvI&7U@!F|vS6*{rkq%uN6XZZ9eT delta 91 zcmaDci|xluwhbXttTz>kekx9um6F|DCe<~K6U1d?0D{S~JIp6LY_xzXP~0HP@0XvF k%D@0par2mv57%VHW$KeJY*gHwxXGN61th$A&gL^W0cPbTsQ>@~ diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index f88d2c601..6266ac2ac 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -910,8 +910,22 @@ td::Status GetOverlaysStatsJsonQuery::receive(td::BufferSlice data) { } else { tail = true; } - - sb << " {\n \"adnl_id\": \"" << n->adnl_id_ << "\",\n \"ip_addr\": \"" << n->ip_addr_ << "\",\n \"broadcast_errors\": " << n->bdcst_errors_ << ",\n \"fec_broadcast_errors\": " << n->fec_bdcst_errors_ << ",\n \"last_in_query_unix\": " << n->last_in_query_ << ",\n \"last_in_query_human\": \"" << time_to_human(n->last_in_query_) << "\",\n" << " \"last_out_query_unix\": " << n->last_out_query_ << ",\n \"last_out_query_human\": \"" << time_to_human(n->last_out_query_) << "\",\n" << "\n \"throughput\": { \"out_bytes_sec\": " << n->t_out_bytes_ << ", \"out_pckts_sec\": " << n->t_out_pckts_ << ", \"in_bytes_sec\": " << n->t_in_bytes_ << ", \"in_pckts_sec\": " << n->t_in_pckts_ << " }\n }"; + + sb << " {\n"; + sb << R"( "adnl_id": ")" << n->adnl_id_ << "\",\n"; + sb << R"( "ip_addr": ")" << n->ip_addr_ << "\",\n"; + sb << R"( "is_neighbour": )" << (n->is_neighbour_ ? "true" : "false") << ",\n"; + sb << R"( "is_alive": )" << (n->is_alive_ ? "true" : "false") << ",\n"; + sb << R"( "node_flags": )" << n->node_flags_ << ",\n"; + sb << R"( "broadcast_errors": )" << n->bdcst_errors_ << ",\n"; + sb << R"( "fec_broadcast_errors": )" << n->fec_bdcst_errors_ << ",\n"; + sb << R"( "last_in_query_unix": )" << n->last_in_query_ << ",\n"; + sb << R"( "last_in_query_human": ")" << time_to_human(n->last_in_query_) << "\",\n"; + sb << R"( "last_out_query_unix": )" << n->last_out_query_ << ",\n"; + sb << R"( "last_out_query_human": ")" << time_to_human(n->last_out_query_) << "\",\n\n"; + sb << R"( "throughput": { "out_bytes_sec": )" << n->t_out_bytes_ << R"(, "out_pckts_sec": )" << n->t_out_pckts_ + << R"(, "in_bytes_sec": )" << n->t_in_bytes_ << R"(, "in_pckts_sec": )" << n->t_in_pckts_ << " }\n"; + sb << " }"; overlay_t_out_bytes += n->t_out_bytes_; overlay_t_out_pckts += n->t_out_pckts_; @@ -920,8 +934,10 @@ td::Status GetOverlaysStatsJsonQuery::receive(td::BufferSlice data) { overlay_t_in_pckts += n->t_in_pckts_; } sb << " ],\n"; - - sb << " \"total_throughput\": { \"out_bytes_sec\": " << overlay_t_out_bytes << ", \"out_pckts_sec\": " << overlay_t_out_pckts << ", \"in_bytes_sec\": " << overlay_t_in_bytes << ", \"in_pckts_sec\": " << overlay_t_in_pckts << " },\n"; + + sb << " \"total_throughput\": { \"out_bytes_sec\": " << overlay_t_out_bytes + << ", \"out_pckts_sec\": " << overlay_t_out_pckts << ", \"in_bytes_sec\": " << overlay_t_in_bytes + << ", \"in_pckts_sec\": " << overlay_t_in_pckts << " },\n"; sb << " \"stats\": {\n"; From 15099f26645d382dba493eb74e645411075c5a09 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 6 Aug 2024 22:11:11 +0300 Subject: [PATCH 109/388] Fix setting bad_peers_ in overlay --- overlay/overlay-peers.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index a21768613..f1621eb44 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -255,12 +255,10 @@ void OverlayImpl::on_ping_result(adnl::AdnlNodeIdShort peer, bool success) { } if (OverlayPeer *p = peer_list_.peers_.get(peer)) { p->on_ping_result(success); - if (!p->is_permanent_member()) { - if (p->is_alive()) { - peer_list_.bad_peers_.erase(peer); - } else { - peer_list_.bad_peers_.insert(peer); - } + if (p->is_alive()) { + peer_list_.bad_peers_.erase(peer); + } else { + peer_list_.bad_peers_.insert(peer); } } } From dcb409462a0f601ac6485428c2056596b5b982cd Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 9 Aug 2024 09:20:42 +0300 Subject: [PATCH 110/388] Cleanup mempool on collators --- validator/manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/manager.cpp b/validator/manager.cpp index 0c4a1a1bc..11454c8e9 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2841,7 +2841,7 @@ void ValidatorManagerImpl::alarm() { } alarm_timestamp().relax(log_ls_stats_at_); if (cleanup_mempool_at_.is_in_past()) { - if (is_validator()) { + if (is_validator() || !collator_nodes_.empty()) { get_external_messages(ShardIdFull{masterchainId, shardIdAll}, [](td::Result, int>>>) {}); get_external_messages(ShardIdFull{basechainId, shardIdAll}, From 7ecc31bc969b792cb4abc22bfe52eb1db7511111 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 9 Aug 2024 15:49:47 +0300 Subject: [PATCH 111/388] Use block data from block candidate in AcceptBlockQuery --- validator/impl/accept-block.cpp | 29 +++++++++++++++++++++++++++++ validator/impl/accept-block.hpp | 2 ++ validator/impl/fabric.cpp | 6 +++--- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index befc17bbf..bada8cca5 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -416,8 +416,37 @@ void AcceptBlockQuery::got_block_handle(BlockHandle handle) { (is_masterchain() ? handle_->inited_proof() && handle_->is_applied() && handle_->inited_is_key_block() : handle_->inited_proof_link())) { finish_query(); + return; + } + if (data_.is_null()) { + td::actor::send_closure(manager_, &ValidatorManager::get_candidate_data_by_block_id_from_db, id_, [SelfId = actor_id(this)](td::Result R) { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &AcceptBlockQuery::got_block_candidate_data, R.move_as_ok()); + } else { + td::actor::send_closure(SelfId, &AcceptBlockQuery::got_block_handle_cont); + } + }); + } else { + got_block_handle_cont(); + } +} + +void AcceptBlockQuery::got_block_candidate_data(td::BufferSlice data) { + auto r_block = create_block(id_, std::move(data)); + if (r_block.is_error()) { + fatal_error("invalid block candidate data in db: " + r_block.error().to_string()); return; } + data_ = r_block.move_as_ok(); + VLOG(VALIDATOR_DEBUG) << "got block candidate data from db"; + if (data_.not_null() && !precheck_header()) { + fatal_error("invalid block header in AcceptBlock"); + return; + } + got_block_handle_cont(); +} + +void AcceptBlockQuery::got_block_handle_cont() { if (data_.not_null() && !handle_->received()) { td::actor::send_closure( manager_, &ValidatorManager::set_block_data, handle_, data_, [SelfId = actor_id(this)](td::Result R) { diff --git a/validator/impl/accept-block.hpp b/validator/impl/accept-block.hpp index dd33ea928..4d547e39b 100644 --- a/validator/impl/accept-block.hpp +++ b/validator/impl/accept-block.hpp @@ -71,6 +71,8 @@ class AcceptBlockQuery : public td::actor::Actor { void written_block_data(); void written_block_signatures(); void got_block_handle(BlockHandle handle); + void got_block_candidate_data(td::BufferSlice data); + void got_block_handle_cont(); void written_block_info(); void got_block_data(td::Ref data); void got_prev_state(td::Ref state); diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index 1b08f7629..8d3fa429b 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -133,9 +133,9 @@ void run_accept_block_query(BlockIdExt id, td::Ref data, std::vector< td::Ref validator_set, td::Ref signatures, td::Ref approve_signatures, bool send_broadcast, bool apply, td::actor::ActorId manager, td::Promise promise) { - td::actor::create_actor("accept", id, std::move(data), prev, std::move(validator_set), - std::move(signatures), std::move(approve_signatures), send_broadcast, apply, - manager, std::move(promise)) + td::actor::create_actor( + PSTRING() << "accept" << id.id.to_str(), id, std::move(data), prev, std::move(validator_set), + std::move(signatures), std::move(approve_signatures), send_broadcast, apply, manager, std::move(promise)) .release(); } From e7da7a558058dcf191ce94be6da673bad18ba801 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 12 Aug 2024 10:07:15 +0300 Subject: [PATCH 112/388] OverlayImpl::get_neighbours bugfix --- overlay/overlay-peers.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index f1621eb44..d37002444 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -624,10 +624,8 @@ std::vector OverlayImpl::get_neighbours(td::uint32 max_si for (td::uint32 i = 0; i < max_size; i++) { td::uint32 t = td::Random::fast(0, static_cast(peer_list_.neighbours_.size()) - 1 - i); td::uint32 j; - for (j = 0; j < i; j++) { - if (ul[j] <= t) { - t++; - } + for (j = 0; j < i && ul[j] <= t; j++) { + t++; } ul.emplace(ul.begin() + j, t); vec.push_back(peer_list_.neighbours_[t]); From 2052b1574fe99503fae6f0564a5e86c49226f831 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 12 Aug 2024 13:56:29 +0300 Subject: [PATCH 113/388] --add-shard parameter --- validator-engine/validator-engine.cpp | 39 +++++++++++++++++++++++++-- validator-engine/validator-engine.hpp | 4 +++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 8191eede5..a6915f409 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -68,6 +68,7 @@ #include #include #include +#include #include "git.h" #include "block-auto.h" #include "block-parse.h" @@ -592,8 +593,11 @@ td::Result Config::config_add_control_process(ton::PublicKeyHash key, td:: } td::Result Config::config_add_shard(ton::ShardIdFull shard) { - if (!shard.is_valid_ext() || shard.is_masterchain()) { - return td::Status::Error(PSTRING() << "invalid shard to collate " << shard.to_str()); + if (shard.is_masterchain()) { + return td::Status::Error("masterchain is monitored by default"); + } + if (!shard.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard " << shard.to_str()); } if (std::find(shards_to_monitor.begin(), shards_to_monitor.end(), shard) != shards_to_monitor.end()) { return false; @@ -1647,6 +1651,14 @@ void ValidatorEngine::load_empty_local_config(td::Promise promise) { } void ValidatorEngine::load_local_config(td::Promise promise) { + for (ton::ShardIdFull shard : add_shard_cmds_) { + auto R = config_.config_add_shard(shard); + if (R.is_error()) { + LOG(WARNING) << "Cannot add shard " << shard.to_str() << " : " << R.move_as_error(); + } else if (R.ok()) { + LOG(WARNING) << "Adding shard to monitor " << shard.to_str(); + } + } if (local_config_.size() == 0) { load_empty_local_config(std::move(promise)); return; @@ -1862,6 +1874,15 @@ void ValidatorEngine::load_config(td::Promise promise) { td::actor::send_closure(keyring_, &ton::keyring::Keyring::add_key_short, key.first, get_key_promise(ig)); } + for (ton::ShardIdFull shard : add_shard_cmds_) { + auto R = config_.config_add_shard(shard); + if (R.is_error()) { + LOG(WARNING) << "Cannot add shard " << shard.to_str() << " : " << R.move_as_error(); + } else if (R.ok()) { + LOG(WARNING) << "Adding shard to monitor " << shard.to_str(); + } + } + write_config(ig.get_promise()); } @@ -4629,6 +4650,20 @@ int main(int argc, char *argv[]) { p.add_option('M', "not-all-shards", "monitor only a necessary set of shards instead of all", [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_not_all_shards); }); }); + p.add_checked_option( + '\0', "add-shard", "add shard to monitor (same as addshard in validator console), format: 0:8000000000000000", + [&](td::Slice arg) -> td::Status { + std::string str = arg.str(); + int wc; + unsigned long long shard; + if (sscanf(str.c_str(), "%d:%016llx", &wc, &shard) != 2) { + return td::Status::Error(PSTRING() << "invalid shard " << str); + } + acts.push_back([=, &x]() { + td::actor::send_closure(x, &ValidatorEngine::add_shard_cmd, ton::ShardIdFull{wc, (ton::ShardId)shard}); + }); + return td::Status::OK(); + }); td::uint32 threads = 7; p.add_checked_option( 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice fname) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 60986b140..6a7680f0e 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -239,6 +239,7 @@ class ValidatorEngine : public td::actor::Actor { std::string session_logs_file_; bool fast_state_serializer_enabled_ = false; bool not_all_shards_ = false; + std::vector add_shard_cmds_; std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; @@ -324,6 +325,9 @@ class ValidatorEngine : public td::actor::Actor { void set_not_all_shards() { not_all_shards_ = true; } + void add_shard_cmd(ton::ShardIdFull shard) { + add_shard_cmds_.push_back(shard); + } void start_up() override; ValidatorEngine() { From 0ca022cc724093877d3c6a58b9575891297e117d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 19 Aug 2024 18:35:50 +0300 Subject: [PATCH 114/388] Improve validator session stats * Collator stats: block limits, number of processed external messages * Collator and validator work time * Last key block seqno * Approvers and signers --- tdutils/td/utils/Timer.cpp | 43 +++++++++++ tdutils/td/utils/Timer.h | 18 +++++ tl/generate/scheme/ton_api.tl | 17 +++-- tl/generate/scheme/ton_api.tlo | Bin 91004 -> 91968 bytes validator-session/validator-session-types.h | 3 + validator-session/validator-session.cpp | 6 ++ validator/impl/collator-impl.h | 5 ++ validator/impl/collator.cpp | 61 ++++++++++++++++ validator/impl/validate-query.cpp | 21 ++++++ validator/impl/validate-query.hpp | 4 ++ validator/interfaces/validator-manager.h | 16 +++++ validator/manager.cpp | 76 ++++++++++++++++---- validator/manager.hpp | 18 ++++- validator/validator-group.cpp | 1 + validator/validator-group.hpp | 11 +-- 15 files changed, 276 insertions(+), 24 deletions(-) diff --git a/tdutils/td/utils/Timer.cpp b/tdutils/td/utils/Timer.cpp index 1f72fba96..24de099aa 100644 --- a/tdutils/td/utils/Timer.cpp +++ b/tdutils/td/utils/Timer.cpp @@ -91,4 +91,47 @@ double PerfWarningTimer::elapsed() const { return Time::now() - start_at_; } +static double thread_cpu_clock() { +#if defined(CLOCK_THREAD_CPUTIME_ID) + timespec ts; + int result = clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts); + CHECK(result == 0); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1e-9; +#else + return 0.0; // TODO: MacOS and Windows support (currently cpu timer is used only in validators) +#endif +} + +ThreadCpuTimer::ThreadCpuTimer(bool is_paused) : is_paused_(is_paused) { + if (is_paused_) { + start_time_ = 0; + } else { + start_time_ = thread_cpu_clock(); + } +} + +void ThreadCpuTimer::pause() { + if (is_paused_) { + return; + } + elapsed_ += thread_cpu_clock() - start_time_; + is_paused_ = true; +} + +void ThreadCpuTimer::resume() { + if (!is_paused_) { + return; + } + start_time_ = thread_cpu_clock(); + is_paused_ = false; +} + +double ThreadCpuTimer::elapsed() const { + double res = elapsed_; + if (!is_paused_) { + res += thread_cpu_clock() - start_time_; + } + return res; +} + } // namespace td diff --git a/tdutils/td/utils/Timer.h b/tdutils/td/utils/Timer.h index 3e0cafbf5..a27cac8a7 100644 --- a/tdutils/td/utils/Timer.h +++ b/tdutils/td/utils/Timer.h @@ -62,4 +62,22 @@ class PerfWarningTimer { std::function callback_; }; +class ThreadCpuTimer { + public: + ThreadCpuTimer() : ThreadCpuTimer(false) { + } + explicit ThreadCpuTimer(bool is_paused); + ThreadCpuTimer(const ThreadCpuTimer &other) = default; + ThreadCpuTimer &operator=(const ThreadCpuTimer &other) = default; + + double elapsed() const; + void pause(); + void resume(); + + private: + double elapsed_{0}; + double start_time_; + bool is_paused_{false}; +}; + } // namespace td diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index b33ca5425..6c5126a11 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -769,13 +769,19 @@ http.server.config dhs:(vector http.server.dnsEntry) local_hosts:(vector http.se ---types--- -validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int comment:string - block_timestamp:double is_accepted:Bool is_ours:Bool got_submit_at:double +validatorSession.collationStats bytes:int gas:int lt_delta:int cat_bytes:int cat_gas:int cat_lt_delta:int + limits_log:string ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int = validadorSession.CollationStats; + +validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int root_hash:int256 file_hash:int256 + comment:string block_timestamp:double is_accepted:Bool is_ours:Bool got_submit_at:double collation_time:double collated_at:double collation_cached:Bool + collation_work_time:double collation_cpu_work_time:double + collation_stats:validatorSession.collationStats validation_time:double validated_at:double validation_cached:Bool + validation_work_time:double validation_cpu_work_time:double gen_utime:double - approved_weight:long approved_33pct_at:double approved_66pct_at:double - signed_weight:long signed_33pct_at:double signed_66pct_at:double + approved_weight:long approved_33pct_at:double approved_66pct_at:double approvers:string + signed_weight:long signed_33pct_at:double signed_66pct_at:double signers:string serialize_time:double deserialize_time:double serialized_size:int = validatorSession.StatsProducer; validatorSession.statsRound timestamp:double producers:(vector validatorSession.statsProducer) = validatorSession.StatsRound; @@ -786,7 +792,8 @@ validatorSession.stats success:Bool id:tonNode.blockIdExt timestamp:double self: first_round:int rounds:(vector validatorSession.statsRound) = validatorSession.Stats; validatorSession.newValidatorGroupStats.node id:int256 weight:long = validatorSession.newValidatorGroupStats.Node; -validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:long cc_seqno:int timestamp:double +validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:long cc_seqno:int + last_key_block_seqno:int timestamp:double self_idx:int nodes:(vector validatorSession.newValidatorGroupStats.node) = validatorSession.NewValidatorGroupStats; ---functions--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index da1aa331d2048803f582b99ede705270f28a22ea..509655394679a5ec39644c3c1973334603d33cf3 100644 GIT binary patch delta 610 zcmex!jP<}dR^CUm^{p77fMX-?d3}|4min3UWr;bNDTyihMZu}X#hLkedd~SdIf*4e zR&YsTN%6!D5}WVne_&);WW;%J@&&^cES-~cgf`b0WiT>!Z+>cQe~J}s%=C*ajKYe@ zIzhrWj|us31MQT4c3+ba^nVR7UuNC;>ijVBsp?Q;!{#{N)jgJh^a^yr3YA z7oV1yQ<7SgI^B?kQ5s1&F*!N4paes>C^aiJ86qq`Il+R11?2wC68{zmvVy#>KKY}O zIwyz)@)O7OxBQHgr`};SnBE}3*Z>Zk?JRvC1v>$lEt8?0mUVUZQDVR zLm8)U6=iG}6@kPiDE5#vH=dW4lt7X}H5w>8-A0UY2^VK#K|xV|S!z-7k(icE86l$stT&R7TW-yLzr4lrwbh6JOCHvmQw-UR>v delta 190 zcmX?bj`hzmR^CUm^{p77fPEwHdHu~21|Jw%_MewHGnvO|1&gOTyZYuGMj4EZU7Hn6 z>`!fVVVok!3X)cz9w@`8&Iw}PJSODBKK)<>qxAF!5k`UO7D9{}(=P}yHh|699wN;6 zfr%AllHBx-(u`u$L^cE?`is>@a WKr=uJr!SCZ>;SX2>&P&Qcmn`#Cp$&} diff --git a/validator-session/validator-session-types.h b/validator-session/validator-session-types.h index e13c36d24..ba166be86 100644 --- a/validator-session/validator-session-types.h +++ b/validator-session/validator-session-types.h @@ -77,6 +77,8 @@ struct ValidatorSessionStats { ValidatorSessionCandidateId candidate_id = ValidatorSessionCandidateId::zero(); int block_status = status_none; double block_timestamp = -1.0; + td::Bits256 root_hash = td::Bits256::zero(); + td::Bits256 file_hash = td::Bits256::zero(); std::string comment; bool is_accepted = false; @@ -159,6 +161,7 @@ struct NewValidatorGroupStats { ValidatorSessionId session_id = ValidatorSessionId::zero(); ShardIdFull shard{masterchainId}; CatchainSeqno cc_seqno = 0; + BlockSeqno last_key_block_seqno = 0; double timestamp = -1.0; td::uint32 self_idx = 0; std::vector nodes; diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 46dd44403..bcd059d36 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -270,6 +270,8 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice } stat->deserialize_time = deserialize_time; stat->serialized_size = data.size(); + stat->root_hash = candidate->root_hash_; + stat->file_hash = file_hash; } if ((td::int32)block_round < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK || @@ -468,6 +470,8 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, ValidatorSessionCan stat->collated_at = td::Clocks::system(); stat->block_timestamp = td::Clocks::system(); stat->collation_cached = collation_cached; + stat->root_hash = root_hash; + stat->file_hash = file_hash; } if (round != cur_round_) { return; @@ -602,6 +606,8 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { if (stat->block_timestamp <= 0.0) { stat->block_timestamp = td::Clocks::system(); } + stat->root_hash = B->root_hash_; + stat->file_hash = td::sha256_bits256(B->data_); } auto P = td::PromiseCreator::lambda([round = cur_round_, hash = block_id, root_hash = block->get_root_hash(), diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 913a0ed87..b8d9e56d3 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -356,6 +356,11 @@ class Collator final : public td::actor::Actor { public: static td::uint32 get_skip_externals_queue_size(); + + private: + td::Timer work_timer_{true}; + td::ThreadCpuTimer cpu_work_timer_{true}; + CollationStats stats_; }; } // namespace validator diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index c6dd7caf2..2dd0f55f6 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -1772,6 +1772,12 @@ bool Collator::register_shard_block_creators(std::vector creator_li * @returns True if collation is successful, false otherwise. */ bool Collator::try_collate() { + work_timer_.resume(); + cpu_work_timer_.resume(); + SCOPE_EXIT { + work_timer_.pause(); + cpu_work_timer_.pause(); + }; if (!preinit_complete) { LOG(WARNING) << "running do_preinit()"; if (!do_preinit()) { @@ -3481,6 +3487,29 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT return true; } +/** + * Creates a string that explains which limit is exceeded. Used for collator stats. + * + * @param block_limit_status Status of block limits. + * @param cls Which limit class is exceeded. + * + * @returns String for collator stats. + */ +static std::string block_full_comment(const block::BlockLimitStatus& block_limit_status, unsigned cls) { + auto bytes = block_limit_status.estimate_block_size(); + if (!block_limit_status.limits.bytes.fits(cls, bytes)) { + return PSTRING() << "block_full bytes " << bytes; + } + if (!block_limit_status.limits.gas.fits(cls, block_limit_status.gas_used)) { + return PSTRING() << "block_full gas " << block_limit_status.gas_used; + } + auto lt_delta = block_limit_status.cur_lt - block_limit_status.limits.start_lt; + if (!block_limit_status.limits.lt_delta.fits(cls, lt_delta)) { + return PSTRING() << "block_full lt_delta " << lt_delta; + } + return ""; +} + /** * Processes inbound internal messages from message queues of the neighbors. * Messages are processed until the normal limit is reached, soft timeout is reached or there are no more messages. @@ -3495,11 +3524,14 @@ bool Collator::process_inbound_internal_messages() { block_full_ = !block_limit_status_->fits(block::ParamLimits::cl_normal); if (block_full_) { LOG(INFO) << "BLOCK FULL, stop processing inbound internal messages"; + stats_.limits_log += PSTRING() << "INBOUND_INT_MESSAGES: " + << block_full_comment(*block_limit_status_, block::ParamLimits::cl_normal) << "\n"; break; } if (soft_timeout_.is_in_past(td::Timestamp::now())) { block_full_ = true; LOG(WARNING) << "soft timeout reached, stop processing inbound internal messages"; + stats_.limits_log += PSTRING() << "INBOUND_INT_MESSAGES: timeout\n"; break; } auto kv = nb_out_msgs_->extract_cur(); @@ -3547,15 +3579,23 @@ bool Collator::process_inbound_external_messages() { } if (full) { LOG(INFO) << "BLOCK FULL, stop processing external messages"; + stats_.limits_log += PSTRING() << "INBOUND_EXT_MESSAGES: " + << block_full_comment(*block_limit_status_, block::ParamLimits::cl_soft) << "\n"; break; } if (medium_timeout_.is_in_past(td::Timestamp::now())) { LOG(WARNING) << "medium timeout reached, stop processing inbound external messages"; + stats_.limits_log += PSTRING() << "INBOUND_EXT_MESSAGES: timeout\n"; break; } auto ext_msg = ext_msg_struct.cell; ton::Bits256 hash{ext_msg->get_hash().bits()}; int r = process_external_message(std::move(ext_msg)); + if (r > 0) { + ++stats_.ext_msgs_accepted; + } else { + ++stats_.ext_msgs_rejected; + } if (r < 0) { bad_ext_msgs_.emplace_back(ext_msg_struct.hash); return false; @@ -3661,11 +3701,15 @@ bool Collator::process_dispatch_queue() { block_full_ = !block_limit_status_->fits(block::ParamLimits::cl_normal); if (block_full_) { LOG(INFO) << "BLOCK FULL, stop processing dispatch queue"; + stats_.limits_log += PSTRING() << "DISPATCH_QUEUE_STAGE_" << iter << ": " + << block_full_comment(*block_limit_status_, block::ParamLimits::cl_normal) + << "\n"; return true; } if (soft_timeout_.is_in_past(td::Timestamp::now())) { block_full_ = true; LOG(WARNING) << "soft timeout reached, stop processing dispatch queue"; + stats_.limits_log += PSTRING() << "DISPATCH_QUEUE_STAGE_" << iter << ": timeout\n"; return true; } StdSmcAddress src_addr; @@ -3715,6 +3759,7 @@ bool Collator::process_dispatch_queue() { ++total_count; if (total_count >= max_total_count[iter]) { dispatch_queue_total_limit_reached_ = true; + stats_.limits_log += PSTRING() << "DISPATCH_QUEUE_STAGE_" << iter << ": total limit reached\n"; break; } } @@ -4064,6 +4109,8 @@ bool Collator::process_new_messages(bool enqueue_only) { if ((block_full_ || have_unprocessed_account_dispatch_queue_) && !enqueue_only) { LOG(INFO) << "BLOCK FULL, enqueue all remaining new messages"; enqueue_only = true; + stats_.limits_log += PSTRING() << "NEW_MESSAGES: " + << block_full_comment(*block_limit_status_, block::ParamLimits::cl_normal) << "\n"; } LOG(DEBUG) << "have message with lt=" << msg.lt; int res = process_one_new_message(std::move(msg), enqueue_only); @@ -5435,6 +5482,18 @@ bool Collator::create_block_candidate() { td::actor::send_closure_later(manager, &ValidatorManager::complete_external_messages, std::move(delay_ext_msgs_), std::move(bad_ext_msgs_)); } + + double work_time = work_timer_.elapsed(); + double cpu_work_time = cpu_work_timer_.elapsed(); + LOG(WARNING) << "Collate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s"; + stats_.bytes = block_limit_status_->estimate_block_size(); + stats_.gas = block_limit_status_->gas_used; + stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt; + stats_.cat_bytes = block_limit_status_->limits.classify_size(stats_.bytes); + stats_.cat_gas = block_limit_status_->limits.classify_gas(stats_.gas); + stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt); + td::actor::send_closure(manager, &ValidatorManager::record_collate_query_stats, block_candidate->id, work_time, + cpu_work_time, std::move(stats_)); return true; } @@ -5539,6 +5598,7 @@ void Collator::after_get_external_messages(td::Result ext_msg_cell = ext_msg->root_cell(); @@ -5550,6 +5610,7 @@ void Collator::after_get_external_messages(td::Resulthash()); } } diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 8c39a1ab4..003b7f9f7 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -118,6 +118,7 @@ bool ValidateQuery::reject_query(std::string error, td::BufferSlice reason) { error = error_ctx() + error; LOG(ERROR) << "REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error; if (main_promise) { + record_stats(); errorlog::ErrorLog::log(PSTRING() << "REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error << ": data=" << block_candidate.id.file_hash.to_hex() << " collated_data=" << block_candidate.collated_file_hash.to_hex()); @@ -155,6 +156,7 @@ bool ValidateQuery::soft_reject_query(std::string error, td::BufferSlice reason) error = error_ctx() + error; LOG(ERROR) << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error; if (main_promise) { + record_stats(); errorlog::ErrorLog::log(PSTRING() << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error << ": data=" << block_candidate.id.file_hash.to_hex() << " collated_data=" << block_candidate.collated_file_hash.to_hex()); @@ -177,6 +179,7 @@ bool ValidateQuery::fatal_error(td::Status error) { error.ensure_error(); LOG(ERROR) << "aborting validation of block candidate for " << shard_.to_str() << " : " << error.to_string(); if (main_promise) { + record_stats(); auto c = error.code(); if (c <= -667 && c >= -670) { errorlog::ErrorLog::log(PSTRING() << "FATAL ERROR: aborting validation of block candidate for " << shard_.to_str() @@ -234,6 +237,7 @@ bool ValidateQuery::fatal_error(std::string err_msg, int err_code) { */ void ValidateQuery::finish_query() { if (main_promise) { + record_stats(); LOG(WARNING) << "validate query done"; main_promise.set_result(now_); } @@ -6764,6 +6768,12 @@ bool ValidateQuery::try_validate() { if (pending) { return true; } + work_timer_.resume(); + cpu_work_timer_.resume(); + SCOPE_EXIT { + work_timer_.pause(); + cpu_work_timer_.pause(); + }; try { if (!stage_) { LOG(WARNING) << "try_validate stage 0"; @@ -6903,6 +6913,17 @@ void ValidateQuery::written_candidate() { finish_query(); } +/** + * Sends validation work time to manager. + */ +void ValidateQuery::record_stats() { + double work_time = work_timer_.elapsed(); + double cpu_work_time = cpu_work_timer_.elapsed(); + LOG(WARNING) << "Validate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s"; + td::actor::send_closure(manager, &ValidatorManager::record_validate_query_stats, block_candidate.id, work_time, + cpu_work_time); +} + } // namespace validator } // namespace ton diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 824afb49d..104950938 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -398,6 +398,10 @@ class ValidateQuery : public td::actor::Actor { } return true; } + + td::Timer work_timer_{true}; + td::ThreadCpuTimer cpu_work_timer_{true}; + void record_stats(); }; } // namespace validator diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 0e9fab73b..5ea074ddf 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -52,6 +52,16 @@ struct AsyncSerializerState { UnixTime last_written_block_ts; }; +struct CollationStats { + td::uint32 bytes, gas, lt_delta; + int cat_bytes, cat_gas, cat_lt_delta; + std::string limits_log; + td::uint32 ext_msgs_total = 0; + td::uint32 ext_msgs_filtered = 0; + td::uint32 ext_msgs_accepted = 0; + td::uint32 ext_msgs_rejected = 0; +}; + using ValidateCandidateResult = td::Variant; class ValidatorManager : public ValidatorManagerInterface { @@ -192,6 +202,12 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void add_lite_query_stats(int lite_query_id) { } + virtual void record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, + CollationStats stats) { + } + virtual void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) { + } + static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) { return ts / (1 << 17) != prev_ts / (1 << 17); } diff --git a/validator/manager.cpp b/validator/manager.cpp index eb082d91e..ec3dbce8d 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -44,6 +44,7 @@ #include "td/utils/JsonBuilder.h" #include "common/delay.h" +#include "td/utils/filesystem.h" #include "validator/stats-merger.h" @@ -2044,7 +2045,7 @@ void ValidatorManagerImpl::update_shards() { } new_validator_groups_.emplace(val_group_id, std::move(it2->second)); } else { - auto G = create_validator_group(val_group_id, shard, val_set, opts, started_); + auto G = create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_); if (!G.empty()) { td::actor::send_closure(G, &ValidatorGroup::start, prev, last_masterchain_block_id_, last_masterchain_state_->get_unix_time()); @@ -2100,7 +2101,7 @@ void ValidatorManagerImpl::update_shards() { } new_validator_groups_.emplace(val_group_id, std::move(it2->second)); } else { - auto G = create_validator_group(val_group_id, shard, val_set, opts, started_); + auto G = create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_); if (!G.empty()) { td::actor::send_closure(G, &ValidatorGroup::start, prev, last_masterchain_block_id_, last_masterchain_state_->get_unix_time()); @@ -2127,7 +2128,7 @@ void ValidatorManagerImpl::update_shards() { } else { new_next_validator_groups_.emplace( val_group_id, - ValidatorGroupEntry{create_validator_group(val_group_id, shard, val_set, opts, started_), shard}); + ValidatorGroupEntry{create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_), shard}); } } } @@ -2230,7 +2231,7 @@ ValidatorSessionId ValidatorManagerImpl::get_validator_set_id(ShardIdFull shard, } td::actor::ActorOwn ValidatorManagerImpl::create_validator_group( - ValidatorSessionId session_id, ShardIdFull shard, td::Ref validator_set, + ValidatorSessionId session_id, ShardIdFull shard, td::Ref validator_set, BlockSeqno key_seqno, validatorsession::ValidatorSessionOptions opts, bool init_session) { if (check_gc_list_.count(session_id) == 1) { return td::actor::ActorOwn{}; @@ -2241,8 +2242,8 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group auto validator_id = get_validator(shard, validator_set); CHECK(!validator_id.is_zero()); auto G = td::actor::create_actor( - "validatorgroup", shard, validator_id, session_id, validator_set, opts, keyring_, adnl_, rldp_, overlays_, - db_root_, actor_id(this), init_session, + "validatorgroup", shard, validator_id, session_id, validator_set, key_seqno, opts, keyring_, adnl_, rldp_, + overlays_, db_root_, actor_id(this), init_session, opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_); return G; } @@ -2831,13 +2832,35 @@ void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id, for (const auto &round : stats.rounds) { std::vector> producers; for (const auto &producer : round.producers) { + BlockIdExt cur_block_id{block_id.id, producer.root_hash, producer.file_hash}; + auto it = recorded_block_stats_.find(cur_block_id); + tl_object_ptr collation_stats; + if (it != recorded_block_stats_.end() && it->second.collator_stats_) { + auto &stats = it->second.collator_stats_.value(); + collation_stats = create_tl_object( + stats.bytes, stats.gas, stats.lt_delta, stats.cat_bytes, stats.cat_gas, stats.cat_lt_delta, + stats.limits_log, stats.ext_msgs_total, stats.ext_msgs_filtered, stats.ext_msgs_accepted, + stats.ext_msgs_rejected); + } + std::string approvers, signers; + for (bool x : producer.approvers) { + approvers += (x ? '1' : '0'); + } + for (bool x : producer.signers) { + signers += (x ? '1' : '0'); + } producers.push_back(create_tl_object( - producer.id.bits256_value(), producer.candidate_id, producer.block_status, producer.comment, - producer.block_timestamp, producer.is_accepted, producer.is_ours, producer.got_submit_at, - producer.collation_time, producer.collated_at, producer.collation_cached, producer.validation_time, - producer.validated_at, producer.validation_cached, producer.gen_utime, producer.approved_weight, - producer.approved_33pct_at, producer.approved_66pct_at, producer.signed_weight, producer.signed_33pct_at, - producer.signed_66pct_at, producer.serialize_time, producer.deserialize_time, producer.serialized_size)); + producer.id.bits256_value(), producer.candidate_id, producer.block_status, producer.root_hash, + producer.file_hash, producer.comment, producer.block_timestamp, producer.is_accepted, producer.is_ours, + producer.got_submit_at, producer.collation_time, producer.collated_at, producer.collation_cached, + it == recorded_block_stats_.end() ? -1.0 : it->second.collator_work_time_, + it == recorded_block_stats_.end() ? -1.0 : it->second.collator_cpu_work_time_, std::move(collation_stats), + producer.validation_time, producer.validated_at, producer.validation_cached, + it == recorded_block_stats_.end() ? -1.0 : it->second.validator_work_time_, + it == recorded_block_stats_.end() ? -1.0 : it->second.validator_cpu_work_time_, producer.gen_utime, + producer.approved_weight, producer.approved_33pct_at, producer.approved_66pct_at, std::move(approvers), + producer.signed_weight, producer.signed_33pct_at, producer.signed_66pct_at, std::move(signers), + producer.serialize_time, producer.deserialize_time, producer.serialized_size)); } rounds.push_back(create_tl_object(round.timestamp, std::move(producers))); } @@ -2869,8 +2892,8 @@ void ValidatorManagerImpl::log_new_validator_group_stats(validatorsession::NewVa create_tl_object(node.id.bits256_value(), node.weight)); } auto obj = create_tl_object( - stats.session_id, stats.shard.workchain, stats.shard.shard, stats.cc_seqno, stats.timestamp, stats.self_idx, - std::move(nodes)); + stats.session_id, stats.shard.workchain, stats.shard.shard, stats.cc_seqno, stats.last_key_block_seqno, + stats.timestamp, stats.self_idx, std::move(nodes)); auto s = td::json_encode(td::ToJson(*obj.get()), false); s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); @@ -3165,6 +3188,31 @@ td::actor::ActorOwn ValidatorManagerFactory::create( rldp, overlays); } +void ValidatorManagerImpl::record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, + CollationStats stats) { + auto &record = new_block_stats_record(block_id); + record.collator_work_time_ = work_time; + record.collator_cpu_work_time_ = cpu_work_time; + record.collator_stats_ = std::move(stats); +} + +void ValidatorManagerImpl::record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) { + auto &record = new_block_stats_record(block_id); + record.validator_work_time_ = work_time; + record.validator_cpu_work_time_ = cpu_work_time; +} + +ValidatorManagerImpl::RecordedBlockStats &ValidatorManagerImpl::new_block_stats_record(BlockIdExt block_id) { + if (!recorded_block_stats_.count(block_id)) { + recorded_block_stats_lru_.push(block_id); + if (recorded_block_stats_lru_.size() > 4096) { + recorded_block_stats_.erase(recorded_block_stats_lru_.front()); + recorded_block_stats_lru_.pop(); + } + } + return recorded_block_stats_[block_id]; +} + size_t ValidatorManagerImpl::CheckedExtMsgCounter::get_msg_count(WorkchainId wc, StdSmcAddress addr) { before_query(); auto it1 = counter_cur_.find({wc, addr}); diff --git a/validator/manager.hpp b/validator/manager.hpp index 12354c634..50fa79cca 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -38,6 +38,7 @@ #include #include #include +#include namespace ton { @@ -261,7 +262,7 @@ class ValidatorManagerImpl : public ValidatorManager { BlockSeqno last_key_block_seqno, const validatorsession::ValidatorSessionOptions &opts); td::actor::ActorOwn create_validator_group(ValidatorSessionId session_id, ShardIdFull shard, - td::Ref validator_set, + td::Ref validator_set, BlockSeqno key_seqno, validatorsession::ValidatorSessionOptions opts, bool create_catchain); struct ValidatorGroupEntry { @@ -708,6 +709,21 @@ class ValidatorManagerImpl : public ValidatorManager { td::uint32 ls_stats_check_ext_messages_{0}; td::actor::ActorOwn candidates_buffer_; + + struct RecordedBlockStats { + double collator_work_time_ = -1.0; + double collator_cpu_work_time_ = -1.0; + td::optional collator_stats_; + double validator_work_time_ = -1.0; + double validator_cpu_work_time_ = -1.0; + }; + std::map recorded_block_stats_; + std::queue recorded_block_stats_lru_; + + void record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, + CollationStats stats) override; + void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) override; + RecordedBlockStats &new_block_stats_record(BlockIdExt block_id); }; } // namespace validator diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index fc3ebe541..60573581b 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -386,6 +386,7 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch stats.session_id = session_id_; stats.shard = shard_; stats.cc_seqno = validator_set_->get_catchain_seqno(); + stats.last_key_block_seqno = last_key_block_seqno_; stats.timestamp = td::Clocks::system(); td::uint32 idx = 0; for (const auto& node : validator_set_->export_vector()) { diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 3499da9d7..936d2fdc7 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -69,15 +69,17 @@ class ValidatorGroup : public td::actor::Actor { } ValidatorGroup(ShardIdFull shard, PublicKeyHash local_id, ValidatorSessionId session_id, - td::Ref validator_set, validatorsession::ValidatorSessionOptions config, - td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays, - std::string db_root, td::actor::ActorId validator_manager, bool create_session, + td::Ref validator_set, BlockSeqno last_key_block_seqno, + validatorsession::ValidatorSessionOptions config, td::actor::ActorId keyring, + td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId overlays, std::string db_root, + td::actor::ActorId validator_manager, bool create_session, bool allow_unsafe_self_blocks_resync, td::Ref opts) : shard_(shard) , local_id_(std::move(local_id)) , session_id_(session_id) , validator_set_(std::move(validator_set)) + , last_key_block_seqno_(last_key_block_seqno) , config_(std::move(config)) , keyring_(keyring) , adnl_(adnl) @@ -115,6 +117,7 @@ class ValidatorGroup : public td::actor::Actor { UnixTime min_ts_; td::Ref validator_set_; + BlockSeqno last_key_block_seqno_; validatorsession::ValidatorSessionOptions config_; td::actor::ActorId keyring_; From 1b4fb42859c84730d79b88372160f892c918d8b1 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 19 Aug 2024 19:55:11 +0300 Subject: [PATCH 115/388] End validator session stats --- catchain/catchain.h | 1 + catchain/catchain.hpp | 9 +++++++ tl/generate/scheme/ton_api.tl | 8 ++++-- tl/generate/scheme/ton_api.tlo | Bin 91968 -> 92472 bytes validator-session/validator-session-types.h | 11 +++++++++ validator-session/validator-session.cpp | 23 +++++++++++++++++ validator-session/validator-session.h | 1 + validator-session/validator-session.hpp | 1 + validator/impl/collator.cpp | 2 ++ validator/interfaces/validator-manager.h | 1 + validator/manager-disk.hpp | 3 +++ validator/manager-hardfork.hpp | 3 +++ validator/manager.cpp | 26 +++++++++++++++++++- validator/manager.hpp | 1 + validator/validator-group.cpp | 10 ++++++++ 15 files changed, 97 insertions(+), 3 deletions(-) diff --git a/catchain/catchain.h b/catchain/catchain.h index 912957e56..c5c8af28d 100644 --- a/catchain/catchain.h +++ b/catchain/catchain.h @@ -96,6 +96,7 @@ class CatChain : public td::actor::Actor { virtual void send_query_via(const PublicKeyHash &dst, std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice query, td::uint64 max_answer_size, td::actor::ActorId via) = 0; + virtual void get_source_heights(td::Promise> promise) = 0; virtual void destroy() = 0; static td::actor::ActorOwn create(std::unique_ptr callback, const CatChainOptions &opts, diff --git a/catchain/catchain.hpp b/catchain/catchain.hpp index 8c8bb99ae..586cf4744 100644 --- a/catchain/catchain.hpp +++ b/catchain/catchain.hpp @@ -115,6 +115,15 @@ class CatChainImpl : public CatChain { td::actor::send_closure(receiver_, &CatChainReceiverInterface::send_custom_query_data_via, dst, name, std::move(promise), timeout, std::move(query), max_answer_size, via); } + void get_source_heights(td::Promise> promise) override { + std::vector heights(top_source_blocks_.size(), 0); + for (size_t i = 0; i < top_source_blocks_.size(); ++i) { + if (top_source_blocks_[i]) { + heights[i] = top_source_blocks_[i]->height(); + } + } + promise.set_result(std::move(heights)); + } void destroy() override; CatChainImpl(std::unique_ptr callback, const CatChainOptions &opts, td::actor::ActorId keyring, td::actor::ActorId adnl, diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 6c5126a11..bf919b0fd 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -388,8 +388,8 @@ tonNode.newShardBlock block:tonNode.blockIdExt cc_seqno:int data:bytes = tonNode tonNode.blockBroadcastCompressed.data signatures:(vector tonNode.blockSignature) proof_data:bytes = tonNode.blockBroadcaseCompressed.Data; -tonNode.blockBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int - signatures:(vector tonNode.blockSignature) +tonNode.blockBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int + signatures:(vector tonNode.blockSignature) proof:bytes data:bytes = tonNode.Broadcast; tonNode.blockBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int flags:# compressed:bytes = tonNode.Broadcast; @@ -796,6 +796,10 @@ validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:lo last_key_block_seqno:int timestamp:double self_idx:int nodes:(vector validatorSession.newValidatorGroupStats.node) = validatorSession.NewValidatorGroupStats; +validatorSession.endValidatorGroupStats.node id:int256 catchain_blocks:int = validatorSession.endValidatorGroupStats.Node; +validatorSession.endValidatorGroupStats session_id:int256 timestamp:double + nodes:(vector validatorSession.endValidatorGroupStats.node) = validatorSession.EndValidatorGroupStats; + ---functions--- ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 509655394679a5ec39644c3c1973334603d33cf3..337dd071e3f746cd822236e2dcb63c194b379de4 100644 GIT binary patch delta 267 zcmX?bj&;W+R^CUm^{p77fNLY~VMEDjuZ-f=%Mx=kQxZ$^i-J>&i!<}{^jz~&CMz0g zZ@yp{!6@0H?XyS+LrE%7iKDUcWDVmVjJ=x`O~miwFbkwm(MXsRWYf)KLO#sX4@NLb zPj3)m6qv3d!l*L2Cs9QLWCtVEVMWetIvlob2SFOP>j*L`aN~7?2zOq7N-9t<$c)Jk kx- nodes; }; +struct EndValidatorGroupStats { + struct Node { + PublicKeyHash id = PublicKeyHash::zero(); + td::uint32 catchain_blocks = 0; + }; + + ValidatorSessionId session_id = ValidatorSessionId::zero(); + double timestamp = -1.0; + std::vector nodes; +}; + } // namespace validatorsession } // namespace ton diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index bcd059d36..be5443785 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -1003,6 +1003,29 @@ void ValidatorSessionImpl::get_current_stats(td::Promise promise.set_result(cur_stats_); } +void ValidatorSessionImpl::get_end_stats(td::Promise promise) { + if (!started_) { + promise.set_error(td::Status::Error(ErrorCode::notready, "not started")); + return; + } + EndValidatorGroupStats stats; + stats.session_id = unique_hash_; + stats.timestamp = td::Clocks::system(); + stats.nodes.resize(description().get_total_nodes()); + for (size_t i = 0; i < stats.nodes.size(); ++i) { + stats.nodes[i].id = description().get_source_id(i); + } + td::actor::send_closure(catchain_, &catchain::CatChain::get_source_heights, + [promise = std::move(promise), + stats = std::move(stats)](td::Result> R) mutable { + TRY_RESULT_PROMISE(promise, heights, std::move(R)); + for (size_t i = 0; i < std::min(heights.size(), stats.nodes.size()); ++i) { + stats.nodes[i].catchain_blocks = heights[i]; + } + promise.set_result(std::move(stats)); + }); +} + void ValidatorSessionImpl::get_validator_group_info_for_litequery( td::uint32 cur_round, td::Promise>> promise) { diff --git a/validator-session/validator-session.h b/validator-session/validator-session.h index 0870f6718..2e1ed9b13 100644 --- a/validator-session/validator-session.h +++ b/validator-session/validator-session.h @@ -105,6 +105,7 @@ class ValidatorSession : public td::actor::Actor { virtual void start() = 0; virtual void destroy() = 0; virtual void get_current_stats(td::Promise promise) = 0; + virtual void get_end_stats(td::Promise promise) = 0; virtual void get_validator_group_info_for_litequery( td::uint32 cur_round, td::Promise>> promise) = 0; diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 580582824..2ee4885b9 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -187,6 +187,7 @@ class ValidatorSessionImpl : public ValidatorSession { void start() override; void destroy() override; void get_current_stats(td::Promise promise) override; + void get_end_stats(td::Promise promise) override; void get_validator_group_info_for_litequery( td::uint32 cur_round, td::Promise>> promise) override; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 2dd0f55f6..f465c0f55 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -4119,6 +4119,8 @@ bool Collator::process_new_messages(bool enqueue_only) { } else if (res == 3) { LOG(INFO) << "All remaining new messages must be enqueued (BLOCK FULL)"; enqueue_only = true; + stats_.limits_log += PSTRING() << "NEW_MESSAGES: " + << block_full_comment(*block_limit_status_, block::ParamLimits::cl_normal) << "\n"; } } return true; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 5ea074ddf..b6016bc2b 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -183,6 +183,7 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) = 0; virtual void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) = 0; + virtual void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) = 0; virtual void get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) = 0; virtual void get_block_data_for_litequery(BlockIdExt block_id, td::Promise> promise) = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index a77be2725..3a77f2301 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -388,6 +388,9 @@ class ValidatorManagerImpl : public ValidatorManager { void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override { UNREACHABLE(); } + void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override { + UNREACHABLE(); + } void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { queue_size_counter_ = diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index e7175b77b..cf4d3799f 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -450,6 +450,9 @@ class ValidatorManagerImpl : public ValidatorManager { void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override { UNREACHABLE(); } + void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override { + UNREACHABLE(); + } void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { queue_size_counter_ = diff --git a/validator/manager.cpp b/validator/manager.cpp index ec3dbce8d..fa592a788 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2902,7 +2902,31 @@ void ValidatorManagerImpl::log_new_validator_group_stats(validatorsession::NewVa file << s << "\n"; file.close(); - LOG(INFO) << "Writing new validator group stats for " << stats.shard.to_str(); + LOG(INFO) << "Writing new validator group stats for " << stats.session_id << " shard=" << stats.shard.to_str() + << " cc_seqno=" << stats.cc_seqno; +} + +void ValidatorManagerImpl::log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) { + std::string fname = opts_->get_session_logs_file(); + if (fname.empty()) { + return; + } + std::vector> nodes; + for (const auto &node : stats.nodes) { + nodes.push_back(create_tl_object(node.id.bits256_value(), + node.catchain_blocks)); + } + auto obj = create_tl_object(stats.session_id, stats.timestamp, + std::move(nodes)); + auto s = td::json_encode(td::ToJson(*obj.get()), false); + s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); + + std::ofstream file; + file.open(fname, std::ios_base::app); + file << s << "\n"; + file.close(); + + LOG(INFO) << "Writing end validator group stats for " << stats.session_id; } void ValidatorManagerImpl::get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 50fa79cca..99aa4e0e1 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -590,6 +590,7 @@ class ValidatorManagerImpl : public ValidatorManager { void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override; void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override; + void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override; void update_options(td::Ref opts) override; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 60573581b..4b61c07cd 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -418,6 +418,16 @@ void ValidatorGroup::destroy() { td::actor::send_closure(manager, &ValidatorManager::log_validator_session_stats, block_id, std::move(stats)); }); + td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_end_stats, + [manager = manager_](td::Result R) { + if (R.is_error()) { + LOG(DEBUG) << "Failed to get validator session end stats: " << R.move_as_error(); + return; + } + auto stats = R.move_as_ok(); + td::actor::send_closure(manager, &ValidatorManager::log_end_validator_group_stats, + std::move(stats)); + }); auto ses = session_.release(); delay_action([ses]() mutable { td::actor::send_closure(ses, &validatorsession::ValidatorSession::destroy); }, td::Timestamp::in(10.0)); From dbe51d6d138c9f8ec1acd928916158ce09283718 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 16 Aug 2024 10:14:54 +0300 Subject: [PATCH 116/388] Status page in blockchain-explorer --- .../blockchain-explorer-query.cpp | 2 +- blockchain-explorer/blockchain-explorer.cpp | 66 +++++++++---- lite-client/ext-client.cpp | 94 +++++++++++++++---- lite-client/ext-client.h | 9 +- 4 files changed, 132 insertions(+), 39 deletions(-) diff --git a/blockchain-explorer/blockchain-explorer-query.cpp b/blockchain-explorer/blockchain-explorer-query.cpp index 1808a3c46..26a6787e1 100644 --- a/blockchain-explorer/blockchain-explorer-query.cpp +++ b/blockchain-explorer/blockchain-explorer-query.cpp @@ -1432,7 +1432,7 @@ void HttpQueryStatus::finish_query() { for (td::uint32 i = 0; i < results_.ips.size(); i++) { A << ""; if (results_.ips[i].is_valid()) { - A << "" << results_.ips[i] << ""; + A << "" << results_.ips[i].get_ip_str() << ":" << results_.ips[i].get_port() << ""; } else { A << "hidden"; } diff --git a/blockchain-explorer/blockchain-explorer.cpp b/blockchain-explorer/blockchain-explorer.cpp index 029c718d6..ca50d5266 100644 --- a/blockchain-explorer/blockchain-explorer.cpp +++ b/blockchain-explorer/blockchain-explorer.cpp @@ -155,8 +155,12 @@ class CoreActor : public CoreActorInterface { td::int32 attempt_ = 0; td::int32 waiting_ = 0; - //void run_queries(); - void got_result(td::uint32 idx, td::int32 attempt, td::Result data); + size_t n_servers_ = 0; + + void run_queries(); + void got_servers_ready(td::int32 attempt, std::vector ready); + void send_ping(td::uint32 idx); + void got_ping_result(td::uint32 idx, td::int32 attempt, td::Result data); void add_result() { if (new_result_) { @@ -175,7 +179,7 @@ class CoreActor : public CoreActorInterface { add_result(); } attempt_ = t; - //run_queries(); + run_queries(); alarm_timestamp() = td::Timestamp::at_unix((attempt_ + 1) * 60); } @@ -450,7 +454,8 @@ class CoreActor : public CoreActorInterface { addrs_.push_back(remote_addr_); servers.push_back(liteclient::LiteServerConfig{ton::adnl::AdnlNodeIdFull{remote_public_key_}, remote_addr_}); } - client_ = liteclient::ExtClient::create(std::move(servers), make_callback()); + n_servers_ = servers.size(); + client_ = liteclient::ExtClient::create(std::move(servers), make_callback(), true); daemon_ = MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, static_cast(http_port_), nullptr, nullptr, &process_http_request, nullptr, MHD_OPTION_NOTIFY_COMPLETED, request_completed, nullptr, MHD_OPTION_THREAD_POOL_SIZE, 16, MHD_OPTION_END); @@ -458,7 +463,46 @@ class CoreActor : public CoreActorInterface { } }; -void CoreActor::got_result(td::uint32 idx, td::int32 attempt, td::Result R) { +void CoreActor::run_queries() { + waiting_ = 0; + new_result_ = std::make_shared(n_servers_, td::Timestamp::at_unix(attempt_ * 60)); + td::actor::send_closure(client_, &liteclient::ExtClient::get_servers_status, + [SelfId = actor_id(this), attempt = attempt_](td::Result> R) { + R.ensure(); + td::actor::send_closure(SelfId, &CoreActor::got_servers_ready, attempt, R.move_as_ok()); + }); +} + +void CoreActor::got_servers_ready(td::int32 attempt, std::vector ready) { + if (attempt != attempt_) { + return; + } + CHECK(ready.size() == n_servers_); + for (td::uint32 i = 0; i < n_servers_; i++) { + if (ready[i]) { + send_ping(i); + } + } + CHECK(waiting_ >= 0); + if (waiting_ == 0) { + add_result(); + } +} + +void CoreActor::send_ping(td::uint32 idx) { + waiting_++; + auto query = ton::create_tl_object(); + auto q = ton::create_tl_object(serialize_tl_object(query, true)); + + auto P = + td::PromiseCreator::lambda([SelfId = actor_id(this), idx, attempt = attempt_](td::Result R) { + td::actor::send_closure(SelfId, &CoreActor::got_ping_result, idx, attempt, std::move(R)); + }); + td::actor::send_closure(client_, &liteclient::ExtClient::send_query_to_server, "query", serialize_tl_object(q, true), + idx, td::Timestamp::in(10.0), std::move(P)); +} + +void CoreActor::got_ping_result(td::uint32 idx, td::int32 attempt, td::Result R) { if (attempt != attempt_) { return; } @@ -499,18 +543,6 @@ void CoreActor::got_result(td::uint32 idx, td::int32 attempt, td::Result(ready_.size(), td::Timestamp::at_unix(attempt_ * 60)); - for (td::uint32 i = 0; i < ready_.size(); i++) { - send_query(i); - } - CHECK(waiting_ >= 0); - if (waiting_ == 0) { - add_result(); - } -}*/ - void CoreActor::send_lite_query(td::BufferSlice query, td::Promise promise) { auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { diff --git a/lite-client/ext-client.cpp b/lite-client/ext-client.cpp index 0232c28f4..a0e48e64a 100644 --- a/lite-client/ext-client.cpp +++ b/lite-client/ext-client.cpp @@ -22,8 +22,8 @@ namespace liteclient { class ExtClientImpl : public ExtClient { public: - ExtClientImpl(std::vector liteservers, td::unique_ptr callback) - : callback_(std::move(callback)) { + ExtClientImpl(std::vector liteservers, td::unique_ptr callback, bool connect_to_all) + : callback_(std::move(callback)), connect_to_all_(connect_to_all) { CHECK(!liteservers.empty()); servers_.resize(liteservers.size()); for (size_t i = 0; i < servers_.size(); ++i) { @@ -36,15 +36,65 @@ class ExtClientImpl : public ExtClient { LOG(INFO) << "Started ext client, " << servers_.size() << " liteservers"; td::Random::Fast rnd; td::random_shuffle(td::as_mutable_span(servers_), rnd); + server_indices_.resize(servers_.size()); + for (size_t i = 0; i < servers_.size(); ++i) { + server_indices_[servers_[i].idx] = i; + } + + if (connect_to_all_) { + for (size_t i = 0; i < servers_.size(); ++i) { + prepare_server(i, nullptr); + } + } } void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, td::Promise promise) override { QueryInfo query_info = get_query_info(data); TRY_RESULT_PROMISE(promise, server_idx, select_server(query_info)); + send_query_internal(std::move(name), std::move(data), std::move(query_info), server_idx, timeout, + std::move(promise)); + } + + void send_query_to_server(std::string name, td::BufferSlice data, size_t server_idx, td::Timestamp timeout, + td::Promise promise) override { + if (server_idx >= servers_.size()) { + promise.set_error(td::Status::Error(PSTRING() << "server idx " << server_idx << " is too big")); + return; + } + server_idx = server_indices_[server_idx]; + QueryInfo query_info = get_query_info(data); + prepare_server(server_idx, &query_info); + send_query_internal(std::move(name), std::move(data), std::move(query_info), server_idx, timeout, + std::move(promise)); + } + + void get_servers_status(td::Promise> promise) override { + std::vector status(servers_.size()); + for (const Server& s : servers_) { + status[s.idx] = s.alive; + } + promise.set_result(std::move(status)); + } + + void reset_servers() override { + LOG(INFO) << "Force resetting all liteservers"; + for (Server& server : servers_) { + server.alive = false; + server.timeout = {}; + server.ignore_until = {}; + server.client.reset(); + } + } + + private: + void send_query_internal(std::string name, td::BufferSlice data, QueryInfo query_info, size_t server_idx, + td::Timestamp timeout, td::Promise promise) { auto& server = servers_[server_idx]; CHECK(!server.client.empty()); - alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT)); + if (!connect_to_all_) { + alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT)); + } td::Promise P = [SelfId = actor_id(this), server_idx, promise = std::move(promise)](td::Result R) mutable { if (R.is_error() && @@ -59,17 +109,6 @@ class ExtClientImpl : public ExtClient { std::move(P)); } - void reset_servers() override { - LOG(INFO) << "Force resetting all liteservers"; - for (Server& server : servers_) { - server.alive = false; - server.timeout = {}; - server.ignore_until = {}; - server.client.reset(); - } - } - - private: td::Result select_server(const QueryInfo& query_info) { for (size_t i = 0; i < servers_.size(); ++i) { if (servers_[i].alive && servers_[i].config.accepts_query(query_info)) { @@ -101,12 +140,22 @@ class ExtClientImpl : public ExtClient { if (server_idx == servers_.size()) { return td::Status::Error(PSTRING() << "no liteserver for query " << query_info.to_str()); } + prepare_server(server_idx, &query_info); + return server_idx; + } + + void prepare_server(size_t server_idx, const QueryInfo* query_info) { Server& server = servers_[server_idx]; + if (server.alive) { + return; + } server.alive = true; server.ignore_until = {}; - alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT)); + if (!connect_to_all_) { + alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT)); + } if (!server.client.empty()) { - return server_idx; + return; } class Callback : public ton::adnl::AdnlExtClient::Callback { @@ -124,10 +173,9 @@ class ExtClientImpl : public ExtClient { size_t idx_; }; LOG(INFO) << "Connecting to liteserver #" << server.idx << " (" << server.config.addr.get_ip_str() << ":" - << server.config.addr.get_port() << ") for query " << query_info.to_str(); + << server.config.addr.get_port() << ") for query " << (query_info ? query_info->to_str() : "[none]"); server.client = ton::adnl::AdnlExtClient::create(server.config.adnl_id, server.config.addr, std::make_unique(actor_id(this), server_idx)); - return server_idx; } struct Server { @@ -139,12 +187,17 @@ class ExtClientImpl : public ExtClient { td::Timestamp ignore_until = td::Timestamp::never(); }; std::vector servers_; + std::vector server_indices_; td::unique_ptr callback_; + bool connect_to_all_ = false; static constexpr double MAX_NO_QUERIES_TIMEOUT = 100.0; static constexpr double BAD_SERVER_TIMEOUT = 30.0; void alarm() override { + if (connect_to_all_) { + return; + } for (Server& server : servers_) { if (server.timeout && server.timeout.is_in_past()) { LOG(INFO) << "Closing connection to liteserver #" << server.idx << " (" << server.config.addr.get_ip_str() @@ -168,7 +221,8 @@ td::actor::ActorOwn ExtClient::create(ton::adnl::AdnlNodeIdFull dst, } td::actor::ActorOwn ExtClient::create(std::vector liteservers, - td::unique_ptr callback) { - return td::actor::create_actor("ExtClient", std::move(liteservers), std::move(callback)); + td::unique_ptr callback, bool connect_to_all) { + return td::actor::create_actor("ExtClient", std::move(liteservers), std::move(callback), + connect_to_all); } } // namespace liteclient diff --git a/lite-client/ext-client.h b/lite-client/ext-client.h index adcff9bed..ef4523fd6 100644 --- a/lite-client/ext-client.h +++ b/lite-client/ext-client.h @@ -30,12 +30,19 @@ class ExtClient : public td::actor::Actor { virtual void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, td::Promise promise) = 0; + virtual void send_query_to_server(std::string name, td::BufferSlice data, size_t server_idx, td::Timestamp timeout, + td::Promise promise) { + promise.set_error(td::Status::Error("not supported")); + } + virtual void get_servers_status(td::Promise> promise) { + promise.set_error(td::Status::Error("not supported")); + } virtual void reset_servers() { } static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, td::unique_ptr callback); static td::actor::ActorOwn create(std::vector liteservers, - td::unique_ptr callback); + td::unique_ptr callback, bool connect_to_all = false); }; } // namespace liteclient From faedb4635cf9c8328777a69a6c9e728a22f38dba Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 19 Aug 2024 20:31:33 +0300 Subject: [PATCH 117/388] Remove unused get validator session stats query --- tl/generate/scheme/ton_api.tl | 6 ---- tl/generate/scheme/ton_api.tlo | Bin 101736 -> 100916 bytes .../validator-engine-console-query.cpp | 20 ----------- .../validator-engine-console-query.h | 19 ---------- .../validator-engine-console.cpp | 1 - validator-engine/validator-engine.cpp | 27 -------------- validator-engine/validator-engine.hpp | 2 -- validator-session/validator-session.cpp | 12 ------- validator-session/validator-session.h | 2 -- validator-session/validator-session.hpp | 2 -- validator/manager-disk.hpp | 5 --- validator/manager-hardfork.hpp | 4 --- validator/manager.cpp | 33 ------------------ validator/manager.hpp | 3 -- validator/validator-group.cpp | 19 ---------- validator/validator-group.hpp | 2 -- validator/validator.h | 3 -- 17 files changed, 160 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 70cb790b7..cc2cc67e1 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -712,10 +712,6 @@ engine.validator.perfTimerStats stats:(vector engine.validator.PerfTimerStatsByN engine.validator.shardOutQueueSize size:long = engine.validator.ShardOutQueueSize; -engine.validator.validatorSessionInfo current_block:tonNode.blockId self:int256 current_round:int next_producers:(vector int256) = engine.validator.ValidatorSessionInfo; -engine.validator.validatorSessionsInfo sessions:(vector engine.validator.validatorSessionInfo) = engine.validator.ValidatorSessionsInfo; - -engine.validator.requiredBlockCandidates block_ids:(vector tonNode.blockId) = engine.validator.RequiredBlockCandidates; ---functions--- @@ -779,8 +775,6 @@ engine.validator.setStateSerializerEnabled enabled:Bool = engine.validator.Succe engine.validator.setCollatorOptionsJson json:string = engine.validator.Success; engine.validator.getCollatorOptionsJson = engine.validator.JsonConfig; -engine.validator.getValidatorSessionsInfo = engine.validator.ValidatorSessionsInfo; - engine.validator.addCollator adnl_id:int256 shard:tonNode.shardId = engine.validator.Success; engine.validator.addShard shard:tonNode.shardId = engine.validator.Success; engine.validator.delCollator adnl_id:int256 shard:tonNode.shardId = engine.validator.Success; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index eae2a5590c1411adc052dbea3dbd6b5832abc004..b788ae4f2aa565cb7262ae3319329ff2c9930420 100644 GIT binary patch delta 65 zcmaDci*3skHr_|G^{p77Kxrdyz4YcQ(h-cCx5z5UGH%#>)#`uM=7en>(#)dN z6sMg0CdPh=Ei88>fcwfbM>hTVxp*qsRU97z4mV?sWRK(_R=`LsCZ|sRxK)0#&1n$_ zkZE8?0g0<}Trt=K4BaJ=00XI=T+m`70Wt*Lb0EphE4F`VXYAWP)1J{@nss)w^fS%r g8zUG+^wFaWVjcrX56EjE2f+dfB)0ub6k~=y0Q&6R)c^nh diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 6266ac2ac..5de0fe1f9 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1291,26 +1291,6 @@ td::Status GetCollatorOptionsJsonQuery::receive(td::BufferSlice data) { return td::Status::OK(); } -td::Status GetValidatorSessionsInfoQuery::run() { - TRY_STATUS(tokenizer_.check_endl()); - return td::Status::OK(); -} - -td::Status GetValidatorSessionsInfoQuery::send() { - auto b = ton::create_serialize_tl_object(); - td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); - return td::Status::OK(); -} - -td::Status GetValidatorSessionsInfoQuery::receive(td::BufferSlice data) { - TRY_RESULT_PREFIX( - f, ton::fetch_tl_object(data.as_slice(), true), - "received incorrect answer: "); - std::string s = td::json_encode(td::ToJson(*f), true); - td::TerminalIO::out() << "---------\n" << s << "--------\n"; - return td::Status::OK(); -} - td::Status AddCollatorQuery::run() { TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 73b2ae835..3cddb80c8 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1293,25 +1293,6 @@ class GetCollatorOptionsJsonQuery : public Query { std::string file_name_; }; -class GetValidatorSessionsInfoQuery : public Query { - public: - GetValidatorSessionsInfoQuery(td::actor::ActorId console, Tokenizer tokenizer) - : Query(console, std::move(tokenizer)) { - } - td::Status run() override; - td::Status send() override; - td::Status receive(td::BufferSlice data) override; - static std::string get_name() { - return "getvalidatorsessions"; - } - static std::string get_help() { - return "getvalidatorsessions\tprint info about validator session"; - } - std::string name() const override { - return get_name(); - } -}; - class AddCollatorQuery : public Query { public: AddCollatorQuery(td::actor::ActorId console, Tokenizer tokenizer) diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index c37d2771b..eae819613 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -150,7 +150,6 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); - add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index a6915f409..705439bce 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -4081,33 +4081,6 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showColla } } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getValidatorSessionsInfo &query, - td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, - td::Promise promise) { - if (!(perm & ValidatorEnginePermissions::vep_default)) { - promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); - return; - } - - if (validator_manager_.empty()) { - promise.set_value( - create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "validator manager not started"))); - return; - } - - auto P = td::PromiseCreator::lambda( - [promise = std::move(promise)]( - td::Result> R) mutable { - if (R.is_error()) { - promise.set_value(create_control_query_error(R.move_as_error())); - } else { - promise.set_value(ton::serialize_tl_object(R.move_as_ok(), true)); - } - }); - td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::get_validator_sessions_info, - std::move(P)); -} - void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollator &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 6a7680f0e..51e0ca9b3 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -504,8 +504,6 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getOverlaysStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); - void run_control_query(ton::ton_api::engine_validator_getValidatorSessionsInfo &query, td::BufferSlice data, - ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_addCollator &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_addShard &query, td::BufferSlice data, diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index d9adf177a..186d6600e 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -1206,18 +1206,6 @@ void ValidatorSessionImpl::stats_process_action(td::uint32 node_id, ton_api::val [](const auto &) {})); } -void ValidatorSessionImpl::get_session_info( - td::Promise> promise) { - std::vector next_producers; - for (td::uint32 round = cur_round_; round < cur_round_ + 20; ++round) { - td::uint32 node = description().get_node_by_priority(round, 0); - next_producers.push_back(description().get_source_id(node).bits256_value()); - } - promise.set_result(create_tl_object( - create_tl_block_id_simple(BlockId{}), description().get_source_id(local_idx()).bits256_value(), - cur_round_, std::move(next_producers))); -} - td::actor::ActorOwn ValidatorSession::create( catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, diff --git a/validator-session/validator-session.h b/validator-session/validator-session.h index d59c4e381..0870f6718 100644 --- a/validator-session/validator-session.h +++ b/validator-session/validator-session.h @@ -110,8 +110,6 @@ class ValidatorSession : public td::actor::Actor { td::Promise>> promise) = 0; virtual void set_catchain_max_block_delay(double value) = 0; - virtual void get_session_info(td::Promise> promise) = 0; - static td::actor::ActorOwn create( catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 6cb476747..580582824 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -175,8 +175,6 @@ class ValidatorSessionImpl : public ValidatorSession { ValidatorSessionCandidateId candidate_id); void stats_process_action(td::uint32 node_id, ton_api::validatorSession_round_Message &action); - void get_session_info(td::Promise> promise) override; - public: ValidatorSessionImpl(catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 01a0c60eb..f4f68468e 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -441,11 +441,6 @@ class ValidatorManagerImpl : public ValidatorManager { void add_persistent_state_description(td::Ref desc) override { } - void get_validator_sessions_info( - td::Promise> promise) override { - UNREACHABLE(); - } - void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { UNREACHABLE(); } diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 28c521f43..371c75706 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -506,10 +506,6 @@ class ValidatorManagerImpl : public ValidatorManager { } void add_persistent_state_description(td::Ref desc) override { } - void get_validator_sessions_info( - td::Promise> promise) override { - UNREACHABLE(); - } void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { UNREACHABLE(); diff --git a/validator/manager.cpp b/validator/manager.cpp index 11454c8e9..d3003a832 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -3394,39 +3394,6 @@ void ValidatorManagerImpl::update_options(td::Ref opts) opts_ = std::move(opts); } -void ValidatorManagerImpl::get_validator_sessions_info( - td::Promise> promise) { - std::vector> groups; - for (const auto& g : validator_groups_) { - groups.push_back(g.second.actor.get()); - } - struct IntermediateData { - std::vector> groups; - std::vector> result; - td::Promise> promise; - - static void step(IntermediateData data) { - if (data.groups.empty()) { - data.promise.set_result( - create_tl_object(std::move(data.result))); - return; - } - auto group = std::move(data.groups.back()); - data.groups.pop_back(); - auto P = td::PromiseCreator::lambda( - [data = - std::move(data)](td::Result> R) mutable { - if (R.is_ok()) { - data.result.push_back(R.move_as_ok()); - } - step(std::move(data)); - }); - td::actor::send_closure(group, &ValidatorGroup::get_session_info, std::move(P)); - } - }; - IntermediateData::step({std::move(groups), {}, std::move(promise)}); -} - void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) { if (shard.is_masterchain() || !shard.is_valid_ext()) { LOG(WARNING) << "cannot collate shard " << shard.to_str(); diff --git a/validator/manager.hpp b/validator/manager.hpp index 007bce35f..8c881f605 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -622,9 +622,6 @@ class ValidatorManagerImpl : public ValidatorManager { void update_options(td::Ref opts) override; - void get_validator_sessions_info( - td::Promise> promise) override; - void add_persistent_state_description(td::Ref desc) override; void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index b708e6ace..83e273588 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -448,25 +448,6 @@ void ValidatorGroup::get_validator_group_info_for_litequery_cont( promise.set_result(std::move(result)); } -void ValidatorGroup::get_session_info( - td::Promise> promise) { - if (session_.empty() || !started_) { - promise.set_error(td::Status::Error(ErrorCode::notready, "session not started")); - } - auto P = td::PromiseCreator::lambda( - [promise = std::move(promise), block_id = create_next_block_id_simple()]( - td::Result> R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - return; - } - auto info = R.move_as_ok(); - info->current_block_ = create_tl_block_id_simple(block_id); - promise.set_result(std::move(info)); - }); - td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_session_info, std::move(P)); -} - void ValidatorGroup::collate_block(td::uint32 round_id, td::Timestamp timeout, td::Promise promise, unsigned max_retries) { if (round_id < last_known_round_id_) { diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index debb7d14a..f44ce868b 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -64,8 +64,6 @@ class ValidatorGroup : public td::actor::Actor { void get_validator_group_info_for_litequery( td::Promise> promise); - void get_session_info(td::Promise> promise); - void update_options(td::Ref opts, bool apply_blocks) { opts_ = std::move(opts); monitoring_shard_ = apply_blocks; diff --git a/validator/validator.h b/validator/validator.h index 03aa5827d..927f21d7c 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -302,9 +302,6 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void update_options(td::Ref opts) = 0; - virtual void get_validator_sessions_info( - td::Promise> promise) = 0; - virtual void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; virtual void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; }; From 9b4a3a8263e90a046e82457ca439e0565845f5c8 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 10 Sep 2024 20:32:05 +0300 Subject: [PATCH 118/388] Synchronize last mc seqno between servers in proxy-liteserver --- utils/proxy-liteserver.cpp | 91 ++++++++++++++++++++++++++++++++++---- 1 file changed, 83 insertions(+), 8 deletions(-) diff --git a/utils/proxy-liteserver.cpp b/utils/proxy-liteserver.cpp index 66161fc45..a9baa7595 100644 --- a/utils/proxy-liteserver.cpp +++ b/utils/proxy-liteserver.cpp @@ -43,8 +43,12 @@ #if TD_DARWIN || TD_LINUX #include #endif +#include "td/utils/overloaded.h" + #include #include +#include +#include "td/utils/tl_storers.h" using namespace ton; @@ -252,26 +256,93 @@ class ProxyLiteserver : public td::actor::Actor { } void receive_query(td::BufferSlice data, td::Promise promise) { + // Like in ValidatorManagerImpl::run_ext_query + auto F = fetch_tl_object(data, true); + if (F.is_ok()) { + data = std::move(F.move_as_ok()->data_); + } else { + auto G = fetch_tl_prefix(data, true); + if (G.is_error()) { + promise.set_error(G.move_as_error()); + return; + } + } + + tl_object_ptr wait_mc_seqno_obj; + auto E = fetch_tl_prefix(data, true); + if (E.is_ok()) { + wait_mc_seqno_obj = E.move_as_ok(); + } liteclient::QueryInfo query_info = liteclient::get_query_info(data); ++ls_stats_[query_info.query_id]; - promise = [promise = std::move(promise), query_info, timer = td::Timer()](td::Result R) mutable { + promise = [promise = std::move(promise), query_info, timer = td::Timer(), + wait_mc_seqno = + (wait_mc_seqno_obj ? wait_mc_seqno_obj->seqno_ : 0)](td::Result R) mutable { if (R.is_ok()) { - LOG(INFO) << "Query " << query_info.to_str() << ": OK, time=" << timer.elapsed() - << ", response_size=" << R.ok().size(); + LOG(INFO) << "Query " << query_info.to_str() + << (wait_mc_seqno ? PSTRING() << " (wait seqno " << wait_mc_seqno << ")" : "") + << ": OK, time=" << timer.elapsed() << ", response_size=" << R.ok().size(); promise.set_value(R.move_as_ok()); return; } - LOG(INFO) << "Query " << query_info.to_str() << ": " << R.error(); + LOG(INFO) << "Query " << query_info.to_str() + << (wait_mc_seqno ? PSTRING() << " (wait seqno " << wait_mc_seqno << ")" : "") << ": " << R.error(); promise.set_value(create_serialize_tl_object( R.error().code(), "Gateway error: " + R.error().message().str())); }; - TRY_RESULT_PROMISE(promise, server_idx, select_server(query_info)); + TRY_RESULT_PROMISE(promise, server_idx, select_server(query_info)); Server& server = servers_[server_idx]; - LOG(INFO) << "Sending query " << query_info.to_str() << ", size=" << data.size() << ", to server #" << server_idx - << " (" << server.config.addr.get_ip_str() << ":" << server.config.addr.get_port() << ")"; + LOG(INFO) << "Sending query " << query_info.to_str() + << (wait_mc_seqno_obj ? PSTRING() << " (wait seqno " << wait_mc_seqno_obj->seqno_ << ")" : "") + << ", size=" << data.size() << ", to server #" << server_idx << " (" << server.config.addr.get_ip_str() + << ":" << server.config.addr.get_port() << ")"; + + BlockSeqno wait_mc_seqno = wait_mc_seqno_obj ? wait_mc_seqno_obj->seqno_ : 0; + wait_mc_seqno = std::max(wait_mc_seqno, last_known_masterchain_seqno_); + if (server.last_known_masterchain_seqno < wait_mc_seqno) { + int timeout_ms = wait_mc_seqno_obj ? wait_mc_seqno_obj->timeout_ms_ : 8000; + data = serialize_tl_object(create_tl_object(wait_mc_seqno, timeout_ms), + true, std::move(data)); + } + data = create_serialize_tl_object(std::move(data)); td::actor::send_closure(server.client, &adnl::AdnlExtClient::send_query, "q", std::move(data), - td::Timestamp::in(8.0), std::move(promise)); + td::Timestamp::in(8.0), + [SelfId = actor_id(this), promise = std::move(promise), server_idx, + wait_mc_seqno](td::Result R) mutable { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &ProxyLiteserver::process_query_response, + R.ok().clone(), server_idx, wait_mc_seqno); + } + promise.set_result(std::move(R)); + }); + } + + void process_query_response(td::BufferSlice data, size_t server_idx, BlockSeqno wait_mc_seqno) { + auto F = fetch_tl_object(data, true); + if (F.is_error() || F.ok()->get_id() == lite_api::liteServer_error::ID) { + return; + } + BlockSeqno new_seqno = wait_mc_seqno; + lite_api::downcast_call(*F.ok_ref(), td::overloaded( + [&](lite_api::liteServer_masterchainInfo& f) { + new_seqno = std::max(new_seqno, f.last_->seqno_); + }, + [&](lite_api::liteServer_masterchainInfoExt& f) { + new_seqno = std::max(new_seqno, f.last_->seqno_); + }, + [&](lite_api::liteServer_accountState& f) { + if (f.id_->workchain_ == masterchainId) { + new_seqno = std::max(new_seqno, f.id_->seqno_); + } + }, + [&](auto& obj) {})); + servers_[server_idx].last_known_masterchain_seqno = + std::max(servers_[server_idx].last_known_masterchain_seqno, new_seqno); + if (new_seqno > last_known_masterchain_seqno_) { + last_known_masterchain_seqno_ = new_seqno; + LOG(INFO) << "Last known masterchain seqno = " << new_seqno; + } } void alarm() override { @@ -307,11 +378,15 @@ class ProxyLiteserver : public td::actor::Actor { liteclient::LiteServerConfig config; td::actor::ActorOwn client; bool alive = false; + BlockSeqno last_known_masterchain_seqno = 0; }; std::vector servers_; std::map ls_stats_; // lite_api ID -> count, 0 for unknown + BlockSeqno last_known_masterchain_seqno_ = 0; + tl_object_ptr last_masterchain_info_; + std::string config_file() const { return db_root_ + "/config.json"; } From 8abdbf422fc7cfb7d9c9e6ba62455156328b5e1c Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 19 Sep 2024 12:14:32 +0300 Subject: [PATCH 119/388] Update query-utils.cpp --- lite-client/query-utils.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lite-client/query-utils.cpp b/lite-client/query-utils.cpp index f48e3e8dd..a3a663be0 100644 --- a/lite-client/query-utils.cpp +++ b/lite-client/query-utils.cpp @@ -216,6 +216,8 @@ QueryInfo get_query_info(const lite_api::Function& f) { /* t_simple */ }, [&](const lite_api::liteServer_getBlockOutMsgQueueSize& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getDispatchQueueInfo& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getDispatchQueueMessages& q) { from_block_id(q.id_); }, [&](const auto&) { /* t_simple */ })); if (info.shard_id.workchain == masterchainId) { info.shard_id.shard = shardIdAll; From 9dfa3fa11a24f072043ebff8ab000677a2dc2059 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 26 Sep 2024 19:24:03 +0300 Subject: [PATCH 120/388] Configure shards for custom overlays --- tl/generate/scheme/ton_api.tl | 3 +- tl/generate/scheme/ton_api.tlo | Bin 104992 -> 105052 bytes .../validator-engine-console-query.cpp | 6 ++ validator/full-node.cpp | 70 ++++++++++-------- validator/full-node.h | 2 + 5 files changed, 48 insertions(+), 33 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 09363b55a..aeb70ca4e 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -660,7 +660,8 @@ engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector eng gc:engine.gc = engine.validator.Config; engine.validator.customOverlayNode adnl_id:int256 msg_sender:Bool msg_sender_priority:int block_sender:Bool = engine.validator.CustomOverlayNode; -engine.validator.customOverlay name:string nodes:(vector engine.validator.customOverlayNode) = engine.validator.CustomOverlay; +engine.validator.customOverlay name:string nodes:(vector engine.validator.customOverlayNode) sender_shards:(vector tonNode.shardId) + = engine.validator.CustomOverlay; engine.validator.customOverlaysConfig overlays:(vector engine.validator.customOverlay) = engine.validator.CustomOverlaysConfig; engine.validator.collatorOptions diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 1132032de0889fc8729f9ac6d49176038d0d3c8a..d2216fbcb73b3697993b70c607ebff9df7003c1f 100644 GIT binary patch delta 108 zcmZ3mh3(E3whap6tY>RdmdH&`6qnoVEbcm;1;m})vrd5%#JG7($cK6QLTN_tN$Yt8 zrJvo`AtWwhap6tg77i*UC*!6qnoVEbcm;1;m})vrYlRICD&Q^4)d%JVnlIIvlob h2SKu%L)M2df~7aF*>GnWm^ED>k5O^^+Hl78P5??=BoF`q diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 761de8402..e81e84939 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1229,6 +1229,12 @@ td::Status ShowCustomOverlaysQuery::receive(td::BufferSlice data) { : "") << (node->block_sender_ ? " (block sender)" : "") << "\n"; } + if (!overlay->sender_shards_.empty()) { + td::TerminalIO::out() << "Sender shards:\n"; + for (const auto &shard : overlay->sender_shards_) { + td::TerminalIO::out() << " " << ton::create_shard_id(shard).to_str() << "\n"; + } + } td::TerminalIO::out() << "\n"; } return td::Status::OK(); diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 651c01494..2cd578ab6 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -21,6 +21,7 @@ #include "td/actor/MultiPromise.h" #include "full-node.h" #include "common/delay.h" +#include "ton/ton-tl.hpp" namespace ton { @@ -209,35 +210,28 @@ void FullNodeImpl::on_new_masterchain_block(td::Ref state, std int min_split = state->monitor_min_split_depth(shard.workchain); return min_split < shard.pfx_len() ? shard_prefix(shard, min_split) : shard; }; - auto set_active = [&](ShardIdFull shard) { - while (new_active.emplace(shard).second && shard.pfx_len() > 0) { - shard = shard_parent(shard); - } - }; for (auto &info : state->get_shards()) { workchains.insert(info->shard().workchain); all_shards.insert(cut_shard(info->shard())); } - for (const auto &wpair : state->get_workchain_list()) { - ton::WorkchainId wc = wpair.first; - const block::WorkchainInfo *winfo = wpair.second.get(); - if (workchains.count(wc) == 0 && winfo->active && winfo->enabled_since <= state->get_unix_time()) { + for (const auto &[wc, winfo] : state->get_workchain_list()) { + if (!workchains.contains(wc) && winfo->active && winfo->enabled_since <= state->get_unix_time()) { all_shards.insert(ShardIdFull(wc)); } } for (ShardIdFull shard : shards_to_monitor) { - set_active(cut_shard(shard)); + new_active.insert(cut_shard(shard)); } for (auto it = shards_.begin(); it != shards_.end(); ) { - if (all_shards.count(it->first)) { + if (all_shards.contains(it->first)) { ++it; } else { it = shards_.erase(it); } } for (ShardIdFull shard : all_shards) { - bool active = new_active.count(shard); + bool active = new_active.contains(shard); bool overlay_exists = !shards_[shard].actor.empty(); if (active || join_all_overlays || overlay_exists) { update_shard_actor(shard, active); @@ -254,8 +248,8 @@ void FullNodeImpl::on_new_masterchain_block(td::Ref state, std if (!use_old_private_overlays_) { std::set my_adnl_ids; my_adnl_ids.insert(adnl_id_); - for (const auto &p : local_collator_nodes_) { - my_adnl_ids.insert(p.first); + for (const auto &[adnl_id, _] : local_collator_nodes_) { + my_adnl_ids.insert(adnl_id); } for (auto key : local_keys_) { auto it = current_validators_.find(key); @@ -307,11 +301,12 @@ void FullNodeImpl::send_ext_message(AccountIdPrefixFull dst, td::BufferSlice dat VLOG(FULL_NODE_WARNING) << "dropping OUT ext message to unknown shard"; return; } - for (auto &private_overlay : custom_overlays_) { - for (auto &actor : private_overlay.second.actors_) { - auto local_id = actor.first; - if (private_overlay.second.params_.msg_senders_.count(local_id)) { - td::actor::send_closure(actor.second, &FullNodeCustomOverlay::send_external_message, data.clone()); + for (auto &[_, private_overlay] : custom_overlays_) { + if (private_overlay.params_.send_shard(dst.as_leaf_shard())) { + for (auto &[local_id, actor] : private_overlay.actors_) { + if (private_overlay.params_.msg_senders_.contains(local_id)) { + td::actor::send_closure(actor, &FullNodeCustomOverlay::send_external_message, data.clone()); + } } } } @@ -743,7 +738,7 @@ void FullNodeImpl::update_custom_overlay(CustomOverlayInfo &overlay) { } } -void FullNodeImpl::send_block_broadcast_to_custom_overlays(const BlockBroadcast& broadcast) { +void FullNodeImpl::send_block_broadcast_to_custom_overlays(const BlockBroadcast &broadcast) { if (!custom_overlays_sent_broadcasts_.insert(broadcast.block_id).second) { return; } @@ -752,11 +747,12 @@ void FullNodeImpl::send_block_broadcast_to_custom_overlays(const BlockBroadcast& custom_overlays_sent_broadcasts_.erase(custom_overlays_sent_broadcasts_lru_.front()); custom_overlays_sent_broadcasts_lru_.pop(); } - for (auto &private_overlay : custom_overlays_) { - for (auto &actor : private_overlay.second.actors_) { - auto local_id = actor.first; - if (private_overlay.second.params_.block_senders_.count(local_id)) { - td::actor::send_closure(actor.second, &FullNodeCustomOverlay::send_broadcast, broadcast.clone()); + for (auto &[_, private_overlay] : custom_overlays_) { + if (private_overlay.params_.send_shard(broadcast.block_id.shard_full())) { + for (auto &[local_id, actor] : private_overlay.actors_) { + if (private_overlay.params_.block_senders_.contains(local_id)) { + td::actor::send_closure(actor, &FullNodeCustomOverlay::send_broadcast, broadcast.clone()); + } } } } @@ -774,12 +770,13 @@ void FullNodeImpl::send_block_candidate_broadcast_to_custom_overlays(const Block custom_overlays_sent_broadcasts_.erase(custom_overlays_sent_broadcasts_lru_.front()); custom_overlays_sent_broadcasts_lru_.pop(); } - for (auto &private_overlay : custom_overlays_) { - for (auto &actor : private_overlay.second.actors_) { - auto local_id = actor.first; - if (private_overlay.second.params_.block_senders_.count(local_id)) { - td::actor::send_closure(actor.second, &FullNodeCustomOverlay::send_block_candidate, block_id, cc_seqno, - validator_set_hash, data.clone()); + for (auto &[_, private_overlay] : custom_overlays_) { + if (private_overlay.params_.send_shard(block_id.shard_full())) { + for (auto &[local_id, actor] : private_overlay.actors_) { + if (private_overlay.params_.block_senders_.contains(local_id)) { + td::actor::send_closure(actor, &FullNodeCustomOverlay::send_block_candidate, block_id, cc_seqno, + validator_set_hash, data.clone()); + } } } } @@ -834,18 +831,27 @@ bool FullNodeConfig::operator!=(const FullNodeConfig &rhs) const { return !(*this == rhs); } +bool CustomOverlayParams::send_shard(const ShardIdFull &shard) const { + return sender_shards_.empty() || std::ranges::any_of(sender_shards_, [&](const ShardIdFull &our_shard) { + return shard_intersects(shard, our_shard); + }); +} + CustomOverlayParams CustomOverlayParams::fetch(const ton_api::engine_validator_customOverlay& f) { CustomOverlayParams c; c.name_ = f.name_; for (const auto &node : f.nodes_) { c.nodes_.emplace_back(node->adnl_id_); if (node->msg_sender_) { - c.msg_senders_[ton::adnl::AdnlNodeIdShort{node->adnl_id_}] = node->msg_sender_priority_; + c.msg_senders_[adnl::AdnlNodeIdShort{node->adnl_id_}] = node->msg_sender_priority_; } if (node->block_sender_) { c.block_senders_.emplace(node->adnl_id_); } } + for (const auto &shard : f.sender_shards_) { + c.sender_shards_.push_back(create_shard_id(shard)); + } return c; } diff --git a/validator/full-node.h b/validator/full-node.h index fa57e4f1c..8b47b9479 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -60,7 +60,9 @@ struct CustomOverlayParams { std::vector nodes_; std::map msg_senders_; std::set block_senders_; + std::vector sender_shards_; + bool send_shard(const ShardIdFull& shard) const; static CustomOverlayParams fetch(const ton_api::engine_validator_customOverlay& f); }; From 7d2110c8b0da73d25a77ffc34c30250406fb2825 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 18 Nov 2024 17:14:13 +0300 Subject: [PATCH 121/388] Fix shard overlays --- validator/full-node.cpp | 54 +++++++++++++++++++++++++++-------------- validator/full-node.hpp | 1 + 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 0cd769e4d..bb806e281 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -206,13 +206,20 @@ void FullNodeImpl::on_new_masterchain_block(td::Ref state, std std::set new_active; all_shards.insert(ShardIdFull(masterchainId)); std::set workchains; + wc_monitor_min_split_ = state->monitor_min_split_depth(basechainId); auto cut_shard = [&](ShardIdFull shard) -> ShardIdFull { - int min_split = state->monitor_min_split_depth(shard.workchain); - return min_split < shard.pfx_len() ? shard_prefix(shard, min_split) : shard; + return wc_monitor_min_split_ < shard.pfx_len() ? shard_prefix(shard, wc_monitor_min_split_) : shard; }; for (auto &info : state->get_shards()) { workchains.insert(info->shard().workchain); - all_shards.insert(cut_shard(info->shard())); + ShardIdFull shard = cut_shard(info->shard()); + while (true) { + all_shards.insert(shard); + if (shard.pfx_len() == 0) { + break; + } + shard = shard_parent(shard); + } } for (const auto &[wc, winfo] : state->get_workchain_list()) { if (!workchains.contains(wc) && winfo->active && winfo->enabled_since <= state->get_unix_time()) { @@ -220,7 +227,14 @@ void FullNodeImpl::on_new_masterchain_block(td::Ref state, std } } for (ShardIdFull shard : shards_to_monitor) { - new_active.insert(cut_shard(shard)); + shard = cut_shard(shard); + while (true) { + new_active.insert(shard); + if (shard.pfx_len() == 0) { + break; + } + shard = shard_parent(shard); + } } for (auto it = shards_.begin(); it != shards_.end(); ) { @@ -478,21 +492,25 @@ void FullNodeImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std::vect } td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { - while (true) { - auto it = shards_.find(shard); - if (it != shards_.end()) { - if (it->second.actor.empty()) { - update_shard_actor(shard, false); - } - return it->second.actor.get(); - } - if (shard.pfx_len() == 0) { - break; - } - shard = shard_parent(shard); + if (shard.is_masterchain()) { + return shards_[ShardIdFull{masterchainId}].actor.get(); + } + if (shard.workchain != basechainId) { + return {}; + } + int pfx_len = shard.pfx_len(); + if (pfx_len > wc_monitor_min_split_) { + shard = shard_prefix(shard, wc_monitor_min_split_); } - update_shard_actor(shard, false); - return shards_[shard].actor.get(); + auto it = shards_.find(shard); + if (it != shards_.end()) { + update_shard_actor(shard, it->second.active); + return it->second.actor.get(); + } + + // Special case if shards_ was not yet initialized. + // This can happen briefly on node startup. + return shards_[ShardIdFull{masterchainId}].actor.get(); } td::actor::ActorId FullNodeImpl::get_shard(AccountIdPrefixFull dst) { diff --git a/validator/full-node.hpp b/validator/full-node.hpp index f6047ed5f..d2de18201 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -124,6 +124,7 @@ class FullNodeImpl : public FullNode { td::actor::ActorId get_shard(AccountIdPrefixFull dst); td::actor::ActorId get_shard(ShardIdFull shard); std::map shards_; + int wc_monitor_min_split_ = 0; td::actor::ActorId keyring_; td::actor::ActorId adnl_; From b3bea413e3e0e6db182a633f7e7cce6345c46d83 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 21 Nov 2024 11:47:39 +0300 Subject: [PATCH 122/388] Rework validator-collator interaction 1) Remove config 41, move "full collated data" to capabilities 2) Whitelist on collator nodes 3) "Ping" request for collator nodes 4) More customizable collators list for validators 5) CollationManager --- crypto/block/block.tlb | 5 - crypto/block/mc-config.cpp | 24 -- crypto/block/mc-config.h | 6 - tl/generate/scheme/ton_api.tl | 28 +- tl/generate/scheme/ton_api.tlo | Bin 105392 -> 107428 bytes ton/ton-types.h | 3 +- .../validator-engine-console-query.cpp | 144 ++++++- .../validator-engine-console-query.h | 104 +++++ .../validator-engine-console.cpp | 5 + validator-engine/validator-engine.cpp | 252 +++++++++--- validator-engine/validator-engine.hpp | 27 +- validator/CMakeLists.txt | 2 + validator/collation-manager.cpp | 373 ++++++++++++++++++ validator/collation-manager.hpp | 87 ++++ validator/collator-node.cpp | 56 ++- validator/collator-node.hpp | 1 + validator/full-node.cpp | 14 +- validator/impl/collator-impl.h | 2 +- validator/impl/collator.cpp | 7 +- validator/impl/shard.hpp | 3 - validator/impl/validate-query.cpp | 9 + validator/impl/validate-query.hpp | 2 +- validator/impl/validator-set.hpp | 3 +- validator/interfaces/shard.h | 1 - validator/interfaces/validator-set.h | 1 + validator/manager-disk.hpp | 5 + validator/manager-hardfork.hpp | 5 + validator/manager.cpp | 77 +++- validator/manager.hpp | 6 + validator/validator-group.cpp | 158 +------- validator/validator-group.hpp | 22 +- validator/validator-options.cpp | 53 ++- validator/validator-options.hpp | 17 +- validator/validator.h | 21 +- 34 files changed, 1204 insertions(+), 319 deletions(-) create mode 100644 validator/collation-manager.cpp create mode 100644 validator/collation-manager.hpp diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 5eddbde38..aa7c0d3f9 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -801,11 +801,6 @@ misbehaviour_punishment_config_v1#01 = MisbehaviourPunishmentConfig; _ MisbehaviourPunishmentConfig = ConfigParam 40; -// collator_nodes: each collator is (workchain:int32 shard:uint64 adnl_id:uint256) -collator_info#00 = CollatorInfo; -colator_config#a0 full_collated_data:Bool collator_nodes:(HashmapE 352 CollatorInfo) = CollatorConfig; -_ CollatorConfig = ConfigParam 41; - size_limits_config#01 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16 max_ext_msg_size:uint32 max_ext_msg_depth:uint16 = SizeLimitsConfig; size_limits_config_v2#02 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16 diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 0a8c0c7b1..56ee85ae3 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2339,28 +2339,4 @@ td::optional PrecompiledContractsConfig::g return c; } -CollatorConfig Config::get_collator_config(bool need_collator_nodes) const { - CollatorConfig collator_config; - gen::CollatorConfig::Record rec; - auto cell = get_config_param(41, -41); - if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { - return collator_config; - } - collator_config.full_collated_data = rec.full_collated_data; - if (need_collator_nodes) { - vm::Dictionary dict{rec.collator_nodes->prefetch_ref(), 32 + 64 + 256}; - dict.check_for_each([&](Ref value, td::ConstBitPtr key, int n) { - CHECK(n == 32 + 64 + 256); - auto workchain = (td::int32)key.get_int(32); - key.advance(32); - td::uint64 shard = key.get_uint(64); - key.advance(64); - td::Bits256 adnl_id(key); - collator_config.collator_nodes.push_back({ton::ShardIdFull(workchain, shard), adnl_id}); - return true; - }); - } - return collator_config; -} - } // namespace block diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 94de329bf..3c4421da5 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -543,11 +543,6 @@ struct CollatorNodeDescr { ton::NodeIdShort adnl_id; }; -struct CollatorConfig { - bool full_collated_data = false; - std::vector collator_nodes; -}; - class Config { enum { default_mc_catchain_lifetime = 200, @@ -664,7 +659,6 @@ class Config { std::vector compute_validator_set(ton::ShardIdFull shard, ton::UnixTime time, ton::CatchainSeqno cc_seqno) const; std::vector compute_total_validator_set(int next) const; - CollatorConfig get_collator_config(bool need_collator_nodes) const; td::Result get_size_limits_config() const; static td::Result do_get_size_limits_config(td::Ref cs); std::unique_ptr get_suspended_addresses(ton::UnixTime now) const; diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 05ea249ce..0839f727d 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -646,8 +646,9 @@ engine.validator.fullNodeMaster port:int adnl:int256 = engine.validator.FullNode engine.validator.fullNodeSlave ip:int port:int adnl:PublicKey = engine.validator.FullNodeSlave; engine.validator.fullNodeConfig ext_messages_broadcast_disabled:Bool = engine.validator.FullNodeConfig; engine.validator.fastSyncMemberCertificate adnl_id:int256 certificate:overlay.MemberCertificate = engine.validator.FastSyncMemberCertificate; +engine.validator.collatorNodeWhitelist enabled:Bool adnl_ids:(vector int256) = engine.validator.CollatorNodeWhitelist; engine.validator.extraConfig state_serializer_enabled:Bool fast_sync_member_certificates:(vector engine.validator.fastSyncMemberCertificate) - = engine.validator.ExtraConfig; + collator_node_whitelist:engine.validator.collatorNodeWhitelist = engine.validator.ExtraConfig; engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) dht:(vector engine.dht) validators:(vector engine.validator) collators:(vector engine.collator) @@ -670,11 +671,10 @@ engine.validator.collatorOptions dispatch_phase_2_max_per_initiator:int dispatch_phase_3_max_per_initiator:int whitelist:(vector string) prioritylist:(vector string) = engine.validator.CollatorOptions; -engine.validator.collatorsList.collator adnl_id:int256 trusted:Bool = engine.validator.collatorsList.Collator; +engine.validator.collatorsList.collator adnl_id:int256 = engine.validator.collatorsList.Collator; engine.validator.collatorsList.shard shard_id:tonNode.shardId collators:(vector engine.validator.collatorsList.collator) - = engine.validator.collatorsList.Shard; -engine.validator.collatorsList self_collate:Bool use_config_41:Bool shards:(vector engine.validator.collatorsList.shard) - = engine.validator.CollatorsList; + self_collate:Bool select_mode:string = engine.validator.collatorsList.Shard; +engine.validator.collatorsList shards:(vector engine.validator.collatorsList.shard) = engine.validator.CollatorsList; ---functions--- ---types--- @@ -740,6 +740,11 @@ engine.validator.perfTimerStats stats:(vector engine.validator.PerfTimerStatsByN engine.validator.shardOutQueueSize size:long = engine.validator.ShardOutQueueSize; +engine.validator.collationManagerStats.shard shard_id:tonNode.shardId self_collate:Bool select_mode:string active:Bool collators:(vector int256) = engine.validator.collationManagerStats.Shard; +engine.validator.collationManagerStats.collator adnl_id:int256 active:Bool alive:Bool ping_in:double = engine.validator.collationManagerStats.Collator; +engine.validator.collationManagerStats.localId adnl_id:int256 shards:(vector engine.validator.collationManagerStats.shard) + collators:(vector engine.validator.collationManagerStats.collator) = engine.validator.collationManagerStats.LocalId; +engine.validator.collationManagerStats local_ids:(vector engine.validator.collationManagerStats.localId) = engine.validator.CollationManagerStats; ---functions--- @@ -811,8 +816,14 @@ engine.validator.addShard shard:tonNode.shardId = engine.validator.Success; engine.validator.delCollator adnl_id:int256 shard:tonNode.shardId = engine.validator.Success; engine.validator.delShard shard:tonNode.shardId = engine.validator.Success; +engine.validator.collatorNodeSetWhitelistedValidator adnl_id:int256 add:Bool = engine.validator.Success; +engine.validator.collatorNodeSetWhitelistEnabled enabled:Bool = engine.validator.Success; +engine.validator.showCollatorNodeWhitelist = engine.validator.CollatorNodeWhitelist; + engine.validator.setCollatorsList list:engine.validator.collatorsList = engine.validator.Success; +engine.validator.clearCollatorsList = engine.validator.Success; engine.validator.showCollatorsList = engine.validator.CollatorsList; +engine.validator.getCollationManagerStats = engine.validator.CollationManagerStats; engine.validator.signOverlayMemberCertificate sign_by:int256 adnl_id:int256 slot:int expire_at:int = overlay.MemberCertificate; engine.validator.importFastSyncMemberCertificate adnl_id:int256 certificate:overlay.MemberCertificate = engine.validator.Success; @@ -887,8 +898,8 @@ validatorSession.stats success:Bool id:tonNode.blockIdExt timestamp:double self: collatorNode.candidate source:PublicKey id:tonNode.blockIdExt data:bytes collated_data:bytes = collatorNode.Candidate; collatorNode.compressedCandidate flags:# source:PublicKey id:tonNode.blockIdExt decompressed_size:int data:bytes = collatorNode.Candidate; -collatorNode.generateBlockSuccess candidate:collatorNode.Candidate = collatorNode.GenerateBlockResult; -collatorNode.generateBlockError code:int message:string = collatorNode.GenerateBlockResult; +collatorNode.pong flags:# = collatorNode.Pong; +collatorNode.error code:int message:string = collatorNode.Error; validatorSession.newValidatorGroupStats.node id:int256 weight:long = validatorSession.newValidatorGroupStats.Node; validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:long cc_seqno:int @@ -901,7 +912,8 @@ validatorSession.endValidatorGroupStats session_id:int256 timestamp:double ---functions--- collatorNode.generateBlock shard:tonNode.shardId cc_seqno:int prev_blocks:(vector tonNode.blockIdExt) - creator:int256 = collatorNode.GenerateBlockResult; + creator:int256 = collatorNode.Candidate; +collatorNode.ping flags:# = collatorNode.Pong; ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index eaa9c416156da6d95b8fbd44bfa6b2ef83d1d1d9..dd2af2ec9de6dbde7d6644ac55b85bf42bba5df1 100644 GIT binary patch delta 1507 zcmZuxVQ3R)7``vw#bj+++oj0`uj!T6HLYDz*ABw+oNe2jUuq( z3AJ=G=ig~@??MyZVV3-bMrQ9RihU6lgDyiS+AO{1%-OT#^(1Qr3FpmcV9y(gN^s@OqdzWGQ9>ceBSc`<%|K!#4vEhwQ zKP$1!t=PyHF+?@EP;kV%oe_Tg;b}kaw7GV4!8$*ASkOdRf<|cQDgF2Ua-Kidz*l#fP)$;~A^G!6p}&GXg8a%F%;QtB%s%4AO^JPb~Y zKmMF?743qHItgNP!vI>QF14iN#;%mbL zZu%S{b=H?qGAThB6)8kiFO{3=fgAJPG)fF!T^Q=cWcA%6++D!Y9FLaR$ZnEbLz4VPqj{Otq6h8{*v+PS)h?xD_Lqkw9*y3uhg` zi2DrA7cNKmMyCl3JiRgih1IAZ(e}OI%d4}Wk}u2BIO!qulUM${ zV-)ujjvi~c^tDmiPZpq}HFsV-`tW_HDf@xJdHk<2%2N|$yeYbG>5mItlA4xPDK>3Q zQpBezmQo}^tJyz4X6Xm!e(h%79=-k7faT+GE9Rb*bVvPrRD;hF)+lAk5HspsYa2qC z?GQIi5)WVvW99(aMQ;&A@hczUjN(DkTrMTaT|cw_#>Bl|4;>!Lvn>{3-u(IT~Ngka1;e^6u z69omE%63}lq{@!>u-6XxO~XEq%U+qQ!sB`~q8a$GU9o-Y0^kK1V1qN4mFkeNIp^WN`EVnIF`ld^SGs()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status CollatorNodeAddWhitelistedValidatorQuery::send() { + auto b = ton::create_serialize_tl_object( + adnl_id_.bits256_value(), true); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status CollatorNodeAddWhitelistedValidatorQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status CollatorNodeDelWhitelistedValidatorQuery::run() { + TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status CollatorNodeDelWhitelistedValidatorQuery::send() { + auto b = ton::create_serialize_tl_object( + adnl_id_.bits256_value(), false); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status CollatorNodeDelWhitelistedValidatorQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status CollatorNodeEnableWhitelistQuery::run() { + TRY_RESULT(value, tokenizer_.get_token()); + if (value != 0 && value != 1) { + return td::Status::Error("expected 0 or 1"); + } + TRY_STATUS(tokenizer_.check_endl()); + enabled_ = value; + return td::Status::OK(); +} + +td::Status CollatorNodeEnableWhitelistQuery::send() { + auto b = ton::create_serialize_tl_object(enabled_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status CollatorNodeEnableWhitelistQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status CollatorNodeShowWhitelistQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status CollatorNodeShowWhitelistQuery::send() { + auto b = ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status CollatorNodeShowWhitelistQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, + ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "Collator node whitelist: " << (f->enabled_ ? "ENABLED" : "DISABLED") << "\n"; + td::TerminalIO::out() << f->adnl_ids_.size() << " validator adnl ids\n"; + for (const auto &id : f->adnl_ids_) { + td::TerminalIO::out() << id.to_hex() << "\n"; + } + return td::Status::OK(); +} + td::Status SetCollatorsListQuery::run() { TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); TRY_STATUS(tokenizer_.check_endl()); @@ -1611,9 +1697,7 @@ td::Status ClearCollatorsListQuery::run() { } td::Status ClearCollatorsListQuery::send() { - auto list = ton::create_tl_object(); - list->self_collate_ = true; - auto b = ton::create_serialize_tl_object(std::move(list)); + auto b = ton::create_serialize_tl_object(); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -1640,20 +1724,60 @@ td::Status ShowCollatorsListQuery::receive(td::BufferSlice data) { TRY_RESULT_PREFIX(list, ton::fetch_tl_object(data.as_slice(), true), "received incorrect answer: "); td::TerminalIO::out() << "Collators list:\n"; - if (list->self_collate_) { - td::TerminalIO::out() << "self_collate = true\n"; - } - if (list->use_config_41_) { - td::TerminalIO::out() << "use_config_41 = true\n"; - } if (list->shards_.empty()) { td::TerminalIO::out() << "Shard list is empty\n"; return td::Status::OK(); } for (const auto &shard : list->shards_) { td::TerminalIO::out() << "Shard " << create_shard_id(shard->shard_id_).to_str() << "\n"; + td::TerminalIO::out() << " Self collate = " << shard->self_collate_ << "\n"; + td::TerminalIO::out() << " Select mode = " << shard->select_mode_ << "\n"; for (const auto &collator : shard->collators_) { - td::TerminalIO::out() << " Collator " << collator->adnl_id_ << (collator->trusted_ ? " (trusted)" : "") << "\n"; + td::TerminalIO::out() << " Collator " << collator->adnl_id_ << "\n"; + } + } + return td::Status::OK(); +} + +td::Status GetCollationManagerStatsQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status GetCollationManagerStatsQuery::send() { + auto b = ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GetCollationManagerStatsQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(list, + ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + if (list->local_ids_.empty()) { + td::TerminalIO::out() << "No stats\n"; + return td::Status::OK();; + } + for (auto &stats : list->local_ids_) { + td::TerminalIO::out() << "VALIDATOR ADNL ID = " << stats->adnl_id_ << "\n"; + std::map collators; + for (auto &collator: stats->collators_) { + collators[collator->adnl_id_] = collator.get(); + } + for (auto &shard : stats->shards_) { + td::TerminalIO::out() << " Shard " << create_shard_id(shard->shard_id_).to_str() << "\n"; + td::TerminalIO::out() << " Self collate = " << shard->self_collate_ << "\n"; + td::TerminalIO::out() << " Select mode = " << shard->select_mode_ << "\n"; + td::TerminalIO::out() << " Active = " << shard->active_ << "\n"; + td::TerminalIO::out() << " Collators: " << shard->collators_.size() << "\n"; + for (auto &id : shard->collators_) { + auto collator = collators[id]; + if (collator == nullptr) { + return td::Status::Error("collator not found"); + } + td::TerminalIO::out() << " " << id << " alive=" << (int)collator->alive_ + << " ping_in=" << collator->ping_in_ << "\n"; + } } } return td::Status::OK(); diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 5708fecda..2d59c29b7 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1457,6 +1457,91 @@ class DelShardQuery : public Query { td::int64 shard_; }; +class CollatorNodeAddWhitelistedValidatorQuery : public Query { + public: + CollatorNodeAddWhitelistedValidatorQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "collatorwhitelistadd"; + } + static std::string get_help() { + return "collatorwhitelistadd \tadd validator adnl id to collator node whitelist"; + } + std::string name() const override { + return get_name(); + } + + private: + ton::PublicKeyHash adnl_id_; +}; + +class CollatorNodeDelWhitelistedValidatorQuery : public Query { + public: + CollatorNodeDelWhitelistedValidatorQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "collatorwhitelistdel"; + } + static std::string get_help() { + return "collatorwhitelistdel \tremove validator adnl id from collator node whitelist"; + } + std::string name() const override { + return get_name(); + } + + private: + ton::PublicKeyHash adnl_id_; +}; + +class CollatorNodeEnableWhitelistQuery : public Query { + public: + CollatorNodeEnableWhitelistQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "collatorwhitelistenable"; + } + static std::string get_help() { + return "collatorwhitelistenable \tenable or disable collator node whiltelist (value is 0 or 1)"; + } + std::string name() const override { + return get_name(); + } + + private: + bool enabled_; +}; + +class CollatorNodeShowWhitelistQuery : public Query { + public: + CollatorNodeShowWhitelistQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "collatorwhitelistshow"; + } + static std::string get_help() { + return "collatorwhitelistshow\tshow collator node whitelist"; + } + std::string name() const override { + return get_name(); + } +}; + class SetCollatorsListQuery : public Query { public: SetCollatorsListQuery(td::actor::ActorId console, Tokenizer tokenizer) @@ -1517,6 +1602,25 @@ class ShowCollatorsListQuery : public Query { } }; +class GetCollationManagerStatsQuery : public Query { + public: + GetCollationManagerStatsQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "collationmanagerstats"; + } + static std::string get_help() { + return "collationmanagerstats\tshow stats of collation manager"; + } + std::string name() const override { + return get_name(); + } +}; + class SignOverlayMemberCertificateQuery : public Query { public: SignOverlayMemberCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 4c4922494..67e932115 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -157,9 +157,14 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); } diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 3722dcd28..8d2ce0bef 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -75,6 +75,7 @@ #include "common/delay.h" #include "block/precompiled-smc/PrecompiledSmartContract.h" #include "interfaces/validator-manager.h" +#include "tl-utils/lite-utils.hpp" #if TON_USE_JEMALLOC #include @@ -147,9 +148,9 @@ Config::Config(const ton::ton_api::engine_validator_config &config) { } } for (auto &col : config.collators_) { - auto key = ton::PublicKeyHash{col->adnl_id_}; + auto id = ton::adnl::AdnlNodeIdShort{col->adnl_id_}; ton::ShardIdFull shard = ton::create_shard_id(col->shard_); - config_add_collator(key, shard).ensure(); + config_add_collator(id, shard).ensure(); } config_add_full_node_adnl_id(ton::PublicKeyHash{config.fullnode_}).ensure(); @@ -175,6 +176,12 @@ Config::Config(const ton::ton_api::engine_validator_config &config) { fast_sync_member_certificates.emplace_back(adnl_id, std::move(certificate)); } } + if (config.extraconfig_->collator_node_whitelist_) { + collator_node_whiltelist_enabled = config.extraconfig_->collator_node_whitelist_->enabled_; + for (const auto& id : config.extraconfig_->collator_node_whitelist_->adnl_ids_) { + collator_node_whitelist.emplace(id); + } + } } else { state_serializer_enabled = true; } @@ -242,9 +249,11 @@ ton::tl_object_ptr Config::tl() const { val.first.tl(), std::move(temp_vec), std::move(adnl_val_vec), val.second.election_date, val.second.expire_at)); } std::vector> col_vec; - for (auto &col : collators) { - col_vec.push_back( - ton::create_tl_object(col.adnl_id.tl(), ton::create_tl_shard_id(col.shard))); + for (auto &[col, shards] : collators) { + for (auto &shard : shards) { + col_vec.push_back( + ton::create_tl_object(col.bits256_value(), ton::create_tl_shard_id(shard))); + } } std::vector> full_node_slaves_vec; @@ -263,8 +272,17 @@ ton::tl_object_ptr Config::tl() const { full_node_config_obj = full_node_config.tl(); } + ton::tl_object_ptr collator_node_whitelist_obj = {}; + if (collator_node_whiltelist_enabled || !collator_node_whitelist.empty()) { + collator_node_whitelist_obj = ton::create_tl_object(); + collator_node_whitelist_obj->enabled_ = collator_node_whiltelist_enabled; + for (const auto& id : collator_node_whitelist) { + collator_node_whitelist_obj->adnl_ids_.push_back(id.bits256_value()); + } + } + ton::tl_object_ptr extra_config_obj = {}; - if (!state_serializer_enabled || !fast_sync_member_certificates.empty()) { + if (!state_serializer_enabled || !fast_sync_member_certificates.empty() || collator_node_whitelist_obj) { // Non-default values extra_config_obj = ton::create_tl_object(); extra_config_obj->state_serializer_enabled_ = state_serializer_enabled; @@ -273,6 +291,7 @@ ton::tl_object_ptr Config::tl() const { ton::create_tl_object(adnl_id.bits256_value(), certificate.tl())); } + extra_config_obj->collator_node_whitelist_ = std::move(collator_node_whitelist_obj); } std::vector> liteserver_vec; @@ -448,28 +467,28 @@ td::Result Config::config_add_validator_adnl_id(ton::PublicKeyHash perm_ke } } -td::Result Config::config_add_collator(ton::PublicKeyHash addr, ton::ShardIdFull shard) { +td::Result Config::config_add_collator(ton::adnl::AdnlNodeIdShort addr, ton::ShardIdFull shard) { if (!shard.is_valid_ext()) { return td::Status::Error(PSTRING() << "invalid shard: " << shard.to_str()); } - Collator c{addr, shard}; - if (std::find(collators.begin(), collators.end(), c) != collators.end()) { + auto& shards = collators[addr]; + if (std::ranges::find(shards, shard) != collators[addr].end()) { return false; } - collators.push_back(c); + shards.push_back(shard); return true; } -td::Result Config::config_del_collator(ton::PublicKeyHash addr, ton::ShardIdFull shard) { +td::Result Config::config_del_collator(ton::adnl::AdnlNodeIdShort addr, ton::ShardIdFull shard) { if (!shard.is_valid_ext()) { return td::Status::Error(PSTRING() << "invalid shard: " << shard.to_str()); } - Collator c{addr, shard}; - auto it = std::find(collators.begin(), collators.end(), c); - if (it == collators.end()) { + auto& shards = collators[addr]; + auto it = std::ranges::find(shards, shard); + if (it == shards.end()) { return false; } - collators.erase(it); + shards.erase(it); return true; } @@ -1564,6 +1583,11 @@ td::Status ValidatorEngine::load_global_config() { } validator_options_.write().set_fast_state_serializer_enabled(fast_state_serializer_enabled_); + for (auto& id : config_.collator_node_whitelist) { + validator_options_.write().set_collator_node_whitelisted_validator(id, true); + } + validator_options_.write().set_collator_node_whitelist_enabled(config_.collator_node_whiltelist_enabled); + return td::Status::OK(); } @@ -1572,14 +1596,16 @@ void ValidatorEngine::set_shard_check_function() { validator_options_.write().set_shard_check_function([](ton::ShardIdFull shard) -> bool { return true; }); } else { std::vector shards = {ton::ShardIdFull(ton::masterchainId)}; - for (const auto& c : config_.collators) { - shards.push_back(c.shard); + for (const auto& [_, collator_shards] : config_.collators) { + for (const auto& shard : collator_shards) { + shards.push_back(shard); + } } for (const auto& s : config_.shards_to_monitor) { shards.push_back(s); } std::sort(shards.begin(), shards.end()); - shards.erase(std::unique(shards.begin(), shards.end()), shards.end()); + shards.erase(std::ranges::unique(shards).begin(), shards.end()); validator_options_.write().set_shard_check_function( [shards = std::move(shards)](ton::ShardIdFull shard) -> bool { for (auto s : shards) { @@ -1613,8 +1639,13 @@ void ValidatorEngine::load_collators_list() { return; } td::Ref list{true}; - list.write().unpack(*collators_list_); - validator_options_.write().set_collators_list(std::move(list)); + S = list.write().unpack(*collators_list_); + if (S.is_ok()) { + validator_options_.write().set_collators_list(std::move(list)); + } else { + LOG(ERROR) << "Invalid collators list: " << S.move_as_error(); + collators_list_ = {}; + } } void ValidatorEngine::load_empty_local_config(td::Promise promise) { @@ -2100,9 +2131,8 @@ void ValidatorEngine::start_full_node() { td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_permanent_key, v.first, [](td::Unit) {}); } - for (auto &c : config_.collators) { - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_collator_adnl_id, - ton::adnl::AdnlNodeIdShort(c.adnl_id)); + for (auto &[c, _] : config_.collators) { + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_collator_adnl_id, c); } for (auto &x : config_.fast_sync_member_certificates) { td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_fast_sync_member_certificate, @@ -2139,9 +2169,10 @@ void ValidatorEngine::started_lite_server() { } void ValidatorEngine::start_collator() { - for (auto &c : config_.collators) { - td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_collator, - ton::adnl::AdnlNodeIdShort(c.adnl_id), c.shard); + for (auto& [id, shards] : config_.collators) { + for (auto& shard : shards) { + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_collator, id, shard); + } } started_collator(); @@ -4054,6 +4085,92 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setCollat promise.set_value(ton::create_serialize_tl_object()); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_collatorNodeSetWhitelistedValidator &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + ton::adnl::AdnlNodeIdShort adnl_id{query.adnl_id_}; + if (query.add_) { + if (!config_.collator_node_whitelist.insert(adnl_id).second) { + promise.set_value(ton::create_serialize_tl_object()); + return; + } + } else { + if (config_.collator_node_whitelist.erase(adnl_id) == 0) { + promise.set_value(ton::create_serialize_tl_object()); + return; + } + } + + validator_options_.write().set_collator_node_whitelisted_validator(adnl_id, query.add_); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_collatorNodeSetWhitelistEnabled &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + if (config_.collator_node_whiltelist_enabled == query.enabled_) { + promise.set_value(ton::create_serialize_tl_object()); + return; + } + config_.collator_node_whiltelist_enabled = query.enabled_; + validator_options_.write().set_collator_node_whitelist_enabled(query.enabled_); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCollatorNodeWhitelist &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + ton::tl_object_ptr result = {}; + result = ton::create_tl_object(); + result->enabled_ = config_.collator_node_whiltelist_enabled; + for (const auto &id : config_.collator_node_whitelist) { + result->adnl_ids_.push_back(id.bits256_value()); + } + promise.set_value(ton::serialize_tl_object(result, true)); +} + void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getCollatorOptionsJson &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { @@ -4107,16 +4224,44 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setCollat return; } + td::Ref list{true}; + auto S = list.write().unpack(*query.list_); + if (S.is_error()) { + promise.set_value(create_control_query_error(S.move_as_error_prefix("Invalid collators list: "))); + return; + } auto s = td::json_encode(td::ToJson(*query.list_), true); - auto S = td::write_file(collators_list_file(), s); + S = td::write_file(collators_list_file(), s); if (S.is_error()) { promise.set_value(create_control_query_error(std::move(S))); return; } - collators_list_ = std::move(query.list_); - td::Ref list{true}; - list.write().unpack(*collators_list_); + validator_options_.write().set_collators_list(std::move(list)); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_clearCollatorsList &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + auto S = td::unlink(collators_list_file()); + if (S.is_error()) { + promise.set_value(create_control_query_error(std::move(S))); + return; + } + + td::Ref list{true, ton::validator::CollatorsList::default_list()}; + collators_list_ = {}; validator_options_.write().set_collators_list(std::move(list)); td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, validator_options_); @@ -4136,12 +4281,33 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showColla if (collators_list_) { promise.set_value(ton::serialize_tl_object(collators_list_, true)); } else { - auto list = ton::create_tl_object(); - list->self_collate_ = true; - promise.set_value(ton::serialize_tl_object(list, true)); + promise.set_value(create_control_query_error(td::Status::Error("collators list is empty"))); } } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getCollationManagerStats &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + td::actor::send_closure( + validator_manager_, &ton::validator::ValidatorManagerInterface::get_collation_manager_stats, + [promise = std::move(promise)]( + td::Result> R) mutable { + if (R.is_ok()) { + promise.set_value(ton::serialize_tl_object(R.move_as_ok(), true)); + } else { + promise.set_value(create_control_query_error(R.move_as_error())); + } + }); +} + void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollator &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { @@ -4154,7 +4320,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollat return; } - auto id = ton::PublicKeyHash{query.adnl_id_}; + auto id = ton::adnl::AdnlNodeIdShort{query.adnl_id_}; auto shard = ton::create_shard_id(query.shard_); auto R = config_.config_add_collator(id, shard); if (R.is_error()) { @@ -4169,12 +4335,10 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollat if (!validator_manager_.empty()) { td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, validator_options_); - td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_collator, - ton::adnl::AdnlNodeIdShort(id), shard); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_collator, id, shard); } if (!full_node_.empty()) { - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_collator_adnl_id, - ton::adnl::AdnlNodeIdShort(id)); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_collator_adnl_id, id); } write_config([promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { @@ -4229,7 +4393,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCollat return; } - auto id = ton::PublicKeyHash{query.adnl_id_}; + auto id = ton::adnl::AdnlNodeIdShort{query.adnl_id_}; auto shard = ton::create_shard_id(query.shard_); auto R = config_.config_del_collator(id, shard); if (R.is_error()) { @@ -4248,12 +4412,10 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCollat if (!validator_manager_.empty()) { td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, validator_options_); - td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::del_collator, - ton::adnl::AdnlNodeIdShort(id), shard); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::del_collator, id, shard); } if (!full_node_.empty()) { - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::del_collator_adnl_id, - ton::adnl::AdnlNodeIdShort(id)); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::del_collator_adnl_id, id); } write_config([promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { @@ -4318,7 +4480,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signOverl int expire_at = query.expire_at_; td::actor::send_closure( keyring_, &ton::keyring::Keyring::get_public_key, public_key_hash, - [=, promise = std::move(promise)](td::Result R) mutable { + [=, keyring = keyring_.get(), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); return; @@ -4330,7 +4492,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signOverl return; } td::BufferSlice to_sign = certificate.to_sign_data(adnl_id); - td::actor::send_closure(keyring_, &ton::keyring::Keyring::sign_message, public_key_hash, std::move(to_sign), + td::actor::send_closure(keyring, &ton::keyring::Keyring::sign_message, public_key_hash, std::move(to_sign), [certificate = std::move(certificate), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 4a8e9bdb4..7daefb57a 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -67,14 +67,6 @@ struct Config { ton::UnixTime election_date; ton::UnixTime expire_at; }; - struct Collator { - ton::PublicKeyHash adnl_id; - ton::ShardIdFull shard; - - bool operator==(const Collator& b) const { - return adnl_id == b.adnl_id && shard == b.shard; - } - }; struct Control { ton::PublicKeyHash key; std::map clients; @@ -90,7 +82,9 @@ struct Config { std::map adnl_ids; std::set dht_ids; std::map validators; - std::vector collators; + std::map> collators; + bool collator_node_whiltelist_enabled = false; + std::set collator_node_whitelist; ton::PublicKeyHash full_node = ton::PublicKeyHash::zero(); std::vector full_node_slaves; std::map full_node_masters; @@ -120,8 +114,8 @@ struct Config { ton::UnixTime expire_at); td::Result config_add_validator_adnl_id(ton::PublicKeyHash perm_key, ton::PublicKeyHash adnl_id, ton::UnixTime expire_at); - td::Result config_add_collator(ton::PublicKeyHash addr, ton::ShardIdFull shard); - td::Result config_del_collator(ton::PublicKeyHash addr, ton::ShardIdFull shard); + td::Result config_add_collator(ton::adnl::AdnlNodeIdShort addr, ton::ShardIdFull shard); + td::Result config_del_collator(ton::adnl::AdnlNodeIdShort addr, ton::ShardIdFull shard); td::Result config_add_full_node_adnl_id(ton::PublicKeyHash id); td::Result config_add_full_node_slave(td::IPAddress addr, ton::PublicKey id); td::Result config_add_full_node_master(td::int32 port, ton::PublicKeyHash id); @@ -537,10 +531,21 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_setCollatorOptionsJson &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_collatorNodeSetWhitelistedValidator &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_collatorNodeSetWhitelistEnabled &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_showCollatorNodeWhitelist &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_setCollatorsList &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_clearCollatorsList &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_showCollatorsList &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_getCollationManagerStats &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_signOverlayMemberCertificate &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_importFastSyncMemberCertificate &query, td::BufferSlice data, diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index 60f66af37..abc602674 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -58,6 +58,7 @@ set(VALIDATOR_HEADERS import-db-slice.hpp queue-size-counter.hpp + collation-manager.hpp collator-node.hpp manager-disk.h manager-disk.hpp @@ -74,6 +75,7 @@ set(VALIDATOR_HEADERS set(VALIDATOR_SOURCE apply-block.cpp block-handle.cpp + collation-manager.cpp collator-node.cpp get-next-key-blocks.cpp import-db-slice.cpp diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp new file mode 100644 index 000000000..2ca3a5f10 --- /dev/null +++ b/validator/collation-manager.cpp @@ -0,0 +1,373 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "collation-manager.hpp" + +#include "collator-node.hpp" +#include "fabric.h" +#include "td/utils/Random.h" + +#include +#include +#include + +namespace ton::validator { + +void CollationManager::start_up() { + td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); + update_collators_list(*opts_->get_collators_list()); +} + +void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, + std::vector prev, Ed25519_PublicKey creator, + td::Ref validator_set, td::uint64 max_answer_size, + td::CancellationToken cancellation_token, td::Promise promise) { + if (shard.is_masterchain()) { + run_collate_query(shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), + opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), std::move(promise), + std::move(cancellation_token), 0); + return; + } + collate_shard_block(shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), + max_answer_size, std::move(cancellation_token), std::move(promise), td::Timestamp::in(10.0)); +} + +void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, + std::vector prev, Ed25519_PublicKey creator, + td::Ref validator_set, td::uint64 max_answer_size, + td::CancellationToken cancellation_token, + td::Promise promise, td::Timestamp timeout) { + TRY_STATUS_PROMISE(promise, cancellation_token.check()); + ShardInfo* s = select_shard_info(shard); + if (s == nullptr) { + promise.set_error( + td::Status::Error(PSTRING() << "shard " << shard.to_str() << " is not configured in collators list")); + return; + } + + adnl::AdnlNodeIdShort selected_collator = adnl::AdnlNodeIdShort::zero(); + size_t selected_idx = 0; + switch (s->select_mode) { + case CollatorsList::mode_random: { + int cnt = 0; + for (size_t i = 0; i < s->collators.size(); ++i) { + adnl::AdnlNodeIdShort collator = s->collators[i]; + if (collators_[collator].alive) { + ++cnt; + if (td::Random::fast(1, cnt) == 1) { + selected_collator = collator; + selected_idx = i; + } + } + } + break; + } + case CollatorsList::mode_ordered: { + for (size_t i = 0; i < s->collators.size(); ++i) { + adnl::AdnlNodeIdShort collator = s->collators[i]; + if (collators_[collator].alive) { + selected_collator = collator; + selected_idx = i; + break; + } + } + break; + } + case CollatorsList::mode_round_robin: { + size_t iters = 0; + for (size_t i = s->cur_idx; iters < s->collators.size(); (++i) %= s->collators.size()) { + adnl::AdnlNodeIdShort& collator = s->collators[i]; + if (collators_[collator].alive) { + selected_collator = collator; + selected_idx = i; + s->cur_idx = (i + 1) % s->collators.size(); + break; + } + } + break; + } + } + + if (selected_collator.is_zero() && s->self_collate) { + run_collate_query(shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), + opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), std::move(promise), + std::move(cancellation_token), 0); + return; + } + + std::vector> prev_blocks; + BlockId next_block_id{shard, 0}; + for (const BlockIdExt& p : prev) { + prev_blocks.push_back(create_tl_block_id(p)); + next_block_id.seqno = std::max(next_block_id.seqno, p.seqno() + 1); + } + + promise = [=, SelfId = actor_id(this), promise = std::move(promise), + retry_at = td::Timestamp::in(0.5)](td::Result R) mutable { + if (R.is_ok()) { + promise.set_value(R.move_as_ok()); + return; + } + if (!selected_collator.is_zero()) { + td::actor::send_closure(SelfId, &CollationManager::on_collate_query_error, selected_collator); + } + LOG(INFO) << "ERROR: collate query for " << next_block_id.to_str() << " to #" << selected_idx << " (" + << selected_collator << "): " << R.error(); + if (timeout < retry_at) { + promise.set_error(R.move_as_error()); + return; + } + delay_action( + [=, promise = std::move(promise)]() mutable { + td::actor::send_closure(SelfId, &CollationManager::collate_shard_block, shard, min_masterchain_block_id, prev, + creator, validator_set, max_answer_size, cancellation_token, std::move(promise), + timeout); + }, + retry_at); + }; + + if (selected_collator.is_zero()) { + promise.set_error(td::Status::Error(PSTRING() << "shard " << shard.to_str() << " has no alive collator node")); + return; + } + + td::BufferSlice query = create_serialize_tl_object( + create_tl_shard_id(shard), validator_set->get_catchain_seqno(), std::move(prev_blocks), creator.as_bits256()); + LOG(INFO) << "sending collate query for " << next_block_id.to_str() << ": send to #" << selected_idx << "(" + << selected_collator << ")"; + + td::Promise P = [=, SelfId = actor_id(this), promise = std::move(promise), + timer = td::Timer()](td::Result R) mutable { + TRY_RESULT_PROMISE_PREFIX(promise, data, std::move(R), "rldp query failed: "); + auto r_error = fetch_tl_object(data, true); + if (r_error.is_ok()) { + auto error = r_error.move_as_ok(); + promise.set_error(td::Status::Error(error->code_, error->message_)); + return; + } + TRY_RESULT_PROMISE(promise, f, fetch_tl_object(data, true)); + TRY_RESULT_PROMISE(promise, candidate, + CollatorNode::deserialize_candidate(std::move(f), td::narrow_cast(max_answer_size))); + if (candidate.pubkey.as_bits256() != creator.as_bits256()) { + promise.set_error(td::Status::Error("collate query: block candidate source mismatch")); + return; + } + if (candidate.id.id != next_block_id) { + promise.set_error(td::Status::Error("collate query: block id mismatch")); + return; + } + LOG(INFO) << "got collated block " << next_block_id.to_str() << " from #" << selected_idx << " (" + << selected_collator << ") in " << timer.elapsed() << "s"; + promise.set_result(std::move(candidate)); + }; + td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_id_, selected_collator, "collatequery", std::move(P), + timeout, std::move(query), max_answer_size); +} + +void CollationManager::update_options(td::Ref opts) { + auto old_list = opts_->get_collators_list(); + opts_ = std::move(opts); + auto list = opts_->get_collators_list(); + if (old_list != list) { + update_collators_list(*list); + } +} + +void CollationManager::validator_group_started(ShardIdFull shard) { + if (active_validator_groups_[shard]++ != 0) { + return; + } + ShardInfo* s = select_shard_info(shard); + if (s == nullptr) { + return; + } + if (s->active_cnt++ != 0) { + return; + } + for (adnl::AdnlNodeIdShort id : s->collators) { + CollatorInfo& collator = collators_[id]; + collator.active_cnt++; + } + alarm(); +} + +void CollationManager::validator_group_finished(ShardIdFull shard) { + if (--active_validator_groups_[shard] != 0) { + return; + } + active_validator_groups_.erase(shard); + ShardInfo* s = select_shard_info(shard); + if (s == nullptr) { + return; + } + if (--s->active_cnt != 0) { + return; + } + for (adnl::AdnlNodeIdShort id : s->collators) { + CollatorInfo& collator = collators_[id]; + --collator.active_cnt; + } + alarm(); +} + +void CollationManager::get_stats( + td::Promise> promise) { + auto stats = create_tl_object(); + stats->adnl_id_ = local_id_.bits256_value(); + for (ShardInfo& s : shards_) { + auto obj = create_tl_object(); + obj->shard_id_ = create_tl_shard_id(s.shard_id); + obj->active_ = s.active_cnt; + obj->self_collate_ = s.self_collate; + switch (s.select_mode) { + case CollatorsList::mode_random: + obj->select_mode_ = "random"; + break; + case CollatorsList::mode_ordered: + obj->select_mode_ = "ordered"; + break; + case CollatorsList::mode_round_robin: + obj->select_mode_ = "round_robin"; + break; + } + for (adnl::AdnlNodeIdShort& id : s.collators) { + obj->collators_.push_back(id.bits256_value()); + } + stats->shards_.push_back(std::move(obj)); + } + for (auto& [id, collator] : collators_) { + auto obj = create_tl_object(); + obj->adnl_id_ = id.bits256_value(); + obj->active_ = collator.active_cnt; + obj->alive_ = collator.alive; + if (collator.active_cnt && !collator.sent_ping) { + obj->ping_in_ = collator.ping_at.in(); + } else { + obj->ping_in_ = -1.0; + } + stats->collators_.push_back(std::move(obj)); + } + promise.set_value(std::move(stats)); +} + +void CollationManager::update_collators_list(const CollatorsList& collators_list) { + shards_.clear(); + for (auto& [_, collator] : collators_) { + collator.active_cnt = 0; + } + auto old_collators = std::move(collators_); + collators_.clear(); + for (const auto& shard : collators_list.shards) { + shards_.push_back({.shard_id = shard.shard_id, .select_mode = shard.select_mode, .collators = shard.collators}); + for (auto id : shard.collators) { + auto it = old_collators.find(id); + if (it == old_collators.end()) { + collators_[id]; + } else { + collators_[id] = std::move(it->second); + old_collators.erase(it); + } + } + } + for (auto& [shard, _] : active_validator_groups_) { + ShardInfo* s = select_shard_info(shard); + if (s == nullptr) { + continue; + } + if (s->active_cnt++ != 0) { + continue; + } + for (adnl::AdnlNodeIdShort id : s->collators) { + CollatorInfo& collator = collators_[id]; + collator.active_cnt++; + } + } + alarm(); +} + +CollationManager::ShardInfo* CollationManager::select_shard_info(ShardIdFull shard) { + for (auto& s : shards_) { + if (shard_intersects(shard, s.shard_id)) { + return &s; + } + } + return nullptr; +} + +void CollationManager::alarm() { + alarm_timestamp() = td::Timestamp::never(); + for (auto& [id, collator] : collators_) { + if (collator.active_cnt == 0 || collator.sent_ping) { + continue; + } + if (collator.ping_at.is_in_past()) { + collator.sent_ping = true; + td::BufferSlice query = create_serialize_tl_object(0); + td::Promise P = [=, SelfId = actor_id(this)](td::Result R) mutable { + td::actor::send_closure(SelfId, &CollationManager::got_pong, id, std::move(R)); + }; + LOG(DEBUG) << "sending ping to " << id; + td::actor::send_closure(rldp_, &rldp::Rldp::send_query, local_id_, id, "collatorping", std::move(P), + td::Timestamp::in(2.0), std::move(query)); + } else { + alarm_timestamp().relax(collator.ping_at); + } + } +} + +void CollationManager::got_pong(adnl::AdnlNodeIdShort id, td::Result R) { + auto it = collators_.find(id); + if (it == collators_.end()) { + return; + } + CollatorInfo& collator = it->second; + collator.sent_ping = false; + + auto r_pong = [&]() -> td::Result> { + TRY_RESULT_PREFIX(data, std::move(R), "rldp query error: "); + auto r_error = fetch_tl_object(data, true); + if (r_error.is_ok()) { + auto error = r_error.move_as_ok(); + return td::Status::Error(error->code_, error->message_); + } + return fetch_tl_object(data, true); + }(); + if (r_pong.is_error()) { + LOG(DEBUG) << "pong from " << id << " : " << r_pong.move_as_error(); + collator.alive = false; + } else { + LOG(DEBUG) << "pong from " << id << " : OK"; + collator.alive = true; + } + collator.ping_at = td::Timestamp::in(td::Random::fast(10.0, 20.0)); + if (collator.active_cnt && !collator.sent_ping) { + alarm_timestamp().relax(collator.ping_at); + } +} + +void CollationManager::on_collate_query_error(adnl::AdnlNodeIdShort id) { + auto it = collators_.find(id); + if (it == collators_.end()) { + return; + } + CollatorInfo& collator = it->second; + collator.ping_at = td::Timestamp::now(); + if (collator.active_cnt && !collator.sent_ping) { + alarm_timestamp().relax(collator.ping_at); + } +} + +} // namespace ton::validator diff --git a/validator/collation-manager.hpp b/validator/collation-manager.hpp new file mode 100644 index 000000000..7ceea1e6b --- /dev/null +++ b/validator/collation-manager.hpp @@ -0,0 +1,87 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "interfaces/validator-manager.h" +#include "rldp/rldp.h" +#include + +namespace ton::validator { + +class ValidatorManager; + +class CollationManager : public td::actor::Actor { + public: + CollationManager(adnl::AdnlNodeIdShort local_id, td::Ref opts, + td::actor::ActorId manager, td::actor::ActorId rldp) + : local_id_(local_id), opts_(opts), manager_(manager), rldp_(rldp) { + } + + void start_up() override; + void alarm() override; + + void collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, + Ed25519_PublicKey creator, td::Ref validator_set, td::uint64 max_answer_size, + td::CancellationToken cancellation_token, td::Promise promise); + + void update_options(td::Ref opts); + + void validator_group_started(ShardIdFull shard); + void validator_group_finished(ShardIdFull shard); + + void get_stats(td::Promise> promise); + + private: + adnl::AdnlNodeIdShort local_id_; + td::Ref opts_; + td::actor::ActorId manager_; + td::actor::ActorId rldp_; + + void collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, + Ed25519_PublicKey creator, td::Ref validator_set, td::uint64 max_answer_size, + td::CancellationToken cancellation_token, td::Promise promise, + td::Timestamp timeout); + + void update_collators_list(const CollatorsList& collators_list); + + struct CollatorInfo { + bool alive = false; + td::Timestamp ping_at = td::Timestamp::now(); + bool sent_ping = false; + size_t active_cnt = 0; + }; + std::map collators_; + + struct ShardInfo { + ShardIdFull shard_id; + CollatorsList::SelectMode select_mode; + std::vector collators; + bool self_collate = false; + size_t cur_idx = 0; + + size_t active_cnt = 0; + }; + std::vector shards_; + + std::map active_validator_groups_; + + ShardInfo* select_shard_info(ShardIdFull shard); + void got_pong(adnl::AdnlNodeIdShort id, td::Result R); + void on_collate_query_error(adnl::AdnlNodeIdShort id); +}; + +} // namespace ton::validator diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 0b4204d86..776fbc9e9 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -54,17 +54,22 @@ void CollatorNode::start_up() { td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlock::ID), std::make_unique(actor_id(this))); + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_ping::ID), + std::make_unique(actor_id(this))); td::actor::send_closure(rldp_, &rldp::Rldp::add_id, adnl::AdnlNodeIdShort(local_id_)); } void CollatorNode::tear_down() { td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlock::ID)); + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_ping::ID)); } void CollatorNode::add_shard(ShardIdFull shard) { CHECK(shard.is_valid_ext() && !shard.is_masterchain()); - if (std::find(collating_shards_.begin(), collating_shards_.end(), shard) != collating_shards_.end()) { + if (std::ranges::find(collating_shards_, shard) != collating_shards_.end()) { return; } LOG(INFO) << "Collator node: local_id=" << local_id_ << " , shard=" << shard.to_str(); @@ -72,7 +77,7 @@ void CollatorNode::add_shard(ShardIdFull shard) { } void CollatorNode::del_shard(ShardIdFull shard) { - auto it = std::find(collating_shards_.begin(), collating_shards_.end(), shard); + auto it = std::ranges::find(collating_shards_, shard); if (it != collating_shards_.end()) { collating_shards_.erase(it); } @@ -127,7 +132,7 @@ void CollatorNode::new_masterchain_block_notification(td::Ref } } for (auto it = validator_groups_.begin(); it != validator_groups_.end();) { - if (new_shards.count(it->first)) { + if (new_shards.contains(it->first)) { ++it; } else { it->second.cleanup(); @@ -253,7 +258,7 @@ void CollatorNode::CacheEntry::cancel(td::Status reason) { } static td::BufferSlice serialize_error(td::Status error) { - return create_serialize_tl_object(error.code(), error.message().c_str()); + return create_serialize_tl_object(error.code(), error.message().c_str()); } static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey creator, CatchainSeqno& cc_seqno, @@ -285,25 +290,32 @@ static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey cre void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise) { - td::Promise new_promise = [promise = std::move(promise), src](td::Result R) mutable { + promise = [promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { - LOG(INFO) << "adnl query from " << src << ", error: " << R.error(); if (R.error().code() == ErrorCode::timeout) { promise.set_error(R.move_as_error()); } else { promise.set_result(serialize_error(R.move_as_error())); } } else { - LOG(INFO) << "adnl query from " << src << ", success"; - promise.set_result(create_serialize_tl_object( - serialize_candidate(R.move_as_ok(), true))); + promise.set_result(R.move_as_ok()); } }; - if (!validator_adnl_ids_.count(src)) { - new_promise.set_error(td::Status::Error("src is not a validator")); + if (!opts_->check_collator_node_whitelist(src)) { + promise.set_error(td::Status::Error("not authorized")); + return; + } + if (!validator_adnl_ids_.contains(src)) { + promise.set_error(td::Status::Error("src is not a validator")); + return; + } + auto r_ping = fetch_tl_object(data, true); + if (r_ping.is_ok()) { + process_ping(src, *r_ping.ok_ref(), std::move(promise)); return; } - TRY_RESULT_PROMISE(new_promise, f, fetch_tl_object(data, true)); + + TRY_RESULT_PROMISE(promise, f, fetch_tl_object(data, true)); ShardIdFull shard = create_shard_id(f->shard_); CatchainSeqno cc_seqno = f->cc_seqno_; std::vector prev_blocks; @@ -311,6 +323,16 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data prev_blocks.push_back(create_block_id(b)); } Ed25519_PublicKey creator(f->creator_); + td::Promise new_promise = [promise = std::move(promise), src, + shard](td::Result R) mutable { + if (R.is_error()) { + LOG(INFO) << "collate query from " << src << ", shard=" << shard.to_str() << ": error: " << R.error(); + promise.set_error(R.move_as_error()); + } else { + LOG(INFO) << "collate query from " << src << ", shard=" << shard.to_str() << ": success"; + promise.set_result(serialize_tl_object(serialize_candidate(R.move_as_ok(), true), true)); + } + }; new_promise = [new_promise = std::move(new_promise), creator, manager = manager_](td::Result R) mutable { TRY_RESULT_PROMISE(new_promise, block, std::move(R)); @@ -427,9 +449,15 @@ void CollatorNode::process_result(std::shared_ptr cache_entry, td::R cache_entry->promises.clear(); } +void CollatorNode::process_ping(adnl::AdnlNodeIdShort src, ton_api::collatorNode_ping& ping, + td::Promise promise) { + LOG(DEBUG) << "got ping from " << src; + promise.set_result(create_serialize_tl_object(0)); +} + bool CollatorNode::can_collate_shard(ShardIdFull shard) const { - return std::any_of(collating_shards_.begin(), collating_shards_.end(), - [&](const ShardIdFull& our_shard) { return shard_intersects(shard, our_shard); }); + return std::ranges::any_of(collating_shards_, + [&](const ShardIdFull& our_shard) { return shard_intersects(shard, our_shard); }); } tl_object_ptr CollatorNode::serialize_candidate(const BlockCandidate& block, diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp index 1a71f08a4..a361ceb30 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node.hpp @@ -43,6 +43,7 @@ class CollatorNode : public td::actor::Actor { private: void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); + void process_ping(adnl::AdnlNodeIdShort src, ton_api::collatorNode_ping& ping, td::Promise promise); bool can_collate_shard(ShardIdFull shard) const; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index bb806e281..4488b77df 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -502,10 +502,16 @@ td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { if (pfx_len > wc_monitor_min_split_) { shard = shard_prefix(shard, wc_monitor_min_split_); } - auto it = shards_.find(shard); - if (it != shards_.end()) { - update_shard_actor(shard, it->second.active); - return it->second.actor.get(); + while (true) { + auto it = shards_.find(shard); + if (it != shards_.end()) { + update_shard_actor(shard, it->second.active); + return it->second.actor.get(); + } + if (shard.pfx_len() == 0) { + break; + } + shard = shard_parent(shard); } // Special case if shards_ was not yet initialized. diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index d91ee1bf6..72154f861 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -45,7 +45,7 @@ class Collator final : public td::actor::Actor { } static constexpr long long supported_capabilities() { return ton::capCreateStatsEnabled | ton::capBounceMsgBody | ton::capReportVersion | ton::capShortDequeue | - ton::capStoreOutMsgQueueSize | ton::capMsgMetadata | ton::capDeferMessages; + ton::capStoreOutMsgQueueSize | ton::capMsgMetadata | ton::capDeferMessages | ton::capFullCollatedData; } using LtCellRef = block::LtCellRef; using NewOutMsg = block::NewOutMsg; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 5b82d481a..677b73ae0 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -730,6 +730,8 @@ bool Collator::unpack_last_mc_state() { store_out_msg_queue_size_ = config_->has_capability(ton::capStoreOutMsgQueueSize); msg_metadata_enabled_ = config_->has_capability(ton::capMsgMetadata); deferring_messages_enabled_ = config_->has_capability(ton::capDeferMessages); + full_collated_data_ = config_->has_capability(capFullCollatedData); + LOG(DEBUG) << "full_collated_data is " << full_collated_data_; shard_conf_ = std::make_unique(*config_); prev_key_block_exists_ = config_->get_last_key_block(prev_key_block_, prev_key_block_lt_); if (prev_key_block_exists_) { @@ -768,8 +770,6 @@ bool Collator::unpack_last_mc_state() { << " have been enabled in global configuration, but we support only " << supported_version() << " (upgrade validator software?)"; } - full_collated_data_ = config_->get_collator_config(false).full_collated_data; - LOG(DEBUG) << "full_collated_data is " << full_collated_data_; // TODO: extract start_lt and end_lt from prev_mc_block as well // std::cerr << " block::gen::ShardState::print_ref(mc_state_root) = "; // block::gen::t_ShardState.print_ref(std::cerr, mc_state_root, 2); @@ -817,6 +817,9 @@ bool Collator::request_neighbor_msg_queues() { auto neighbor_list = shard_conf_->get_neighbor_shard_hash_ids(shard_); LOG(DEBUG) << "got a preliminary list of " << neighbor_list.size() << " neighbors for " << shard_.to_str(); for (ton::BlockId blk_id : neighbor_list) { + if (blk_id.seqno == 0 && blk_id.shard_full() != shard_) { + continue; + } auto shard_ptr = shard_conf_->get_shard_hash(ton::ShardIdFull(blk_id)); if (shard_ptr.is_null()) { return fatal_error(-667, "cannot obtain shard hash for neighbor "s + blk_id.to_str()); diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index c8c7aca20..b13be0219 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -170,9 +170,6 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { block::WorkchainSet get_workchain_list() const override { return config_ ? config_->get_workchain_list() : block::WorkchainSet(); } - block::CollatorConfig get_collator_config(bool need_collator_nodes) const override { - return config_ ? config_->get_collator_config(need_collator_nodes) : block::CollatorConfig(); - } private: ZeroStateIdExt zerostate_id_; diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index ca00e00e9..4c6264025 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -1536,6 +1536,9 @@ bool ValidateQuery::request_neighbor_queues() { auto neighbor_list = new_shard_conf_->get_neighbor_shard_hash_ids(shard_); LOG(DEBUG) << "got a preliminary list of " << neighbor_list.size() << " neighbors for " << shard_.to_str(); for (ton::BlockId blk_id : neighbor_list) { + if (blk_id.seqno == 0 && blk_id.shard_full() != shard_) { + continue; + } auto shard_ptr = new_shard_conf_->get_shard_hash(ton::ShardIdFull(blk_id)); if (shard_ptr.is_null()) { return reject_query("cannot obtain shard hash for neighbor "s + blk_id.to_str()); @@ -2305,6 +2308,12 @@ bool ValidateQuery::prepare_out_msg_queue_size() { have_out_msg_queue_size_in_state_ = true; return true; } + if (ps_.out_msg_queue_->is_empty()) { + old_out_msg_queue_size_ = 0; + out_msg_queue_size_known_ = true; + have_out_msg_queue_size_in_state_ = true; + return true; + } if (!store_out_msg_queue_size_) { // Don't need it return true; } diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index a21a56e90..585d553c5 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -113,7 +113,7 @@ class ValidateQuery : public td::actor::Actor { } static constexpr long long supported_capabilities() { return ton::capCreateStatsEnabled | ton::capBounceMsgBody | ton::capReportVersion | ton::capShortDequeue | - ton::capStoreOutMsgQueueSize | ton::capMsgMetadata | ton::capDeferMessages; + ton::capStoreOutMsgQueueSize | ton::capMsgMetadata | ton::capDeferMessages | ton::capFullCollatedData; } public: diff --git a/validator/impl/validator-set.hpp b/validator/impl/validator-set.hpp index 3141f36c5..8b833e047 100644 --- a/validator/impl/validator-set.hpp +++ b/validator/impl/validator-set.hpp @@ -50,6 +50,7 @@ class ValidatorSetQ : public ValidatorSet { td::Ref signatures) const override; td::Result check_approve_signatures(RootHash root_hash, FileHash file_hash, td::Ref signatures) const override; + const ValidatorDescr* find_validator(const NodeIdShort& id) const override; ValidatorSetQ* make_copy() const override; @@ -62,8 +63,6 @@ class ValidatorSetQ : public ValidatorSet { ValidatorWeight total_weight_; std::vector ids_; std::vector> ids_map_; - - const ValidatorDescr* find_validator(const NodeIdShort& id) const; }; class ValidatorSetCompute { diff --git a/validator/interfaces/shard.h b/validator/interfaces/shard.h index 3383f6fc2..3546f0a39 100644 --- a/validator/interfaces/shard.h +++ b/validator/interfaces/shard.h @@ -85,7 +85,6 @@ class MasterchainState : virtual public ShardState { virtual bool check_old_mc_block_id(const ton::BlockIdExt& blkid, bool strict = false) const = 0; virtual td::Result> get_config_holder() const = 0; virtual block::WorkchainSet get_workchain_list() const = 0; - virtual block::CollatorConfig get_collator_config(bool need_collator_nodes) const = 0; virtual td::Status prepare() { return td::Status::OK(); } diff --git a/validator/interfaces/validator-set.h b/validator/interfaces/validator-set.h index b71c0bfea..20569df75 100644 --- a/validator/interfaces/validator-set.h +++ b/validator/interfaces/validator-set.h @@ -35,6 +35,7 @@ class ValidatorSet : public td::CntObject { virtual td::uint32 get_validator_set_hash() const = 0; virtual ShardId get_validator_set_from() const = 0; virtual std::vector export_vector() const = 0; + virtual const ValidatorDescr* find_validator(const NodeIdShort& id) const = 0; virtual td::Result check_signatures(RootHash root_hash, FileHash file_hash, td::Ref signatures) const = 0; virtual td::Result check_approve_signatures(RootHash root_hash, FileHash file_hash, diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index c54ccde55..c6d7337f9 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -457,6 +457,11 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } + void get_collation_manager_stats( + td::Promise> promise) override { + UNREACHABLE(); + } + void update_options(td::Ref opts) override { opts_ = std::move(opts); } diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index daf83a197..860243fb9 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -525,6 +525,11 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } + void get_collation_manager_stats( + td::Promise> promise) override { + UNREACHABLE(); + } + private: td::Ref opts_; diff --git a/validator/manager.cpp b/validator/manager.cpp index f97263570..3f59d784d 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -799,7 +799,13 @@ void ValidatorManagerImpl::wait_neighbor_msg_queue_proofs( public: Worker(size_t pending, td::Promise>> promise) : pending_(pending), promise_(std::move(promise)) { - CHECK(pending_ > 0); + } + + void start_up() override { + if (pending_ == 0) { + promise_.set_result(std::move(result_)); + stop(); + } } void on_result(td::Ref res) { @@ -2462,15 +2468,27 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group auto validator_id = get_validator(shard, validator_set); CHECK(!validator_id.is_zero()); + auto descr = validator_set->find_validator(validator_id.bits256_value()); + CHECK(descr); + auto adnl_id = adnl::AdnlNodeIdShort{ + descr->addr.is_zero() ? ValidatorFullId{descr->key}.compute_short_id().bits256_value() : descr->addr}; auto G = td::actor::create_actor( - PSTRING() << "valgroup" << shard.to_str(), shard, validator_id, session_id, validator_set, key_seqno, - last_masterchain_state_->get_collator_config(true), opts, keyring_, adnl_, rldp_, overlays_, db_root_, - actor_id(this), init_session, opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_, + PSTRING() << "valgroup" << shard.to_str(), shard, validator_id, session_id, validator_set, key_seqno, opts, + keyring_, adnl_, rldp_, overlays_, db_root_, actor_id(this), get_collation_manager(adnl_id), init_session, + opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_, opts_->need_monitor(shard, last_masterchain_state_)); return G; } } +td::actor::ActorId ValidatorManagerImpl::get_collation_manager(adnl::AdnlNodeIdShort adnl_id) { + auto &actor = collation_managers_[adnl_id]; + if (actor.empty()) { + actor = td::actor::create_actor("collation", adnl_id, opts_, actor_id(this), rldp_); + } + return actor.get(); +} + void ValidatorManagerImpl::add_handle_to_lru(BlockHandle handle) { auto it = handle_lru_map_.find(handle->id()); if (it != handle_lru_map_.end()) { @@ -3458,6 +3476,9 @@ void ValidatorManagerImpl::update_options(td::Ref opts) for (auto &collator : collator_nodes_) { td::actor::send_closure(collator.second.actor, &CollatorNode::update_options, opts); } + for (auto &[_, c] : collation_managers_) { + td::actor::send_closure(c, &CollationManager::update_options, opts); + } opts_ = std::move(opts); } @@ -3492,6 +3513,54 @@ void ValidatorManagerImpl::del_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh } } +void ValidatorManagerImpl::get_collation_manager_stats( + td::Promise> promise) { + class Cb : public td::actor::Actor { + public: + explicit Cb(td::Promise> promise) + : promise_(std::move(promise)) { + } + + void got_stats(tl_object_ptr s) { + result_.push_back(std::move(s)); + dec_pending(); + } + + void inc_pending() { + ++pending_; + } + + void dec_pending() { + CHECK(pending_ > 0); + --pending_; + if (pending_ == 0) { + promise_.set_result(create_tl_object(std::move(result_))); + stop(); + } + } + + private: + td::Promise> promise_; + size_t pending_ = 1; + std::vector> result_; + }; + auto callback = td::actor::create_actor("stats", std::move(promise)).release(); + + for (auto &[_, actor] : collation_managers_) { + td::actor::send_closure(callback, &Cb::inc_pending); + td::actor::send_closure( + actor, &CollationManager::get_stats, + [callback](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(callback, &Cb::dec_pending); + } else { + td::actor::send_closure(callback, &Cb::got_stats, R.move_as_ok()); + } + }); + } + td::actor::send_closure(callback, &Cb::dec_pending); +} + void ValidatorManagerImpl::add_persistent_state_description(td::Ref desc) { auto now = (UnixTime)td::Clocks::system(); if (desc->end_time <= now) { diff --git a/validator/manager.hpp b/validator/manager.hpp index b7e457e27..055adf304 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -283,12 +283,15 @@ class ValidatorManagerImpl : public ValidatorManager { td::Ref validator_set, BlockSeqno key_seqno, validatorsession::ValidatorSessionOptions opts, bool create_catchain); + td::actor::ActorId get_collation_manager(adnl::AdnlNodeIdShort adnl_id); + struct ValidatorGroupEntry { td::actor::ActorOwn actor; ShardIdFull shard; }; std::map validator_groups_; std::map next_validator_groups_; + std::map> collation_managers_; std::set check_gc_list_; std::vector gc_list_; @@ -634,6 +637,9 @@ class ValidatorManagerImpl : public ValidatorManager { void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; + void get_collation_manager_stats( + td::Promise> promise) override; + void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { if (last_masterchain_state_.is_null()) { diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 55798a482..f95266b33 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -65,7 +65,10 @@ void ValidatorGroup::generate_block_candidate( td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_candidate, source_info, std::move(cache), std::move(R)); }; - collate_block(source_info, td::Timestamp::in(10.0), std::move(P)); + td::uint64 max_answer_size = config_.max_block_size + config_.max_collated_data_size + 1024; + td::actor::send_closure(collation_manager_, &CollationManager::collate_block, shard_, min_masterchain_block_id_, + prev_block_ids_, Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, validator_set_, + max_answer_size, cancellation_token_source_.get_cancellation_token(), std::move(P)); } void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo source_info, @@ -218,9 +221,9 @@ void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref return; } LOG_CHECK(R.error().code() == ErrorCode::timeout || R.error().code() == ErrorCode::notready) << R.move_as_error(); - td::actor::send_closure(SelfId, &ValidatorGroup::accept_block_query, block_id, std::move(block), - std::move(prev), std::move(sig_set), std::move(approve_sig_set), send_broadcast_mode, - std::move(promise), true); + td::actor::send_closure(SelfId, &ValidatorGroup::accept_block_query, block_id, std::move(block), std::move(prev), + std::move(sig_set), std::move(approve_sig_set), send_broadcast_mode, std::move(promise), + true); } else { promise.set_value(R.move_as_ok()); } @@ -292,7 +295,8 @@ std::unique_ptr ValidatorGroup::ma td::actor::send_closure(id_, &ValidatorGroup::generate_block_candidate, std::move(source_info), std::move(promise)); } - void on_block_committed(validatorsession::BlockSourceInfo source_info, validatorsession::ValidatorSessionRootHash root_hash, + void on_block_committed(validatorsession::BlockSourceInfo source_info, + validatorsession::ValidatorSessionRootHash root_hash, validatorsession::ValidatorSessionFileHash file_hash, td::BufferSlice data, std::vector> signatures, std::vector> approve_signatures, @@ -409,7 +413,7 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch stats.last_key_block_seqno = last_key_block_seqno_; stats.timestamp = td::Clocks::system(); td::uint32 idx = 0; - for (const auto& node : validator_set_->export_vector()) { + for (const auto &node : validator_set_->export_vector()) { PublicKeyHash id = ValidatorFullId{node.key}.compute_short_id(); if (id == local_id_) { stats.self_idx = idx; @@ -489,7 +493,7 @@ void ValidatorGroup::get_validator_group_info_for_litequery_cont( auto result = create_tl_object(); result->next_block_id_ = create_tl_lite_block_id_simple(next_block_id); - for (const BlockIdExt& prev : prev_block_ids_) { + for (const BlockIdExt &prev : prev_block_ids_) { result->prev_.push_back(create_tl_lite_block_id(prev)); } result->cc_seqno_ = validator_set_->get_catchain_seqno(); @@ -497,146 +501,6 @@ void ValidatorGroup::get_validator_group_info_for_litequery_cont( promise.set_result(std::move(result)); } -void ValidatorGroup::collate_block(validatorsession::BlockSourceInfo source_info, td::Timestamp timeout, - td::Promise promise, unsigned max_retries) { - if (source_info.round < last_known_round_id_) { - promise.set_error(td::Status::Error("too old")); - return; - } - BlockId next_block_id = create_next_block_id_simple(); - adnl::AdnlNodeIdShort collator_adnl_id = adnl::AdnlNodeIdShort::zero(); - bool self_collate = false; - bool trusted_collator = false; - - if (shard_.is_masterchain()) { - self_collate = true; - } else { - for (const auto &s : opts_->get_collators_list()->shards) { - if (!shard_intersects(s.shard_id, shard_)) { - continue; - } - if (!s.collators.empty()) { - const CollatorsList::Collator &col = s.collators[td::Random::fast(0, s.collators.size() - 1)]; - collator_adnl_id = col.adnl_id; - trusted_collator = col.trusted; - break; - } - } - if (collator_adnl_id.is_zero()) { - if (opts_->get_collators_list()->self_collate) { - self_collate = true; - } else if (opts_->get_collators_list()->use_config_41) { - // TODO: some way to choose node (similar to "unreliability" in full-node) - int cnt = 0; - for (const block::CollatorNodeDescr &c : collator_config_.collator_nodes) { - if (shard_intersects(shard_, c.shard)) { - if (td::Random::fast(0, cnt) == 0) { - collator_adnl_id = adnl::AdnlNodeIdShort(c.adnl_id); - } - ++cnt; - } - } - } - } - } - if (self_collate) { - run_collate_query(shard_, min_masterchain_block_id_, prev_block_ids_, - Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, validator_set_, - opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), std::move(promise), - cancellation_token_source_.get_cancellation_token(), 0); - return; - } - if (collator_adnl_id.is_zero()) { - promise.set_error(td::Status::Error(PSTRING() << "no collator for shard " << shard_.to_str())); - return; - } - - promise = td::PromiseCreator::lambda([=, SelfId = actor_id(this), promise = std::move(promise), - timer = td::Timer()](td::Result R) mutable { - if (R.is_ok()) { - LOG(INFO) << "collate query for " << next_block_id.to_str() << ": success, time=" << timer.elapsed() << "s"; - promise.set_result(R.move_as_ok()); - return; - } - bool retry = (!timeout || !timeout.is_in_past()) && max_retries > 0; - LOG(WARNING) << "collate query for " << next_block_id.to_str() << ": " << R.error() << ", time=" << timer.elapsed() - << "s, " << (retry ? "retrying" : "giving up"); - if (retry) { - td::actor::send_closure(SelfId, &ValidatorGroup::collate_block, source_info, timeout, std::move(promise), - max_retries - 1); - } else { - promise.set_result(td::Status::Error(ErrorCode::timeout, "timeout")); - } - }); - - std::vector> prev_blocks; - for (const BlockIdExt &p : prev_block_ids_) { - prev_blocks.push_back(create_tl_block_id(p)); - } - td::BufferSlice query = create_serialize_tl_object( - create_tl_shard_id(shard_), validator_set_->get_catchain_seqno(), std::move(prev_blocks), - local_id_full_.ed25519_value().raw()); - - auto P = td::PromiseCreator::lambda( - [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error_prefix("rldp query failed: ")); - return; - } - td::actor::send_closure(SelfId, &ValidatorGroup::receive_collate_query_response, source_info, R.move_as_ok(), - trusted_collator, std::move(promise)); - }); - LOG(INFO) << "sending collate query for " << next_block_id.to_str() << ": send to " << collator_adnl_id; - size_t max_answer_size = config_.max_block_size + config_.max_collated_data_size + 1024; - td::Timestamp query_timeout = td::Timestamp::in(10.0); - query_timeout.relax(timeout); - td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_adnl_id_, collator_adnl_id, "collatequery", - std::move(P), timeout, std::move(query), max_answer_size); -} - -void ValidatorGroup::receive_collate_query_response(validatorsession::BlockSourceInfo source_info, td::BufferSlice data, - bool trusted_collator, td::Promise promise) { - if (source_info.round < last_known_round_id_) { - promise.set_error(td::Status::Error("too old")); - return; - } - TRY_RESULT_PROMISE(promise, f, fetch_tl_object(data, true)); - td::Result res; - ton_api::downcast_call(*f, td::overloaded( - [&](ton_api::collatorNode_generateBlockError &r) { - td::Status error = td::Status::Error(r.code_, r.message_); - res = error.move_as_error_prefix("collate query: "); - }, - [&](ton_api::collatorNode_generateBlockSuccess &r) { - res = CollatorNode::deserialize_candidate( - std::move(r.candidate_), - config_.max_block_size + config_.max_collated_data_size + 1024); - })); - TRY_RESULT_PROMISE(promise, candidate, std::move(res)); - if (candidate.pubkey.as_bits256() != local_id_full_.ed25519_value().raw()) { - promise.set_error(td::Status::Error("collate query: block candidate source mismatch")); - return; - } - if (candidate.id.shard_full() != shard_) { - promise.set_error(td::Status::Error("collate query: shard mismatch")); - return; - } - - if (trusted_collator) { - promise.set_result(std::move(candidate)); - return; - } - auto P = td::PromiseCreator::lambda( - [candidate = candidate.clone(), promise = std::move(promise)](td::Result> R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error_prefix("validate received block error: ")); - return; - } - promise.set_result(std::move(candidate)); - }); - validate_block_candidate(source_info, std::move(candidate), std::move(P)); -} - } // namespace validator } // namespace ton diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 38fc9fb44..fb7a4dcfb 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -18,6 +18,7 @@ */ #pragma once +#include "collation-manager.hpp" #include "interfaces/validator-manager.h" #include "validator-session/validator-session.h" @@ -59,6 +60,10 @@ class ValidatorGroup : public td::actor::Actor { init_ = false; create_session(); } + td::actor::send_closure(collation_manager_, &CollationManager::validator_group_started, shard_); + } + void tear_down() override { + td::actor::send_closure(collation_manager_, &CollationManager::validator_group_finished, shard_); } void get_validator_group_info_for_litequery( @@ -71,17 +76,17 @@ class ValidatorGroup : public td::actor::Actor { ValidatorGroup(ShardIdFull shard, PublicKeyHash local_id, ValidatorSessionId session_id, td::Ref validator_set, BlockSeqno last_key_block_seqno, - block::CollatorConfig collator_config, validatorsession::ValidatorSessionOptions config, - td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays, - std::string db_root, td::actor::ActorId validator_manager, bool create_session, + validatorsession::ValidatorSessionOptions config, td::actor::ActorId keyring, + td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId overlays, std::string db_root, + td::actor::ActorId validator_manager, + td::actor::ActorId collation_manager, bool create_session, bool allow_unsafe_self_blocks_resync, td::Ref opts, bool monitoring_shard) : shard_(shard) , local_id_(std::move(local_id)) , session_id_(session_id) , validator_set_(std::move(validator_set)) , last_key_block_seqno_(last_key_block_seqno) - , collator_config_(std::move(collator_config)) , config_(std::move(config)) , keyring_(keyring) , adnl_(adnl) @@ -89,6 +94,7 @@ class ValidatorGroup : public td::actor::Actor { , overlays_(overlays) , db_root_(std::move(db_root)) , manager_(validator_manager) + , collation_manager_(collation_manager) , init_(create_session) , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) , opts_(std::move(opts)) @@ -97,10 +103,6 @@ class ValidatorGroup : public td::actor::Actor { private: std::unique_ptr make_validator_session_callback(); - void collate_block(validatorsession::BlockSourceInfo source_info, td::Timestamp timeout, - td::Promise promise, unsigned max_retries = 4); - void receive_collate_query_response(validatorsession::BlockSourceInfo source_info, td::BufferSlice data, - bool trusted_collator, td::Promise promise); struct PostponedAccept { RootHash root_hash; @@ -124,7 +126,6 @@ class ValidatorGroup : public td::actor::Actor { td::Ref validator_set_; BlockSeqno last_key_block_seqno_; - block::CollatorConfig collator_config_; validatorsession::ValidatorSessionOptions config_; td::actor::ActorId keyring_; @@ -133,6 +134,7 @@ class ValidatorGroup : public td::actor::Actor { td::actor::ActorId overlays_; std::string db_root_; td::actor::ActorId manager_; + td::actor::ActorId collation_manager_; td::actor::ActorOwn session_; adnl::AdnlNodeIdShort local_adnl_id_; diff --git a/validator/validator-options.cpp b/validator/validator-options.cpp index 4f9b8c538..230a5df46 100644 --- a/validator/validator-options.cpp +++ b/validator/validator-options.cpp @@ -26,29 +26,58 @@ namespace ton { namespace validator { -void CollatorsList::unpack(const ton_api::engine_validator_collatorsList& obj) { +td::Status CollatorsList::unpack(const ton_api::engine_validator_collatorsList& obj) { shards.clear(); - self_collate = obj.self_collate_; - use_config_41 = obj.use_config_41_; + self_collate = false; for (const auto& shard_obj : obj.shards_) { + ShardIdFull shard_id = create_shard_id(shard_obj->shard_id_); + if (shard_id.is_masterchain()) { + return td::Status::Error("masterchain shard in collators list"); + } + if (!shard_id.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard " << shard_id.to_str()); + } shards.emplace_back(); Shard& shard = shards.back(); - shard.shard_id = create_shard_id(shard_obj->shard_id_); + shard.shard_id = shard_id; + shard.self_collate = shard_obj->self_collate_; + if (shard.self_collate) { + self_collate = true; + } + if (shard_obj->select_mode_.empty() || shard_obj->select_mode_ == "random") { + shard.select_mode = mode_random; + } else if (shard_obj->select_mode_ == "ordered") { + shard.select_mode = mode_ordered; + } else if (shard_obj->select_mode_ == "round_robin") { + shard.select_mode = mode_round_robin; + } else { + return td::Status::Error(PSTRING() << "invalid select mode '" << shard_obj->select_mode_ + << "' (allowed: 'random', 'ordered', 'round_robin')"); + } for (const auto& collator : shard_obj->collators_) { - shard.collators.push_back({adnl::AdnlNodeIdShort{collator->adnl_id_}, collator->trusted_}); + shard.collators.push_back(adnl::AdnlNodeIdShort{collator->adnl_id_}); } } + return td::Status::OK(); +} + +CollatorsList CollatorsList::default_list() { + CollatorsList list; + list.shards.push_back( + {.shard_id = ShardIdFull{basechainId, shardIdAll}, .select_mode = mode_random, .self_collate = true}); + list.self_collate = true; + return list; } -td::Ref ValidatorManagerOptions::create( - BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard, bool allow_blockchain_init, - double sync_blocks_before, double block_ttl, double state_ttl, double max_mempool_num, - double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) { +td::Ref ValidatorManagerOptions::create(BlockIdExt zero_block_id, BlockIdExt init_block_id, + std::function check_shard, + bool allow_blockchain_init, double sync_blocks_before, + double block_ttl, double state_ttl, + double max_mempool_num, double archive_ttl, + double key_proof_ttl, bool initial_sync_disabled) { return td::make_ref(zero_block_id, init_block_id, std::move(check_shard), allow_blockchain_init, sync_blocks_before, block_ttl, state_ttl, - max_mempool_num, - archive_ttl, key_proof_ttl, initial_sync_disabled); + max_mempool_num, archive_ttl, key_proof_ttl, initial_sync_disabled); } } // namespace validator diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 8e876625d..05cc73d31 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -157,6 +157,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { td::Ref get_collators_list() const override { return collators_list_; } + bool check_collator_node_whitelist(adnl::AdnlNodeIdShort id) const override { + return !collator_node_whitelist_enabled_ || collator_node_whitelist_.contains(id); + } void set_zero_block_id(BlockIdExt block_id) override { zero_block_id_ = block_id; @@ -255,6 +258,16 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_collators_list(td::Ref list) override { collators_list_ = std::move(list); } + void set_collator_node_whitelisted_validator(adnl::AdnlNodeIdShort id, bool add) override { + if (add) { + collator_node_whitelist_.insert(id); + } else { + collator_node_whitelist_.erase(id); + } + } + void set_collator_node_whitelist_enabled(bool enabled) override { + collator_node_whitelist_enabled_ = enabled; + } ValidatorManagerOptionsImpl *make_copy() const override { return new ValidatorManagerOptionsImpl(*this); @@ -308,7 +321,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool state_serializer_enabled_ = true; td::Ref collator_options_{true}; bool fast_state_serializer_enabled_ = false; - td::Ref collators_list_{true, CollatorsList{}}; + td::Ref collators_list_{true, CollatorsList::default_list()}; + std::set collator_node_whitelist_; + bool collator_node_whitelist_enabled_ = false; }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 458076768..3eaaaddf9 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -73,19 +73,20 @@ struct CollatorOptions : public td::CntObject { }; struct CollatorsList : public td::CntObject { - struct Collator { - adnl::AdnlNodeIdShort adnl_id; - bool trusted; + enum SelectMode { + mode_random, mode_ordered, mode_round_robin }; struct Shard { ShardIdFull shard_id; - std::vector collators; + SelectMode select_mode = mode_random; + std::vector collators; + bool self_collate = false; }; - bool self_collate = true; - bool use_config_41 = false; std::vector shards; + bool self_collate = false; - void unpack(const ton_api::engine_validator_collatorsList& obj); + td::Status unpack(const ton_api::engine_validator_collatorsList& obj); + static CollatorsList default_list(); }; struct ValidatorManagerOptions : public td::CntObject { @@ -130,6 +131,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual td::Ref get_collator_options() const = 0; virtual bool get_fast_state_serializer_enabled() const = 0; virtual td::Ref get_collators_list() const = 0; + virtual bool check_collator_node_whitelist(adnl::AdnlNodeIdShort id) const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; @@ -163,6 +165,8 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_collator_options(td::Ref value) = 0; virtual void set_fast_state_serializer_enabled(bool value) = 0; virtual void set_collators_list(td::Ref list) = 0; + virtual void set_collator_node_whitelisted_validator(adnl::AdnlNodeIdShort id, bool add) = 0; + virtual void set_collator_node_whitelist_enabled(bool enabled) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, @@ -314,6 +318,9 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; virtual void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; + + virtual void get_collation_manager_stats( + td::Promise> promise) = 0; }; } // namespace validator From b60f6ee72f41cd5003860482309632c81c80d3ab Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 26 Nov 2024 08:54:19 +0300 Subject: [PATCH 123/388] Write collation stats to session stats, add collator options for collated data --- tl/generate/scheme/ton_api.tl | 13 +++++-- tl/generate/scheme/ton_api.tlo | Bin 107428 -> 108128 bytes validator-engine/validator-engine.cpp | 4 ++ validator/collator-node.cpp | 3 +- validator/fabric.h | 2 +- validator/impl/collator.cpp | 46 +++++++++++++++++++---- validator/impl/validate-query.cpp | 2 +- validator/interfaces/validator-manager.h | 18 +++++++-- validator/manager.cpp | 29 ++++++++++---- validator/manager.hpp | 3 +- validator/validator.h | 5 +++ 11 files changed, 98 insertions(+), 27 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 0839f727d..f7b23d1b8 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -669,7 +669,8 @@ engine.validator.collatorOptions deferring_enabled:Bool defer_messages_after:int defer_out_queue_size_limit:long dispatch_phase_2_max_total:int dispatch_phase_3_max_total:int dispatch_phase_2_max_per_initiator:int dispatch_phase_3_max_per_initiator:int - whitelist:(vector string) prioritylist:(vector string) = engine.validator.CollatorOptions; + whitelist:(vector string) prioritylist:(vector string) + force_full_collated_data:Bool ignore_collated_data_limits:Bool = engine.validator.CollatorOptions; engine.validator.collatorsList.collator adnl_id:int256 = engine.validator.collatorsList.Collator; engine.validator.collatorsList.shard shard_id:tonNode.shardId collators:(vector engine.validator.collatorsList.collator) @@ -874,8 +875,12 @@ http.server.config dhs:(vector http.server.dnsEntry) local_hosts:(vector http.se ---types--- -validatorSession.collationStats bytes:int gas:int lt_delta:int cat_bytes:int cat_gas:int cat_lt_delta:int - limits_log:string ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int = validadorSession.CollationStats; +validatorSession.collationStats actual_bytes:int actual_collated_data_bytes:int + bytes:int gas:int lt_delta:int collated_data_bytes:int + cat_bytes:int cat_gas:int cat_lt_delta:int cat_collated_data_bytes:int + limits_log:string ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int + work_time:double cpu_work_time:double + serialized_size:int serialized_size_no_collated_data:int = validadorSession.CollationStats; validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int root_hash:int256 file_hash:int256 comment:string block_timestamp:double is_accepted:Bool is_ours:Bool got_submit_at:double @@ -896,6 +901,8 @@ validatorSession.stats success:Bool id:tonNode.blockIdExt timestamp:double self: signatures:int signatures_weight:long approve_signatures:int approve_signatures_weight:long first_round:int rounds:(vector validatorSession.statsRound) = validatorSession.Stats; +validatorSession.statsCollatedBlock timestamp:double id:tonNode.blockIdExt stats:validatorSession.collationStats = validatorSession.StatsCollatedBlock; + collatorNode.candidate source:PublicKey id:tonNode.blockIdExt data:bytes collated_data:bytes = collatorNode.Candidate; collatorNode.compressedCandidate flags:# source:PublicKey id:tonNode.blockIdExt decompressed_size:int data:bytes = collatorNode.Candidate; collatorNode.pong flags:# = collatorNode.Pong; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index dd2af2ec9de6dbde7d6644ac55b85bf42bba5df1..2cff142655e991800ee801a06bd7a6e6a29e0657 100644 GIT binary patch delta 536 zcmZ2-obACGHr_|G^{p77z+@xuX)%^Y4Lg?2cf@>@SXTcz^nJ6Ar6jY4@q**ZWr;bN zDTyWdMZu}X#hLkedch@$CB@G9IXQ_XsVPo5`N`RglMgb9Z+5ZS!N_=U^IzN3IjkVV z6efQ>At~%%Py*Cc473U)cJr8!5BKJutDZAXHrUQ0DUp_6l$;u$R+^I&pA0oI9%xzO zWXBWIGSZppdHF@Ds50?6nYo!I#ghxp$v~X^W{p7;*vZos^cfXoL5>I6A`G-q`q_O= zp2Xym(!`wjq{@=i;>nJyWh9^iDE5J*rwh6>N>2~aXG{Jd88Dujyup=~vpl~jJH8|{H+3>!x)dL8azSZ4Txj||A4a>$0*)+- z3dN~KnZO{fN==C`29oi4`KU%uW(<-C`)~Uf1IC+TETA}GlKej8m8(=0 delta 183 zcmaEGhHc4lHr_|G^{p77z<49?X)%@;Q(jNld`HYjiRGQ8e&%K!OG)O zwL6`|3X)ftu9(RvDePZRl9``Z4Ac%1ym?H>hjX*#>gSA;AMD|Pm@{RqK@%&;9Qo-R z?HGlp-_d8};RNwPin*uz8Zf3$e_+6vGrh);@yz58_8eeKx3d^C-W1#3>BZR21QwWl NVY|q74PQo!NB~(wMzsI{ diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 8d2ce0bef..cf3ca3ff5 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -2707,6 +2707,8 @@ static td::Result> parse_collator_optio f.dispatch_phase_2_max_per_initiator_ = opts.dispatch_phase_2_max_per_initiator; f.dispatch_phase_3_max_per_initiator_ = opts.dispatch_phase_3_max_per_initiator ? opts.dispatch_phase_3_max_per_initiator.value() : -1; + f.force_full_collated_data_ = false; + f.ignore_collated_data_limits_ = false; TRY_RESULT_PREFIX(json, td::json_decode(json_str), "failed to parse json: "); TRY_STATUS_PREFIX(ton::ton_api::from_json(f, json.get_object()), "json does not fit TL scheme: "); @@ -2746,6 +2748,8 @@ static td::Result> parse_collator_optio TRY_RESULT(addr, block::StdAddress::parse(s)); opts.prioritylist.emplace(addr.workchain, addr.addr); } + opts.force_full_collated_data = f.force_full_collated_data_; + opts.ignore_collated_data_limits = f.ignore_collated_data_limits_; return ref; } diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 776fbc9e9..cf2770d93 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -431,7 +431,8 @@ void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std << ", time=" << timer.elapsed() << ": " << (R.is_ok() ? "OK" : R.error().to_string()); td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R)); }, - cache_entry->cancellation_token_source.get_cancellation_token(), CollateMode::skip_store_candidate); + cache_entry->cancellation_token_source.get_cancellation_token(), + CollateMode::skip_store_candidate | CollateMode::from_collator_node); } void CollatorNode::process_result(std::shared_ptr cache_entry, td::Result R) { diff --git a/validator/fabric.h b/validator/fabric.h index 629065176..20358822e 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -27,7 +27,7 @@ namespace ton { namespace validator { enum ValidateMode { fake = 1 }; -enum CollateMode { skip_store_candidate = 1 }; +enum CollateMode { skip_store_candidate = 1, from_collator_node = 2 }; td::actor::ActorOwn create_db_actor(td::actor::ActorId manager, std::string db_root_, td::Ref opts); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 677b73ae0..0054ea98a 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -16,6 +16,7 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "candidate-serializer.h" #include "collator-impl.h" #include "vm/boc.h" #include "td/db/utils/BlobView.h" @@ -77,7 +78,7 @@ static inline bool dbg(int c) { * @param timeout The timeout for the collator. * @param promise The promise to return the result. * @param cancellation_token Token to cancel collation. - * @param mode +1 - skip storing candidate to disk. + * @param mode +1 - skip storing candidate to disk, +2 - called from CollatorNode. * @param attempt_idx The index of the attempt, starting from 0. On later attempts collator decreases block limits and skips some steps. */ Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, @@ -730,7 +731,7 @@ bool Collator::unpack_last_mc_state() { store_out_msg_queue_size_ = config_->has_capability(ton::capStoreOutMsgQueueSize); msg_metadata_enabled_ = config_->has_capability(ton::capMsgMetadata); deferring_messages_enabled_ = config_->has_capability(ton::capDeferMessages); - full_collated_data_ = config_->has_capability(capFullCollatedData); + full_collated_data_ = config_->has_capability(capFullCollatedData) || collator_opts_->force_full_collated_data; LOG(DEBUG) << "full_collated_data is " << full_collated_data_; shard_conf_ = std::make_unique(*config_); prev_key_block_exists_ = config_->get_last_key_block(prev_key_block_, prev_key_block_lt_); @@ -751,15 +752,24 @@ bool Collator::unpack_last_mc_state() { LOG(INFO) << "Attempt #3: bytes, gas limits /= 2"; block_limits_->bytes.multiply_by(0.5); block_limits_->gas.multiply_by(0.5); + block_limits_->collated_data.multiply_by(0.5); } else if (attempt_idx_ == 4) { LOG(INFO) << "Attempt #4: bytes, gas limits /= 4"; block_limits_->bytes.multiply_by(0.25); block_limits_->gas.multiply_by(0.25); + block_limits_->collated_data.multiply_by(0.25); + } + if (collator_opts_->ignore_collated_data_limits) { + block_limits_->collated_data = block::ParamLimits{1 << 30, 1 << 30, 1 << 30}; } LOG(DEBUG) << "block limits: bytes [" << block_limits_->bytes.underload() << ", " << block_limits_->bytes.soft() << ", " << block_limits_->bytes.hard() << "]"; LOG(DEBUG) << "block limits: gas [" << block_limits_->gas.underload() << ", " << block_limits_->gas.soft() << ", " << block_limits_->gas.hard() << "]"; + LOG(DEBUG) << "block limits: lt_delta [" << block_limits_->lt_delta.underload() << ", " + << block_limits_->lt_delta.soft() << ", " << block_limits_->lt_delta.hard() << "]"; + LOG(DEBUG) << "block limits: collated_data_bytes [" << block_limits_->collated_data.underload() << ", " + << block_limits_->collated_data.soft() << ", " << block_limits_->collated_data.hard() << "]"; if (config_->has_capabilities() && (config_->get_capabilities() & ~supported_capabilities())) { LOG(ERROR) << "block generation capabilities " << config_->get_capabilities() << " have been enabled in global configuration, but we support only " << supported_capabilities() @@ -3678,6 +3688,10 @@ static std::string block_full_comment(const block::BlockLimitStatus& block_limit if (!block_limit_status.limits.lt_delta.fits(cls, lt_delta)) { return PSTRING() << "block_full lt_delta " << lt_delta; } + auto collated_data_bytes = block_limit_status.collated_data_stat.estimate_proof_size(); + if (!block_limit_status.limits.collated_data.fits(cls, collated_data_bytes)) { + return PSTRING() << "block_full collated_data " << collated_data_bytes; + } return ""; } @@ -5810,7 +5824,8 @@ bool Collator::create_block_candidate() { << ") exceeds the limit in consensus config (" << consensus_config.max_block_size << ")"); } - if (block_candidate->collated_data.size() > consensus_config.max_collated_data_size) { + if (block_candidate->collated_data.size() > consensus_config.max_collated_data_size && + !collator_opts_->ignore_collated_data_limits) { return fatal_error(PSTRING() << "collated data size (" << block_candidate->collated_data.size() << ") exceeds the limit in consensus config (" << consensus_config.max_collated_data_size << ")"); @@ -5838,14 +5853,31 @@ bool Collator::create_block_candidate() { double work_time = work_timer_.elapsed(); double cpu_work_time = cpu_work_timer_.elapsed(); LOG(WARNING) << "Collate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s"; - stats_.bytes = block_limit_status_->estimate_block_size(); + stats_.actual_bytes = block_candidate->data.size(); + stats_.actual_collated_data_bytes = block_candidate->collated_data.size(); + stats_.estimated_bytes = block_limit_status_->estimate_block_size(); stats_.gas = block_limit_status_->gas_used; stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt; - stats_.cat_bytes = block_limit_status_->limits.classify_size(stats_.bytes); + stats_.estimated_collated_data_bytes = block_limit_status_->collated_data_stat.estimate_proof_size(); + stats_.cat_bytes = block_limit_status_->limits.classify_size(stats_.estimated_bytes); stats_.cat_gas = block_limit_status_->limits.classify_gas(stats_.gas); stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt); - td::actor::send_closure(manager, &ValidatorManager::record_collate_query_stats, block_candidate->id, work_time, - cpu_work_time, std::move(stats_)); + stats_.cat_collated_data_bytes = + block_limit_status_->limits.classify_collated_data_size(stats_.estimated_collated_data_bytes); + stats_.work_time = work_time; + stats_.cpu_work_time = cpu_work_time; + + // TODO: remove this later (currently needed to collect stats) + if (mode_ & CollateMode::from_collator_node) { + size_t d; + stats_.serialized_size = + validatorsession::compress_candidate_data(block_candidate->data, block_candidate->collated_data, d).ok().size(); + stats_.serialized_size_no_collated_data = + validatorsession::compress_candidate_data(block_candidate->data, td::Slice{}, d).ok().size(); + } + + td::actor::send_closure(manager, &ValidatorManager::record_collate_query_stats, block_candidate->id, + std::move(stats_)); return true; } diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 4c6264025..34c164eee 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -65,7 +65,7 @@ std::string ErrorCtx::as_string() const { * @param manager The ActorId of the ValidatorManager. * @param timeout The timeout for the validation. * @param promise The Promise to return the ValidateCandidateResult to. - * @param is_fake A boolean indicating if the validation is fake (performed when creating a hardfork). + * @param mode +1 - fake mode */ ValidateQuery::ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, BlockCandidate candidate, Ref validator_set, diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index a728c93b9..b52aab76d 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -54,13 +54,24 @@ struct AsyncSerializerState { }; struct CollationStats { - td::uint32 bytes, gas, lt_delta; - int cat_bytes, cat_gas, cat_lt_delta; + td::uint32 actual_bytes = 0, actual_collated_data_bytes = 0; + td::uint32 estimated_bytes = 0, gas = 0, lt_delta = 0, estimated_collated_data_bytes = 0; + int cat_bytes = 0, cat_gas = 0, cat_lt_delta = 0, cat_collated_data_bytes = 0; std::string limits_log; td::uint32 ext_msgs_total = 0; td::uint32 ext_msgs_filtered = 0; td::uint32 ext_msgs_accepted = 0; td::uint32 ext_msgs_rejected = 0; + double work_time = 0.0, cpu_work_time = 0.0; + td::uint32 serialized_size = 0, serialized_size_no_collated_data = 0; + + tl_object_ptr tl() const { + return create_tl_object( + actual_bytes, actual_collated_data_bytes, estimated_bytes, gas, lt_delta, estimated_collated_data_bytes, + cat_bytes, cat_gas, cat_lt_delta, cat_collated_data_bytes, limits_log, ext_msgs_total, ext_msgs_filtered, + ext_msgs_accepted, ext_msgs_rejected, work_time, cpu_work_time, serialized_size, + serialized_size_no_collated_data); + } }; using ValidateCandidateResult = td::Variant; @@ -208,8 +219,7 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void add_lite_query_stats(int lite_query_id) { } - virtual void record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, - CollationStats stats) { + virtual void record_collate_query_stats(BlockIdExt block_id, CollationStats stats) { } virtual void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) { } diff --git a/validator/manager.cpp b/validator/manager.cpp index 3f59d784d..caaa3a796 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -3099,10 +3099,7 @@ void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id, tl_object_ptr collation_stats; if (it != recorded_block_stats_.end() && it->second.collator_stats_) { auto &stats = it->second.collator_stats_.value(); - collation_stats = create_tl_object( - stats.bytes, stats.gas, stats.lt_delta, stats.cat_bytes, stats.cat_gas, stats.cat_lt_delta, - stats.limits_log, stats.ext_msgs_total, stats.ext_msgs_filtered, stats.ext_msgs_accepted, - stats.ext_msgs_rejected); + collation_stats = stats.tl(); } std::string approvers, signers; for (bool x : producer.approvers) { @@ -3616,12 +3613,28 @@ td::actor::ActorOwn ValidatorManagerFactory::create( rldp, overlays); } -void ValidatorManagerImpl::record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, - CollationStats stats) { +void ValidatorManagerImpl::record_collate_query_stats(BlockIdExt block_id, CollationStats stats) { auto &record = new_block_stats_record(block_id); - record.collator_work_time_ = work_time; - record.collator_cpu_work_time_ = cpu_work_time; + record.collator_work_time_ = stats.work_time; + record.collator_cpu_work_time_ = stats.cpu_work_time; record.collator_stats_ = std::move(stats); + + std::string fname = opts_->get_session_logs_file(); + if (fname.empty()) { + return; + } + + auto obj = create_tl_object(td::Clocks::system(), + create_tl_block_id(block_id), stats.tl()); + auto s = td::json_encode(td::ToJson(*obj.get()), false); + s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); + + std::ofstream file; + file.open(fname, std::ios_base::app); + file << s << "\n"; + file.close(); + + LOG(DEBUG) << "Writing collation stats stats for " << block_id.id.to_str(); } void ValidatorManagerImpl::record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 055adf304..e123f27e9 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -780,8 +780,7 @@ class ValidatorManagerImpl : public ValidatorManager { std::map recorded_block_stats_; std::queue recorded_block_stats_lru_; - void record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, - CollationStats stats) override; + void record_collate_query_stats(BlockIdExt block_id, CollationStats stats) override; void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) override; RecordedBlockStats &new_block_stats_record(BlockIdExt block_id); diff --git a/validator/validator.h b/validator/validator.h index 3eaaaddf9..393dc69f4 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -70,6 +70,11 @@ struct CollatorOptions : public td::CntObject { std::set> whitelist; // Prioritize these accounts on each phase of process_dispatch_queue std::set> prioritylist; + + // Always enable full_collated_data + bool force_full_collated_data = false; + // Ignore collated data size limits from block limits and catchain config + bool ignore_collated_data_limits = false; }; struct CollatorsList : public td::CntObject { From d9aeab07db471d314b6f9a5ce7641daeb91fd4eb Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 26 Nov 2024 10:53:55 +0300 Subject: [PATCH 124/388] Send telemetry broadcasts to fast sync overlays --- validator-engine/validator-engine.cpp | 2 +- validator/full-node-fast-sync-overlays.cpp | 79 ++++++++++++++++++++-- validator/full-node-fast-sync-overlays.hpp | 13 +++- validator/full-node.cpp | 68 ++++++++++++++----- validator/full-node.hpp | 2 +- validator/impl/validator-set.hpp | 1 - validator/interfaces/validator-set.h | 1 - validator/manager.cpp | 2 +- 8 files changed, 141 insertions(+), 27 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 6b91d3ebc..16aabde5f 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -4985,7 +4985,7 @@ int main(int argc, char *argv[]) { }); p.add_option( '\0', "collect-validator-telemetry", - "store validator telemetry from private block overlay to a given file (json format)", + "store validator telemetry from fast sync overlay to a given file (json format)", [&](td::Slice s) { acts.push_back( [&x, s = s.str()]() { diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index a5789771b..270c53f07 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -87,9 +87,42 @@ void FullNodeFastSyncOverlay::process_block_candidate_broadcast(PublicKeyHash sr validator_set_hash, std::move(data)); } +void FullNodeFastSyncOverlay::process_telemetry_broadcast( + adnl::AdnlNodeIdShort src, const tl_object_ptr &telemetry) { + if (telemetry->adnl_id_ != src.bits256_value()) { + VLOG(FULL_NODE_WARNING) << "Invalid telemetry broadcast from " << src << ": adnl_id mismatch"; + return; + } + auto now = (td::int32)td::Clocks::system(); + if (telemetry->timestamp_ < now - 60) { + VLOG(FULL_NODE_WARNING) << "Invalid telemetry broadcast from " << src << ": too old (" + << now - telemetry->timestamp_ << "s ago)"; + return; + } + if (telemetry->timestamp_ > now + 60) { + VLOG(FULL_NODE_WARNING) << "Invalid telemetry broadcast from " << src << ": too new (" + << telemetry->timestamp_ - now << "s in the future)"; + return; + } + VLOG(FULL_NODE_DEBUG) << "Got telemetry broadcast from " << src; + auto s = td::json_encode(td::ToJson(*telemetry), false); + std::erase_if(s, [](char c) { return c == '\n' || c == '\r'; }); + telemetry_file_ << s << "\n"; + telemetry_file_.flush(); + if (telemetry_file_.fail()) { + VLOG(FULL_NODE_WARNING) << "Failed to write telemetry to file"; + } +} + void FullNodeFastSyncOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { auto B = fetch_tl_object(std::move(broadcast), true); if (B.is_error()) { + if (collect_telemetry_ && src != local_id_.pubkey_hash()) { + auto R = fetch_tl_prefix(broadcast, true); + if (R.is_ok()) { + process_telemetry_broadcast(adnl::AdnlNodeIdShort{src}, R.ok()); + } + } return; } @@ -143,6 +176,30 @@ void FullNodeFastSyncOverlay::send_block_candidate(BlockIdExt block_id, Catchain local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); } +void FullNodeFastSyncOverlay::send_validator_telemetry(tl_object_ptr telemetry) { + process_telemetry_broadcast(local_id_, telemetry); + auto data = serialize_tl_object(telemetry, true); + if (data.size() <= overlay::Overlays::max_simple_broadcast_size()) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), 0, std::move(data)); + } else { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), 0, std::move(data)); + } +} + +void FullNodeFastSyncOverlay::collect_validator_telemetry(std::string filename) { + if (collect_telemetry_) { + telemetry_file_.close(); + } + collect_telemetry_ = true; + LOG(FULL_NODE_WARNING) << "Collecting validator telemetry to " << filename << " (local id: " << local_id_ << ")"; + telemetry_file_.open(filename, std::ios_base::app); + if (!telemetry_file_.is_open()) { + LOG(WARNING) << "Cannot open file " << filename << " for validator telemetry"; + } +} + void FullNodeFastSyncOverlay::start_up() { auto X = create_hash_tl_object(zero_state_file_hash_, create_tl_shard_id(shard_)); td::BufferSlice b{32}; @@ -261,14 +318,15 @@ void FullNodeFastSyncOverlay::get_stats_extra(td::Promise promise) promise.set_result(td::json_encode(td::ToJson(*res), true)); } -td::actor::ActorId FullNodeFastSyncOverlays::choose_overlay(ShardIdFull shard) { +std::pair, adnl::AdnlNodeIdShort> FullNodeFastSyncOverlays::choose_overlay( + ShardIdFull shard) { for (auto &p : id_to_overlays_) { auto &overlays = p.second.overlays_; ShardIdFull cur_shard = shard; while (true) { auto it = overlays.find(cur_shard); if (it != overlays.end()) { - return it->second.get(); + return {it->second.get(), p.first}; } if (cur_shard.pfx_len() == 0) { break; @@ -276,7 +334,20 @@ td::actor::ActorId FullNodeFastSyncOverlays::choose_ove cur_shard = shard_parent(cur_shard); } } - return {}; + return {td::actor::ActorId{}, adnl::AdnlNodeIdShort::zero()}; +} + +td::actor::ActorId FullNodeFastSyncOverlays::get_masterchain_overlay_for( + adnl::AdnlNodeIdShort adnl_id) { + auto it = id_to_overlays_.find(adnl_id); + if (it == id_to_overlays_.end()) { + return {}; + } + auto it2 = it->second.overlays_.find(ShardIdFull{masterchainId}); + if (it2 == it->second.overlays_.end()) { + return {}; + } + return it2->second.get(); } void FullNodeFastSyncOverlays::update_overlays(td::Ref state, @@ -291,7 +362,7 @@ void FullNodeFastSyncOverlays::update_overlays(td::Ref state, monitoring_shards.insert(ShardIdFull{masterchainId}); std::set all_shards; all_shards.insert(ShardIdFull{masterchainId}); - for (const auto& desc : state->get_shards()) { + for (const auto &desc : state->get_shards()) { ShardIdFull shard = desc->shard(); td::uint32 monitor_min_split = state->monitor_min_split_depth(shard.workchain); if (shard.pfx_len() > monitor_min_split) { diff --git a/validator/full-node-fast-sync-overlays.hpp b/validator/full-node-fast-sync-overlays.hpp index b89f88024..05d83071d 100644 --- a/validator/full-node-fast-sync-overlays.hpp +++ b/validator/full-node-fast-sync-overlays.hpp @@ -17,6 +17,7 @@ #pragma once #include "full-node.h" +#include namespace ton::validator::fullnode { @@ -32,6 +33,9 @@ class FullNodeFastSyncOverlay : public td::actor::Actor { void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed& query); void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast& query); + void process_telemetry_broadcast(adnl::AdnlNodeIdShort src, + const tl_object_ptr& telemetry); + template void process_broadcast(PublicKeyHash, T&) { VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast"; @@ -42,6 +46,9 @@ class FullNodeFastSyncOverlay : public td::actor::Actor { void send_broadcast(BlockBroadcast broadcast); void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::BufferSlice data); + void send_validator_telemetry(tl_object_ptr telemetry); + + void collect_validator_telemetry(std::string filename); void start_up() override; void tear_down() override; @@ -96,11 +103,15 @@ class FullNodeFastSyncOverlay : public td::actor::Actor { void try_init(); void init(); void get_stats_extra(td::Promise promise); + + bool collect_telemetry_ = false; + std::ofstream telemetry_file_; }; class FullNodeFastSyncOverlays { public: - td::actor::ActorId choose_overlay(ShardIdFull shard); + std::pair, adnl::AdnlNodeIdShort> choose_overlay(ShardIdFull shard); + td::actor::ActorId get_masterchain_overlay_for(adnl::AdnlNodeIdShort adnl_id); void update_overlays(td::Ref state, std::set my_adnl_ids, std::set monitoring_shards, const FileHash& zero_state_file_hash, const td::actor::ActorId& keyring, const td::actor::ActorId& adnl, diff --git a/validator/full-node.cpp b/validator/full-node.cpp index ed3b5bcb9..d927eef2a 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -279,6 +279,7 @@ void FullNodeImpl::on_new_masterchain_block(td::Ref state, std fast_sync_overlays_.update_overlays(state, std::move(my_adnl_ids), std::move(monitoring_shards), zero_state_file_hash_, keyring_, adnl_, overlays_, validator_manager_, actor_id(this)); + update_validator_telemetry_collector(); } } @@ -338,7 +339,7 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_shard_block_info, block_id, cc_seqno, data.clone()); } - auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(ShardIdFull(masterchainId)); + auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(ShardIdFull(masterchainId)).first; if (!fast_sync_overlay.empty()) { td::actor::send_closure(fast_sync_overlay, &FullNodeFastSyncOverlay::send_shard_block_info, block_id, cc_seqno, data.clone()); @@ -358,7 +359,7 @@ void FullNodeImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_se td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_block_candidate, block_id, cc_seqno, validator_set_hash, data.clone()); } - auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(block_id.shard_full()); + auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(block_id.shard_full()).first; if (!fast_sync_overlay.empty()) { td::actor::send_closure(fast_sync_overlay, &FullNodeFastSyncOverlay::send_block_candidate, block_id, cc_seqno, validator_set_hash, data.clone()); @@ -383,7 +384,7 @@ void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, int mode) { td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_broadcast, broadcast.clone()); } - auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(broadcast.block_id.shard_full()); + auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(broadcast.block_id.shard_full()).first; if (!fast_sync_overlay.empty()) { td::actor::send_closure(fast_sync_overlay, &FullNodeFastSyncOverlay::send_broadcast, broadcast.clone()); } @@ -595,12 +596,21 @@ void FullNodeImpl::new_key_block(BlockHandle handle) { } void FullNodeImpl::send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) { - auto it = private_block_overlays_.find(key); - if (it == private_block_overlays_.end()) { - VLOG(FULL_NODE_INFO) << "Cannot send validator telemetry for " << key << " : no private block overlay"; - return; + if (use_old_private_overlays_) { + auto it = private_block_overlays_.find(key); + if (it == private_block_overlays_.end()) { + VLOG(FULL_NODE_INFO) << "Cannot send validator telemetry for " << key << " : no private block overlay"; + return; + } + td::actor::send_closure(it->second, &FullNodePrivateBlockOverlay::send_validator_telemetry, std::move(telemetry)); + } else { + auto overlay = fast_sync_overlays_.get_masterchain_overlay_for(adnl::AdnlNodeIdShort{telemetry->adnl_id_}); + if (overlay.empty()) { + VLOG(FULL_NODE_INFO) << "Cannot send validator telemetry for adnl id " << key << " : no fast sync overlay"; + return; + } + td::actor::send_closure(overlay, &FullNodeFastSyncOverlay::send_validator_telemetry, std::move(telemetry)); } - td::actor::send_closure(it->second, &FullNodePrivateBlockOverlay::send_validator_telemetry, std::move(telemetry)); } void FullNodeImpl::process_block_broadcast(BlockBroadcast broadcast) { @@ -631,19 +641,43 @@ void FullNodeImpl::set_validator_telemetry_filename(std::string value) { } void FullNodeImpl::update_validator_telemetry_collector() { - if (validator_telemetry_filename_.empty() || private_block_overlays_.empty()) { - validator_telemetry_collector_key_ = PublicKeyHash::zero(); - return; - } - if (!private_block_overlays_.contains(validator_telemetry_collector_key_)) { - auto it = private_block_overlays_.begin(); - validator_telemetry_collector_key_ = it->first; - td::actor::send_closure(it->second, &FullNodePrivateBlockOverlay::collect_validator_telemetry, - validator_telemetry_filename_); + if (use_old_private_overlays_) { + if (validator_telemetry_filename_.empty() || private_block_overlays_.empty()) { + validator_telemetry_collector_key_ = PublicKeyHash::zero(); + return; + } + if (!private_block_overlays_.contains(validator_telemetry_collector_key_)) { + auto it = private_block_overlays_.begin(); + validator_telemetry_collector_key_ = it->first; + td::actor::send_closure(it->second, &FullNodePrivateBlockOverlay::collect_validator_telemetry, + validator_telemetry_filename_); + } + } else { + if (validator_telemetry_filename_.empty()) { + validator_telemetry_collector_key_ = PublicKeyHash::zero(); + return; + } + if (fast_sync_overlays_.get_masterchain_overlay_for(adnl::AdnlNodeIdShort{validator_telemetry_collector_key_}) + .empty()) { + auto [actor, adnl_id] = fast_sync_overlays_.choose_overlay(ShardIdFull{masterchainId}); + validator_telemetry_collector_key_ = adnl_id.pubkey_hash(); + if (!actor.empty()) { + td::actor::send_closure(actor, &FullNodeFastSyncOverlay::collect_validator_telemetry, + validator_telemetry_filename_); + } + } } } void FullNodeImpl::start_up() { + // TODO: enable fast sync overlays by other means (e.g. some config param) + // TODO: in the future - remove the old private overlay entirely + // This env var is for testing + auto fast_sync_env = getenv("TON_FAST_SYNC_OVERLAYS"); + if (fast_sync_env && !strcmp(fast_sync_env, "1")) { + use_old_private_overlays_ = false; + } + update_shard_actor(ShardIdFull{masterchainId}, true); if (local_id_.is_zero()) { if (adnl_id_.is_zero()) { diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 4194407d6..9e254d7d4 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -154,7 +154,7 @@ class FullNodeImpl : public FullNode { // Old overlays - one private overlay for all validators // New overlays (fast sync overlays) - semiprivate overlay per shard (monitor_min_split depth) // for validators and authorized nodes - bool use_old_private_overlays_ = false; // TODO: set from config or something + bool use_old_private_overlays_ = true; std::map> private_block_overlays_; bool broadcast_block_candidates_in_public_overlay_ = false; FullNodeFastSyncOverlays fast_sync_overlays_; diff --git a/validator/impl/validator-set.hpp b/validator/impl/validator-set.hpp index 67fd9cd20..951ca4b71 100644 --- a/validator/impl/validator-set.hpp +++ b/validator/impl/validator-set.hpp @@ -51,7 +51,6 @@ class ValidatorSetQ : public ValidatorSet { td::Ref signatures) const override; td::Result check_approve_signatures(RootHash root_hash, FileHash file_hash, td::Ref signatures) const override; - const ValidatorDescr* find_validator(const NodeIdShort& id) const override; ValidatorSetQ* make_copy() const override; diff --git a/validator/interfaces/validator-set.h b/validator/interfaces/validator-set.h index d690937e0..ad7fb9b55 100644 --- a/validator/interfaces/validator-set.h +++ b/validator/interfaces/validator-set.h @@ -36,7 +36,6 @@ class ValidatorSet : public td::CntObject { virtual td::uint32 get_validator_set_hash() const = 0; virtual ShardId get_validator_set_from() const = 0; virtual std::vector export_vector() const = 0; - virtual const ValidatorDescr* find_validator(const NodeIdShort& id) const = 0; virtual td::Result check_signatures(RootHash root_hash, FileHash file_hash, td::Ref signatures) const = 0; virtual td::Result check_approve_signatures(RootHash root_hash, FileHash file_hash, diff --git a/validator/manager.cpp b/validator/manager.cpp index c928090f4..4c8e6ecc8 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2475,7 +2475,7 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group auto validator_id = get_validator(shard, validator_set); CHECK(!validator_id.is_zero()); - auto descr = validator_set->find_validator(validator_id.bits256_value()); + auto descr = validator_set->get_validator(validator_id.bits256_value()); CHECK(descr); auto adnl_id = adnl::AdnlNodeIdShort{ descr->addr.is_zero() ? ValidatorFullId{descr->key}.compute_short_id().bits256_value() : descr->addr}; From bf572f9599f92841581de08092db46eb911651e7 Mon Sep 17 00:00:00 2001 From: birydrad <> Date: Tue, 26 Nov 2024 14:01:20 +0400 Subject: [PATCH 125/388] optimistic out-msg-queue broadcast --- tdutils/td/utils/StringBuilder.h | 15 ++ tdutils/td/utils/Timer.cpp | 39 +++- tdutils/td/utils/Timer.h | 43 ++++- tdutils/td/utils/logging.h | 10 +- tl/generate/scheme/ton_api.tl | 6 +- tl/generate/scheme/ton_api.tlo | Bin 108568 -> 108888 bytes ton/ton-types.h | 49 ++++- validator/collation-manager.cpp | 11 +- validator/collation-manager.hpp | 6 +- validator/collator-node.cpp | 119 ++++++++++-- validator/collator-node.hpp | 7 +- validator/full-node-fast-sync-overlays.cpp | 43 +++++ validator/full-node-fast-sync-overlays.hpp | 2 + validator/full-node-shard.hpp | 3 + validator/full-node.cpp | 11 ++ validator/full-node.hpp | 1 + validator/impl/collator-impl.h | 15 +- validator/impl/collator.cpp | 205 ++++++++++++++------- validator/impl/out-msg-queue-proof.cpp | 113 ++++++++++-- validator/impl/out-msg-queue-proof.hpp | 18 +- validator/manager.cpp | 17 +- validator/manager.hpp | 1 + validator/validator-group.cpp | 8 +- validator/validator.h | 7 + 24 files changed, 623 insertions(+), 126 deletions(-) diff --git a/tdutils/td/utils/StringBuilder.h b/tdutils/td/utils/StringBuilder.h index 99e9d5172..685416fe3 100644 --- a/tdutils/td/utils/StringBuilder.h +++ b/tdutils/td/utils/StringBuilder.h @@ -149,4 +149,19 @@ std::enable_if_t::value, string> to_string(const T &x) { return sb.as_cslice().str(); } +template +struct LambdaPrintHelper { + SB& sb; +}; +template +SB& operator<<(const LambdaPrintHelper& helper, F&& f) { + f(helper.sb); + return helper.sb; +} +struct LambdaPrint {}; + +inline LambdaPrintHelper operator<<(td::StringBuilder& sb, const LambdaPrint&) { + return LambdaPrintHelper{sb}; +} + } // namespace td diff --git a/tdutils/td/utils/Timer.cpp b/tdutils/td/utils/Timer.cpp index 24de099aa..c2c678955 100644 --- a/tdutils/td/utils/Timer.cpp +++ b/tdutils/td/utils/Timer.cpp @@ -22,6 +22,8 @@ #include "td/utils/logging.h" #include "td/utils/Time.h" +#include + namespace td { Timer::Timer(bool is_paused) : is_paused_(is_paused) { @@ -60,12 +62,15 @@ StringBuilder &operator<<(StringBuilder &string_builder, const Timer &timer) { return string_builder << format::as_time(timer.elapsed()); } -PerfWarningTimer::PerfWarningTimer(string name, double max_duration, std::function&& callback) +PerfWarningTimer::PerfWarningTimer(string name, double max_duration, std::function &&callback) : name_(std::move(name)), start_at_(Time::now()), max_duration_(max_duration), callback_(std::move(callback)) { } PerfWarningTimer::PerfWarningTimer(PerfWarningTimer &&other) - : name_(std::move(other.name_)), start_at_(other.start_at_), max_duration_(other.max_duration_), callback_(std::move(other.callback_)) { + : name_(std::move(other.name_)) + , start_at_(other.start_at_) + , max_duration_(other.max_duration_) + , callback_(std::move(other.callback_)) { other.start_at_ = 0; } @@ -134,4 +139,34 @@ double ThreadCpuTimer::elapsed() const { return res; } +PerfLogAction PerfLog::start_action(std::string name) { + auto i = entries_.size(); + entries_.push_back({.name = std::move(name), .begin = td::Timestamp::now().at()}); + return PerfLogAction{i, std::unique_ptr(this)}; +} +td::StringBuilder &operator<<(StringBuilder &sb, const PerfLog &log) { + sb << "{"; + std::vector ids(log.entries_.size()); + std::iota(ids.begin(), ids.end(), 0); + std::sort(ids.begin(), ids.end(), [&](auto a, auto b) { + return log.entries_[a].end - log.entries_[a].begin > log.entries_[b].end - log.entries_[b].begin; + }); + sb << "{"; + for (size_t i = 0; i < log.entries_.size(); i++) { + sb << "\n\t"; + auto &entry = log.entries_[ids[i]]; + sb << "{" << entry.name << ":" << entry.begin << "->" << entry.end << "(" << entry.end - entry.begin << ")" + << td::format::cond(entry.status.is_error(), entry.status, "") << "}"; + } + sb << "\n}"; + return sb; +} + +double PerfLog::finish_action(size_t i, td::Status status) { + auto &entry = entries_[i]; + CHECK(entry.end == 0); + entry.end = td::Timestamp::now().at(); + entry.status = std::move(status); + return entry.end - entry.begin; +} } // namespace td diff --git a/tdutils/td/utils/Timer.h b/tdutils/td/utils/Timer.h index a27cac8a7..d787f1ca2 100644 --- a/tdutils/td/utils/Timer.h +++ b/tdutils/td/utils/Timer.h @@ -19,6 +19,7 @@ #pragma once #include "td/utils/StringBuilder.h" +#include "td/utils/Status.h" #include @@ -46,7 +47,7 @@ class Timer { class PerfWarningTimer { public: - explicit PerfWarningTimer(string name, double max_duration = 0.1, std::function&& callback = {}); + explicit PerfWarningTimer(string name, double max_duration = 0.1, std::function &&callback = {}); PerfWarningTimer(const PerfWarningTimer &) = delete; PerfWarningTimer &operator=(const PerfWarningTimer &) = delete; PerfWarningTimer(PerfWarningTimer &&other); @@ -80,4 +81,44 @@ class ThreadCpuTimer { bool is_paused_{false}; }; +class PerfLog; +struct EmptyDeleter { + template + void operator()(T *) { + } +}; +class PerfLogAction { + public: + template + double finish(const T &result); + size_t i_{0}; + std::unique_ptr perf_log_; +}; + +class PerfLog { + public: + PerfLogAction start_action(std::string name); + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const PerfLog &log); + + private: + struct Entry { + std::string name; + double begin{}; + double end{}; + td::Status status; + }; + std::vector entries_; + friend class PerfLogAction; + + double finish_action(size_t i, td::Status status); +}; +template +double PerfLogAction::finish(const T &result) { + if (result.is_ok()) { + return perf_log_->finish_action(i_, td::Status::OK()); + } else { + return perf_log_->finish_action(i_, result.error().clone()); + } +} + } // namespace td diff --git a/tdutils/td/utils/logging.h b/tdutils/td/utils/logging.h index d00fba154..5c9a0621f 100644 --- a/tdutils/td/utils/logging.h +++ b/tdutils/td/utils/logging.h @@ -74,6 +74,7 @@ #define LOG(level) LOG_IMPL(level, level, true, ::td::Slice()) #define LOG_IF(level, condition) LOG_IMPL(level, level, condition, #condition) +#define FLOG(level) LOG_IMPL(level, level, true, ::td::Slice()) << td::LambdaPrint{} << [&](auto &sb) #define VLOG(level) LOG_IMPL(DEBUG, level, true, TD_DEFINE_STR(level)) #define VLOG_IF(level, condition) LOG_IMPL(DEBUG, level, condition, TD_DEFINE_STR(level) " " #condition) @@ -95,13 +96,13 @@ inline bool no_return_func() { #define DUMMY_LOG_CHECK(condition) LOG_IF(NEVER, !(condition)) #ifdef TD_DEBUG - #if TD_MSVC +#if TD_MSVC #define LOG_CHECK(condition) \ __analysis_assume(!!(condition)); \ LOG_IMPL(FATAL, FATAL, !(condition), #condition) - #else +#else #define LOG_CHECK(condition) LOG_IMPL(FATAL, FATAL, !(condition) && no_return_func(), #condition) - #endif +#endif #else #define LOG_CHECK DUMMY_LOG_CHECK #endif @@ -263,6 +264,9 @@ class Logger { sb_ << other; return *this; } + LambdaPrintHelper operator<<(const LambdaPrint &) { + return LambdaPrintHelper{*this}; + } MutableCSlice as_cslice() { return sb_.as_cslice(); diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 5b272f268..720f8d496 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -433,6 +433,10 @@ tonNode.newBlockCandidateBroadcast id:tonNode.blockIdExt catchain_seqno:int vali tonNode.newBlockCandidateBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int collator_signature:tonNode.blockSignature flags:# compressed:bytes = tonNode.Broadcast; +// optimistic broadcast of response to tonNode.getOutMsgQueueProof with dst_shard, block and limits arguments +tonNode.outMsgQueueProofBroadcast dst_shard:tonNode.shardId block:tonNode.blockIdExt + limits:ImportedMsgQueueLimits proof:tonNode.OutMsgQueueProof = tonNode.Broadcast; + tonNode.shardPublicOverlayId workchain:int shard:long zero_state_file_hash:int256 = tonNode.ShardPublicOverlayId; tonNode.privateBlockOverlayId zero_state_file_hash:int256 nodes:(vector int256) = tonNode.PrivateBlockOverlayId; @@ -924,7 +928,7 @@ validatorSession.endValidatorGroupStats session_id:int256 timestamp:double ---functions--- collatorNode.generateBlock shard:tonNode.shardId cc_seqno:int prev_blocks:(vector tonNode.blockIdExt) - creator:int256 = collatorNode.Candidate; + creator:int256 round:int first_block_round:int priority:int = collatorNode.Candidate; collatorNode.ping flags:# = collatorNode.Pong; ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index c984fba0b30b868520b6ea2239c9d376838d2378..19a8d13138d85dd2bea8b7e28793cbc85e441aa9 100644 GIT binary patch delta 392 zcmbPnf$hd6whensSWchfai4s^M1+xJqNDicHzp07jE6Vp+7;h}$V_-DG2K9pF=cv! zEMpB=X!-+LMiWkuwwuRFZQDVRw(SW@j2f0KCPv{;ruRfM zsxY!o-x$p}fAWHS*6DAp8RdBe(=v;SOX8Dq@{_aUr(d*Ul%L)+nNef%nG%8R88M6- FyaE5biE#h` delta 264 zcmca{iEYLOwhensSgKTm3MU^h5n*JX=qSGVjY$J1Ofcx@{BUm4dfX`AY2c5Mu=t8 zff5D~u`O`cf>}VtEDA8iG71=0Oi*NOnC$ab1)@_$k6uUgvx diff --git a/ton/ton-types.h b/ton/ton-types.h index 11741c5ec..aeb0595ad 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -425,14 +425,47 @@ struct Ed25519_PublicKey { }; // represents (the contents of) a block + +struct OutMsgQueueProofBroadcast : public td::CntObject { + OutMsgQueueProofBroadcast(ShardIdFull dst_shard, BlockIdExt block_id, td::int32 max_bytes, td::int32 max_msgs, + td::BufferSlice queue_proofs, td::BufferSlice block_state_proofs, + std::vector msg_counts) + : dst_shard(std::move(dst_shard)) + , block_id(block_id) + , max_bytes(max_bytes) + , max_msgs(max_msgs) + , queue_proofs(std::move(queue_proofs)) + , block_state_proofs(std::move(block_state_proofs)) + , msg_counts(std::move(msg_counts)) { + } + ShardIdFull dst_shard; + BlockIdExt block_id; + + // importedMsgQueueLimits + td::uint32 max_bytes; + td::uint32 max_msgs; + + // outMsgQueueProof + td::BufferSlice queue_proofs; + td::BufferSlice block_state_proofs; + std::vector msg_counts; + + virtual OutMsgQueueProofBroadcast* make_copy() const { + return new OutMsgQueueProofBroadcast(dst_shard, block_id, max_bytes, max_msgs, queue_proofs.clone(), + block_state_proofs.clone(), msg_counts); + } +}; + struct BlockCandidate { BlockCandidate(Ed25519_PublicKey pubkey, BlockIdExt id, FileHash collated_file_hash, td::BufferSlice data, - td::BufferSlice collated_data) + td::BufferSlice collated_data, + std::vector> out_msg_queue_broadcasts = {}) : pubkey(pubkey) , id(id) , collated_file_hash(collated_file_hash) , data(std::move(data)) - , collated_data(std::move(collated_data)) { + , collated_data(std::move(collated_data)) + , out_msg_queue_proof_broadcasts(std::move(out_msg_queue_broadcasts)) { } Ed25519_PublicKey pubkey; BlockIdExt id; @@ -440,11 +473,21 @@ struct BlockCandidate { td::BufferSlice data; td::BufferSlice collated_data; + // used only locally + std::vector> out_msg_queue_proof_broadcasts; + BlockCandidate clone() const { - return BlockCandidate{pubkey, id, collated_file_hash, data.clone(), collated_data.clone()}; + return BlockCandidate{ + pubkey, id, collated_file_hash, data.clone(), collated_data.clone(), out_msg_queue_proof_broadcasts}; } }; +struct BlockCandidatePriority { + td::uint32 round{}; + td::uint32 first_block_round{}; + td::int32 priority{}; +}; + struct ValidatorDescr { /* ton::validator::ValidatorFullId */ Ed25519_PublicKey key; ValidatorWeight weight; diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index 2ca3a5f10..e84470595 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -33,6 +33,7 @@ void CollationManager::start_up() { void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, Ed25519_PublicKey creator, + BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, td::CancellationToken cancellation_token, td::Promise promise) { if (shard.is_masterchain()) { @@ -41,12 +42,13 @@ void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_mastercha std::move(cancellation_token), 0); return; } - collate_shard_block(shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), + collate_shard_block(shard, min_masterchain_block_id, std::move(prev), creator, priority, std::move(validator_set), max_answer_size, std::move(cancellation_token), std::move(promise), td::Timestamp::in(10.0)); } void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, Ed25519_PublicKey creator, + BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, td::CancellationToken cancellation_token, td::Promise promise, td::Timestamp timeout) { @@ -133,8 +135,8 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas delay_action( [=, promise = std::move(promise)]() mutable { td::actor::send_closure(SelfId, &CollationManager::collate_shard_block, shard, min_masterchain_block_id, prev, - creator, validator_set, max_answer_size, cancellation_token, std::move(promise), - timeout); + creator, priority, validator_set, max_answer_size, cancellation_token, + std::move(promise), timeout); }, retry_at); }; @@ -145,7 +147,8 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas } td::BufferSlice query = create_serialize_tl_object( - create_tl_shard_id(shard), validator_set->get_catchain_seqno(), std::move(prev_blocks), creator.as_bits256()); + create_tl_shard_id(shard), validator_set->get_catchain_seqno(), std::move(prev_blocks), creator.as_bits256(), + priority.round, priority.first_block_round, priority.priority); LOG(INFO) << "sending collate query for " << next_block_id.to_str() << ": send to #" << selected_idx << "(" << selected_collator << ")"; diff --git a/validator/collation-manager.hpp b/validator/collation-manager.hpp index 7ceea1e6b..9ca69814b 100644 --- a/validator/collation-manager.hpp +++ b/validator/collation-manager.hpp @@ -35,7 +35,8 @@ class CollationManager : public td::actor::Actor { void alarm() override; void collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, - Ed25519_PublicKey creator, td::Ref validator_set, td::uint64 max_answer_size, + Ed25519_PublicKey creator, BlockCandidatePriority priority, + td::Ref validator_set, td::uint64 max_answer_size, td::CancellationToken cancellation_token, td::Promise promise); void update_options(td::Ref opts); @@ -52,7 +53,8 @@ class CollationManager : public td::actor::Actor { td::actor::ActorId rldp_; void collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, - Ed25519_PublicKey creator, td::Ref validator_set, td::uint64 max_answer_size, + Ed25519_PublicKey creator, BlockCandidatePriority priority, + td::Ref validator_set, td::uint64 max_answer_size, td::CancellationToken cancellation_token, td::Promise promise, td::Timestamp timeout); diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index cf2770d93..1b8eb79e8 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -191,17 +191,41 @@ void CollatorNode::update_validator_group_info(ShardIdFull shard, std::vectorblock_seqno < info.next_block_seqno) { cache_entry->cancel(td::Status::Error(PSTRING() << "next block seqno " << cache_entry->block_seqno << " is too small, expected " << info.next_block_seqno)); + if (!cache_entry->has_external_query_at && cache_entry->has_internal_query_at) { + LOG(INFO) << "generate block query" + << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno + << ", next_block_seqno=" << cache_entry->block_seqno + << ": nobody asked for block, but we tried to generate it"; + } + if (cache_entry->has_external_query_at && !cache_entry->has_internal_query_at) { + LOG(INFO) << "generate block query" + << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno + << ", next_block_seqno=" << cache_entry->block_seqno + << ": somebody asked for block we didn't even tried to generate"; + } cache_it = info.cache.erase(cache_it); continue; } if (cache_entry->block_seqno == info.next_block_seqno && cached_prev != info.prev) { cache_entry->cancel(td::Status::Error("invalid prev blocks")); + if (!cache_entry->has_external_query_at && cache_entry->has_internal_query_at) { + LOG(INFO) << "generate block query" + << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno + << ", next_block_seqno=" << cache_entry->block_seqno + << ": nobody asked for block, but we tried to generate it"; + } + if (cache_entry->has_external_query_at && !cache_entry->has_internal_query_at) { + LOG(INFO) << "generate block query" + << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno + << ", next_block_seqno=" << cache_entry->block_seqno + << ": somebody asked for block we didn't even tried to generate"; + } cache_it = info.cache.erase(cache_it); continue; } ++cache_it; } - generate_block(shard, cc_seqno, info.prev, td::Timestamp::in(10.0), [](td::Result) {}); + generate_block(shard, cc_seqno, info.prev, {}, td::Timestamp::in(10.0), [](td::Result) {}); } return; } @@ -285,6 +309,14 @@ static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey cre cc_seqno = info.gen_catchain_seqno; val_set_hash = info.gen_validator_list_hash_short; + + for (auto& broadcast_ref : block.out_msg_queue_proof_broadcasts) { + auto block_state_proof = create_block_state_proof(root).move_as_ok(); + + auto &broadcast = broadcast_ref.write(); + broadcast.block_id = block.id; + broadcast.block_state_proofs = vm::std_boc_serialize(std::move(block_state_proof), 31).move_as_ok(); + } return block; } @@ -322,6 +354,11 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data for (const auto& b : f->prev_blocks_) { prev_blocks.push_back(create_block_id(b)); } + auto priority = BlockCandidatePriority { + .round = static_cast(f->round_), + .first_block_round = static_cast(f->first_block_round_), + .priority = f->priority_ + }; Ed25519_PublicKey creator(f->creator_); td::Promise new_promise = [promise = std::move(promise), src, shard](td::Result R) mutable { @@ -353,11 +390,13 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data return; } LOG(INFO) << "got adnl query from " << src << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno; - generate_block(shard, cc_seqno, std::move(prev_blocks), td::Timestamp::in(10.0), std::move(new_promise)); + generate_block(shard, cc_seqno, std::move(prev_blocks), priority, td::Timestamp::in(10.0), std::move(new_promise)); } void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std::vector prev_blocks, - td::Timestamp timeout, td::Promise promise) { + std::optional o_priority, td::Timestamp timeout, + td::Promise promise) { + bool is_external = !o_priority; if (last_masterchain_state_.is_null()) { promise.set_error(td::Status::Error(ErrorCode::notready, "not ready")); return; @@ -379,8 +418,8 @@ void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std promise.set_error(td::Status::Error(ErrorCode::timeout)); return; } - td::actor::send_closure(SelfId, &CollatorNode::generate_block, shard, cc_seqno, std::move(prev_blocks), timeout, - std::move(promise)); + td::actor::send_closure(SelfId, &CollatorNode::generate_block, shard, cc_seqno, std::move(prev_blocks), + std::move(o_priority), timeout, std::move(promise)); }); return; } @@ -399,36 +438,81 @@ void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std return; } + static auto prefix_inner = [] (auto &sb, auto &shard, auto cc_seqno, auto block_seqno, + const std::optional &o_priority) { + sb << "generate block query" + << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno; + if (o_priority) { + sb << " external{"; + sb << "round_offset=" << o_priority->round - o_priority->first_block_round << ",priority=" << o_priority->priority; + sb << ",first_block_round=" << o_priority->first_block_round; + sb << "}"; + } else { + sb << " internal" ; + } + }; + auto prefix = [&] (auto &sb) { + prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority); + }; + auto cache_entry = validator_group_info.cache[prev_blocks]; if (cache_entry == nullptr) { cache_entry = validator_group_info.cache[prev_blocks] = std::make_shared(); } + if (is_external && !cache_entry->has_external_query_at) { + cache_entry->has_external_query_at = td::Timestamp::now(); + if (cache_entry->has_internal_query_at && cache_entry->has_external_query_at) { + FLOG(INFO) { + prefix(sb); + sb << ": got external query " << cache_entry->has_external_query_at.at() - cache_entry->has_internal_query_at.at() + << "s after internal query [WON]"; + }; + } + } + if (!is_external && !cache_entry->has_internal_query_at) { + cache_entry->has_internal_query_at = td::Timestamp::now(); + if (cache_entry->has_internal_query_at && cache_entry->has_external_query_at) { + FLOG(INFO) { + prefix(sb); + sb << ": got internal query " << cache_entry->has_internal_query_at.at() - cache_entry->has_external_query_at.at() + << "s after external query [LOST]"; + }; + } + } if (cache_entry->result) { - LOG(INFO) << "generate block query" - << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno - << ": using cached result"; + auto has_result_ago = td::Timestamp::now().at() - cache_entry->has_result_at.at(); + FLOG(INFO) { + prefix(sb); + sb << ": using cached result " << " generated " << has_result_ago << "s ago"; + sb << (is_external ? " for external query [WON]" : " for internal query "); + }; + promise.set_result(cache_entry->result.value().clone()); return; } cache_entry->promises.push_back(std::move(promise)); + if (cache_entry->started) { - LOG(INFO) << "generate block query" - << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno - << ": collation in progress, waiting"; + FLOG(INFO) { + prefix(sb); + sb << ": collation in progress, waiting"; + }; return; } - LOG(INFO) << "generate block query" - << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno - << ": starting collation"; + FLOG(INFO) { + prefix(sb); + sb << ": starting collation"; + }; cache_entry->started = true; cache_entry->block_seqno = block_seqno; run_collate_query( shard, last_masterchain_state_->get_block_id(), std::move(prev_blocks), Ed25519_PublicKey{td::Bits256::zero()}, last_masterchain_state_->get_validator_set(shard), opts_->get_collator_options(), manager_, timeout, [=, SelfId = actor_id(this), timer = td::Timer{}](td::Result R) { - LOG(INFO) << "generate block result" - << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno - << ", time=" << timer.elapsed() << ": " << (R.is_ok() ? "OK" : R.error().to_string()); + FLOG(INFO) { + prefix_inner(sb, shard, cc_seqno, block_seqno,o_priority); + sb << timer.elapsed() << ": " << (R.is_ok() ? "OK" : R.error().to_string()); + }; td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R)); }, cache_entry->cancellation_token_source.get_cancellation_token(), @@ -443,6 +527,7 @@ void CollatorNode::process_result(std::shared_ptr cache_entry, td::R } } else { cache_entry->result = R.move_as_ok(); + cache_entry->has_result_at = td::Timestamp::now(); for (auto& p : cache_entry->promises) { p.set_result(cache_entry->result.value().clone()); } diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp index a361ceb30..54876c35f 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node.hpp @@ -19,6 +19,7 @@ #include "interfaces/validator-manager.h" #include "rldp/rldp.h" #include +#include namespace ton::validator { @@ -57,6 +58,9 @@ class CollatorNode : public td::actor::Actor { struct CacheEntry { bool started = false; + td::Timestamp has_internal_query_at; + td::Timestamp has_external_query_at; + td::Timestamp has_result_at; BlockSeqno block_seqno = 0; td::optional result; td::CancellationTokenSource cancellation_token_source; @@ -84,7 +88,8 @@ class CollatorNode : public td::actor::Actor { td::Result get_future_validator_group(ShardIdFull shard, CatchainSeqno cc_seqno); void generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std::vector prev_blocks, - td::Timestamp timeout, td::Promise promise); + std::optional o_priority, td::Timestamp timeout, + td::Promise promise); void process_result(std::shared_ptr cache_entry, td::Result R); public: diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index 270c53f07..4e0d11e48 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -46,6 +46,33 @@ void FullNodeFastSyncOverlay::process_block_broadcast(PublicKeyHash src, ton_api td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok()); } +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_outMsgQueueProofBroadcast &query) { + BlockIdExt block_id = create_block_id(query.block_); + ShardIdFull shard_id = create_shard_id(query.dst_shard_); + if (query.proof_->get_id() != ton_api::tonNode_outMsgQueueProof::ID) { + LOG(ERROR) << "got tonNode.outMsgQueueProofBroadcast with proof not tonNode.outMsgQueueProof"; + return; + } + auto tl_proof = move_tl_object_as(query.proof_); + auto R = OutMsgQueueProof::fetch(shard_id, {block_id}, + block::ImportedMsgQueueLimits{.max_bytes = td::uint32(query.limits_->max_bytes_), + .max_msgs = td::uint32(query.limits_->max_msgs_)}, + *tl_proof); + if (R.is_error()) { + LOG(ERROR) << "got tonNode.outMsgQueueProofBroadcast with invalid proof: " << R.error(); + return; + } + if (R.ok().size() != 1) { + LOG(ERROR) << "got tonNode.outMsgQueueProofBroadcast with invalid proofs count=" << R.ok().size(); + return; + } + auto proof = std::move(R.move_as_ok()[0]); + + LOG(INFO) << "got tonNode.outMsgQueueProofBroadcast " << shard_id.to_str() << " " << block_id.to_str(); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::add_out_msg_queue_proof, shard_id, + std::move(proof)); +} + void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query) { BlockIdExt block_id = create_block_id(query.block_->block_); VLOG(FULL_NODE_DEBUG) << "Received newShardBlockBroadcast in fast sync overlay from " << src << ": " @@ -200,6 +227,22 @@ void FullNodeFastSyncOverlay::collect_validator_telemetry(std::string filename) } } +void FullNodeFastSyncOverlay::send_out_msg_queue_proof_broadcast(td::Ref broadcast) { + if (!inited_) { + return; + } + auto B = create_serialize_tl_object( + create_tl_shard_id(broadcast->dst_shard), create_tl_block_id(broadcast->block_id), + create_tl_object(broadcast->max_bytes, broadcast->max_msgs), + create_tl_object(broadcast->queue_proofs.clone(), + broadcast->block_state_proofs.clone(), + std::vector(broadcast->msg_counts))); + VLOG(FULL_NODE_DEBUG) << "Sending outMsgQueueProof in fast sync overlay: " << broadcast->dst_shard.to_str() << " " + << broadcast->block_id.to_str(); + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); +} + void FullNodeFastSyncOverlay::start_up() { auto X = create_hash_tl_object(zero_state_file_hash_, create_tl_shard_id(shard_)); td::BufferSlice b{32}; diff --git a/validator/full-node-fast-sync-overlays.hpp b/validator/full-node-fast-sync-overlays.hpp index 05d83071d..e0db87b7f 100644 --- a/validator/full-node-fast-sync-overlays.hpp +++ b/validator/full-node-fast-sync-overlays.hpp @@ -25,6 +25,7 @@ class FullNodeFastSyncOverlay : public td::actor::Actor { public: void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast& query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed& query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_outMsgQueueProofBroadcast& query); void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast& query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast& query); @@ -46,6 +47,7 @@ class FullNodeFastSyncOverlay : public td::actor::Actor { void send_broadcast(BlockBroadcast broadcast); void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::BufferSlice data); + void send_out_msg_queue_proof_broadcast(td::Ref broadcast); void send_validator_telemetry(tl_object_ptr telemetry); void collect_validator_telemetry(std::string filename); diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 8ac5185d9..fd1ef943e 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -151,6 +151,9 @@ class FullNodeShardImpl : public FullNodeShard { void process_broadcast(PublicKeyHash src, ton_api::tonNode_ihrMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_outMsgQueueProofBroadcast &query) { + LOG(ERROR) << "Ignore outMsgQueueProofBroadcast"; + } void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 0278c9ae4..e7527d768 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -371,6 +371,14 @@ void FullNodeImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_se } } +void FullNodeImpl::send_out_msg_queue_proof_broadcast(td::Ref broadcast) { + auto fast_sync_overlay = fast_sync_overlays_.choose_overlay(broadcast->dst_shard).first; + if (!fast_sync_overlay.empty()) { + td::actor::send_closure(fast_sync_overlay, &FullNodeFastSyncOverlay::send_out_msg_queue_proof_broadcast, + std::move(broadcast)); + } +} + void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, int mode) { if (mode & broadcast_mode_custom) { send_block_broadcast_to_custom_overlays(broadcast); @@ -713,6 +721,9 @@ void FullNodeImpl::start_up() { td::actor::send_closure(id_, &FullNodeImpl::send_block_candidate, block_id, cc_seqno, validator_set_hash, std::move(data)); } + void send_out_msg_queue_proof_broadcast(td::Ref broadcast) override { + td::actor::send_closure(id_, &FullNodeImpl::send_out_msg_queue_proof_broadcast, std::move(broadcast)); + } void send_broadcast(BlockBroadcast broadcast, int mode) override { td::actor::send_closure(id_, &FullNodeImpl::send_broadcast, std::move(broadcast), mode); } diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 9e254d7d4..5533b1f43 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -70,6 +70,7 @@ class FullNodeImpl : public FullNode { void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::BufferSlice data); void send_broadcast(BlockBroadcast broadcast, int mode); + void send_out_msg_queue_proof_broadcast(td::Ref broadcats); void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 72154f861..0090e95de 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -237,6 +237,7 @@ class Collator final : public td::actor::Actor { bool store_out_msg_queue_size_ = false; td::PerfWarningTimer perf_timer_; + td::PerfLog perf_log_; // block::Account* lookup_account(td::ConstBitPtr addr) const; std::unique_ptr make_account_from(td::ConstBitPtr addr, Ref account, @@ -252,18 +253,18 @@ class Collator final : public td::actor::Actor { bool fatal_error(int err_code, std::string err_msg); bool fatal_error(std::string err_msg, int err_code = -666); void check_pending(); - void after_get_mc_state(td::Result, BlockIdExt>> res); - void after_get_shard_state(int idx, td::Result> res); - void after_get_block_data(int idx, td::Result> res); - void after_get_shard_blocks(td::Result>> res); + void after_get_mc_state(td::Result, BlockIdExt>> res, td::PerfLogAction token); + void after_get_shard_state(int idx, td::Result> res, td::PerfLogAction token); + void after_get_block_data(int idx, td::Result> res, td::PerfLogAction token); + void after_get_shard_blocks(td::Result>> res, td::PerfLogAction token); bool preprocess_prev_mc_state(); bool register_mc_state(Ref other_mc_state); bool request_aux_mc_state(BlockSeqno seqno, Ref& state); Ref get_aux_mc_state(BlockSeqno seqno) const; - void after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res); + void after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res, td::PerfLogAction token); bool fix_one_processed_upto(block::MsgProcessedUpto& proc, const ton::ShardIdFull& owner); bool fix_processed_upto(block::MsgProcessedUptoCollection& upto); - void got_neighbor_msg_queues(td::Result>> R); + void got_neighbor_msg_queues(td::Result>> R, td::PerfLogAction token); void got_neighbor_msg_queue(unsigned i, Ref res); void got_out_queue_size(size_t i, td::Result res); bool adjust_shard_config(); @@ -309,7 +310,7 @@ class Collator final : public td::actor::Actor { bool is_our_address(Ref addr_ref) const; bool is_our_address(ton::AccountIdPrefixFull addr_prefix) const; bool is_our_address(const ton::StdSmcAddress& addr) const; - void after_get_external_messages(td::Result, int>>> res); + void after_get_external_messages(td::Result, int>>> res, td::PerfLogAction token); td::Result register_external_message_cell(Ref ext_msg, const ExtMessage::Hash& ext_hash, int priority); // td::Result register_external_message(td::Slice ext_msg_boc); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 0054ea98a..cd97e3fcc 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -217,25 +217,30 @@ void Collator::start_up() { // 2. learn latest masterchain state and block id LOG(DEBUG) << "sending get_top_masterchain_state_block() to Manager"; ++pending; + auto token = perf_log_.start_action("get_top_masterchain_state_block"); if (!is_hardfork_) { td::actor::send_closure_later(manager, &ValidatorManager::get_top_masterchain_state_block, - [self = get_self()](td::Result, BlockIdExt>> res) { + [self = get_self(), token = std::move(token)]( + td::Result, BlockIdExt>> res) mutable { LOG(DEBUG) << "got answer to get_top_masterchain_state_block"; td::actor::send_closure_later(std::move(self), &Collator::after_get_mc_state, - std::move(res)); + std::move(res), std::move(token)); }); } else { - td::actor::send_closure_later( - manager, &ValidatorManager::get_shard_state_from_db_short, min_mc_block_id, - [self = get_self(), block_id = min_mc_block_id](td::Result> res) { - LOG(DEBUG) << "got answer to get_top_masterchain_state_block"; - if (res.is_error()) { - td::actor::send_closure_later(std::move(self), &Collator::after_get_mc_state, res.move_as_error()); - } else { - td::actor::send_closure_later(std::move(self), &Collator::after_get_mc_state, - std::make_pair(Ref(res.move_as_ok()), block_id)); - } - }); + td::actor::send_closure_later(manager, &ValidatorManager::get_shard_state_from_db_short, min_mc_block_id, + [self = get_self(), block_id = min_mc_block_id, + token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to get_top_masterchain_state_block"; + if (res.is_error()) { + td::actor::send_closure_later(std::move(self), &Collator::after_get_mc_state, + res.move_as_error(), std::move(token)); + } else { + td::actor::send_closure_later( + std::move(self), &Collator::after_get_mc_state, + std::make_pair(Ref(res.move_as_ok()), block_id), + std::move(token)); + } + }); } } // 3. load previous block(s) and corresponding state(s) @@ -245,23 +250,27 @@ void Collator::start_up() { // 3.1. load state LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), - timeout, [self = get_self(), i](td::Result> res) { - LOG(DEBUG) << "got answer to wait_block_state query #" << i; - td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_state, i, - std::move(res)); - }); + auto token = perf_log_.start_action(PSTRING() << "wait_block_state #" << i); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), timeout, + [self = get_self(), i, token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state query #" << i; + td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_state, i, std::move(res), + std::move(token)); + }); if (prev_blocks[i].seqno()) { // 3.2. load block // NB: we need the block itself only for extracting start_lt and end_lt to create correct prev_blk:ExtBlkRef and related Merkle proofs LOG(DEBUG) << "sending wait_block_data() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_data_short, prev_blocks[i], priority(), - timeout, [self = get_self(), i](td::Result> res) { - LOG(DEBUG) << "got answer to wait_block_data query #" << i; - td::actor::send_closure_later(std::move(self), &Collator::after_get_block_data, i, - std::move(res)); - }); + auto token = perf_log_.start_action(PSTRING() << "wait_block_data #" << i); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_data_short, prev_blocks[i], priority(), timeout, + [self = get_self(), i, token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_data query #" << i; + td::actor::send_closure_later(std::move(self), &Collator::after_get_block_data, i, std::move(res), + std::move(token)); + }); } } if (is_hardfork_) { @@ -271,22 +280,28 @@ void Collator::start_up() { if (!is_hardfork_) { LOG(DEBUG) << "sending get_external_messages() query to Manager"; ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::get_external_messages, shard_, - [self = get_self()](td::Result, int>>> res) -> void { + auto token = perf_log_.start_action("get_external_messages"); + td::actor::send_closure_later( + manager, &ValidatorManager::get_external_messages, shard_, + [self = get_self(), + token = std::move(token)](td::Result, int>>> res) mutable -> void { LOG(DEBUG) << "got answer to get_external_messages() query"; - td::actor::send_closure_later(std::move(self), &Collator::after_get_external_messages, std::move(res)); + td::actor::send_closure_later(std::move(self), &Collator::after_get_external_messages, std::move(res), + std::move(token)); }); } if (is_masterchain() && !is_hardfork_) { // 5. load shard block info messages LOG(DEBUG) << "sending get_shard_blocks_for_collator() query to Manager"; ++pending; - td::actor::send_closure_later( - manager, &ValidatorManager::get_shard_blocks_for_collator, prev_blocks[0], - [self = get_self()](td::Result>> res) -> void { - LOG(DEBUG) << "got answer to get_shard_blocks_for_collator() query"; - td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_blocks, std::move(res)); - }); + auto token = perf_log_.start_action("get_shard_blocks_for_collator"); + td::actor::send_closure_later(manager, &ValidatorManager::get_shard_blocks_for_collator, prev_blocks[0], + [self = get_self(), token = std::move(token)]( + td::Result>> res) mutable -> void { + LOG(DEBUG) << "got answer to get_shard_blocks_for_collator() query"; + td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_blocks, + std::move(res), std::move(token)); + }); } // 6. set timeout alarm_timestamp() = timeout; @@ -362,6 +377,8 @@ bool Collator::fatal_error(td::Status error) { td::Timestamp::in(10.0), std::move(main_promise), std::move(cancellation_token_), mode_, attempt_idx_ + 1); } else { + LOG(INFO) << "collation failed in " << perf_timer_.elapsed() << " s " << error; + LOG(INFO) << perf_log_; main_promise(std::move(error)); } busy_ = false; @@ -482,12 +499,14 @@ bool Collator::request_aux_mc_state(BlockSeqno seqno, Ref& st CHECK(blkid.is_valid_ext() && blkid.is_masterchain()); LOG(DEBUG) << "sending auxiliary wait_block_state() query for " << blkid.to_str() << " to Manager"; ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, blkid, priority(), timeout, - [self = get_self(), blkid](td::Result> res) { - LOG(DEBUG) << "got answer to wait_block_state query for " << blkid.to_str(); - td::actor::send_closure_later(std::move(self), &Collator::after_get_aux_shard_state, - blkid, std::move(res)); - }); + auto token = perf_log_.start_action(PSTRING() << "auxiliary wait_block_state " << blkid.to_str()); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_short, blkid, priority(), timeout, + [self = get_self(), blkid, token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state query for " << blkid.to_str(); + td::actor::send_closure_later(std::move(self), &Collator::after_get_aux_shard_state, blkid, std::move(res), + std::move(token)); + }); state.clear(); return true; } @@ -515,9 +534,11 @@ Ref Collator::get_aux_mc_state(BlockSeqno seqno) const { * @param blkid The BlockIdExt of the shard state. * @param res The result of retrieving the shard state. */ -void Collator::after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res) { +void Collator::after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res, + td::PerfLogAction token) { LOG(DEBUG) << "in Collator::after_get_aux_shard_state(" << blkid.to_str() << ")"; --pending; + token.finish(res); if (res.is_error()) { fatal_error("cannot load auxiliary masterchain state for "s + blkid.to_str() + " : " + res.move_as_error().to_string()); @@ -579,9 +600,11 @@ bool Collator::preprocess_prev_mc_state() { * * @param res The retrieved masterchain state. */ -void Collator::after_get_mc_state(td::Result, BlockIdExt>> res) { +void Collator::after_get_mc_state(td::Result, BlockIdExt>> res, + td::PerfLogAction token) { LOG(WARNING) << "in Collator::after_get_mc_state()"; --pending; + token.finish(res); if (res.is_error()) { fatal_error(res.move_as_error()); return; @@ -598,12 +621,14 @@ void Collator::after_get_mc_state(td::Result, Bl // NB. it is needed only for creating a correct ExtBlkRef reference to it, which requires start_lt and end_lt LOG(DEBUG) << "sending wait_block_data() query #-1 for " << mc_block_id_.to_str() << " to Manager"; ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_data_short, mc_block_id_, priority(), timeout, - [self = get_self()](td::Result> res) { - LOG(DEBUG) << "got answer to wait_block_data query #-1"; - td::actor::send_closure_later(std::move(self), &Collator::after_get_block_data, -1, - std::move(res)); - }); + auto token = perf_log_.start_action("wait_block_data #-1"); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_data_short, mc_block_id_, priority(), timeout, + [self = get_self(), token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_data query #-1"; + td::actor::send_closure_later(std::move(self), &Collator::after_get_block_data, -1, std::move(res), + std::move(token)); + }); } check_pending(); } @@ -614,9 +639,10 @@ void Collator::after_get_mc_state(td::Result, Bl * @param idx The index of the previous shard block (0 or 1). * @param res The retrieved shard state. */ -void Collator::after_get_shard_state(int idx, td::Result> res) { +void Collator::after_get_shard_state(int idx, td::Result> res, td::PerfLogAction token) { LOG(WARNING) << "in Collator::after_get_shard_state(" << idx << ")"; --pending; + token.finish(res); if (res.is_error()) { fatal_error(res.move_as_error()); return; @@ -647,9 +673,10 @@ void Collator::after_get_shard_state(int idx, td::Result> res) { * @param idx The index of the previous block (0 or 1). * @param res The retreved block data. */ -void Collator::after_get_block_data(int idx, td::Result> res) { +void Collator::after_get_block_data(int idx, td::Result> res, td::PerfLogAction token) { LOG(DEBUG) << "in Collator::after_get_block_data(" << idx << ")"; --pending; + token.finish(res); if (res.is_error()) { fatal_error(res.move_as_error()); return; @@ -691,8 +718,10 @@ void Collator::after_get_block_data(int idx, td::Result> res) { * * @param res The retrieved shard block descriptions. */ -void Collator::after_get_shard_blocks(td::Result>> res) { +void Collator::after_get_shard_blocks(td::Result>> res, + td::PerfLogAction token) { --pending; + token.finish(res); if (res.is_error()) { fatal_error(res.move_as_error()); return; @@ -848,10 +877,13 @@ bool Collator::request_neighbor_msg_queues() { ++i; } ++pending; + auto token = perf_log_.start_action("neighbor_msg_queues"); td::actor::send_closure_later( manager, &ValidatorManager::wait_neighbor_msg_queue_proofs, shard_, std::move(top_blocks), timeout, - [self = get_self()](td::Result>> res) { - td::actor::send_closure_later(std::move(self), &Collator::got_neighbor_msg_queues, std::move(res)); + [self = get_self(), + token = std::move(token)](td::Result>> res) mutable { + td::actor::send_closure_later(std::move(self), &Collator::got_neighbor_msg_queues, std::move(res), + std::move(token)); }); return true; } @@ -883,13 +915,15 @@ bool Collator::request_out_msg_queue_size() { * @param i The index of the neighbor. * @param res The obtained outbound queue. */ -void Collator::got_neighbor_msg_queues(td::Result>> R) { +void Collator::got_neighbor_msg_queues(td::Result>> R, + td::PerfLogAction token) { --pending; + double duration = token.finish(R); if (R.is_error()) { fatal_error(R.move_as_error_prefix("failed to get neighbor msg queues: ")); return; } - LOG(INFO) << "neighbor output queues fetched"; + LOG(INFO) << "neighbor output queues fetched, took " << duration << "s"; auto res = R.move_as_ok(); unsigned i = 0; for (block::McShardDescr& descr : neighbors_) { @@ -2091,12 +2125,9 @@ bool Collator::init_lt() { * @returns True if the configuration parameters were successfully fetched and initialized, false otherwise. */ bool Collator::fetch_config_params() { - auto res = block::FetchConfigParams::fetch_config_params(*config_, - &old_mparams_, &storage_prices_, &storage_phase_cfg_, - &rand_seed_, &compute_phase_cfg_, &action_phase_cfg_, - &masterchain_create_fee_, &basechain_create_fee_, - workchain(), now_ - ); + auto res = block::FetchConfigParams::fetch_config_params( + *config_, &old_mparams_, &storage_prices_, &storage_phase_cfg_, &rand_seed_, &compute_phase_cfg_, + &action_phase_cfg_, &masterchain_create_fee_, &basechain_create_fee_, workchain(), now_); if (res.is_error()) { return fatal_error(res.move_as_error()); } @@ -2217,6 +2248,11 @@ bool Collator::init_value_create() { bool Collator::do_collate() { // After do_collate started it will not be interrupted by timeout alarm_timestamp() = td::Timestamp::never(); + auto token = perf_log_.start_action("do_collate"); + td::Status status = td::Status::Error("some error"); + SCOPE_EXIT { + token.finish(status); + }; LOG(WARNING) << "do_collate() : start"; if (!fetch_config_params()) { @@ -2342,6 +2378,7 @@ bool Collator::do_collate() { if (!create_block_candidate()) { return fatal_error("cannot serialize a new Block candidate"); } + status = td::Status::OK(); return true; } @@ -5811,12 +5848,43 @@ bool Collator::create_block_candidate() { << block_limit_status_->transactions; LOG(INFO) << "serialized collated data size " << cdata_slice.size() << " bytes (preliminary estimate was " << block_limit_status_->collated_data_stat.estimate_proof_size() << ")"; + auto new_block_id_ext = ton::BlockIdExt{ton::BlockId{shard_, new_block_seqno}, new_block->get_hash().bits(), + block::compute_file_hash(blk_slice.as_slice())}; // 3. create a BlockCandidate - block_candidate = std::make_unique( - created_by_, - ton::BlockIdExt{ton::BlockId{shard_, new_block_seqno}, new_block->get_hash().bits(), - block::compute_file_hash(blk_slice.as_slice())}, - block::compute_file_hash(cdata_slice.as_slice()), blk_slice.clone(), cdata_slice.clone()); + block_candidate = + std::make_unique(created_by_, new_block_id_ext, block::compute_file_hash(cdata_slice.as_slice()), + blk_slice.clone(), cdata_slice.clone()); + const bool need_out_msg_queue_broadcasts = true; + if (need_out_msg_queue_broadcasts) { + // we can't generate two proofs at the same time for the same root (it is not currently supported by cells) + // so we have can't reuse new state and have to regenerate it with merkle update + auto new_state = vm::MerkleUpdate::apply(prev_state_root_pure_, state_update); + CHECK(new_state.not_null()); + CHECK(new_state->get_hash() == state_root->get_hash()); + assert(config_ && shard_conf_); + auto neighbor_list = shard_conf_->get_neighbor_shard_hash_ids(shard_); + LOG(INFO) << "Build OutMsgQueueProofs for " << neighbor_list.size() << " neighbours"; + for (ton::BlockId blk_id : neighbor_list) { + auto prefix = blk_id.shard_full(); + auto limits = mc_state_->get_imported_msg_queue_limits(blk_id.workchain); + + // one could use monitor_min_split_depth here, to decrease number of broadcasts + // but current implementation OutMsgQueueImporter doesn't support it + + auto r_proof = OutMsgQueueProof::build( + prefix, {OutMsgQueueProof::OneBlock{.id = new_block_id_ext, .state_root = new_state, .block_root = new_block}}, limits); + if (r_proof.is_ok()) { + auto proof = r_proof.move_as_ok(); + block_candidate->out_msg_queue_proof_broadcasts.push_back(td::Ref( + true, OutMsgQueueProofBroadcast(prefix, new_block_id_ext, limits.max_bytes, limits.max_msgs, + std::move(proof->queue_proofs_), std::move(proof->block_state_proofs_), + std::move(proof->msg_counts_)))); + } else { + LOG(ERROR) << "Failed to build OutMsgQueueProof: " << r_proof.error(); + } + } + } + // 3.1 check block and collated data size auto consensus_config = config_->get_consensus_config(); if (block_candidate->data.size() > consensus_config.max_block_size) { @@ -5896,6 +5964,7 @@ void Collator::return_block_candidate(td::Result saved) { CHECK(block_candidate); LOG(WARNING) << "sending new BlockCandidate to Promise"; LOG(WARNING) << "collation took " << perf_timer_.elapsed() << " s"; + LOG(WARNING) << perf_log_; main_promise(block_candidate->clone()); busy_ = false; stop(); @@ -5974,9 +6043,11 @@ td::Result Collator::register_external_message_cell(Ref ext_msg, * * @param res The result of the external message retrieval operation. */ -void Collator::after_get_external_messages(td::Result, int>>> res) { +void Collator::after_get_external_messages(td::Result, int>>> res, + td::PerfLogAction token) { // res: pair {ext msg, priority} --pending; + token.finish(res); if (res.is_error()) { fatal_error(res.move_as_error()); return; diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 444d5846f..4e8802c7e 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -320,13 +320,23 @@ void OutMsgQueueImporter::get_neighbor_msg_queue_proofs( if (prefix.pfx_len() > min_split) { prefix = shard_prefix(prefix, min_split); } - new_queries[prefix].push_back(block); + + LOG(INFO) << "search for out msg queue proof " << prefix.to_str() << block.to_str(); + auto& small_entry = small_cache_[std::make_pair(dst_shard, block)]; + if (!small_entry.result.is_null()) { + entry->result[block] = small_entry.result; + entry->from_small_cache++; + alarm_timestamp().relax(small_entry.timeout = td::Timestamp::in(CACHE_TTL)); + } else { + small_entry.pending_entries.push_back(entry); + ++entry->pending; + new_queries[prefix].push_back(block); + } } }; auto limits = last_masterchain_state_->get_imported_msg_queue_limits(dst_shard.workchain); for (auto& p : new_queries) { for (size_t i = 0; i < p.second.size(); i += 16) { - ++entry->pending; size_t j = std::min(i + 16, p.second.size()); get_proof_import(entry, std::vector(p.second.begin() + i, p.second.begin() + j), limits * (td::uint32)(j - i)); @@ -355,7 +365,7 @@ void OutMsgQueueImporter::get_proof_local(std::shared_ptr entry, Blo if (block.seqno() == 0) { std::vector> proof = { td::Ref(true, block, state->root_cell(), td::Ref{})}; - td::actor::send_closure(SelfId, &OutMsgQueueImporter::got_proof, entry, std::move(proof)); + td::actor::send_closure(SelfId, &OutMsgQueueImporter::got_proof, entry, std::move(proof), ProofSource::Local); return; } td::actor::send_closure( @@ -371,7 +381,8 @@ void OutMsgQueueImporter::get_proof_local(std::shared_ptr entry, Blo Ref block_state_proof = create_block_state_proof(R.ok()->root_cell()).move_as_ok(); std::vector> proof = { td::Ref(true, block, state->root_cell(), std::move(block_state_proof))}; - td::actor::send_closure(SelfId, &OutMsgQueueImporter::got_proof, entry, std::move(proof)); + td::actor::send_closure(SelfId, &OutMsgQueueImporter::got_proof, entry, std::move(proof), + ProofSource::Local); }); }); } @@ -395,27 +406,70 @@ void OutMsgQueueImporter::get_proof_import(std::shared_ptr entry, st retry_after); return; } - td::actor::send_closure(SelfId, &OutMsgQueueImporter::got_proof, entry, R.move_as_ok()); + td::actor::send_closure(SelfId, &OutMsgQueueImporter::got_proof, entry, R.move_as_ok(), ProofSource::Query); }); } -void OutMsgQueueImporter::got_proof(std::shared_ptr entry, std::vector> proofs) { +void OutMsgQueueImporter::got_proof(std::shared_ptr entry, std::vector> proofs, + ProofSource proof_source) { if (!check_timeout(entry)) { return; } + // TODO: maybe save proof to small cache? It would allow other queries to reuse this result for (auto& p : proofs) { - entry->result[p->block_id_] = std::move(p); - } - CHECK(entry->pending > 0); - if (--entry->pending == 0) { - finish_query(entry); + auto block_id = p->block_id_; + if (entry->result.emplace(block_id, std::move(p)).second) { + CHECK(entry->pending > 0); + switch (proof_source) { + case ProofSource::SmallCache: + entry->from_small_cache++; + break; + case ProofSource::Broadcast: + entry->from_broadcast++; + break; + case ProofSource::Query: + entry->from_query++; + break; + case ProofSource::Local: + entry->from_local++; + break; + } + if (--entry->pending == 0) { + finish_query(entry); + } + } } } void OutMsgQueueImporter::finish_query(std::shared_ptr entry) { - LOG(DEBUG) << "Done importing neighbor msg queues for shard " << entry->dst_shard.to_str() << ", " - << entry->blocks.size() << " blocks in " << entry->timer.elapsed() << "s"; + FLOG(INFO) { + sb << "Done importing neighbor msg queues for shard " << entry->dst_shard.to_str() << ", " << entry->blocks.size() + << " blocks in " << entry->timer.elapsed() << "s"; + sb << " sources{"; + if (entry->from_broadcast) { + sb << " broadcast=" << entry->from_broadcast; + } + if (entry->from_small_cache) { + sb << " small_cache=" << entry->from_small_cache; + } + if (entry->from_local) { + sb << " local=" << entry->from_local; + } + if (entry->from_query) { + sb << " query=" << entry->from_query; + } + sb << "}"; + + if (!small_cache_.empty()) { + sb << " small_cache_size=" << small_cache_.size(); + } + if (!cache_.empty()) { + sb << " cache_size=" << cache_.size(); + } + }; + entry->done = true; + CHECK(entry->blocks.size() == entry->result.size()); alarm_timestamp().relax(entry->timeout = td::Timestamp::in(CACHE_TTL)); for (auto& p : entry->promises) { p.first.set_result(entry->result); @@ -441,8 +495,7 @@ bool OutMsgQueueImporter::check_timeout(std::shared_ptr entry) { } void OutMsgQueueImporter::alarm() { - auto it = cache_.begin(); - while (it != cache_.end()) { + for (auto it = cache_.begin(); it != cache_.end();) { auto& promises = it->second->promises; if (it->second->timeout.is_in_past()) { if (!it->second->done) { @@ -469,6 +522,36 @@ void OutMsgQueueImporter::alarm() { promises.resize(j); ++it; } + + for (auto it = small_cache_.begin(); it != small_cache_.end();) { + td::remove_if(it->second.pending_entries, + [](const std::shared_ptr& entry) { return entry->done || entry->promises.empty(); }); + if (it->second.timeout.is_in_past()) { + if (it->second.pending_entries.empty()) { + it = small_cache_.erase(it); + } else { + ++it; + } + } else { + alarm_timestamp().relax(it->second.timeout); + ++it; + } + } +} + +void OutMsgQueueImporter::add_out_msg_queue_proof(ShardIdFull dst_shard, td::Ref proof) { + LOG(INFO) << "add out msg queue proof " << dst_shard.to_str() << proof->block_id_.to_str(); + auto& small_entry = small_cache_[std::make_pair(dst_shard, proof->block_id_)]; + if (!small_entry.result.is_null()) { + return; + } + alarm_timestamp().relax(small_entry.timeout = td::Timestamp::in(CACHE_TTL)); + small_entry.result = proof; + CHECK(proof.not_null()); + auto pending_entries = std::move(small_entry.pending_entries); + for (auto& entry : pending_entries) { + got_proof(entry, {proof}, ProofSource::Broadcast); + } } void BuildOutMsgQueueProof::abort_query(td::Status reason) { diff --git a/validator/impl/out-msg-queue-proof.hpp b/validator/impl/out-msg-queue-proof.hpp index c115a2763..6f4c08d1d 100644 --- a/validator/impl/out-msg-queue-proof.hpp +++ b/validator/impl/out-msg-queue-proof.hpp @@ -41,6 +41,7 @@ class OutMsgQueueImporter : public td::actor::Actor { void new_masterchain_block_notification(td::Ref state, std::set collating_shards); void get_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, td::Promise>> promise); + void add_out_msg_queue_proof(ShardIdFull dst_shard, td::Ref proof); void update_options(td::Ref opts) { opts_ = std::move(opts); @@ -62,13 +63,28 @@ class OutMsgQueueImporter : public td::actor::Actor { td::Timestamp timeout = td::Timestamp::never(); td::Timer timer; size_t pending = 0; + size_t from_small_cache = 0; + size_t from_broadcast = 0; + size_t from_query = 0; + size_t from_local = 0; }; std::map>, std::shared_ptr> cache_; + // This cache has smaller granularity, proof is stored for each block separately + struct SmallCacheEntry { + td::Ref result; + std::vector> pending_entries; + td::Timestamp timeout = td::Timestamp::never(); + }; + std::map, SmallCacheEntry> small_cache_; + void get_proof_local(std::shared_ptr entry, BlockIdExt block); void get_proof_import(std::shared_ptr entry, std::vector blocks, block::ImportedMsgQueueLimits limits); - void got_proof(std::shared_ptr entry, std::vector> proofs); + enum class ProofSource { + SmallCache, Broadcast, Query, Local + }; + void got_proof(std::shared_ptr entry, std::vector> proofs, ProofSource proof_source); void finish_query(std::shared_ptr entry); bool check_timeout(std::shared_ptr entry); diff --git a/validator/manager.cpp b/validator/manager.cpp index d4ea68101..3ff33d234 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1439,6 +1439,11 @@ void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate can if (!id.is_masterchain()) { add_cached_block_candidate(ReceivedBlock{id, candidate.data.clone()}); } + LOG(INFO) << "Got candidate " << id.to_str() << " with " << candidate.out_msg_queue_proof_broadcasts.size() + << " out msg queue proof broadcasts"; + for (auto broadcast : candidate.out_msg_queue_proof_broadcasts) { + callback_->send_out_msg_queue_proof_broadcast(broadcast); + } td::actor::send_closure(db_, &Db::store_block_candidate, std::move(candidate), std::move(promise)); } @@ -3519,7 +3524,7 @@ void ValidatorManagerImpl::del_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh } else { td::actor::send_closure(it->second.actor, &CollatorNode::del_shard, shard); } -} +}; void ValidatorManagerImpl::get_collation_manager_stats( td::Promise> promise) { @@ -3569,6 +3574,16 @@ void ValidatorManagerImpl::get_collation_manager_stats( td::actor::send_closure(callback, &Cb::dec_pending); } +void ValidatorManagerImpl::add_out_msg_queue_proof(ShardIdFull dst_shard, td::Ref proof) { + if (!collator_nodes_.empty()) { + if (out_msg_queue_importer_.empty()) { + out_msg_queue_importer_ = td::actor::create_actor("outmsgqueueimporter", actor_id(this), + opts_, last_masterchain_state_); + } + td::actor::send_closure(out_msg_queue_importer_, &OutMsgQueueImporter::add_out_msg_queue_proof, dst_shard, + std::move(proof)); + } +} void ValidatorManagerImpl::add_persistent_state_description(td::Ref desc) { auto now = (UnixTime)td::Clocks::system(); if (desc->end_time <= now) { diff --git a/validator/manager.hpp b/validator/manager.hpp index c0d74456f..649d51014 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -640,6 +640,7 @@ class ValidatorManagerImpl : public ValidatorManager { void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; + void add_out_msg_queue_proof(ShardIdFull dst_shard, td::Ref proof) override; void get_collation_manager_stats( td::Promise> promise) override; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index f95266b33..344f5c1c3 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -66,8 +66,14 @@ void ValidatorGroup::generate_block_candidate( std::move(R)); }; td::uint64 max_answer_size = config_.max_block_size + config_.max_collated_data_size + 1024; + auto block_candidate_priority = BlockCandidatePriority{ + .round = source_info.round, + .first_block_round = source_info.first_block_round, + .priority = source_info.source_priority + }; td::actor::send_closure(collation_manager_, &CollationManager::collate_block, shard_, min_masterchain_block_id_, - prev_block_ids_, Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, validator_set_, + prev_block_ids_, Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, + block_candidate_priority, validator_set_, max_answer_size, cancellation_token_source_.get_cancellation_token(), std::move(P)); } diff --git a/validator/validator.h b/validator/validator.h index 2cd771021..c4a779cfd 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -198,6 +198,9 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::BufferSlice data) = 0; virtual void send_broadcast(BlockBroadcast broadcast, int mode) = 0; + virtual void send_out_msg_queue_proof_broadcast(td::Ref broadcats) { + LOG(ERROR) << "Unimplemented send_out_msg_queue_proof_broadcast - ignore broadcast"; + } virtual void download_block(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; virtual void download_zero_state(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, @@ -325,6 +328,10 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; virtual void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; + virtual void add_out_msg_queue_proof(ShardIdFull dst_shard, td::Ref proof) { + LOG(ERROR) << "Unimplemented add_out_msg_queu_proof - ignore broadcast"; + } + virtual void get_collation_manager_stats( td::Promise> promise) = 0; }; From 5d79855c94711ad161b41b75c18edff5396be842 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 27 Nov 2024 18:12:23 +0300 Subject: [PATCH 126/388] Out msg queues: improve logs, various small changes --- ton/ton-types.h | 15 ++--- validator-session/validator-session-types.h | 3 +- validator-session/validator-session.cpp | 18 ++--- validator/full-node-fast-sync-overlays.cpp | 13 ++-- validator/full-node-shard.cpp | 9 ++- validator/impl/collator.cpp | 18 +++-- validator/impl/out-msg-queue-proof.cpp | 75 +++++++++++---------- validator/manager.cpp | 16 +++-- validator/validator-group.cpp | 18 ++--- 9 files changed, 106 insertions(+), 79 deletions(-) diff --git a/ton/ton-types.h b/ton/ton-types.h index aeb0595ad..f8eb49df1 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -428,15 +428,14 @@ struct Ed25519_PublicKey { struct OutMsgQueueProofBroadcast : public td::CntObject { OutMsgQueueProofBroadcast(ShardIdFull dst_shard, BlockIdExt block_id, td::int32 max_bytes, td::int32 max_msgs, - td::BufferSlice queue_proofs, td::BufferSlice block_state_proofs, - std::vector msg_counts) + td::BufferSlice queue_proof, td::BufferSlice block_state_proof, int msg_count) : dst_shard(std::move(dst_shard)) , block_id(block_id) , max_bytes(max_bytes) , max_msgs(max_msgs) - , queue_proofs(std::move(queue_proofs)) - , block_state_proofs(std::move(block_state_proofs)) - , msg_counts(std::move(msg_counts)) { + , queue_proofs(std::move(queue_proof)) + , block_state_proofs(std::move(block_state_proof)) + , msg_count(std::move(msg_count)) { } ShardIdFull dst_shard; BlockIdExt block_id; @@ -448,11 +447,11 @@ struct OutMsgQueueProofBroadcast : public td::CntObject { // outMsgQueueProof td::BufferSlice queue_proofs; td::BufferSlice block_state_proofs; - std::vector msg_counts; + int msg_count; - virtual OutMsgQueueProofBroadcast* make_copy() const { + OutMsgQueueProofBroadcast* make_copy() const override { return new OutMsgQueueProofBroadcast(dst_shard, block_id, max_bytes, max_msgs, queue_proofs.clone(), - block_state_proofs.clone(), msg_counts); + block_state_proofs.clone(), msg_count); } }; diff --git a/validator-session/validator-session-types.h b/validator-session/validator-session-types.h index 7147bf2d3..2edc42bbd 100644 --- a/validator-session/validator-session-types.h +++ b/validator-session/validator-session-types.h @@ -179,9 +179,8 @@ struct EndValidatorGroupStats { }; struct BlockSourceInfo { - td::uint32 round, first_block_round; PublicKey source; - td::int32 source_priority; + BlockCandidatePriority priority; }; } // namespace validatorsession diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 3a913990c..99ee61f23 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -553,9 +553,9 @@ void ValidatorSessionImpl::check_generate_slot() { LOG(WARNING) << print_id << ": failed to generate block candidate: " << R.move_as_error(); } }); - callback_->on_generate_slot( - BlockSourceInfo{cur_round_, first_block_round_, description().get_source_public_key(local_idx()), priority}, - std::move(P)); + callback_->on_generate_slot(BlockSourceInfo{description().get_source_public_key(local_idx()), + BlockCandidatePriority{cur_round_, first_block_round_, priority}}, + std::move(P)); } else { alarm_timestamp().relax(t); } @@ -634,8 +634,9 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { pending_approve_.insert(block_id); callback_->on_candidate( - BlockSourceInfo{cur_round_, first_block_round_, description().get_source_public_key(block->get_src_idx()), - description().get_node_priority(block->get_src_idx(), cur_round_)}, + BlockSourceInfo{description().get_source_public_key(block->get_src_idx()), + BlockCandidatePriority{cur_round_, first_block_round_, + description().get_node_priority(block->get_src_idx(), cur_round_)}}, B->root_hash_, B->data_.clone(), B->collated_data_.clone(), std::move(P)); } else if (T.is_in_past()) { if (!active_requests_.count(block_id)) { @@ -909,9 +910,10 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) { stats.rounds.pop_back(); } - BlockSourceInfo source_info{cur_round_, first_block_round_, - description().get_source_public_key(block->get_src_idx()), - description().get_node_priority(block->get_src_idx(), cur_round_)}; + BlockSourceInfo source_info{ + description().get_source_public_key(block->get_src_idx()), + BlockCandidatePriority{cur_round_, first_block_round_, + description().get_node_priority(block->get_src_idx(), cur_round_)}}; if (it == blocks_.end()) { callback_->on_block_committed(std::move(source_info), block->get_root_hash(), block->get_file_hash(), td::BufferSlice(), std::move(export_sigs), std::move(export_approve_sigs), diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index 4e0d11e48..947f9dee8 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -47,6 +47,9 @@ void FullNodeFastSyncOverlay::process_block_broadcast(PublicKeyHash src, ton_api } void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_outMsgQueueProofBroadcast &query) { + if (src == local_id_.pubkey_hash()) { + return; // dropping broadcast from self + } BlockIdExt block_id = create_block_id(query.block_); ShardIdFull shard_id = create_shard_id(query.dst_shard_); if (query.proof_->get_id() != ton_api::tonNode_outMsgQueueProof::ID) { @@ -68,7 +71,8 @@ void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonN } auto proof = std::move(R.move_as_ok()[0]); - LOG(INFO) << "got tonNode.outMsgQueueProofBroadcast " << shard_id.to_str() << " " << block_id.to_str(); + LOG(INFO) << "got tonNode.outMsgQueueProofBroadcast to " << shard_id.to_str() << " from " << block_id.to_str() + << ", msgs=" << proof->msg_count_ << ", size=" << tl_proof->queue_proofs_.size(); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::add_out_msg_queue_proof, shard_id, std::move(proof)); } @@ -236,9 +240,10 @@ void FullNodeFastSyncOverlay::send_out_msg_queue_proof_broadcast(td::Ref(broadcast->max_bytes, broadcast->max_msgs), create_tl_object(broadcast->queue_proofs.clone(), broadcast->block_state_proofs.clone(), - std::vector(broadcast->msg_counts))); - VLOG(FULL_NODE_DEBUG) << "Sending outMsgQueueProof in fast sync overlay: " << broadcast->dst_shard.to_str() << " " - << broadcast->block_id.to_str(); + std::vector(1, broadcast->msg_count))); + VLOG(FULL_NODE_DEBUG) << "Sending outMsgQueueProof in fast sync overlay to " << broadcast->dst_shard.to_str() + << " from " << broadcast->block_id.to_str() << ", msgs=" << broadcast->msg_count + << " bytes=" << broadcast->queue_proofs.size(); td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); } diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 29950363d..c351653f4 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -703,8 +703,13 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod promise.set_result(serialize_tl_object(R.move_as_ok(), true)); } }); - VLOG(FULL_NODE_DEBUG) << "Got query getOutMsgQueueProof (" << blocks.size() << " blocks) to shard " - << dst_shard.to_str() << " from " << src; + FLOG(DEBUG) { + sb << "Got query getOutMsgQueueProof to shard " << dst_shard.to_str() << " from blocks"; + for (const BlockIdExt &id : blocks) { + sb << " " << id.id.to_str(); + } + sb << " from " << src; + }; td::actor::create_actor("buildqueueproof", dst_shard, std::move(blocks), limits, validator_manager_, std::move(P)) .release(); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index cd97e3fcc..dd8d55169 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -5854,33 +5854,39 @@ bool Collator::create_block_candidate() { block_candidate = std::make_unique(created_by_, new_block_id_ext, block::compute_file_hash(cdata_slice.as_slice()), blk_slice.clone(), cdata_slice.clone()); - const bool need_out_msg_queue_broadcasts = true; + const bool need_out_msg_queue_broadcasts = !is_masterchain(); if (need_out_msg_queue_broadcasts) { // we can't generate two proofs at the same time for the same root (it is not currently supported by cells) // so we have can't reuse new state and have to regenerate it with merkle update auto new_state = vm::MerkleUpdate::apply(prev_state_root_pure_, state_update); CHECK(new_state.not_null()); CHECK(new_state->get_hash() == state_root->get_hash()); - assert(config_ && shard_conf_); + CHECK(shard_conf_); auto neighbor_list = shard_conf_->get_neighbor_shard_hash_ids(shard_); LOG(INFO) << "Build OutMsgQueueProofs for " << neighbor_list.size() << " neighbours"; - for (ton::BlockId blk_id : neighbor_list) { + for (BlockId blk_id : neighbor_list) { auto prefix = blk_id.shard_full(); + if (shard_intersects(prefix, shard_)) { + continue; + } auto limits = mc_state_->get_imported_msg_queue_limits(blk_id.workchain); // one could use monitor_min_split_depth here, to decrease number of broadcasts // but current implementation OutMsgQueueImporter doesn't support it auto r_proof = OutMsgQueueProof::build( - prefix, {OutMsgQueueProof::OneBlock{.id = new_block_id_ext, .state_root = new_state, .block_root = new_block}}, limits); + prefix, + {OutMsgQueueProof::OneBlock{.id = new_block_id_ext, .state_root = new_state, .block_root = new_block}}, + limits); if (r_proof.is_ok()) { auto proof = r_proof.move_as_ok(); + CHECK(proof->msg_counts_.size() == 1); block_candidate->out_msg_queue_proof_broadcasts.push_back(td::Ref( true, OutMsgQueueProofBroadcast(prefix, new_block_id_ext, limits.max_bytes, limits.max_msgs, std::move(proof->queue_proofs_), std::move(proof->block_state_proofs_), - std::move(proof->msg_counts_)))); + proof->msg_counts_[0]))); } else { - LOG(ERROR) << "Failed to build OutMsgQueueProof: " << r_proof.error(); + LOG(ERROR) << "Failed to build OutMsgQueueProof to " << prefix.to_str() << ": " << r_proof.error(); } } } diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index 4e8802c7e..edd20a8d2 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -92,29 +92,6 @@ static td::Result> process_queue( ++msg_count[kv->source]; ++msg_count_total; - // TODO: Get processed_upto from destination shard (in request?) - /* - // Parse message to check if it was processed (as in Collator::process_inbound_message) - ton::LogicalTime enqueued_lt = kv->msg->prefetch_ulong(64); - auto msg_env = kv->msg->prefetch_ref(); - block::tlb::MsgEnvelope::Record_std env; - if (!tlb::unpack_cell(msg_env, env)) { - return td::Status::Error("cannot unpack MsgEnvelope of an internal message"); - } - vm::CellSlice cs{vm::NoVmOrd{}, env.msg}; - block::gen::CommonMsgInfo::Record_int_msg_info info; - if (!tlb::unpack(cs, info)) { - return td::Status::Error("cannot unpack CommonMsgInfo of an internal message"); - } - auto src_prefix = block::tlb::MsgAddressInt::get_prefix(info.src); - auto dest_prefix = block::tlb::MsgAddressInt::get_prefix(info.dest); - auto cur_prefix = block::interpolate_addr(src_prefix, dest_prefix, env.cur_addr); - auto next_prefix = block::interpolate_addr(src_prefix, dest_prefix, env.next_addr); - block::EnqueuedMsgDescr descr{cur_prefix, next_prefix, kv->lt, enqueued_lt, env.msg->get_hash().bits()}; - if (dst_processed_upto->already_processed(descr)) { - } else { - }*/ - dfs_cs(*kv->msg); TRY_STATUS_PREFIX(check_no_prunned(*kv->msg), "invalid message proof: ") if (estimated_proof_size >= limits.max_bytes || msg_count_total >= (long long)limits.max_msgs) { @@ -301,7 +278,12 @@ void OutMsgQueueImporter::get_neighbor_msg_queue_proofs( return; } - LOG(DEBUG) << "Importing neighbor msg queues for shard " << dst_shard.to_str() << ", " << blocks.size() << " blocks"; + FLOG(DEBUG) { + sb << "Importing neighbor msg queues for shard " << dst_shard.to_str() << ", " << blocks.size() << " blocks:"; + for (const BlockIdExt& block : blocks) { + sb << " " << block.id.to_str(); + } + }; cache_[{dst_shard, blocks}] = entry = std::make_shared(); entry->dst_shard = dst_shard; @@ -321,7 +303,7 @@ void OutMsgQueueImporter::get_neighbor_msg_queue_proofs( prefix = shard_prefix(prefix, min_split); } - LOG(INFO) << "search for out msg queue proof " << prefix.to_str() << block.to_str(); + LOG(DEBUG) << "search for out msg queue proof " << prefix.to_str() << " " << block.to_str(); auto& small_entry = small_cache_[std::make_pair(dst_shard, block)]; if (!small_entry.result.is_null()) { entry->result[block] = small_entry.result; @@ -397,7 +379,13 @@ void OutMsgQueueImporter::get_proof_import(std::shared_ptr entry, st [=, SelfId = actor_id(this), retry_after = td::Timestamp::in(0.1), dst_shard = entry->dst_shard](td::Result>> R) { if (R.is_error()) { - LOG(DEBUG) << "Failed to get out msg queue for " << dst_shard.to_str() << ": " << R.move_as_error(); + FLOG(DEBUG) { + sb << "Failed to get out msg queue for " << dst_shard.to_str() << " from"; + for (const BlockIdExt &block : blocks) { + sb << " " << block.id.to_str(); + } + sb << ": " << R.move_as_error(); + }; delay_action( [=]() { td::actor::send_closure(SelfId, &OutMsgQueueImporter::get_proof_import, entry, std::move(blocks), @@ -443,8 +431,11 @@ void OutMsgQueueImporter::got_proof(std::shared_ptr entry, std::vect void OutMsgQueueImporter::finish_query(std::shared_ptr entry) { FLOG(INFO) { - sb << "Done importing neighbor msg queues for shard " << entry->dst_shard.to_str() << ", " << entry->blocks.size() - << " blocks in " << entry->timer.elapsed() << "s"; + sb << "Done importing neighbor msg queues for shard " << entry->dst_shard.to_str() << " from"; + for (const BlockIdExt &block : entry->blocks) { + sb << " " << block.id.to_str(); + } + sb << " in " << entry->timer.elapsed() << "s"; sb << " sources{"; if (entry->from_broadcast) { sb << " broadcast=" << entry->from_broadcast; @@ -479,8 +470,13 @@ void OutMsgQueueImporter::finish_query(std::shared_ptr entry) { bool OutMsgQueueImporter::check_timeout(std::shared_ptr entry) { if (entry->timeout.is_in_past()) { - LOG(DEBUG) << "Aborting importing neighbor msg queues for shard " << entry->dst_shard.to_str() << ", " - << entry->blocks.size() << " blocks: timeout"; + FLOG(DEBUG) { + sb << "Aborting importing neighbor msg queues for shard " << entry->dst_shard.to_str() << " from"; + for (const BlockIdExt &block : entry->blocks) { + sb << " " << block.id.to_str(); + } + sb << ": timeout"; + }; for (auto& p : entry->promises) { p.first.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); } @@ -499,8 +495,13 @@ void OutMsgQueueImporter::alarm() { auto& promises = it->second->promises; if (it->second->timeout.is_in_past()) { if (!it->second->done) { - LOG(DEBUG) << "Aborting importing neighbor msg queues for shard " << it->second->dst_shard.to_str() << ", " - << it->second->blocks.size() << " blocks: timeout"; + FLOG(DEBUG) { + sb << "Aborting importing neighbor msg queues for shard " << it->second->dst_shard.to_str() << " from"; + for (const BlockIdExt &block : it->second->blocks) { + sb << " " << block.id.to_str(); + } + sb << ": timeout"; + }; for (auto& p : promises) { p.first.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); } @@ -540,7 +541,7 @@ void OutMsgQueueImporter::alarm() { } void OutMsgQueueImporter::add_out_msg_queue_proof(ShardIdFull dst_shard, td::Ref proof) { - LOG(INFO) << "add out msg queue proof " << dst_shard.to_str() << proof->block_id_.to_str(); + LOG(INFO) << "add out msg queue proof " << dst_shard.to_str() << " " << proof->block_id_.to_str(); auto& small_entry = small_cache_[std::make_pair(dst_shard, proof->block_id_)]; if (!small_entry.result.is_null()) { return; @@ -556,7 +557,13 @@ void OutMsgQueueImporter::add_out_msg_queue_proof(ShardIdFull dst_shard, td::Ref void BuildOutMsgQueueProof::abort_query(td::Status reason) { if (promise_) { - LOG(DEBUG) << "failed to build msg queue proof to " << dst_shard_.to_str() << ": " << reason; + FLOG(DEBUG) { + sb << "failed to build msg queue proof to " << dst_shard_.to_str() << " from"; + for (const auto& block : blocks_) { + sb << " " << block.id.id.to_str(); + } + sb << ": " << reason; + }; promise_.set_error( reason.move_as_error_prefix(PSTRING() << "failed to build msg queue proof to " << dst_shard_.to_str() << ": ")); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 3ff33d234..903e31554 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -798,7 +798,9 @@ void ValidatorManagerImpl::wait_neighbor_msg_queue_proofs( if (dst_shard.is_masterchain()) { // We spit queries for masterchain {dst_shard, {block_1, ..., block_n}} into separate queries // {dst_shard, {block_1}}, ..., {dst_shard, {block_n}} - // Also, use cache + // Also, use cache. + // This is performed here and not in OutMsgQueueImporter because it's important to use + // cached_msg_queue_to_masterchain_, which is related to the current list of shard block descriptions class Worker : public td::actor::Actor { public: Worker(size_t pending, td::Promise>> promise) @@ -2958,12 +2960,15 @@ PublicKeyHash ValidatorManagerImpl::get_validator(ShardIdFull shard, td::Refget_collators_list()->self_collate; } bool ValidatorManagerImpl::Collator::can_collate_shard(ShardIdFull shard) const { @@ -3524,7 +3529,7 @@ void ValidatorManagerImpl::del_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh } else { td::actor::send_closure(it->second.actor, &CollatorNode::del_shard, shard); } -}; +} void ValidatorManagerImpl::get_collation_manager_stats( td::Promise> promise) { @@ -3575,15 +3580,18 @@ void ValidatorManagerImpl::get_collation_manager_stats( } void ValidatorManagerImpl::add_out_msg_queue_proof(ShardIdFull dst_shard, td::Ref proof) { - if (!collator_nodes_.empty()) { + if (is_shard_collator(dst_shard)) { if (out_msg_queue_importer_.empty()) { out_msg_queue_importer_ = td::actor::create_actor("outmsgqueueimporter", actor_id(this), opts_, last_masterchain_state_); } td::actor::send_closure(out_msg_queue_importer_, &OutMsgQueueImporter::add_out_msg_queue_proof, dst_shard, std::move(proof)); + } else { + VLOG(VALIDATOR_DEBUG) << "Dropping unneeded out msg queue proof to shard " << dst_shard.to_str(); } } + void ValidatorManagerImpl::add_persistent_state_description(td::Ref desc) { auto now = (UnixTime)td::Clocks::system(); if (desc->end_time <= now) { diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 344f5c1c3..97f34ea7c 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -32,13 +32,14 @@ namespace ton { namespace validator { static bool need_send_candidate_broadcast(const validatorsession::BlockSourceInfo &source_info, bool is_masterchain) { - return source_info.first_block_round == source_info.round && source_info.source_priority == 0 && !is_masterchain; + return source_info.priority.first_block_round == source_info.priority.round && source_info.priority.priority == 0 && + !is_masterchain; } void ValidatorGroup::generate_block_candidate( validatorsession::BlockSourceInfo source_info, td::Promise promise) { - td::uint32 round_id = source_info.round; + td::uint32 round_id = source_info.priority.round; if (round_id > last_known_round_id_) { last_known_round_id_ = round_id; } @@ -66,15 +67,10 @@ void ValidatorGroup::generate_block_candidate( std::move(R)); }; td::uint64 max_answer_size = config_.max_block_size + config_.max_collated_data_size + 1024; - auto block_candidate_priority = BlockCandidatePriority{ - .round = source_info.round, - .first_block_round = source_info.first_block_round, - .priority = source_info.source_priority - }; td::actor::send_closure(collation_manager_, &CollationManager::collate_block, shard_, min_masterchain_block_id_, prev_block_ids_, Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, - block_candidate_priority, validator_set_, - max_answer_size, cancellation_token_source_.get_cancellation_token(), std::move(P)); + source_info.priority, validator_set_, max_answer_size, + cancellation_token_source_.get_cancellation_token(), std::move(P)); } void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo source_info, @@ -103,7 +99,7 @@ void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo void ValidatorGroup::validate_block_candidate(validatorsession::BlockSourceInfo source_info, BlockCandidate block, td::Promise> promise) { - td::uint32 round_id = source_info.round; + td::uint32 round_id = source_info.priority.round; if (round_id > last_known_round_id_) { last_known_round_id_ = round_id; } @@ -174,7 +170,7 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so validatorsession::ValidatorSessionStats stats, td::Promise promise) { stats.cc_seqno = validator_set_->get_catchain_seqno(); - td::uint32 round_id = source_info.round; + td::uint32 round_id = source_info.priority.round; if (round_id >= last_known_round_id_) { last_known_round_id_ = round_id + 1; } From 5fae8db7a03d9d16b8bbaedd16a3d70202d22c28 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 28 Nov 2024 13:09:40 +0300 Subject: [PATCH 127/388] Adapt "get msg queue sizes" in lite-client and tonlib to non-full liteservers --- lite-client/lite-client.cpp | 90 +++++++++++++++--- lite-client/lite-client.h | 3 +- tonlib/tonlib/TonlibClient.cpp | 167 ++++++++++++++++++++++++++++++--- tonlib/tonlib/tonlib-cli.cpp | 19 ++++ 4 files changed, 253 insertions(+), 26 deletions(-) diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index dc09ae52b..ce3dd4b20 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -1627,27 +1627,93 @@ void TestNode::send_compute_complaint_price_query(ton::StdSmcAddress elector_add } bool TestNode::get_msg_queue_sizes() { - auto q = ton::serialize_tl_object(ton::create_tl_object(0, 0, 0), true); - return envelope_send_query(std::move(q), [Self = actor_id(this)](td::Result res) -> void { - if (res.is_error()) { - LOG(ERROR) << "liteServer.getOutMsgQueueSizes error: " << res.move_as_error(); + ton::BlockIdExt blkid = mc_last_id_; + if (!blkid.is_valid_full()) { + return set_error("must obtain last block information before making other queries"); + } + if (!(ready_ && !client_.empty())) { + return set_error("server connection not ready"); + } + auto b = + ton::create_serialize_tl_object(ton::create_tl_lite_block_id(blkid)); + LOG(INFO) << "requesting recent shard configuration"; + return envelope_send_query(std::move(b), [Self = actor_id(this), blkid](td::Result R) -> void { + if (R.is_error()) { return; } - auto F = ton::fetch_tl_object(res.move_as_ok(), true); + auto F = ton::fetch_tl_object(R.move_as_ok(), true); if (F.is_error()) { - LOG(ERROR) << "cannot parse answer to liteServer.getOutMsgQueueSizes"; - return; + LOG(ERROR) << "cannot parse answer to liteServer.getAllShardsInfo"; + } else { + auto f = F.move_as_ok(); + td::actor::send_closure_later(Self, &TestNode::get_msg_queue_sizes_cont, blkid, std::move(f->data_)); } - td::actor::send_closure_later(Self, &TestNode::got_msg_queue_sizes, F.move_as_ok()); }); } -void TestNode::got_msg_queue_sizes(ton::tl_object_ptr f) { +void TestNode::get_msg_queue_sizes_cont(ton::BlockIdExt mc_blkid, td::BufferSlice data) { + LOG(INFO) << "got shard configuration with respect to block " << mc_blkid.to_str(); + std::vector blocks; + blocks.push_back(mc_blkid); + auto R = vm::std_boc_deserialize(data.clone()); + if (R.is_error()) { + set_error(R.move_as_error_prefix("cannot deserialize shard configuration: ")); + return; + } + auto root = R.move_as_ok(); + block::ShardConfig sh_conf; + if (!sh_conf.unpack(vm::load_cell_slice_ref(root))) { + set_error("cannot extract shard block list from shard configuration"); + return; + } + auto ids = sh_conf.get_shard_hash_ids(true); + for (auto id : ids) { + auto ref = sh_conf.get_shard_hash(ton::ShardIdFull(id)); + if (ref.not_null()) { + blocks.push_back(ref->top_block_id()); + } + } + + struct QueryInfo { + std::vector blocks; + std::vector sizes; + size_t pending; + }; + auto info = std::make_shared(); + info->blocks = std::move(blocks); + info->sizes.resize(info->blocks.size(), 0); + info->pending = info->blocks.size(); + + for (size_t i = 0; i < info->blocks.size(); ++i) { + ton::BlockIdExt block_id = info->blocks[i]; + auto b = ton::create_serialize_tl_object( + 0, ton::create_tl_lite_block_id(block_id), false); + LOG(DEBUG) << "requesting queue size for block " << block_id.to_str(); + envelope_send_query(std::move(b), [=, this](td::Result R) -> void { + if (R.is_error()) { + return; + } + auto F = ton::fetch_tl_object(R.move_as_ok(), true); + if (F.is_error()) { + set_error(F.move_as_error_prefix("failed to get queue size: ")); + return; + } + auto f = F.move_as_ok(); + LOG(DEBUG) << "got queue size for block " << block_id.to_str() << " : " << f->size_; + info->sizes[i] = f->size_; + if (--info->pending == 0) { + get_msg_queue_sizes_finish(std::move(info->blocks), std::move(info->sizes)); + } + }); + } +} + +void TestNode::get_msg_queue_sizes_finish(std::vector blocks, std::vector sizes) { + CHECK(blocks.size() == sizes.size()); td::TerminalIO::out() << "Outbound message queue sizes:" << std::endl; - for (auto &x : f->shards_) { - td::TerminalIO::out() << ton::create_block_id(x->id_).id.to_str() << " " << x->size_ << std::endl; + for (size_t i = 0; i < blocks.size(); ++i) { + td::TerminalIO::out() << blocks[i].id.to_str() << " " << sizes[i] << std::endl; } - td::TerminalIO::out() << "External message queue size limit: " << f->ext_msg_queue_size_limit_ << std::endl; } bool TestNode::get_dispatch_queue_info(ton::BlockIdExt block_id) { diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index 721d2b20d..57804418f 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -324,7 +324,8 @@ class TestNode : public td::actor::Actor { void send_compute_complaint_price_query(ton::StdSmcAddress elector_addr, unsigned expires_in, unsigned bits, unsigned refs, td::Bits256 chash, std::string filename); bool get_msg_queue_sizes(); - void got_msg_queue_sizes(ton::tl_object_ptr f); + void get_msg_queue_sizes_cont(ton::BlockIdExt mc_blkid, td::BufferSlice data); + void get_msg_queue_sizes_finish(std::vector blocks, std::vector sizes); bool get_dispatch_queue_info(ton::BlockIdExt block_id); bool get_dispatch_queue_info_cont(ton::BlockIdExt block_id, bool first, td::Bits256 after_addr); void got_dispatch_queue_info(ton::BlockIdExt block_id, diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index b9ff4899c..73cb2a16a 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -1800,6 +1800,132 @@ class GetShardBlockProof : public td::actor::Actor { std::vector> links_; }; +class GetOutMsgQueueSizes : public td::actor::Actor { + public: + GetOutMsgQueueSizes(ExtClientRef ext_client_ref, std::vector blocks, + td::actor::ActorShared<> parent, + td::Promise>&& promise) + : blocks_(std::move(blocks)), parent_(std::move(parent)), promise_(std::move(promise)) { + client_.set_client(ext_client_ref); + } + + void start_up() override { + sizes_.resize(blocks_.size()); + pending_ = blocks_.size() + 1; + + for (size_t i = 0; i < blocks_.size(); ++i) { + client_.send_query( + ton::lite_api::liteServer_getBlockOutMsgQueueSize(1, ton::create_tl_lite_block_id(blocks_[i]), true), + [SelfId = actor_id(this), i](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &GetOutMsgQueueSizes::abort, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &GetOutMsgQueueSizes::got_block_queue_size, i, R.move_as_ok()); + } + }); + } + + client_.send_query( + ton::lite_api::liteServer_getOutMsgQueueSizes(1, ton::masterchainId, ton::shardIdAll), + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &GetOutMsgQueueSizes::abort, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &GetOutMsgQueueSizes::got_ext_msg_queue_size_limit, + R.ok()->ext_msg_queue_size_limit_); + } + }); + } + + void got_block_queue_size(size_t i, lite_api_ptr f) { + try { + auto S = [&, this]() -> td::Status { + TRY_RESULT_PREFIX(roots, vm::std_boc_deserialize_multi(f->proof_), "cannot deserialize proof: "); + if (roots.size() != 2) { + return td::Status::Error("expected 2 roots in proof"); + } + auto state_root = vm::MerkleProof::virtualize(std::move(roots[1]), 1); + if (state_root.is_null()) { + return td::Status::Error("state proof is invalid"); + } + ton::Bits256 state_hash = state_root->get_hash().bits(); + TRY_STATUS_PREFIX(block::check_block_header_proof(vm::MerkleProof::virtualize(std::move(roots[0]), 1), + blocks_[i], &state_hash, true, nullptr, nullptr), + "error in block header proof: "); + + block::gen::ShardStateUnsplit::Record sstate; + block::gen::OutMsgQueueInfo::Record out_msg_queue_info; + if (!tlb::unpack_cell(state_root, sstate) || !tlb::unpack_cell(sstate.out_msg_queue_info, out_msg_queue_info)) { + return td::Status::Error("cannot unpack shard state"); + } + vm::CellSlice& extra_slice = out_msg_queue_info.extra.write(); + if (extra_slice.fetch_long(1) == 0) { + return td::Status::Error("no out_msg_queue_size in shard state"); + } + block::gen::OutMsgQueueExtra::Record out_msg_queue_extra; + if (!tlb::unpack(extra_slice, out_msg_queue_extra)) { + return td::Status::Error("cannot unpack OutMsgQueueExtra"); + } + vm::CellSlice& size_slice = out_msg_queue_extra.out_queue_size.write(); + if (size_slice.fetch_long(1) == 0) { + return td::Status::Error("no out_msg_queue_size in shard state"); + } + td::uint64 size = size_slice.prefetch_ulong(48); + if (size != f->size_) { + return td::Status::Error("queue size mismatch"); + } + return td::Status::OK(); + }(); + if (S.is_error()) { + abort(std::move(S)); + return; + } + } catch (vm::VmError& err) { + abort(err.as_status()); + return; + } catch (vm::VmVirtError& err) { + abort(err.as_status()); + return; + } + + sizes_[i] = f->size_; + dec_pending(); + } + + void got_ext_msg_queue_size_limit(td::uint32 value) { + ext_msg_queue_size_limit_ = value; + dec_pending(); + } + + void dec_pending() { + if (--pending_ == 0) { + std::vector> shards; + for (size_t i = 0; i < blocks_.size(); ++i) { + shards.push_back( + tonlib_api::make_object(to_tonlib_api(blocks_[i]), sizes_[i])); + } + promise_.set_result( + tonlib_api::make_object(std::move(shards), ext_msg_queue_size_limit_)); + stop(); + } + } + + void abort(td::Status error) { + promise_.set_error(std::move(error)); + stop(); + } + + private: + std::vector blocks_; + td::actor::ActorShared<> parent_; + td::Promise> promise_; + ExtClient client_; + + std::vector sizes_; + td::uint32 ext_msg_queue_size_limit_ = 0; + size_t pending_ = 0; +}; + auto to_lite_api(const tonlib_api::ton_blockIdExt& blk) -> td::Result>; auto to_tonlib_api(const ton::lite_api::liteServer_transactionId& txid) -> tonlib_api_ptr; @@ -6129,19 +6255,34 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getShardBlockProof& td::Status TonlibClient::do_request(const tonlib_api::blocks_getOutMsgQueueSizes& request, td::Promise>&& promise) { - client_.send_query(ton::lite_api::liteServer_getOutMsgQueueSizes(request.mode_, request.wc_, request.shard_), - promise.wrap([](lite_api_ptr&& queue_sizes) { - tonlib_api::blocks_outMsgQueueSizes result; - result.ext_msg_queue_size_limit_ = queue_sizes->ext_msg_queue_size_limit_; - for (auto &x : queue_sizes->shards_) { - tonlib_api::blocks_outMsgQueueSize shard; - shard.id_ = to_tonlib_api(*x->id_); - shard.size_ = x->size_; - result.shards_.push_back(tonlib_api::make_object(std::move(shard))); - } - return tonlib_api::make_object(std::move(result)); - })); - + auto req_mode = request.mode_; + auto req_shard = ton::ShardIdFull{request.wc_, (ton::ShardId)request.shard_}; + if ((req_mode & 1) && !req_shard.is_valid_ext()) { + return td::Status::Error("invalid shard"); + } + client_.with_last_block( + [=, self = this, promise = std::move(promise)](td::Result r_last_block) mutable { + TRY_RESULT_PROMISE_PREFIX(promise, last_block, std::move(r_last_block), "get last block failed: "); + do_request(tonlib_api::blocks_getShards(to_tonlib_api(last_block.last_block_id)), + [=, mc_blkid = last_block.last_block_id, + promise = std::move(promise)](td::Result> R) mutable { + TRY_RESULT_PROMISE_PREFIX(promise, shards, std::move(R), "get shards failed: "); + std::vector blocks; + if (!(req_mode & 1) || ton::shard_intersects(mc_blkid.shard_full(), req_shard)) { + blocks.push_back(mc_blkid); + } + for (const auto& shard : shards->shards_) { + TRY_RESULT_PROMISE(promise, block_id, to_block_id(*shard)); + if (!(req_mode & 1) || ton::shard_intersects(block_id.shard_full(), req_shard)) { + blocks.push_back(block_id); + } + } + auto actor_id = self->actor_id_++; + self->actors_[actor_id] = td::actor::create_actor( + "GetOutMsgQueueSizes", self->client_.get_client(), std::move(blocks), + actor_shared(this, actor_id), std::move(promise)); + }); + }); return td::Status::OK(); } diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index 2c7100f24..364b8f663 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -430,6 +430,7 @@ class TonlibCli : public td::actor::Actor { << "\t 'k' modifier - use fake key\n" << "\t 'c' modifier - just esmitate fees\n"; td::TerminalIO::out() << "getmasterchainsignatures - get sigratures of masterchain block \n"; + td::TerminalIO::out() << "msgqueuesizes - get out msg queue sizes in the latest shard states\n"; } else if (cmd == "genkey") { generate_key(); } else if (cmd == "exit" || cmd == "quit") { @@ -517,6 +518,8 @@ class TonlibCli : public td::actor::Actor { } else if (cmd == "getmasterchainsignatures") { auto seqno = parser.read_word(); run_get_masterchain_block_signatures(seqno, std::move(cmd_promise)); + } else if (cmd == "msgqueuesizes") { + run_get_out_msg_queue_sizes(std::move(cmd_promise)); } else if (cmd == "showtransactions") { run_show_transactions(parser, std::move(cmd_promise)); } else { @@ -2161,6 +2164,22 @@ class TonlibCli : public td::actor::Actor { })); } + void run_get_out_msg_queue_sizes(td::Promise promise) { + send_query(make_object(0, 0, 0), + promise.wrap([](tonlib_api::object_ptr&& f) { + td::TerminalIO::out() << "Outbound message queue sizes:" << std::endl; + for (const auto& shard : f->shards_) { + td::TerminalIO::out() << ton::BlockId{shard->id_->workchain_, (ton::ShardId)shard->id_->shard_, + (ton::BlockSeqno)shard->id_->seqno_} + .to_str() + << " " << shard->size_ << std::endl; + } + td::TerminalIO::out() << "External message queue size limit: " << f->ext_msg_queue_size_limit_ + << std::endl; + return td::Unit(); + })); + } + void run_show_transactions(td::ConstParser& parser, td::Promise promise) { TRY_RESULT_PROMISE(promise, address, to_account_address(parser.read_word(), false)); TRY_RESULT_PROMISE(promise, lt, td::to_integer_safe(parser.read_word())); From 923f1cd69b6d31f1e131764f0f9d2a208cfd4d63 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 28 Nov 2024 14:13:49 +0300 Subject: [PATCH 128/388] Improve collator node pings and collation manager stats --- tl/generate/scheme/ton_api.tl | 2 +- tl/generate/scheme/ton_api.tlo | Bin 108888 -> 108972 bytes .../validator-engine-console-query.cpp | 20 +++++++++++++-- validator/collation-manager.cpp | 11 +++++--- validator/collation-manager.hpp | 2 ++ validator/collator-node.cpp | 24 +++++++++++++++++- validator/collator-node.hpp | 4 +++ validator/manager.cpp | 10 ++++++++ 8 files changed, 66 insertions(+), 7 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 720f8d496..4bfd57883 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -750,7 +750,7 @@ engine.validator.perfTimerStats stats:(vector engine.validator.PerfTimerStatsByN engine.validator.shardOutQueueSize size:long = engine.validator.ShardOutQueueSize; engine.validator.collationManagerStats.shard shard_id:tonNode.shardId self_collate:Bool select_mode:string active:Bool collators:(vector int256) = engine.validator.collationManagerStats.Shard; -engine.validator.collationManagerStats.collator adnl_id:int256 active:Bool alive:Bool ping_in:double = engine.validator.collationManagerStats.Collator; +engine.validator.collationManagerStats.collator adnl_id:int256 active:Bool alive:Bool ping_in:double last_ping_ago:double last_ping_status:string = engine.validator.collationManagerStats.Collator; engine.validator.collationManagerStats.localId adnl_id:int256 shards:(vector engine.validator.collationManagerStats.shard) collators:(vector engine.validator.collationManagerStats.collator) = engine.validator.collationManagerStats.LocalId; engine.validator.collationManagerStats local_ids:(vector engine.validator.collationManagerStats.localId) = engine.validator.CollationManagerStats; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 19a8d13138d85dd2bea8b7e28793cbc85e441aa9..d5ebffaa061ab442de15cdea2f087fbaf6254856 100644 GIT binary patch delta 126 zcmca{iEYhgwhcP+ERLK?;+qZRcPt09CL73W2qx#}UB)%XsFFignJ%4gtwzL33q_`xpq_lW);BJTwleZi{ L1F@>*#FYmC=kqRX delta 77 zcmZ2;neE0UwhcP+ELtBY&)IAszhgO=HQ7L3Lohi%CnvEazX&9C^O%qi%jCcl8k2pG WM^B!xhXbOFlast_ping_status_; + std::erase_if(status, [](char c) { return c < (char)32; }); + if (status.size() > 128) { + status.resize(128); + } + sb << td::StringBuilder::FixedDouble(collator->last_ping_ago_, 3) << ": " << status; + } + td::TerminalIO::out() << sb.as_cslice() << "\n"; } } } diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index e84470595..879b8d895 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -261,6 +261,8 @@ void CollationManager::get_stats( } else { obj->ping_in_ = -1.0; } + obj->last_ping_ago_ = collator.last_ping_at ? td::Time::now() - collator.last_ping_at.at() : -1.0; + obj->last_ping_status_ = collator.last_ping_status.is_ok() ? "OK" : collator.last_ping_status.message().str(); stats->collators_.push_back(std::move(obj)); } promise.set_value(std::move(stats)); @@ -323,7 +325,7 @@ void CollationManager::alarm() { td::actor::send_closure(SelfId, &CollationManager::got_pong, id, std::move(R)); }; LOG(DEBUG) << "sending ping to " << id; - td::actor::send_closure(rldp_, &rldp::Rldp::send_query, local_id_, id, "collatorping", std::move(P), + td::actor::send_closure(rldp_, &rldp::Rldp::send_query, local_id_, id, "ping", std::move(P), td::Timestamp::in(2.0), std::move(query)); } else { alarm_timestamp().relax(collator.ping_at); @@ -340,7 +342,7 @@ void CollationManager::got_pong(adnl::AdnlNodeIdShort id, td::Result td::Result> { - TRY_RESULT_PREFIX(data, std::move(R), "rldp query error: "); + TRY_RESULT(data, std::move(R)); auto r_error = fetch_tl_object(data, true); if (r_error.is_ok()) { auto error = r_error.move_as_ok(); @@ -348,12 +350,15 @@ void CollationManager::got_pong(adnl::AdnlNodeIdShort id, td::Result(data, true); }(); + collator.last_ping_at = td::Timestamp::now(); if (r_pong.is_error()) { - LOG(DEBUG) << "pong from " << id << " : " << r_pong.move_as_error(); + LOG(DEBUG) << "pong from " << id << " : " << r_pong.error(); collator.alive = false; + collator.last_ping_status = r_pong.move_as_error(); } else { LOG(DEBUG) << "pong from " << id << " : OK"; collator.alive = true; + collator.last_ping_status = td::Status::OK(); } collator.ping_at = td::Timestamp::in(td::Random::fast(10.0, 20.0)); if (collator.active_cnt && !collator.sent_ping) { diff --git a/validator/collation-manager.hpp b/validator/collation-manager.hpp index 9ca69814b..0ca4617d0 100644 --- a/validator/collation-manager.hpp +++ b/validator/collation-manager.hpp @@ -65,6 +65,8 @@ class CollationManager : public td::actor::Actor { td::Timestamp ping_at = td::Timestamp::now(); bool sent_ping = false; size_t active_cnt = 0; + td::Timestamp last_ping_at = td::Timestamp::never(); + td::Status last_ping_status = td::Status::Error("not pinged"); }; std::map collators_; diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 1b8eb79e8..968835c22 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -167,6 +167,10 @@ void CollatorNode::new_masterchain_block_notification(td::Ref } } +void CollatorNode::update_shard_client_handle(BlockHandle shard_client_handle) { + shard_client_handle_ = shard_client_handle; +} + void CollatorNode::update_validator_group_info(ShardIdFull shard, std::vector prev, CatchainSeqno cc_seqno) { if (!can_collate_shard(shard)) { @@ -225,7 +229,12 @@ void CollatorNode::update_validator_group_info(ShardIdFull shard, std::vector) {}); + auto S = check_out_of_sync(); + if (S.is_ok()) { + generate_block(shard, cc_seqno, info.prev, {}, td::Timestamp::in(10.0), [](td::Result) {}); + } else { + LOG(DEBUG) << "not generating block automatically: " << S; + } } return; } @@ -535,9 +544,22 @@ void CollatorNode::process_result(std::shared_ptr cache_entry, td::R cache_entry->promises.clear(); } +td::Status CollatorNode::check_out_of_sync() { + if (last_masterchain_state_.is_null() || !shard_client_handle_) { + return td::Status::Error("not inited"); + } + auto now = (UnixTime)td::Clocks::system(); + if (last_masterchain_state_->get_unix_time() < now - 60 || shard_client_handle_->unix_time() < now - 60) { + return td::Status::Error(PSTRING() << "out of sync: mc " << now - last_masterchain_state_->get_unix_time() + << "s ago, shardclient " << now - shard_client_handle_->unix_time() << "s ago"); + } + return td::Status::OK(); +} + void CollatorNode::process_ping(adnl::AdnlNodeIdShort src, ton_api::collatorNode_ping& ping, td::Promise promise) { LOG(DEBUG) << "got ping from " << src; + TRY_STATUS_PROMISE(promise, check_out_of_sync()); promise.set_result(create_serialize_tl_object(0)); } diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp index 54876c35f..cca77d26c 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node.hpp @@ -36,6 +36,7 @@ class CollatorNode : public td::actor::Actor { void del_shard(ShardIdFull shard); void new_masterchain_block_notification(td::Ref state); + void update_shard_client_handle(BlockHandle shard_client_handle); void update_validator_group_info(ShardIdFull shard, std::vector prev, CatchainSeqno cc_seqno); void update_options(td::Ref opts) { @@ -84,6 +85,7 @@ class CollatorNode : public td::actor::Actor { std::map, FutureValidatorGroup> future_validator_groups_; td::Ref last_masterchain_state_; + BlockHandle shard_client_handle_; td::Result get_future_validator_group(ShardIdFull shard, CatchainSeqno cc_seqno); @@ -92,6 +94,8 @@ class CollatorNode : public td::actor::Actor { td::Promise promise); void process_result(std::shared_ptr cache_entry, td::Result R); + td::Status check_out_of_sync(); + public: static tl_object_ptr serialize_candidate(const BlockCandidate& block, bool compress); static td::Result deserialize_candidate(tl_object_ptr f, diff --git a/validator/manager.cpp b/validator/manager.cpp index 903e31554..b8394bdc5 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2754,6 +2754,9 @@ void ValidatorManagerImpl::update_shard_client_block_handle(BlockHandle handle, last_liteserver_state_ = std::move(state); } } + for (auto &c : collator_nodes_) { + td::actor::send_closure(c.second.actor, &CollatorNode::update_shard_client_handle, shard_client_handle_); + } shard_client_update(seqno); promise.set_value(td::Unit()); } @@ -3509,6 +3512,13 @@ void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh if (it == collator_nodes_.end()) { it = collator_nodes_.emplace(id, Collator()).first; it->second.actor = td::actor::create_actor("collatornode", id, opts_, actor_id(this), adnl_, rldp_); + if (last_masterchain_state_.not_null()) { + td::actor::send_closure(it->second.actor, &CollatorNode::new_masterchain_block_notification, + last_masterchain_state_); + } + if (shard_client_handle_) { + td::actor::send_closure(it->second.actor, &CollatorNode::update_shard_client_handle, shard_client_handle_); + } } if (!it->second.shards.insert(shard).second) { return; From 0280a288c6c038f2cfcce001cd71e828f76aee58 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 29 Nov 2024 10:34:01 +0300 Subject: [PATCH 129/388] Check supported version in collator-node --- validator/collator-node.cpp | 88 +++++++++++++++++++++++++--------- validator/collator-node.hpp | 4 ++ validator/impl/collator-impl.h | 3 ++ 3 files changed, 73 insertions(+), 22 deletions(-) diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 968835c22..869ac9ba0 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -21,6 +21,7 @@ #include "block-db.h" #include "td/utils/lz4.h" #include "checksum.h" +#include "impl/collator-impl.h" #include "impl/shard.hpp" #include "validator-session/candidate-serializer.h" @@ -85,6 +86,15 @@ void CollatorNode::del_shard(ShardIdFull shard) { void CollatorNode::new_masterchain_block_notification(td::Ref state) { last_masterchain_state_ = state; + + if (state->last_key_block_id().seqno() != last_key_block_seqno_) { + last_key_block_seqno_ = state->last_key_block_id().seqno(); + mc_config_status_ = check_mc_config(); + if (mc_config_status_.is_error()) { + LOG(ERROR) << "Cannot validate masterchain config (possibly outdated software):" << mc_config_status_; + } + } + if (validator_adnl_ids_.empty() || state->is_key_state()) { validator_adnl_ids_.clear(); for (int next : {-1, 0, 1}) { @@ -216,7 +226,7 @@ void CollatorNode::update_validator_group_info(ShardIdFull shard, std::vectorhas_external_query_at && !cache_entry->has_internal_query_at) { LOG(INFO) << "generate block query" @@ -230,11 +240,15 @@ void CollatorNode::update_validator_group_info(ShardIdFull shard, std::vector) {}); - } else { + if (S.is_error()) { LOG(DEBUG) << "not generating block automatically: " << S; + return; + } + if (mc_config_status_.is_error()) { + LOG(DEBUG) << "not generating block automatically: unsupported mc config: " << mc_config_status_; + return; } + generate_block(shard, cc_seqno, info.prev, {}, td::Timestamp::in(10.0), [](td::Result) {}); } return; } @@ -322,7 +336,7 @@ static BlockCandidate change_creator(BlockCandidate block, Ed25519_PublicKey cre for (auto& broadcast_ref : block.out_msg_queue_proof_broadcasts) { auto block_state_proof = create_block_state_proof(root).move_as_ok(); - auto &broadcast = broadcast_ref.write(); + auto& broadcast = broadcast_ref.write(); broadcast.block_id = block.id; broadcast.block_state_proofs = vm::std_boc_serialize(std::move(block_state_proof), 31).move_as_ok(); } @@ -363,11 +377,9 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data for (const auto& b : f->prev_blocks_) { prev_blocks.push_back(create_block_id(b)); } - auto priority = BlockCandidatePriority { - .round = static_cast(f->round_), - .first_block_round = static_cast(f->first_block_round_), - .priority = f->priority_ - }; + auto priority = BlockCandidatePriority{.round = static_cast(f->round_), + .first_block_round = static_cast(f->first_block_round_), + .priority = f->priority_}; Ed25519_PublicKey creator(f->creator_); td::Promise new_promise = [promise = std::move(promise), src, shard](td::Result R) mutable { @@ -447,22 +459,21 @@ void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std return; } - static auto prefix_inner = [] (auto &sb, auto &shard, auto cc_seqno, auto block_seqno, - const std::optional &o_priority) { + static auto prefix_inner = [](auto& sb, auto& shard, auto cc_seqno, auto block_seqno, + const std::optional& o_priority) { sb << "generate block query" << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno; if (o_priority) { sb << " external{"; - sb << "round_offset=" << o_priority->round - o_priority->first_block_round << ",priority=" << o_priority->priority; + sb << "round_offset=" << o_priority->round - o_priority->first_block_round + << ",priority=" << o_priority->priority; sb << ",first_block_round=" << o_priority->first_block_round; sb << "}"; } else { - sb << " internal" ; + sb << " internal"; } }; - auto prefix = [&] (auto &sb) { - prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority); - }; + auto prefix = [&](auto& sb) { prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority); }; auto cache_entry = validator_group_info.cache[prev_blocks]; if (cache_entry == nullptr) { @@ -473,7 +484,8 @@ void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std if (cache_entry->has_internal_query_at && cache_entry->has_external_query_at) { FLOG(INFO) { prefix(sb); - sb << ": got external query " << cache_entry->has_external_query_at.at() - cache_entry->has_internal_query_at.at() + sb << ": got external query " + << cache_entry->has_external_query_at.at() - cache_entry->has_internal_query_at.at() << "s after internal query [WON]"; }; } @@ -483,7 +495,8 @@ void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std if (cache_entry->has_internal_query_at && cache_entry->has_external_query_at) { FLOG(INFO) { prefix(sb); - sb << ": got internal query " << cache_entry->has_internal_query_at.at() - cache_entry->has_external_query_at.at() + sb << ": got internal query " + << cache_entry->has_internal_query_at.at() - cache_entry->has_external_query_at.at() << "s after external query [LOST]"; }; } @@ -503,8 +516,8 @@ void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std if (cache_entry->started) { FLOG(INFO) { - prefix(sb); - sb << ": collation in progress, waiting"; + prefix(sb); + sb << ": collation in progress, waiting"; }; return; } @@ -519,7 +532,7 @@ void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std last_masterchain_state_->get_validator_set(shard), opts_->get_collator_options(), manager_, timeout, [=, SelfId = actor_id(this), timer = td::Timer{}](td::Result R) { FLOG(INFO) { - prefix_inner(sb, shard, cc_seqno, block_seqno,o_priority); + prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority); sb << timer.elapsed() << ": " << (R.is_ok() ? "OK" : R.error().to_string()); }; td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R)); @@ -556,10 +569,41 @@ td::Status CollatorNode::check_out_of_sync() { return td::Status::OK(); } +td::Status CollatorNode::check_mc_config() { + if (last_masterchain_state_.is_null()) { + return td::Status::Error("not inited"); + } + TRY_RESULT_PREFIX( + config, + block::ConfigInfo::extract_config(last_masterchain_state_->root_cell(), block::ConfigInfo::needCapabilities), + "cannot unpack masterchain config"); + if (config->get_global_version() > Collator::supported_version()) { + return td::Status::Error(PSTRING() << "unsupported global version " << config->get_global_version() + << " (supported: " << Collator::supported_version() << ")"); + } + if (config->get_capabilities() & ~Collator::supported_capabilities()) { + return td::Status::Error(PSTRING() << "unsupported capabilities " << config->get_capabilities() + << " (supported: " << Collator::supported_capabilities() << ")"); + } + td::Status S = td::Status::OK(); + config->foreach_config_param([&](int idx, td::Ref param) { + if (idx < 0) { + return true; + } + if (!block::gen::ConfigParam{idx}.validate_ref(1024, std::move(param))) { + S = td::Status::Error(PSTRING() << "unknown ConfigParam " << idx); + return false; + } + return true; + }); + return S; +} + void CollatorNode::process_ping(adnl::AdnlNodeIdShort src, ton_api::collatorNode_ping& ping, td::Promise promise) { LOG(DEBUG) << "got ping from " << src; TRY_STATUS_PROMISE(promise, check_out_of_sync()); + TRY_STATUS_PROMISE_PREFIX(promise, mc_config_status_.clone(), "unsupported mc config: "); promise.set_result(create_serialize_tl_object(0)); } diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp index cca77d26c..5aab24635 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node.hpp @@ -87,6 +87,9 @@ class CollatorNode : public td::actor::Actor { td::Ref last_masterchain_state_; BlockHandle shard_client_handle_; + td::Status mc_config_status_ = td::Status::Error("not inited"); + BlockSeqno last_key_block_seqno_ = (BlockSeqno)-1; + td::Result get_future_validator_group(ShardIdFull shard, CatchainSeqno cc_seqno); void generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std::vector prev_blocks, @@ -95,6 +98,7 @@ class CollatorNode : public td::actor::Actor { void process_result(std::shared_ptr cache_entry, td::Result R); td::Status check_out_of_sync(); + td::Status check_mc_config(); public: static tl_object_ptr serialize_candidate(const BlockCandidate& block, bool compress); diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 0090e95de..b207242fb 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -40,6 +40,7 @@ namespace validator { using td::Ref; class Collator final : public td::actor::Actor { + public: static constexpr int supported_version() { return SUPPORTED_VERSION; } @@ -47,6 +48,8 @@ class Collator final : public td::actor::Actor { return ton::capCreateStatsEnabled | ton::capBounceMsgBody | ton::capReportVersion | ton::capShortDequeue | ton::capStoreOutMsgQueueSize | ton::capMsgMetadata | ton::capDeferMessages | ton::capFullCollatedData; } + + private: using LtCellRef = block::LtCellRef; using NewOutMsg = block::NewOutMsg; const ShardIdFull shard_; From 4704de76c6e26a49ee85693521ecc4f8dc49cb9d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 29 Nov 2024 13:49:24 +0300 Subject: [PATCH 130/388] Fix compilation errors --- tdutils/td/utils/Timer.cpp | 1 + validator-engine/validator-engine.cpp | 6 +++--- validator/collation-manager.cpp | 2 +- validator/collator-node.cpp | 8 ++++---- validator/full-node.cpp | 6 +++--- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/tdutils/td/utils/Timer.cpp b/tdutils/td/utils/Timer.cpp index c2c678955..abb69b100 100644 --- a/tdutils/td/utils/Timer.cpp +++ b/tdutils/td/utils/Timer.cpp @@ -23,6 +23,7 @@ #include "td/utils/Time.h" #include +#include namespace td { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 3b9715b35..2e03ce458 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -472,7 +472,7 @@ td::Result Config::config_add_collator(ton::adnl::AdnlNodeIdShort addr, to return td::Status::Error(PSTRING() << "invalid shard: " << shard.to_str()); } auto& shards = collators[addr]; - if (std::ranges::find(shards, shard) != collators[addr].end()) { + if (std::find(shards.begin(), shards.end(), shard) != shards.end()) { return false; } shards.push_back(shard); @@ -484,7 +484,7 @@ td::Result Config::config_del_collator(ton::adnl::AdnlNodeIdShort addr, to return td::Status::Error(PSTRING() << "invalid shard: " << shard.to_str()); } auto& shards = collators[addr]; - auto it = std::ranges::find(shards, shard); + auto it = std::find(shards.begin(), shards.end(), shard); if (it == shards.end()) { return false; } @@ -1605,7 +1605,7 @@ void ValidatorEngine::set_shard_check_function() { shards.push_back(s); } std::sort(shards.begin(), shards.end()); - shards.erase(std::ranges::unique(shards).begin(), shards.end()); + shards.erase(std::unique(shards.begin(), shards.end()), shards.end()); validator_options_.write().set_shard_check_function( [shards = std::move(shards)](ton::ShardIdFull shard) -> bool { for (auto s : shards) { diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index 879b8d895..eb9aa626d 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -321,7 +321,7 @@ void CollationManager::alarm() { if (collator.ping_at.is_in_past()) { collator.sent_ping = true; td::BufferSlice query = create_serialize_tl_object(0); - td::Promise P = [=, SelfId = actor_id(this)](td::Result R) mutable { + td::Promise P = [=, id = id, SelfId = actor_id(this)](td::Result R) mutable { td::actor::send_closure(SelfId, &CollationManager::got_pong, id, std::move(R)); }; LOG(DEBUG) << "sending ping to " << id; diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 869ac9ba0..94bb18a1b 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -70,7 +70,7 @@ void CollatorNode::tear_down() { void CollatorNode::add_shard(ShardIdFull shard) { CHECK(shard.is_valid_ext() && !shard.is_masterchain()); - if (std::ranges::find(collating_shards_, shard) != collating_shards_.end()) { + if (std::find(collating_shards_.begin(), collating_shards_.end(), shard) != collating_shards_.end()) { return; } LOG(INFO) << "Collator node: local_id=" << local_id_ << " , shard=" << shard.to_str(); @@ -78,7 +78,7 @@ void CollatorNode::add_shard(ShardIdFull shard) { } void CollatorNode::del_shard(ShardIdFull shard) { - auto it = std::ranges::find(collating_shards_, shard); + auto it = std::find(collating_shards_.begin(), collating_shards_.end(), shard); if (it != collating_shards_.end()) { collating_shards_.erase(it); } @@ -608,8 +608,8 @@ void CollatorNode::process_ping(adnl::AdnlNodeIdShort src, ton_api::collatorNode } bool CollatorNode::can_collate_shard(ShardIdFull shard) const { - return std::ranges::any_of(collating_shards_, - [&](const ShardIdFull& our_shard) { return shard_intersects(shard, our_shard); }); + return std::any_of(collating_shards_.begin(), collating_shards_.end(), + [&](const ShardIdFull& our_shard) { return shard_intersects(shard, our_shard); }); } tl_object_ptr CollatorNode::serialize_candidate(const BlockCandidate& block, diff --git a/validator/full-node.cpp b/validator/full-node.cpp index e7527d768..374397139 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -936,9 +936,9 @@ bool FullNodeConfig::operator!=(const FullNodeConfig &rhs) const { } bool CustomOverlayParams::send_shard(const ShardIdFull &shard) const { - return sender_shards_.empty() || std::ranges::any_of(sender_shards_, [&](const ShardIdFull &our_shard) { - return shard_intersects(shard, our_shard); - }); + return sender_shards_.empty() || + std::any_of(sender_shards_.begin(), sender_shards_.end(), + [&](const ShardIdFull &our_shard) { return shard_intersects(shard, our_shard); }); } CustomOverlayParams CustomOverlayParams::fetch(const ton_api::engine_validator_customOverlay& f) { From 378b5e94c8051cd0fc872a05e92128987328fedf Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 4 Dec 2024 18:06:19 +0300 Subject: [PATCH 131/388] Automatically issue and import fast sync overlay certificates --- adnl/adnl-node-id.hpp | 4 +- dht-server/dht-server.cpp | 7 +- tl/generate/scheme/ton_api.tl | 10 +- tl/generate/scheme/ton_api.tlo | Bin 109280 -> 110212 bytes .../validator-engine-console-query.cpp | 103 +-- .../validator-engine-console-query.h | 71 ++- .../validator-engine-console.cpp | 2 + validator-engine/validator-engine.cpp | 594 ++++++++++++++---- validator-engine/validator-engine.hpp | 40 +- validator/full-node-fast-sync-overlays.cpp | 2 +- validator/full-node.h | 2 + validator/full-node.hpp | 5 +- 12 files changed, 659 insertions(+), 181 deletions(-) diff --git a/adnl/adnl-node-id.hpp b/adnl/adnl-node-id.hpp index 2d3ade164..43e3629f4 100644 --- a/adnl/adnl-node-id.hpp +++ b/adnl/adnl-node-id.hpp @@ -37,7 +37,9 @@ class AdnlNodeIdShort { } explicit AdnlNodeIdShort(td::Bits256 value) : hash_(value) { } - explicit AdnlNodeIdShort(tl_object_ptr obj) : hash_(obj->id_) { + explicit AdnlNodeIdShort(tl_object_ptr &&obj) : hash_(obj->id_) { + } + explicit AdnlNodeIdShort(const tl_object_ptr &obj) : hash_(obj->id_) { } const auto &pubkey_hash() const { diff --git a/dht-server/dht-server.cpp b/dht-server/dht-server.cpp index 006e74082..46a011352 100644 --- a/dht-server/dht-server.cpp +++ b/dht-server/dht-server.cpp @@ -171,8 +171,8 @@ ton::tl_object_ptr Config::tl() const { } return ton::create_tl_object( out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec), - ton::PublicKeyHash::zero().tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), - nullptr, nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(shard_vec), std::move(gc_vec)); + ton::PublicKeyHash::zero().tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), nullptr, + nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(shard_vec), std::move(gc_vec)); } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, @@ -1195,7 +1195,8 @@ int main(int argc, char *argv[]) { SET_VERBOSITY_LEVEL(v); }); p.add_option('V', "version", "shows dht-server build information", [&]() { - std::cout << "dht-server build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::cout << "dht-server build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); }); p.add_option('h', "help", "prints_help", [&]() { diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index a4d34428e..396c06bc1 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -464,6 +464,8 @@ tonNode.outMsgQueueProof queue_proofs:bytes block_state_proofs:bytes msg_counts: tonNode.outMsgQueueProofEmpty = tonNode.OutMsgQueueProof; tonNode.forgetPeer = tonNode.ForgetPeer; +tonNode.newFastSyncMemberCertificate adnl_id:int256 certificate:overlay.MemberCertificate = tonNode.NewFastSyncMemberCertificate; + ---functions--- @@ -501,6 +503,8 @@ tonNode.slave.sendExtMessage message:tonNode.externalMessage = tonNode.Success; tonNode.query = Object; +tonNode.requestFastSyncOverlayMemberCertificate sign_by:int256 adnl_id:int256 slot:int = overlay.MemberCertificate; + ---types--- // bit 0 - started @@ -654,9 +658,11 @@ engine.validator.fullNodeMaster port:int adnl:int256 = engine.validator.FullNode engine.validator.fullNodeSlave ip:int port:int adnl:PublicKey = engine.validator.FullNodeSlave; engine.validator.fullNodeConfig ext_messages_broadcast_disabled:Bool = engine.validator.FullNodeConfig; engine.validator.fastSyncMemberCertificate adnl_id:int256 certificate:overlay.MemberCertificate = engine.validator.FastSyncMemberCertificate; +engine.validator.fastSyncOverlayClient adnl_id:int256 slot:int = engine.validator.FastSyncOverlayClient; engine.validator.collatorNodeWhitelist enabled:Bool adnl_ids:(vector int256) = engine.validator.CollatorNodeWhitelist; engine.validator.extraConfig state_serializer_enabled:Bool fast_sync_member_certificates:(vector engine.validator.fastSyncMemberCertificate) - collator_node_whitelist:engine.validator.collatorNodeWhitelist = engine.validator.ExtraConfig; + collator_node_whitelist:engine.validator.collatorNodeWhitelist + fast_sync_overlay_clients:(vector engine.validator.fastSyncOverlayClient) = engine.validator.ExtraConfig; engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) dht:(vector engine.dht) validators:(vector engine.validator) collators:(vector engine.collator) @@ -840,6 +846,8 @@ engine.validator.getCollationManagerStats = engine.validator.CollationManagerSta engine.validator.signOverlayMemberCertificate sign_by:int256 adnl_id:int256 slot:int expire_at:int = overlay.MemberCertificate; engine.validator.importFastSyncMemberCertificate adnl_id:int256 certificate:overlay.MemberCertificate = engine.validator.Success; +engine.validator.addFastSyncClient adnl_id:int256 slot:int = engine.validator.Success; +engine.validator.delFastSyncClient adnl_id:int256 = engine.validator.Success; ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index cfb78d6fdf63bab403a3e0491b987293043a6759..faef1564525053095bedcd3d1a7a3d48fe583ec0 100644 GIT binary patch delta 449 zcmaEGm96D28}Fmp`c@23V6l<6T9WzIsrskDQTratq zN9u`;!1}NAi&aYU^ZfEtQuX{&%O~HGl$xx>Ew%ZB*#UOOW1BbH@fU$i*?f3|BpcYc z``Z*bfdWQ1j|us(Oy=CaSyuYleND-<#Nv|p;>x__`24ceqMXFa_++q6lLKeUi-W9Z zggUIqnN5eow(TIuX~(xOg*vTphXOamX$&C6lP4_JnXb^pC^I?0iGu|svN>VLoG8RYhG2`PKS*Pg*#0AxaRWDN;tlq9O49=i7)8*Pug_+5-M*rb@r}9d(og20`mm5I zN-Zo+EiQo>?hgq9-_+cs)FS89qLR$C%;dz9)ZJ%Qm_XsiJpE=PWAgNbCZLs5RxdGFUXj7LfqVOu62>>?+bz~I`Y-|jRq8fS diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 2d78b412e..c0170a067 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -807,21 +807,21 @@ td::Status SignCertificateQuery::send() { auto sign = ton::create_serialize_tl_object(signer_.tl(), std::move(cid)); auto pub = ton::create_serialize_tl_object(signer_.tl()); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(pub), - td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &SignCertificateQuery::receive_pubkey, R.move_as_ok()); - } - })); + td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &SignCertificateQuery::receive_pubkey, R.move_as_ok()); + } + })); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(sign), - td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &SignCertificateQuery::receive_signature, R.move_as_ok()); - } - })); + td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &SignCertificateQuery::handle_error, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &SignCertificateQuery::receive_signature, R.move_as_ok()); + } + })); return td::Status::OK(); } @@ -833,33 +833,32 @@ void SignCertificateQuery::receive_pubkey(td::BufferSlice R) { } pubkey_ = f.move_as_ok(); has_pubkey_ = true; - if(has_signature_) { + if (has_signature_) { save_certificate(); } } - td::Status SignCertificateQuery::receive(td::BufferSlice data) { UNREACHABLE(); } void SignCertificateQuery::receive_signature(td::BufferSlice R) { auto f = ton::fetch_tl_object(R.as_slice(), true); - if(f.is_error()){ + if (f.is_error()) { handle_error(f.move_as_error_prefix("Failed to get signature: ")); return; } signature_ = std::move(f.move_as_ok()->signature_); - if(has_pubkey_) { + if (has_pubkey_) { save_certificate(); } } void SignCertificateQuery::save_certificate() { - auto c = ton::create_serialize_tl_object( - std::move(pubkey_), expire_at_, max_size_, std::move(signature_)); + auto c = ton::create_serialize_tl_object(std::move(pubkey_), expire_at_, max_size_, + std::move(signature_)); auto w = td::write_file(out_file_, c.as_slice()); - if(w.is_error()) { + if (w.is_error()) { handle_error(w.move_as_error_prefix("Failed to write certificate to file: ")); return; } @@ -880,11 +879,8 @@ td::Status ImportCertificateQuery::send() { TRY_RESULT_PREFIX(cert, ton::fetch_tl_object(data.as_slice(), true), "incorrect certificate"); auto b = ton::create_serialize_tl_object( - overlay_, - ton::create_tl_object(id_), - ton::create_tl_object(kh_.tl()), - std::move(cert) - ); + overlay_, ton::create_tl_object(id_), + ton::create_tl_object(kh_.tl()), std::move(cert)); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -1031,7 +1027,6 @@ td::Status GetOverlaysStatsJsonQuery::receive(td::BufferSlice data) { return td::Status::OK(); } - td::Status ImportCertificateQuery::receive(td::BufferSlice data) { TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), "received incorrect answer: "); @@ -1039,10 +1034,9 @@ td::Status ImportCertificateQuery::receive(td::BufferSlice data) { return td::Status::OK(); } - td::Status SignShardOverlayCertificateQuery::run() { TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); - TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(key_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(expire_at_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(max_size_, tokenizer_.get_token()); @@ -1052,8 +1046,8 @@ td::Status SignShardOverlayCertificateQuery::run() { } td::Status SignShardOverlayCertificateQuery::send() { - auto b = ton::create_serialize_tl_object - (wc_, shard_, ton::create_tl_object(key_.tl()), expire_at_, max_size_); + auto b = ton::create_serialize_tl_object( + wc_, shard_, ton::create_tl_object(key_.tl()), expire_at_, max_size_); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -1062,7 +1056,7 @@ td::Status SignShardOverlayCertificateQuery::receive(td::BufferSlice data) { TRY_RESULT_PREFIX(c, ton::fetch_tl_object(data.as_slice(), true), "received incorrect cert: "); auto w = td::write_file(out_file_, data.as_slice()); - if(w.is_error()) { + if (w.is_error()) { return w.move_as_error_prefix("Failed to write certificate to file: "); } td::TerminalIO::out() << "saved certificate\n"; @@ -1072,7 +1066,7 @@ td::Status SignShardOverlayCertificateQuery::receive(td::BufferSlice data) { td::Status ImportShardOverlayCertificateQuery::run() { TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); - TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(key_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(in_file_, tokenizer_.get_token()); @@ -1083,8 +1077,8 @@ td::Status ImportShardOverlayCertificateQuery::send() { TRY_RESULT(data, td::read_file(in_file_)); TRY_RESULT_PREFIX(cert, ton::fetch_tl_object(data.as_slice(), true), "incorrect certificate"); - auto b = ton::create_serialize_tl_object - (wc_, shard_, ton::create_tl_object(key_.tl()), std::move(cert)); + auto b = ton::create_serialize_tl_object( + wc_, shard_, ton::create_tl_object(key_.tl()), std::move(cert)); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -1908,3 +1902,42 @@ td::Status ImportFastSyncMemberCertificateQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << "success\n"; return td::Status::OK(); } + +td::Status AddFastSyncOverlayClientQuery::run() { + TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); + TRY_RESULT_ASSIGN(slot_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status AddFastSyncOverlayClientQuery::send() { + auto b = ton::create_serialize_tl_object(adnl_id_, slot_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status AddFastSyncOverlayClientQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status DelFastSyncOverlayClientQuery::run() { + TRY_RESULT_ASSIGN(adnl_id_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status DelFastSyncOverlayClientQuery::send() { + auto b = ton::create_serialize_tl_object(adnl_id_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status DelFastSyncOverlayClientQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index c34f4f5b9..60e05d59c 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -979,9 +979,9 @@ class GetOverlaysStatsJsonQuery : public Query { std::string name() const override { return get_name(); } - -private: - std::string file_name_; + + private: + std::string file_name_; }; class SignCertificateQuery : public Query { @@ -996,7 +996,8 @@ class SignCertificateQuery : public Query { return "signcert"; } static std::string get_help() { - return "signcert \tsign overlay certificate by key"; + return "signcert \tsign overlay certificate by " + " key"; } std::string name() const override { return get_name(); @@ -1004,9 +1005,8 @@ class SignCertificateQuery : public Query { void receive_pubkey(td::BufferSlice R); void receive_signature(td::BufferSlice R); - private: - void save_certificate(); + void save_certificate(); td::Bits256 overlay_; td::Bits256 id_; @@ -1057,14 +1057,14 @@ class SignShardOverlayCertificateQuery : public Query { return "signshardoverlaycert"; } static std::string get_help() { - return "signshardoverlaycert \tsign certificate for in currently active shard overlay"; + return "signshardoverlaycert \tsign certificate for " + " in currently active shard overlay"; } std::string name() const override { return get_name(); } private: - td::int32 wc_; td::int64 shard_; td::int32 expire_at_; @@ -1073,7 +1073,6 @@ class SignShardOverlayCertificateQuery : public Query { std::string out_file_; }; - class ImportShardOverlayCertificateQuery : public Query { public: ImportShardOverlayCertificateQuery(td::actor::ActorId console, Tokenizer tokenizer) @@ -1086,14 +1085,14 @@ class ImportShardOverlayCertificateQuery : public Query { return "importshardoverlaycert"; } static std::string get_help() { - return "importshardoverlaycert \timport certificate for in currently active shard overlay"; + return "importshardoverlaycert \timport certificate for in " + "currently active shard overlay"; } std::string name() const override { return get_name(); } private: - td::int32 wc_; td::int64 shard_; ton::PublicKeyHash key_; @@ -1134,7 +1133,8 @@ class GetPerfTimerStatsJsonQuery : public Query { return "getperftimerstatsjson"; } static std::string get_help() { - return "getperftimerstatsjson \tgets min, average and max event processing time for last 60, 300 and 3600 seconds and writes to json file"; + return "getperftimerstatsjson \tgets min, average and max event processing time for last 60, 300 and 3600 " + "seconds and writes to json file"; } std::string name() const override { return get_name(); @@ -1696,3 +1696,50 @@ class ImportFastSyncMemberCertificateQuery : public Query { td::Bits256 adnl_id_; std::string file_name_; }; + +class AddFastSyncOverlayClientQuery : public Query { + public: + AddFastSyncOverlayClientQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "addfastsyncoverlayclient"; + } + static std::string get_help() { + return "addfastsyncoverlayclient \tstarts issuing member certificates " + "to (hex) on slot (int)"; + } + std::string name() const override { + return get_name(); + } + + private: + td::Bits256 adnl_id_; + td::int32 slot_; +}; + +class DelFastSyncOverlayClientQuery : public Query { + public: + DelFastSyncOverlayClientQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "delfastsyncoverlayclient"; + } + static std::string get_help() { + return "delfastsyncoverlayclient \tstops issuing member certificates " + "to (hex)"; + } + std::string name() const override { + return get_name(); + } + + private: + td::Bits256 adnl_id_; +}; diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 9e85b567e..d80316812 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -168,6 +168,8 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 5f4c7c5cc..abba70c57 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -27,9 +27,17 @@ */ #include "validator-engine.hpp" +#include "adnl/adnl-node-id.hpp" #include "auto/tl/ton_api.h" +#include "errorcode.h" +#include "keys/keys.hpp" #include "overlay-manager.h" +#include "overlays.h" +#include "td/actor/PromiseFuture.h" #include "td/actor/actor.h" +#include "td/utils/Status.h" +#include "td/utils/Time.h" +#include "td/utils/buffer.h" #include "tl-utils/tl-utils.hpp" #include "tl/TlObject.h" #include "ton/ton-types.h" @@ -59,6 +67,8 @@ #include "memprof/memprof.h" #include "dht/dht.hpp" +#include +#include #if TD_DARWIN || TD_LINUX #include @@ -182,6 +192,10 @@ Config::Config(const ton::ton_api::engine_validator_config &config) { collator_node_whitelist.emplace(id); } } + for (auto &client : config.extraconfig_->fast_sync_overlay_clients_) { + auto key = ton::adnl::AdnlNodeIdShort{client->adnl_id_}; + fast_sync_overlay_clients.emplace_back(std::move(key), client->slot_); + } } else { state_serializer_enabled = true; } @@ -282,7 +296,8 @@ ton::tl_object_ptr Config::tl() const { } ton::tl_object_ptr extra_config_obj = {}; - if (!state_serializer_enabled || !fast_sync_member_certificates.empty() || collator_node_whitelist_obj) { + if (!state_serializer_enabled || !fast_sync_member_certificates.empty() || collator_node_whitelist_obj || + !fast_sync_overlay_clients.empty()) { // Non-default values extra_config_obj = ton::create_tl_object(); extra_config_obj->state_serializer_enabled_ = state_serializer_enabled; @@ -292,6 +307,11 @@ ton::tl_object_ptr Config::tl() const { certificate.tl())); } extra_config_obj->collator_node_whitelist_ = std::move(collator_node_whitelist_obj); + for (const auto &client : fast_sync_overlay_clients) { + extra_config_obj->fast_sync_overlay_clients_.push_back( + ton::create_tl_object(client.id.bits256_value(), + client.slot)); + } } std::vector> liteserver_vec; @@ -1352,10 +1372,12 @@ void ValidatorEngine::schedule_shutdown(double at) { LOG(DEBUG) << "Scheduled shutdown is in past (" << at << ")"; } else { LOG(INFO) << "Schedule shutdown for " << at << " (in " << ts.in() << "s)"; - ton::delay_action([]() { - LOG(WARNING) << "Shutting down as scheduled"; - std::_Exit(0); - }, ts); + ton::delay_action( + []() { + LOG(WARNING) << "Shutting down as scheduled"; + std::_Exit(0); + }, + ts); } } void ValidatorEngine::start_up() { @@ -1388,12 +1410,14 @@ void ValidatorEngine::alarm() { auto cur_t = config->get_validator_set_start_stop(0); CHECK(cur_t.first > 0); - auto val_set = state_->get_total_validator_set(0); - auto e = val_set->export_vector(); std::set to_del; for (auto &val : config_.validators) { bool is_validator = false; - if (val_set->is_validator(ton::NodeIdShort{val.first.bits256_value()})) { + if (validator_set_next_.not_null() && + validator_set_next_->is_validator(ton::NodeIdShort{val.first.bits256_value()})) { + is_validator = true; + } + if (validator_set_.not_null() && validator_set_->is_validator(ton::NodeIdShort{val.first.bits256_value()})) { is_validator = true; } if (!is_validator && val.second.election_date < cur_t.first && cur_t.first + 600 < state_->get_unix_time()) { @@ -1414,9 +1438,41 @@ void ValidatorEngine::alarm() { need_write = true; } + { + std::set fs_to_del; + for (auto &x : config_.fast_sync_member_certificates) { + if (x.second.is_expired()) { + fs_to_del.insert(x.first); + continue; + } + auto issued_by = x.second.issued_by().compute_short_id().bits256_value(); + if (validator_set_.not_null() && validator_set_->is_validator(issued_by)) { + continue; + } + if (validator_set_prev_.not_null() && validator_set_prev_->is_validator(issued_by)) { + continue; + } + if (validator_set_next_.not_null() && validator_set_next_->is_validator(issued_by)) { + continue; + } + fs_to_del.insert(x.first); + } + if (!fs_to_del.empty()) { + need_write = true; + std::erase_if(config_.fast_sync_member_certificates, + [&](const std::pair &e) { + return !fs_to_del.contains(e.first); + }); + } + } + if (need_write) { write_config([](td::Unit) {}); } + if (issue_fast_sync_overlay_certificates_at_.is_in_past()) { + issue_fast_sync_overlay_certificates_at_ = td::Timestamp::in(60.0); + issue_fast_sync_overlay_certificates(); + } } for (auto &x : config_.gc) { if (running_gc_.count(x) == 0) { @@ -1443,6 +1499,16 @@ void ValidatorEngine::deleted_key(ton::PublicKeyHash x) { } } +void ValidatorEngine::got_state(td::Ref state) { + if (state_.not_null() && state_->get_block_id() == state->get_block_id()) { + return; + } + state_ = std::move(state); + validator_set_ = state_->get_total_validator_set(0); + validator_set_next_ = state_->get_total_validator_set(1); + validator_set_prev_ = state_->get_total_validator_set(-1); +} + td::Status ValidatorEngine::load_global_config() { TRY_RESULT_PREFIX(conf_data, td::read_file(global_config_), "failed to read: "); TRY_RESULT_PREFIX(conf_json, td::json_decode(conf_data.as_slice()), "failed to parse json: "); @@ -1596,25 +1662,24 @@ void ValidatorEngine::set_shard_check_function() { validator_options_.write().set_shard_check_function([](ton::ShardIdFull shard) -> bool { return true; }); } else { std::vector shards = {ton::ShardIdFull(ton::masterchainId)}; - for (const auto& [_, collator_shards] : config_.collators) { - for (const auto& shard : collator_shards) { + for (const auto &[_, collator_shards] : config_.collators) { + for (const auto &shard : collator_shards) { shards.push_back(shard); } } - for (const auto& s : config_.shards_to_monitor) { + for (const auto &s : config_.shards_to_monitor) { shards.push_back(s); } std::sort(shards.begin(), shards.end()); shards.erase(std::unique(shards.begin(), shards.end()), shards.end()); - validator_options_.write().set_shard_check_function( - [shards = std::move(shards)](ton::ShardIdFull shard) -> bool { - for (auto s : shards) { - if (shard_intersects(shard, s)) { - return true; - } - } - return false; - }); + validator_options_.write().set_shard_check_function([shards = std::move(shards)](ton::ShardIdFull shard) -> bool { + for (auto s : shards) { + if (shard_intersects(shard, s)) { + return true; + } + } + return false; + }); } } @@ -2100,7 +2165,8 @@ void ValidatorEngine::started_validator() { } void ValidatorEngine::start_full_node() { - if (!config_.full_node.is_zero() || config_.full_node_slaves.size() > 0) { + if (!config_.full_node.is_zero() || !config_.full_node_slaves.empty()) { + full_node_id_ = ton::adnl::AdnlNodeIdShort{config_.full_node}; auto pk = ton::PrivateKey{ton::privkeys::Ed25519::random()}; auto short_id = pk.compute_short_id(); td::actor::send_closure(keyring_, &ton::keyring::Keyring::add_key, std::move(pk), true, [](td::Unit) {}); @@ -2123,8 +2189,8 @@ void ValidatorEngine::start_full_node() { td::actor::send_closure(SelfId, &ValidatorEngine::started_full_node); }); full_node_ = ton::validator::fullnode::FullNode::create( - short_id, ton::adnl::AdnlNodeIdShort{config_.full_node}, validator_options_->zero_block_id().file_hash, - config_.full_node_config, keyring_.get(), adnl_.get(), rldp_.get(), rldp2_.get(), + short_id, full_node_id_, validator_options_->zero_block_id().file_hash, config_.full_node_config, + keyring_.get(), adnl_.get(), rldp_.get(), rldp2_.get(), default_dht_node_.is_zero() ? td::actor::ActorId{} : dht_nodes_[default_dht_node_].get(), overlay_manager_.get(), validator_manager_.get(), full_node_client_.get(), db_root_, std::move(P)); for (auto &v : config_.validators) { @@ -2143,6 +2209,7 @@ void ValidatorEngine::start_full_node() { validator_telemetry_filename_); } load_custom_overlays_config(); + register_fast_sync_certificate_callback(); } else { started_full_node(); } @@ -2382,9 +2449,17 @@ void ValidatorEngine::try_add_full_node_adnl_addr(ton::PublicKeyHash id, td::Pro return; } - if (!full_node_.empty()) { - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::update_adnl_id, - ton::adnl::AdnlNodeIdShort{id}, [](td::Unit) {}); + if (!full_node_.empty() && id != full_node_id_.pubkey_hash()) { + td::actor::send_closure( + adnl_.get(), &ton::adnl::Adnl::unsubscribe, full_node_id_, + ton::adnl::Adnl::int_to_bytestring(ton::ton_api::tonNode_newFastSyncMemberCertificate::ID)); + td::actor::send_closure( + adnl_.get(), &ton::adnl::Adnl::unsubscribe, full_node_id_, + ton::adnl::Adnl::int_to_bytestring(ton::ton_api::tonNode_requestFastSyncOverlayMemberCertificate::ID)); + full_node_id_ = ton::adnl::AdnlNodeIdShort{id}; + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::update_adnl_id, full_node_id_, + [](td::Unit) {}); + register_fast_sync_certificate_callback(); } write_config(std::move(promise)); @@ -2644,9 +2719,225 @@ void ValidatorEngine::try_del_proxy(td::uint32 ip, td::int32 port, std::vector validator_engine) + : validator_engine_(std::move(validator_engine)) { + } + void receive_message(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, + td::BufferSlice data) override { + auto R = ton::fetch_tl_object( + std::move(data), true); + if (R.is_error()) { + return; + } + auto res = R.move_as_ok(); + auto cert = ton::overlay::OverlayMemberCertificate(res->certificate_.get()); + if (cert.empty()) { + return; + } + LOG(DEBUG) << "Received tonNode.newFastSyncMemberCertificate from " << src; + td::actor::send_closure(validator_engine_, &ValidatorEngine::try_import_fast_sync_member_certificate, dst, + std::move(cert), td::PromiseCreator::lambda([](td::Result R) { + if (R.is_error()) { + LOG(WARNING) << "failed to import overlay member certificate: " << R.move_as_error(); + } + })); + } + void receive_query(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + auto R = + ton::fetch_tl_object(std::move(data), true); + if (R.is_error()) { + return; + } + auto q = R.move_as_ok(); + td::actor::send_closure( + validator_engine_, &ValidatorEngine::process_fast_sync_overlay_certificate_request, + ton::PublicKeyHash{q->sign_by_}, ton::adnl::AdnlNodeIdShort{q->adnl_id_}, 0, q->slot_, + (td::int32)td::Clocks::system() + 3600, + td::PromiseCreator::lambda( + [promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + return; + } + auto cert = R.move_as_ok(); + promise.set_value(ton::serialize_tl_object(cert.tl(), true)); + })); + } + + private: + td::actor::ActorId validator_engine_; + }; + td::actor::send_closure( + adnl_.get(), &ton::adnl::Adnl::subscribe, full_node_id_, + ton::adnl::Adnl::int_to_bytestring(ton::ton_api::tonNode_newFastSyncMemberCertificate::ID), + std::make_unique(actor_id(this))); + td::actor::send_closure( + adnl_.get(), &ton::adnl::Adnl::subscribe, full_node_id_, + ton::adnl::Adnl::int_to_bytestring(ton::ton_api::tonNode_requestFastSyncOverlayMemberCertificate::ID), + std::make_unique(actor_id(this))); +} + +void ValidatorEngine::try_import_fast_sync_member_certificate(ton::adnl::AdnlNodeIdShort id, + ton::overlay::OverlayMemberCertificate certificate, + td::Promise promise) { + if (!started_ || state_.is_null()) { + return promise.set_error(td::Status::Error("not started")); + } + if (certificate.slot() < 0 || + certificate.slot() >= ton::validator::fullnode::FullNode::MAX_FAST_SYNC_OVERLAY_CLIENTS) { + return promise.set_error(td::Status::Error( + PSTRING() << "invalid slot (max " << ton::validator::fullnode::FullNode::MAX_FAST_SYNC_OVERLAY_CLIENTS + << " clients)")); + } + if (certificate.expire_at() < td::Clocks::system() + 60) { + return promise.set_error(td::Status::Error("certificate expires too soon")); + } + TRY_STATUS_PROMISE_PREFIX(promise, certificate.check_signature(id), "invalid certificate: "); + + auto cert_score = [this](ton::overlay::OverlayMemberCertificate &cert) -> td::int64 { + auto issued_by = cert.issued_by().compute_short_id().bits256_value(); + if (validator_set_next_.not_null() && validator_set_next_->is_validator(issued_by)) { + return cert.expire_at() + (1ll << 32); + } + if (validator_set_.not_null() && validator_set_->is_validator(issued_by)) { + return cert.expire_at() + (1ll << 32); + } + if (validator_set_prev_.not_null() && validator_set_prev_->is_validator(issued_by)) { + return cert.expire_at() + (0ll << 32); + } + return -1; + }; + for (auto &x : config_.fast_sync_member_certificates) { + if (x.first == id) { + // fast check + if (x.second.issued_by() == certificate.issued_by()) { + if (x.second.expire_at() < certificate.expire_at()) { + x.second = std::move(certificate); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_fast_sync_member_certificate, + x.first, x.second); + write_config(std::move(promise)); + return; + } + LOG(DEBUG) << "Not importing certificate: certificate from the same issuer exists with bigger ttl"; + promise.set_value(td::Unit()); + return; + } + auto new_score = cert_score(certificate); + auto old_score = cert_score(x.second); + if (new_score > old_score) { + x.second = std::move(certificate); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_fast_sync_member_certificate, + x.first, x.second); + write_config(std::move(promise)); + return; + } + LOG(DEBUG) << "Not importing certificate: certificate with better score exists"; + promise.set_value(td::Unit()); + return; + } + } + + auto new_score = cert_score(certificate); + if (new_score < 0) { + LOG(DEBUG) << "Not importing certificate: issuer is not a validator"; + promise.set_value(td::Unit()); + return; + } + + auto &x = config_.fast_sync_member_certificates.emplace_back(std::move(id), std::move(certificate)); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_fast_sync_member_certificate, x.first, + x.second); + write_config(std::move(promise)); +} + +void ValidatorEngine::issue_fast_sync_overlay_certificates() { + if (state_.is_null() || config_.fast_sync_overlay_clients.empty() || full_node_id_.is_zero()) { + return; + } + auto issue_by = find_local_validator_for_cert_issuing(); + if (issue_by.is_zero()) { + return; + } + auto src = full_node_id_; + td::int32 expire_at = static_cast(td::Clocks::system()) + 3600; + for (auto &client : config_.fast_sync_overlay_clients) { + issue_fast_sync_overlay_certificate( + issue_by, client.id, 0, client.slot, expire_at, + [src, dst = client.id, adnl = adnl_.get()](td::Result R) { + if (R.is_error()) { + LOG(WARNING) << "cannot issue fast sync overlay certificate for " << dst << ": " << R.move_as_error(); + return; + } + auto cert = R.move_as_ok(); + LOG(INFO) << "Sending fast sync overlay certificate issued by " << cert.issued_by().compute_short_id() + << " to " << dst << " slot " << cert.slot(); + td::actor::send_closure(adnl, &ton::adnl::Adnl::send_message, src, dst, + ton::create_serialize_tl_object( + dst.bits256_value(), cert.tl())); + }); + } +} + +void ValidatorEngine::issue_fast_sync_overlay_certificate(ton::PublicKeyHash issue_by, + ton::adnl::AdnlNodeIdShort issue_to, td::uint32 flags, + td::int32 slot, td::int32 expire_at, + td::Promise promise) { + if (issue_by.is_zero()) { + issue_by = find_local_validator_for_cert_issuing(); + if (issue_by.is_zero()) { + return promise.set_error(td::Status::Error(ton::ErrorCode::notready, "cannot find a local validator")); + } + } + if (expire_at < td::Clocks::system() + 10) { + return promise.set_error(td::Status::Error(ton::ErrorCode::error, "expire at is in too near future")); + } + ton::overlay::OverlayMemberCertificate cert(ton::PublicKey(), flags, slot, expire_at, td::BufferSlice()); + auto to_sign = cert.to_sign_data(issue_to); + td::actor::send_closure(keyring_, &ton::keyring::Keyring::sign_add_get_public_key, issue_by, std::move(to_sign), + [slot, flags, expire_at, promise = std::move(promise)]( + td::Result> R) mutable { + TRY_RESULT_PROMISE_PREFIX(promise, res, std::move(R), "failed to sign certificate: "); + ton::overlay::OverlayMemberCertificate cert(std::move(res.second), flags, slot, expire_at, + std::move(res.first)); + promise.set_value(std::move(cert)); + }); +} + +void ValidatorEngine::process_fast_sync_overlay_certificate_request( + ton::PublicKeyHash issue_by, ton::adnl::AdnlNodeIdShort issue_to, td::uint32 flags, td::int32 slot, + td::int32 expire_at, td::Promise promise) { + for (auto &client : config_.fast_sync_overlay_clients) { + if (client.id == issue_to && (slot < 0 || slot == client.slot)) { + return issue_fast_sync_overlay_certificate(std::move(issue_by), std::move(issue_to), flags, client.slot, + expire_at, std::move(promise)); + } + } + promise.set_error(td::Status::Error(ton::ErrorCode::error, "cannot issue certificate to unknown adnl id")); +} + +ton::PublicKeyHash ValidatorEngine::find_local_validator_for_cert_issuing() { + if (state_.is_null()) { + return ton::PublicKeyHash{}; + } + for (auto& val_set : {validator_set_, validator_set_next_, validator_set_prev_}) { + if (val_set.is_null()) { + continue; + } + for (auto &[val_id, _]: config_.validators) { + if (val_set->is_validator(ton::NodeIdShort{val_id.bits256_value()})) { + return val_id; + } + } + } + return ton::PublicKeyHash::zero(); +} + void ValidatorEngine::load_custom_overlays_config() { - custom_overlays_config_ = - ton::create_tl_object(); + custom_overlays_config_ = ton::create_tl_object(); auto data_R = td::read_file(custom_overlays_config_file()); if (data_R.is_error()) { return; @@ -2699,7 +2990,7 @@ void ValidatorEngine::del_custom_overlay_from_config(std::string name, td::Promi static td::Result> parse_collator_options(td::MutableSlice json_str) { td::Ref ref{true}; - ton::validator::CollatorOptions& opts = ref.write(); + ton::validator::CollatorOptions &opts = ref.write(); // Set default values (from_json leaves missing fields as is) ton::ton_api::engine_validator_collatorOptions f; @@ -3706,7 +3997,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importCer return; } auto r = ton::overlay::Certificate::create(std::move(query.cert_)); - if(r.is_error()) { + if (r.is_error()) { promise.set_value(create_control_query_error(r.move_as_error_prefix("Invalid certificate: "))); } //TODO force Overlays::update_certificate to return result @@ -3721,17 +4012,15 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importCer }); */ td::actor::send_closure(overlay_manager_, &ton::overlay::Overlays::update_certificate, - ton::adnl::AdnlNodeIdShort{query.local_id_->id_}, - ton::overlay::OverlayIdShort{query.overlay_id_}, - ton::PublicKeyHash{query.signed_key_->key_hash_}, - r.move_as_ok()); - promise.set_value( - ton::serialize_tl_object(ton::create_tl_object(), true) - ); + ton::adnl::AdnlNodeIdShort{query.local_id_->id_}, + ton::overlay::OverlayIdShort{query.overlay_id_}, + ton::PublicKeyHash{query.signed_key_->key_hash_}, r.move_as_ok()); + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importShardOverlayCertificate &query, td::BufferSlice data, - ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importShardOverlayCertificate &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_modify)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -3746,7 +4035,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importSha return; } auto r = ton::overlay::Certificate::create(std::move(query.cert_)); - if(r.is_error()) { + if (r.is_error()) { promise.set_value(create_control_query_error(r.move_as_error_prefix("Invalid certificate: "))); } auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { @@ -3758,12 +4047,13 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importSha } }); ton::ShardIdFull shard_id{ton::WorkchainId{query.workchain_}, static_cast(query.shard_)}; - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_shard_overlay_certificate, - shard_id, ton::PublicKeyHash{query.signed_key_->key_hash_}, r.move_as_ok(), std::move(P)); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::import_shard_overlay_certificate, shard_id, + ton::PublicKeyHash{query.signed_key_->key_hash_}, r.move_as_ok(), std::move(P)); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signShardOverlayCertificate &query, td::BufferSlice data, - ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signShardOverlayCertificate &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_modify)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -3785,11 +4075,11 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signShard promise.set_value(R.move_as_ok()); } }); - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::sign_shard_overlay_certificate, - shard_id, ton::PublicKeyHash{query.signed_key_->key_hash_}, query.expire_at_, query.max_size_, std::move(P)); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::sign_shard_overlay_certificate, shard_id, + ton::PublicKeyHash{query.signed_key_->key_hash_}, query.expire_at_, query.max_size_, + std::move(P)); } - void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getOverlaysStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_default)) { @@ -3857,42 +4147,46 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getPerfTi return; } - auto P = td::PromiseCreator::lambda( - [promise = std::move(promise), query = std::move(query)](td::Result> R) mutable { - const std::vector times{60, 300, 3600}; - double now = td::Time::now(); - if (R.is_error()) { - promise.set_value(create_control_query_error(R.move_as_error())); - } else { - auto r = R.move_as_ok(); - std::vector> by_name; - for (const auto &stats : r) { - if (stats.name == query.name_ || query.name_.empty()) { - std::vector> by_time; - for (const auto &t : times) { - double min = std::numeric_limits::lowest(); - double max = std::numeric_limits::max(); - double sum = 0; - int cnt = 0; - for (const auto &stat : stats.stats) { - double time = stat.first; - double duration = stat.second; - if (now - time <= static_cast(t)) { - min = td::min(min, duration); - max = td::max(max, duration); - sum += duration; - ++cnt; - } - } - by_time.push_back(ton::create_tl_object(t, min, sum / static_cast(cnt), max)); + auto P = td::PromiseCreator::lambda([promise = std::move(promise), query = std::move(query)]( + td::Result> R) mutable { + const std::vector times{60, 300, 3600}; + double now = td::Time::now(); + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + auto r = R.move_as_ok(); + std::vector> by_name; + for (const auto &stats : r) { + if (stats.name == query.name_ || query.name_.empty()) { + std::vector> by_time; + for (const auto &t : times) { + double min = std::numeric_limits::lowest(); + double max = std::numeric_limits::max(); + double sum = 0; + int cnt = 0; + for (const auto &stat : stats.stats) { + double time = stat.first; + double duration = stat.second; + if (now - time <= static_cast(t)) { + min = td::min(min, duration); + max = td::max(max, duration); + sum += duration; + ++cnt; } - by_name.push_back(ton::create_tl_object(stats.name, std::move(by_time))); } + by_time.push_back(ton::create_tl_object( + t, min, sum / static_cast(cnt), max)); } - promise.set_value(ton::create_serialize_tl_object(std::move(by_name))); + by_name.push_back(ton::create_tl_object( + stats.name, std::move(by_time))); } - }); - td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::prepare_perf_timer_stats, std::move(P)); + } + promise.set_value( + ton::create_serialize_tl_object(std::move(by_name))); + } + }); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::prepare_perf_timer_stats, + std::move(P)); } void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getShardOutQueueSize &query, @@ -4015,9 +4309,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setExtMes }); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCustomOverlay &query, - td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, - td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCustomOverlay &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_modify)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -4086,9 +4379,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCustom }); } -void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCustomOverlays &query, - td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, - td::Promise promise) { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCustomOverlays &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_default)) { promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); return; @@ -4312,7 +4604,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addShard if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); } else { - promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); } }); } @@ -4348,7 +4641,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delShard if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); } else { - promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); } }); } @@ -4484,7 +4778,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCollat if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); } else { - promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); } }); } @@ -4529,7 +4824,8 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCollat if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); } else { - promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); } }); } @@ -4550,30 +4846,16 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_signOverl ton::adnl::AdnlNodeIdShort adnl_id{query.adnl_id_}; int slot = query.slot_; int expire_at = query.expire_at_; - td::actor::send_closure( - keyring_, &ton::keyring::Keyring::get_public_key, public_key_hash, - [=, keyring = keyring_.get(), promise = std::move(promise)](td::Result R) mutable { + + issue_fast_sync_overlay_certificate( + public_key_hash, adnl_id, 0, slot, expire_at, + [promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); return; } - ton::overlay::OverlayMemberCertificate certificate{R.move_as_ok(), 0, slot, expire_at, td::BufferSlice{}}; - if (certificate.is_expired()) { - promise.set_value( - create_control_query_error(td::Status::Error(ton::ErrorCode::error, "certificate is expired"))); - return; - } - td::BufferSlice to_sign = certificate.to_sign_data(adnl_id); - td::actor::send_closure(keyring, &ton::keyring::Keyring::sign_message, public_key_hash, std::move(to_sign), - [certificate = std::move(certificate), - promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_value(create_control_query_error(R.move_as_error())); - return; - } - certificate.set_signature(R.move_as_ok()); - promise.set_value(ton::serialize_tl_object(certificate.tl(), true)); - }); + auto cert = R.move_as_ok(); + promise.set_value(ton::serialize_tl_object(cert.tl(), true)); }); } @@ -4600,23 +4882,92 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_importFas promise.set_value(create_control_query_error(std::move(S))); return; } - for (auto &old_cert : config_.fast_sync_member_certificates) { - if (old_cert.first == adnl_id && old_cert.second.issued_by() == certificate.issued_by() && - old_cert.second.expire_at() == certificate.expire_at() && old_cert.second.slot() == certificate.slot()) { - promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "duplicate certificate"))); + + try_import_fast_sync_member_certificate( + std::move(adnl_id), std::move(certificate), [promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addFastSyncClient &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + ton::adnl::AdnlNodeIdShort adnl_id{query.adnl_id_}; + td::int32 slot = query.slot_; + if (slot < 0 || slot >= ton::validator::fullnode::FullNode::MAX_FAST_SYNC_OVERLAY_CLIENTS) { + promise.set_value(create_control_query_error(td::Status::Error( + PSTRING() << "invalid slot (max " << ton::validator::fullnode::FullNode::MAX_FAST_SYNC_OVERLAY_CLIENTS + << " clients)"))); + return; + } + + bool found = false; + for (auto &c : config_.fast_sync_overlay_clients) { + if (c.slot == slot) { + if (c.id == adnl_id) { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + issue_fast_sync_overlay_certificates(); + } else { + promise.set_value(create_control_query_error(td::Status::Error("duplicate slot"))); + } return; } + if (c.id == adnl_id) { + found = true; + c.slot = slot; + } } + if (!found) { + config_.fast_sync_overlay_clients.emplace_back(adnl_id, slot); + } + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value( + ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); + issue_fast_sync_overlay_certificates(); +} - config_.fast_sync_member_certificates.emplace_back(adnl_id, certificate); - write_config([=, promise = std::move(promise), full_node = full_node_.get()](td::Result R) mutable { +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delFastSyncClient &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + ton::adnl::AdnlNodeIdShort adnl_id{query.adnl_id_}; + for (auto &c : config_.fast_sync_overlay_clients) { + if (c.id == adnl_id) { + std::swap(c, config_.fast_sync_overlay_clients.back()); + config_.fast_sync_overlay_clients.pop_back(); + break; + } + } + write_config([promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); } else { - if (!full_node.empty()) { - td::actor::send_closure(full_node, &ton::validator::fullnode::FullNode::import_fast_sync_member_certificate, - adnl_id, std::move(certificate)); - } promise.set_value( ton::serialize_tl_object(ton::create_tl_object(), true)); } @@ -4692,9 +5043,8 @@ void ValidatorEngine::get_current_validator_perm_key(td::Promiseget_total_validator_set(0); - CHECK(val_set.not_null()); - auto vec = val_set->export_vector(); + CHECK(validator_set_.not_null()); + auto vec = validator_set_->export_vector(); for (size_t idx = 0; idx < vec.size(); idx++) { auto &el = vec[idx]; ton::PublicKey pub{ton::pubkeys::Ed25519{el.key.as_bits256()}}; diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 89dd22cc4..eac31f46a 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -27,11 +27,14 @@ */ #pragma once +#include "adnl/adnl-node-id.hpp" #include "adnl/adnl.h" #include "auto/tl/ton_api.h" +#include "overlays.h" #include "rldp/rldp.h" #include "rldp2/rldp.h" #include "dht/dht.h" +#include "td/actor/PromiseFuture.h" #include "validator/manager.h" #include "validator/validator.h" #include "validator/full-node.h" @@ -75,6 +78,13 @@ struct Config { ton::PublicKey key; td::IPAddress addr; }; + struct FastSyncOverlayClient { + FastSyncOverlayClient() = default; + FastSyncOverlayClient(ton::adnl::AdnlNodeIdShort id, td::int32 slot) : id(id), slot(slot) { + } + ton::adnl::AdnlNodeIdShort id; + td::int32 slot; + }; std::map keys_refcnt; td::uint16 out_port; @@ -93,6 +103,7 @@ struct Config { std::map controls; std::set gc; std::vector shards_to_monitor; + std::vector fast_sync_overlay_clients; bool state_serializer_enabled = true; std::vector> @@ -158,6 +169,7 @@ class ValidatorEngine : public td::actor::Actor { td::actor::ActorOwn validator_manager_; td::actor::ActorOwn full_node_client_; td::actor::ActorOwn full_node_; + ton::adnl::AdnlNodeIdShort full_node_id_ = ton::adnl::AdnlNodeIdShort::zero(); std::map> full_node_masters_; td::actor::ActorOwn control_ext_server_; @@ -187,13 +199,13 @@ class ValidatorEngine : public td::actor::Actor { std::map keys_; td::Ref state_; + td::Ref validator_set_, validator_set_prev_, validator_set_next_; + td::Timestamp issue_fast_sync_overlay_certificates_at_ = td::Timestamp::now(); td::Promise get_key_promise(td::MultiPromise::InitGuard &ig); void got_key(ton::PublicKey key); void deleted_key(ton::PublicKeyHash key); - void got_state(td::Ref state) { - state_ = std::move(state); - } + void got_state(td::Ref state); void write_config(td::Promise promise); @@ -420,6 +432,20 @@ class ValidatorEngine : public td::actor::Actor { void try_del_proxy(td::uint32 ip, td::int32 port, std::vector cats, std::vector prio_cats, td::Promise promise); + void register_fast_sync_certificate_callback(); + void try_import_fast_sync_member_certificate(ton::adnl::AdnlNodeIdShort id, + ton::overlay::OverlayMemberCertificate certificate, + td::Promise promise); + + void issue_fast_sync_overlay_certificates(); + void issue_fast_sync_overlay_certificate(ton::PublicKeyHash issue_by, ton::adnl::AdnlNodeIdShort issue_to, + td::uint32 flags, td::int32 slot, td::int32 expire_at, + td::Promise promise); + void process_fast_sync_overlay_certificate_request(ton::PublicKeyHash issue_by, ton::adnl::AdnlNodeIdShort issue_to, + td::uint32 flags, td::int32 slot, td::int32 expire_at, + td::Promise promise); + ton::PublicKeyHash find_local_validator_for_cert_issuing(); + std::string custom_overlays_config_file() const { return db_root_ + "/custom-overlays.json"; } @@ -432,8 +458,8 @@ class ValidatorEngine : public td::actor::Actor { void load_custom_overlays_config(); td::Status write_custom_overlays_config(); - void add_custom_overlay_to_config( - ton::tl_object_ptr overlay, td::Promise promise); + void add_custom_overlay_to_config(ton::tl_object_ptr overlay, + td::Promise promise); void del_custom_overlay_from_config(std::string name, td::Promise promise); void load_collator_options(); @@ -560,6 +586,10 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getAdnlStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_addFastSyncClient &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_delFastSyncClient &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index 947f9dee8..6554c5180 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -308,7 +308,7 @@ void FullNodeFastSyncOverlay::init() { options.default_permanent_members_flags_ = overlay::OverlayMemberFlags::DoNotReceiveBroadcasts; } options.local_overlay_member_flags_ = receive_broadcasts_ ? 0 : overlay::OverlayMemberFlags::DoNotReceiveBroadcasts; - options.max_slaves_in_semiprivate_overlay_ = 100000; // TODO: set lower limit (high limit for testing) + options.max_slaves_in_semiprivate_overlay_ = FullNode::MAX_FAST_SYNC_OVERLAY_CLIENTS; td::actor::send_closure(overlays_, &overlay::Overlays::create_semiprivate_overlay, local_id_, overlay_id_full_.clone(), current_validators_adnl_, root_public_keys_, member_certificate_, std::make_unique(actor_id(this)), rules, std::move(scope), options); diff --git a/validator/full-node.h b/validator/full-node.h index f8fc123be..c9a49472d 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -111,6 +111,8 @@ class FullNode : public td::actor::Actor { } enum { broadcast_mode_public = 1, broadcast_mode_private_block = 2, broadcast_mode_custom = 4 }; + static constexpr td::int32 MAX_FAST_SYNC_OVERLAY_CLIENTS = 5000; // TODO: set lower limit (high limit for testing) + static td::actor::ActorOwn create( ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 414b3170a..b625e4c2d 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -100,7 +100,10 @@ class FullNodeImpl : public FullNode { void set_validator_telemetry_filename(std::string value) override; void import_fast_sync_member_certificate(adnl::AdnlNodeIdShort local_id, - overlay::OverlayMemberCertificate cert) override { + overlay::OverlayMemberCertificate cert) override { + VLOG(FULL_NODE_DEBUG) << "Importing fast sync overlay certificate for " << local_id << " issued by " + << cert.issued_by().compute_short_id() << " expires in " + << (double)cert.expire_at() - td::Clocks::system(); fast_sync_overlays_.add_member_certificate(local_id, std::move(cert)); } From 20c20e236b4a9341174dd648e98c9d5585f1fcc7 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 10 Dec 2024 15:46:28 +0300 Subject: [PATCH 132/388] Check peer version before getOutMsgQueueProof --- validator/full-node-shard.cpp | 30 +++++++++++++++++++++--------- validator/full-node-shard.hpp | 2 +- validator/manager.cpp | 2 +- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 7d33a1950..e62bfe42b 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -1005,7 +1005,7 @@ void FullNodeShardImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std: block::ImportedMsgQueueLimits limits, td::Timestamp timeout, td::Promise>> promise) { // TODO: maybe more complex download (like other requests here) - auto &b = choose_neighbour(); + auto &b = choose_neighbour(3, 0); // Required version: 3.0 if (b.adnl_id == adnl::AdnlNodeIdShort::zero()) { promise.set_error(td::Status::Error(ErrorCode::notready, "no nodes")); return; @@ -1252,24 +1252,36 @@ void FullNodeShardImpl::got_neighbours(std::vector vec) { } } -const Neighbour &FullNodeShardImpl::choose_neighbour() const { +const Neighbour &FullNodeShardImpl::choose_neighbour(td::uint32 required_version_major, + td::uint32 required_version_minor) const { if (neighbours_.size() == 0) { return Neighbour::zero; } + auto is_eligible = + [&](const Neighbour &n) { + return n.version_major > required_version_major || + (n.version_major == required_version_major && n.version_minor >= required_version_minor); + }; double min_unreliability = 1e9; - for (auto &x : neighbours_) { - min_unreliability = std::min(min_unreliability, x.second.unreliability); + for (auto &[_, x] : neighbours_) { + if (!is_eligible(x)) { + continue; + } + min_unreliability = std::min(min_unreliability, x.unreliability); } const Neighbour *best = nullptr; td::uint32 sum = 0; - for (auto &x : neighbours_) { - auto unr = static_cast(x.second.unreliability - min_unreliability); + for (auto &[_, x] : neighbours_) { + if (!is_eligible(x)) { + continue; + } + auto unr = static_cast(x.unreliability - min_unreliability); - if (x.second.version_major < proto_version_major()) { + if (x.version_major < proto_version_major()) { unr += 4; - } else if (x.second.version_major == proto_version_major() && x.second.version_minor < proto_version_minor()) { + } else if (x.version_major == proto_version_major() && x.version_minor < proto_version_minor()) { unr += 2; } @@ -1279,7 +1291,7 @@ const Neighbour &FullNodeShardImpl::choose_neighbour() const { auto w = 1 << (f - unr); sum += w; if (td::Random::fast(0, sum - 1) <= w - 1) { - best = &x.second; + best = &x; } } } diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index fd1ef943e..59f671c9a 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -209,7 +209,7 @@ class FullNodeShardImpl : public FullNodeShard { void got_neighbours(std::vector res); void update_neighbour_stats(adnl::AdnlNodeIdShort adnl_id, double t, bool success); void got_neighbour_capabilities(adnl::AdnlNodeIdShort adnl_id, double t, td::BufferSlice data); - const Neighbour &choose_neighbour() const; + const Neighbour &choose_neighbour(td::uint32 required_version_major = 0, td::uint32 required_version_minor = 0) const; template td::Promise create_neighbour_promise(const Neighbour &x, td::Promise p, bool require_state = false) { diff --git a/validator/manager.cpp b/validator/manager.cpp index b8394bdc5..31278788c 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1794,7 +1794,7 @@ void ValidatorManagerImpl::send_validator_telemetry(PublicKeyHash key, void ValidatorManagerImpl::send_get_out_msg_queue_proof_request( ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) { - callback_->download_out_msg_queue_proof(dst_shard, std::move(blocks), limits, td::Timestamp::in(10.0), + callback_->download_out_msg_queue_proof(dst_shard, std::move(blocks), limits, td::Timestamp::in(5.0), std::move(promise)); } From 160b539eaad7bc97b7e238168756cca676a5f3be Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 11 Dec 2024 16:07:48 +0300 Subject: [PATCH 133/388] Add logs add-collator --- .../validator-engine-console-query.cpp | 8 ++++++-- validator-engine-console/validator-engine-console-query.h | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index d229b9329..182234067 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1607,7 +1607,9 @@ td::Status AddCollatorQuery::send() { td::Status AddCollatorQuery::receive(td::BufferSlice data) { TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), "received incorrect answer: "); - td::TerminalIO::out() << "successfully added collator\n"; + td::TerminalIO::out() << "successfully added collator for shard " << shard_.to_str() << "\n"; + td::TerminalIO::out() << "ADNL ID = " << adnl_id_.bits256_value().to_hex() << " (" << adnl_id_.bits256_value() + << ")\n"; return td::Status::OK(); } @@ -1627,7 +1629,9 @@ td::Status DelCollatorQuery::send() { td::Status DelCollatorQuery::receive(td::BufferSlice data) { TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), "received incorrect answer: "); - td::TerminalIO::out() << "successfully removed collator\n"; + td::TerminalIO::out() << "successfully removed collator for shard " << shard_.to_str() << "\n"; + td::TerminalIO::out() << "ADNL ID = " << adnl_id_.bits256_value().to_hex() << " (" << adnl_id_.bits256_value() + << ")\n"; return td::Status::OK(); } diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 417374518..ad9f7c3b3 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -151,8 +151,8 @@ inline td::Result Tokenizer::get_token() { auto r_wc = td::to_integer_safe(word); if (r_wc.is_ok()) { TRY_RESULT_ASSIGN(word, get_raw_token()); - TRY_RESULT(shard, td::to_integer_safe(word)); - return ton::ShardIdFull{r_wc.move_as_ok(), shard}; + TRY_RESULT(shard, td::to_integer_safe(word)); + return ton::ShardIdFull{r_wc.move_as_ok(), (ton::ShardId)shard}; } return ton::ShardIdFull::parse(word); } From c955a5333d7c8643b67d76ae42950af7e97796d1 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 24 Dec 2024 11:58:51 +0300 Subject: [PATCH 134/388] Fix loading block candidates in WaitBlockData --- validator/downloaders/wait-block-data.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/downloaders/wait-block-data.cpp b/validator/downloaders/wait-block-data.cpp index 53a3d351b..86e8f4b8c 100644 --- a/validator/downloaders/wait-block-data.cpp +++ b/validator/downloaders/wait-block-data.cpp @@ -106,7 +106,7 @@ void WaitBlockData::start() { }); td::actor::send_closure(manager_, &ValidatorManager::try_get_static_file, handle_->id().file_hash, std::move(P)); - } else if (try_get_candidate_) { + } else if (try_get_candidate_ && !handle_->id().is_masterchain()) { try_get_candidate_ = false; td::actor::send_closure( manager_, &ValidatorManager::get_candidate_data_by_block_id_from_db, handle_->id(), From 392cf64758a0698c4b237c3e233a9727b11685aa Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 26 Dec 2024 14:03:00 +0300 Subject: [PATCH 135/388] Support optional fields in tl json generator --- tdtl/td/tl/tl_simple.h | 13 +++++++++++++ tl/generate/tl_json_converter.cpp | 29 +++++++++++++++++++++++------ 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/tdtl/td/tl/tl_simple.h b/tdtl/td/tl/tl_simple.h index dd5a1e53a..06d6ea133 100644 --- a/tdtl/td/tl/tl_simple.h +++ b/tdtl/td/tl/tl_simple.h @@ -81,12 +81,16 @@ struct Type { struct Arg { const Type *type; std::string name; + int var_num = -1; + int exist_var_num = -1; + int exist_var_bit = -1; }; struct Constructor { std::string name; std::int32_t id; std::vector args; + int var_count = 0; const CustomType *type; }; @@ -100,6 +104,7 @@ struct CustomType { struct Function { std::string name; + int var_count = 0; std::int32_t id; std::vector args; const Type *type; @@ -248,11 +253,15 @@ class Schema { constructor = constructors_.back().get(); constructor->id = from->id; constructor->name = from->name; + constructor->var_count = from->var_count; constructor->type = get_custom_type(config_->get_type(from->type_id)); for (auto &from_arg : from->args) { Arg arg; arg.name = from_arg.name; arg.type = get_type(from_arg.type); + arg.var_num = from_arg.var_num; + arg.exist_var_num = from_arg.exist_var_num; + arg.exist_var_bit = from_arg.exist_var_bit; constructor->args.push_back(std::move(arg)); } } @@ -266,11 +275,15 @@ class Schema { function = functions_.back().get(); function->id = from->id; function->name = from->name; + function->var_count = from->var_count; function->type = get_type(config_->get_type(from->type_id)); for (auto &from_arg : from->args) { Arg arg; arg.name = from_arg.name; arg.type = get_type(from_arg.type); + arg.var_num = from_arg.var_num; + arg.exist_var_num = from_arg.exist_var_num; + arg.exist_var_bit = from_arg.exist_var_bit; function->args.push_back(std::move(arg)); } } diff --git a/tl/generate/tl_json_converter.cpp b/tl/generate/tl_json_converter.cpp index a0213ef51..d1a0488fb 100644 --- a/tl/generate/tl_json_converter.cpp +++ b/tl/generate/tl_json_converter.cpp @@ -48,13 +48,30 @@ void gen_to_json_constructor(StringBuilder &sb, const T *constructor, bool is_he sb << " {\n"; sb << " auto jo = jv.enter_object();\n"; sb << " jo(\"@type\", \"" << constructor->name << "\");\n"; + std::vector var_names(constructor->var_count); + for (auto &arg : constructor->args) { + if (arg.var_num >= 0) { + CHECK(arg.var_num < (int)var_names.size()); + var_names[arg.var_num] = tl::simple::gen_cpp_field_name(arg.name); + } + } for (auto &arg : constructor->args) { auto field_name = tl::simple::gen_cpp_field_name(arg.name); - // TODO: or as null - bool is_custom = arg.type->type == tl::simple::Type::Custom; - - if (is_custom) { - sb << " if (object." << field_name << ") {\n "; + bool is_optional = arg.type->type == tl::simple::Type::Custom || arg.exist_var_num >= 0; + + if (is_optional) { + sb << " if ("; + if (arg.type->type == tl::simple::Type::Custom) { + sb << "object." << field_name; + if (arg.exist_var_num >= 0) { + sb << " && "; + } + } + if (arg.exist_var_num >= 0) { + CHECK(arg.exist_var_num < (int)var_names.size()); + sb << "(object." << var_names[arg.exist_var_num] << " & " << (1 << arg.exist_var_bit) << ")"; + } + sb << ") {\n "; } auto object = PSTRING() << "object." << tl::simple::gen_cpp_field_name(arg.name); if (arg.type->type == tl::simple::Type::Bytes || arg.type->type == tl::simple::Type::SecureBytes) { @@ -72,7 +89,7 @@ void gen_to_json_constructor(StringBuilder &sb, const T *constructor, bool is_he object = PSTRING() << "JsonVectorInt64{" << object << "}"; } sb << " jo(\"" << arg.name << "\", ToJson(" << object << "));\n"; - if (is_custom) { + if (is_optional) { sb << " }\n"; } } From 71342bdcd4ba387f0629f797bf9de722c05752cd Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 26 Dec 2024 14:03:12 +0300 Subject: [PATCH 136/388] Rework session stats --- tl/generate/scheme/ton_api.tl | 90 +++++++++------ tl/generate/scheme/ton_api.tlo | Bin 110212 -> 111624 bytes ton/ton-types.h | 11 ++ validator-session/validator-session-types.h | 116 ++++++++++++++++--- validator-session/validator-session.cpp | 70 +++++------ validator-session/validator-session.h | 5 - validator-session/validator-session.hpp | 3 +- validator/collation-manager.cpp | 64 +++++----- validator/collation-manager.hpp | 4 +- validator/collator-node.cpp | 16 ++- validator/fabric.h | 3 +- validator/impl/collator-impl.h | 4 +- validator/impl/collator.cpp | 31 ++--- validator/impl/fabric.cpp | 13 ++- validator/impl/validate-query.cpp | 36 +++--- validator/impl/validate-query.hpp | 2 +- validator/interfaces/validator-manager.h | 78 ++++++++++--- validator/manager-disk.cpp | 3 +- validator/manager-disk.hpp | 9 -- validator/manager-hardfork.hpp | 9 -- validator/manager.cpp | 122 ++++++-------------- validator/manager.hpp | 18 +-- validator/validator-group.cpp | 55 ++++----- validator/validator-group.hpp | 12 +- 24 files changed, 438 insertions(+), 336 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 396c06bc1..bdcf0a296 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -895,48 +895,66 @@ http.server.config dhs:(vector http.server.dnsEntry) local_hosts:(vector http.se ---types--- -validatorSession.collationStats actual_bytes:int actual_collated_data_bytes:int - bytes:int gas:int lt_delta:int collated_data_bytes:int - cat_bytes:int cat_gas:int cat_lt_delta:int cat_collated_data_bytes:int - limits_log:string ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int - work_time:double cpu_work_time:double - serialized_size:int serialized_size_no_collated_data:int = validadorSession.CollationStats; - -validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int root_hash:int256 file_hash:int256 - comment:string block_timestamp:double is_accepted:Bool is_ours:Bool got_submit_at:double - collation_time:double collated_at:double collation_cached:Bool - collation_work_time:double collation_cpu_work_time:double - collation_stats:validatorSession.collationStats - validation_time:double validated_at:double validation_cached:Bool - validation_work_time:double validation_cpu_work_time:double - gen_utime:double - approved_weight:long approved_33pct_at:double approved_66pct_at:double approvers:string - signed_weight:long signed_33pct_at:double signed_66pct_at:double signers:string - serialize_time:double deserialize_time:double serialized_size:int = validatorSession.StatsProducer; - -validatorSession.statsRound timestamp:double producers:(vector validatorSession.statsProducer) = validatorSession.StatsRound; - -validatorSession.stats success:Bool id:tonNode.blockIdExt timestamp:double self:int256 session_id:int256 cc_seqno:int - creator:int256 total_validators:int total_weight:long - signatures:int signatures_weight:long approve_signatures:int approve_signatures_weight:long - first_round:int rounds:(vector validatorSession.statsRound) = validatorSession.Stats; - -validatorSession.statsCollatedBlock timestamp:double id:tonNode.blockIdExt stats:validatorSession.collationStats = validatorSession.StatsCollatedBlock; +validatorStats.stats.producer flags:# + validator_id:int256 block_status:int + candidate_id:flags.0?int256 block_id:flags.0?tonNode.blockIdExt collated_data_hash:flags.0?int256 + is_accepted:flags.0?Bool is_ours:flags.0?Bool + got_block_at:flags.0?double got_submit_at:flags.0?double gen_utime:flags.0?int comment:flags.0?string + collation_time:flags.1?double collated_at:flags.1?double collation_cached:flags.1?Bool self_collated:flags.1?Bool collator_node_id:flags.2?int256 + validation_time:flags.3?double validated_at:flags.3?double validation_cached:flags.3?Bool + approved_weight:flags.0?long approved_33pct_at:flags.0?double approved_66pct_at:flags.0?double approvers:flags.0?string + signed_weight:flags.0?long signed_33pct_at:flags.0?double signed_66pct_at:flags.0?double signers:flags.0?string + serialize_time:flags.4?double deserialize_time:flags.4?double serialized_size:flags.4?int = validatorStats.stats.Producer; + +validatorStats.stats.round started_at:double producers:(vector validatorStats.stats.producer) = validatorStats.stats.Round; + +validatorStats.stats flags:# + session_id:int256 self:int256 block_id:tonNode.blockIdExt cc_seqno:int success:Bool timestamp:double + creator:flags.0?int256 + total_validators:int total_weight:long + signatures:flags.0?int signatures_weight:flags.0?long + approve_signatures:flags.0?int approve_signatures_weight:flags.0?long + first_round:int rounds:(vector validatorStats.stats.round) = validatorStats.Stats; + +validatorStats.blockLimitsStatus + bytes:int gas:int lt_delta:int collated_data_bytes:int + cat_bytes:int cat_gas:int cat_lt_delta:int cat_collated_data_bytes:int + limits_log:string = validatorStats.BlockLimitsStatus; +validatorStats.extMsgsStats + ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int = validatorStats.ExtMsgsStats; + +validatorStats.collatedBlock flags:# + block_id:tonNode.blockIdExt collated_data_hash:int256 cc_seqno:int collated_at:double + bytes:int collated_data_bytes:int attempt:int + collator_node_id:flags.0?int256 validator_id:flags.1?int256 + total_time:double work_time:double cpu_work_time:double time_stats:string + block_limits:validatorStats.blockLimitsStatus + ext_msgs_stats:validatorStats.extMsgsStats = validatorSession.stats.CollatedBlock; + +validatorStats.validatedBlock + block_id:tonNode.blockIdExt collated_data_hash:int256 validated_at:double + valid:Bool comment:string + bytes:int collated_data_bytes:int + total_time:double work_time:double cpu_work_time:double = validatorStats.ValidatedBlock; + +validatorStats.newValidatorGroup.node id:int256 weight:long = validatorStats.newValidatorGroup.Node; +validatorStats.newValidatorGroup session_id:int256 shard:tonNode.shardId cc_seqno:int + last_key_block_seqno:int started_at:double + self_idx:int nodes:(vector validatorStats.newValidatorGroup.node) = validatorStats.NewValidatorGroup; + +validatorStats.endValidatorGroup.node id:int256 catchain_blocks:int = validatorStats.endValidatorGroup.Node; +validatorStats.endValidatorGroup session_id:int256 timestamp:double + nodes:(vector validatorStats.endValidatorGroup.node) = validatorStats.EndValidatorGroup; + +validatorStats.collatorNodeResponse collator_node_id:int256 validator_id:int256 timestamp:double + block_id:tonNode.blockIdExt original_block_id:tonNode.blockIdExt collated_data_hash:int256 + = validatorStats.CollatorNodeResponse; collatorNode.candidate source:PublicKey id:tonNode.blockIdExt data:bytes collated_data:bytes = collatorNode.Candidate; collatorNode.compressedCandidate flags:# source:PublicKey id:tonNode.blockIdExt decompressed_size:int data:bytes = collatorNode.Candidate; collatorNode.pong flags:# = collatorNode.Pong; collatorNode.error code:int message:string = collatorNode.Error; -validatorSession.newValidatorGroupStats.node id:int256 weight:long = validatorSession.newValidatorGroupStats.Node; -validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:long cc_seqno:int - last_key_block_seqno:int timestamp:double - self_idx:int nodes:(vector validatorSession.newValidatorGroupStats.node) = validatorSession.NewValidatorGroupStats; - -validatorSession.endValidatorGroupStats.node id:int256 catchain_blocks:int = validatorSession.endValidatorGroupStats.Node; -validatorSession.endValidatorGroupStats session_id:int256 timestamp:double - nodes:(vector validatorSession.endValidatorGroupStats.node) = validatorSession.EndValidatorGroupStats; - ---functions--- collatorNode.generateBlock shard:tonNode.shardId cc_seqno:int prev_blocks:(vector tonNode.blockIdExt) creator:int256 round:int first_block_round:int priority:int = collatorNode.Candidate; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index faef1564525053095bedcd3d1a7a3d48fe583ec0..f091213cf90e633cb8909ffc33ba05310ad4d1f0 100644 GIT binary patch delta 2515 zcma)8Urbw781J{W1zP^JP^8$h7HJg%B2$pTn1IN}q(CWuB#Pd4?G-AexAtB(GFc+H z8BE+99?f|WP08Yum^t@=F>!GZ8cCKojV6+8I*rlA_~3(ydyvfY-CnxOrAy?Y;e6+J ze&2V#^ZS17$?L|&o5q4)-g&51r~K`|Q1E-p#fH|KK*rnlkssO`$?K0vhF?kHfQQbH zz&(ZaT|z7tlK9BuF(EufzInt+puKsiLGS&8n@G8jhGNl3ND`7mQb-c(WsomA9hF23 z%!s7Z8lFHB~kTwQT+TF%8?w{Z9M0|2$#SkjaQs|(p_Nme5TDT%FXsCg~&{p+$eIo z;v<`$do_vEpnxzFk1U-ph4qUq?Qi_OW^0DIVmkgLr07W>1lN2|LD2XhKDbkJKCFIt zrWeF)d|B+X&`K-Rk=GWg3X8We9MeFEo|y(0{g49>-57)_I@Sno`bi_y(bzNKwph|& zA<4zrn{!cWSq2-O^Mlo%$;?F~EYmJUoj@*}D;m8G*4Wya_WB+d}b3aIr)C1 z=+pAC{3EDME+*?vIh%Xn)yt?*P}Yv$eyS-a#DdXbYF@GS*=XZ%b3p z!dp$>?=g*M)4H5gt#!tSIP6j0+F&CL0J+hgJAyH#2pw`|n})5{sHXvO$Yo@QvHETO zqtkwH=d1fhR!%wT^;W1NiwmAotB{ONMB||tm+p<+S2F)nSzhMN$i|N@a|3_udAo;HonX%?~Ge^h;UeWS(=-TYV5Dsj!aw*$fI2Qq=>BD> zcsmFNbYwGaTSgVda_H?ru;$53tTjNax9oP2k3G+A`MKJIS3W4K*ycUVyVWke3tgJ7 z2EdYG+)XNDWvHO-Av@g+K#2;)s4YdI1Z(_JHjKyk=)|OymipNjSjj_BqFndq0Gk2B zaAwV}@3bfflcG{BHh>NfvvZK#zlM=z>It+cy;DYIWN@rVxhQDyiJXz(gmlE3S&jzF~?NwuSn2iNMJgE9Yq DLoy>h delta 1670 zcmb7EUrbwN6z@@5I_y|S3*1@)C19Xghy2k9%5-dXkZfaRAb%X}_O|yX>(=$M-di>o z6ZbF;7!zS%+%eIIy3A-I%TOL_j1P1n^JR&?m<&^5VivcUJ=BE6J?z2n+``(GzU)5S zeBb%=JLfy+ch2ks%jI>;&cD8R`E}Fg*Zuv@`#sIyJ-+o{!;BP-$datahZRkWsIj(# zYBVb8{4}gfy0)++EN}EY0N>SrI>nZpv7R@6J0_1B09@7|R}<5Q6I2h!sGKk5lx%mg>fWyE_Ef`%HN}xwqq)I5*OmOGsaVrygv@2F$Ooco_zbZ0bG1!# zELv4x+By(dGl@-=k;qG|U zmD#)9D^~LLPrTSbuYP4;H{4!cf9ul?v(Y2`suvvI!nTlj?RM1>2XCnR??1gZDXb@? zs2F-rS2Wgmqt22O;L33jSigTa%N{Ue&oO97-5Y{d!08Z}aq1oThRyezac=+)By6Lz_7Si?5buOzE1DEfv9vM@L}~JN2<2KOcn=yf6;UXdiBEVE+w zQss*g$@D2i;yfW%ZBg7rV?bz)tnce4ygS94EPo*2(iF6!?M)a!&k69@cZFzhL_Km= z(RAtT^k-RcVdNydz>;%M<#T4L!#|z&j#1 zS@SKy>3Cuyo>${JeDZbG<`v=TL|$pabO;1@Udyx5HF_fPgi63xesW?o2v%GIp5xlP z(1NQ$9&cmO<=mW6zP<90*yjhgO(;;|PyifQEIBbY-4!2c8Q<%On{WcbzN>0iJ0+#rS&S&g6sp qCt+lOPK9XCw6-?U$cHU`+lr4Tp}CB1sSF=6%he&6P5l^#K;VDclU@)2 diff --git a/ton/ton-types.h b/ton/ton-types.h index 220af6e02..51d9e2d2e 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -501,6 +501,17 @@ struct BlockCandidate { } }; +struct GeneratedCandidate { + BlockCandidate candidate; + bool is_cached = false; + bool self_collated = false; + td::Bits256 collator_node_id = td::Bits256::zero(); + + GeneratedCandidate clone() const { + return {candidate.clone(), is_cached, self_collated, collator_node_id}; + } +}; + struct BlockCandidatePriority { td::uint32 round{}; td::uint32 first_block_round{}; diff --git a/validator-session/validator-session-types.h b/validator-session/validator-session-types.h index 2edc42bbd..59b2141aa 100644 --- a/validator-session/validator-session-types.h +++ b/validator-session/validator-session-types.h @@ -23,6 +23,8 @@ #include "adnl/adnl-node-id.hpp" #include "ton/ton-types.h" +#include + namespace ton { namespace validatorsession { @@ -73,24 +75,25 @@ struct ValidatorSessionStats { enum { status_none = 0, status_received = 1, status_rejected = 2, status_approved = 3 }; struct Producer { - PublicKeyHash id = PublicKeyHash::zero(); - ValidatorSessionCandidateId candidate_id = ValidatorSessionCandidateId::zero(); int block_status = status_none; - double block_timestamp = -1.0; - td::Bits256 root_hash = td::Bits256::zero(); - td::Bits256 file_hash = td::Bits256::zero(); - std::string comment; - + PublicKeyHash validator_id = PublicKeyHash::zero(); + ValidatorSessionCandidateId candidate_id = ValidatorSessionCandidateId::zero(); + BlockIdExt block_id{workchainIdNotYet, 0, 0, td::Bits256::zero(), td::Bits256::zero()}; + td::Bits256 collated_data_hash = td::Bits256::zero(); bool is_accepted = false; bool is_ours = false; + double got_block_at = -1.0; double got_submit_at = -1.0; - double collation_time = -1.0; - double validation_time = -1.0; - double collated_at = -1.0; - double validated_at = -1.0; + td::int32 gen_utime = -1; + std::string comment; + + double collation_time = -1.0, collated_at = -1.0; bool collation_cached = false; + bool self_collated = false; + td::Bits256 collator_node_id = td::Bits256::zero(); + + double validation_time = -1.0, validated_at = -1.0; bool validation_cached = false; - double gen_utime = -1.0; std::vector approvers, signers; ValidatorWeight approved_weight = 0; @@ -129,20 +132,52 @@ struct ValidatorSessionStats { } } } + + tl_object_ptr tl() const { + std::string approvers_str(approvers.size(), '0'); + for (size_t i = 0; i < approvers.size(); ++i) { + approvers_str[i] = '0' + approvers[i]; + } + std::string signers_str(signers.size(), '0'); + for (size_t i = 0; i < signers.size(); ++i) { + signers_str[i] = '0' + signers[i]; + } + int flags = + (block_status != status_none ? ton_api::validatorStats_stats_producer::Flags::BLOCK_ID_MASK : 0) | + (collated_at >= 0.0 ? ton_api::validatorStats_stats_producer::Flags::COLLATED_AT_MASK : 0) | + (!collator_node_id.is_zero() ? ton_api::validatorStats_stats_producer::Flags::COLLATOR_NODE_ID_MASK : 0) | + (validated_at >= 0.0 ? ton_api::validatorStats_stats_producer::Flags::VALIDATED_AT_MASK : 0) | + (serialize_time >= 0.0 || deserialize_time >= 0.0 || serialized_size >= 0 + ? ton_api::validatorStats_stats_producer::Flags::SERIALIZE_TIME_MASK + : 0); + return create_tl_object( + flags, validator_id.bits256_value(), block_status, candidate_id, create_tl_block_id(block_id), + collated_data_hash, is_accepted, is_ours, got_block_at, got_submit_at, gen_utime, comment, collation_time, + collated_at, collation_cached, self_collated, collator_node_id, validation_time, validated_at, + validation_cached, approved_weight, approved_33pct_at, approved_66pct_at, std::move(approvers_str), + signed_weight, signed_33pct_at, signed_66pct_at, std::move(signers_str), serialize_time, deserialize_time, + serialized_size); + } }; struct Round { - double timestamp = -1.0; + double started_at = -1.0; std::vector producers; - }; - td::uint32 first_round; - std::vector rounds; + tl_object_ptr tl() const { + std::vector> producers_tl; + for (const auto &producer : producers) { + producers_tl.push_back(producer.tl()); + } + return create_tl_object(started_at, std::move(producers_tl)); + } + }; - bool success = false; ValidatorSessionId session_id = ValidatorSessionId::zero(); + PublicKeyHash self = PublicKeyHash::zero(); + BlockIdExt block_id{workchainIdNotYet, 0, 0, td::Bits256::zero(), td::Bits256::zero()}; CatchainSeqno cc_seqno = 0; + bool success = false; double timestamp = -1.0; - PublicKeyHash self = PublicKeyHash::zero(); PublicKeyHash creator = PublicKeyHash::zero(); td::uint32 total_validators = 0; ValidatorWeight total_weight = 0; @@ -150,6 +185,29 @@ struct ValidatorSessionStats { ValidatorWeight signatures_weight = 0; td::uint32 approve_signatures = 0; ValidatorWeight approve_signatures_weight = 0; + + td::uint32 first_round = 0; + std::vector rounds; + + void fix_block_ids() { + for (auto &round : rounds) { + for (auto &producer : round.producers) { + producer.block_id.id = block_id.id; + } + } + } + + tl_object_ptr tl() const { + std::vector> rounds_tl; + for (const auto &round : rounds) { + rounds_tl.push_back(round.tl()); + } + int flags = success ? ton_api::validatorStats_stats::Flags::CREATOR_MASK : 0; + return create_tl_object( + flags, session_id, self.bits256_value(), create_tl_block_id(block_id), cc_seqno, success, timestamp, + creator.bits256_value(), total_validators, total_weight, signatures, signatures_weight, approve_signatures, + approve_signatures_weight, first_round, std::move(rounds_tl)); + } }; struct NewValidatorGroupStats { @@ -162,9 +220,20 @@ struct NewValidatorGroupStats { ShardIdFull shard{masterchainId}; CatchainSeqno cc_seqno = 0; BlockSeqno last_key_block_seqno = 0; - double timestamp = -1.0; + double started_at = -1.0; td::uint32 self_idx = 0; std::vector nodes; + + tl_object_ptr tl() const { + std::vector> nodes_arr; + for (const auto &node : nodes) { + nodes_arr.push_back( + create_tl_object(node.id.bits256_value(), node.weight)); + } + return create_tl_object(session_id, create_tl_shard_id(shard), cc_seqno, + last_key_block_seqno, started_at, self_idx, + std::move(nodes_arr)); + } }; struct EndValidatorGroupStats { @@ -176,6 +245,15 @@ struct EndValidatorGroupStats { ValidatorSessionId session_id = ValidatorSessionId::zero(); double timestamp = -1.0; std::vector nodes; + + tl_object_ptr tl() const { + std::vector> nodes_arr; + for (const auto &node : nodes) { + nodes_arr.push_back(create_tl_object(node.id.bits256_value(), + node.catchain_blocks)); + } + return create_tl_object(session_id, timestamp, std::move(nodes_arr)); + } }; struct BlockSourceInfo { diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 99ee61f23..5cc31a7ba 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -266,13 +266,14 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice if (stat->block_status == ValidatorSessionStats::status_none) { stat->block_status = ValidatorSessionStats::status_received; } - if (stat->block_timestamp <= 0.0) { - stat->block_timestamp = td::Clocks::system(); + if (stat->got_block_at <= 0.0) { + stat->got_block_at = td::Clocks::system(); } stat->deserialize_time = deserialize_time; stat->serialized_size = data.size(); - stat->root_hash = candidate->root_hash_; - stat->file_hash = file_hash; + stat->block_id.root_hash = candidate->root_hash_; + stat->block_id.file_hash = file_hash; + stat->collated_data_hash = collated_data_file_hash; } if ((td::int32)block_round < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK || @@ -451,35 +452,36 @@ void ValidatorSessionImpl::candidate_approved_signed(td::uint32 round, Validator } } -void ValidatorSessionImpl::generated_block(td::uint32 round, ValidatorSessionCandidateId root_hash, - td::BufferSlice data, td::BufferSlice collated_data, double collation_time, - bool collation_cached) { - if (data.size() > description().opts().max_block_size || - collated_data.size() > description().opts().max_collated_data_size) { - LOG(ERROR) << this << ": generated candidate is too big. Dropping. size=" << data.size() << " " - << collated_data.size(); +void ValidatorSessionImpl::generated_block(td::uint32 round, GeneratedCandidate c, double collation_time) { + if (c.candidate.data.size() > description().opts().max_block_size || + c.candidate.collated_data.size() > description().opts().max_collated_data_size) { + LOG(ERROR) << this << ": generated candidate is too big. Dropping. size=" << c.candidate.data.size() << " " + << c.candidate.collated_data.size(); return; } - auto file_hash = sha256_bits256(data.as_slice()); - auto collated_data_file_hash = sha256_bits256(collated_data.as_slice()); - auto block_id = description().candidate_id(local_idx(), root_hash, file_hash, collated_data_file_hash); + auto file_hash = sha256_bits256(c.candidate.data.as_slice()); + auto collated_data_file_hash = sha256_bits256(c.candidate.collated_data.as_slice()); + auto block_id = description().candidate_id(local_idx(), c.candidate.id.root_hash, file_hash, collated_data_file_hash); auto stat = stats_get_candidate_stat(round, local_id(), block_id); if (stat) { stat->block_status = ValidatorSessionStats::status_received; stat->collation_time = collation_time; stat->collated_at = td::Clocks::system(); - stat->block_timestamp = td::Clocks::system(); - stat->collation_cached = collation_cached; - stat->root_hash = root_hash; - stat->file_hash = file_hash; + stat->got_block_at = td::Clocks::system(); + stat->collation_cached = c.is_cached; + stat->self_collated = c.self_collated; + stat->collator_node_id = c.collator_node_id; + stat->block_id = c.candidate.id; + stat->collated_data_hash = collated_data_file_hash; } if (round != cur_round_) { return; } td::Timer serialize_timer; - auto b = create_tl_object(local_id().tl(), round, root_hash, std::move(data), - std::move(collated_data)); + auto b = create_tl_object(local_id().tl(), round, c.candidate.id.root_hash, + std::move(c.candidate.data), + std::move(c.candidate.collated_data)); auto B = serialize_candidate(b, compress_block_candidates_).move_as_ok(); if (stat) { stat->serialize_time = serialize_timer.elapsed(); @@ -546,9 +548,8 @@ void ValidatorSessionImpl::check_generate_slot() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), print_id = print_id(), timer = std::move(timer), round = cur_round_](td::Result R) { if (R.is_ok()) { - auto c = std::move(R.ok_ref().candidate); - td::actor::send_closure(SelfId, &ValidatorSessionImpl::generated_block, round, c.id.root_hash, - c.data.clone(), c.collated_data.clone(), timer.elapsed(), R.ok().is_cached); + td::actor::send_closure(SelfId, &ValidatorSessionImpl::generated_block, round, R.move_as_ok(), + timer.elapsed()); } else { LOG(WARNING) << print_id << ": failed to generate block candidate: " << R.move_as_error(); } @@ -606,11 +607,12 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { if (stat->block_status == ValidatorSessionStats::status_none) { stat->block_status = ValidatorSessionStats::status_received; } - if (stat->block_timestamp <= 0.0) { - stat->block_timestamp = td::Clocks::system(); + if (stat->got_block_at <= 0.0) { + stat->got_block_at = td::Clocks::system(); } - stat->root_hash = B->root_hash_; - stat->file_hash = td::sha256_bits256(B->data_); + stat->block_id.root_hash = B->root_hash_; + stat->block_id.file_hash = td::sha256_bits256(B->data_); + stat->collated_data_hash = td::sha256_bits256(B->collated_data_); } auto P = td::PromiseCreator::lambda([round = cur_round_, hash = block_id, root_hash = block->get_root_hash(), @@ -933,7 +935,7 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) { while (round_idx >= cur_stats_.rounds.size()) { stats_add_round(); } - cur_stats_.rounds[round_idx].timestamp = td::Clocks::system(); + cur_stats_.rounds[round_idx].started_at = td::Clocks::system(); } auto it2 = blocks_.begin(); while (it2 != blocks_.end()) { @@ -1034,9 +1036,7 @@ void ValidatorSessionImpl::get_end_stats(td::Promise pro promise.set_error(td::Status::Error(ErrorCode::notready, "not started")); return; } - EndValidatorGroupStats stats; - stats.session_id = unique_hash_; - stats.timestamp = td::Clocks::system(); + EndValidatorGroupStats stats{.session_id = unique_hash_, .timestamp = td::Clocks::system()}; stats.nodes.resize(description().get_total_nodes()); for (size_t i = 0; i < stats.nodes.size(); ++i) { stats.nodes[i].id = description().get_source_id(i); @@ -1145,7 +1145,7 @@ void ValidatorSessionImpl::stats_init() { if (cur_stats_.rounds.empty()) { stats_add_round(); } - cur_stats_.rounds[0].timestamp = td::Clocks::system(); + cur_stats_.rounds[0].started_at = td::Clocks::system(); stats_inited_ = true; } @@ -1158,13 +1158,13 @@ void ValidatorSessionImpl::stats_add_round() { td::int32 priority = description().get_node_priority(i, round); if (priority >= 0) { CHECK((size_t)priority < stat.producers.size()); - stat.producers[priority].id = description().get_source_id(i); + stat.producers[priority].validator_id = description().get_source_id(i); stat.producers[priority].is_ours = (local_idx() == i); stat.producers[priority].approvers.resize(description().get_total_nodes(), false); stat.producers[priority].signers.resize(description().get_total_nodes(), false); } } - while (!stat.producers.empty() && stat.producers.back().id.is_zero()) { + while (!stat.producers.empty() && stat.producers.back().validator_id.is_zero()) { stat.producers.pop_back(); } } @@ -1179,7 +1179,7 @@ ValidatorSessionStats::Producer *ValidatorSessionImpl::stats_get_candidate_stat( } auto &stats_round = cur_stats_.rounds[round - cur_stats_.first_round]; auto it = std::find_if(stats_round.producers.begin(), stats_round.producers.end(), - [&](const ValidatorSessionStats::Producer &p) { return p.id == src; }); + [&](const ValidatorSessionStats::Producer &p) { return p.validator_id == src; }); if (it == stats_round.producers.end()) { return nullptr; } diff --git a/validator-session/validator-session.h b/validator-session/validator-session.h index b099d65ec..641fb4866 100644 --- a/validator-session/validator-session.h +++ b/validator-session/validator-session.h @@ -78,11 +78,6 @@ class ValidatorSession : public td::actor::Actor { bool is_cached_ = false; }; - struct GeneratedCandidate { - BlockCandidate candidate; - bool is_cached = false; - }; - class Callback { public: virtual void on_candidate(BlockSourceInfo source_info, ValidatorSessionRootHash root_hash, td::BufferSlice data, diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 690346f70..2d996101a 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -217,8 +217,7 @@ class ValidatorSessionImpl : public ValidatorSession { void candidate_approved_signed(td::uint32 round, ValidatorSessionCandidateId hash, td::uint32 ok_from, td::BufferSlice signature); - void generated_block(td::uint32 round, ValidatorSessionRootHash root_hash, td::BufferSlice data, - td::BufferSlice collated, double collation_time, bool collation_cached); + void generated_block(td::uint32 round, GeneratedCandidate c, double collation_time); void signed_block(td::uint32 round, ValidatorSessionCandidateId hash, td::BufferSlice signature); void end_request(td::uint32 round, ValidatorSessionCandidateId block_id) { diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index eb9aa626d..8fabfcaee 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -33,13 +33,16 @@ void CollationManager::start_up() { void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, Ed25519_PublicKey creator, - BlockCandidatePriority priority, - td::Ref validator_set, td::uint64 max_answer_size, - td::CancellationToken cancellation_token, td::Promise promise) { + BlockCandidatePriority priority, td::Ref validator_set, + td::uint64 max_answer_size, td::CancellationToken cancellation_token, + td::Promise promise) { if (shard.is_masterchain()) { - run_collate_query(shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), - opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), std::move(promise), - std::move(cancellation_token), 0); + run_collate_query( + shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), + opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), promise.wrap([](BlockCandidate&& candidate) { + return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true}; + }), + adnl::AdnlNodeIdShort::zero(), std::move(cancellation_token), 0); return; } collate_shard_block(shard, min_masterchain_block_id, std::move(prev), creator, priority, std::move(validator_set), @@ -48,10 +51,9 @@ void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_mastercha void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, Ed25519_PublicKey creator, - BlockCandidatePriority priority, - td::Ref validator_set, td::uint64 max_answer_size, - td::CancellationToken cancellation_token, - td::Promise promise, td::Timestamp timeout) { + BlockCandidatePriority priority, td::Ref validator_set, + td::uint64 max_answer_size, td::CancellationToken cancellation_token, + td::Promise promise, td::Timestamp timeout) { TRY_STATUS_PROMISE(promise, cancellation_token.check()); ShardInfo* s = select_shard_info(shard); if (s == nullptr) { @@ -104,9 +106,12 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas } if (selected_collator.is_zero() && s->self_collate) { - run_collate_query(shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), - opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), std::move(promise), - std::move(cancellation_token), 0); + run_collate_query( + shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), + opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), promise.wrap([](BlockCandidate&& candidate) { + return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true}; + }), + adnl::AdnlNodeIdShort::zero(), std::move(cancellation_token), 0); return; } @@ -117,10 +122,13 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas next_block_id.seqno = std::max(next_block_id.seqno, p.seqno() + 1); } - promise = [=, SelfId = actor_id(this), promise = std::move(promise), - retry_at = td::Timestamp::in(0.5)](td::Result R) mutable { + td::Promise P = [=, SelfId = actor_id(this), promise = std::move(promise), + retry_at = td::Timestamp::in(0.5)](td::Result R) mutable { if (R.is_ok()) { - promise.set_value(R.move_as_ok()); + promise.set_value(GeneratedCandidate{.candidate = R.move_as_ok(), + .is_cached = false, + .self_collated = false, + .collator_node_id = selected_collator.bits256_value()}); return; } if (!selected_collator.is_zero()) { @@ -142,7 +150,7 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas }; if (selected_collator.is_zero()) { - promise.set_error(td::Status::Error(PSTRING() << "shard " << shard.to_str() << " has no alive collator node")); + P.set_error(td::Status::Error(PSTRING() << "shard " << shard.to_str() << " has no alive collator node")); return; } @@ -152,32 +160,32 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas LOG(INFO) << "sending collate query for " << next_block_id.to_str() << ": send to #" << selected_idx << "(" << selected_collator << ")"; - td::Promise P = [=, SelfId = actor_id(this), promise = std::move(promise), - timer = td::Timer()](td::Result R) mutable { - TRY_RESULT_PROMISE_PREFIX(promise, data, std::move(R), "rldp query failed: "); + td::Promise P2 = [=, SelfId = actor_id(this), P = std::move(P), + timer = td::Timer()](td::Result R) mutable { + TRY_RESULT_PROMISE_PREFIX(P, data, std::move(R), "rldp query failed: "); auto r_error = fetch_tl_object(data, true); if (r_error.is_ok()) { auto error = r_error.move_as_ok(); - promise.set_error(td::Status::Error(error->code_, error->message_)); + P.set_error(td::Status::Error(error->code_, error->message_)); return; } - TRY_RESULT_PROMISE(promise, f, fetch_tl_object(data, true)); - TRY_RESULT_PROMISE(promise, candidate, + TRY_RESULT_PROMISE(P, f, fetch_tl_object(data, true)); + TRY_RESULT_PROMISE(P, candidate, CollatorNode::deserialize_candidate(std::move(f), td::narrow_cast(max_answer_size))); if (candidate.pubkey.as_bits256() != creator.as_bits256()) { - promise.set_error(td::Status::Error("collate query: block candidate source mismatch")); + P.set_error(td::Status::Error("collate query: block candidate source mismatch")); return; } if (candidate.id.id != next_block_id) { - promise.set_error(td::Status::Error("collate query: block id mismatch")); + P.set_error(td::Status::Error("collate query: block id mismatch")); return; } LOG(INFO) << "got collated block " << next_block_id.to_str() << " from #" << selected_idx << " (" << selected_collator << ") in " << timer.elapsed() << "s"; - promise.set_result(std::move(candidate)); + P.set_result(std::move(candidate)); }; - td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_id_, selected_collator, "collatequery", std::move(P), - timeout, std::move(query), max_answer_size); + td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_id_, selected_collator, "collatequery", + std::move(P2), timeout, std::move(query), max_answer_size); } void CollationManager::update_options(td::Ref opts) { diff --git a/validator/collation-manager.hpp b/validator/collation-manager.hpp index 0ca4617d0..72cc02e5a 100644 --- a/validator/collation-manager.hpp +++ b/validator/collation-manager.hpp @@ -37,7 +37,7 @@ class CollationManager : public td::actor::Actor { void collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, Ed25519_PublicKey creator, BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, - td::CancellationToken cancellation_token, td::Promise promise); + td::CancellationToken cancellation_token, td::Promise promise); void update_options(td::Ref opts); @@ -55,7 +55,7 @@ class CollationManager : public td::actor::Actor { void collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, Ed25519_PublicKey creator, BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, - td::CancellationToken cancellation_token, td::Promise promise, + td::CancellationToken cancellation_token, td::Promise promise, td::Timestamp timeout); void update_collators_list(const CollatorsList& collators_list); diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 94bb18a1b..cbab8b795 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -391,12 +391,24 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data promise.set_result(serialize_tl_object(serialize_candidate(R.move_as_ok(), true), true)); } }; - new_promise = [new_promise = std::move(new_promise), creator, + new_promise = [new_promise = std::move(new_promise), creator, local_id = local_id_, manager = manager_](td::Result R) mutable { TRY_RESULT_PROMISE(new_promise, block, std::move(R)); + + CollatorNodeResponseStats stats; + stats.collator_node_id = local_id.bits256_value(); + stats.validator_id = creator.as_bits256(); + stats.original_block_id = block.id; + stats.collated_data_hash = block.collated_file_hash; + CatchainSeqno cc_seqno; td::uint32 val_set_hash; block = change_creator(std::move(block), creator, cc_seqno, val_set_hash); + + stats.block_id = block.id; + stats.timestamp = td::Clocks::system(); + td::actor::send_closure(manager, &ValidatorManager::log_collator_node_response_stats, std::move(stats)); + td::Promise P = new_promise.wrap([block = block.clone()](td::Unit&&) mutable -> BlockCandidate { return std::move(block); }); td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, block.id, std::move(block), cc_seqno, @@ -537,7 +549,7 @@ void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std }; td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R)); }, - cache_entry->cancellation_token_source.get_cancellation_token(), + local_id_, cache_entry->cancellation_token_source.get_cancellation_token(), CollateMode::skip_store_candidate | CollateMode::from_collator_node); } diff --git a/validator/fabric.h b/validator/fabric.h index 20358822e..d2e9b4735 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -89,7 +89,8 @@ void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_bloc Ed25519_PublicKey creator, td::Ref validator_set, td::Ref collator_opts, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, - td::CancellationToken cancellation_token, unsigned mode, int attempt_idx = 0); + adnl::AdnlNodeIdShort collator_node_id, td::CancellationToken cancellation_token, unsigned mode, + int attempt_idx = 0); void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index b207242fb..3572109d9 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -79,6 +79,7 @@ class Collator final : public td::actor::Actor { td::Timestamp timeout; td::Timestamp queue_cleanup_timeout_, soft_timeout_, medium_timeout_; td::Promise main_promise; + adnl::AdnlNodeIdShort collator_node_id_ = adnl::AdnlNodeIdShort::zero(); unsigned mode_ = 0; int attempt_idx_; bool allow_repeat_collation_ = false; @@ -97,7 +98,8 @@ class Collator final : public td::actor::Actor { Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector prev, Ref validator_set, Ed25519_PublicKey collator_id, Ref collator_opts, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, - td::CancellationToken cancellation_token, unsigned mode, int attempt_idx); + adnl::AdnlNodeIdShort collator_node_id, td::CancellationToken cancellation_token, unsigned mode, + int attempt_idx); ~Collator() override = default; bool is_busy() const { return busy_; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index dd8d55169..94d98ac59 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -77,6 +77,7 @@ static inline bool dbg(int c) { * @param manager The ActorId of the ValidatorManager. * @param timeout The timeout for the collator. * @param promise The promise to return the result. + * @param collator_node_id ADNL id of the collator node that generates the block (zero if it's not a collator node) * @param cancellation_token Token to cancel collation. * @param mode +1 - skip storing candidate to disk, +2 - called from CollatorNode. * @param attempt_idx The index of the attempt, starting from 0. On later attempts collator decreases block limits and skips some steps. @@ -84,8 +85,8 @@ static inline bool dbg(int c) { Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector prev, td::Ref validator_set, Ed25519_PublicKey collator_id, Ref collator_opts, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise promise, td::CancellationToken cancellation_token, - unsigned mode, int attempt_idx) + td::Timestamp timeout, td::Promise promise, adnl::AdnlNodeIdShort collator_node_id, + td::CancellationToken cancellation_token, unsigned mode, int attempt_idx) : shard_(shard) , is_hardfork_(is_hardfork) , min_mc_block_id{min_masterchain_block_id} @@ -100,6 +101,7 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_mastercha , soft_timeout_(td::Timestamp::at(timeout.at() - 3.0)) , medium_timeout_(td::Timestamp::at(timeout.at() - 1.5)) , main_promise(std::move(promise)) + , collator_node_id_(collator_node_id) , mode_(mode) , attempt_idx_(attempt_idx) , perf_timer_("collate", 0.1, @@ -374,8 +376,8 @@ bool Collator::fatal_error(td::Status error) { !is_hardfork_ && !timeout.is_in_past()) { LOG(WARNING) << "Repeating collation (attempt #" << attempt_idx_ + 1 << ")"; run_collate_query(shard_, min_mc_block_id, prev_blocks, created_by_, validator_set_, collator_opts_, manager, - td::Timestamp::in(10.0), std::move(main_promise), std::move(cancellation_token_), mode_, - attempt_idx_ + 1); + td::Timestamp::in(10.0), std::move(main_promise), collator_node_id_, + std::move(cancellation_token_), mode_, attempt_idx_ + 1); } else { LOG(INFO) << "collation failed in " << perf_timer_.elapsed() << " s " << error; LOG(INFO) << perf_log_; @@ -5927,8 +5929,15 @@ bool Collator::create_block_candidate() { double work_time = work_timer_.elapsed(); double cpu_work_time = cpu_work_timer_.elapsed(); LOG(WARNING) << "Collate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s"; + stats_.block_id = block_candidate->id; + stats_.collated_data_hash = block_candidate->collated_file_hash; + stats_.cc_seqno = validator_set_->get_catchain_seqno(); + stats_.collated_at = td::Clocks::system(); stats_.actual_bytes = block_candidate->data.size(); stats_.actual_collated_data_bytes = block_candidate->collated_data.size(); + stats_.attempt = attempt_idx_; + stats_.collator_node_id = collator_node_id_.bits256_value(); + stats_.validator_id = created_by_.as_bits256(); stats_.estimated_bytes = block_limit_status_->estimate_block_size(); stats_.gas = block_limit_status_->gas_used; stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt; @@ -5938,20 +5947,12 @@ bool Collator::create_block_candidate() { stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt); stats_.cat_collated_data_bytes = block_limit_status_->limits.classify_collated_data_size(stats_.estimated_collated_data_bytes); + stats_.total_time = perf_timer_.elapsed(); stats_.work_time = work_time; stats_.cpu_work_time = cpu_work_time; + stats_.time_stats = (PSTRING() << perf_log_); - // TODO: remove this later (currently needed to collect stats) - if (mode_ & CollateMode::from_collator_node) { - size_t d; - stats_.serialized_size = - validatorsession::compress_candidate_data(block_candidate->data, block_candidate->collated_data, d).ok().size(); - stats_.serialized_size_no_collated_data = - validatorsession::compress_candidate_data(block_candidate->data, td::Slice{}, d).ok().size(); - } - - td::actor::send_closure(manager, &ValidatorManager::record_collate_query_stats, block_candidate->id, - std::move(stats_)); + td::actor::send_closure(manager, &ValidatorManager::log_collate_query_stats, std::move(stats_)); return true; } diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index db15b1edb..d1555edc5 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -216,7 +216,8 @@ void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_bloc Ed25519_PublicKey creator, td::Ref validator_set, td::Ref collator_opts, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, - td::CancellationToken cancellation_token, unsigned mode, int attempt_idx) { + adnl::AdnlNodeIdShort collator_node_id, td::CancellationToken cancellation_token, unsigned mode, + int attempt_idx) { BlockSeqno seqno = 0; for (auto& p : prev) { if (p.seqno() > seqno) { @@ -227,7 +228,7 @@ void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_bloc << (attempt_idx ? "_" + td::to_string(attempt_idx) : ""), shard, false, min_masterchain_block_id, std::move(prev), std::move(validator_set), creator, std::move(collator_opts), std::move(manager), timeout, std::move(promise), - std::move(cancellation_token), mode, attempt_idx) + collator_node_id, std::move(cancellation_token), mode, attempt_idx) .release(); } @@ -240,10 +241,10 @@ void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_b seqno = p.seqno(); } } - td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, - min_masterchain_block_id, std::move(prev), td::Ref{}, - Ed25519_PublicKey{Bits256::zero()}, td::Ref{true}, - std::move(manager), timeout, std::move(promise), td::CancellationToken{}, 0, 0) + td::actor::create_actor( + PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, min_masterchain_block_id, + std::move(prev), td::Ref{}, Ed25519_PublicKey{Bits256::zero()}, td::Ref{true}, + std::move(manager), timeout, std::move(promise), adnl::AdnlNodeIdShort::zero(), td::CancellationToken{}, 0, 0) .release(); } diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 34c164eee..92b32b75c 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -116,13 +116,12 @@ bool ValidateQuery::reject_query(std::string error, td::BufferSlice reason) { error = error_ctx() + error; LOG(ERROR) << "REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error; if (main_promise) { - record_stats(); + record_stats(false, error); errorlog::ErrorLog::log(PSTRING() << "REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error << ": data=" << block_candidate.id.file_hash.to_hex() << " collated_data=" << block_candidate.collated_file_hash.to_hex()); errorlog::ErrorLog::log_file(block_candidate.data.clone()); errorlog::ErrorLog::log_file(block_candidate.collated_data.clone()); - LOG(INFO) << "validation took " << perf_timer_.elapsed() << " s"; main_promise.set_result(CandidateReject{std::move(error), std::move(reason)}); } stop(); @@ -155,13 +154,12 @@ bool ValidateQuery::soft_reject_query(std::string error, td::BufferSlice reason) error = error_ctx() + error; LOG(ERROR) << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error; if (main_promise) { - record_stats(); + record_stats(false, error); errorlog::ErrorLog::log(PSTRING() << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error << ": data=" << block_candidate.id.file_hash.to_hex() << " collated_data=" << block_candidate.collated_file_hash.to_hex()); errorlog::ErrorLog::log_file(block_candidate.data.clone()); errorlog::ErrorLog::log_file(block_candidate.collated_data.clone()); - LOG(INFO) << "validation took " << perf_timer_.elapsed() << " s"; main_promise.set_result(CandidateReject{std::move(error), std::move(reason)}); } stop(); @@ -179,7 +177,7 @@ bool ValidateQuery::fatal_error(td::Status error) { error.ensure_error(); LOG(ERROR) << "aborting validation of block candidate for " << shard_.to_str() << " : " << error.to_string(); if (main_promise) { - record_stats(); + record_stats(false, error.message().str()); auto c = error.code(); if (c <= -667 && c >= -670) { errorlog::ErrorLog::log(PSTRING() << "FATAL ERROR: aborting validation of block candidate for " << shard_.to_str() @@ -188,7 +186,6 @@ bool ValidateQuery::fatal_error(td::Status error) { errorlog::ErrorLog::log_file(block_candidate.data.clone()); errorlog::ErrorLog::log_file(block_candidate.collated_data.clone()); } - LOG(INFO) << "validation took " << perf_timer_.elapsed() << " s"; main_promise(std::move(error)); } stop(); @@ -238,9 +235,8 @@ bool ValidateQuery::fatal_error(std::string err_msg, int err_code) { */ void ValidateQuery::finish_query() { if (main_promise) { - record_stats(); + record_stats(true); LOG(WARNING) << "validate query done"; - LOG(WARNING) << "validation took " << perf_timer_.elapsed() << " s"; main_promise.set_result(now_); } stop(); @@ -7061,13 +7057,25 @@ void ValidateQuery::written_candidate() { /** * Sends validation work time to manager. */ -void ValidateQuery::record_stats() { - double work_time = work_timer_.elapsed(); - double cpu_work_time = cpu_work_timer_.elapsed(); +void ValidateQuery::record_stats(bool valid, std::string error_message) { + ValidationStats stats; + stats.block_id = id_; + stats.collated_data_hash = block_candidate.collated_file_hash; + stats.validated_at = td::Clocks::system(); + stats.valid = valid; + if (valid) { + stats.comment = (PSTRING() << "OK ts=" << now_); + } else { + stats.comment = std::move(error_message); + } + stats.actual_bytes = block_candidate.data.size(); + stats.actual_collated_data_bytes = block_candidate.collated_data.size(); + stats.total_time = perf_timer_.elapsed(); + stats.work_time = work_timer_.elapsed(); + stats.cpu_work_time = cpu_work_timer_.elapsed(); LOG(WARNING) << "validation took " << perf_timer_.elapsed() << "s"; - LOG(WARNING) << "Validate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s"; - td::actor::send_closure(manager, &ValidatorManager::record_validate_query_stats, block_candidate.id, work_time, - cpu_work_time); + LOG(WARNING) << "Validate query work time = " << stats.work_time << "s, cpu time = " << stats.cpu_work_time << "s"; + td::actor::send_closure(manager, &ValidatorManager::log_validate_query_stats, std::move(stats)); } } // namespace validator diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 585d553c5..328a9e1eb 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -407,7 +407,7 @@ class ValidateQuery : public td::actor::Actor { td::Timer work_timer_{true}; td::ThreadCpuTimer cpu_work_timer_{true}; - void record_stats(); + void record_stats(bool valid, std::string error_message = ""); }; } // namespace validator diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 14d10eccc..59f114b33 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -32,6 +32,8 @@ #include "auto/tl/lite_api.h" #include "impl/out-msg-queue-proof.hpp" +#include + namespace ton { namespace validator { @@ -54,7 +56,14 @@ struct AsyncSerializerState { }; struct CollationStats { + BlockIdExt block_id; + td::Bits256 collated_data_hash = td::Bits256::zero(); + CatchainSeqno cc_seqno = 0; + double collated_at = -1.0; td::uint32 actual_bytes = 0, actual_collated_data_bytes = 0; + int attempt = 0; + td::Bits256 collator_node_id = td::Bits256::zero(); + td::Bits256 validator_id = td::Bits256::zero(); td::uint32 estimated_bytes = 0, gas = 0, lt_delta = 0, estimated_collated_data_bytes = 0; int cat_bytes = 0, cat_gas = 0, cat_lt_delta = 0, cat_collated_data_bytes = 0; std::string limits_log; @@ -62,15 +71,51 @@ struct CollationStats { td::uint32 ext_msgs_filtered = 0; td::uint32 ext_msgs_accepted = 0; td::uint32 ext_msgs_rejected = 0; - double work_time = 0.0, cpu_work_time = 0.0; - td::uint32 serialized_size = 0, serialized_size_no_collated_data = 0; - - tl_object_ptr tl() const { - return create_tl_object( - actual_bytes, actual_collated_data_bytes, estimated_bytes, gas, lt_delta, estimated_collated_data_bytes, - cat_bytes, cat_gas, cat_lt_delta, cat_collated_data_bytes, limits_log, ext_msgs_total, ext_msgs_filtered, - ext_msgs_accepted, ext_msgs_rejected, work_time, cpu_work_time, serialized_size, - serialized_size_no_collated_data); + double total_time = 0.0, work_time = 0.0, cpu_work_time = 0.0; + std::string time_stats; + + tl_object_ptr tl() const { + int flags = (collator_node_id.is_zero() ? 0 : ton_api::validatorStats_collatedBlock::Flags::COLLATOR_NODE_ID_MASK) | + (validator_id.is_zero() ? 0 : ton_api::validatorStats_collatedBlock::Flags::VALIDATOR_ID_MASK); + return create_tl_object( + flags, create_tl_block_id(block_id), collated_data_hash, cc_seqno, collated_at, actual_bytes, + actual_collated_data_bytes, attempt, collator_node_id, validator_id, total_time, work_time, cpu_work_time, + time_stats, + create_tl_object(estimated_bytes, gas, lt_delta, + estimated_collated_data_bytes, cat_bytes, cat_gas, + cat_lt_delta, cat_collated_data_bytes, limits_log), + create_tl_object(ext_msgs_total, ext_msgs_filtered, ext_msgs_accepted, + ext_msgs_rejected)); + } +}; + +struct ValidationStats { + BlockIdExt block_id; + td::Bits256 collated_data_hash = td::Bits256::zero(); + double validated_at = -1.0; + bool valid = false; + std::string comment; + td::uint32 actual_bytes = 0, actual_collated_data_bytes = 0; + double total_time = 0.0, work_time = 0.0, cpu_work_time = 0.0; + + tl_object_ptr tl() const { + return create_tl_object( + create_tl_block_id(block_id), collated_data_hash, validated_at, valid, comment, actual_bytes, + actual_collated_data_bytes, total_time, work_time, cpu_work_time); + } +}; + +struct CollatorNodeResponseStats { + td::Bits256 collator_node_id = td::Bits256::zero(); + td::Bits256 validator_id = td::Bits256::zero(); + double timestamp = -1.0; + BlockIdExt block_id, original_block_id; + td::Bits256 collated_data_hash = td::Bits256::zero(); + + tl_object_ptr tl() const { + return create_tl_object( + collator_node_id, validator_id, timestamp, create_tl_block_id(block_id), create_tl_block_id(original_block_id), + collated_data_hash);; } }; @@ -198,9 +243,12 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) = 0; - virtual void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) = 0; - virtual void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) = 0; - virtual void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) = 0; + virtual void log_validator_session_stats(validatorsession::ValidatorSessionStats stats) { + } + virtual void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) { + } + virtual void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) { + } virtual void get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) = 0; virtual void get_block_data_for_litequery(BlockIdExt block_id, td::Promise> promise) = 0; @@ -220,9 +268,11 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void add_lite_query_stats(int lite_query_id) { } - virtual void record_collate_query_stats(BlockIdExt block_id, CollationStats stats) { + virtual void log_collate_query_stats(CollationStats stats) { + } + virtual void log_validate_query_stats(ValidationStats stats) { } - virtual void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) { + virtual void log_collator_node_response_stats(CollatorNodeResponseStats stats) { } virtual void add_persistent_state_description(td::Ref desc) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 542382520..5ebe54678 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -129,7 +129,8 @@ void ValidatorManagerImpl::sync_complete(td::Promise promise) { Ed25519_PublicKey created_by{td::Bits256::zero()}; td::as(created_by.as_bits256().data() + 32 - 4) = ((unsigned)std::time(nullptr) >> 8); run_collate_query(shard_id, last_masterchain_block_id_, prev, created_by, val_set, td::Ref{true}, - actor_id(this), td::Timestamp::in(10.0), std::move(P), td::CancellationToken{}, 0); + actor_id(this), td::Timestamp::in(10.0), std::move(P), adnl::AdnlNodeIdShort::zero(), + td::CancellationToken{}, 0); } void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vector prev, BlockIdExt last, diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 30bc7756b..24e9e30a8 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -403,15 +403,6 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) override { UNREACHABLE(); } - void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override { - UNREACHABLE(); - } - void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override { - UNREACHABLE(); - } - void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override { - UNREACHABLE(); - } void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { queue_size_counter_ = td::actor::create_actor("queuesizecounter", td::Ref{}, diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 804ef0c73..3191adf0f 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -468,15 +468,6 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) override { UNREACHABLE(); } - void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override { - UNREACHABLE(); - } - void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override { - UNREACHABLE(); - } - void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override { - UNREACHABLE(); - } void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { queue_size_counter_ = td::actor::create_actor("queuesizecounter", td::Ref{}, diff --git a/validator/manager.cpp b/validator/manager.cpp index 31278788c..ff142c570 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -3107,61 +3107,20 @@ void ValidatorManagerImpl::wait_shard_client_state(BlockSeqno seqno, td::Timesta shard_client_waiters_[seqno].waiting_.emplace_back(timeout, 0, std::move(promise)); } -void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id, - validatorsession::ValidatorSessionStats stats) { +void ValidatorManagerImpl::log_validator_session_stats(validatorsession::ValidatorSessionStats stats) { std::string fname = opts_->get_session_logs_file(); if (fname.empty()) { return; } - - std::vector> rounds; - for (const auto &round : stats.rounds) { - std::vector> producers; - for (const auto &producer : round.producers) { - BlockIdExt cur_block_id{block_id.id, producer.root_hash, producer.file_hash}; - auto it = recorded_block_stats_.find(cur_block_id); - tl_object_ptr collation_stats; - if (it != recorded_block_stats_.end() && it->second.collator_stats_) { - auto &stats = it->second.collator_stats_.value(); - collation_stats = stats.tl(); - } - std::string approvers, signers; - for (bool x : producer.approvers) { - approvers += (x ? '1' : '0'); - } - for (bool x : producer.signers) { - signers += (x ? '1' : '0'); - } - producers.push_back(create_tl_object( - producer.id.bits256_value(), producer.candidate_id, producer.block_status, producer.root_hash, - producer.file_hash, producer.comment, producer.block_timestamp, producer.is_accepted, producer.is_ours, - producer.got_submit_at, producer.collation_time, producer.collated_at, producer.collation_cached, - it == recorded_block_stats_.end() ? -1.0 : it->second.collator_work_time_, - it == recorded_block_stats_.end() ? -1.0 : it->second.collator_cpu_work_time_, std::move(collation_stats), - producer.validation_time, producer.validated_at, producer.validation_cached, - it == recorded_block_stats_.end() ? -1.0 : it->second.validator_work_time_, - it == recorded_block_stats_.end() ? -1.0 : it->second.validator_cpu_work_time_, producer.gen_utime, - producer.approved_weight, producer.approved_33pct_at, producer.approved_66pct_at, std::move(approvers), - producer.signed_weight, producer.signed_33pct_at, producer.signed_66pct_at, std::move(signers), - producer.serialize_time, producer.deserialize_time, producer.serialized_size)); - } - rounds.push_back(create_tl_object(round.timestamp, std::move(producers))); - } - - auto obj = create_tl_object( - stats.success, create_tl_block_id(block_id), stats.timestamp, stats.self.bits256_value(), stats.session_id, - stats.cc_seqno, stats.creator.bits256_value(), stats.total_validators, stats.total_weight, stats.signatures, - stats.signatures_weight, stats.approve_signatures, stats.approve_signatures_weight, stats.first_round, - std::move(rounds)); + stats.fix_block_ids(); + auto obj = stats.tl(); auto s = td::json_encode(td::ToJson(*obj.get()), false); s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - std::ofstream file; file.open(fname, std::ios_base::app); file << s << "\n"; file.close(); - - LOG(INFO) << "Writing validator session stats for " << block_id.id.to_str(); + LOG(INFO) << "Writing validator session stats for " << stats.block_id.to_str(); } void ValidatorManagerImpl::log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) { @@ -3169,22 +3128,13 @@ void ValidatorManagerImpl::log_new_validator_group_stats(validatorsession::NewVa if (fname.empty()) { return; } - std::vector> nodes; - for (const auto &node : stats.nodes) { - nodes.push_back( - create_tl_object(node.id.bits256_value(), node.weight)); - } - auto obj = create_tl_object( - stats.session_id, stats.shard.workchain, stats.shard.shard, stats.cc_seqno, stats.last_key_block_seqno, - stats.timestamp, stats.self_idx, std::move(nodes)); + auto obj = stats.tl(); auto s = td::json_encode(td::ToJson(*obj.get()), false); s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - std::ofstream file; file.open(fname, std::ios_base::app); file << s << "\n"; file.close(); - LOG(INFO) << "Writing new validator group stats for " << stats.session_id << " shard=" << stats.shard.to_str() << " cc_seqno=" << stats.cc_seqno; } @@ -3194,21 +3144,13 @@ void ValidatorManagerImpl::log_end_validator_group_stats(validatorsession::EndVa if (fname.empty()) { return; } - std::vector> nodes; - for (const auto &node : stats.nodes) { - nodes.push_back(create_tl_object(node.id.bits256_value(), - node.catchain_blocks)); - } - auto obj = create_tl_object(stats.session_id, stats.timestamp, - std::move(nodes)); + auto obj = stats.tl(); auto s = td::json_encode(td::ToJson(*obj.get()), false); s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - std::ofstream file; file.open(fname, std::ios_base::app); file << s << "\n"; file.close(); - LOG(INFO) << "Writing end validator group stats for " << stats.session_id; } @@ -3657,45 +3599,49 @@ td::actor::ActorOwn ValidatorManagerFactory::create( rldp, overlays); } -void ValidatorManagerImpl::record_collate_query_stats(BlockIdExt block_id, CollationStats stats) { - auto &record = new_block_stats_record(block_id); - record.collator_work_time_ = stats.work_time; - record.collator_cpu_work_time_ = stats.cpu_work_time; - record.collator_stats_ = std::move(stats); - +void ValidatorManagerImpl::log_collate_query_stats(CollationStats stats) { std::string fname = opts_->get_session_logs_file(); if (fname.empty()) { return; } - - auto obj = create_tl_object(td::Clocks::system(), - create_tl_block_id(block_id), stats.tl()); + auto obj = stats.tl(); auto s = td::json_encode(td::ToJson(*obj.get()), false); s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - std::ofstream file; file.open(fname, std::ios_base::app); file << s << "\n"; file.close(); - - LOG(DEBUG) << "Writing collation stats stats for " << block_id.id.to_str(); + LOG(DEBUG) << "Writing collation stats stats for " << stats.block_id.to_str(); } -void ValidatorManagerImpl::record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) { - auto &record = new_block_stats_record(block_id); - record.validator_work_time_ = work_time; - record.validator_cpu_work_time_ = cpu_work_time; +void ValidatorManagerImpl::log_validate_query_stats(ValidationStats stats) { + std::string fname = opts_->get_session_logs_file(); + if (fname.empty()) { + return; + } + auto obj = stats.tl(); + auto s = td::json_encode(td::ToJson(*obj.get()), false); + s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); + std::ofstream file; + file.open(fname, std::ios_base::app); + file << s << "\n"; + file.close(); + LOG(DEBUG) << "Writing validation stats stats for " << stats.block_id.to_str(); } -ValidatorManagerImpl::RecordedBlockStats &ValidatorManagerImpl::new_block_stats_record(BlockIdExt block_id) { - if (!recorded_block_stats_.count(block_id)) { - recorded_block_stats_lru_.push(block_id); - if (recorded_block_stats_lru_.size() > 4096) { - recorded_block_stats_.erase(recorded_block_stats_lru_.front()); - recorded_block_stats_lru_.pop(); - } +void ValidatorManagerImpl::log_collator_node_response_stats(CollatorNodeResponseStats stats) { + std::string fname = opts_->get_session_logs_file(); + if (fname.empty()) { + return; } - return recorded_block_stats_[block_id]; + auto obj = stats.tl(); + auto s = td::json_encode(td::ToJson(*obj.get()), false); + s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); + std::ofstream file; + file.open(fname, std::ios_base::app); + file << s << "\n"; + file.close(); + LOG(DEBUG) << "Writing collator node response stats stats for " << stats.block_id.to_str(); } size_t ValidatorManagerImpl::CheckedExtMsgCounter::get_msg_count(WorkchainId wc, StdSmcAddress addr) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 3353989fd..febed73df 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -630,7 +630,7 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) override; - void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override; + void log_validator_session_stats(validatorsession::ValidatorSessionStats stats) override; void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override; void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override; @@ -775,19 +775,9 @@ class ValidatorManagerImpl : public ValidatorManager { td::actor::ActorOwn candidates_buffer_; - struct RecordedBlockStats { - double collator_work_time_ = -1.0; - double collator_cpu_work_time_ = -1.0; - td::optional collator_stats_; - double validator_work_time_ = -1.0; - double validator_cpu_work_time_ = -1.0; - }; - std::map recorded_block_stats_; - std::queue recorded_block_stats_lru_; - - void record_collate_query_stats(BlockIdExt block_id, CollationStats stats) override; - void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) override; - RecordedBlockStats &new_block_stats_record(BlockIdExt block_id); + void log_collate_query_stats(CollationStats stats) override; + void log_validate_query_stats(ValidationStats stats) override; + void log_collator_node_response_stats(CollatorNodeResponseStats stats) override; std::map> validator_telemetry_; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 97f34ea7c..dad298841 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -36,9 +36,8 @@ static bool need_send_candidate_broadcast(const validatorsession::BlockSourceInf !is_masterchain; } -void ValidatorGroup::generate_block_candidate( - validatorsession::BlockSourceInfo source_info, - td::Promise promise) { +void ValidatorGroup::generate_block_candidate(validatorsession::BlockSourceInfo source_info, + td::Promise promise) { td::uint32 round_id = source_info.priority.round; if (round_id > last_known_round_id_) { last_known_round_id_ = round_id; @@ -49,20 +48,21 @@ void ValidatorGroup::generate_block_candidate( } if (cached_collated_block_) { if (cached_collated_block_->result) { - promise.set_value({cached_collated_block_->result.value().clone(), true}); + auto res = cached_collated_block_->result.value().clone(); + res.is_cached = true; + promise.set_value(std::move(res)); } else { - cached_collated_block_->promises.push_back(promise.wrap([](BlockCandidate &&res) { - return validatorsession::ValidatorSession::GeneratedCandidate{std::move(res), true}; + cached_collated_block_->promises.push_back(promise.wrap([](GeneratedCandidate &&res) { + res.is_cached = true; + return std::move(res); })); } return; } cached_collated_block_ = std::make_shared(); - cached_collated_block_->promises.push_back(promise.wrap([](BlockCandidate &&res) { - return validatorsession::ValidatorSession::GeneratedCandidate{std::move(res), false}; - })); - td::Promise P = [SelfId = actor_id(this), cache = cached_collated_block_, - source_info](td::Result R) { + cached_collated_block_->promises.push_back(std::move(promise)); + td::Promise P = [SelfId = actor_id(this), cache = cached_collated_block_, + source_info](td::Result R) { td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_candidate, source_info, std::move(cache), std::move(R)); }; @@ -75,7 +75,7 @@ void ValidatorGroup::generate_block_candidate( void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo source_info, std::shared_ptr cache, - td::Result R) { + td::Result R) { if (R.is_error()) { for (auto &p : cache->promises) { p.set_error(R.error().clone()); @@ -84,12 +84,12 @@ void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo cached_collated_block_ = nullptr; } } else { - auto candidate = R.move_as_ok(); - add_available_block_candidate(candidate.pubkey.as_bits256(), candidate.id, candidate.collated_file_hash); + auto c = R.move_as_ok(); + add_available_block_candidate(c.candidate.pubkey.as_bits256(), c.candidate.id, c.candidate.collated_file_hash); if (need_send_candidate_broadcast(source_info, shard_.is_masterchain())) { - send_block_candidate_broadcast(candidate.id, candidate.data.clone()); + send_block_candidate_broadcast(c.candidate.id, c.candidate.data.clone()); } - cache->result = std::move(candidate); + cache->result = std::move(c); for (auto &p : cache->promises) { p.set_value(cache->result.value().clone()); } @@ -186,7 +186,8 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so } auto next_block_id = create_next_block_id(root_hash, file_hash); LOG(WARNING) << "Accepted block " << next_block_id; - td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, next_block_id, std::move(stats)); + stats.block_id = next_block_id; + td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, std::move(stats)); auto block = block_data.size() > 0 ? create_block(next_block_id, std::move(block_data)).move_as_ok() : td::Ref{}; @@ -293,7 +294,7 @@ std::unique_ptr ValidatorGroup::ma std::move(candidate), std::move(P)); } void on_generate_slot(validatorsession::BlockSourceInfo source_info, - td::Promise promise) override { + td::Promise promise) override { td::actor::send_closure(id_, &ValidatorGroup::generate_block_candidate, std::move(source_info), std::move(promise)); } @@ -397,8 +398,8 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch for (auto &p : postponed_accept_) { auto next_block_id = create_next_block_id(p.root_hash, p.file_hash); - td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, next_block_id, - std::move(p.stats)); + p.stats.block_id = next_block_id; + td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, std::move(p.stats)); auto block = p.block.size() > 0 ? create_block(next_block_id, std::move(p.block)).move_as_ok() : td::Ref{}; @@ -408,12 +409,11 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch } postponed_accept_.clear(); - validatorsession::NewValidatorGroupStats stats; - stats.session_id = session_id_; - stats.shard = shard_; - stats.cc_seqno = validator_set_->get_catchain_seqno(); - stats.last_key_block_seqno = last_key_block_seqno_; - stats.timestamp = td::Clocks::system(); + validatorsession::NewValidatorGroupStats stats{.session_id = session_id_, + .shard = shard_, + .cc_seqno = validator_set_->get_catchain_seqno(), + .last_key_block_seqno = last_key_block_seqno_, + .started_at = td::Clocks::system()}; td::uint32 idx = 0; for (const auto &node : validator_set_->export_vector()) { PublicKeyHash id = ValidatorFullId{node.key}.compute_short_id(); @@ -441,7 +441,8 @@ void ValidatorGroup::destroy() { return; } stats.cc_seqno = cc_seqno; - td::actor::send_closure(manager, &ValidatorManager::log_validator_session_stats, block_id, + stats.block_id = block_id; + td::actor::send_closure(manager, &ValidatorManager::log_validator_session_stats, std::move(stats)); }); td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_end_stats, diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index fb7a4dcfb..675885fcc 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -35,8 +35,7 @@ class ValidatorManager; class ValidatorGroup : public td::actor::Actor { public: - void generate_block_candidate(validatorsession::BlockSourceInfo source_info, - td::Promise promise); + void generate_block_candidate(validatorsession::BlockSourceInfo source_info, td::Promise promise); void validate_block_candidate(validatorsession::BlockSourceInfo source_info, BlockCandidate block, td::Promise> promise); void accept_block_candidate(validatorsession::BlockSourceInfo source_info, td::BufferSlice block, RootHash root_hash, @@ -146,14 +145,14 @@ class ValidatorGroup : public td::actor::Actor { bool monitoring_shard_ = true; struct CachedCollatedBlock { - td::optional result; - std::vector> promises; + td::optional result; + std::vector> promises; }; std::shared_ptr cached_collated_block_; td::CancellationTokenSource cancellation_token_source_; void generated_block_candidate(validatorsession::BlockSourceInfo source_info, - std::shared_ptr cache, td::Result R); + std::shared_ptr cache, td::Result R); using CacheKey = std::tuple; std::map approved_candidates_cache_; @@ -165,8 +164,7 @@ class ValidatorGroup : public td::actor::Actor { } void get_validator_group_info_for_litequery_cont( - td::uint32 expected_round, - std::vector> candidates, + td::uint32 expected_round, std::vector> candidates, td::Promise> promise); std::set> available_block_candidates_; // source, id, collated hash From e6984e4799519cd6d1751e3aa06853a01333a560 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 27 Dec 2024 12:20:45 +0300 Subject: [PATCH 137/388] Add "self" to all session stats records --- tl/generate/scheme/ton_api.tl | 11 +++++---- tl/generate/scheme/ton_api.tlo | Bin 111624 -> 111644 bytes validator-session/validator-session-types.h | 7 ++++-- validator-session/validator-session.cpp | 2 +- validator/collator-node.cpp | 4 ++-- validator/fabric.h | 2 +- validator/impl/collator.cpp | 5 ++-- validator/impl/fabric.cpp | 8 +++---- validator/impl/validate-query.cpp | 10 ++++---- validator/impl/validate-query.hpp | 3 ++- validator/interfaces/validator-manager.h | 24 ++++++++++---------- validator/manager-disk.cpp | 2 +- validator/validator-group.cpp | 7 +++--- 13 files changed, 47 insertions(+), 38 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index bdcf0a296..3ddb69490 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -923,16 +923,17 @@ validatorStats.blockLimitsStatus validatorStats.extMsgsStats ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int = validatorStats.ExtMsgsStats; -validatorStats.collatedBlock flags:# +validatorStats.collatedBlock block_id:tonNode.blockIdExt collated_data_hash:int256 cc_seqno:int collated_at:double bytes:int collated_data_bytes:int attempt:int - collator_node_id:flags.0?int256 validator_id:flags.1?int256 + self:int256 is_validator:Bool total_time:double work_time:double cpu_work_time:double time_stats:string block_limits:validatorStats.blockLimitsStatus ext_msgs_stats:validatorStats.extMsgsStats = validatorSession.stats.CollatedBlock; validatorStats.validatedBlock block_id:tonNode.blockIdExt collated_data_hash:int256 validated_at:double + self:int256 valid:Bool comment:string bytes:int collated_data_bytes:int total_time:double work_time:double cpu_work_time:double = validatorStats.ValidatedBlock; @@ -940,13 +941,13 @@ validatorStats.validatedBlock validatorStats.newValidatorGroup.node id:int256 weight:long = validatorStats.newValidatorGroup.Node; validatorStats.newValidatorGroup session_id:int256 shard:tonNode.shardId cc_seqno:int last_key_block_seqno:int started_at:double - self_idx:int nodes:(vector validatorStats.newValidatorGroup.node) = validatorStats.NewValidatorGroup; + self_idx:int self:int256 nodes:(vector validatorStats.newValidatorGroup.node) = validatorStats.NewValidatorGroup; validatorStats.endValidatorGroup.node id:int256 catchain_blocks:int = validatorStats.endValidatorGroup.Node; -validatorStats.endValidatorGroup session_id:int256 timestamp:double +validatorStats.endValidatorGroup session_id:int256 timestamp:double self:int256 nodes:(vector validatorStats.endValidatorGroup.node) = validatorStats.EndValidatorGroup; -validatorStats.collatorNodeResponse collator_node_id:int256 validator_id:int256 timestamp:double +validatorStats.collatorNodeResponse self:int256 validator_id:int256 timestamp:double block_id:tonNode.blockIdExt original_block_id:tonNode.blockIdExt collated_data_hash:int256 = validatorStats.CollatorNodeResponse; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index f091213cf90e633cb8909ffc33ba05310ad4d1f0..9522bf9a607a2cf5097dbf4d58cfb0421d3a5d8f 100644 GIT binary patch delta 333 zcmeBp!8Ydw+lD*=u zxMgtj8oTXl>=@_pgLO>z@nclr1nIqbOvs0SdSeu0`1B8cj9aFE2xnB_<;g6LFH6kH zOi3)sFPhFcnNenXZ!}}}WQ8o&>6HPDspShy6;6DC{O6|ksm zcR1O+#%}u>JH|QuU>(za{1_D=tO? nodes; tl_object_ptr tl() const { @@ -232,7 +233,7 @@ struct NewValidatorGroupStats { } return create_tl_object(session_id, create_tl_shard_id(shard), cc_seqno, last_key_block_seqno, started_at, self_idx, - std::move(nodes_arr)); + self.bits256_value(), std::move(nodes_arr)); } }; @@ -244,6 +245,7 @@ struct EndValidatorGroupStats { ValidatorSessionId session_id = ValidatorSessionId::zero(); double timestamp = -1.0; + PublicKeyHash self = PublicKeyHash::zero(); std::vector nodes; tl_object_ptr tl() const { @@ -252,7 +254,8 @@ struct EndValidatorGroupStats { nodes_arr.push_back(create_tl_object(node.id.bits256_value(), node.catchain_blocks)); } - return create_tl_object(session_id, timestamp, std::move(nodes_arr)); + return create_tl_object(session_id, timestamp, self.bits256_value(), + std::move(nodes_arr)); } }; diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 5cc31a7ba..e18dd1eb5 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -1036,7 +1036,7 @@ void ValidatorSessionImpl::get_end_stats(td::Promise pro promise.set_error(td::Status::Error(ErrorCode::notready, "not started")); return; } - EndValidatorGroupStats stats{.session_id = unique_hash_, .timestamp = td::Clocks::system()}; + EndValidatorGroupStats stats{.session_id = unique_hash_, .timestamp = td::Clocks::system(), .self = local_id()}; stats.nodes.resize(description().get_total_nodes()); for (size_t i = 0; i < stats.nodes.size(); ++i) { stats.nodes[i].id = description().get_source_id(i); diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index cbab8b795..ff84456bd 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -396,8 +396,8 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data TRY_RESULT_PROMISE(new_promise, block, std::move(R)); CollatorNodeResponseStats stats; - stats.collator_node_id = local_id.bits256_value(); - stats.validator_id = creator.as_bits256(); + stats.self = local_id.pubkey_hash(); + stats.validator_id = PublicKey(pubkeys::Ed25519(creator)).compute_short_id(); stats.original_block_id = block.id; stats.collated_data_hash = block.collated_file_hash; diff --git a/validator/fabric.h b/validator/fabric.h index d2e9b4735..86868f766 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -82,7 +82,7 @@ void run_check_proof_query(BlockIdExt id, td::Ref proof, td::actor::Actor void run_check_proof_link_query(BlockIdExt id, td::Ref proof, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, - BlockCandidate candidate, td::Ref validator_set, + BlockCandidate candidate, td::Ref validator_set, PublicKeyHash local_validator_id, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, unsigned mode = 0); void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 94d98ac59..bc8e0ef53 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -5936,8 +5936,9 @@ bool Collator::create_block_candidate() { stats_.actual_bytes = block_candidate->data.size(); stats_.actual_collated_data_bytes = block_candidate->collated_data.size(); stats_.attempt = attempt_idx_; - stats_.collator_node_id = collator_node_id_.bits256_value(); - stats_.validator_id = created_by_.as_bits256(); + stats_.is_validator = !(mode_ & CollateMode::from_collator_node); + stats_.self = stats_.is_validator ? PublicKey(pubkeys::Ed25519(created_by_)).compute_short_id() + : collator_node_id_.pubkey_hash(); stats_.estimated_bytes = block_limit_status_->estimate_block_size(); stats_.gas = block_limit_status_->gas_used; stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt; diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index d1555edc5..ae341a5cb 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -192,8 +192,8 @@ void run_check_proof_link_query(BlockIdExt id, td::Ref proof, td::act .release(); } -void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, - std::vector prev, BlockCandidate candidate, td::Ref validator_set, +void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, + BlockCandidate candidate, td::Ref validator_set, PublicKeyHash local_validator_id, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, unsigned mode) { BlockSeqno seqno = 0; @@ -207,8 +207,8 @@ void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, td::actor::create_actor(PSTRING() << (is_fake ? "fakevalidate" : "validateblock") << shard.to_str() << ":" << (seqno + 1) << "#" << idx.fetch_add(1), shard, min_masterchain_block_id, std::move(prev), std::move(candidate), - std::move(validator_set), std::move(manager), timeout, std::move(promise), - mode) + std::move(validator_set), local_validator_id, std::move(manager), timeout, + std::move(promise), mode) .release(); } diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 92b32b75c..b38d2f37d 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -67,16 +67,17 @@ std::string ErrorCtx::as_string() const { * @param promise The Promise to return the ValidateCandidateResult to. * @param mode +1 - fake mode */ -ValidateQuery::ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block_id, - std::vector prev, BlockCandidate candidate, Ref validator_set, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, unsigned mode) +ValidateQuery::ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, + BlockCandidate candidate, Ref validator_set, + PublicKeyHash local_validator_id, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise promise, unsigned mode) : shard_(shard) , id_(candidate.id) , min_mc_block_id(min_masterchain_block_id) , prev_blocks(std::move(prev)) , block_candidate(std::move(candidate)) , validator_set_(std::move(validator_set)) + , local_validator_id_(local_validator_id) , manager(manager) , timeout(timeout) , main_promise(std::move(promise)) @@ -7062,6 +7063,7 @@ void ValidateQuery::record_stats(bool valid, std::string error_message) { stats.block_id = id_; stats.collated_data_hash = block_candidate.collated_file_hash; stats.validated_at = td::Clocks::system(); + stats.self = local_validator_id_; stats.valid = valid; if (valid) { stats.comment = (PSTRING() << "OK ts=" << now_); diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 328a9e1eb..05a397c85 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -118,7 +118,7 @@ class ValidateQuery : public td::actor::Actor { public: ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, - BlockCandidate candidate, td::Ref validator_set, + BlockCandidate candidate, td::Ref validator_set, PublicKeyHash local_validator_id, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, unsigned mode = 0); @@ -132,6 +132,7 @@ class ValidateQuery : public td::actor::Actor { std::vector> prev_states; BlockCandidate block_candidate; td::Ref validator_set_; + PublicKeyHash local_validator_id_ = PublicKeyHash::zero(); td::actor::ActorId manager; td::Timestamp timeout; td::Promise main_promise; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 59f114b33..6c31fe22f 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -62,8 +62,8 @@ struct CollationStats { double collated_at = -1.0; td::uint32 actual_bytes = 0, actual_collated_data_bytes = 0; int attempt = 0; - td::Bits256 collator_node_id = td::Bits256::zero(); - td::Bits256 validator_id = td::Bits256::zero(); + PublicKeyHash self = PublicKeyHash::zero(); + bool is_validator = false; td::uint32 estimated_bytes = 0, gas = 0, lt_delta = 0, estimated_collated_data_bytes = 0; int cat_bytes = 0, cat_gas = 0, cat_lt_delta = 0, cat_collated_data_bytes = 0; std::string limits_log; @@ -75,11 +75,9 @@ struct CollationStats { std::string time_stats; tl_object_ptr tl() const { - int flags = (collator_node_id.is_zero() ? 0 : ton_api::validatorStats_collatedBlock::Flags::COLLATOR_NODE_ID_MASK) | - (validator_id.is_zero() ? 0 : ton_api::validatorStats_collatedBlock::Flags::VALIDATOR_ID_MASK); return create_tl_object( - flags, create_tl_block_id(block_id), collated_data_hash, cc_seqno, collated_at, actual_bytes, - actual_collated_data_bytes, attempt, collator_node_id, validator_id, total_time, work_time, cpu_work_time, + create_tl_block_id(block_id), collated_data_hash, cc_seqno, collated_at, actual_bytes, + actual_collated_data_bytes, attempt, self.bits256_value(), is_validator, total_time, work_time, cpu_work_time, time_stats, create_tl_object(estimated_bytes, gas, lt_delta, estimated_collated_data_bytes, cat_bytes, cat_gas, @@ -93,6 +91,7 @@ struct ValidationStats { BlockIdExt block_id; td::Bits256 collated_data_hash = td::Bits256::zero(); double validated_at = -1.0; + PublicKeyHash self = PublicKeyHash::zero(); bool valid = false; std::string comment; td::uint32 actual_bytes = 0, actual_collated_data_bytes = 0; @@ -100,22 +99,23 @@ struct ValidationStats { tl_object_ptr tl() const { return create_tl_object( - create_tl_block_id(block_id), collated_data_hash, validated_at, valid, comment, actual_bytes, - actual_collated_data_bytes, total_time, work_time, cpu_work_time); + create_tl_block_id(block_id), collated_data_hash, validated_at, self.bits256_value(), valid, comment, + actual_bytes, actual_collated_data_bytes, total_time, work_time, cpu_work_time); } }; struct CollatorNodeResponseStats { - td::Bits256 collator_node_id = td::Bits256::zero(); - td::Bits256 validator_id = td::Bits256::zero(); + PublicKeyHash self = PublicKeyHash::zero(); + PublicKeyHash validator_id = PublicKeyHash::zero(); double timestamp = -1.0; BlockIdExt block_id, original_block_id; td::Bits256 collated_data_hash = td::Bits256::zero(); tl_object_ptr tl() const { return create_tl_object( - collator_node_id, validator_id, timestamp, create_tl_block_id(block_id), create_tl_block_id(original_block_id), - collated_data_hash);; + self.bits256_value(), validator_id.bits256_value(), timestamp, create_tl_block_id(block_id), + create_tl_block_id(original_block_id), collated_data_hash); + ; } }; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 5ebe54678..4bbd48540 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -153,7 +153,7 @@ void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vector prev, BlockIdExt min_masterch .shard = shard_, .cc_seqno = validator_set_->get_catchain_seqno(), .last_key_block_seqno = last_key_block_seqno_, - .started_at = td::Clocks::system()}; + .started_at = td::Clocks::system(), + .self = local_id_}; td::uint32 idx = 0; for (const auto &node : validator_set_->export_vector()) { PublicKeyHash id = ValidatorFullId{node.key}.compute_short_id(); From ca1093d9bad844acf6a29280292c8f64629a452c Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 27 Dec 2024 13:00:00 +0300 Subject: [PATCH 138/388] Improve handling sessino stats file --- validator/manager.cpp | 128 ++++++++++++++++++------------------------ validator/manager.hpp | 8 +++ 2 files changed, 63 insertions(+), 73 deletions(-) diff --git a/validator/manager.cpp b/validator/manager.cpp index ff142c570..545f0ec73 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1866,6 +1866,7 @@ void ValidatorManagerImpl::start_up() { } validator_manager_init(opts_, actor_id(this), db_.get(), std::move(P)); + init_session_stats(); check_waiters_at_ = td::Timestamp::in(1.0); alarm_timestamp().relax(check_waiters_at_); @@ -3108,50 +3109,16 @@ void ValidatorManagerImpl::wait_shard_client_state(BlockSeqno seqno, td::Timesta } void ValidatorManagerImpl::log_validator_session_stats(validatorsession::ValidatorSessionStats stats) { - std::string fname = opts_->get_session_logs_file(); - if (fname.empty()) { - return; - } stats.fix_block_ids(); - auto obj = stats.tl(); - auto s = td::json_encode(td::ToJson(*obj.get()), false); - s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - std::ofstream file; - file.open(fname, std::ios_base::app); - file << s << "\n"; - file.close(); - LOG(INFO) << "Writing validator session stats for " << stats.block_id.to_str(); + write_session_stats(stats); } void ValidatorManagerImpl::log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) { - std::string fname = opts_->get_session_logs_file(); - if (fname.empty()) { - return; - } - auto obj = stats.tl(); - auto s = td::json_encode(td::ToJson(*obj.get()), false); - s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - std::ofstream file; - file.open(fname, std::ios_base::app); - file << s << "\n"; - file.close(); - LOG(INFO) << "Writing new validator group stats for " << stats.session_id << " shard=" << stats.shard.to_str() - << " cc_seqno=" << stats.cc_seqno; + write_session_stats(stats); } void ValidatorManagerImpl::log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) { - std::string fname = opts_->get_session_logs_file(); - if (fname.empty()) { - return; - } - auto obj = stats.tl(); - auto s = td::json_encode(td::ToJson(*obj.get()), false); - s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - std::ofstream file; - file.open(fname, std::ios_base::app); - file << s << "\n"; - file.close(); - LOG(INFO) << "Writing end validator group stats for " << stats.session_id; + write_session_stats(stats); } void ValidatorManagerImpl::get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) { @@ -3443,6 +3410,7 @@ void ValidatorManagerImpl::update_options(td::Ref opts) td::actor::send_closure(c, &CollationManager::update_options, opts); } opts_ = std::move(opts); + init_session_stats(); } void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) { @@ -3600,48 +3568,15 @@ td::actor::ActorOwn ValidatorManagerFactory::create( } void ValidatorManagerImpl::log_collate_query_stats(CollationStats stats) { - std::string fname = opts_->get_session_logs_file(); - if (fname.empty()) { - return; - } - auto obj = stats.tl(); - auto s = td::json_encode(td::ToJson(*obj.get()), false); - s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - std::ofstream file; - file.open(fname, std::ios_base::app); - file << s << "\n"; - file.close(); - LOG(DEBUG) << "Writing collation stats stats for " << stats.block_id.to_str(); + write_session_stats(stats); } void ValidatorManagerImpl::log_validate_query_stats(ValidationStats stats) { - std::string fname = opts_->get_session_logs_file(); - if (fname.empty()) { - return; - } - auto obj = stats.tl(); - auto s = td::json_encode(td::ToJson(*obj.get()), false); - s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - std::ofstream file; - file.open(fname, std::ios_base::app); - file << s << "\n"; - file.close(); - LOG(DEBUG) << "Writing validation stats stats for " << stats.block_id.to_str(); + write_session_stats(stats); } void ValidatorManagerImpl::log_collator_node_response_stats(CollatorNodeResponseStats stats) { - std::string fname = opts_->get_session_logs_file(); - if (fname.empty()) { - return; - } - auto obj = stats.tl(); - auto s = td::json_encode(td::ToJson(*obj.get()), false); - s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - std::ofstream file; - file.open(fname, std::ios_base::app); - file << s << "\n"; - file.close(); - LOG(DEBUG) << "Writing collator node response stats stats for " << stats.block_id.to_str(); + write_session_stats(stats); } size_t ValidatorManagerImpl::CheckedExtMsgCounter::get_msg_count(WorkchainId wc, StdSmcAddress addr) { @@ -3702,6 +3637,53 @@ void ValidatorManagerImpl::init_validator_telemetry() { } } +void ValidatorManagerImpl::init_session_stats() { + if (opts_->get_session_logs_file() == session_stats_filename_) { + return; + } + session_stats_filename_ = opts_->get_session_logs_file(); + if (session_stats_filename_.empty()) { + session_stats_enabled_ = false; + session_stats_fd_.close(); + return; + } + auto r_fd = td::FileFd::open(session_stats_filename_, + td::FileFd::Flags::Write | td::FileFd::Flags::Append | td::FileFd::Create); + if (r_fd.is_error()) { + LOG(ERROR) << "Failed to open session stats file for writing: " << r_fd.move_as_error(); + session_stats_filename_.clear(); + session_stats_enabled_ = false; + return; + } + session_stats_fd_ = r_fd.move_as_ok(); + session_stats_enabled_ = true; +} + +template +void ValidatorManagerImpl::write_session_stats(const T &obj) { + if (!session_stats_enabled_) { + return; + } + auto s = td::json_encode(td::ToJson(*obj.tl()), false); + s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); + s += '\n'; + td::Slice slice{s}; + while (!slice.empty()) { + auto R = session_stats_fd_.write(slice); + if (R.is_error()) { + LOG(WARNING) << "Failed to write to session stats: " << R.move_as_error(); + } + if (R.ok() == 0) { + LOG(WARNING) << "Failed to write to session stats"; + } + slice.remove_prefix(R.ok()); + } + auto S = session_stats_fd_.sync(); + if (S.is_error()) { + LOG(WARNING) << "Failed to write to session stats: " << S; + } +} + } // namespace validator } // namespace ton diff --git a/validator/manager.hpp b/validator/manager.hpp index febed73df..d8c30fb5b 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -793,6 +793,14 @@ class ValidatorManagerImpl : public ValidatorManager { std::map> persistent_state_descriptions_; std::map> persistent_state_blocks_; + + bool session_stats_enabled_ = false; + std::string session_stats_filename_; + td::FileFd session_stats_fd_; + + void init_session_stats(); + template + void write_session_stats(const T &obj); }; } // namespace validator From 4ec34edc0c8e963412c62da67ef47d7c958dcc2d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 27 Dec 2024 13:01:00 +0300 Subject: [PATCH 139/388] Fix telementry collection --- validator/full-node-fast-sync-overlays.cpp | 4 +++- validator/full-node-private-overlay.cpp | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index 6554c5180..90bec5933 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -208,7 +208,9 @@ void FullNodeFastSyncOverlay::send_block_candidate(BlockIdExt block_id, Catchain } void FullNodeFastSyncOverlay::send_validator_telemetry(tl_object_ptr telemetry) { - process_telemetry_broadcast(local_id_, telemetry); + if (collect_telemetry_) { + process_telemetry_broadcast(local_id_, telemetry); + } auto data = serialize_tl_object(telemetry, true); if (data.size() <= overlay::Overlays::max_simple_broadcast_size()) { td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_, diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp index 1acfbd4ed..67e5f451a 100644 --- a/validator/full-node-private-overlay.cpp +++ b/validator/full-node-private-overlay.cpp @@ -185,7 +185,9 @@ void FullNodePrivateBlockOverlay::send_broadcast(BlockBroadcast broadcast) { } void FullNodePrivateBlockOverlay::send_validator_telemetry(tl_object_ptr telemetry) { - process_telemetry_broadcast(local_id_.pubkey_hash(), telemetry); + if (collect_telemetry_) { + process_telemetry_broadcast(local_id_.pubkey_hash(), telemetry); + } auto data = serialize_tl_object(telemetry, true); if (data.size() <= overlay::Overlays::max_simple_broadcast_size()) { td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_, From 5ce9d0bcdbeab6be5105c698583bab4901c911ab Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 27 Dec 2024 15:23:31 +0300 Subject: [PATCH 140/388] Add more information to validatorStats.newValidatorGroup.node --- tl/generate/scheme/ton_api.tl | 2 +- tl/generate/scheme/ton_api.tlo | Bin 111644 -> 111708 bytes validator-session/validator-session-types.h | 6 ++++-- validator/validator-group.cpp | 10 +++++++--- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 3ddb69490..bae1077c6 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -938,7 +938,7 @@ validatorStats.validatedBlock bytes:int collated_data_bytes:int total_time:double work_time:double cpu_work_time:double = validatorStats.ValidatedBlock; -validatorStats.newValidatorGroup.node id:int256 weight:long = validatorStats.newValidatorGroup.Node; +validatorStats.newValidatorGroup.node id:int256 pubkey:PublicKey adnl_id:int256 weight:long = validatorStats.newValidatorGroup.Node; validatorStats.newValidatorGroup session_id:int256 shard:tonNode.shardId cc_seqno:int last_key_block_seqno:int started_at:double self_idx:int self:int256 nodes:(vector validatorStats.newValidatorGroup.node) = validatorStats.NewValidatorGroup; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 9522bf9a607a2cf5097dbf4d58cfb0421d3a5d8f..98305606b167e16c79843b3c684fe4c366a0bfbe 100644 GIT binary patch delta 87 zcmbR9g6+-=whbEgEG*eFdzy9Zx9iw5DoB8N+XKQGGg!gQ>3d=rH8?>^ZXOfzVVUf> gRC#hph|FYx7p&83H!;dj7l>t4foQUbWqc9_00yNT?*IS* delta 69 zcmccff^E(VwhbEgEc>!G-J5mnx9iw5DoB8N+XKQGGg!gQ>3d=rHNdRN8&+sc4~S({ MfhehnWqc9_0BZ3VtpET3 diff --git a/validator-session/validator-session-types.h b/validator-session/validator-session-types.h index a2a3ebc1f..e613b08a5 100644 --- a/validator-session/validator-session-types.h +++ b/validator-session/validator-session-types.h @@ -213,6 +213,8 @@ struct ValidatorSessionStats { struct NewValidatorGroupStats { struct Node { PublicKeyHash id = PublicKeyHash::zero(); + PublicKey pubkey; + adnl::AdnlNodeIdShort adnl_id = adnl::AdnlNodeIdShort::zero(); ValidatorWeight weight = 0; }; @@ -228,8 +230,8 @@ struct NewValidatorGroupStats { tl_object_ptr tl() const { std::vector> nodes_arr; for (const auto &node : nodes) { - nodes_arr.push_back( - create_tl_object(node.id.bits256_value(), node.weight)); + nodes_arr.push_back(create_tl_object( + node.id.bits256_value(), node.pubkey.tl(), node.adnl_id.bits256_value(), node.weight)); } return create_tl_object(session_id, create_tl_shard_id(shard), cc_seqno, last_key_block_seqno, started_at, self_idx, diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index efa2a428c..d3819f07d 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -155,8 +155,8 @@ void ValidatorGroup::validate_block_candidate(validatorsession::BlockSourceInfo return; } VLOG(VALIDATOR_DEBUG) << "validating block candidate " << next_block_id; - run_validate_query(shard_, min_masterchain_block_id_, prev_block_ids_, std::move(block), validator_set_, - local_id_, manager_, td::Timestamp::in(15.0), std::move(P)); + run_validate_query(shard_, min_masterchain_block_id_, prev_block_ids_, std::move(block), validator_set_, local_id_, + manager_, td::Timestamp::in(15.0), std::move(P)); } void ValidatorGroup::update_approve_cache(CacheKey key, UnixTime value) { @@ -421,7 +421,11 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch if (id == local_id_) { stats.self_idx = idx; } - stats.nodes.push_back(validatorsession::NewValidatorGroupStats::Node{id, node.weight}); + stats.nodes.push_back(validatorsession::NewValidatorGroupStats::Node{ + .id = id, + .pubkey = PublicKey(pubkeys::Ed25519(node.key)), + .adnl_id = (node.addr.is_zero() ? adnl::AdnlNodeIdShort{id} : adnl::AdnlNodeIdShort{node.addr}), + .weight = node.weight}); ++idx; } td::actor::send_closure(manager_, &ValidatorManager::log_new_validator_group_stats, std::move(stats)); From e8e7883329ab97883ee03cb7bb919213f4951524 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 7 Jan 2025 10:54:09 +0300 Subject: [PATCH 141/388] Add more required cells to the proof in collated data --- validator/impl/collator-impl.h | 2 +- validator/impl/collator.cpp | 33 ++++++++++++++++++++++++++++++- validator/impl/validate-query.cpp | 6 +++--- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 3572109d9..9d73746b7 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -224,7 +224,7 @@ class Collator final : public td::actor::Actor { std::unique_ptr block_candidate; - std::unique_ptr dispatch_queue_; + std::unique_ptr dispatch_queue_, old_dispatch_queue_; std::map sender_generated_messages_count_; unsigned dispatch_queue_ops_{0}; std::map last_dispatch_queue_emitted_lt_; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index bc8e0ef53..e9e286c96 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -1215,6 +1215,7 @@ bool Collator::import_shard_state_data(block::ShardState& ss) { processed_upto_ = std::move(ss.processed_upto_); ihr_pending = std::move(ss.ihr_pending_); dispatch_queue_ = std::move(ss.dispatch_queue_); + old_dispatch_queue_ = std::make_unique(*dispatch_queue_); block_create_stats_ = std::move(ss.block_create_stats_); if (ss.out_msg_queue_size_) { have_out_msg_queue_size_in_state_ = true; @@ -5708,6 +5709,11 @@ Ref Collator::collate_shard_block_descr_set() { return cell; } +/** + * Visits certain cells in out msg queue and dispatch queue to add them to the proof + * + * @returns True on success, Falise if error occurred + */ bool Collator::prepare_msg_queue_proof() { auto res = old_out_msg_queue_->scan_diff( *out_msg_queue_, @@ -5732,7 +5738,32 @@ bool Collator::prepare_msg_queue_proof() { } return true; }, - 3); + 2); + if (!res) { + return false; + } + res = old_dispatch_queue_->scan_diff( + *dispatch_queue_, + [this](td::ConstBitPtr, int, Ref old_value, Ref new_value) { + if (old_value.not_null()) { + old_value = old_dispatch_queue_->extract_value(std::move(old_value)); + vm::Dictionary dispatch_dict{64}; + td::uint64 dispatch_dict_size; + CHECK(block::unpack_account_dispatch_queue(old_value, dispatch_dict, dispatch_dict_size)); + td::BitArray<64> max_lt; + CHECK(dispatch_dict.get_minmax_key(max_lt, true).not_null()); + } + if (new_value.not_null()) { + new_value = dispatch_queue_->extract_value(std::move(new_value)); + vm::Dictionary dispatch_dict{64}; + td::uint64 dispatch_dict_size; + CHECK(block::unpack_account_dispatch_queue(new_value, dispatch_dict, dispatch_dict_size)); + td::BitArray<64> min_lt; + CHECK(dispatch_dict.get_minmax_key(min_lt, false).not_null()); + } + return true; + }, + 2); return res; } diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index b38d2f37d..2d1e95a5b 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -2923,7 +2923,7 @@ bool ValidateQuery::precheck_account_updates() { CHECK(key_len == 256); return precheck_one_account_update(key, std::move(old_val_extra), std::move(new_val_extra)); }, - 3 /* check augmentation of changed nodes */)) { + 2 /* check augmentation of changed nodes in the new dict */)) { return reject_query("invalid ShardAccounts dictionary in the new state"); } } catch (vm::VmError& err) { @@ -3372,7 +3372,7 @@ bool ValidateQuery::precheck_message_queue_update() { CHECK(key_len == 352); return precheck_one_message_queue_update(key, std::move(old_val_extra), std::move(new_val_extra)); }, - 3 /* check augmentation of changed nodes */)) { + 2 /* check augmentation of changed nodes in the new dict */)) { return reject_query("invalid OutMsgQueue dictionary in the new state"); } } catch (vm::VmError& err) { @@ -3533,7 +3533,7 @@ bool ValidateQuery::unpack_dispatch_queue_update() { return check_account_dispatch_queue_update(key, ps_.dispatch_queue_->extract_value(std::move(old_val_extra)), ns_.dispatch_queue_->extract_value(std::move(new_val_extra))); }, - 3 /* check augmentation of changed nodes */); + 2 /* check augmentation of changed nodes in the new dict */); if (!res) { return reject_query("invalid DispatchQueue dictionary in the new state"); } From 1ee3e5d6a72d688f2f48f4c08e9f230137ce1bbd Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 7 Jan 2025 12:56:29 +0300 Subject: [PATCH 142/388] Fix self_collate --- validator/collation-manager.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index 8fabfcaee..2b06ad482 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -284,7 +284,10 @@ void CollationManager::update_collators_list(const CollatorsList& collators_list auto old_collators = std::move(collators_); collators_.clear(); for (const auto& shard : collators_list.shards) { - shards_.push_back({.shard_id = shard.shard_id, .select_mode = shard.select_mode, .collators = shard.collators}); + shards_.push_back({.shard_id = shard.shard_id, + .select_mode = shard.select_mode, + .collators = shard.collators, + .self_collate = shard.self_collate}); for (auto id : shard.collators) { auto it = old_collators.find(id); if (it == old_collators.end()) { From 0763691f7099d54a552969a50364dce9c4cddf08 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 27 Jan 2025 11:49:18 +0300 Subject: [PATCH 143/388] Simplify loading virtual states from collated data, fix message queue cleanup --- validator/fabric.h | 2 +- validator/impl/collator.cpp | 23 +++++--- validator/impl/fabric.cpp | 2 +- validator/impl/validate-query.cpp | 88 +++++++++---------------------- validator/impl/validate-query.hpp | 3 +- 5 files changed, 44 insertions(+), 74 deletions(-) diff --git a/validator/fabric.h b/validator/fabric.h index 86868f766..19319f36e 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -40,7 +40,7 @@ td::Result> create_proof(BlockIdExt masterchain_block_id, td::Buf td::Result> create_proof_link(BlockIdExt block_id, td::BufferSlice proof); td::Result> create_signature_set(td::BufferSlice sig_set); td::Result> create_shard_state(BlockIdExt block_id, td::BufferSlice data); -td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell); +td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell); td::Result create_block_handle(td::BufferSlice data); td::Result create_block_handle(td::Slice data); td::Result create_temp_block_handle(td::BufferSlice data); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index e9e286c96..cf4c1fad8 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -2450,8 +2450,7 @@ bool Collator::out_msg_queue_cleanup() { auto pure_out_msg_queue = std::make_unique(r_cell.move_as_ok().data_cell, 352, block::tlb::aug_OutMsgQueue); td::uint32 deleted = 0; - bool fail = false; - pure_out_msg_queue->check_for_each([&](Ref value, td::ConstBitPtr key, int n) -> bool { + bool ok = pure_out_msg_queue->check_for_each([&](Ref value, td::ConstBitPtr key, int n) -> bool { vm::CellSlice& cs = value.write(); assert(n == 352); block::EnqueuedMsgDescr enq_msg_descr; @@ -2461,7 +2460,6 @@ bool Collator::out_msg_queue_cleanup() { && enq_msg_descr.check_key(key) // check key && enq_msg_descr.lt_ == created_lt)) { LOG(ERROR) << "cannot unpack EnqueuedMsg with key " << key.to_hex(n); - fail = true; return false; } LOG(DEBUG) << "scanning outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," @@ -2495,7 +2493,6 @@ bool Collator::out_msg_queue_cleanup() { if (!dequeue_message(std::move(enq_msg_descr.msg_env_), deliver_lt)) { fatal_error(PSTRING() << "cannot dequeue outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," << enq_msg_descr.hash_.to_hex() << ") by inserting a msg_export_deq record"); - fail = true; return false; } register_out_msg_queue_op(); @@ -2504,11 +2501,11 @@ bool Collator::out_msg_queue_cleanup() { block_limit_class_ = std::max(block_limit_class_, block_limit_status_->classify()); } } - return !delivered; + return true; }); LOG(WARNING) << "deleted " << deleted << " messages from out_msg_queue after merge, remaining queue size is " << out_msg_queue_size_; - if (fail) { + if (!ok) { return fatal_error("error scanning/updating OutMsgQueue"); } } else { @@ -5801,7 +5798,19 @@ bool Collator::create_collated_data() { if (state_proof.is_null()) { return fatal_error("cannot generate Merkle proof for previous state"); } - proofs[prev_state_root_->get_hash().bits()] = std::move(state_proof); + if (after_merge_) { + bool special; + auto cs = vm::load_cell_slice_special(state_proof, special); + CHECK(cs.special_type() == vm::CellTraits::SpecialType::MerkleProof); + cs = vm::load_cell_slice(cs.prefetch_ref(0)); + CHECK(cs.size_refs() == 2); + CHECK(cs.size() == 32); + CHECK(cs.prefetch_ulong(32) == 0x5f327da5U); + proofs[cs.prefetch_ref(0)->get_hash(0).bits()] = vm::CellBuilder::create_merkle_proof(cs.prefetch_ref(0)); + proofs[cs.prefetch_ref(1)->get_hash(0).bits()] = vm::CellBuilder::create_merkle_proof(cs.prefetch_ref(1)); + } else { + proofs[prev_state_root_->get_hash().bits()] = std::move(state_proof); + } } // 4. Proofs for message queues for (vm::MerkleProofBuilder &mpb : neighbor_proof_builders_) { diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index ae341a5cb..2f6ddf1eb 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -84,7 +84,7 @@ td::Result> create_shard_state(BlockIdExt block_id, td::Buff } } -td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell) { +td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell) { auto res = ShardStateQ::fetch(block_id, {}, std::move(root_cell)); if (res.is_error()) { return res.move_as_error(); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 3b3d766c4..18cc55d0e 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -1243,10 +1243,21 @@ bool ValidateQuery::check_this_shard_mc_info() { * @returns True if the previous state is computed successfully, false otherwise. */ bool ValidateQuery::compute_prev_state() { + CHECK(prev_states.size() == 1u + after_merge_); + CHECK(prev_states.size() == prev_blocks.size()); if (!is_masterchain() && full_collated_data_) { - return compute_prev_state_from_collated_data(); + for (size_t i = 0; i < prev_states.size(); i++) { + Ref root = get_virt_state_root(prev_blocks[i]); + if (root.is_null()) { + return reject_query(PSTRING() << "cannot get previous state from collated data: " << prev_blocks[i].to_str()); + } + auto r_state = create_shard_state(prev_blocks[i], std::move(root)); + if (r_state.is_error()) { + return reject_query("failed to parse previous state from collated data", r_state.move_as_error()); + } + prev_states[i] = r_state.move_as_ok(); + } } - CHECK(prev_states.size() == 1u + after_merge_); // Extend validator timeout if previous block is too old UnixTime prev_ts = prev_states[0]->get_unix_time(); if (after_merge_) { @@ -1274,46 +1285,6 @@ bool ValidateQuery::compute_prev_state() { return true; } -bool ValidateQuery::compute_prev_state_from_collated_data() { - td::Bits256 state_hash; - if (id_.seqno() == 1) { - if (prev_blocks.size() != 1) { - return reject_query("seqno is 1, but number of previous blocks is not 1"); - } - state_hash = prev_blocks[0].root_hash; - } else { - std::vector> prev_state_roots(prev_blocks.size()); - for (size_t i = 0; i < prev_blocks.size(); ++i) { - prev_state_roots[i] = get_virt_state_root(prev_blocks[i].root_hash); - if (prev_state_roots[i].is_null()) { - return reject_query(PSTRING() << "cannot get hash of previous state root: " << prev_blocks[i]); - } - } - - if (prev_state_roots.size() == 1) { - state_hash = prev_state_roots[0]->get_hash().bits(); - } else { - CHECK(prev_state_roots.size() == 2); - Ref merged; - if (!block::gen::t_ShardState.cell_pack_split_state(merged, prev_state_roots[0], prev_state_roots[1])) { - return fatal_error(-667, "cannot construct mechanically merged previously state"); - } - state_hash = merged->get_hash().bits(); - } - } - if (state_hash != prev_state_hash_) { - return reject_query("previous state hash mismatch for block "s + id_.to_str() + " : block header declares " + - prev_state_hash_.to_hex() + " , actual " + state_hash.to_hex()); - } - auto it = virt_roots_.find(state_hash); - if (it == virt_roots_.end()) { - return reject_query(PSTRING() << "no state root for previous block in collated data (hash = " - << state_hash.to_hex() << ")"); - } - prev_state_root_ = it->second; - return true; -} - /** * Computes the next shard state using the previous state and the block's Merkle update. */ @@ -1574,21 +1545,11 @@ bool ValidateQuery::request_neighbor_queues() { ++i; continue; } - td::Bits256 state_root_hash; - if (descr.blk_.seqno() == 0) { - state_root_hash = descr.blk_.root_hash; - } else { - Ref state_root = get_virt_state_root(descr.blk_.root_hash); - if (state_root.is_null()) { - return reject_query(PSTRING() << "cannot get hash of state root: " << descr.blk_); - } - state_root_hash = state_root->get_hash().bits(); - } - auto it = virt_roots_.find(state_root_hash); - if (it == virt_roots_.end()) { - return reject_query(PSTRING() << "cannot get state root form collated data: " << descr.blk_); + auto state_root = get_virt_state_root(descr.blk_); + if (state_root.is_null()) { + return reject_query(PSTRING() << "cannot get state root form collated data: " << descr.blk_.to_str()); } - auto state = ShardStateQ::fetch(descr.blk_, {}, it->second); + auto state = ShardStateQ::fetch(descr.blk_, {}, std::move(state_root)); if (state.is_error()) { return reject_query("cannot fetch shard state from collated data", state.move_as_error()); } @@ -6898,22 +6859,23 @@ bool ValidateQuery::postcheck_value_flow() { return true; } -Ref ValidateQuery::get_virt_state_root(td::Bits256 block_root_hash) { - auto it = virt_roots_.find(block_root_hash); +Ref ValidateQuery::get_virt_state_root(const BlockIdExt& block_id) { + auto it = virt_roots_.find(block_id.root_hash); if (it == virt_roots_.end()) { return {}; } Ref root = it->second; + if (block_id.seqno() == 0) { + return root; + } block::gen::Block::Record block; if (!tlb::unpack_cell(root, block)) { return {}; } vm::CellSlice upd_cs{vm::NoVmSpec(), block.state_update}; - if (!(upd_cs.is_special() && upd_cs.prefetch_long(8) == 4 // merkle update - && upd_cs.size_ext() == 0x20228)) { - return {}; - } - return vm::MerkleProof::virtualize_raw(upd_cs.prefetch_ref(1), {0, 1}); + td::Bits256 state_root_hash = upd_cs.prefetch_ref(1)->get_hash(0).bits(); + it = virt_roots_.find(state_root_hash); + return it == virt_roots_.end() ? Ref{} : it->second; } /** diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 289319d13..32d8e9f46 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -306,7 +306,6 @@ class ValidateQuery : public td::actor::Actor { bool extract_collated_data(); bool try_validate(); bool compute_prev_state(); - bool compute_prev_state_from_collated_data(); bool compute_next_state(); bool unpack_merge_prev_state(); bool unpack_prev_state(); @@ -397,7 +396,7 @@ class ValidateQuery : public td::actor::Actor { const block::CurrencyCollection& create); bool check_mc_block_extra(); - Ref get_virt_state_root(td::Bits256 block_root_hash); + Ref get_virt_state_root(const BlockIdExt& block_id); bool check_timeout() { if (timeout && timeout.is_in_past()) { From c863c42ed18be62d43f0055b712d5e9a5c2451ac Mon Sep 17 00:00:00 2001 From: birydrad <> Date: Wed, 11 Dec 2024 14:48:48 +0300 Subject: [PATCH 144/388] celldb: version 2 - thread safe cache - parallel commit - multiple optimizations - support of key-value merge operations - improved tests and benchmarks - in-memory version won't read from key value after start - uses vector in-memory table now - use rocksdb::WALRecoveryMode::kTolerateCorruptedTailRecords - do not silently ignore errors during recovery --- CMakeLists.txt | 1 + crypto/CMakeLists.txt | 3 +- crypto/test/test-db.cpp | 949 ++++++++++++--- crypto/vm/cells/Cell.h | 1 + crypto/vm/cells/DataCell.cpp | 5 +- crypto/vm/cells/DataCell.h | 8 +- crypto/vm/cells/ExtCell.h | 20 + crypto/vm/cells/PrunnedCell.h | 4 + crypto/vm/cells/UsageCell.h | 3 + crypto/vm/cells/VirtualCell.h | 3 + crypto/vm/db/CellHashTable.h | 13 +- crypto/vm/db/CellStorage.cpp | 85 ++ crypto/vm/db/CellStorage.h | 23 + crypto/vm/db/DynamicBagOfCellsDb.cpp | 79 +- crypto/vm/db/DynamicBagOfCellsDb.h | 71 +- crypto/vm/db/DynamicBagOfCellsDbV2.cpp | 1505 ++++++++++++++++++++++++ crypto/vm/db/InMemoryBagOfCellsDb.cpp | 173 ++- crypto/vm/db/StaticBagOfCellsDb.cpp | 18 +- tddb/td/db/KeyValue.h | 53 +- tddb/td/db/MemoryKeyValue.cpp | 82 +- tddb/td/db/MemoryKeyValue.h | 36 +- tddb/td/db/RocksDb.cpp | 123 +- tddb/td/db/RocksDb.h | 26 +- tdutils/td/utils/MpmcQueue.h | 4 +- tdutils/td/utils/Status.h | 7 + tdutils/td/utils/ThreadSafeCounter.h | 60 +- validator-engine/validator-engine.cpp | 12 +- validator-engine/validator-engine.hpp | 4 + validator/db/celldb.cpp | 202 +++- validator/db/celldb.hpp | 8 +- validator/validator-options.hpp | 7 + validator/validator.h | 2 + 32 files changed, 3274 insertions(+), 316 deletions(-) create mode 100644 crypto/vm/db/DynamicBagOfCellsDbV2.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index cea3fc7ec..dfbbd60ac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -156,6 +156,7 @@ if (TON_USE_ROCKSDB) set(WITH_GFLAGS OFF CACHE BOOL "build with GFlags") set(WITH_TESTS OFF CACHE BOOL "build with tests") set(WITH_TOOLS OFF CACHE BOOL "build with tools") + set(USE_RTTI ON CACHE BOOL "use rtti") set(FAIL_ON_WARNINGS OFF CACHE BOOL "fail on warnings") message("Add rocksdb") add_subdirectory(third-party/rocksdb EXCLUDE_FROM_ALL) diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 069083381..530dc7ba5 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -144,6 +144,7 @@ set(TON_CRYPTO_SOURCE set(TON_DB_SOURCE vm/db/DynamicBagOfCellsDb.cpp + vm/db/DynamicBagOfCellsDbV2.cpp vm/db/CellStorage.cpp vm/db/TonDb.cpp @@ -541,7 +542,7 @@ target_include_directories(create-state PUBLIC $ #include #include @@ -59,84 +54,87 @@ #include #include "openssl/digest.hpp" +#include "storage/db.h" +#include "td/utils/VectorQueue.h" #include "vm/dict.h" -#include #include #include #include -#include +#include -namespace vm { -class ThreadExecutor : public DynamicBagOfCellsDb::AsyncExecutor { +#include +#include +#include + +#include "td/actor/actor.h" +#include "td/utils/overloaded.h" + + +class ActorExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor { public: - explicit ThreadExecutor(size_t threads_n) { - for (size_t i = 0; i < threads_n; ++i) { - threads_.emplace_back([this]() { - while (true) { - auto task = pop_task(); - if (!task) { - break; - } - CHECK(generation_.load() % 2 == 1); - task(); - } - }); - } + ActorExecutor(size_t tn) : tn_(tn) { + scheduler_.run_in_context([&] { worker_ = td::actor::create_actor("Worker"); }); + thread_ = td::thread([this]() { scheduler_.run(); }); } - - ~ThreadExecutor() override { - for (size_t i = 0; i < threads_.size(); ++i) { - push_task({}); + ~ActorExecutor() { + scheduler_.run_in_context_external([&] { send_closure(worker_, &Worker::close); }); + thread_.join(); + } + std::string describe() const override { + return PSTRING() << "ActorExecutor(tn=" << tn_ << ")"; + } + class Worker : public td::actor::Actor { + public: + void close() { + td::actor::core::SchedulerContext::get()->stop(); + stop(); } - for (auto &t : threads_) { - t.join(); + void execute_sync(std::function f) { + f(); } - } + }; void execute_async(std::function f) override { - push_task(std::move(f)); + class Runner : public td::actor::Actor { + public: + explicit Runner(std::function f) : f_(std::move(f)) { + } + void start_up() override { + f_(); + stop(); + } + + private: + std::function f_; + }; + auto context = td::actor::SchedulerContext::get(); + if (context) { + td::actor::create_actor("executeasync", std::move(f)).release(); + } else { + scheduler_.run_in_context_external( + [&] { td::actor::create_actor("executeasync", std::move(f)).release(); }); + } } void execute_sync(std::function f) override { - auto x = generation_.load(); - std::scoped_lock lock(sync_mutex_); - CHECK(x == generation_); - CHECK(generation_.load() % 2 == 1); - f(); - CHECK(generation_.load() % 2 == 1); - } - void inc_generation() { - generation_.fetch_add(1); + auto context = td::actor::SchedulerContext::get(); + if (context) { + td::actor::send_closure(worker_, &Worker::execute_sync, std::move(f)); + } else { + scheduler_.run_in_context_external( + [&] { td::actor::send_closure(worker_, &Worker::execute_sync, std::move(f)); }); + } } private: - std::atomic generation_{0}; - std::queue, size_t>> queue_; - std::mutex queue_mutex_; - std::condition_variable cv_; - std::mutex sync_mutex_; - std::vector threads_; - - std::function pop_task() { - std::unique_lock lock(queue_mutex_); - cv_.wait(lock, [&] { return !queue_.empty(); }); - CHECK(!queue_.empty()); - auto task = std::move(queue_.front()); - queue_.pop(); - CHECK(task.second == generation_); - return task.first; - } - - void push_task(std::function task) { - { - std::scoped_lock lock(queue_mutex_); - queue_.emplace(std::move(task), generation_.load()); - } - cv_.notify_one(); - } + size_t tn_; + td::actor::Scheduler scheduler_{{tn_}, false, td::actor::Scheduler::Paused}; + td::actor::ActorOwn worker_; + td::thread thread_; }; +namespace vm { std::vector do_get_serialization_modes() { std::vector res; for (int i = 0; i < 32; i++) { @@ -324,6 +322,34 @@ TEST(Cell, sha_benchmark_threaded) { bench_threaded([n]() { return BenchSha256(n); }); } } +class BenchTasks : public td::Benchmark { + public: + explicit BenchTasks(size_t tn) : tn_(tn) { + } + + std::string get_description() const override { + return PSTRING() << "bench_tasks(threads_n=" << tn_ << ")"; + } + + void run(int n) override { + ActorExecutor executor(tn_); + for (int i = 0; i < n; i++) { + std::latch latch(tn_); + for (size_t j = 0; j < tn_; j++) { + executor.execute_async([&]() { latch.count_down(); }); + } + latch.wait(); + } + } + + private: + size_t tn_{}; +}; +TEST(Bench, Tasks) { + for (size_t tn : {1, 4, 16}) { + bench(BenchTasks(tn)); + } +} std::string serialize_boc(Ref cell, int mode = 31) { CHECK(cell.not_null()); @@ -437,6 +463,8 @@ class CellExplorer { cs_ = {}; break; } + default: + UNREACHABLE(); } } @@ -474,6 +502,8 @@ class CellExplorer { case op.ReadCellSlice: log_ << "read slice " << op.children_mask << "\n"; break; + default: + UNREACHABLE(); } } void log_cell(const Ref &cell) { @@ -627,7 +657,9 @@ TEST(Cell, MerkleProof) { auto exploration2 = CellExplorer::explore(usage_cell, exploration.ops); ASSERT_EQ(exploration.log, exploration2.log); - auto is_prunned = [&](const Ref &cell) { return exploration.visited.count(cell->get_hash()) == 0; }; + auto is_prunned = [&](const Ref &cell_to_check) { + return exploration.visited.count(cell_to_check->get_hash()) == 0; + }; auto proof = MerkleProof::generate(cell, is_prunned); // CellBuilder::virtualize(proof, 1); //ASSERT_EQ(1u, proof->get_level()); @@ -706,7 +738,7 @@ TEST(Cell, MerkleProofCombine) { check(proof_union_fast); } { - auto cell = MerkleProof::virtualize(proof12, 1); + cell = MerkleProof::virtualize(proof12, 1); auto usage_tree = std::make_shared(); auto usage_cell = UsageCell::create(cell, usage_tree->root_ptr()); @@ -927,7 +959,6 @@ TEST(TonDb, InMemoryDynamicBocSimple) { auto before = counter(); SCOPE_EXIT { LOG_CHECK(before == counter()) << before << " vs " << counter(); - ; }; td::Random::Xorshift128plus rnd{123}; auto kv = std::make_shared(); @@ -963,26 +994,193 @@ TEST(TonDb, InMemoryDynamicBocSimple) { int VERBOSITY_NAME(boc) = VERBOSITY_NAME(DEBUG) + 10; +struct CellMerger : td::Merger { + void merge_value_and_update(std::string &value, td::Slice update) override { + return CellStorer::merge_value_and_refcnt_diff(value, update); + } + void merge_update_and_update(std::string &left_update, td::Slice right_update) override { + LOG(ERROR) << "update_and_update"; + UNREACHABLE(); + return CellStorer::merge_refcnt_diffs(left_update, right_update); + } +}; +struct CompactionFilterEraseEmptyValues : public rocksdb::CompactionFilter { + bool Filter(int level, const rocksdb::Slice & /*key*/, const rocksdb::Slice &existing_value, std::string *new_value, + bool *value_changed) const override { + return existing_value.empty(); + } + bool FilterMergeOperand(int, const rocksdb::Slice & /*key*/, const rocksdb::Slice &operand) const override { + return operand.empty(); + } + + // Name of the compaction filter + const char *Name() const override { + return "CompactionFilterEraseEmptyValues"; + } +}; +auto to_td(rocksdb::Slice value) -> td::Slice { + return td::Slice(value.data(), value.size()); +} + +struct MergeOperatorAddCellRefcnt : public rocksdb::MergeOperator { + const char *Name() const override { + return "MergeOperatorAddCellRefcnt"; + } + bool FullMergeV2(const MergeOperationInput &merge_in, MergeOperationOutput *merge_out) const override { + CHECK(merge_in.existing_value); + auto &value = *merge_in.existing_value; + CHECK(merge_in.operand_list.size() >= 1); + td::Slice diff; + std::string diff_buf; + if (merge_in.operand_list.size() == 1) { + diff = to_td(merge_in.operand_list[0]); + } else { + diff_buf = merge_in.operand_list[0].ToString(); + for (size_t i = 1; i < merge_in.operand_list.size(); ++i) { + CellStorer::merge_refcnt_diffs(diff_buf, to_td(merge_in.operand_list[i])); + } + diff = diff_buf; + } + + merge_out->new_value = value.ToString(); + CellStorer::merge_value_and_refcnt_diff(merge_out->new_value, diff); + return true; + } + bool PartialMerge(const rocksdb::Slice & /*key*/, const rocksdb::Slice &left, const rocksdb::Slice &right, + std::string *new_value, rocksdb::Logger *logger) const override { + *new_value = left.ToString(); + CellStorer::merge_refcnt_diffs(*new_value, to_td(right)); + return true; + } +}; + +struct DB { + std::unique_ptr dboc; + std::shared_ptr kv; + void reset_loader() { + dboc->set_loader(std::make_unique(kv->snapshot())); + } +}; struct BocOptions { - std::shared_ptr async_executor; - std::optional o_in_memory; + using AsyncExecutor = DynamicBagOfCellsDb::AsyncExecutor; + + using CreateInMemoryOptions = DynamicBagOfCellsDb::CreateInMemoryOptions; + using CreateV1Options = DynamicBagOfCellsDb::CreateV1Options; + using CreateV2Options = DynamicBagOfCellsDb::CreateV2Options; + + std::shared_ptr async_executor; + struct KvOptions { + enum KvType { InMemory, RocksDb } kv_type{InMemory}; + bool experimental{false}; + bool no_transactions{false}; + size_t cache_size{0}; + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const KvOptions &kv_options) { + if (kv_options.kv_type == KvType::InMemory) { + return sb << "InMemory{}"; + } + return sb << "RockDb{cache_size=" << kv_options.cache_size << ", no_transactions=" << kv_options.no_transactions + << ", experimental=" << kv_options.experimental << "}"; + } + }; + KvOptions kv_options; + std::variant options; td::uint64 seed{123}; - auto create_dboc(td::KeyValueReader *kv, std::optional o_root_n) { - if (o_in_memory) { - auto res = DynamicBagOfCellsDb::create_in_memory(kv, *o_in_memory); - auto stats = res->get_stats().move_as_ok(); - if (o_root_n) { - ASSERT_EQ(*o_root_n, stats.roots_total_count); + std::shared_ptr create_kv(std::shared_ptr old_key_value, bool no_reads = false) { + if (kv_options.kv_type == KvOptions::InMemory) { + if (old_key_value) { + return old_key_value; } - VLOG(boc) << "reset roots_n=" << stats.roots_total_count << " cells_n=" << stats.cells_total_count; - return res; + return std::make_shared(std::make_shared()); + } else if (kv_options.kv_type == KvOptions::RocksDb) { + auto merge_operator = std::make_shared(); + static const CompactionFilterEraseEmptyValues compaction_filter; + CHECK(!old_key_value || old_key_value.use_count() == 1); + std::string db_path = "test_celldb"; + if (old_key_value) { + //LOG(ERROR) << "Reload rocksdb"; + old_key_value.reset(); + } else { + //LOG(ERROR) << "New rocksdb"; + td::RocksDb::destroy(db_path).ensure(); + } + auto db_options = td::RocksDbOptions{ + .block_cache = {}, + .merge_operator = merge_operator, + .compaction_filter = &compaction_filter, + .experimental = kv_options.experimental, + .no_reads = no_reads, + .no_transactions = kv_options.no_transactions, + .use_direct_reads = true, + .no_block_cache = true, + }; + if (kv_options.cache_size != 0) { + db_options.no_block_cache = false; + db_options.block_cache = rocksdb::NewLRUCache(kv_options.cache_size); + } + return std::make_shared(td::RocksDb::open(db_path, std::move(db_options)).move_as_ok()); + } else { + UNREACHABLE(); + } + } + void check_kv_is_empty(KeyValue &kv) { + if (kv_options.kv_type == KvOptions::InMemory) { + ASSERT_EQ(0u, kv.count("").move_as_ok()); + return; } - return DynamicBagOfCellsDb::create(); + + size_t non_empty_values = 0; + kv.for_each([&](auto key, auto value) { + non_empty_values += !value.empty(); + return td::Status::OK(); + }); + if (non_empty_values != 0) { + kv.for_each([&](auto key, auto value) { + LOG(ERROR) << "Key: " << td::hex_encode(key) << " Value: " << td::hex_encode(value); + std::string x; + LOG(ERROR) << int(kv.get(key, x).move_as_ok()); + return td::Status::OK(); + }); + } + ASSERT_EQ(0u, non_empty_values); + } + + [[nodiscard]] auto create_db(DB db, std::optional o_root_n) { + auto old_boc = std::move(db.dboc); + auto old_kv = std::move(db.kv); + old_boc.reset(); + using ResT = DB; + return std::visit(td::overloaded( + [&](CreateV1Options &) -> ResT { + auto new_kv = create_kv(std::move(old_kv)); + auto res = DynamicBagOfCellsDb::create(); + res->set_loader(std::make_unique(new_kv->snapshot())); + return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; + }, + [&](CreateV2Options &options) -> ResT { + auto new_kv = create_kv(std::move(old_kv)); + auto res = DynamicBagOfCellsDb::create_v2(options); + res->set_loader(std::make_unique(new_kv->snapshot())); + return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; + }, + [&](CreateInMemoryOptions &options) -> ResT { + auto read_kv = create_kv(std::move(old_kv), false); + auto res = DynamicBagOfCellsDb::create_in_memory(read_kv.get(), options); + auto new_kv = create_kv(std::move(read_kv), true); + res->set_loader(std::make_unique(new_kv->snapshot())); + auto stats = res->get_stats().move_as_ok(); + if (o_root_n) { + ASSERT_EQ(*o_root_n, stats.roots_total_count); + } + VLOG(boc) << "reset roots_n=" << stats.roots_total_count + << " cells_n=" << stats.cells_total_count; + return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; + }), + options); }; void prepare_commit(DynamicBagOfCellsDb &dboc) { + td::PerfWarningTimer warning_timer("test_db_prepare_commit"); if (async_executor) { - async_executor->inc_generation(); std::latch latch(1); td::Result res; async_executor->execute_sync([&] { @@ -993,70 +1191,172 @@ struct BocOptions { }); latch.wait(); async_executor->execute_sync([&] {}); - async_executor->inc_generation(); + res.ensure(); } else { dboc.prepare_commit(); } } + enum CacheAction { ResetCache, KeepCache }; + void write_commit(DynamicBagOfCellsDb &dboc, std::shared_ptr kv, CacheAction action) { + td::PerfWarningTimer warning_timer("test_db_write_commit"); + kv->begin_write_batch().ensure(); + CellStorer cell_storer(*kv); + { + td::PerfWarningTimer timer("test_db_commit"); + dboc.commit(cell_storer).ensure(); + } + { + td::PerfWarningTimer timer("test_db_commit_write_batch"); + kv->commit_write_batch().ensure(); + } + switch (action) { + case ResetCache: { + td::PerfWarningTimer timer("test_db_reset_cache"); + dboc.set_loader(std::make_unique(kv->snapshot())); + break; + } + case KeepCache: + break; + } + } + + void commit(DB &db, CacheAction action = ResetCache) { + prepare_commit(*db.dboc); + write_commit(*db.dboc, db.kv, action); + } + + std::string description() const { + td::StringBuilder sb; + + sb << "DBOC(type="; + std::visit(td::overloaded([&](const CreateV1Options &) { sb << "V1"; }, + [&](const CreateV2Options &options) { + sb << "V2(concurrency=" << options.extra_threads + 1; + if (options.executor) { + sb << ", executor=" << options.executor->describe(); + } else { + sb << ", executor=threads"; + } + sb << ")"; + }, + [&](const CreateInMemoryOptions &options) { + sb << "InMemory(use_arena=" << options.use_arena + << ", less_memory=" << options.use_less_memory_during_creation << ")"; + }), + options); + sb << kv_options; + if (async_executor) { + sb << ", executor=" << async_executor->describe(); + } + sb << ")"; + + return sb.as_cslice().str(); + } }; template -void with_all_boc_options(F &&f, size_t tests_n = 500) { +void with_all_boc_options(F &&f, size_t tests_n, bool single_thread = false) { LOG(INFO) << "Test dynamic boc"; auto counter = [] { return td::NamedThreadSafeCounter::get_default().get_counter("DataCell").sum(); }; + std::map>> benches; auto run = [&](BocOptions options) { - LOG(INFO) << "\t" << (options.o_in_memory ? "in memory" : "on disk") << (options.async_executor ? " async" : ""); - if (options.o_in_memory) { - LOG(INFO) << "\t\tuse_arena=" << options.o_in_memory->use_arena - << " less_memory=" << options.o_in_memory->use_less_memory_during_creation; - } + auto description = options.description(); + LOG(INFO) << "Running " << description; + auto start = td::Timestamp::now(); + DynamicBagOfCellsDb::Stats stats; + auto o_in_memory = std::get_if(&options.options); for (td::uint32 i = 0; i < tests_n; i++) { auto before = counter(); + options.seed = i == 0 ? 123 : i; - f(options); + auto stats_diff = f(options); + stats.apply_diff(stats_diff); + auto after = counter(); - LOG_CHECK((options.o_in_memory && options.o_in_memory->use_arena) || before == after) - << before << " vs " << after; + LOG_CHECK((o_in_memory && o_in_memory->use_arena) || before == after) << before << " vs " << after; + } + LOG(INFO) << "\ttook " << td::Timestamp::now().at() - start.at() << " seconds"; + LOG(INFO) << stats; + for (auto &[key, value] : stats.named_stats.stats_int) { + if (td::begins_with(key, "bench_")) { + benches[key].emplace_back(value, description); + } } }; - run({.async_executor = std::make_shared(4)}); - run({}); - for (auto use_arena : {false, true}) { - for (auto less_memory : {false, true}) { - run({.o_in_memory = - DynamicBagOfCellsDb::CreateInMemoryOptions{.extra_threads = std::thread::hardware_concurrency(), - .verbose = false, - .use_arena = use_arena, - .use_less_memory_during_creation = less_memory}}); + + // NB: use .experimental to play with different RocksDb parameters + // Note, that new benchmark are necessary to fully understand the effect of different RocksDb options + std::vector kv_options_list = { + // BocOptions::KvOptions{.kv_type = BocOptions::KvOptions::InMemory}, + // BocOptions::KvOptions{.kv_type = BocOptions::KvOptions::RocksDb, .experimental = false, .cache_size = 0}, + BocOptions::KvOptions{ + .kv_type = BocOptions::KvOptions::RocksDb, .experimental = false, .cache_size = size_t{128 << 20}}, + }; + std::vector has_executor_options = {false, true}; + for (auto kv_options : kv_options_list) { + for (bool has_executor : has_executor_options) { + std::shared_ptr executor; + if (has_executor) { + executor = std::make_shared( + 4); // 4 - to compare V1 and V2, because V1 has parallel_load = 4 by default + } + // V2 - 4 threads + run({.async_executor = executor, + .kv_options = kv_options, + .options = DynamicBagOfCellsDb::CreateV2Options{ + .extra_threads = 3, .executor = executor, .cache_ttl_max = 5}}); + + // V1 + run({.async_executor = executor, .kv_options = kv_options, .options = DynamicBagOfCellsDb::CreateV1Options{}}); + + // V2 - one thread + run({.async_executor = executor, + .kv_options = kv_options, + .options = DynamicBagOfCellsDb::CreateV2Options{.extra_threads = 0, .executor = executor, .cache_ttl_max = 5}}); + + // InMemory + for (auto use_arena : {false, true}) { + for (auto less_memory : {false, true}) { + run({.async_executor = executor, + .kv_options = kv_options, + .options = + DynamicBagOfCellsDb::CreateInMemoryOptions{.extra_threads = std::thread::hardware_concurrency(), + .verbose = false, + .use_arena = use_arena, + .use_less_memory_during_creation = less_memory}}); + } + } + } + } + + for (auto &[name, v] : benches) { + std::sort(v.begin(), v.end()); + LOG(INFO) << "Bench " << name; + for (auto &[t, name] : v) { + LOG(INFO) << "\t" << name << " " << double(t) / 1000 << "s"; } } } -void test_dynamic_boc(BocOptions options) { - auto counter = [] { return td::NamedThreadSafeCounter::get_default().get_counter("DataCell").sum(); }; - auto before = counter(); - SCOPE_EXIT { - LOG_CHECK((options.o_in_memory && options.o_in_memory->use_arena) || before == counter()) - << before << " vs " << counter(); - }; +DynamicBagOfCellsDb::Stats test_dynamic_boc(BocOptions options) { + DynamicBagOfCellsDb::Stats stats; td::Random::Xorshift128plus rnd{options.seed}; std::string old_root_hash; std::string old_root_serialization; - auto kv = std::make_shared(); - auto create_dboc = [&]() { + DB db; + auto reload_db = [&]() { auto roots_n = old_root_hash.empty() ? 0 : 1; - return options.create_dboc(kv.get(), roots_n); + db = options.create_db(std::move(db), roots_n); }; - auto dboc = create_dboc(); - dboc->set_loader(std::make_unique(kv)); + reload_db(); for (int t = 1000; t >= 0; t--) { if (rnd() % 10 == 0) { - dboc = create_dboc(); + reload_db(); } - dboc->set_loader(std::make_unique(kv)); + db.reset_loader(); Ref old_root; if (!old_root_hash.empty()) { - old_root = dboc->load_cell(old_root_hash).move_as_ok(); + old_root = db.dboc->load_cell(old_root_hash).move_as_ok(); auto serialization = serialize_boc(old_root); ASSERT_EQ(old_root_serialization, serialization); } @@ -1071,47 +1371,61 @@ void test_dynamic_boc(BocOptions options) { ->get_root_cell(0) .move_as_ok(); - dboc->dec(old_root); + db.dboc->dec(old_root); if (t != 0) { - dboc->inc(cell); - } - dboc->prepare_commit().ensure(); - { - CellStorer cell_storer(*kv); - dboc->commit(cell_storer).ensure(); + db.dboc->inc(cell); } + options.commit(db, BocOptions::ResetCache); } - ASSERT_EQ(0u, kv->count("").ok()); + options.check_kv_is_empty(*db.kv); + + stats.named_stats.apply_diff(db.kv->get_usage_stats().to_named_stats()); + return stats; } TEST(TonDb, DynamicBoc) { with_all_boc_options(test_dynamic_boc, 1); }; -void test_dynamic_boc2(BocOptions options) { +DynamicBagOfCellsDb::Stats test_dynamic_boc2(BocOptions options) { td::Random::Xorshift128plus rnd{options.seed}; + DynamicBagOfCellsDb::Stats stats; - int total_roots = rnd.fast(1, !rnd.fast(0, 10) * 100 + 10); + int total_roots = rnd.fast(1, !rnd.fast(0, 30) * 100 + 10); int max_roots = rnd.fast(1, 20); + int max_cells = 20; + + // VERBOSITY_NAME(boc) = 1; + // LOG(WARNING) << "====================================================\n\n"; + // max_roots = 2; + // total_roots = 2; + // max_cells = 2; + + auto meta_key = [](size_t i) { return PSTRING() << "meta." << i; }; + std::array meta; + int last_commit_at = 0; int first_root_id = 0; int last_root_id = 0; - auto kv = std::make_shared(); - auto create_dboc = [&](td::int64 root_n) { return options.create_dboc(kv.get(), root_n); }; - auto dboc = create_dboc(0); - dboc->set_loader(std::make_unique(kv)); + DB db; + auto reload_db = [&](td::int64 root_n) { db = options.create_db(std::move(db), root_n); }; + reload_db(0); auto counter = [] { return td::NamedThreadSafeCounter::get_default().get_counter("DataCell").sum(); }; auto before = counter(); - SCOPE_EXIT{ - // LOG_CHECK((options.o_in_memory && options.o_in_memory->use_arena) || before == counter()) - // << before << " vs " << counter(); + SCOPE_EXIT { + bool skip_check = false; + if (std::holds_alternative(options.options) && + std::get(options.options).use_arena) { + skip_check = true; + } + LOG_IF(FATAL, !(skip_check || before == counter())) << before << " vs " << counter(); }; std::vector> roots(max_roots); std::vector root_hashes(max_roots); auto add_root = [&](Ref root) { - dboc->inc(root); + db.dboc->inc(root); root_hashes[last_root_id % max_roots] = (root->get_hash().as_slice().str()); roots[last_root_id % max_roots] = root; last_root_id++; @@ -1124,9 +1438,9 @@ void test_dynamic_boc2(BocOptions options) { VLOG(boc) << " from db"; auto from_root_hash = root_hashes[root_id % max_roots]; if (rnd() % 2 == 0) { - from_root = dboc->load_root(from_root_hash).move_as_ok(); + from_root = db.dboc->load_root(from_root_hash).move_as_ok(); } else { - from_root = dboc->load_cell(from_root_hash).move_as_ok(); + from_root = db.dboc->load_cell(from_root_hash).move_as_ok(); } } else { VLOG(boc) << "FROM MEMORY"; @@ -1147,31 +1461,69 @@ void test_dynamic_boc2(BocOptions options) { from_root = get_root(rnd.fast(first_root_id, last_root_id - 1)); } VLOG(boc) << " ..."; - auto new_root = gen_random_cell(rnd.fast(1, 20), from_root, rnd); - root_cnt[new_root->get_hash()]++; - add_root(std::move(new_root)); + auto new_root_cell = gen_random_cell(rnd.fast(1, max_cells), from_root, rnd); + root_cnt[new_root_cell->get_hash()]++; + add_root(std::move(new_root_cell)); VLOG(boc) << " OK"; }; - auto commit = [&] { - VLOG(boc) << "commit"; - //rnd.fast(0, 1); - options.prepare_commit(*dboc); - { - CellStorer cell_storer(*kv); - dboc->commit(cell_storer); + td::UsageStats commit_stats{}; + auto commit = [&](bool finish = false) { + for (size_t i = 0; i < meta.size(); i++) { + std::string value; + auto status = db.dboc->meta_get(meta_key(i), value).move_as_ok(); + if (status == KeyValue::GetStatus::Ok) { + ASSERT_EQ(value, meta[i]); + ASSERT_TRUE(!meta[i].empty()); + } else { + ASSERT_TRUE(meta[i].empty()); + } + + if (meta[i].empty()) { + if (!finish && rnd() % 2 == 0) { + meta[i] = td::to_string(rnd()); + db.dboc->meta_set(meta_key(i), meta[i]); + VLOG(boc) << "meta set " << meta_key(i) << " " << meta[i]; + } + } else { + auto f = finish ? 1 : rnd() % 3; + if (f == 0) { + meta[i] = td::to_string(rnd()); + db.dboc->meta_set(meta_key(i), meta[i]); + VLOG(boc) << "meta set " << meta_key(i) << " " << meta[i]; + } else if (f == 1) { + meta[i] = ""; + db.dboc->meta_erase(meta_key(i)); + VLOG(boc) << "meta erase " << meta_key(i); + } + } } - dboc->set_loader(std::make_unique(kv)); + + VLOG(boc) << "before commit cells_in_db=" << db.kv->count(""); + //rnd.fast(0, 1); + auto stats_before = db.kv->get_usage_stats(); + options.commit(db, BocOptions::ResetCache); + auto stats_after = db.kv->get_usage_stats(); + commit_stats = commit_stats + stats_after - stats_before; + VLOG(boc) << "after commit cells_in_db=" << db.kv->count(""); + + // db.reset_loader(); for (int i = last_commit_at; i < last_root_id; i++) { roots[i % max_roots].clear(); } last_commit_at = last_root_id; }; - auto reset = [&] { + auto reset = [&](bool force_full = false) { VLOG(boc) << "reset"; commit(); - dboc = create_dboc(td::int64(root_cnt.size())); - dboc->set_loader(std::make_unique(kv)); + if (rnd() % 3 == 0 || force_full) { + // very slow for rocksdb + auto r_stats = db.dboc->get_stats(); + if (r_stats.is_ok()) { + stats.apply_diff(r_stats.ok()); + } + reload_db(root_cnt.size()); + } }; auto delete_root = [&] { @@ -1187,22 +1539,28 @@ void test_dynamic_boc2(BocOptions options) { root_cnt.erase(it); } - dboc->dec(std::move(old_root)); + db.dboc->dec(std::move(old_root)); first_root_id++; VLOG(boc) << " OK"; }; td::RandomSteps steps({{new_root, 10}, {delete_root, 9}, {commit, 2}, {reset, 1}}); while (first_root_id != total_roots) { - VLOG(boc) << first_root_id << " " << last_root_id << " " << kv->count("").ok(); + VLOG(boc) << first_root_id << " " << last_root_id; // << " " << db.kv->count("").ok(); steps.step(rnd); } - commit(); - ASSERT_EQ(0u, kv->count("").ok()); + commit(true); + options.check_kv_is_empty(*db.kv); + + // auto stats = kv->get_usage_stats(); + // LOG(ERROR) << "total: " << stats; + reset(true); + stats.named_stats.apply_diff(db.kv->get_usage_stats().to_named_stats()); + return stats; } TEST(TonDb, DynamicBoc2) { - with_all_boc_options(test_dynamic_boc2); + with_all_boc_options(test_dynamic_boc2, 50); } template @@ -1341,6 +1699,10 @@ class CompactArray { size_t size() const { return size_; } + void reset() { + size_ = 0; + root_ = {}; + } Ref merkle_proof(std::vector keys) { std::set hashes; @@ -1435,6 +1797,283 @@ class FastCompactArray { std::vector v_; }; +struct BocTestHelper { + public: + BocTestHelper() = default; + BocTestHelper(td::int64 seed) : rnd_(seed) { + } + + CompactArray create_array(size_t size, td::uint64 max_value) { + std::vector v(size); + td::Random::Xorshift128plus rnd{123}; + for (auto &x : v) { + x = rnd() % max_value; + } + return CompactArray(v); + } + + private: + td::Random::Xorshift128plus rnd_{123}; +}; + +DynamicBagOfCellsDb::Stats bench_dboc_get_and_set(BocOptions options) { + BocTestHelper helper(options.seed); + size_t n = 1 << 20; + size_t max_value = 1 << 26; + auto arr = helper.create_array(n, max_value); + + // auto kv = std::make_shared(); + td::Slice db_path = "compact_array_db"; + td::RocksDb::destroy(db_path).ensure(); + + DB db = options.create_db({}, {}); + DynamicBagOfCellsDb::Stats stats; + + td::Timer total_timer; + + auto bench = [&](td::Slice desc, auto &&f) { + auto before = db.dboc->get_stats().move_as_ok(); + td::Timer timer; + LOG(ERROR) << "Benchmarking " << desc; + f(); + stats.named_stats.stats_int[desc.str()] = td::int64(timer.elapsed() * 1000); + LOG(ERROR) << "Benchmarking " << desc << " done: " << timer.elapsed() << "s\n"; + auto after = db.dboc->get_stats().move_as_ok(); + after.named_stats.subtract_diff(before.named_stats); + LOG(ERROR) << after; + }; + + td::VectorQueue roots; + // Save array in db + bench(PSLICE() << "bench_inc_large_db(n=" << n << ")", [&] { + db.dboc->inc(arr.root()); + roots.push(arr.root()->get_hash()); + options.commit(db, BocOptions::ResetCache); + }); + bench("bench_compactify", [&] { + auto status = dynamic_cast(*db.kv).raw_db()->CompactRange({}, nullptr, nullptr); + LOG_IF(FATAL, !status.ok()) << status.ToString(); + }); + db = options.create_db(std::move(db), {}); + + bench(PSLICE() << "bench_inc_large_existed_db(n=" << n << ")", [&] { + db.dboc->inc(arr.root()); + roots.push(arr.root()->get_hash()); + options.commit(db, BocOptions::ResetCache); + }); + + td::Random::Xorshift128plus rnd{123}; + while (false) { + auto hash = arr.root()->get_hash(); + arr = CompactArray{n, db.dboc->load_root(hash.as_slice()).move_as_ok()}; + td::Timer timer; + for (size_t i = 0; i < 10000; i++) { + auto pos = rnd() % n; + arr.get(pos); + } + LOG(ERROR) << timer.elapsed() << "s\n"; + db.reset_loader(); + } + + for (auto p : + std::vector>{{10000, 0}, {10000, 5}, {5000, 5000}, {5, 10000}, {0, 10000}}) { + auto get_n = p.first; + auto set_n = p.second; + auto hash = arr.root()->get_hash(); + arr = CompactArray{n, db.dboc->load_root(hash.as_slice()).move_as_ok()}; + bench(PSTRING() << "bench_changes(get_n=" << get_n << ", set_n=" << set_n << ")", [&] { + for (size_t i = 0; i < get_n; i++) { + auto pos = rnd() % n; + arr.get(pos); + } + for (size_t i = 0; i < set_n; i++) { + auto pos = rnd() % n; + auto value = rnd() % max_value; + arr.set(pos, value); + } + }); + bench(PSTRING() << "bench_commit(get_n=" << get_n << ", set_n=" << set_n << ")", [&] { + db.dboc->inc(arr.root()); + roots.push(arr.root()->get_hash()); + options.commit(db, BocOptions::ResetCache); + }); + } + arr.reset(); + + bench(PSLICE() << "bench_dec_some_roots()", [&] { + while (roots.size() > 1) { + auto hash = roots.pop(); + auto cell = db.dboc->load_cell(hash.as_slice()).move_as_ok(); + db.dboc->dec(cell); + } + options.commit(db, BocOptions::ResetCache); + }); + + db = options.create_db(std::move(db), {}); + + bench(PSLICE() << "bench_dec_large_root(n=" << n << ")", [&] { + while (!roots.empty()) { + auto hash = roots.pop(); + auto cell = db.dboc->load_cell(hash.as_slice()).move_as_ok(); + db.dboc->dec(cell); + + /* + do { + auto cell = db.dboc->load_cell(hash.as_slice()).move_as_ok(); + db.dboc->dec(cell); + cell = {}; + options.prepare_commit(*db.dboc); + //db.dboc->prepare_commit().ensure(); + db.reset_loader(); + db = options.create_db(std::move(db), {}); + } while (true); + */ + } + options.commit(db, BocOptions::ResetCache); + }); + stats.named_stats.stats_int["bench_total"] = td::int64(total_timer.elapsed() * 1000); + + return stats; +} + +TEST(TonDb, BenchDynamicBocGetAndSet) { + with_all_boc_options(bench_dboc_get_and_set, 1); +} + +TEST(TonDb, DynamicBocIncSimple) { + auto kv = std::make_shared(std::make_shared()); + auto db = DynamicBagOfCellsDb::create_v2({.extra_threads = 0}); + db->set_loader(std::make_unique(kv)); + + td::Random::Xorshift128plus rnd(123); + size_t size = 4; + std::vector values(size); + for (auto &v : values) { + //v = rnd() % 2; + v = rnd(); + } + // 1. Create large dictionary and store it in db + auto arr_ptr = std::make_unique(values); + auto &arr = *arr_ptr; + td::VectorQueue queue; + auto push = [&]() { + //LOG(ERROR) << "PUSH ROOT"; + auto begin_stats = kv->get_usage_stats(); + db->inc(arr.root()); + queue.push(arr.root()->get_hash()); + vm::CellStorer cell_storer(*kv); + db->commit(cell_storer); + auto end_stats = kv->get_usage_stats(); + LOG(ERROR) << end_stats - begin_stats; + db->set_loader(std::make_unique(kv)); + auto hash = arr.root()->get_hash(); + arr = CompactArray{size, db->load_root(hash.as_slice()).move_as_ok()}; + //LOG(ERROR) << "CELLS IN DB: " << kv->count("").move_as_ok(); + }; + auto pop = [&]() { + if (queue.empty()) { + return; + } + //LOG(ERROR) << "POP ROOT"; + auto begin_stats = kv->get_usage_stats(); + auto cell = db->load_cell(queue.pop().as_slice()).move_as_ok(); + db->dec(cell); + vm::CellStorer cell_storer(*kv); + db->commit(cell_storer); + auto end_stats = kv->get_usage_stats(); + db->set_loader(std::make_unique(kv)); + //LOG(ERROR) << end_stats - begin_stats; + //LOG(ERROR) << "CELLS IN DB: " << kv->count("").move_as_ok(); + }; + auto upd = [&] { + for (int i = 0; i < 20; i++) { + auto pos = rnd.fast(0, td::narrow_cast(size) - 1); + if (rnd() % 2) { + auto value = rnd() % 2; + arr.set(pos, value); + } else { + arr.get(pos); + } + } + }; + + //LOG(ERROR) << "Created compact array"; + push(); + pop(); + //CHECK(kv->count("").move_as_ok() == 0); + + // 2. Lets change first 20 keys and read last 20 keys + /* + for (size_t i = 0; i < 20 && i < size; i++) { + arr.set(i, rnd()); + } + */ + //arr.set(0, rnd()); + arr.set(size - 1, rnd()); + for (size_t i = 0; i < 20 && i < size; i++) { + arr.get(size - i - 1); + } + + // 3. And now commit diff with stats + push(); + push(); + upd(); + upd(); + push(); + push(); + upd(); + pop(); + pop(); + upd(); + push(); + push(); + while (!queue.empty()) { + pop(); + } + LOG(ERROR) << "CELLS IN DB: " << kv->count("").move_as_ok(); +} + +class BenchCellStorerMergeRefcntDiffs : public td::Benchmark { + public: + std::string get_description() const override { + return PSTRING() << "bench_cells_storer_merge_refcnt_diffs"; + } + + void run(int n) override { + auto cell = vm::CellBuilder().store_bytes(std::string(32, 'A')).finalize(); + auto left_update = CellStorer::serialize_refcnt_diffs(1); + auto right_update = CellStorer::serialize_refcnt_diffs(1); + for (int i = 0; i < n; i++) { + CellStorer::merge_refcnt_diffs(left_update, right_update); + } + } + + private: + size_t tn_{}; +}; +class BenchCellStorerMergeValueAndRefcntDiff : public td::Benchmark { + public: + std::string get_description() const override { + return PSTRING() << "bench_cells_storer_merge_value_and_refcnt_diffs"; + } + + void run(int n) override { + auto cell = vm::CellBuilder().store_bytes(std::string(32, 'A')).finalize(); + auto value = CellStorer::serialize_value(10, cell, false); + auto update = CellStorer::serialize_refcnt_diffs(1); + for (int i = 0; i < n; i++) { + CellStorer::merge_value_and_refcnt_diff(value, update); + } + } + + private: + size_t tn_{}; +}; +TEST(Bench, CellStorerMerge) { + bench(BenchCellStorerMergeRefcntDiffs()); + bench(BenchCellStorerMergeValueAndRefcntDiff()); +} + TEST(Cell, BocHands) { serialize_boc(CellBuilder{}.store_bytes("AAAAAAAA").finalize()); auto a = CellBuilder{}.store_bytes("abcd").store_ref(CellBuilder{}.store_bytes("???").finalize()).finalize(); @@ -2262,7 +2901,8 @@ TEST(TonDb, BocRespectsUsageCell) { ASSERT_STREQ(serialization, serialization_of_virtualized_cell); } -void test_dynamic_boc_respectes_usage_cell(vm::BocOptions options) { +/* +vm::DynamicBagOfCellsDb::Stats test_dynamic_boc_respects_usage_cell(vm::BocOptions options) { td::Random::Xorshift128plus rnd(options.seed); auto cell = vm::gen_random_cell(20, rnd, true); auto usage_tree = std::make_shared(); @@ -2283,11 +2923,14 @@ void test_dynamic_boc_respectes_usage_cell(vm::BocOptions options) { auto serialization_of_virtualized_cell = serialize_boc(virtualized_proof); auto serialization = serialize_boc(cell); ASSERT_STREQ(serialization, serialization_of_virtualized_cell); + vm::DynamicBagOfCellsDb::Stats stats; + return stats; } TEST(TonDb, DynamicBocRespectsUsageCell) { - vm::with_all_boc_options(test_dynamic_boc_respectes_usage_cell, 20); + vm::with_all_boc_options(test_dynamic_boc_respects_usage_cell, 20, true); } +*/ TEST(TonDb, LargeBocSerializer) { td::Random::Xorshift128plus rnd{123}; diff --git a/crypto/vm/cells/Cell.h b/crypto/vm/cells/Cell.h index a75371dbb..e2b47ffc0 100644 --- a/crypto/vm/cells/Cell.h +++ b/crypto/vm/cells/Cell.h @@ -55,6 +55,7 @@ class Cell : public CellTraits { } // load interface + virtual td::Status set_data_cell(Ref &&data_cell) const = 0; virtual td::Result load_cell() const = 0; virtual Ref virtualize(VirtualizationParameters virt) const; virtual td::uint32 get_virtualization() const = 0; diff --git a/crypto/vm/cells/DataCell.cpp b/crypto/vm/cells/DataCell.cpp index 4dd301616..73e86517a 100644 --- a/crypto/vm/cells/DataCell.cpp +++ b/crypto/vm/cells/DataCell.cpp @@ -36,7 +36,8 @@ struct ArenaAllocator { T* obj = new (ptr) T(std::forward(args)...); return std::unique_ptr(obj); } -private: + + private: td::MutableSlice alloc_batch() { size_t batch_size = 1 << 20; auto batch = std::make_unique(batch_size); @@ -53,7 +54,7 @@ struct ArenaAllocator { return res; } }; -} +} // namespace std::unique_ptr DataCell::create_empty_data_cell(Info info) { if (use_arena) { ArenaAllocator allocator; diff --git a/crypto/vm/cells/DataCell.h b/crypto/vm/cells/DataCell.h index 6d3c845fc..b39ee1d4b 100644 --- a/crypto/vm/cells/DataCell.h +++ b/crypto/vm/cells/DataCell.h @@ -31,6 +31,9 @@ class DataCell : public Cell { static thread_local bool use_arena; DataCell(const DataCell& other) = delete; + DataCell(DataCell&& other) = delete; + DataCell& operator=(const DataCell& other) = delete; + DataCell& operator=(DataCell&& other) = delete; ~DataCell() override; static void store_depth(td::uint8* dest, td::uint16 depth) { @@ -126,6 +129,10 @@ class DataCell : public Cell { explicit DataCell(Info info); public: + td::Status set_data_cell(Ref&& data_cell) const override { + CHECK(get_hash() == data_cell->get_hash()); + return td::Status::OK(); + } td::Result load_cell() const override { return LoadedCell{Ref{this}, {}, {}}; } @@ -228,4 +235,3 @@ inline CellHash as_cell_hash(const Ref& cell) { } } // namespace vm - diff --git a/crypto/vm/cells/ExtCell.h b/crypto/vm/cells/ExtCell.h index 401bb0483..dbbd8575b 100644 --- a/crypto/vm/cells/ExtCell.h +++ b/crypto/vm/cells/ExtCell.h @@ -65,6 +65,9 @@ class ExtCell : public Cell { bool is_loaded() const override { return CellView(this)->is_loaded(); } + Ref> get_prunned_cell() const { + return prunned_cell_.load(); + } private: mutable td::AtomicRef data_cell_; @@ -112,6 +115,23 @@ class ExtCell : public Cell { return CellView(this)->get_depth(level); } + td::Status set_data_cell(Ref&& new_data_cell) const override { + auto prunned_cell = prunned_cell_.load(); + if (prunned_cell.is_null()) { + auto old_data_cell = data_cell_.get_unsafe(); + DCHECK(old_data_cell); + TRY_STATUS(old_data_cell->check_equals_unloaded(new_data_cell)); + return td::Status::OK(); + } + + TRY_STATUS(prunned_cell->check_equals_unloaded(new_data_cell)); + if (data_cell_.store_if_empty(new_data_cell)) { + prunned_cell_.store({}); + get_thread_safe_counter_unloaded().add(-1); + } + return td::Status::OK(); + } + td::Result> load_data_cell() const { auto data_cell = data_cell_.get_unsafe(); if (data_cell) { diff --git a/crypto/vm/cells/PrunnedCell.h b/crypto/vm/cells/PrunnedCell.h index a58b245cc..6e8b77093 100644 --- a/crypto/vm/cells/PrunnedCell.h +++ b/crypto/vm/cells/PrunnedCell.h @@ -142,6 +142,10 @@ class PrunnedCell : public Cell { return info_.get_depth(get_storage())[get_level_mask().apply(level).get_hash_i()]; } + td::Status set_data_cell(Ref &&data_cell) const override { + return td::Status::OK(); + } + td::Result load_cell() const override { return td::Status::Error("Can't load prunned branch"); } diff --git a/crypto/vm/cells/UsageCell.h b/crypto/vm/cells/UsageCell.h index 3e6e88982..978b91f76 100644 --- a/crypto/vm/cells/UsageCell.h +++ b/crypto/vm/cells/UsageCell.h @@ -36,6 +36,9 @@ class UsageCell : public Cell { return Ref{true, std::move(cell), std::move(tree_node), PrivateTag{}}; } + td::Status set_data_cell(Ref &&data_cell) const override { + return cell_->set_data_cell(std::move(data_cell)); + } // load interface td::Result load_cell() const override { TRY_RESULT(loaded_cell, cell_->load_cell()); diff --git a/crypto/vm/cells/VirtualCell.h b/crypto/vm/cells/VirtualCell.h index 02abc1c88..a75bdf9de 100644 --- a/crypto/vm/cells/VirtualCell.h +++ b/crypto/vm/cells/VirtualCell.h @@ -37,6 +37,9 @@ class VirtualCell : public Cell { } // load interface + td::Status set_data_cell(Ref &&data_cell) const override { + return cell_->set_data_cell(std::move(data_cell)); + } td::Result load_cell() const override { TRY_RESULT(loaded_cell, cell_->load_cell()); loaded_cell.virt = loaded_cell.virt.apply(virt_); diff --git a/crypto/vm/db/CellHashTable.h b/crypto/vm/db/CellHashTable.h index 522c987be..a38980638 100644 --- a/crypto/vm/db/CellHashTable.h +++ b/crypto/vm/db/CellHashTable.h @@ -40,6 +40,17 @@ class CellHashTable { return res; } + template + std::pair emplace(td::Slice hash, ArgsT &&...args) { + auto it = set_.find(hash); + if (it != set_.end()) { + return std::pair(const_cast(*it), false); + } + auto res = set_.emplace(std::forward(args)...); + CHECK(res.second); + return std::pair(const_cast(*res.first), res.second); + } + template void for_each(F &&f) { for (auto &info : set_) { @@ -64,7 +75,7 @@ class CellHashTable { size_t size() const { return set_.size(); } - InfoT* get_if_exists(td::Slice hash) { + InfoT *get_if_exists(td::Slice hash) { auto it = set_.find(hash); if (it != set_.end()) { return &const_cast(*it); diff --git a/crypto/vm/db/CellStorage.cpp b/crypto/vm/db/CellStorage.cpp index 06df461ef..a07d85e87 100644 --- a/crypto/vm/db/CellStorage.cpp +++ b/crypto/vm/db/CellStorage.cpp @@ -17,14 +17,19 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "vm/db/CellStorage.h" + +#include "td/utils/Parser.h" #include "vm/db/DynamicBagOfCellsDb.h" #include "vm/boc.h" #include "td/utils/base64.h" #include "td/utils/tl_parsers.h" #include "td/utils/tl_helpers.h" +#include + namespace vm { namespace { + class RefcntCellStorer { public: RefcntCellStorer(td::int32 refcnt, const td::Ref &cell, bool as_boc) @@ -43,7 +48,9 @@ class RefcntCellStorer { storer.store_slice(data); return; } + CHECK(refcnt_ > 0); store(refcnt_, storer); + CHECK(cell_.not_null()) store(*cell_, storer); for (unsigned i = 0; i < cell_->size_refs(); i++) { auto cell = cell_->get_ref(i); @@ -91,6 +98,7 @@ class RefcntCellParser { stored_boc_ = true; parse(refcnt, parser); } + CHECK(refcnt > 0); if (!need_data_) { return; } @@ -159,6 +167,9 @@ td::Result CellLoader::load(td::Slice hash, bool need_da DCHECK(get_status == KeyValue::GetStatus::NotFound); return LoadResult{}; } + if (serialized.empty()) { + return LoadResult{}; + } TRY_RESULT(res, load(hash, serialized, need_data, ext_cell_creator)); if (on_load_callback_) { on_load_callback_(res); @@ -198,6 +209,7 @@ td::Result CellLoader::load_refcnt(td::Slice hash) { if (res.refcnt_ == -1) { parse(res.refcnt_, parser); } + CHECK(res.refcnt_ > 0); TRY_STATUS(parser.get_status()); return res; } @@ -216,4 +228,77 @@ std::string CellStorer::serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc) { return kv_.set(cell->get_hash().as_slice(), serialize_value(refcnt, cell, as_boc)); } + +td::Status CellStorer::merge(td::Slice hash, td::int32 refcnt_diff) { + return kv_.merge(hash, serialize_refcnt_diffs(refcnt_diff)); +} + +void CellStorer::merge_value_and_refcnt_diff(std::string &left, td::Slice right) { + if (right.empty()) { + return; + } + CHECK(left.size() > 4); + CHECK(right.size() == 4); + + td::int32 left_refcnt = td::as(left.data()); + size_t shift = 0; + if (left_refcnt == -1) { + CHECK(left.size() >= 8); + left_refcnt = td::as(left.data() + 4); + shift = 4; + } + td::int32 right_refcnt_diff = td::as(right.data()); + td::int32 new_refcnt = left_refcnt + right_refcnt_diff; + CHECK(new_refcnt > 0); + td::as(left.data() + shift) = new_refcnt; +} +void CellStorer::merge_refcnt_diffs(std::string &left, td::Slice right) { + if (right.empty()) { + return; + } + if (left.empty()) { + left = right.str(); + return; + } + CHECK(left.size() == 4); + CHECK(right.size() == 4); + td::int32 left_refcnt_diff = td::as(left.data()); + td::int32 right_refcnt_diff = td::as(right.data()); + td::int32 total_refcnt_diff = left_refcnt_diff + right_refcnt_diff; + td::as(left.data()) = total_refcnt_diff; +} + +std::string CellStorer::serialize_refcnt_diffs(td::int32 refcnt_diff) { + TD_PERF_COUNTER(cell_store_refcnt_diff); + std::string s(4, 0); + td::as(s.data()) = refcnt_diff; + return s; +} + +td::Status CellStorer::apply_diff(const Diff &diff) { + switch (diff.type) { + case Diff::Set: + return kv_.set(diff.key.as_slice(), diff.value); + case Diff::Erase: + return kv_.erase(diff.key.as_slice()); + case Diff::Merge: + return kv_.merge(diff.key.as_slice(), diff.value); + default: + UNREACHABLE(); + } +} +td::Status CellStorer::apply_meta_diff(const MetaDiff &diff) { + switch (diff.type) { + case MetaDiff::Set: + CHECK(diff.key.size() != CellTraits::hash_bytes); + CHECK(!diff.value.empty()); + return kv_.set(diff.key, diff.value); + case MetaDiff::Erase: + CHECK(diff.key.size() != CellTraits::hash_bytes); + CHECK(diff.value.empty()); + return kv_.erase(diff.key); + default: + UNREACHABLE(); + } +} } // namespace vm diff --git a/crypto/vm/db/CellStorage.h b/crypto/vm/db/CellStorage.h index cabd7fdcb..ca32a8007 100644 --- a/crypto/vm/db/CellStorage.h +++ b/crypto/vm/db/CellStorage.h @@ -51,6 +51,9 @@ class CellLoader { td::Result load(td::Slice hash, bool need_data, ExtCellCreator &ext_cell_creator); static td::Result load(td::Slice hash, td::Slice value, bool need_data, ExtCellCreator &ext_cell_creator); td::Result load_refcnt(td::Slice hash); // This only loads refcnt_, cell_ == null + KeyValueReader &key_value_reader() const { + return *reader_; + } private: std::shared_ptr reader_; @@ -62,8 +65,28 @@ class CellStorer { CellStorer(KeyValue &kv); td::Status erase(td::Slice hash); td::Status set(td::int32 refcnt, const td::Ref &cell, bool as_boc); + td::Status merge(td::Slice hash, td::int32 refcnt_diff); + + static void merge_value_and_refcnt_diff(std::string &value, td::Slice right); + static void merge_refcnt_diffs(std::string &left, td::Slice right); + static std::string serialize_refcnt_diffs(td::int32 refcnt_diff); + static std::string serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc); + struct Diff { + enum Type { Set, Erase, Merge } type{Set}; + CellHash key; + std::string value{}; + }; + td::Status apply_diff(const Diff &diff); + + struct MetaDiff { + enum Type { Set, Erase } type{Set}; + std::string key; + std::string value{}; + }; + td::Status apply_meta_diff(const MetaDiff &diff); + private: KeyValue &kv_; }; diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp index 093037583..bd23733e0 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDb.cpp @@ -66,19 +66,27 @@ struct CellInfo { struct Eq { using is_transparent = void; // Pred to use - bool operator()(const CellInfo &info, const CellInfo &other_info) const { return info.key() == other_info.key();} - bool operator()(const CellInfo &info, td::Slice hash) const { return info.key().as_slice() == hash;} - bool operator()(td::Slice hash, const CellInfo &info) const { return info.key().as_slice() == hash;} - + bool operator()(const CellInfo &info, const CellInfo &other_info) const { + return info.key() == other_info.key(); + } + bool operator()(const CellInfo &info, td::Slice hash) const { + return info.key().as_slice() == hash; + } + bool operator()(td::Slice hash, const CellInfo &info) const { + return info.key().as_slice() == hash; + } }; struct Hash { using is_transparent = void; // Pred to use using transparent_key_equal = Eq; - size_t operator()(td::Slice hash) const { return cell_hash_slice_hash(hash); } - size_t operator()(const CellInfo &info) const { return cell_hash_slice_hash(info.key().as_slice());} + size_t operator()(td::Slice hash) const { + return cell_hash_slice_hash(hash); + } + size_t operator()(const CellInfo &info) const { + return cell_hash_slice_hash(info.key().as_slice()); + } }; }; - bool operator<(const CellInfo &a, td::Slice b) { return a.key().as_slice() < b; } @@ -99,6 +107,30 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { return get_cell_info_lazy(level_mask, hash, depth).cell; } + td::Result>> meta_get_all() const override { + std::vector> result; + auto s = loader_->key_value_reader().for_each_in_range("desc", "desd", + [&](const td::Slice &key, const td::Slice &value) { + if (td::begins_with(key, "desc") && key.size() != 32) { + result.emplace_back(key.str(), value.str()); + } + return td::Status::OK(); + }); + TRY_STATUS(std::move(s)); + return result; + } + td::Result meta_get(td::Slice key, std::string &value) override { + return loader_->key_value_reader().get(key, value); + } + td::Status meta_set(td::Slice key, td::Slice value) override { + meta_diffs_.push_back( + CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Set, .key = key.str(), .value = value.str()}); + return td::Status::OK(); + } + td::Status meta_erase(td::Slice key) override { + meta_diffs_.push_back(CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Erase, .key = key.str()}); + return td::Status::OK(); + } td::Result> load_cell(td::Slice hash) override { auto info = hash_table_.get_if_exists(hash); if (info && info->sync_with_db) { @@ -198,21 +230,29 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat if (is_prepared_for_commit()) { return td::Status::OK(); } + td::PerfWarningTimer timer_dfs_new_cells_in_db("dfs_new_cells_in_db"); for (auto &new_cell : to_inc_) { auto &new_cell_info = get_cell_info(new_cell); dfs_new_cells_in_db(new_cell_info); } + timer_dfs_new_cells_in_db.reset(); + td::PerfWarningTimer timer_dfs_new_cells("dfs_new_cells"); for (auto &new_cell : to_inc_) { auto &new_cell_info = get_cell_info(new_cell); dfs_new_cells(new_cell_info); } + timer_dfs_new_cells.reset(); + td::PerfWarningTimer timer_dfs_old_cells("dfs_old_cells"); for (auto &old_cell : to_dec_) { auto &old_cell_info = get_cell_info(old_cell); dfs_old_cells(old_cell_info); } + timer_dfs_old_cells.reset(); + td::PerfWarningTimer timer_save_diff_prepare("save_diff_prepare"); save_diff_prepare(); + timer_save_diff_prepare.reset(); to_inc_.clear(); to_dec_.clear(); @@ -222,6 +262,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat td::Status commit(CellStorer &storer) override { prepare_commit(); + td::PerfWarningTimer times_save_diff("save diff", 0.01); save_diff(storer); // Some elements are erased from hash table, to keep it small. // Hash table is no longer represents the difference between the loader and @@ -249,7 +290,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat celldb_compress_depth_ = value; } - vm::ExtCellCreator& as_ext_cell_creator() override { + vm::ExtCellCreator &as_ext_cell_creator() override { return *this; } @@ -259,6 +300,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat std::vector> to_dec_; CellHashTable hash_table_; std::vector visited_; + std::vector meta_diffs_; Stats stats_diff_; td::uint32 celldb_compress_depth_{0}; @@ -269,8 +311,9 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat class SimpleExtCellCreator : public ExtCellCreator { public: - explicit SimpleExtCellCreator(std::shared_ptr cell_db_reader) : - cell_db_reader_(std::move(cell_db_reader)) {} + explicit SimpleExtCellCreator(std::shared_ptr cell_db_reader) + : cell_db_reader_(std::move(cell_db_reader)) { + } td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { TRY_RESULT(ext_cell, DynamicBocExtCell::create(PrunnedCellInfo{level_mask, hash, depth}, @@ -279,7 +322,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat return std::move(ext_cell); } - std::vector>& get_created_cells() { + std::vector> &get_created_cells() { return created_cells_; } @@ -382,8 +425,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat } bool not_in_db = false; - for_each( - info, [¬_in_db, this](auto &child_info) { not_in_db |= !dfs_new_cells_in_db(child_info); }, false); + for_each(info, [¬_in_db, this](auto &child_info) { not_in_db |= !dfs_new_cells_in_db(child_info); }, false); if (not_in_db) { CHECK(!info.in_db); @@ -441,6 +483,10 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat for (auto info_ptr : visited_) { save_cell(*info_ptr, storer); } + for (auto meta_diff : meta_diffs_) { + storer.apply_meta_diff(meta_diff); + } + meta_diffs_.clear(); visited_.clear(); } @@ -558,6 +604,8 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat } auto res = r_res.move_as_ok(); if (res.status != CellLoader::LoadResult::Ok) { + LOG_CHECK(info.cell.not_null()) << "Trying to load nonexistent cell from db " + << CellHash::from_slice(hash).to_hex(); break; } info.cell = std::move(res.cell()); @@ -651,7 +699,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat CellHashTable cells_; - std::queue load_queue_; + std::queue load_queue_; td::uint32 active_load_ = 0; td::uint32 max_parallel_load_ = 4; }; @@ -814,11 +862,10 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat pca_state_->promise_.set_result(td::Unit()); pca_state_ = {}; } - }; } // namespace -std::unique_ptr DynamicBagOfCellsDb::create() { +std::unique_ptr DynamicBagOfCellsDb::create(CreateV1Options) { return std::make_unique(); } } // namespace vm diff --git a/crypto/vm/db/DynamicBagOfCellsDb.h b/crypto/vm/db/DynamicBagOfCellsDb.h index 62864ad97..ca23d72f2 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.h +++ b/crypto/vm/db/DynamicBagOfCellsDb.h @@ -17,6 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "td/db/KeyValue.h" #include "vm/cells.h" #include "td/utils/Slice.h" @@ -49,13 +50,23 @@ class CellDbReader { class DynamicBagOfCellsDb { public: virtual ~DynamicBagOfCellsDb() = default; + + virtual td::Result>> meta_get_all() const = 0; + virtual td::Result meta_get(td::Slice key, std::string &value) = 0; + virtual td::Status meta_set(td::Slice key, td::Slice value) = 0; + virtual td::Status meta_erase(td::Slice key) = 0; + virtual td::Result> load_cell(td::Slice hash) = 0; virtual td::Result> load_root(td::Slice hash) = 0; virtual td::Result> load_root_thread_safe(td::Slice hash) const = 0; + virtual td::Result>> load_known_roots() const { + return std::vector>(); + } struct Stats { td::int64 roots_total_count{0}; td::int64 cells_total_count{0}; td::int64 cells_total_size{0}; + td::NamedStats named_stats; std::vector> custom_stats; void apply_diff(const Stats &diff) { roots_total_count += diff.roots_total_count; @@ -64,6 +75,20 @@ class DynamicBagOfCellsDb { CHECK(roots_total_count >= 0); CHECK(cells_total_count >= 0); CHECK(cells_total_size >= 0); + named_stats.apply_diff(diff.named_stats); + } + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const Stats &stats) { + sb << "STATS\n"; + for (auto &p : stats.custom_stats) { + sb << "\t" << p.first << "\t" << p.second << "\n"; + } + for (auto &p : stats.named_stats.stats_int) { + sb << "\t" << p.first << "\t" << p.second << "\n"; + } + for (auto &p : stats.named_stats.stats_str) { + sb << "\t" << p.first << "\t" << p.second << "\n"; + } + return sb; } }; virtual void inc(const Ref &old_root) = 0; @@ -72,7 +97,7 @@ class DynamicBagOfCellsDb { virtual td::Status prepare_commit() = 0; virtual Stats get_stats_diff() = 0; virtual td::Result get_stats() { - return td::Status::Error("Not implemented"); + return Stats{}; } virtual td::Status commit(CellStorer &) = 0; virtual std::shared_ptr get_cell_db_reader() = 0; @@ -83,25 +108,49 @@ class DynamicBagOfCellsDb { virtual void set_celldb_compress_depth(td::uint32 value) = 0; virtual vm::ExtCellCreator &as_ext_cell_creator() = 0; - static std::unique_ptr create(); + class AsyncExecutor { + public: + virtual ~AsyncExecutor() { + } + virtual void execute_async(std::function f) = 0; + virtual void execute_sync(std::function f) = 0; + virtual std::string describe() const { + return "AsyncExecutor"; + } + }; + + struct CreateV1Options { + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const CreateV1Options &options) { + return sb << "V1{}"; + } + }; + static std::unique_ptr create(CreateV1Options = {}); + + struct CreateV2Options { + size_t extra_threads{std::thread::hardware_concurrency()}; + std::shared_ptr executor{}; + size_t cache_ttl_max{2000}; + size_t cache_size_max{1000000}; + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const CreateV2Options &options) { + return sb << "V2{extra_threads=" << options.extra_threads << ", cache_ttl_max=" << options.cache_ttl_max + << ", cache_size_max=" << options.cache_size_max << "}"; + } + }; + static std::unique_ptr create_v2(CreateV2Options options); struct CreateInMemoryOptions { size_t extra_threads{std::thread::hardware_concurrency()}; bool verbose{true}; - // Allocated DataCels will never be deleted + // Allocated DataCells will never be deleted bool use_arena{false}; // Almost no overhead in memory during creation, but will scan database twice bool use_less_memory_during_creation{true}; - }; - static std::unique_ptr create_in_memory(td::KeyValueReader *kv, CreateInMemoryOptions options); - - class AsyncExecutor { - public: - virtual ~AsyncExecutor() { + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const CreateInMemoryOptions &options) { + return sb << "InMemory{extra_threads=" << options.extra_threads << ", use_arena=" << options.use_arena + << ", use_less_memory_during_creation=" << options.use_less_memory_during_creation << "}"; } - virtual void execute_async(std::function f) = 0; - virtual void execute_sync(std::function f) = 0; }; + static std::unique_ptr create_in_memory(td::KeyValueReader *kv, CreateInMemoryOptions options); virtual void load_cell_async(td::Slice hash, std::shared_ptr executor, td::Promise> promise) = 0; diff --git a/crypto/vm/db/DynamicBagOfCellsDbV2.cpp b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp new file mode 100644 index 000000000..45fdffee8 --- /dev/null +++ b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp @@ -0,0 +1,1505 @@ +#include "vm/db/DynamicBagOfCellsDb.h" +#include "vm/db/CellStorage.h" +#include "vm/db/CellHashTable.h" + +#include "vm/cells/ExtCell.h" + +#include "td/utils/base64.h" +#include "td/utils/format.h" +#include "td/utils/ThreadSafeCounter.h" +#include "td/utils/misc.h" +#include "validator/validator.h" + +#include "vm/cellslice.h" + +#include + +namespace vm { +namespace { + +// Very stupid Vector/MpmcQueue +template +struct TsVector { + TsVector() { + first_block_size_ = 64; + blocks_[0].data.resize(first_block_size_); + blocks_[0].is_ready = true; + } + TsVector(std::vector base) { + first_block_size_ = base.size(); + blocks_[0].data = std::move(base); + blocks_[0].is_ready = true; + } + struct Block { + std::mutex mutex; + std::atomic is_ready{false}; + std::vector data; + }; + T &at(size_t i) { + td::uint64 j = i / first_block_size_; + td::int32 hb = 63 - td::count_leading_zeroes64(j); // hb = -1 if j=0, else hb>=0 + + // If j=0, hb<0, so hb>>31 = -1 => mask=0 + // If j>0, hb>=0, so hb>>31=0 => mask=~0 (all ones) + td::uint64 mask = ~(td::uint64)(hb >> 31); + + size_t block_i = hb + 1; + uint64_t shift = hb & 63ULL; + uint64_t start = ((1ULL << shift) * first_block_size_) & mask; + size_t pos_in_block = i - start; + auto &block = blocks_[block_i]; + if (block.is_ready.load(std::memory_order_acquire)) { + return block.data.at(pos_in_block); + } + + std::unique_lock lock(block.mutex); + if (block.is_ready.load(std::memory_order_acquire)) { + return block.data.at(pos_in_block); + } + block.resize(start); + block.is_ready.store(true, std::memory_order_release); + return block.data.at(pos_in_block); + } + template + void push_back(S &&value) { + at(end_.fetch_add(1, std::memory_order_relaxed)) = std::forward(value); + } + T pop_front() { + auto pos = begin_.fetch_add(1, std::memory_order_relaxed); + while (pos >= end_.load(std::memory_order_acquire)) { + // This may (or may not) use too much CPU + td::this_thread::yield(); + } + return std::move(at(pos)); + } + size_t size() const { + return end_.load(); + } + + std::array blocks_; + size_t first_block_size_{0}; + std::atomic begin_{0}; + std::atomic end_{0}; +}; +struct CellInfo; + +class CellDbReaderExt; +struct DynamicBocExtCellExtra { + std::shared_ptr reader; +}; + +class DynamicBocCellLoader { + public: + static td::Result> load_data_cell(const ExtCell &cell, + const DynamicBocExtCellExtra &extra); +}; +using DynamicBocExtCell = ExtCell; + +class CellDbReaderExt : public CellDbReader { + public: + virtual td::Result> load_ext_cell(Ref cell) = 0; +}; + +td::Result> DynamicBocCellLoader::load_data_cell(const DynamicBocExtCell &cell, + const DynamicBocExtCellExtra &extra) { + return extra.reader->load_ext_cell(Ref(&cell)); +} + +#define S(x) \ + td::NamedThreadSafeCounter::CounterRef x { \ + nc.get_counter(#x) \ + } + +struct CacheStats { + td::NamedThreadSafeCounter nc; + S(load_cell_ext); + S(load_cell_ext_cache_hits); + S(load_cell_sync); + S(load_cell_sync_cache_hits); + S(load_cell_async); + S(load_cell_async_cache_hits); + S(ext_cells); + S(ext_cells_load); + S(ext_cells_load_cache_hits); + + S(kv_read_found); + S(kv_read_not_found); + + S(sync_with_db); + S(sync_with_db_only_ref); + S(load_cell_no_cache); +}; + +struct CommitStats { + td::NamedThreadSafeCounter nc; + + S(to_inc); + S(to_dec); + + S(gather_new_cells_calls); + S(gather_new_cells_calls_it); + S(update_parents_calls); + S(update_parents_calls_it); + S(dec_calls); + S(dec_calls_it); + + S(new_cells); + S(new_cells_leaves); + + S(new_cells_loaded_not_in_db); + S(new_cells_loaded_in_db); + S(new_cells_not_in_db_fast); + + S(dec_loaded); + S(dec_to_zero); + + S(changes_loaded); + + // new diff logic + S(diff_zero); + S(diff_full); + S(diff_erase); + S(diff_ref_cnt); + + // old full data logic + S(inc_save); + S(inc_save_full); + S(inc_save_only_ref_cnt); + S(inc_new_cell); + S(inc_just_ref_cnt); + + S(dec_save); + S(dec_save_full); + S(dec_save_only_refcnt); + S(dec_save_erase); + S(dec_erase_cell); + S(dec_just_ref_cnt); +}; + +template +struct AtomicPod { + T load() const { + while (true) { + if (auto res = try_read_stable()) { + return res->second; + } + } + } + + template + std::pair update(F &&f) { + while (true) { + auto res = try_read_stable(); + if (!res) { + continue; + } + auto [before, old_data] = *res; + + auto o_new_data = f(old_data); + if (!o_new_data) { + return {old_data, false}; + } + + if (!lock_.compare_exchange_weak(before, before + 1, std::memory_order_acq_rel, std::memory_order_relaxed)) { + continue; + } + + data_ = *o_new_data; // relaxed store inside lock + lock_.fetch_add(1, std::memory_order_release); + return {*o_new_data, true}; + } + } + + private: + mutable std::atomic lock_{0}; + T data_{}; + + std::optional> try_read_stable() const { + auto before = lock_.load(std::memory_order_acquire); + if (before % 2 == 1) { + return std::nullopt; + } + T temp = data_; // relaxed read is ok, checked by versioning + auto after = lock_.load(std::memory_order_acquire); + if (after != before) { + return std::nullopt; + } + return std::make_pair(before, temp); + } +}; + +struct InDbInfo { + std::vector parents; + std::atomic pending_children{0}; + std::atomic maybe_in_db{true}; + std::atomic visited_in_gather_new_cells{false}; +}; +td::StringBuilder &operator<<(td::StringBuilder &sb, const InDbInfo &info) { + sb << "mb_in_db:" << info.maybe_in_db.load() << " chld_n:" << info.pending_children + << " prnt_n:" << info.parents.size(); + return sb; +} +struct CellInfo { + struct State { + // db_ref_cnt and in_db are correct + bool sync_with_db{false}; + + // ignore if sync_with_db is false + td::int32 db_ref_cnt{0}; + td::int32 db_refcnt_fixup{0}; + + // if true - cell is definitely in db + // if false - we know that cell is not in db only is sync_with_db=true + bool in_db{false}; + + // diff to be applied + }; + AtomicPod state; + std::atomic ref_cnt_diff{0}; + + std::atomic visited{false}; + td::unique_ptr in_db_info_ptr; + std::mutex mutex; + + // Could be AtomicRef, but is am not sure that it is worth it + const Ref cell; + + explicit CellInfo(Ref cell) : cell(std::move(cell)) { + } + + InDbInfo &in_db_info() { + return *in_db_info_ptr; + } + const InDbInfo &in_db_info() const { + return *in_db_info_ptr; + } + InDbInfo &in_db_info_create() { // NOT thread safe + if (!in_db_info_ptr) { + in_db_info_ptr = td::make_unique(); + } + return in_db_info(); + } + InDbInfo &in_db_info_create(CellInfo *parent) { // Thread Safe + std::unique_lock lock(mutex); + if (!in_db_info_ptr) { + in_db_info_ptr = td::make_unique(); + } + auto &res = *in_db_info_ptr; + if (parent != nullptr) { + res.parents.emplace_back(parent); + } + lock.unlock(); + return res; + } + void in_db_info_destroy() { + in_db_info_ptr = nullptr; + } + td::int32 inc_ref_cnt() { + return ref_cnt_diff.fetch_add(1, std::memory_order_relaxed) + 1; + } + td::int32 dec_ref_cnt() { + return ref_cnt_diff.fetch_sub(1, std::memory_order_relaxed) - 1; + } + td::int32 get_ref_cnt_diff() const { + return ref_cnt_diff.load(std::memory_order_relaxed); + } + + void set_not_in_db() { + state.update([&](State state) -> std::optional { + if (state.sync_with_db) { + CHECK(state.db_ref_cnt == 0); + CHECK(!state.in_db); + return {}; + } + state.sync_with_db = true; + state.in_db = false; + state.db_ref_cnt = 0; + return state; + }); + } + void set_in_db() { + state.update([&](State state) -> std::optional { + if (state.sync_with_db) { + //LOG_CHECK(state.in_db) << *this; + return {}; + } + state.in_db = true; + return state; + }); + } + void synced_with_db(td::int32 db_ref_cnt) { + state.update([&](State state) -> std::optional { + if (state.sync_with_db) { + CHECK(state.in_db); + CHECK(state.db_ref_cnt == db_ref_cnt); + return {}; + } + state.in_db = true; + state.db_ref_cnt = db_ref_cnt; + return state; + }); + } + bool visit() { + return !visited.exchange(true); + } + void on_written_to_db() { + auto diff = ref_cnt_diff.exchange(0); + state.update([&](State state) -> std::optional { + if (diff == 0) { + return {}; + } + if (state.sync_with_db) { + state.db_ref_cnt += diff; + CHECK(state.db_ref_cnt >= 0); + state.in_db = state.db_ref_cnt > 0; + } else { + CHECK(diff > 0); + state.in_db = true; + state.db_refcnt_fixup += diff; + } + return state; + }); + } + + td::Result> get_data_cell() { + TRY_RESULT(loaded_cell, cell->load_cell()); + return loaded_cell.data_cell; + } + Cell::Hash key() const { + return cell->get_hash(); + } + bool operator<(const CellInfo &other) const { + return key() < other.key(); + } + + struct Eq { + using is_transparent = void; // Pred to use + bool operator()(const CellInfo &info, const CellInfo &other_info) const { + return info.key() == other_info.key(); + } + bool operator()(const CellInfo &info, td::Slice hash) const { + return info.key().as_slice() == hash; + } + bool operator()(td::Slice hash, const CellInfo &info) const { + return info.key().as_slice() == hash; + } + }; + struct Hash { + using is_transparent = void; // Pred to use + using transparent_key_equal = Eq; + size_t operator()(td::Slice hash) const { + return cell_hash_slice_hash(hash); + } + size_t operator()(const CellInfo &info) const { + return cell_hash_slice_hash(info.key().as_slice()); + } + }; +}; +td::StringBuilder &operator<<(td::StringBuilder &sb, const CellInfo &info) { + if (info.cell->is_loaded()) { + auto data_cell = info.cell->load_cell().move_as_ok().data_cell; + vm::CellSlice cs(vm::NoVm{}, data_cell); + sb << data_cell->get_hash().to_hex().substr(0, 8) << " refs:" << data_cell->size_refs() + << " data:" << cs.data_bits().to_hex(cs.size()) << " data_ptr=" << data_cell.get() << " data_ref_cnt(" + << data_cell->get_refcnt() << ")"; + } else { + sb << info.cell->get_hash().to_hex().substr(0, 8); + } + auto state = info.state.load(); + sb << " " << &info; + sb << "\n\tin_db=" << state.in_db << " sync_with_db=" << state.sync_with_db + << " ref_cnt_diff=" << info.get_ref_cnt_diff() << " db_ref_cnt=" << state.db_ref_cnt + << " db_ref_cnt_fixup=" << state.db_refcnt_fixup; + if (state.sync_with_db) { + sb << " REFS(" << info.get_ref_cnt_diff() + state.db_ref_cnt << ")"; + } + if (info.in_db_info_ptr) { + sb << " " << info.in_db_info(); + } + sb << " visited=" << info.visited.load(); + return sb; +} + +struct ExecutorOptions { + size_t extra_threads_n{0}; + std::shared_ptr async_executor; +}; +template +class ExecutorImpl { + public: + ExecutorImpl(ExecutorOptions options) : options_(options) { + } + ExecutorOptions options_; + using InputData = std::vector>; + using OutputData = std::vector>; + struct InputChunk { + td::Span infos; + size_t begin{}; + size_t end{}; + }; + + template + OutputData process(const InputData &data, const F &process_task_f) { + if (options_.extra_threads_n > 0) { + return process_parallel(data, process_task_f); + } else { + return process_sequential(data, process_task_f); + } + } + template + struct SingleThreadWorker { + const F &process_task_f; + mutable std::vector results{}; + void add_task(InputT input) const { + process_task_f(input, *this); + } + void add_result(OutputT output) const { + results.push_back(output); + } + }; + template + OutputData process_sequential(const InputData &data, const F &process_task_f) { + auto w = SingleThreadWorker{process_task_f}; + for (auto &chunk : data) { + for (auto &info : chunk) { + process_task_f(info, w); + } + } + + return {std::move(w.results)}; + } + + template + struct Shared; + + template + struct Worker { + size_t worker_i{}; + std::shared_ptr> shared; + + void add_task(InputT input) const { + shared->delay_or_process_task(input, *this); + } + void add_result(OutputT value) const { + shared->add_result(value, worker_i); + } + void loop() const { + shared->loop(*this); + } + }; + + template + struct Shared { + Shared(size_t workers_n, const InputData &input_data, const ProcessTaskF &process_task_f) + : input_chunks(prepare_input_chunks(input_data)) + , workers_n(workers_n) + , input_size(input_chunks.empty() ? 0 : input_chunks.back().end) + , batch_size(std::clamp(input_size / workers_n / 4, size_t(1), size_t(128))) + , process_task_f(process_task_f) { + } + + const std::vector input_chunks; + + const size_t workers_n{0}; + const size_t input_size{0}; + const size_t batch_size{128}; + + const ProcessTaskF &process_task_f; + + // Position in input + std::atomic next_input_i{0}; + + // Shared queue + // Probably a simpler queue would also work fine + td::MpmcQueue mpmc_queue{workers_n}; + using Waiter = td::MpmcSleepyWaiter; + Waiter waiter; + std::atomic mpmc_queue_size{workers_n}; // guard + + // Output vectors + struct ThreadData { + std::vector output; + char pad[TD_CONCURRENCY_PAD - sizeof(output)]; + }; + std::vector thread_data{workers_n}; + + auto prepare_input_chunks(const InputData &input_data) { + std::vector chunks; + for (auto &chunk : input_data) { + size_t prev_end = chunks.empty() ? 0 : chunks.back().end; + chunks.push_back({.infos = td::as_span(chunk), .begin = prev_end, .end = prev_end + chunk.size()}); + } + return chunks; + } + + void delay_or_process_task(InputT input, const Worker &worker) { + // if there is enough tasks in queue, we continue recursion + if (mpmc_queue_size.load(std::memory_order_acquire) > 256) { + process_task_f(input, worker); + } else { + mpmc_queue_size.fetch_add(1, std::memory_order_acq_rel); + mpmc_queue.push(input, worker.worker_i); + waiter.notify(); + } + } + + void add_result(OutputT result, size_t worker_i) { + thread_data[worker_i].output.push_back(std::move(result)); + } + + void process_initial_input(const Worker &worker) { + size_t input_chunk_i = 0; + while (true) { + auto begin_i = next_input_i.fetch_add(batch_size, std::memory_order_relaxed); + auto end_i = begin_i + batch_size; + if (begin_i >= input_size) { + break; + } + for (size_t i = begin_i; i < end_i && i < input_size; i++) { + while (input_chunks[input_chunk_i].end <= i) { + input_chunk_i++; + } + auto offset = i - input_chunks[input_chunk_i].begin; + auto task = input_chunks[input_chunk_i].infos[offset]; + process_task_f(task, worker); + } + } + } + + void on_processed_task_from_queue(size_t worker_i) { + if (mpmc_queue_size.fetch_add(-1, std::memory_order_acq_rel) == 1) { + for (size_t i = 0; i < workers_n; i++) { + mpmc_queue.push(nullptr, worker_i); + waiter.notify(); + } + } + } + + void process_queue(const Worker &worker) { + on_processed_task_from_queue(worker.worker_i); + + Waiter::Slot slot; + waiter.init_slot(slot, td::narrow_cast(worker.worker_i)); + + while (true) { + InputT input{}; + if (mpmc_queue.try_pop(input, worker.worker_i)) { + waiter.stop_wait(slot); + if (!input) { + break; + } + process_task_f(input, worker); + on_processed_task_from_queue(worker.worker_i); + } else { + waiter.wait(slot); + } + } + } + void loop(const Worker &worker) { + process_initial_input(worker); + process_queue(worker); + } + void finish() const { + CHECK(mpmc_queue_size == 0); + } + }; + + template + OutputData process_parallel(const InputData &input_data, const F &process_task_f) { + const size_t workers_n = options_.extra_threads_n + 1; + auto shared = std::make_shared>(workers_n, input_data, process_task_f); + + CHECK(workers_n >= 1); + for (size_t i = 0; i < workers_n; i++) { + auto to_run = [worker = Worker{.worker_i = i, .shared = shared}] { worker.loop(); }; + + if (i + 1 == workers_n) { + to_run(); + } else if (options_.async_executor) { + options_.async_executor->execute_async(std::move(to_run)); + } else { + // NB: td::thread, NOT std::thread + td::thread(std::move(to_run)).detach(); + } + } + shared->finish(); + return td::transform(shared->thread_data, [](auto &&x) { return std::move(x.output); }); + } +}; +struct Executor { + Executor(ExecutorOptions options = {}) : options_(options) { + } + + template + auto operator()(const std::vector> &data, const F &process_task_f) { + return ExecutorImpl(options_).process(data, process_task_f); + } + + private: + ExecutorOptions options_; +}; + +// Thread safe storage for CellInfo +// Will be used by everybody as shared cache. Yes there is some overhead, but it don't want to create other hash table +struct CellInfoStorage { + public: + // All methods are thead safe + // All CellInfo pointers lives as long as CellInfoStorage + + // returns CellInfo, only if it is already exists + CellInfo *get_cell_info(td::Slice hash) { + return lock(hash)->hash_table.get_if_exists(hash); + } + + CellInfo &create_cell_info_from_db(Ref data_cell, td::int32 ref_cnt) { + auto &info = create_cell_info_from_data_cell(std::move(data_cell)); + info.synced_with_db(ref_cnt); + return info; + } + + // Creates CellInfo from data_cell, or updates existing CellInfo if is not yet loaded + CellInfo &create_cell_info_from_data_cell(Ref cell) { + CHECK(cell.not_null()); + CHECK(cell->is_loaded()); + + auto hash = cell->get_hash(); + auto [info, created] = lock(hash.as_slice())->hash_table.emplace(hash.as_slice(), std::move(cell)); + + if (!created) { + info.cell->set_data_cell(std::move(cell)); + } + return info; + } + + // Creates CellInfo from cell. If cell is loaded, it will be used to rewrite or udpate current cell + CellInfo &create_cell_info(Ref cell, CellDbReaderExt *from_reader, CacheStats &stats) { + if (cell->is_loaded()) { + return create_cell_info_from_data_cell(cell->load_cell().move_as_ok().data_cell); + } + + bool our_ext_cell = false; + auto ext_cell = dynamic_cast(cell.get()); + if (ext_cell) { + auto prunned_cell = ext_cell->get_prunned_cell(); + if (prunned_cell.not_null()) { + our_ext_cell = prunned_cell->get_extra().reader.get() == from_reader; + } + our_ext_cell = true; + } else if (!cell->is_loaded()) { + // if we cached cell from OTHER db is good idea to drop it ASAP + force_drop_cache_.store(true, std::memory_order_relaxed); + } + + auto hash = cell->get_hash(); + auto [info, created] = lock(hash.as_slice())->hash_table.emplace(hash.as_slice(), std::move(cell)); + if (our_ext_cell) { + stats.ext_cells_load.inc(); + if (info.cell->is_loaded()) { + stats.ext_cells_load_cache_hits.inc(); + } + info.set_in_db(); + } + return info; + } + + void dump() { + LOG(ERROR) << "===========BEGIN DUMP==========="; + for (auto &bucket : buckets_) { + std::lock_guard guard(bucket.mutex); + bucket.hash_table.for_each([&](auto &info) { LOG(INFO) << info; }); + } + LOG(ERROR) << "===========END DUMP==========="; + } + + size_t cache_size() { + size_t res = 0; + for (auto &bucket : buckets_) { + std::lock_guard guard(bucket.mutex); + res += bucket.hash_table.size(); + } + return res; + } + bool force_drop_cache() { + return force_drop_cache_.load(std::memory_order_relaxed); + } + + private: + struct Bucket { + std::mutex mutex; + CellHashTable hash_table; + }; + constexpr static size_t buckets_n = 8192; + std::array bucket_; + + struct Unlock { + void operator()(Bucket *bucket) const { + bucket->mutex.unlock(); + } + }; + std::array buckets_{}; + std::atomic force_drop_cache_{false}; + + std::unique_ptr lock(Bucket &bucket) { + bucket.mutex.lock(); + return std::unique_ptr(&bucket); + } + std::unique_ptr lock(td::Slice key) { + auto hash = td::as(key.substr(16, 8).ubegin()); + auto bucket_i = hash % buckets_n; + return lock(buckets_[bucket_i]); + } +}; + +class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { + public: + explicit DynamicBagOfCellsDbImplV2(CreateV2Options options) : options_(options) { + get_thread_safe_counter().inc(); + // LOG(ERROR) << "Constructor called for DynamicBagOfCellsDbImplV2"; + } + ~DynamicBagOfCellsDbImplV2() { + // LOG(ERROR) << "Destructor called for DynamicBagOfCellsDbImplV2"; + get_thread_safe_counter().add(-1); + + if (cell_db_reader_) { + cell_db_reader_->drop_cache(); + } + } + + td::Result>> meta_get_all() const override { + CHECK(meta_db_fixup_.empty()); + std::vector> result; + auto s = cell_db_reader_->key_value_reader().for_each_in_range( + "desc", "desd", [&](const td::Slice &key, const td::Slice &value) { + if (td::begins_with(key, "desc") && key.size() != 32) { + result.emplace_back(key.str(), value.str()); + } + return td::Status::OK(); + }); + TRY_STATUS(std::move(s)); + return result; + } + td::Result meta_get(td::Slice key, std::string &value) override { + auto it = meta_db_fixup_.find(key); + if (it != meta_db_fixup_.end()) { + if (it->second.empty()) { + return KeyValue::GetStatus::NotFound; + } + value = it->second; + return KeyValue::GetStatus::Ok; + } + return cell_db_reader_->key_value_reader().get(key, value); + } + td::Status meta_set(td::Slice key, td::Slice value) override { + meta_diffs_.push_back( + CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Set, .key = key.str(), .value = value.str()}); + return td::Status::OK(); + } + td::Status meta_erase(td::Slice key) override { + meta_diffs_.push_back(CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Erase, .key = key.str()}); + return td::Status::OK(); + } + td::Result> load_cell(td::Slice hash) override { + CHECK(cell_db_reader_); + return cell_db_reader_->load_cell(hash); + } + td::Result> load_root(td::Slice hash) override { + return load_cell(hash); + } + td::Result> load_root_thread_safe(td::Slice hash) const override { + // TODO: it is better to use AtomicRef, or atomic shared pointer + // But to use AtomicRef we need a little refactoring + // And std::atomic> is still unsupported by clang + std::unique_lock lock(atomic_cell_db_reader_mutex_); + auto reader = atomic_cell_db_reader_; + lock.unlock(); + if (!reader) { + return td::Status::Error("Empty reader"); + } + return reader->load_cell(hash); + } + void load_cell_async(td::Slice hash, std::shared_ptr executor, + td::Promise> promise) override { + CHECK(cell_db_reader_); + return cell_db_reader_->load_cell_async(hash, std::move(executor), std::move(promise)); + } + void prepare_commit_async(std::shared_ptr executor, td::Promise promise) override { + auto promise_ptr = std::make_shared>(std::move(promise)); + executor->execute_async([this, promise_ptr = std::move(promise_ptr)] { + prepare_commit(); + promise_ptr->set_value(td::Unit()); + }); + } + + void inc(const Ref &cell) override { + if (cell.is_null()) { + return; + } + if (cell->get_virtualization() != 0) { + return; + } + to_inc_.push_back(cell); + } + void dec(const Ref &cell) override { + if (cell.is_null()) { + return; + } + if (cell->get_virtualization() != 0) { + return; + } + to_dec_.push_back(cell); + } + + bool is_prepared_for_commit() { + return to_inc_.empty() && to_dec_.empty(); + } + + Stats get_stats_diff() override { + return {}; + } + + td::Status prepare_commit() override { + if (is_prepared_for_commit()) { + return td::Status::OK(); + } + // NB: we don't use options.executor, because it is prone to deadlocks. We need extra_threads_n threads + // available for blocking + Executor executor{{.extra_threads_n = options_.extra_threads, .async_executor = {}}}; + // calculate in_db for all vertices reachable from to_inc_ roots + // - for ext cells we already know they are in db + // - calculate in_db up from leaves + // - if at least one child is not in db, then the cell is definitely not in db + // - so in best case only leaves will be loaded from db + // - this is optional step. All other logic must work in any case + // - only already loaded cells are loaded from db + + stats_.to_inc.add(to_inc_.size()); + stats_.to_dec.add(to_dec_.size()); + + std::vector> visited_cells; + auto add_visited_cells = [&](std::vector> new_visited_cells) { + for (auto &x : new_visited_cells) { + visited_cells.push_back(std::move(x)); + } + }; + + std::vector> new_cells_leaves; + { + td::PerfWarningTimer timer("celldb_v2: gather_new_cells"); + std::vector prepared_to_inc; + std::vector visited_roots; + for (auto &cell : to_inc_) { + auto &info = cell_db_reader_->cell_info(cell); + if (info.inc_ref_cnt() == 1 && info.visit()) { + visited_roots.push_back(&info); + } + if (info.state.load().in_db) { + continue; + } + auto &in_db_info = info.in_db_info_create(nullptr); + if (!in_db_info.visited_in_gather_new_cells.exchange(true)) { + prepared_to_inc.push_back(&info); + } + } + new_cells_leaves = + executor({std::move(prepared_to_inc)}, [&](CellInfo *info, auto &worker) { gather_new_cells(info, worker); }); + visited_cells.push_back(std::move(visited_roots)); + } + + // LOG(WARNING) << "new_cells_leaves: " << new_cells_leaves.size(); + { + td::PerfWarningTimer timer("celldb_v2: update_parents"); + add_visited_cells( + executor({std::move(new_cells_leaves)}, [&](CellInfo *info, auto &worker) { update_parents(info, worker); })); + } + { + td::PerfWarningTimer timer("dec"); + std::vector prepared_to_dec; + for (auto &cell : to_dec_) { + auto &info = cell_db_reader_->cell_info(cell); + prepared_to_dec.push_back(&info); + } + add_visited_cells( + executor({std::move(prepared_to_dec)}, [&](CellInfo *info, auto &worker) { dec_cell(info, worker); })); + } + + td::PerfWarningTimer timer_serialize("celldb_v2: save_diff_serialize", 0.01); + // LOG(INFO) << "threads_n = " << options_.extra_threads + 1; + diff_chunks_ = executor.operator()( + visited_cells, [&](CellInfo *info, auto &worker) { serialize_diff(info, worker); }); + timer_serialize.reset(); + + { + td::PerfWarningTimer timer("celldb_v2: clear"); + to_inc_.clear(); + to_dec_.clear(); + } + + //cell_db_reader_->dump(); + return td::Status::OK(); + } + + td::Status commit(CellStorer &storer) override { + prepare_commit(); + save_diff(storer); + // We DON'T delete entries from cache, so cache actually represents diff with snapshot in reader + // But we don't want took keep old snapshot forever + LOG_IF(ERROR, dbg) << "clear cell_db_reader"; + //cell_db_reader_->dump(); + //TODO: call drop_cache reliably via rtti + + constexpr bool always_drop_cache = false; + if (always_drop_cache) { + td::PerfWarningTimer timer("celldb_v2: reset reader"); + cell_db_reader_->drop_cache(); + cache_stats_.apply_diff(cell_db_reader_->get_stats()); + cache_stats_.stats_int["commits"] += 1; + cell_db_reader_ = {}; + // keep atomic reader, to it will be reused + } + return td::Status::OK(); + } + + std::shared_ptr get_cell_db_reader() override { + CHECK(cell_db_reader_); + return cell_db_reader_; + } + + td::Status set_loader(std::unique_ptr loader) override { + if (cell_db_reader_) { + auto cache_size = cell_db_reader_->cache_size(); + bool force_drop_cache = cell_db_reader_->force_drop_cache(); + if (loader && cache_size < options_.cache_size_max && cell_db_reader_ttl_ < options_.cache_ttl_max && + !force_drop_cache) { + // keep cache + cell_db_reader_ttl_++; + return td::Status::OK(); + } + + td::PerfWarningTimer timer(PSTRING() << "celldb_v2: reset reader, TTL=" << cell_db_reader_ttl_ << "/" + << options_.cache_ttl_max << ", cache_size=" << cache_size + << ", force_drop_cache=" << force_drop_cache); + cache_stats_.apply_diff(cell_db_reader_->get_stats()); + cell_db_reader_->drop_cache(); + cell_db_reader_ = {}; + meta_db_fixup_ = {}; + cell_db_reader_ttl_ = 0; + } + + if (loader) { + cell_db_reader_ = std::make_shared(std::move(loader)); + cell_db_reader_ttl_ = 0; + } + + { + std::lock_guard guard(atomic_cell_db_reader_mutex_); + atomic_cell_db_reader_ = cell_db_reader_; + } + return td::Status::OK(); + } + + void set_celldb_compress_depth(td::uint32 value) override { + CHECK(value == 0); + } + + vm::ExtCellCreator &as_ext_cell_creator() override { + CHECK(cell_db_reader_); + return *cell_db_reader_; + } + td::Result get_stats() override { + auto ps = stats_.nc.get_stats().with_prefix("storage_"); + ps.apply_diff(cache_stats_.with_prefix("cache_cum_")); + if (cell_db_reader_) { + ps.apply_diff(cell_db_reader_->get_stats().with_prefix("cache_now_")); + ps.apply_diff(cell_db_reader_->get_stats().with_prefix("cache_cum_")); + } + Stats res; + res.named_stats = std::move(ps); + res.named_stats.stats_int["cache.size"] = cell_db_reader_ ? cell_db_reader_->cache_size() : 0; + res.named_stats.stats_int["cache.size_max"] = options_.cache_size_max; + res.named_stats.stats_int["cache.ttl"] = cell_db_reader_ttl_; + res.named_stats.stats_int["cache.ttl_max"] = options_.cache_ttl_max; + return res; + } + + private: + static td::NamedThreadSafeCounter::CounterRef get_thread_safe_counter() { + static auto res = td::NamedThreadSafeCounter::get_default().get_counter("DynamicBagOfCellsDb"); + return res; + } + + class CellDbReaderImpl : public CellDbReaderExt, + public ExtCellCreator, + public std::enable_shared_from_this { + public: + explicit CellDbReaderImpl(std::unique_ptr cell_loader) : cell_loader_(std::move(cell_loader)) { + } + + size_t cache_size() const { + // NOT thread safe + if (internal_storage_) { + return internal_storage_->cache_size(); + } + return 0; + } + bool force_drop_cache() const { + // NOT thread safe + if (internal_storage_) { + return internal_storage_->force_drop_cache(); + } + return false; + } + void drop_cache() { + // NOT thread safe + internal_storage_.reset(); + } + + td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { + // thread safe function + stats_.ext_cells.inc(); + TRY_RESULT(ext_cell, DynamicBocExtCell::create(PrunnedCellInfo{level_mask, hash, depth}, + DynamicBocExtCellExtra{shared_from_this()})); + + return ext_cell; + } + CellInfo *register_ext_cell_inner(Ref ext_cell, CellInfoStorage &storage) { + auto &info = storage.create_cell_info(std::move(ext_cell), this, stats_); + return &info; + } + + void load_cell_async(td::Slice hash, std::shared_ptr executor, td::Promise> promise) { + // thread safe function + stats_.load_cell_async.inc(); + auto maybe_cell = load_cell_fast_path(hash, false, nullptr); + if (maybe_cell.not_null()) { + stats_.load_cell_async_cache_hits.inc(); + return promise.set_value(std::move(maybe_cell)); + } + auto promise_ptr = std::make_shared>>(std::move(promise)); + + executor->execute_async( + [self = shared_from_this(), promise_ptr = std::move(promise_ptr), hash = CellHash::from_slice(hash)]() { + promise_ptr->set_result(self->load_cell(hash.as_slice())); + }); + } + + td::Result> load_cell(td::Slice hash) override { + // thread safe function + stats_.load_cell_sync.inc(); + bool loaded{false}; + auto maybe_cell = load_cell_fast_path(hash, true, &loaded); + if (maybe_cell.not_null()) { + if (!loaded) { + stats_.load_cell_sync_cache_hits.inc(); + } + return maybe_cell; + } + return load_cell_slow_path(hash); + } + + td::Result> load_ext_cell(Ref ext_cell) override { + // thread safe function. + // Called by external cell + stats_.load_cell_ext.inc(); + auto storage = weak_storage_.lock(); + if (!storage) { + TRY_RESULT(load_result, load_cell_no_cache(ext_cell->get_hash().as_slice())); + return load_result.cell_; + } + // we delayed registering ext cell till this moment + auto cell_info = register_ext_cell_inner(std::move(ext_cell), *storage); + + CHECK(cell_info != nullptr); // currently all ext_cells are registered in cache + if (!cell_info->cell->is_loaded()) { + sync_with_db(*cell_info, true); + CHECK(cell_info->cell->is_loaded()); // critical, better to fail + } else { + stats_.load_cell_ext_cache_hits.inc(); + } + return cell_info->cell->load_cell().move_as_ok().data_cell; + } + + CellInfo &cell_info(Ref cell) { + // thread safe function, but called only by DB + CHECK(internal_storage_) + return internal_storage_->create_cell_info(std::move(cell), this, stats_); + } + + std::pair sync_with_db(CellInfo &info, bool need_data) { + // thread safe function, but called only by DB + auto effective_need_data = need_data; + if (info.cell->is_loaded()) { + effective_need_data = false; + } + return info.state.update([&](CellInfo::State state) -> std::optional { + if (state.sync_with_db) { + return {}; + } + stats_.sync_with_db.inc(); + if (!effective_need_data) { + stats_.sync_with_db_only_ref.inc(); + } + auto load_result = + cell_loader_->load(info.cell->get_hash().as_slice(), effective_need_data, *this).move_as_ok(); + + state.sync_with_db = true; + if (load_result.status == CellLoader::LoadResult::NotFound) { + CHECK(state.in_db == false); + CHECK(state.db_ref_cnt == 0); + stats_.kv_read_not_found.inc(); + return state; + } + stats_.kv_read_found.inc(); + + state.in_db = true; + state.db_ref_cnt = load_result.refcnt() + state.db_refcnt_fixup; + if (load_result.cell().not_null()) { + info.cell->set_data_cell(std::move(load_result.cell())); + } + CHECK(!need_data || info.cell->is_loaded()); + return state; + }); + } + + void dump() { + internal_storage_->dump(); + } + + td::NamedStats get_stats() const { + return stats_.nc.get_stats(); + } + td::KeyValueReader &key_value_reader() { + return cell_loader_->key_value_reader(); + } + + private: + static td::NamedThreadSafeCounter::CounterRef get_thread_safe_counter() { + static auto res = td::NamedThreadSafeCounter::get_default().get_counter("DynamicBagOfCellsDbLoader"); + return res; + } + std::shared_ptr internal_storage_{std::make_shared()}; + std::weak_ptr weak_storage_{internal_storage_}; + std::unique_ptr cell_loader_; + CacheStats stats_; + + Ref load_cell_fast_path(td::Slice hash, bool may_block, bool *loaded) { + auto storage = weak_storage_.lock(); + if (!storage) { + return {}; + } + auto cell_info = storage->get_cell_info(hash); + if (cell_info != nullptr) { + if (!cell_info->cell->is_loaded()) { + if (may_block) { + if (loaded) { + *loaded = true; + } + CHECK(cell_info->state.load().in_db); + sync_with_db(*cell_info, true); + CHECK(cell_info->cell->is_loaded()); + } else { + return {}; + } + } + return cell_info->cell->load_cell().move_as_ok().data_cell; + } + return {}; + } + td::Result load_cell_no_cache(td::Slice hash) { + stats_.load_cell_no_cache.inc(); + TRY_RESULT(load_result, cell_loader_->load(hash, true, *this)); + if (load_result.status == CellLoader::LoadResult::NotFound) { + stats_.kv_read_not_found.inc(); + return td::Status::Error("Cell load failed: not in db"); + } + stats_.kv_read_found.inc(); + return load_result; + } + td::Result> load_cell_slow_path(td::Slice hash) { + TRY_RESULT(load_result, load_cell_no_cache(hash)); + auto storage = weak_storage_.lock(); + if (!storage) { + return load_result.cell_; + } + auto &cell_info = storage->create_cell_info_from_db(std::move(load_result.cell()), load_result.refcnt()); + return cell_info.cell->load_cell().move_as_ok().data_cell; + } + }; + + CreateV2Options options_; + std::vector> to_inc_; + std::vector> to_dec_; + std::vector> diff_chunks_; + std::vector meta_diffs_; + std::map> meta_db_fixup_; + + mutable std::mutex atomic_cell_db_reader_mutex_; + std::shared_ptr atomic_cell_db_reader_; + + std::shared_ptr cell_db_reader_; + size_t cell_db_reader_ttl_{0}; + td::NamedStats cache_stats_; + CommitStats stats_; + bool dbg{false}; + + template + void gather_new_cells(CellInfo *info, WorkerT &worker) { + stats_.gather_new_cells_calls.inc(); + do { + // invariant: info is not in DB; with created in_db_info + // we enter into each root only once + stats_.gather_new_cells_calls_it.inc(); + stats_.new_cells.inc(); + auto &in_db_info = info->in_db_info(); + + CellSlice cs(vm::NoVm{}, info->cell); // ensure cell is loaded + CellInfo *prev_child_info = nullptr; + while (cs.have_refs()) { + auto *child_info = &cell_db_reader_->cell_info(cs.fetch_ref()); + auto child_state = child_info->state.load(); + + if (child_state.in_db) { + LOG_IF(INFO, dbg) << "gather_new_cells: IN DB\n\tchld: " << *child_info; + continue; + } + + auto &child_in_db_info = child_info->in_db_info_create(info); + in_db_info.pending_children.fetch_add(1, std::memory_order_relaxed); + + if (child_in_db_info.visited_in_gather_new_cells.exchange(true)) { + continue; + } + + if (prev_child_info != nullptr) { + worker.add_task(prev_child_info); + } + prev_child_info = child_info; + } + LOG_IF(INFO, dbg) << "gather_new_cells: NOT IN DB\n\t" << *info; + if (in_db_info.pending_children.load(std::memory_order_relaxed) == 0) { + worker.add_result(info); + stats_.new_cells_leaves.inc(); + LOG_IF(WARNING, dbg) << "gather_new_cells: ADD LEAVE\n\t" << *info; + } + info = prev_child_info; + } while (info != nullptr); + } + + template + void update_parents(CellInfo *info, const WorkerT &worker) { + stats_.update_parents_calls.inc(); + size_t it = 0; + do { + stats_.update_parents_calls_it.inc(); + it++; + //LOG(INFO) << "update_parents: it=" << it << "\n\t"; + auto &in_db_info = info->in_db_info(); + bool in_db = false; + if (in_db_info.maybe_in_db.load(std::memory_order_relaxed)) { + auto [state, loaded] = cell_db_reader_->sync_with_db(*info, false); + in_db = state.in_db; + if (in_db) { + stats_.new_cells_loaded_in_db.inc(); + } else { + stats_.new_cells_loaded_not_in_db.inc(); + } + } else { + stats_.new_cells_not_in_db_fast.inc(); + info->set_not_in_db(); + } + LOG_IF(INFO, dbg) << "update_parents: it=" << it << "\n\t" << *info; + + CellInfo *prev_parent{nullptr}; + for (auto &parent : in_db_info.parents) { + auto &parent_in_db_info = parent->in_db_info(); + if (!in_db) { + parent_in_db_info.maybe_in_db.store(false, std::memory_order_relaxed); + } + if (parent_in_db_info.pending_children.fetch_sub(1, std::memory_order_release) == 1) { + if (prev_parent) { + worker.add_task(prev_parent); + } + prev_parent = parent; + } + } + if (!in_db) { + CellSlice cs(vm::NoVm{}, info->cell); + while (cs.have_refs()) { + auto child = cs.fetch_ref(); + auto &child_info = cell_db_reader_->cell_info(std::move(child)); + if (child_info.inc_ref_cnt() == 1 && child_info.visit()) { + worker.add_result(&child_info); + } + } + } + info->in_db_info_destroy(); + info = prev_parent; + } while (info); + } + + template + void dec_cell(CellInfo *info, WorkerT &worker) { + stats_.dec_calls.inc(); + + while (true) { + stats_.dec_calls_it.inc(); + if (info->visit()) { + worker.add_result(info); + } + auto ref_cnt_diff = info->dec_ref_cnt(); + if (ref_cnt_diff > 0) { + LOG_IF(INFO, dbg) << "NOT DEC" + << "\n\t" << info; + break; + } + auto state = info->state.load(); + if (ref_cnt_diff == 0 && state.in_db) { + LOG_IF(INFO, dbg) << "NOT DEC (in_db) " + << "\n\t" << info; + break; + } + if (!state.sync_with_db) { + state = cell_db_reader_->sync_with_db(*info, true).first; + stats_.dec_loaded.inc(); + CHECK(ref_cnt_diff == 0 || state.in_db); + } + auto ref_cnt = state.db_ref_cnt + ref_cnt_diff; + if (ref_cnt > 0) { + LOG_IF(INFO, dbg) << "DEC " << ref_cnt << "\n\t" << info; + } else { + LOG_IF(ERROR, dbg) << "DEC " << ref_cnt << "\n\t" << info; + } + CHECK(ref_cnt >= 0); + if (ref_cnt > 0) { + break; + } + stats_.dec_to_zero.inc(); + CellSlice cs(vm::NoVm{}, info->cell); + if (!cs.have_refs()) { + break; + } + while (cs.size_refs() > 1) { + worker.add_task(&cell_db_reader_->cell_info(cs.fetch_ref())); + } + info = &cell_db_reader_->cell_info(cs.fetch_ref()); + } + } + + template + void serialize_diff(CellInfo *info, Worker &worker) { + info->visited.store(false, std::memory_order_relaxed); + auto ref_cnt_diff = info->get_ref_cnt_diff(); + if (ref_cnt_diff == 0) { + stats_.diff_zero.inc(); + return; + } + + bool merge_supported = true; + if (merge_supported) { + auto state = info->state.load(); + if (ref_cnt_diff < 0) { + CHECK(state.sync_with_db); + /* + if (state.db_ref_cnt + ref_cnt_diff == 0) { + LOG(ERROR) << "DEC ERASE " << info->cell->get_hash().to_hex(); + } else { + LOG(ERROR) << "DEC MERGE " << info->cell->get_hash().to_hex() << *info; + } + */ + } + if (ref_cnt_diff < 0 && state.sync_with_db && state.db_ref_cnt + ref_cnt_diff == 0) { + // Erase is better than Merge+CompactionFilter + // So I see no reason for CompactionFilter at all + worker.add_result({.type = CellStorer::Diff::Erase, .key = info->cell->get_hash()}); + stats_.diff_erase.inc(); + } else { + bool with_data = ref_cnt_diff > 0 && !state.in_db; + if (with_data) { + CHECK(state.sync_with_db); + auto data_cell = info->cell->load_cell().move_as_ok().data_cell; + stats_.diff_full.inc(); + worker.add_result({.type = CellStorer::Diff::Set, + .key = info->cell->get_hash(), + .value = CellStorer::serialize_value(ref_cnt_diff + state.db_ref_cnt, data_cell, false)}); + } else { + stats_.diff_ref_cnt.inc(); + worker.add_result({.type = CellStorer::Diff::Merge, + .key = info->cell->get_hash(), + .value = CellStorer::serialize_refcnt_diffs(ref_cnt_diff)}); + } + } + info->on_written_to_db(); + return; + } + + auto state = info->state.load(); + if (!state.sync_with_db) { + stats_.changes_loaded.inc(); + state = cell_db_reader_->sync_with_db(*info, true).first; + } + CHECK(state.sync_with_db); + auto new_ref_cnt = ref_cnt_diff + state.db_ref_cnt; + + if (ref_cnt_diff < 0) { + stats_.dec_save.inc(); + if (new_ref_cnt == 0) { + stats_.dec_erase_cell.inc(); + + LOG_IF(ERROR, dbg) << "DEC ERASE " << *info; + worker.add_result({.type = CellStorer::Diff::Erase, .key = info->cell->get_hash()}); + stats_.dec_save_erase.inc(); + } else { + stats_.dec_just_ref_cnt.inc(); + + LOG_IF(ERROR, dbg) << "DEC REFCNT " << *info; + CHECK(info->cell->is_loaded()); + worker.add_result( + {.type = CellStorer::Diff::Set, + .key = info->cell->get_hash(), + .value = CellStorer::serialize_value(new_ref_cnt, info->cell->load_cell().move_as_ok().data_cell, false)}); + stats_.dec_save_full.inc(); + } + } else { + stats_.inc_save.inc(); + CHECK(info->cell->is_loaded()); + if (state.db_ref_cnt == 0) { + stats_.inc_new_cell.inc(); + LOG_IF(ERROR, dbg) << "INC CREATE " << *info; + } else { + stats_.inc_just_ref_cnt.inc(); + LOG_IF(ERROR, dbg) << "INC REFCNT " << *info; + } + + worker.add_result( + {.type = CellStorer::Diff::Set, + .key = info->cell->get_hash(), + .value = CellStorer::serialize_value(new_ref_cnt, info->cell->load_cell().move_as_ok().data_cell, false)}); + stats_.inc_save_full.inc(); + } + } + + void save_diff(CellStorer &storer) { + td::PerfWarningTimer timer("celldb_v2: save_diff"); + td::PerfWarningTimer timer_store_to_db("celldb_v2: save_diff_store_to_db", 0.01); + // Have no idea hot to parallelize this in case of rocksdb + for (auto &diffs : diff_chunks_) { + for (auto &diff : diffs) { + storer.apply_diff(diff).ensure(); + } + } + for (auto &meta_diff : meta_diffs_) { + meta_db_fixup_[meta_diff.key] = meta_diff.value; + storer.apply_meta_diff(meta_diff).ensure(); + } + timer_store_to_db.reset(); + td::PerfWarningTimer timer_clear("celldb_v2: save_diff_clear"); + diff_chunks_.clear(); + meta_diffs_.clear(); + timer_clear.reset(); + } +}; +} // namespace + +std::unique_ptr DynamicBagOfCellsDb::create_v2(CreateV2Options options) { + return std::make_unique(options); +} +} // namespace vm diff --git a/crypto/vm/db/InMemoryBagOfCellsDb.cpp b/crypto/vm/db/InMemoryBagOfCellsDb.cpp index 03cad0934..b0a30cfb2 100644 --- a/crypto/vm/db/InMemoryBagOfCellsDb.cpp +++ b/crypto/vm/db/InMemoryBagOfCellsDb.cpp @@ -413,6 +413,7 @@ class CellStorage { size_t dense_ht_size = 0; size_t new_ht_size = 0; for_each_bucket(0, [&](auto bucket_id, CellBucket &bucket) { + // TODO: this leads to CE when use_dense_hash_map == false dense_ht_capacity += bucket.infos_.dense_ht_values_.size(); dense_ht_size += bucket.infos_.dense_ht_size_; new_ht_capacity += bucket.infos_.new_ht_.bucket_count(); @@ -468,6 +469,14 @@ class CellStorage { } return td::Status::Error("not found"); } + td::Result>> load_known_roots_local() const { + auto lock = local_access_.lock(); + std::vector> result; + for (auto &root : roots_) { + result.emplace_back(root); + } + return result; + } td::Result> load_root_shared(const CellHash &hash) const { std::lock_guard lock(root_mutex_); if (auto it = roots_.find(hash); it != roots_.end()) { @@ -620,7 +629,7 @@ class CellStorage { sb << "\n\t" << key << "=" << value; } LOG_IF(ERROR, desc_count != 0 && desc_count != stats.roots_total_count + 1) - << "desc<> keys count is " << desc_count << " wich is different from roots count " << stats.roots_total_count; + << "desc<> keys count is " << desc_count << " which is different from roots count " << stats.roots_total_count; LOG_IF(WARNING, verbose) << P << "done in " << full_timer.elapsed() << "\n\troots_count=" << stats.roots_total_count << "\n\t" << desc_count << "\n\tcells_count=" << stats.cells_total_count @@ -757,15 +766,77 @@ class CellStorage { } }; +class MetaStorage { + public: + explicit MetaStorage(std::vector> values) + : meta_(std::move_iterator(values.begin()), std::move_iterator(values.end())) { + for (auto &p : meta_) { + CHECK(p.first.size() != CellTraits::hash_bytes); + } + } + std::vector> meta_get_all() const { + return td::transform(meta_, [](const auto &p) { return std::make_pair(p.first, p.second); }); + } + KeyValue::GetStatus meta_get(td::Slice key, std::string &value) const { + auto lock = local_access_.lock(); + auto it = meta_.find(key.str()); + if (it == meta_.end()) { + return KeyValue::GetStatus::NotFound; + } + value = it->second; + return KeyValue::GetStatus::Ok; + } + void meta_set(td::Slice key, td::Slice value) { + auto lock = local_access_.lock(); + meta_[key.str()] = value.str(); + meta_diffs_.push_back( + CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Set, .key = key.str(), .value = value.str()}); + } + void meta_erase(td::Slice key) { + auto lock = local_access_.lock(); + meta_.erase(key.str()); + meta_diffs_.push_back(CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Erase, .key = key.str()}); + } + std::vector extract_diffs() { + auto lock = local_access_.lock(); + return std::move(meta_diffs_); + } + + private: + mutable UniqueAccess local_access_; + std::unordered_map meta_; + std::vector meta_diffs_; +}; + class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { public: - explicit InMemoryBagOfCellsDb(td::unique_ptr storage) : storage_(std::move(storage)) { + explicit InMemoryBagOfCellsDb(td::unique_ptr storage, td::unique_ptr meta_storage) + : storage_(std::move(storage)), meta_storage_(std::move(meta_storage)) { + } + + td::Result>> meta_get_all() const override { + return meta_storage_->meta_get_all(); + } + td::Result meta_get(td::Slice key, std::string &value) override { + CHECK(key.size() != CellTraits::hash_bytes); + return meta_storage_->meta_get(key, value); + } + td::Status meta_set(td::Slice key, td::Slice value) override { + meta_storage_->meta_set(key, value); + return td::Status::OK(); + } + td::Status meta_erase(td::Slice key) override { + meta_storage_->meta_erase(key); + return td::Status::OK(); } td::Result> load_cell(td::Slice hash) override { return storage_->load_cell(CellHash::from_slice(hash)); } + td::Result>> load_known_roots() const override { + return storage_->load_known_roots_local(); + } td::Result> load_root(td::Slice hash) override { return storage_->load_root_local(CellHash::from_slice(hash)); } @@ -798,29 +869,37 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { TRY_STATUS(prepare_commit()); } + td::PerfWarningTimer times_save_diff("save diff"); Stats diff; CHECK(to_dec_.empty()); - for (auto &it : info_) { - auto &info = it.second; + for (auto &info : info_) { if (info.diff_refcnt == 0) { continue; } auto refcnt = td::narrow_cast(static_cast(info.db_refcnt) + info.diff_refcnt); - CHECK(refcnt >= 0); + LOG_CHECK(refcnt >= 0) << info.db_refcnt << " + " << info.diff_refcnt; if (refcnt > 0) { - cell_storer.set(refcnt, info.cell, false); + if (info.db_refcnt == 0) { + TRY_STATUS(cell_storer.set(refcnt, info.cell, false)); + } else { + TRY_STATUS(cell_storer.merge(info.cell->get_hash().as_slice(), info.diff_refcnt)); + } storage_->set(refcnt, info.cell); if (info.db_refcnt == 0) { diff.cells_total_count++; diff.cells_total_size += static_cast(info.cell->get_storage_size()); } } else { - cell_storer.erase(info.cell->get_hash().as_slice()); + TRY_STATUS(cell_storer.erase(info.cell->get_hash().as_slice())); storage_->erase(info.cell->get_hash()); diff.cells_total_count--; diff.cells_total_size -= static_cast(info.cell->get_storage_size()); } } + auto meta_diffs = meta_storage_->extract_diffs(); + for (const auto &meta_diff : meta_diffs) { + TRY_STATUS(cell_storer.apply_meta_diff(meta_diff)); + } storage_->apply_stats_diff(diff); info_ = {}; return td::Status::OK(); @@ -872,13 +951,39 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { private: td::unique_ptr storage_; + td::unique_ptr meta_storage_; struct Info { - td::int32 db_refcnt{0}; - td::int32 diff_refcnt{0}; + mutable td::int32 db_refcnt{0}; + mutable td::int32 diff_refcnt{0}; Ref cell; + vm::CellHash key() const { + return cell->get_hash(); + } + struct Eq { + using is_transparent = void; // Pred to use + bool operator()(const Info &info, const Info &other_info) const { + return info.key() == other_info.key(); + } + bool operator()(const Info &info, td::Slice hash) const { + return info.key().as_slice() == hash; + } + bool operator()(td::Slice hash, const Info &info) const { + return info.key().as_slice() == hash; + } + }; + struct Hash { + using is_transparent = void; // Pred to use + using transparent_key_equal = Eq; + size_t operator()(td::Slice hash) const { + return cell_hash_slice_hash(hash); + } + size_t operator()(const Info &info) const { + return cell_hash_slice_hash(info.key().as_slice()); + } + }; }; - td::HashMap info_; + td::HashSet info_; std::unique_ptr loader_; std::vector> to_inc_; @@ -886,13 +991,13 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { Ref do_inc(Ref cell) { auto cell_hash = cell->get_hash(); - if (auto it = info_.find(cell_hash); it != info_.end()) { - CHECK(it->second.diff_refcnt != std::numeric_limits::max()); - it->second.diff_refcnt++; - return it->second.cell; + if (auto it = info_.find(cell_hash.as_slice()); it != info_.end()) { + CHECK(it->diff_refcnt != std::numeric_limits::max()); + it->diff_refcnt++; + return it->cell; } if (auto o_info = storage_->get_info(cell_hash)) { - info_.emplace(cell_hash, Info{.db_refcnt = o_info->db_refcnt, .diff_refcnt = 1, .cell = o_info->cell}); + info_.emplace(Info{.db_refcnt = o_info->db_refcnt, .diff_refcnt = 1, .cell = o_info->cell}); return std::move(o_info->cell); } @@ -905,21 +1010,21 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { } auto res = cb.finalize(cs.is_special()); CHECK(res->get_hash() == cell_hash); - info_.emplace(cell_hash, Info{.db_refcnt = 0, .diff_refcnt = 1, .cell = res}); + info_.emplace(Info{.db_refcnt = 0, .diff_refcnt = 1, .cell = res}); return res; } void do_dec(Ref cell) { auto cell_hash = cell->get_hash(); - auto it = info_.find(cell_hash); + auto it = info_.find(cell_hash.as_slice()); if (it != info_.end()) { - CHECK(it->second.diff_refcnt != std::numeric_limits::min()); - --it->second.diff_refcnt; + CHECK(it->diff_refcnt != std::numeric_limits::min()); + --it->diff_refcnt; } else { auto info = *storage_->get_info(cell_hash); - it = info_.emplace(cell_hash, Info{.db_refcnt = info.db_refcnt, .diff_refcnt = -1, .cell = info.cell}).first; + it = info_.emplace(Info{.db_refcnt = info.db_refcnt, .diff_refcnt = -1, .cell = info.cell}).first; } - if (it->second.diff_refcnt + it->second.db_refcnt != 0) { + if (it->diff_refcnt + it->db_refcnt != 0) { return; } CellSlice cs(NoVm{}, std::move(cell)); @@ -936,7 +1041,8 @@ std::unique_ptr DynamicBagOfCellsDb::create_in_memory(td::K if (kv == nullptr) { LOG_IF(WARNING, options.verbose) << "Create empty in-memory cells database (no key value is given)"; auto storage = CellStorage::build(options, [](auto, auto, auto) { return std::make_pair(0, 0); }); - return std::make_unique(std::move(storage)); + auto meta_storage = td::make_unique(std::vector>{}); + return std::make_unique(std::move(storage), std::move(meta_storage)); } std::vector keys; @@ -962,6 +1068,9 @@ std::unique_ptr DynamicBagOfCellsDb::create_in_memory(td::K local_desc_count++; return td::Status::OK(); } + if (key.size() != 32) { + return td::Status::OK(); + } auto r_res = CellLoader::load(key, value.str(), true, pc_creator); if (r_res.is_error()) { LOG(ERROR) << r_res.error() << " at " << td::format::escaped(key); @@ -983,6 +1092,24 @@ std::unique_ptr DynamicBagOfCellsDb::create_in_memory(td::K }; auto storage = CellStorage::build(options, parallel_scan_cells); - return std::make_unique(std::move(storage)); + + std::vector> meta; + // NB: it scans 1/(2^32) of the database which is not much + kv->for_each_in_range("desc", "desd", [&meta](td::Slice key, td::Slice value) { + if (key.size() != 32) { + meta.emplace_back(key.str(), value.str()); + } + return td::Status::OK(); + }); + // this is for tests mostly. desc* keys are expected to correspond to roots + kv->for_each_in_range("meta", "metb", [&meta](td::Slice key, td::Slice value) { + if (key.size() != 32) { + meta.emplace_back(key.str(), value.str()); + } + return td::Status::OK(); + }); + auto meta_storage = td::make_unique(std::move(meta)); + + return std::make_unique(std::move(storage), std::move(meta_storage)); } } // namespace vm diff --git a/crypto/vm/db/StaticBagOfCellsDb.cpp b/crypto/vm/db/StaticBagOfCellsDb.cpp index 80dbfbf0b..c65d2624f 100644 --- a/crypto/vm/db/StaticBagOfCellsDb.cpp +++ b/crypto/vm/db/StaticBagOfCellsDb.cpp @@ -40,6 +40,9 @@ class RootCell : public Cell { struct PrivateTag {}; public: + td::Status set_data_cell(Ref &&data_cell) const override { + return cell_->set_data_cell(std::move(data_cell)); + } td::Result load_cell() const override { return cell_->load_cell(); } @@ -94,11 +97,11 @@ class DataCellCacheNoop { class DataCellCacheMutex { public: Ref store(int idx, Ref cell) { - auto lock = cells_rw_mutex_.lock_write(); + std::lock_guard lock(mutex_); return cells_.emplace(idx, std::move(cell)).first->second; } Ref load(int idx) { - auto lock = cells_rw_mutex_.lock_read(); + std::lock_guard lock(mutex_); auto it = cells_.find(idx); if (it != cells_.end()) { return it->second; @@ -106,12 +109,13 @@ class DataCellCacheMutex { return {}; } void clear() { - auto guard = cells_rw_mutex_.lock_write(); + std::lock_guard lock(mutex_); cells_.clear(); } private: - td::RwMutex cells_rw_mutex_; + std::mutex mutex_; + // NB: in case of high contention, one should use multiple buckets with per bucket mutexes td::HashMap> cells_; }; @@ -246,7 +250,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { BagOfCells::Info info_; std::mutex index_i_mutex_; - td::RwMutex index_data_rw_mutex_; + std::mutex index_mutex_; std::string index_data_; std::atomic index_i_{0}; size_t index_offset_{0}; @@ -319,7 +323,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { info_.index_offset + (td::int64)idx * info_.offset_byte_size)); offset_view = new_offset_view; } else { - guard = index_data_rw_mutex_.lock_read().move_as_ok(); + std::lock_guard guard(index_mutex_); offset_view = td::Slice(index_data_).substr((td::int64)idx * info_.offset_byte_size, info_.offset_byte_size); } @@ -432,7 +436,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { } td::uint8 tmp[8]; info_.write_offset(tmp, index_offset_); - auto guard = index_data_rw_mutex_.lock_write(); + std::lock_guard guard(index_mutex_); index_data_.append(reinterpret_cast(tmp), info_.offset_byte_size); } return td::Status::OK(); diff --git a/tddb/td/db/KeyValue.h b/tddb/td/db/KeyValue.h index 12c3a4f8d..c3f83919b 100644 --- a/tddb/td/db/KeyValue.h +++ b/tddb/td/db/KeyValue.h @@ -20,19 +20,51 @@ #include "td/utils/Status.h" #include "td/utils/Time.h" #include "td/utils/logging.h" +#include "td/utils/ThreadSafeCounter.h" #include namespace td { +struct UsageStats { + size_t get_count{}; + size_t get_found_count{}; + size_t get_not_found_count{}; + size_t set_count{}; + UsageStats operator+(const UsageStats& other) const { + return UsageStats{.get_count = get_count + other.get_count, + .get_found_count = get_found_count + other.get_found_count, + .get_not_found_count = get_not_found_count + other.get_not_found_count, + .set_count = set_count + other.set_count}; + } + UsageStats operator-(const UsageStats& other) const { + return UsageStats{.get_count = get_count - other.get_count, + .get_found_count = get_found_count - other.get_found_count, + .get_not_found_count = get_not_found_count - other.get_not_found_count, + .set_count = set_count - other.set_count}; + } + NamedStats to_named_stats() const { + NamedStats ns; + ns.stats_int["usage_get_count"] += get_count; + ns.stats_int["usage_get_found_count"] += get_found_count; + ns.stats_int["usage_get_not_found_count"] += get_not_found_count; + ns.stats_int["usage_set_count"] += set_count; + return ns; + } +}; +inline td::StringBuilder& operator<<(td::StringBuilder& sb, const UsageStats& stats) { + sb << "get: " << stats.get_count << ", +" << stats.get_found_count << ", -" << stats.get_not_found_count; + return sb; +} + class KeyValueReader { public: virtual ~KeyValueReader() = default; enum class GetStatus : int32 { Ok, NotFound }; - virtual Result get(Slice key, std::string &value) = 0; + virtual Result get(Slice key, std::string& value) = 0; virtual Result count(Slice prefix) = 0; virtual Status for_each(std::function f) { return Status::Error("for_each is not supported"); } - virtual Status for_each_in_range (Slice begin, Slice end, std::function f) { + virtual Status for_each_in_range(Slice begin, Slice end, std::function f) { return td::Status::Error("foreach_range is not supported"); } }; @@ -42,7 +74,7 @@ class PrefixedKeyValueReader : public KeyValueReader { PrefixedKeyValueReader(std::shared_ptr reader, Slice prefix) : reader_(std::move(reader)), prefix_(prefix.str()) { } - Result get(Slice key, std::string &value) override { + Result get(Slice key, std::string& value) override { return reader_->get(PSLICE() << prefix_ << key, value); } Result count(Slice prefix) override { @@ -54,14 +86,16 @@ class PrefixedKeyValueReader : public KeyValueReader { std::string prefix_; }; -class KeyValueUtils { - public: -}; - class KeyValue : public KeyValueReader { public: virtual Status set(Slice key, Slice value) = 0; virtual Status erase(Slice key) = 0; + virtual Status merge(Slice key, Slice value) { + return Status::Error("merge is not supported"); + } + virtual Status run_gc() { + return Status::OK(); + } virtual Status begin_write_batch() = 0; virtual Status commit_write_batch() = 0; @@ -80,12 +114,15 @@ class KeyValue : public KeyValueReader { virtual Status flush() { return Status::OK(); } + virtual UsageStats get_usage_stats() { + return {}; + } }; class PrefixedKeyValue : public KeyValue { public: PrefixedKeyValue(std::shared_ptr kv, Slice prefix) : kv_(std::move(kv)), prefix_(prefix.str()) { } - Result get(Slice key, std::string &value) override { + Result get(Slice key, std::string& value) override { return kv_->get(PSLICE() << prefix_ << key, value); } Result count(Slice prefix) override { diff --git a/tddb/td/db/MemoryKeyValue.cpp b/tddb/td/db/MemoryKeyValue.cpp index 080133602..7105f72b9 100644 --- a/tddb/td/db/MemoryKeyValue.cpp +++ b/tddb/td/db/MemoryKeyValue.cpp @@ -22,57 +22,99 @@ namespace td { Result MemoryKeyValue::get(Slice key, std::string &value) { - auto it = map_.find(key); - if (it == map_.end()) { + auto bucket = lock(key); + auto &map = bucket->map; + + usage_stats_.get_count++; + auto it = map.find(key); + if (it == map.end()) { + usage_stats_.get_not_found_count++; return GetStatus::NotFound; } value = it->second; + usage_stats_.get_found_count++; return GetStatus::Ok; } +std::unique_ptr MemoryKeyValue::lock(td::Slice key) { + auto bucket_id = std::hash()(std::string_view(key.data(), key.size())) % buckets_.size(); + return lock(buckets_[bucket_id]); +} + Status MemoryKeyValue::for_each(std::function f) { - for (auto &it : map_) { - TRY_STATUS(f(it.first, it.second)); + for (auto &unlocked_bucket : buckets_) { + auto bucket = lock(unlocked_bucket); + for (auto &it : bucket->map) { + TRY_STATUS(f(it.first, it.second)); + } } return Status::OK(); } Status MemoryKeyValue::for_each_in_range(Slice begin, Slice end, std::function f) { - for (auto it = map_.lower_bound(begin); it != map_.end(); it++) { - if (it->first < end) { - TRY_STATUS(f(it->first, it->second)); - } else { - break; + for (auto &unlocked_bucket : buckets_) { + auto bucket = lock(unlocked_bucket); + auto &map = bucket->map; + for (auto it = map.lower_bound(begin); it != map.end(); it++) { + if (it->first < end) { + TRY_STATUS(f(it->first, it->second)); + } else { + break; + } } } return Status::OK(); } Status MemoryKeyValue::set(Slice key, Slice value) { - map_[key.str()] = value.str(); + auto bucket = lock(key); + auto &map = bucket->map; + + usage_stats_.set_count++; + map[key.str()] = value.str(); return Status::OK(); } +Status MemoryKeyValue::merge(Slice key, Slice update) { + CHECK(merger_); + auto bucket = lock(key); + auto &map = bucket->map; + auto &value = map[key.str()]; + merger_->merge_value_and_update(value, update); + if (value.empty()) { + map.erase(key.str()); + } + return td::Status::OK(); +} Status MemoryKeyValue::erase(Slice key) { - auto it = map_.find(key); - if (it != map_.end()) { - map_.erase(it); + auto bucket = lock(key); + auto &map = bucket->map; + auto it = map.find(key); + if (it != map.end()) { + map.erase(it); } return Status::OK(); } Result MemoryKeyValue::count(Slice prefix) { size_t res = 0; - for (auto it = map_.lower_bound(prefix); it != map_.end(); it++) { - if (Slice(it->first).truncate(prefix.size()) != prefix) { - break; + for (auto &unlocked_bucket : buckets_) { + auto bucket = lock(unlocked_bucket); + auto &map = bucket->map; + for (auto it = map.lower_bound(prefix); it != map.end(); it++) { + if (Slice(it->first).truncate(prefix.size()) != prefix) { + break; + } + res++; } - res++; } return res; } std::unique_ptr MemoryKeyValue::snapshot() { auto res = std::make_unique(); - res->map_ = map_; + for (size_t i = 0; i < buckets_.size(); i++) { + auto bucket = lock(buckets_[i]); + res->buckets_[i].map = bucket->map; + } return std::move(res); } @@ -80,10 +122,10 @@ std::string MemoryKeyValue::stats() const { return PSTRING() << "MemoryKeyValueStats{" << tag("get_count", get_count_) << "}"; } Status MemoryKeyValue::begin_write_batch() { - UNREACHABLE(); + return Status::OK(); } Status MemoryKeyValue::commit_write_batch() { - UNREACHABLE(); + return Status::OK(); } Status MemoryKeyValue::abort_write_batch() { UNREACHABLE(); diff --git a/tddb/td/db/MemoryKeyValue.h b/tddb/td/db/MemoryKeyValue.h index f0b5faa08..cf896095d 100644 --- a/tddb/td/db/MemoryKeyValue.h +++ b/tddb/td/db/MemoryKeyValue.h @@ -22,12 +22,22 @@ #include namespace td { + +struct Merger { + virtual ~Merger() = default; + virtual void merge_value_and_update(std::string &value, Slice update) = 0; + virtual void merge_update_and_update(std::string &left_update, Slice right_update) = 0; +}; class MemoryKeyValue : public KeyValue { public: - Result get(Slice key, std::string &value) override; + MemoryKeyValue() = default; + MemoryKeyValue(std::shared_ptr merger) : merger_(std::move(merger)) { + } + Result get(Slice key, std::string& value) override; Status for_each(std::function f) override; Status for_each_in_range(Slice begin, Slice end, std::function f) override; Status set(Slice key, Slice value) override; + Status merge(Slice key, Slice value) override; Status erase(Slice key) override; Result count(Slice prefix) override; @@ -43,8 +53,30 @@ class MemoryKeyValue : public KeyValue { std::string stats() const override; + UsageStats get_usage_stats() override { + return usage_stats_; + } + private: - std::map> map_; + static constexpr size_t buckets_n = 64; + struct Bucket { + std::mutex mutex; + std::map> map; + }; + struct Unlock { + void operator()(Bucket* bucket) const { + bucket->mutex.unlock(); + } + }; + std::array buckets_{}; int64 get_count_{0}; + UsageStats usage_stats_{}; + std::shared_ptr merger_; + + std::unique_ptr lock(Bucket& bucket) { + bucket.mutex.lock(); + return std::unique_ptr(&bucket); + } + std::unique_ptr lock(td::Slice key); }; } // namespace td diff --git a/tddb/td/db/RocksDb.cpp b/tddb/td/db/RocksDb.cpp index f1aa64a5d..b56f3b145 100644 --- a/tddb/td/db/RocksDb.cpp +++ b/tddb/td/db/RocksDb.cpp @@ -24,10 +24,13 @@ #include "rocksdb/write_batch.h" #include "rocksdb/utilities/optimistic_transaction_db.h" #include "rocksdb/utilities/transaction.h" +#include "td/utils/misc.h" + +#include namespace td { namespace { -static Status from_rocksdb(rocksdb::Status status) { +static Status from_rocksdb(const rocksdb::Status &status) { if (status.ok()) { return Status::OK(); } @@ -56,62 +59,83 @@ RocksDb::~RocksDb() { } RocksDb RocksDb::clone() const { + if (transaction_db_) { + return RocksDb{transaction_db_, options_}; + } return RocksDb{db_, options_}; } Result RocksDb::open(std::string path, RocksDbOptions options) { - rocksdb::OptimisticTransactionDB *db; - { - rocksdb::Options db_options; + rocksdb::Options db_options; + db_options.merge_operator = options.merge_operator; + db_options.compaction_filter = options.compaction_filter; - static auto default_cache = rocksdb::NewLRUCache(1 << 30); - if (!options.no_block_cache && options.block_cache == nullptr) { - options.block_cache = default_cache; - } + static auto default_cache = rocksdb::NewLRUCache(1 << 30); + if (!options.no_block_cache && options.block_cache == nullptr) { + options.block_cache = default_cache; + } - rocksdb::BlockBasedTableOptions table_options; - if (options.no_block_cache) { - table_options.no_block_cache = true; - } else { - table_options.block_cache = options.block_cache; - } - db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); - - db_options.use_direct_reads = options.use_direct_reads; - db_options.manual_wal_flush = true; - db_options.create_if_missing = true; - db_options.max_background_compactions = 4; - db_options.max_background_flushes = 2; - db_options.bytes_per_sync = 1 << 20; - db_options.writable_file_max_buffer_size = 2 << 14; - db_options.statistics = options.statistics; - db_options.max_log_file_size = 100 << 20; - db_options.keep_log_file_num = 1; - rocksdb::OptimisticTransactionDBOptions occ_options; - occ_options.validate_policy = rocksdb::OccValidationPolicy::kValidateSerial; + rocksdb::BlockBasedTableOptions table_options; + if (options.no_block_cache) { + table_options.no_block_cache = true; + } else { + table_options.block_cache = options.block_cache; + } + db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); + + // table_options.block_align = true; + if (options.no_reads) { + db_options.memtable_factory.reset(new rocksdb::VectorRepFactory()); + db_options.allow_concurrent_memtable_write = false; + } + + db_options.wal_recovery_mode = rocksdb::WALRecoveryMode::kTolerateCorruptedTailRecords; + db_options.use_direct_reads = options.use_direct_reads; + db_options.manual_wal_flush = true; + db_options.create_if_missing = true; + db_options.max_background_compactions = 4; + db_options.max_background_flushes = 2; + db_options.bytes_per_sync = 1 << 20; + db_options.writable_file_max_buffer_size = 2 << 14; + db_options.statistics = options.statistics; + db_options.max_log_file_size = 100 << 20; + db_options.keep_log_file_num = 1; + + if (options.experimental) { + // Place your experimental options here + } + + if (options.no_transactions) { + rocksdb::DB *db{nullptr}; + TRY_STATUS(from_rocksdb(rocksdb::DB::Open(db_options, std::move(path), &db))); + return RocksDb(std::shared_ptr(db), std::move(options)); + } else { + rocksdb::OptimisticTransactionDB *db{nullptr}; rocksdb::ColumnFamilyOptions cf_options(db_options); std::vector column_families; column_families.push_back(rocksdb::ColumnFamilyDescriptor(rocksdb::kDefaultColumnFamilyName, cf_options)); std::vector handles; + rocksdb::OptimisticTransactionDBOptions occ_options; + occ_options.validate_policy = rocksdb::OccValidationPolicy::kValidateSerial; TRY_STATUS(from_rocksdb(rocksdb::OptimisticTransactionDB::Open(db_options, occ_options, std::move(path), column_families, &handles, &db))); CHECK(handles.size() == 1); // i can delete the handle since DBImpl is always holding a reference to // default column family delete handles[0]; + return RocksDb(std::shared_ptr(db), std::move(options)); } - return RocksDb(std::shared_ptr(db), std::move(options)); } std::shared_ptr RocksDb::create_statistics() { return rocksdb::CreateDBStatistics(); } -std::string RocksDb::statistics_to_string(const std::shared_ptr statistics) { +std::string RocksDb::statistics_to_string(const std::shared_ptr &statistics) { return statistics->ToString(); } -void RocksDb::reset_statistics(const std::shared_ptr statistics) { +void RocksDb::reset_statistics(const std::shared_ptr &statistics) { statistics->Reset(); } @@ -133,7 +157,9 @@ std::string RocksDb::stats() const { } Result RocksDb::get(Slice key, std::string &value) { - //LOG(ERROR) << "GET"; + if (options_.no_reads) { + return td::Status::Error("trying to read from write-only database"); + } rocksdb::Status status; if (snapshot_) { rocksdb::ReadOptions options; @@ -162,6 +188,18 @@ Status RocksDb::set(Slice key, Slice value) { } return from_rocksdb(db_->Put({}, to_rocksdb(key), to_rocksdb(value))); } +Status RocksDb::merge(Slice key, Slice value) { + if (write_batch_) { + return from_rocksdb(write_batch_->Merge(to_rocksdb(key), to_rocksdb(value))); + } + if (transaction_) { + return from_rocksdb(transaction_->Merge(to_rocksdb(key), to_rocksdb(value))); + } + return from_rocksdb(db_->Merge({}, to_rocksdb(key), to_rocksdb(value))); +} +Status RocksDb::run_gc() { + return from_rocksdb(db_->CompactRange({}, nullptr, nullptr)); +} Status RocksDb::erase(Slice key) { if (write_batch_) { @@ -174,7 +212,11 @@ Status RocksDb::erase(Slice key) { } Result RocksDb::count(Slice prefix) { + if (options_.no_reads) { + return td::Status::Error("trying to read from write-only database"); + } rocksdb::ReadOptions options; + options.auto_prefix_mode = true; options.snapshot = snapshot_.get(); std::unique_ptr iterator; if (snapshot_ || !transaction_) { @@ -197,7 +239,11 @@ Result RocksDb::count(Slice prefix) { } Status RocksDb::for_each(std::function f) { + if (options_.no_reads) { + return td::Status::Error("trying to read from write-only database"); + } rocksdb::ReadOptions options; + options.auto_prefix_mode = true; options.snapshot = snapshot_.get(); std::unique_ptr iterator; if (snapshot_ || !transaction_) { @@ -219,7 +265,11 @@ Status RocksDb::for_each(std::function f) { } Status RocksDb::for_each_in_range(Slice begin, Slice end, std::function f) { + if (options_.no_reads) { + return td::Status::Error("trying to read from write-only database"); + } rocksdb::ReadOptions options; + options.auto_prefix_mode = true; options.snapshot = snapshot_.get(); std::unique_ptr iterator; if (snapshot_ || !transaction_) { @@ -252,9 +302,10 @@ Status RocksDb::begin_write_batch() { Status RocksDb::begin_transaction() { CHECK(!write_batch_); + CHECK(transaction_db_); rocksdb::WriteOptions options; options.sync = true; - transaction_.reset(db_->BeginTransaction(options, {})); + transaction_.reset(transaction_db_->BeginTransaction(options, {})); return Status::OK(); } @@ -307,7 +358,11 @@ Status RocksDb::end_snapshot() { } RocksDb::RocksDb(std::shared_ptr db, RocksDbOptions options) - : db_(std::move(db)), options_(options) { + : transaction_db_{db}, db_(std::move(db)), options_(std::move(options)) { +} + +RocksDb::RocksDb(std::shared_ptr db, RocksDbOptions options) + : db_(std::move(db)), options_(std::move(options)) { } void RocksDbSnapshotStatistics::begin_snapshot(const rocksdb::Snapshot *snapshot) { diff --git a/tddb/td/db/RocksDb.h b/tddb/td/db/RocksDb.h index 499a33281..d24a20dd7 100644 --- a/tddb/td/db/RocksDb.h +++ b/tddb/td/db/RocksDb.h @@ -36,12 +36,16 @@ #include namespace rocksdb { +class DB; +class Comparator; class Cache; class OptimisticTransactionDB; class Transaction; class WriteBatch; class Snapshot; class Statistics; +class MergeOperator; +class CompactionFilter; } // namespace rocksdb namespace td { @@ -61,6 +65,14 @@ struct RocksDbOptions { std::shared_ptr statistics = nullptr; std::shared_ptr block_cache; // Default - one 1G cache for all RocksDb std::shared_ptr snapshot_statistics = nullptr; + + std::shared_ptr merge_operator = nullptr; + const rocksdb::CompactionFilter *compaction_filter = nullptr; + + bool experimental = false; + bool no_reads = false; + bool no_transactions = false; + bool use_direct_reads = false; bool no_block_cache = false; }; @@ -73,10 +85,12 @@ class RocksDb : public KeyValue { Result get(Slice key, std::string &value) override; Status set(Slice key, Slice value) override; + Status merge(Slice key, Slice value) override; Status erase(Slice key) override; + Status run_gc() override; Result count(Slice prefix) override; Status for_each(std::function f) override; - Status for_each_in_range (Slice begin, Slice end, std::function f) override; + Status for_each_in_range(Slice begin, Slice end, std::function f) override; Status begin_write_batch() override; Status commit_write_batch() override; @@ -94,8 +108,8 @@ class RocksDb : public KeyValue { std::string stats() const override; static std::shared_ptr create_statistics(); - static std::string statistics_to_string(const std::shared_ptr statistics); - static void reset_statistics(const std::shared_ptr statistics); + static std::string statistics_to_string(const std::shared_ptr &statistics); + static void reset_statistics(const std::shared_ptr &statistics); static std::shared_ptr create_cache(size_t capacity); @@ -103,12 +117,13 @@ class RocksDb : public KeyValue { RocksDb &operator=(RocksDb &&); ~RocksDb(); - std::shared_ptr raw_db() const { + std::shared_ptr raw_db() const { return db_; }; private: - std::shared_ptr db_; + std::shared_ptr transaction_db_; + std::shared_ptr db_; RocksDbOptions options_; std::unique_ptr transaction_; @@ -123,5 +138,6 @@ class RocksDb : public KeyValue { std::unique_ptr snapshot_; explicit RocksDb(std::shared_ptr db, RocksDbOptions options); + explicit RocksDb(std::shared_ptr db, RocksDbOptions options); }; } // namespace td diff --git a/tdutils/td/utils/MpmcQueue.h b/tdutils/td/utils/MpmcQueue.h index e6504e358..1a5f8fa36 100644 --- a/tdutils/td/utils/MpmcQueue.h +++ b/tdutils/td/utils/MpmcQueue.h @@ -414,7 +414,9 @@ class MpmcQueue { while (true) { auto node = hazard_pointers_.protect(thread_id, 0, read_pos_); auto &block = node->block; - if (block.write_pos <= block.read_pos && node->next.load(std::memory_order_relaxed) == nullptr) { + auto read_pos = block.read_pos.load(); + auto write_pos = block.write_pos.load(); + if (write_pos <= read_pos && node->next.load(std::memory_order_relaxed) == nullptr) { return false; } auto pos = block.read_pos++; diff --git a/tdutils/td/utils/Status.h b/tdutils/td/utils/Status.h index cff808143..f75de466a 100644 --- a/tdutils/td/utils/Status.h +++ b/tdutils/td/utils/Status.h @@ -619,6 +619,13 @@ inline Result::Result(Status &&status) : status_(std::move(status)) { inline StringBuilder &operator<<(StringBuilder &string_builder, const Status &status) { return status.print(string_builder); } +template +StringBuilder &operator<<(StringBuilder &sb, const Result &result) { + if (result.is_ok()) { + return sb << "Ok{" << result.ok() << "}"; + } + return sb << result.error(); +} namespace detail { diff --git a/tdutils/td/utils/ThreadSafeCounter.h b/tdutils/td/utils/ThreadSafeCounter.h index aa976b2fb..46dc16bf7 100644 --- a/tdutils/td/utils/ThreadSafeCounter.h +++ b/tdutils/td/utils/ThreadSafeCounter.h @@ -19,6 +19,7 @@ #pragma once +#include "port/thread.h" #include "td/utils/common.h" #include "td/utils/Slice.h" #include "td/utils/StringBuilder.h" @@ -26,6 +27,7 @@ #include #include +#include #include namespace td { @@ -69,6 +71,50 @@ class ThreadSafeCounter { ThreadSafeMultiCounter<1> counter_; }; +struct NamedStats { + std::map stats_int; + std::map stats_str; + + NamedStats with_suffix(const std::string &suffix) const { + NamedStats res; + for (auto &p : stats_int) { + res.stats_int[p.first + suffix] = p.second; + } + for (auto &p : stats_str) { + res.stats_str[p.first + suffix] = p.second; + } + return res; + } + NamedStats with_prefix(const std::string &prefix) const { + NamedStats res; + for (auto &p : stats_int) { + res.stats_int[prefix + p.first] = p.second; + } + for (auto &p : stats_str) { + res.stats_str[prefix + p.first] = p.second; + } + return res; + } + void apply_diff(const NamedStats &other) { + for (auto &p : other.stats_int) { + stats_int[p.first] += p.second; + } + for (auto &p : other.stats_str) { + stats_str[p.first] = p.second; + } + } + void subtract_diff(const NamedStats &other) { + for (auto &p : other.stats_int) { + stats_int[p.first] -= p.second; + } + } + NamedStats combine_with(const NamedStats &other) const { + NamedStats res = *this; + res.apply_diff(other); + return res; + } +}; + class NamedThreadSafeCounter { static constexpr int N = 128; using Counter = ThreadSafeMultiCounter; @@ -79,6 +125,9 @@ class NamedThreadSafeCounter { CounterRef() = default; CounterRef(size_t index, Counter *counter) : index_(index), counter_(counter) { } + void inc() { + add(1); + } void add(int64 diff) { counter_->add(index_, diff); } @@ -119,6 +168,11 @@ class NamedThreadSafeCounter { f(names_[i], counter_.sum(i)); } } + NamedStats get_stats() const { + NamedStats res; + for_each([&](Slice name, int64 cnt) { res.stats_int.emplace(name.str(), cnt); }); + return res; + } void clear() { std::unique_lock guard(mutex_); @@ -181,11 +235,11 @@ struct NamedPerfCounter { } // namespace td -#define TD_PERF_COUNTER(name) \ +#define TD_PERF_COUNTER(name) \ static auto perf_##name = td::NamedPerfCounter::get_default().get_counter(td::Slice(#name)); \ auto scoped_perf_##name = td::NamedPerfCounter::ScopedPerfCounterRef{.perf_counter = perf_##name}; -#define TD_PERF_COUNTER_SINCE(name, since) \ +#define TD_PERF_COUNTER_SINCE(name, since) \ static auto perf_##name = td::NamedPerfCounter::get_default().get_counter(td::Slice(#name)); \ - auto scoped_perf_##name = \ + auto scoped_perf_##name = \ td::NamedPerfCounter::ScopedPerfCounterRef{.perf_counter = perf_##name, .started_at_ticks = since}; diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 2ea04e183..cbcc3ab1f 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1414,6 +1414,9 @@ td::Status ValidatorEngine::load_global_config() { if (zero_state.root_hash.is_zero() || zero_state.file_hash.is_zero()) { return td::Status::Error(ton::ErrorCode::error, "[validator] section contains incomplete [zero_state]"); } + if (celldb_in_memory_ && celldb_v2_) { + return td::Status::Error(ton::ErrorCode::error, "at most one of --celldb-in-memory --celldb-v2 could be used"); + } ton::BlockIdExt init_block; if (!conf.validator_->init_block_) { @@ -1461,11 +1464,12 @@ td::Status ValidatorEngine::load_global_config() { if (!session_logs_file_.empty()) { validator_options_.write().set_session_logs_file(session_logs_file_); } - if (celldb_in_memory_) { + if (celldb_in_memory_ || celldb_v2_) { celldb_compress_depth_ = 0; } validator_options_.write().set_celldb_compress_depth(celldb_compress_depth_); validator_options_.write().set_celldb_in_memory(celldb_in_memory_); + validator_options_.write().set_celldb_v2(celldb_v2_); validator_options_.write().set_max_open_archive_files(max_open_archive_files_); validator_options_.write().set_archive_preload_period(archive_preload_period_); validator_options_.write().set_disable_rocksdb_stats(disable_rocksdb_stats_); @@ -4526,6 +4530,12 @@ int main(int argc, char *argv[]) { [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_in_memory, true); }); }); + p.add_option( + '\0', "celldb-v2", + "use new version off celldb", + [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_v2, true); }); + }); p.add_checked_option( '\0', "catchain-max-block-delay", "delay before creating a new catchain block, in seconds (default: 0.4)", [&](td::Slice s) -> td::Status { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index e0dc91f13..5a1db7f3f 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -218,6 +218,7 @@ class ValidatorEngine : public td::actor::Actor { bool celldb_direct_io_ = false; bool celldb_preload_all_ = false; bool celldb_in_memory_ = false; + bool celldb_v2_ = false; td::optional catchain_max_block_delay_, catchain_max_block_delay_slow_; bool read_config_ = false; bool started_keyring_ = false; @@ -311,6 +312,9 @@ class ValidatorEngine : public td::actor::Actor { void set_celldb_in_memory(bool value) { celldb_in_memory_ = value; } + void set_celldb_v2(bool value) { + celldb_v2_ = value; + } void set_catchain_max_block_delay(double value) { catchain_max_block_delay_ = value; } diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index e86a373d1..5fc5a83a4 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -28,6 +28,9 @@ #include "ton/ton-io.hpp" #include "common/delay.h" +#include +#include + namespace ton { namespace validator { @@ -73,6 +76,41 @@ CellDbIn::CellDbIn(td::actor::ActorId root_db, td::actor::ActorId td::Slice { + return td::Slice(value.data(), value.size()); + } + bool FullMergeV2(const MergeOperationInput& merge_in, MergeOperationOutput* merge_out) const override { + CHECK(merge_in.existing_value); + auto& value = *merge_in.existing_value; + CHECK(merge_in.operand_list.size() >= 1); + td::Slice diff; + std::string diff_buf; + if (merge_in.operand_list.size() == 1) { + diff = to_td(merge_in.operand_list[0]); + } else { + diff_buf = merge_in.operand_list[0].ToString(); + for (size_t i = 1; i < merge_in.operand_list.size(); ++i) { + vm::CellStorer::merge_refcnt_diffs(diff_buf, to_td(merge_in.operand_list[i])); + } + diff = diff_buf; + } + + merge_out->new_value = value.ToString(); + vm::CellStorer::merge_value_and_refcnt_diff(merge_out->new_value, diff); + return true; + } + bool PartialMerge(const rocksdb::Slice& /*key*/, const rocksdb::Slice& left, const rocksdb::Slice& right, + std::string* new_value, rocksdb::Logger* logger) const override { + *new_value = left.ToString(); + vm::CellStorer::merge_refcnt_diffs(*new_value, to_td(right)); + return true; + } +}; + void CellDbIn::start_up() { on_load_callback_ = [actor = std::make_shared>( td::actor::create_actor("celldbmigration", actor_id(this))), @@ -96,46 +134,147 @@ void CellDbIn::start_up() { db_options.snapshot_statistics = snapshot_statistics_; } db_options.statistics = statistics_; - if (opts_->get_celldb_cache_size()) { - db_options.block_cache = td::RocksDb::create_cache(opts_->get_celldb_cache_size().value()); - LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(opts_->get_celldb_cache_size().value()); + auto o_celldb_cache_size = opts_->get_celldb_cache_size(); + + std::optional boc_in_memory_options; + std::optional boc_v1_options; + std::optional boc_v2_options; + + if (opts_->get_celldb_v2()) { + boc_v2_options = + vm::DynamicBagOfCellsDb::CreateV2Options{.extra_threads = std::clamp(std::thread::hardware_concurrency() / 2, 1u, 8u), + .executor = {}, + .cache_ttl_max = 2000, + .cache_size_max = 1000000}; + size_t min_rocksdb_cache = std::max(size_t{1} << 30, boc_v2_options->cache_size_max * 5000); + if (!o_celldb_cache_size || o_celldb_cache_size.value() < min_rocksdb_cache) { + LOG(WARNING) << "Increase CellDb block cache size to " << td::format::as_size(min_rocksdb_cache) << " from " + << td::format::as_size(o_celldb_cache_size.value()); + o_celldb_cache_size = min_rocksdb_cache; + } + LOG(WARNING) << "Using V2 DynamicBagOfCells with options " << *boc_v2_options; + } else if (opts_->get_celldb_in_memory()) { + // default options + boc_in_memory_options = vm::DynamicBagOfCellsDb::CreateInMemoryOptions{ + .extra_threads = std::thread::hardware_concurrency(), + .verbose = true, + .use_arena = false, + .use_less_memory_during_creation = true, + }; + LOG(WARNING) << "Using InMemory DynamicBagOfCells with options " << *boc_v2_options; + } else { + boc_v1_options = vm::DynamicBagOfCellsDb::CreateV1Options{}; + LOG(WARNING) << "Using V1 DynamicBagOfCells with options " << *boc_v1_options; + } + + if (o_celldb_cache_size) { + db_options.block_cache = td::RocksDb::create_cache(o_celldb_cache_size.value()); + LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(o_celldb_cache_size.value()); } db_options.use_direct_reads = opts_->get_celldb_direct_io(); + // NB: from now on we MUST use this merge operator + // Only V2 and InMemory BoC actually use them, but it still should be kept for V1, + // to handle updates written by V2 or InMemory BoCs + db_options.merge_operator = std::make_shared(); + if (opts_->get_celldb_in_memory()) { td::RocksDbOptions read_db_options; read_db_options.use_direct_reads = true; read_db_options.no_block_cache = true; read_db_options.block_cache = {}; + read_db_options.merge_operator = std::make_shared(); LOG(WARNING) << "Loading all cells in memory (because of --celldb-in-memory)"; td::Timer timer; auto read_cell_db = std::make_shared(td::RocksDb::open(path_, std::move(read_db_options)).move_as_ok()); - boc_ = vm::DynamicBagOfCellsDb::create_in_memory(read_cell_db.get(), {}); + boc_ = vm::DynamicBagOfCellsDb::create_in_memory(read_cell_db.get(), *boc_in_memory_options); in_memory_load_time_ = timer.elapsed(); - td::actor::send_closure(parent_, &CellDb::set_in_memory_boc, boc_); + + // no reads will be allowed from rocksdb, only writes + db_options.no_reads = true; } auto rocks_db = std::make_shared(td::RocksDb::open(path_, std::move(db_options)).move_as_ok()); rocks_db_ = rocks_db->raw_db(); cell_db_ = std::move(rocks_db); if (!opts_->get_celldb_in_memory()) { - boc_ = vm::DynamicBagOfCellsDb::create(); + if (opts_->get_celldb_v2()) { + boc_ = vm::DynamicBagOfCellsDb::create_v2(*boc_v2_options); + } else { + boc_ = vm::DynamicBagOfCellsDb::create(*boc_v1_options); + } boc_->set_celldb_compress_depth(opts_->get_celldb_compress_depth()); boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); - td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); } + auto meta = boc_->meta_get_all().move_as_ok(); + size_t missing_roots = 0; + size_t unknown_roots = 0; + std::set root_hashes; + for (auto [k, v] : meta) { + if (k == "desczero") { + continue; + } + auto obj = fetch_tl_object(td::BufferSlice{v}, true); + obj.ensure(); + auto entry = DbEntry{obj.move_as_ok()}; + root_hashes.insert(vm::CellHash::from_slice(entry.root_hash.as_slice())); + auto cell = boc_->load_cell(entry.root_hash.as_slice()); + missing_roots += cell.is_error(); + LOG_IF(ERROR, cell.is_error()) << "Cannot load root from meta: " << entry.block_id.to_str() << " " << cell.error(); + } + auto known_roots = boc_->load_known_roots().move_as_ok(); + for (auto& root : known_roots) { + block::gen::ShardStateUnsplit::Record info; + block::gen::OutMsgQueueInfo::Record qinfo; + block::ShardId shard; + if (!(tlb::unpack_cell(root, info) && shard.deserialize(info.shard_id.write()) && + tlb::unpack_cell(info.out_msg_queue_info, qinfo))) { + LOG(FATAL) << "cannot create ShardDescr from a root in celldb"; + } + if (!root_hashes.contains(root->get_hash())) { + unknown_roots++; + LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no; + constexpr bool delete_unknown_roots = false; + if (delete_unknown_roots) { + vm::CellStorer stor{*cell_db_}; + cell_db_->begin_write_batch().ensure(); + boc_->dec(root); + boc_->commit(stor).ensure(); + cell_db_->commit_write_batch().ensure(); + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + } + LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no << " REMOVED"; + } + } + } + + LOG_IF(ERROR, missing_roots != 1) << "Missing root hashes: " << missing_roots; + LOG_IF(ERROR, unknown_roots != 0) << "Unknown roots: " << unknown_roots; + + LOG_IF(FATAL, missing_roots > 1) << "Missing root hashes: " << missing_roots; + LOG_IF(FATAL, unknown_roots != 0) << "Unknown roots: " << unknown_roots; + alarm_timestamp() = td::Timestamp::in(10.0); auto empty = get_empty_key_hash(); if (get_block(empty).is_error()) { DbEntry e{get_empty_key(), empty, empty, RootHash::zero()}; + vm::CellStorer stor{*cell_db_}; cell_db_->begin_write_batch().ensure(); set_block(empty, std::move(e)); + boc_->commit(stor); cell_db_->commit_write_batch().ensure(); } + if (opts_->get_celldb_v2() || opts_->get_celldb_in_memory()) { + send_closure(parent_, &CellDb::set_thread_safe_boc, boc_); + } else { + send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); + } + if (opts_->get_celldb_preload_all()) { // Iterate whole DB in a separate thread delay_action( @@ -161,7 +300,7 @@ void CellDbIn::start_up() { { std::string key = "stats.last_deleted_mc_seqno", value; - auto R = cell_db_->get(td::as_slice(key), value); + auto R = boc_->meta_get(td::as_slice(key), value); R.ensure(); if (R.ok() == td::KeyValue::GetStatus::Ok) { auto r_value = td::to_integer_safe(value); @@ -240,10 +379,10 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi td::Timer timer_write; vm::CellStorer stor{*cell_db_}; cell_db_->begin_write_batch().ensure(); - boc_->commit(stor).ensure(); set_block(get_empty_key_hash(), std::move(E)); set_block(D.prev, std::move(P)); set_block(key_hash, std::move(D)); + boc_->commit(stor).ensure(); cell_db_->commit_write_batch().ensure(); timer_write.pause(); @@ -266,11 +405,10 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi void CellDbIn::get_cell_db_reader(td::Promise> promise) { if (db_busy_) { - action_queue_.push( - [self = this, promise = std::move(promise)](td::Result R) mutable { - R.ensure(); - self->get_cell_db_reader(std::move(promise)); - }); + action_queue_.push([self = this, promise = std::move(promise)](td::Result R) mutable { + R.ensure(); + self->get_cell_db_reader(std::move(promise)); + }); return; } promise.set_result(boc_->get_cell_db_reader()); @@ -440,9 +578,16 @@ void CellDbIn::gc_cont2(BlockHandle handle) { timer_get_keys.reset(); td::PerfWarningTimer timer_boc{"gccell_boc", 0.05}; - auto cell = boc_->load_cell(F.root_hash.as_slice()).move_as_ok(); + auto r_cell = boc_->load_cell(F.root_hash.as_slice()); + td::Ref cell; + if (r_cell.is_ok()) { + cell = r_cell.move_as_ok(); + boc_->dec(cell); + LOG(ERROR) << "GC of " << handle->id().to_str(); + } else { + LOG(ERROR) << "GC of UNKNOWN root: " << handle->id().to_str(); + } - boc_->dec(cell); db_busy_ = true; boc_->prepare_commit_async( async_executor, [this, SelfId = actor_id(this), timer_boc = std::move(timer_boc), F = std::move(F), key_hash, @@ -458,17 +603,19 @@ void CellDbIn::gc_cont2(BlockHandle handle) { td::PerfWarningTimer timer_write_batch{"gccell_write_batch", 0.05}; cell_db_->begin_write_batch().ensure(); - boc_->commit(stor).ensure(); - cell_db_->erase(get_key(key_hash)).ensure(); + boc_->meta_erase(get_key(key_hash)).ensure(); set_block(F.prev, std::move(P)); set_block(F.next, std::move(N)); if (handle->id().is_masterchain()) { last_deleted_mc_state_ = handle->id().seqno(); std::string key = "stats.last_deleted_mc_seqno", value = td::to_string(last_deleted_mc_state_); - cell_db_->set(td::as_slice(key), td::as_slice(value)); + boc_->meta_set(td::as_slice(key), td::as_slice(value)); } + + boc_->commit(stor).ensure(); cell_db_->commit_write_batch().ensure(); + alarm_timestamp() = td::Timestamp::now(); timer_write_batch.reset(); @@ -530,7 +677,7 @@ CellDbIn::KeyHash CellDbIn::get_empty_key_hash() { td::Result CellDbIn::get_block(KeyHash key_hash) { const auto key = get_key(key_hash); std::string value; - auto R = cell_db_->get(td::as_slice(key), value); + auto R = boc_->meta_get(td::as_slice(key), value); R.ensure(); auto S = R.move_as_ok(); if (S == td::KeyValue::GetStatus::NotFound) { @@ -543,7 +690,7 @@ td::Result CellDbIn::get_block(KeyHash key_hash) { void CellDbIn::set_block(KeyHash key_hash, DbEntry e) { const auto key = get_key(key_hash); - cell_db_->set(td::as_slice(key), e.release()).ensure(); + boc_->meta_set(td::as_slice(key), e.release()); } void CellDbIn::migrate_cell(td::Bits256 hash) { @@ -631,12 +778,14 @@ void CellDb::alarm() { } void CellDb::load_cell(RootHash hash, td::Promise> promise) { - if (in_memory_boc_) { - auto result = in_memory_boc_->load_root_thread_safe(hash.as_slice()); + if (thread_safe_boc_) { + auto result = thread_safe_boc_->load_root_thread_safe(hash.as_slice()); if (result.is_ok()) { return async_apply("load_cell_result", std::move(promise), std::move(result)); } else { LOG(ERROR) << "load_root_thread_safe failed - this is suspicious"; + send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise)); + return; } } if (!started_) { @@ -710,6 +859,13 @@ std::vector> CellDbIn::CellDbStatistics::pre for (auto& [key, value] : boc_stats_->custom_stats) { stats.emplace_back(key, value); } + + for (auto& [key, value] : boc_stats_->named_stats.stats_str) { + stats.emplace_back(key, value); + } + for (auto& [key, value] : boc_stats_->named_stats.stats_int) { + stats.emplace_back(key, td::to_string(value)); + } } return stats; } diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index 5639b9748..8d97e13c7 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -195,13 +195,13 @@ class CellDb : public CellDbBase { started_ = true; boc_->set_loader(std::make_unique(std::move(snapshot), on_load_callback_)).ensure(); } - void set_in_memory_boc(std::shared_ptr in_memory_boc) { - CHECK(opts_->get_celldb_in_memory()); + void set_thread_safe_boc(std::shared_ptr thread_safe_boc) { + CHECK(opts_->get_celldb_in_memory() || opts_->get_celldb_v2()); if (!started_) { alarm(); } started_ = true; - in_memory_boc_ = std::move(in_memory_boc); + thread_safe_boc_ = std::move(thread_safe_boc); } void get_cell_db_reader(td::Promise> promise); @@ -219,7 +219,7 @@ class CellDb : public CellDbBase { td::actor::ActorOwn cell_db_; std::unique_ptr boc_; - std::shared_ptr in_memory_boc_; + std::shared_ptr thread_safe_boc_; bool started_ = false; std::vector> prepared_stats_{{"started", "false"}}; diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index ace6b1066..45b8d7ec2 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -139,6 +139,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool get_celldb_in_memory() const override { return celldb_in_memory_; } + bool get_celldb_v2() const override { + return celldb_v2_; + } td::optional get_catchain_max_block_delay() const override { return catchain_max_block_delay_; } @@ -237,6 +240,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_celldb_in_memory(bool value) override { celldb_in_memory_ = value; } + void set_celldb_v2(bool value) override { + celldb_v2_ = value; + } void set_catchain_max_block_delay(double value) override { catchain_max_block_delay_ = value; } @@ -304,6 +310,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool celldb_direct_io_ = false; bool celldb_preload_all_ = false; bool celldb_in_memory_ = false; + bool celldb_v2_ = false; td::optional catchain_max_block_delay_, catchain_max_block_delay_slow_; bool state_serializer_enabled_ = true; td::Ref collator_options_{true}; diff --git a/validator/validator.h b/validator/validator.h index 5d6c0173c..66795cece 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -104,6 +104,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual std::string get_session_logs_file() const = 0; virtual td::uint32 get_celldb_compress_depth() const = 0; virtual bool get_celldb_in_memory() const = 0; + virtual bool get_celldb_v2() const = 0; virtual size_t get_max_open_archive_files() const = 0; virtual double get_archive_preload_period() const = 0; virtual bool get_disable_rocksdb_stats() const = 0; @@ -144,6 +145,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_celldb_direct_io(bool value) = 0; virtual void set_celldb_preload_all(bool value) = 0; virtual void set_celldb_in_memory(bool value) = 0; + virtual void set_celldb_v2(bool value) = 0; virtual void set_catchain_max_block_delay(double value) = 0; virtual void set_catchain_max_block_delay_slow(double value) = 0; virtual void set_state_serializer_enabled(bool value) = 0; From 597f3b7020c4671c2007184b45d4fd3798a4d802 Mon Sep 17 00:00:00 2001 From: birydrad <> Date: Wed, 15 Jan 2025 14:25:25 +0000 Subject: [PATCH 145/388] celldb: add test for load nonexisting cell, test thread safeness of CellUsageTree, fixes --- crypto/test/test-db.cpp | 35 ++++++- crypto/vm/db/DynamicBagOfCellsDb.cpp | 8 +- crypto/vm/db/DynamicBagOfCellsDb.h | 2 +- crypto/vm/db/DynamicBagOfCellsDbV2.cpp | 8 +- crypto/vm/db/InMemoryBagOfCellsDb.cpp | 15 ++- validator/db/celldb.cpp | 122 ++++++++++++++----------- validator/db/celldb.hpp | 1 + 7 files changed, 130 insertions(+), 61 deletions(-) diff --git a/crypto/test/test-db.cpp b/crypto/test/test-db.cpp index bc4dac606..2b77e7118 100644 --- a/crypto/test/test-db.cpp +++ b/crypto/test/test-db.cpp @@ -50,6 +50,7 @@ #include #include #include +#include #include @@ -1312,7 +1313,8 @@ void with_all_boc_options(F &&f, size_t tests_n, bool single_thread = false) { // V2 - one thread run({.async_executor = executor, .kv_options = kv_options, - .options = DynamicBagOfCellsDb::CreateV2Options{.extra_threads = 0, .executor = executor, .cache_ttl_max = 5}}); + .options = + DynamicBagOfCellsDb::CreateV2Options{.extra_threads = 0, .executor = executor, .cache_ttl_max = 5}}); // InMemory for (auto use_arena : {false, true}) { @@ -1353,6 +1355,8 @@ DynamicBagOfCellsDb::Stats test_dynamic_boc(BocOptions options) { if (rnd() % 10 == 0) { reload_db(); } + db.dboc->load_cell(vm::CellHash{}.as_slice()).ensure_error(); + db.reset_loader(); Ref old_root; if (!old_root_hash.empty()) { @@ -2901,6 +2905,35 @@ TEST(TonDb, BocRespectsUsageCell) { ASSERT_STREQ(serialization, serialization_of_virtualized_cell); } +TEST(UsageTree, ThreadSafe) { + size_t test_n = 100; + td::Random::Xorshift128plus rnd(123); + for (size_t test_i = 0; test_i < test_n; test_i++) { + auto cell = vm::gen_random_cell(rnd.fast(2, 100), rnd, false); + auto usage_tree = std::make_shared(); + auto usage_cell = vm::UsageCell::create(cell, usage_tree->root_ptr()); + std::ptrdiff_t threads_n = 1; // TODO: when CellUsageTree is thread safe, change it to 4 + auto barrier = std::barrier{threads_n}; + std::vector threads; + std::vector explorations(threads_n); + for (std::ptrdiff_t i = 0; i < threads_n; i++) { + threads.emplace_back([&, i = i]() { + barrier.arrive_and_wait(); + explorations[i] = vm::CellExplorer::random_explore(usage_cell, rnd); + }); + } + for (auto &thread : threads) { + thread.join(); + } + auto proof = vm::MerkleProof::generate(cell, usage_tree.get()); + auto virtualized_proof = vm::MerkleProof::virtualize(proof, 1); + for (auto &exploration : explorations) { + auto new_exploration = vm::CellExplorer::explore(virtualized_proof, exploration.ops); + ASSERT_EQ(exploration.log, new_exploration.log); + } + } +} + /* vm::DynamicBagOfCellsDb::Stats test_dynamic_boc_respects_usage_cell(vm::BocOptions options) { td::Random::Xorshift128plus rnd(options.seed); diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp index bd23733e0..d6731b039 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDb.cpp @@ -107,15 +107,21 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { return get_cell_info_lazy(level_mask, hash, depth).cell; } - td::Result>> meta_get_all() const override { + td::Result>> meta_get_all(size_t max_count) const override { std::vector> result; auto s = loader_->key_value_reader().for_each_in_range("desc", "desd", [&](const td::Slice &key, const td::Slice &value) { + if (result.size() >= max_count) { + return td::Status::Error("COUNT_LIMIT"); + } if (td::begins_with(key, "desc") && key.size() != 32) { result.emplace_back(key.str(), value.str()); } return td::Status::OK(); }); + if (s.message() == "COUNT_LIMIT") { + s = td::Status::OK(); + } TRY_STATUS(std::move(s)); return result; } diff --git a/crypto/vm/db/DynamicBagOfCellsDb.h b/crypto/vm/db/DynamicBagOfCellsDb.h index ca23d72f2..82028f3fe 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.h +++ b/crypto/vm/db/DynamicBagOfCellsDb.h @@ -51,7 +51,7 @@ class DynamicBagOfCellsDb { public: virtual ~DynamicBagOfCellsDb() = default; - virtual td::Result>> meta_get_all() const = 0; + virtual td::Result>> meta_get_all(size_t max_count) const = 0; virtual td::Result meta_get(td::Slice key, std::string &value) = 0; virtual td::Status meta_set(td::Slice key, td::Slice value) = 0; virtual td::Status meta_erase(td::Slice key) = 0; diff --git a/crypto/vm/db/DynamicBagOfCellsDbV2.cpp b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp index 45fdffee8..eff74e214 100644 --- a/crypto/vm/db/DynamicBagOfCellsDbV2.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp @@ -765,16 +765,22 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { } } - td::Result>> meta_get_all() const override { + td::Result>> meta_get_all(size_t max_count) const override { CHECK(meta_db_fixup_.empty()); std::vector> result; auto s = cell_db_reader_->key_value_reader().for_each_in_range( "desc", "desd", [&](const td::Slice &key, const td::Slice &value) { + if (result.size() >= max_count) { + return td::Status::Error("COUNT_LIMIT"); + } if (td::begins_with(key, "desc") && key.size() != 32) { result.emplace_back(key.str(), value.str()); } return td::Status::OK(); }); + if (s.message() == "COUNT_LIMIT") { + s = td::Status::OK(); + } TRY_STATUS(std::move(s)); return result; } diff --git a/crypto/vm/db/InMemoryBagOfCellsDb.cpp b/crypto/vm/db/InMemoryBagOfCellsDb.cpp index b0a30cfb2..e43cfde4e 100644 --- a/crypto/vm/db/InMemoryBagOfCellsDb.cpp +++ b/crypto/vm/db/InMemoryBagOfCellsDb.cpp @@ -774,8 +774,15 @@ class MetaStorage { CHECK(p.first.size() != CellTraits::hash_bytes); } } - std::vector> meta_get_all() const { - return td::transform(meta_, [](const auto &p) { return std::make_pair(p.first, p.second); }); + std::vector> meta_get_all(size_t max_count) const { + std::vector> res; + for (const auto &[k, v] : meta_) { + if (res.size() >= max_count) { + break; + } + res.emplace_back(k, v); + } + return res; } KeyValue::GetStatus meta_get(td::Slice key, std::string &value) const { auto lock = local_access_.lock(); @@ -814,8 +821,8 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { : storage_(std::move(storage)), meta_storage_(std::move(meta_storage)) { } - td::Result>> meta_get_all() const override { - return meta_storage_->meta_get_all(); + td::Result>> meta_get_all(size_t max_count) const override { + return meta_storage_->meta_get_all(max_count); } td::Result meta_get(td::Slice key, std::string &value) override { CHECK(key.size() != CellTraits::hash_bytes); diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 5fc5a83a4..90c659cc4 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -111,6 +111,66 @@ struct MergeOperatorAddCellRefcnt : public rocksdb::MergeOperator { } }; +void CellDbIn::validate_meta() { + LOG(INFO) << "Validating metadata\n"; + size_t max_meta_keys_loaded = opts_->get_celldb_in_memory() ? std::numeric_limits::max() : 10000; + auto meta = boc_->meta_get_all(max_meta_keys_loaded).move_as_ok(); + bool partial_check = meta.size() == max_meta_keys_loaded; + if (partial_check) { + LOG(ERROR) << "Too much metadata in the database, do only partial check"; + } + size_t missing_roots = 0; + size_t unknown_roots = 0; + std::set root_hashes; + for (auto [k, v] : meta) { + if (k == "desczero") { + continue; + } + auto obj = fetch_tl_object(td::BufferSlice{v}, true); + obj.ensure(); + auto entry = DbEntry{obj.move_as_ok()}; + root_hashes.insert(vm::CellHash::from_slice(entry.root_hash.as_slice())); + auto cell = boc_->load_cell(entry.root_hash.as_slice()); + missing_roots += cell.is_error(); + LOG_IF(ERROR, cell.is_error()) << "Cannot load root from meta: " << entry.block_id.to_str() << " " << cell.error(); + } + + // load_known_roots is only supported by InMemory database, so it is ok to check all known roots here + auto known_roots = boc_->load_known_roots().move_as_ok(); + for (auto& root : known_roots) { + block::gen::ShardStateUnsplit::Record info; + block::gen::OutMsgQueueInfo::Record qinfo; + block::ShardId shard; + if (!(tlb::unpack_cell(root, info) && shard.deserialize(info.shard_id.write()) && + tlb::unpack_cell(info.out_msg_queue_info, qinfo))) { + LOG(FATAL) << "cannot create ShardDescr from a root in celldb"; + } + if (!partial_check && !root_hashes.contains(root->get_hash())) { + unknown_roots++; + LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no; + constexpr bool delete_unknown_roots = false; + if (delete_unknown_roots) { + vm::CellStorer stor{*cell_db_}; + cell_db_->begin_write_batch().ensure(); + boc_->dec(root); + boc_->commit(stor).ensure(); + cell_db_->commit_write_batch().ensure(); + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + } + LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no << " REMOVED"; + } + } + } + + LOG_IF(ERROR, missing_roots != 0) << "Missing root hashes: " << missing_roots; + LOG_IF(ERROR, unknown_roots != 0) << "Unknown roots: " << unknown_roots; + + LOG_IF(FATAL, missing_roots != 0) << "Missing root hashes: " << missing_roots; + LOG_IF(FATAL, unknown_roots != 0) << "Unknown roots: " << unknown_roots; + LOG(INFO) << "Validating metadata: OK\n"; +} + void CellDbIn::start_up() { on_load_callback_ = [actor = std::make_shared>( td::actor::create_actor("celldbmigration", actor_id(this))), @@ -141,11 +201,11 @@ void CellDbIn::start_up() { std::optional boc_v2_options; if (opts_->get_celldb_v2()) { - boc_v2_options = - vm::DynamicBagOfCellsDb::CreateV2Options{.extra_threads = std::clamp(std::thread::hardware_concurrency() / 2, 1u, 8u), - .executor = {}, - .cache_ttl_max = 2000, - .cache_size_max = 1000000}; + boc_v2_options = vm::DynamicBagOfCellsDb::CreateV2Options{ + .extra_threads = std::clamp(std::thread::hardware_concurrency() / 2, 1u, 8u), + .executor = {}, + .cache_ttl_max = 2000, + .cache_size_max = 1000000}; size_t min_rocksdb_cache = std::max(size_t{1} << 30, boc_v2_options->cache_size_max * 5000); if (!o_celldb_cache_size || o_celldb_cache_size.value() < min_rocksdb_cache) { LOG(WARNING) << "Increase CellDb block cache size to " << td::format::as_size(min_rocksdb_cache) << " from " @@ -208,54 +268,7 @@ void CellDbIn::start_up() { boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); } - auto meta = boc_->meta_get_all().move_as_ok(); - size_t missing_roots = 0; - size_t unknown_roots = 0; - std::set root_hashes; - for (auto [k, v] : meta) { - if (k == "desczero") { - continue; - } - auto obj = fetch_tl_object(td::BufferSlice{v}, true); - obj.ensure(); - auto entry = DbEntry{obj.move_as_ok()}; - root_hashes.insert(vm::CellHash::from_slice(entry.root_hash.as_slice())); - auto cell = boc_->load_cell(entry.root_hash.as_slice()); - missing_roots += cell.is_error(); - LOG_IF(ERROR, cell.is_error()) << "Cannot load root from meta: " << entry.block_id.to_str() << " " << cell.error(); - } - auto known_roots = boc_->load_known_roots().move_as_ok(); - for (auto& root : known_roots) { - block::gen::ShardStateUnsplit::Record info; - block::gen::OutMsgQueueInfo::Record qinfo; - block::ShardId shard; - if (!(tlb::unpack_cell(root, info) && shard.deserialize(info.shard_id.write()) && - tlb::unpack_cell(info.out_msg_queue_info, qinfo))) { - LOG(FATAL) << "cannot create ShardDescr from a root in celldb"; - } - if (!root_hashes.contains(root->get_hash())) { - unknown_roots++; - LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no; - constexpr bool delete_unknown_roots = false; - if (delete_unknown_roots) { - vm::CellStorer stor{*cell_db_}; - cell_db_->begin_write_batch().ensure(); - boc_->dec(root); - boc_->commit(stor).ensure(); - cell_db_->commit_write_batch().ensure(); - if (!opts_->get_celldb_in_memory()) { - boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); - } - LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no << " REMOVED"; - } - } - } - - LOG_IF(ERROR, missing_roots != 1) << "Missing root hashes: " << missing_roots; - LOG_IF(ERROR, unknown_roots != 0) << "Unknown roots: " << unknown_roots; - - LOG_IF(FATAL, missing_roots > 1) << "Missing root hashes: " << missing_roots; - LOG_IF(FATAL, unknown_roots != 0) << "Unknown roots: " << unknown_roots; + validate_meta(); alarm_timestamp() = td::Timestamp::in(10.0); @@ -267,6 +280,9 @@ void CellDbIn::start_up() { set_block(empty, std::move(e)); boc_->commit(stor); cell_db_->commit_write_batch().ensure(); + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + } } if (opts_->get_celldb_v2() || opts_->get_celldb_in_memory()) { diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index 8d97e13c7..1e1ccddab 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -74,6 +74,7 @@ class CellDbIn : public CellDbBase { CellDbIn(td::actor::ActorId root_db, td::actor::ActorId parent, std::string path, td::Ref opts); + void validate_meta(); void start_up() override; void alarm() override; From 57d7c2a8958de3a30a319887b18458545378d964 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 28 Feb 2025 18:00:36 +0300 Subject: [PATCH 146/388] New account storage stat --- common/global-version.h | 2 +- crypto/CMakeLists.txt | 2 + crypto/block/account-storage-stat.cpp | 172 ++++++++++++++++++++ crypto/block/account-storage-stat.h | 119 ++++++++++++++ crypto/block/block-parse.cpp | 45 +++--- crypto/block/block-parse.h | 7 - crypto/block/block.tlb | 17 +- crypto/block/create-state.cpp | 7 +- crypto/block/mc-config.h | 6 +- crypto/block/transaction.cpp | 225 +++++++++++++------------- crypto/block/transaction.h | 12 +- crypto/vm/boc.h | 8 +- crypto/vm/cells/CellSlice.cpp | 8 + crypto/vm/cells/CellSlice.h | 1 + doc/GlobalVersions.md | 7 + tonlib/tonlib/TonlibClient.cpp | 15 +- validator/impl/validate-query.cpp | 1 + 17 files changed, 478 insertions(+), 176 deletions(-) create mode 100644 crypto/block/account-storage-stat.cpp create mode 100644 crypto/block/account-storage-stat.h diff --git a/common/global-version.h b/common/global-version.h index 2308ce3e9..b54a3bdc5 100644 --- a/common/global-version.h +++ b/common/global-version.h @@ -19,6 +19,6 @@ namespace ton { // See doc/GlobalVersions.md -constexpr int SUPPORTED_VERSION = 10; +constexpr int SUPPORTED_VERSION = 11; } diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 069083381..59b43d13a 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -214,6 +214,8 @@ set(BLOCK_SOURCE block/mc-config.cpp block/output-queue-merger.cpp block/transaction.cpp + block/account-storage-stat.h + block/account-storage-stat.cpp block/precompiled-smc/PrecompiledSmartContract.cpp ${TLB_BLOCK_AUTO} diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp new file mode 100644 index 000000000..8e8b6f7e1 --- /dev/null +++ b/crypto/block/account-storage-stat.cpp @@ -0,0 +1,172 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "account-storage-stat.h" + +namespace block { + +AccountStorageStat::AccountStorageStat() : AccountStorageStat({}, {}, 0, 0) { +} + +AccountStorageStat::AccountStorageStat(Ref dict_root, std::vector> roots, + td::uint64 total_cells, td::uint64 total_bits) + : dict_(std::move(dict_root), 256), roots_(std::move(roots)), total_cells_(total_cells), total_bits_(total_bits) { +} + +AccountStorageStat::AccountStorageStat(const AccountStorageStat& other) + : AccountStorageStat(other.dict_.get_root_cell(), other.roots_, other.total_cells_, other.total_bits_) { +} + +AccountStorageStat::AccountStorageStat(AccountStorageStat&& other) + : AccountStorageStat(other.dict_.get_root_cell(), std::move(other.roots_), other.total_cells_, other.total_bits_) { + cache_ = std::move(other.cache_); +} + +AccountStorageStat& AccountStorageStat::operator=(const AccountStorageStat& other) { + dict_ = other.dict_; + total_cells_ = other.total_cells_; + total_bits_ = other.total_bits_; + roots_ = other.roots_; + cache_ = {}; + return *this; +} + +AccountStorageStat& AccountStorageStat::operator=(AccountStorageStat&& other) { + dict_ = std::move(other.dict_); + total_cells_ = other.total_cells_; + total_bits_ = other.total_bits_; + roots_ = std::move(other.roots_); + cache_ = std::move(other.cache_); + return *this; +} + +td::Result AccountStorageStat::add_root(const Ref& cell) { + roots_.push_back(cell); + return add_cell(cell); +} + +td::Status AccountStorageStat::remove_root(const Ref& cell) { + auto it = std::find_if(roots_.begin(), roots_.end(), + [&](const Ref& c) { return c->get_hash() == cell->get_hash(); }); + if (it == roots_.end()) { + return td::Status::Error(PSTRING() << "no such root " << cell->get_hash().to_hex()); + } + roots_.erase(it); + return remove_cell(cell); +} + +td::Result AccountStorageStat::replace_roots(std::vector> new_roots) { + std::vector> old_roots = roots_; + td::uint32 max_merkle_depth = 0; + for (const Ref& root : new_roots) { + if (root.is_null()) { + continue; + } + TRY_RESULT(info, add_root(root)); + max_merkle_depth = std::max(max_merkle_depth, info.max_merkle_depth); + } + for (const Ref& root : old_roots) { + TRY_STATUS(remove_root(root)); + } + return CellInfo{max_merkle_depth}; +} + +td::Result AccountStorageStat::add_cell(const Ref& cell) { + Entry& e = get_entry(cell); + ++e.refcnt; + if (e.refcnt == 0) { + return td::Status::Error(PSTRING() << "cell " << cell->get_hash().to_hex() << ": refcnt overflow"); + } + if (e.refcnt != 1) { + update_dict(e); + return CellInfo{e.max_merkle_depth}; + } + td::uint32 max_merkle_depth = 0; + vm::CellSlice cs{vm::NoVm{}, cell}; + ++total_cells_; + total_bits_ += cs.size(); + for (unsigned i = 0; i < cs.size_refs(); ++i) { + TRY_RESULT(info, add_cell(cs.prefetch_ref(i))); + max_merkle_depth = std::max(max_merkle_depth, info.max_merkle_depth); + } + if (cs.special_type() == vm::CellTraits::SpecialType::MerkleProof || + cs.special_type() == vm::CellTraits::SpecialType::MerkleUpdate) { + ++max_merkle_depth; + } + max_merkle_depth = std::min(max_merkle_depth, MERKLE_DEPTH_LIMIT); + Entry& e2 = get_entry(cell); + e2.max_merkle_depth = max_merkle_depth; + update_dict(e2); + return CellInfo{max_merkle_depth}; +} + +td::Status AccountStorageStat::remove_cell(const Ref& cell) { + Entry& e = get_entry(cell); + if (e.refcnt == 0) { + return td::Status::Error(PSTRING() << "cell " << cell->get_hash().to_hex() << " is not in the dict"); + } + --e.refcnt; + update_dict(e); + if (e.refcnt != 0) { + return td::Status::OK(); + } + vm::CellSlice cs{vm::NoVm{}, std::move(cell)}; + if (total_cells_ == 0 || total_bits_ < cs.size()) { + return td::Status::Error("total_cell/total_bits becomes negative"); + } + --total_cells_; + total_bits_ -= cs.size(); + for (unsigned i = 0; i < cs.size_refs(); ++i) { + TRY_STATUS(remove_cell(cs.prefetch_ref(i))); + } + return td::Status::OK(); +} + +bool AccountStorageStat::Entry::serialize(vm::CellBuilder& cb) const { + return cb.store_long_bool(refcnt, 32) && cb.store_long_bool(max_merkle_depth, 2); +} + +void AccountStorageStat::Entry::fetch(Ref cs) { + if (cs.is_null()) { + refcnt = max_merkle_depth = 0; + } else { + refcnt = (td::uint32)cs.write().fetch_ulong(32); + max_merkle_depth = (td::uint32)cs.write().fetch_ulong(2); + } +} + +AccountStorageStat::Entry& AccountStorageStat::get_entry(const Ref& cell) { + return cache_.apply(cell->get_hash().as_slice(), [&](Entry& e) { + if (e.inited) { + return; + } + e.inited = true; + e.hash = cell->get_hash(); + e.fetch(dict_.lookup(e.hash.as_bitslice())); + }); +} + +void AccountStorageStat::update_dict(const Entry& e) { + if (e.refcnt == 0) { + dict_.lookup_delete(e.hash.as_bitslice()); + } else { + vm::CellBuilder cb; + CHECK(e.serialize(cb)); + dict_.set_builder(e.hash.as_bitslice(), cb); + } +} + +} // namespace block \ No newline at end of file diff --git a/crypto/block/account-storage-stat.h b/crypto/block/account-storage-stat.h new file mode 100644 index 000000000..894192d4f --- /dev/null +++ b/crypto/block/account-storage-stat.h @@ -0,0 +1,119 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#pragma once +#include "common/refcnt.hpp" +#include "vm/dict.h" +#include "ton/ton-types.h" +#include "ton/ton-shard.h" +#include "common/bitstring.h" +#include "block.h" +#include "vm/db/CellHashTable.h" + +namespace block { +using td::Ref; + +class AccountStorageStat { + public: + struct CellInfo { + td::uint32 max_merkle_depth = 0; + }; + + AccountStorageStat(); + AccountStorageStat(Ref dict_root, std::vector> roots, td::uint64 total_cells, + td::uint64 total_bits); + AccountStorageStat(const AccountStorageStat &other); + AccountStorageStat(AccountStorageStat &&other); + ~AccountStorageStat() = default; + + AccountStorageStat &operator=(const AccountStorageStat &other); + AccountStorageStat &operator=(AccountStorageStat &&other); + + td::uint64 get_total_cells() const { + return total_cells_; + } + + td::uint64 get_total_bits() const { + return total_bits_; + } + + Ref get_dict_root() const { + return dict_.get_root_cell(); + } + + td::Bits256 get_dict_hash() const { + return dict_.is_empty() ? td::Bits256::zero() : td::Bits256{dict_.get_root_cell()->get_hash().bits()}; + } + + td::Result add_root(const Ref &cell); + td::Status remove_root(const Ref &cell); + td::Result replace_roots(std::vector> new_roots); + + private: + vm::Dictionary dict_; + td::uint64 total_cells_, total_bits_; + std::vector> roots_; + + td::Result add_cell(const Ref &cell); + td::Status remove_cell(const Ref &cell); + + struct Entry { + bool inited = false; + vm::Cell::Hash hash; + td::uint32 refcnt = 0; + td::uint32 max_merkle_depth = 0; + + void fetch(Ref cs); + bool serialize(vm::CellBuilder &cb) const; + + vm::Cell::Hash key() const { + return hash; + } + bool operator<(const Entry &other) const { + return key() < other.key(); + } + struct Eq { + using is_transparent = void; // Pred to use + bool operator()(const Entry &info, const Entry &other_info) const { + return info.key() == other_info.key(); + } + bool operator()(const Entry &info, td::Slice hash) const { + return info.key().as_slice() == hash; + } + bool operator()(td::Slice hash, const Entry &info) const { + return info.key().as_slice() == hash; + } + }; + struct Hash { + using is_transparent = void; // Pred to use + using transparent_key_equal = Eq; + size_t operator()(td::Slice hash) const { + return cell_hash_slice_hash(hash); + } + size_t operator()(const Entry &info) const { + return cell_hash_slice_hash(info.key().as_slice()); + } + }; + }; + vm::CellHashTable cache_; + + Entry &get_entry(const Ref &cell); + void update_dict(const Entry &e); + + static constexpr td::uint32 MERKLE_DEPTH_LIMIT = 3; +}; + +} // namespace block diff --git a/crypto/block/block-parse.cpp b/crypto/block/block-parse.cpp index 50851c795..6d645ac79 100644 --- a/crypto/block/block-parse.cpp +++ b/crypto/block/block-parse.cpp @@ -960,41 +960,34 @@ const MsgEnvelope t_MsgEnvelope; const RefTo t_Ref_MsgEnvelope; bool StorageUsed::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { - return t_VarUInteger_7.validate_skip(ops, cs, weak) // cells:(VarUInteger 7) - && t_VarUInteger_7.validate_skip(ops, cs, weak) // bits:(VarUInteger 7) - && t_VarUInteger_7.validate_skip(ops, cs, weak); // public_cells:(VarUInteger 7) -} - -bool StorageUsed::skip(vm::CellSlice& cs) const { - return t_VarUInteger_7.skip(cs) // cells:(VarUInteger 7) - && t_VarUInteger_7.skip(cs) // bits:(VarUInteger 7) - && t_VarUInteger_7.skip(cs); // public_cells:(VarUInteger 7) -} - -const StorageUsed t_StorageUsed; - -bool StorageUsedShort::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { return t_VarUInteger_7.validate_skip(ops, cs, weak) // cells:(VarUInteger 7) && t_VarUInteger_7.validate_skip(ops, cs, weak); // bits:(VarUInteger 7) } -bool StorageUsedShort::skip(vm::CellSlice& cs) const { +bool StorageUsed::skip(vm::CellSlice& cs) const { return t_VarUInteger_7.skip(cs) // cells:(VarUInteger 7) && t_VarUInteger_7.skip(cs); // bits:(VarUInteger 7) } -const StorageUsedShort t_StorageUsedShort; +const StorageUsed t_StorageUsed; const Maybe t_Maybe_Grams; bool StorageInfo::skip(vm::CellSlice& cs) const { - return t_StorageUsed.skip(cs) // used:StorageUsed - && cs.advance(32) // last_paid:uint32 - && t_Maybe_Grams.skip(cs); // due_payment:(Maybe Grams) + int extra_tag = 0; + return t_StorageUsed.skip(cs) // used:StorageUsed + && cs.fetch_uint_to(3, extra_tag) // storage_extra:StorageExtraInfo + && (extra_tag == 0 || cs.advance(256)) // storage_extra_info$001 dict_hash:uint256 = StorageExtraInfo; + && cs.advance(32) // last_paid:uint32 + && t_Maybe_Grams.skip(cs); // due_payment:(Maybe Grams) } bool StorageInfo::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { - return t_StorageUsed.validate_skip(ops, cs, weak) // used:StorageUsed + int extra_tag = 0; + return t_StorageUsed.validate_skip(ops, cs, weak) // used:StorageUsed + && cs.fetch_uint_to(3, extra_tag) // storage_extra:StorageExtraInfo + && (extra_tag == 0 || + (extra_tag == 1 && cs.advance(256))) // storage_extra_info$001 dict_hash:uint256 = StorageExtraInfo; && cs.advance(32) // last_paid:uint32 && t_Maybe_Grams.validate_skip(ops, cs, weak); // due_payment:(Maybe Grams) } @@ -1368,7 +1361,7 @@ bool TrActionPhase::skip(vm::CellSlice& cs) const { && cs.advance(16 * 4 + 256) // tot_actions:uint16 spec_actions:uint16 // skipped_actions:uint16 msgs_created:uint16 // action_list_hash:uint256 - && t_StorageUsedShort.skip(cs); // tot_msg_size:StorageUsedShort + && t_StorageUsed.skip(cs); // tot_msg_size:StorageUsed } bool TrActionPhase::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { @@ -1381,7 +1374,7 @@ bool TrActionPhase::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const && cs.advance(16 * 4 + 256) // tot_actions:uint16 spec_actions:uint16 // skipped_actions:uint16 msgs_created:uint16 // action_list_hash:uint256 - && t_StorageUsedShort.validate_skip(ops, cs, weak); // tot_msg_size:StorageUsed + && t_StorageUsed.validate_skip(ops, cs, weak); // tot_msg_size:StorageUsed } const TrActionPhase t_TrActionPhase; @@ -1392,11 +1385,11 @@ bool TrBouncePhase::skip(vm::CellSlice& cs) const { return cs.advance(2); // tr_phase_bounce_negfunds$00 case tr_phase_bounce_nofunds: return cs.advance(2) // tr_phase_bounce_nofunds$01 - && t_StorageUsedShort.skip(cs) // msg_size:StorageUsedShort + && t_StorageUsed.skip(cs) // msg_size:StorageUsed && t_Grams.skip(cs); // req_fwd_fees:Grams case tr_phase_bounce_ok: return cs.advance(1) // tr_phase_bounce_ok$1 - && t_StorageUsedShort.skip(cs) // msg_size:StorageUsedShort + && t_StorageUsed.skip(cs) // msg_size:StorageUsed && t_Grams.skip(cs) // msg_fees:Grams && t_Grams.skip(cs); // fwd_fees:Grams } @@ -1409,11 +1402,11 @@ bool TrBouncePhase::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const return cs.advance(2); // tr_phase_bounce_negfunds$00 case tr_phase_bounce_nofunds: return cs.advance(2) // tr_phase_bounce_nofunds$01 - && t_StorageUsedShort.validate_skip(ops, cs, weak) // msg_size:StorageUsedShort + && t_StorageUsed.validate_skip(ops, cs, weak) // msg_size:StorageUsed && t_Grams.validate_skip(ops, cs, weak); // req_fwd_fees:Grams case tr_phase_bounce_ok: return cs.advance(1) // tr_phase_bounce_ok$1 - && t_StorageUsedShort.validate_skip(ops, cs, weak) // msg_size:StorageUsedShort + && t_StorageUsed.validate_skip(ops, cs, weak) // msg_size:StorageUsed && t_Grams.validate_skip(ops, cs, weak) // msg_fees:Grams && t_Grams.validate_skip(ops, cs, weak); // fwd_fees:Grams } diff --git a/crypto/block/block-parse.h b/crypto/block/block-parse.h index 65f8b91fe..fd17c6579 100644 --- a/crypto/block/block-parse.h +++ b/crypto/block/block-parse.h @@ -493,13 +493,6 @@ struct StorageUsed final : TLB_Complex { extern const StorageUsed t_StorageUsed; -struct StorageUsedShort final : TLB_Complex { - bool skip(vm::CellSlice& cs) const override; - bool validate_skip(int* ops, vm::CellSlice& cs, bool weak = false) const override; -}; - -extern const StorageUsedShort t_StorageUsedShort; - struct StorageInfo final : TLB_Complex { bool skip(vm::CellSlice& cs) const override; bool validate_skip(int* ops, vm::CellSlice& cs, bool weak = false) const override; diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 4a8bbc065..5cba3c69f 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -246,14 +246,13 @@ out_msg_queue_extra#0 dispatch_queue:DispatchQueue out_queue_size:(Maybe uint48) _ out_queue:OutMsgQueue proc_info:ProcessedInfo extra:(Maybe OutMsgQueueExtra) = OutMsgQueueInfo; -// -storage_used$_ cells:(VarUInteger 7) bits:(VarUInteger 7) - public_cells:(VarUInteger 7) = StorageUsed; -storage_used_short$_ cells:(VarUInteger 7) - bits:(VarUInteger 7) = StorageUsedShort; +storage_extra_none$000 = StorageExtraInfo; +storage_extra_info$001 dict_hash:uint256 = StorageExtraInfo; + +storage_used$_ cells:(VarUInteger 7) bits:(VarUInteger 7) = StorageUsed; -storage_info$_ used:StorageUsed last_paid:uint32 +storage_info$_ used:StorageUsed storage_extra:StorageExtraInfo last_paid:uint32 due_payment:(Maybe Grams) = StorageInfo; account_none$0 = Account; @@ -341,13 +340,13 @@ tr_phase_action$_ success:Bool valid:Bool no_funds:Bool total_fwd_fees:(Maybe Grams) total_action_fees:(Maybe Grams) result_code:int32 result_arg:(Maybe int32) tot_actions:uint16 spec_actions:uint16 skipped_actions:uint16 msgs_created:uint16 - action_list_hash:bits256 tot_msg_size:StorageUsedShort + action_list_hash:bits256 tot_msg_size:StorageUsed = TrActionPhase; tr_phase_bounce_negfunds$00 = TrBouncePhase; -tr_phase_bounce_nofunds$01 msg_size:StorageUsedShort +tr_phase_bounce_nofunds$01 msg_size:StorageUsed req_fwd_fees:Grams = TrBouncePhase; -tr_phase_bounce_ok$1 msg_size:StorageUsedShort +tr_phase_bounce_ok$1 msg_size:StorageUsed msg_fees:Grams fwd_fees:Grams = TrBouncePhase; // trans_ord$0000 credit_first:Bool diff --git a/crypto/block/create-state.cpp b/crypto/block/create-state.cpp index c8c8b970d..4a74ac0f6 100644 --- a/crypto/block/create-state.cpp +++ b/crypto/block/create-state.cpp @@ -338,10 +338,11 @@ td::RefInt256 create_smartcontract(td::RefInt256 smc_addr, Ref code, R PDO(cb.store_long_rchk_bool(workchain_id, ctor == 2 ? 8 : 32) && cb.store_bits_bool(addr.cbits(), 256)); THRERR("Cannot serialize addr:MsgAddressInt of the new smart contract"); // storage_stat:StorageInfo -> storage_stat.used:StorageUsed - PDO(block::store_UInt7(cb, stats.cells) // cells:(VarUInteger 7) - && block::store_UInt7(cb, stats.bits) // bits:(VarUInteger 7) - && block::store_UInt7(cb, stats.public_cells)); // public_cells:(VarUInteger 7) + PDO(block::store_UInt7(cb, stats.cells) // cells:(VarUInteger 7) + && block::store_UInt7(cb, stats.bits)) // bits:(VarUInteger 7) THRERR("Cannot serialize used:StorageUsed of the new smart contract"); + PDO(cb.store_zeroes_bool(3)); // extra:StorageExtraInfo + THRERR("Cannot serialize storage_extra:StorageExtraInfo of the new smart contract"); PDO(cb.store_long_bool(0, 33)); // last_paid:uint32 due_payment:(Maybe Grams) PDO(cb.append_data_cell_bool(storage)); // storage:AccountStorage THRERR("Cannot create Account of the new smart contract"); diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 98e6a26df..bb4825ead 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -156,6 +156,10 @@ class McShardHashI : public td::CntObject { virtual bool before_merge() const = 0; }; +struct StorageUsed { + td::uint64 cells = 0, bits = 0; +}; + struct McShardHash : public McShardHashI { ton::BlockIdExt blk_; ton::LogicalTime start_lt_, end_lt_; @@ -336,7 +340,7 @@ struct StoragePrices { , mc_cell_price(_mc_cprice) { } static td::RefInt256 compute_storage_fees(ton::UnixTime now, const std::vector& pricing, - const vm::CellStorageStat& storage_stat, ton::UnixTime last_paid, + const StorageUsed& storage_used, ton::UnixTime last_paid, bool is_special, bool is_masterchain); }; diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 34d235114..4e92b8f6c 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -257,6 +257,11 @@ bool Account::unpack_storage_info(vm::CellSlice& cs) { return false; } last_paid = info.last_paid; + if (info.storage_extra.write().fetch_long(3) == 1) { + info.storage_extra->prefetch_bits_to(storage_dict_hash.value_force()); + } else { + storage_dict_hash = {}; + } if (info.due_payment->prefetch_ulong(1) == 1) { vm::CellSlice& cs2 = info.due_payment.write(); cs2.advance(1); @@ -268,11 +273,9 @@ bool Account::unpack_storage_info(vm::CellSlice& cs) { due_payment = td::zero_refint(); } unsigned long long u = 0; - u |= storage_stat.cells = block::tlb::t_VarUInteger_7.as_uint(*used.cells); - u |= storage_stat.bits = block::tlb::t_VarUInteger_7.as_uint(*used.bits); - u |= storage_stat.public_cells = block::tlb::t_VarUInteger_7.as_uint(*used.public_cells); - LOG(DEBUG) << "last_paid=" << last_paid << "; cells=" << storage_stat.cells << " bits=" << storage_stat.bits - << " public_cells=" << storage_stat.public_cells; + u |= storage_used.cells = block::tlb::t_VarUInteger_7.as_uint(*used.cells); + u |= storage_used.bits = block::tlb::t_VarUInteger_7.as_uint(*used.bits); + LOG(DEBUG) << "last_paid=" << last_paid << "; cells=" << storage_used.cells << " bits=" << storage_used.bits; return (u != std::numeric_limits::max()); } @@ -527,7 +530,8 @@ bool Account::init_new(ton::UnixTime now) { last_trans_hash_.set_zero(); now_ = now; last_paid = 0; - storage_stat.clear(); + storage_used = {}; + storage_dict_hash = {}; due_payment = td::zero_refint(); balance.set_zero(); if (my_addr_exact.is_null()) { @@ -617,12 +621,12 @@ bool Account::belongs_to_shard(ton::ShardIdFull shard) const { * @param payment The total sum to be updated. * @param delta The time delta for which the payment is calculated. * @param prices The storage prices. - * @param storage Account storage statistics. + * @param storage_used Account storage statistics. * @param is_mc A flag indicating whether the account is in the masterchain. */ void add_partial_storage_payment(td::BigInt256& payment, ton::UnixTime delta, const block::StoragePrices& prices, - const vm::CellStorageStat& storage, bool is_mc) { - td::BigInt256 c{(long long)storage.cells}, b{(long long)storage.bits}; + const StorageUsed& storage_used, bool is_mc) { + td::BigInt256 c{(long long)storage_used.cells}, b{(long long)storage_used.bits}; if (is_mc) { // storage.cells * prices.mc_cell_price + storage.bits * prices.mc_bit_price; c.mul_short(prices.mc_cell_price); @@ -643,7 +647,7 @@ void add_partial_storage_payment(td::BigInt256& payment, ton::UnixTime delta, co * * @param now The current Unix time. * @param pricing The vector of storage prices. - * @param storage_stat Account storage statistics. + * @param storage_used Account storage statistics. * @param last_paid The Unix time when the last payment was made. * @param is_special A flag indicating if the account is special. * @param is_masterchain A flag indicating if the account is in the masterchain. @@ -651,7 +655,7 @@ void add_partial_storage_payment(td::BigInt256& payment, ton::UnixTime delta, co * @returns The computed storage fees as RefInt256. */ td::RefInt256 StoragePrices::compute_storage_fees(ton::UnixTime now, const std::vector& pricing, - const vm::CellStorageStat& storage_stat, ton::UnixTime last_paid, + const StorageUsed& storage_used, ton::UnixTime last_paid, bool is_special, bool is_masterchain) { if (now <= last_paid || !last_paid || is_special || pricing.empty() || now <= pricing[0].valid_since) { return td::zero_refint(); @@ -669,7 +673,7 @@ td::RefInt256 StoragePrices::compute_storage_fees(ton::UnixTime now, const std:: ton::UnixTime valid_until = (i < n - 1 ? std::min(now, pricing[i + 1].valid_since) : now); if (upto < valid_until) { assert(upto >= pricing[i].valid_since); - add_partial_storage_payment(total.unique_write(), valid_until - upto, pricing[i], storage_stat, is_masterchain); + add_partial_storage_payment(total.unique_write(), valid_until - upto, pricing[i], storage_used, is_masterchain); } upto = valid_until; } @@ -685,7 +689,7 @@ td::RefInt256 StoragePrices::compute_storage_fees(ton::UnixTime now, const std:: * @returns The computed storage fees as RefInt256. */ td::RefInt256 Account::compute_storage_fees(ton::UnixTime now, const std::vector& pricing) const { - return StoragePrices::compute_storage_fees(now, pricing, storage_stat, last_paid, is_special, is_masterchain()); + return StoragePrices::compute_storage_fees(now, pricing, storage_used, last_paid, is_special, is_masterchain()); } namespace transaction { @@ -1848,7 +1852,7 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { if (S.is_error()) { // Rollback changes to state, fail action phase LOG(INFO) << "Account state size exceeded limits: " << S.move_as_error(); - new_storage_stat.clear(); + new_account_storage_stat = {}; new_code = old_code; new_data = old_data; new_library = old_library; @@ -2104,7 +2108,7 @@ int Transaction::try_action_change_library(vm::CellSlice& cs, ActionPhase& ap, c LOG(DEBUG) << "added " << ((rec.mode >> 1) ? "public" : "private") << " library with hash " << hash.to_hex(); } new_library = std::move(dict).extract_root_cell(); - } catch (vm::VmError& vme) { + } catch (vm::VmError&) { return 42; } ap.spec_actions++; @@ -2931,7 +2935,7 @@ static td::uint32 get_public_libraries_diff_count(const td::Ref& old_l * This function is not called for special accounts. * * @param size_limits The size limits configuration. - * @param update_storage_stat Store storage stat in the Transaction's CellStorageStat. + * @param update_storage_stat Store storage stat in the Transaction's AccountStorageStat. * * @returns A `td::Status` indicating the result of the check. * - If the state limits are within the allowed range, returns OK. @@ -2939,60 +2943,47 @@ static td::uint32 get_public_libraries_diff_count(const td::Ref& old_l */ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, bool update_storage_stat) { auto cell_equal = [](const td::Ref& a, const td::Ref& b) -> bool { - if (a.is_null()) { - return b.is_null(); - } - if (b.is_null()) { - return false; - } - return a->get_hash() == b->get_hash(); + return a.is_null() || b.is_null() ? a.is_null() == b.is_null() : a->get_hash() == b->get_hash(); }; if (cell_equal(account.code, new_code) && cell_equal(account.data, new_data) && cell_equal(account.library, new_library)) { return td::Status::OK(); } - vm::CellStorageStat storage_stat; - storage_stat.limit_cells = size_limits.max_acc_state_cells; - storage_stat.limit_bits = size_limits.max_acc_state_bits; + AccountStorageStat storage_stat; + if (update_storage_stat && account.account_storage_stat) { + storage_stat = account.account_storage_stat.value(); + } { TD_PERF_COUNTER(transaction_storage_stat_a); td::Timer timer; - auto add_used_storage = [&](const td::Ref& cell) -> td::Status { - if (cell.not_null()) { - TRY_RESULT(res, storage_stat.add_used_storage(cell)); - if (res.max_merkle_depth > max_allowed_merkle_depth) { - return td::Status::Error("too big merkle depth"); - } - } - return td::Status::OK(); - }; - TRY_STATUS(add_used_storage(new_code)); - TRY_STATUS(add_used_storage(new_data)); - TRY_STATUS(add_used_storage(new_library)); + TRY_RESULT(info, storage_stat.replace_roots({new_code, new_data, new_library})); + if (info.max_merkle_depth > max_allowed_merkle_depth) { + return td::Status::Error("too big Merkle depth"); + } if (timer.elapsed() > 0.1) { - LOG(INFO) << "Compute used storage took " << timer.elapsed() << "s"; + LOG(INFO) << "Compute used storage (1) took " << timer.elapsed() << "s"; } } - if (acc_status == Account::acc_active) { - storage_stat.clear_limit(); - } else { - storage_stat.clear(); - } - td::Status res; - if (storage_stat.cells > size_limits.max_acc_state_cells || storage_stat.bits > size_limits.max_acc_state_bits) { - res = td::Status::Error(PSTRING() << "account state is too big"); - } else if (account.is_masterchain() && !cell_equal(account.library, new_library) && - get_public_libraries_count(new_library) > size_limits.max_acc_public_libraries) { - res = td::Status::Error("too many public libraries"); - } else { - res = td::Status::OK(); + if (storage_stat.get_total_cells() > size_limits.max_acc_state_cells || + storage_stat.get_total_bits() > size_limits.max_acc_state_bits) { + return td::Status::Error(PSTRING() << "account state is too big: cells=" << storage_stat.get_total_cells() + << ", bits=" << storage_stat.get_total_bits() + << " (max cells=" << size_limits.max_acc_state_cells + << ", max bits=" << size_limits.max_acc_state_bits << ")"); + } + if (account.is_masterchain() && !cell_equal(account.library, new_library)) { + auto libraries_count = get_public_libraries_count(new_library); + if (libraries_count > size_limits.max_acc_public_libraries) { + return td::Status::Error(PSTRING() << "too many public libraries: " << libraries_count << " (max " + << size_limits.max_acc_public_libraries << ")"); + } } if (update_storage_stat) { // storage_stat will be reused in compute_state() - new_storage_stat = std::move(storage_stat); + new_account_storage_stat.value_force() = std::move(storage_stat); } - return res; + return td::Status::OK(); } /** @@ -3140,44 +3131,6 @@ bool Account::store_acc_status(vm::CellBuilder& cb, int acc_status) const { return cb.store_long_bool(v, 2); } -/** - * Tries to update the storage statistics based on the old storage statistics and old account state without fully recomputing it. - * - * It succeeds if only root cell of AccountStorage is changed. - * old_cs and new_cell are AccountStorage without extra currencies (if global_version >= 10). - * - * @param old_stat The old storage statistics. - * @param old_cs The old AccountStorage. - * @param new_cell The new AccountStorage. - * - * @returns An optional value of type vm::CellStorageStat. If the update is successful, it returns the new storage statistics. Otherwise, it returns an empty optional. - */ -static td::optional try_update_storage_stat(const vm::CellStorageStat& old_stat, - td::Ref old_cs, - td::Ref new_cell) { - if (old_stat.cells == 0 || old_cs.is_null()) { - return {}; - } - vm::CellSlice new_cs = vm::CellSlice(vm::NoVm(), new_cell); - if (old_cs->size_refs() != new_cs.size_refs()) { - return {}; - } - for (unsigned i = 0; i < old_cs->size_refs(); ++i) { - if (old_cs->prefetch_ref(i)->get_hash() != new_cs.prefetch_ref(i)->get_hash()) { - return {}; - } - } - if (old_stat.bits < old_cs->size()) { - return {}; - } - - vm::CellStorageStat new_stat; - new_stat.cells = old_stat.cells; - new_stat.bits = old_stat.bits - old_cs->size() + new_cs.size(); - new_stat.public_cells = old_stat.public_cells; - return new_stat; -} - /** * Removes extra currencies dict from AccountStorage. * @@ -3185,9 +3138,9 @@ static td::optional try_update_storage_stat(const vm::CellS * * @param storage_cs AccountStorage as CellSlice. * - * @returns AccountStorage without extra currencies as Cell. + * @returns AccountStorage without extra currencies as CellSlice. */ -static td::Ref storage_without_extra_currencies(td::Ref storage_cs) { +static td::Ref storage_without_extra_currencies(td::Ref storage_cs) { block::gen::AccountStorage::Record rec; if (!block::gen::csr_unpack(storage_cs, rec)) { LOG(ERROR) << "failed to unpack AccountStorage"; @@ -3205,18 +3158,20 @@ static td::Ref storage_without_extra_currencies(td::Ref return {}; } } - td::Ref cell; - if (!block::gen::pack_cell(cell, rec)) { + td::Ref result; + if (!block::gen::csr_pack(result, rec)) { LOG(ERROR) << "failed to pack AccountStorage"; return {}; } - return cell; + return result; } namespace transaction { /** * Computes the new state of the account. * + * @param cfg The configuration for the serialization phase. + * * @returns True if the state computation is successful, false otherwise. */ bool Transaction::compute_state(const SerializeConfig& cfg) { @@ -3290,45 +3245,82 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { } else { new_inner_state.clear(); } - vm::CellStorageStat& stats = new_storage_stat; + td::Ref old_storage_for_stat = account.storage; - td::Ref new_storage_for_stat = storage; + td::Ref new_storage_for_stat = new_storage; if (cfg.extra_currency_v2) { new_storage_for_stat = storage_without_extra_currencies(new_storage); if (new_storage_for_stat.is_null()) { return false; } if (old_storage_for_stat.not_null()) { - old_storage_for_stat = vm::load_cell_slice_ref(storage_without_extra_currencies(old_storage_for_stat)); + old_storage_for_stat = storage_without_extra_currencies(old_storage_for_stat); if (old_storage_for_stat.is_null()) { return false; } } + } else if (cfg.store_storage_dict_hash) { + LOG(ERROR) << "unsupported store_storage_dict_hash=true, extra_currency_v2=false"; + return false; } - auto new_stats = try_update_storage_stat(account.storage_stat, old_storage_for_stat, storage); - if (new_stats) { - stats = new_stats.unwrap(); + + bool storage_refs_changed = false; + if (old_storage_for_stat.is_null() || new_storage_for_stat->size_refs() != old_storage_for_stat->size_refs()) { + storage_refs_changed = true; } else { + for (unsigned i = 0; i < new_storage_for_stat->size_refs(); i++) { + if (new_storage_for_stat->prefetch_ref(i)->get_hash() != old_storage_for_stat->prefetch_ref(i)->get_hash()) { + storage_refs_changed = true; + break; + } + } + } + + if (storage_refs_changed || (cfg.store_storage_dict_hash && !account.storage_dict_hash)) { TD_PERF_COUNTER(transaction_storage_stat_b); td::Timer timer; - stats.add_used_storage(new_storage_for_stat).ensure(); + if (!new_account_storage_stat && account.account_storage_stat) { + new_account_storage_stat = account.account_storage_stat; + } + AccountStorageStat& stats = new_account_storage_stat.value_force(); + // Don't check Merkle depth and size here - they were checked in check_state_limits + auto S = stats.replace_roots(new_storage->prefetch_all_refs()).move_as_status(); + if (S.is_error()) { + LOG(ERROR) << "Cannot recompute storage stats for account " << account.addr.to_hex() << ": " << S.move_as_error(); + return false; + } + new_storage_dict_hash = stats.get_dict_hash(); + // Root of AccountStorage is not counted in AccountStorageStat + new_storage_used.cells = stats.get_total_cells() + 1; + new_storage_used.bits = stats.get_total_bits() + new_storage->size(); if (timer.elapsed() > 0.1) { - LOG(INFO) << "Compute used storage took " << timer.elapsed() << "s"; + LOG(INFO) << "Compute used storage (2) took " << timer.elapsed() << "s"; } - } - CHECK(cb.store_long_bool(1, 1) // account$1 - && cb.append_cellslice_bool(account.my_addr) // addr:MsgAddressInt - && block::store_UInt7(cb, stats.cells) // storage_used$_ cells:(VarUInteger 7) - && block::store_UInt7(cb, stats.bits) // bits:(VarUInteger 7) - && block::store_UInt7(cb, stats.public_cells) // public_cells:(VarUInteger 7) - && cb.store_long_bool(last_paid, 32)); // last_paid:uint32 + } else { + new_storage_used = account.storage_used; + new_storage_used.bits -= old_storage_for_stat->size(); + new_storage_used.bits += new_storage_for_stat->size(); + new_storage_dict_hash = account.storage_dict_hash; + new_account_storage_stat = account.account_storage_stat; + } + if (!cfg.store_storage_dict_hash) { + new_storage_dict_hash = {}; + } + + CHECK(cb.store_long_bool(1, 1) // account$1 + && cb.append_cellslice_bool(account.my_addr) // addr:MsgAddressInt + && block::store_UInt7(cb, new_storage_used.cells) // storage_used$_ cells:(VarUInteger 7) + && block::store_UInt7(cb, new_storage_used.bits) // bits:(VarUInteger 7) + && cb.store_long_bool(new_storage_dict_hash ? 1 : 0, 3) // extra:StorageExtraInfo + && (!new_storage_dict_hash || cb.store_bits_bool(new_storage_dict_hash.value())) // dict_hash:uint256 + && cb.store_long_bool(last_paid, 32)); // last_paid:uint32 if (due_payment.not_null() && td::sgn(due_payment) != 0) { CHECK(cb.store_long_bool(1, 1) && block::tlb::t_Grams.store_integer_ref(cb, due_payment)); // due_payment:(Maybe Grams) } else { CHECK(cb.store_long_bool(0, 1)); } - CHECK(cb.append_data_cell_bool(std::move(storage))); + CHECK(cb.append_cellslice_bool(new_storage)); new_total_state = cb.finalize(); if (verbosity > 2) { FLOG(INFO) { @@ -3345,6 +3337,8 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { * * Updates root. * + * @param cfg The configuration for the serialization. + * * @returns True if the serialization is successful, False otherwise. */ bool Transaction::serialize(const SerializeConfig& cfg) { @@ -3688,7 +3682,9 @@ Ref Transaction::commit(Account& acc) { acc.last_trans_end_lt_ = end_lt; acc.last_trans_hash_ = root->get_hash().bits(); acc.last_paid = last_paid; - acc.storage_stat = new_storage_stat; + acc.storage_used = new_storage_used; + acc.account_storage_stat = std::move(new_account_storage_stat); + acc.storage_dict_hash = new_storage_dict_hash; acc.storage = new_storage; acc.balance = std::move(balance); acc.due_payment = std::move(due_payment); @@ -3936,6 +3932,7 @@ td::Status FetchConfigParams::fetch_config_params( } { serialize_cfg->extra_currency_v2 = config.get_global_version() >= 10; + serialize_cfg->store_storage_dict_hash = config.get_global_version() >= 11; } { // fetch block_grams_created diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index 8e612e6a5..aa08719a8 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -17,6 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "account-storage-stat.h" #include "common/refcnt.hpp" #include "common/refint.h" #include "vm/cells.h" @@ -179,6 +180,7 @@ struct ActionPhaseConfig { struct SerializeConfig { bool extra_currency_v2{false}; + bool store_storage_dict_hash{false}; }; struct CreditPhase { @@ -266,8 +268,12 @@ struct Account { ton::LogicalTime last_trans_lt_; ton::Bits256 last_trans_hash_; ton::LogicalTime block_lt; + ton::UnixTime last_paid; - vm::CellStorageStat storage_stat; + StorageUsed storage_used; + td::optional storage_dict_hash; + td::optional account_storage_stat; + block::CurrencyCollection balance; td::RefInt256 due_payment; Ref orig_total_state; // ^Account @@ -377,7 +383,9 @@ struct Transaction { std::unique_ptr compute_phase; std::unique_ptr action_phase; std::unique_ptr bounce_phase; - vm::CellStorageStat new_storage_stat; + StorageUsed new_storage_used; + td::optional new_account_storage_stat; + td::optional new_storage_dict_hash; bool gas_limit_overridden{false}; Transaction(const Account& _account, int ttype, ton::LogicalTime req_start_lt, ton::UnixTime _now, Ref _inmsg = {}); diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index 17e7eb69d..3a7ddc48d 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -113,21 +113,19 @@ class NewCellStorageStat { struct CellStorageStat { unsigned long long cells; unsigned long long bits; - unsigned long long public_cells; struct CellInfo { td::uint32 max_merkle_depth = 0; }; td::HashMap seen; - CellStorageStat() : cells(0), bits(0), public_cells(0) { + CellStorageStat() : cells(0), bits(0) { } - explicit CellStorageStat(unsigned long long limit_cells) - : cells(0), bits(0), public_cells(0), limit_cells(limit_cells) { + explicit CellStorageStat(unsigned long long limit_cells) : cells(0), bits(0), limit_cells(limit_cells) { } void clear_seen() { seen.clear(); } void clear() { - cells = bits = public_cells = 0; + cells = bits = 0; clear_limit(); clear_seen(); } diff --git a/crypto/vm/cells/CellSlice.cpp b/crypto/vm/cells/CellSlice.cpp index 9cd3e931a..bea20f95d 100644 --- a/crypto/vm/cells/CellSlice.cpp +++ b/crypto/vm/cells/CellSlice.cpp @@ -773,6 +773,14 @@ bool CellSlice::prefetch_maybe_ref(Ref& res) const { } } +std::vector> CellSlice::prefetch_all_refs() const { + std::vector> res(size_refs()); + for (unsigned i = 0; i < size_refs(); ++i) { + res[i] = prefetch_ref(i); + } + return res; +} + bool CellSlice::fetch_maybe_ref(Ref& res) { auto z = prefetch_ulong(1); if (!z) { diff --git a/crypto/vm/cells/CellSlice.h b/crypto/vm/cells/CellSlice.h index ecce30f5c..7525272b5 100644 --- a/crypto/vm/cells/CellSlice.h +++ b/crypto/vm/cells/CellSlice.h @@ -190,6 +190,7 @@ class CellSlice : public td::CntObject { } bool fetch_maybe_ref(Ref& ref); bool prefetch_maybe_ref(Ref& ref) const; + std::vector> prefetch_all_refs() const; td::BitSlice fetch_bits(unsigned bits); td::BitSlice prefetch_bits(unsigned bits) const; td::Ref fetch_subslice(unsigned bits, unsigned refs = 0); diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index 77963e959..c0be0b108 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -156,3 +156,10 @@ Example: if the last masterchain block seqno is `19071` then the list contains b ### TVM changes - `SENDMSG` calculates messages size and fees without extra currencies, uses new +64 and +128 mode behavior. - `SENDMSG` does not check the number of extra currencies. + +## Version 11 +### New account storage stat +Along with the storage stat (cells and bits count), each account now stores the hash of the **storage dict**. + +**Storage dict** is the dictionary that stores refcnt for each cell in the account state. +This is required to help computing storage stats in the future, after collator-validator separation. \ No newline at end of file diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index d73e715c9..f91f2bf14 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -180,7 +180,7 @@ struct RawAccountState { td::Ref extra_currencies; ton::UnixTime storage_last_paid{0}; - vm::CellStorageStat storage_stat; + block::StorageUsed storage_used; td::Ref code; td::Ref data; @@ -1036,7 +1036,7 @@ class Query { TRY_RESULT(basechain_msg_prices, cfg->get_msg_prices(false)); block::MsgPrices* msg_prices[2] = {&basechain_msg_prices, &masterchain_msg_prices}; auto storage_fee_256 = block::StoragePrices::compute_storage_fees( - raw_.source->get_sync_time(), storage_prices, raw_.source->raw().storage_stat, + raw_.source->get_sync_time(), storage_prices, raw_.source->raw().storage_used, raw_.source->raw().storage_last_paid, false, is_masterchain); auto storage_fee = storage_fee_256.is_null() ? 0 : storage_fee_256->to_long(); @@ -1085,7 +1085,7 @@ class Query { TRY_RESULT(dest_gas_limits_prices, cfg->get_gas_limits_prices(dest_is_masterchain)); auto dest_storage_fee_256 = destination ? block::StoragePrices::compute_storage_fees( - destination->get_sync_time(), storage_prices, destination->raw().storage_stat, + destination->get_sync_time(), storage_prices, destination->raw().storage_used, destination->raw().storage_last_paid, false, is_masterchain) : td::make_refint(0); Fee dst_fee; @@ -1399,17 +1399,16 @@ class GetRawAccountState : public td::actor::Actor { return td::Status::Error("Failed to unpack StorageInfo"); } unsigned long long u = 0; - vm::CellStorageStat storage_stat; + block::StorageUsed storage_stat; u |= storage_stat.cells = block::tlb::t_VarUInteger_7.as_uint(*storage_used.cells); u |= storage_stat.bits = block::tlb::t_VarUInteger_7.as_uint(*storage_used.bits); - u |= storage_stat.public_cells = block::tlb::t_VarUInteger_7.as_uint(*storage_used.public_cells); //LOG(DEBUG) << "last_paid=" << res.storage_last_paid << "; cells=" << storage_stat.cells - //<< " bits=" << storage_stat.bits << " public_cells=" << storage_stat.public_cells; + //<< " bits=" << storage_stat.bits; if (u == std::numeric_limits::max()) { return td::Status::Error("Failed to unpack StorageStat"); } - res.storage_stat = storage_stat; + res.storage_used = storage_stat; } block::gen::AccountStorage::Record storage; @@ -2089,7 +2088,7 @@ class RunEmulator : public TonlibQueryActor { raw.balance = balance.grams->to_long(); raw.extra_currencies = balance.extra; raw.storage_last_paid = std::move(account.last_paid); - raw.storage_stat = std::move(account.storage_stat); + raw.storage_used = account.storage_used; raw.code = std::move(account.code); raw.data = std::move(account.data); raw.state = std::move(account.total_state); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 90966d820..5f2ba6615 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -1008,6 +1008,7 @@ bool ValidateQuery::fetch_config_params() { } { serialize_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; + serialize_cfg_.store_storage_dict_hash = config_->get_global_version() >= 11; } { // fetch block_grams_created From f92cdf327c1b26d3bcb0379ff2f3451505915afb Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 5 Mar 2025 16:10:50 +0300 Subject: [PATCH 147/388] Fix nullptr error in Collator::finalize_stats --- validator/impl/collator.cpp | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 400d17a63..fee6f21d1 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -6178,21 +6178,23 @@ void Collator::finalize_stats() { } else { stats_.block_id.id = new_id; } - stats_.cc_seqno = validator_set_->get_catchain_seqno(); + stats_.cc_seqno = validator_set_.not_null() ? validator_set_->get_catchain_seqno() : 0; stats_.collated_at = td::Clocks::system(); stats_.attempt = attempt_idx_; stats_.is_validator = !(mode_ & CollateMode::from_collator_node); stats_.self = stats_.is_validator ? PublicKey(pubkeys::Ed25519(created_by_)).compute_short_id() : collator_node_id_.pubkey_hash(); - stats_.estimated_bytes = block_limit_status_->estimate_block_size(); - stats_.gas = block_limit_status_->gas_used; - stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt; - stats_.estimated_collated_data_bytes = block_limit_status_->collated_data_stat.estimate_proof_size(); - stats_.cat_bytes = block_limit_status_->limits.classify_size(stats_.estimated_bytes); - stats_.cat_gas = block_limit_status_->limits.classify_gas(stats_.gas); - stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt); - stats_.cat_collated_data_bytes = - block_limit_status_->limits.classify_collated_data_size(stats_.estimated_collated_data_bytes); + if (block_limit_status_) { + stats_.estimated_bytes = block_limit_status_->estimate_block_size(); + stats_.gas = block_limit_status_->gas_used; + stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt; + stats_.estimated_collated_data_bytes = block_limit_status_->collated_data_stat.estimate_proof_size(); + stats_.cat_bytes = block_limit_status_->limits.classify_size(stats_.estimated_bytes); + stats_.cat_gas = block_limit_status_->limits.classify_gas(stats_.gas); + stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt); + stats_.cat_collated_data_bytes = + block_limit_status_->limits.classify_collated_data_size(stats_.estimated_collated_data_bytes); + } stats_.total_time = perf_timer_.elapsed(); stats_.work_time = work_time; stats_.cpu_work_time = cpu_work_time; From 928f02e6a17a4f8582678c83cf3aee348d7db29d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 5 Mar 2025 17:02:16 +0300 Subject: [PATCH 148/388] New account storage stat for accelerator --- crypto/block/account-storage-stat.cpp | 195 +++++++++++++++++--------- crypto/block/account-storage-stat.h | 21 ++- crypto/block/block.tlb | 1 + crypto/block/transaction.cpp | 176 ++++++++++++++++++----- crypto/block/transaction.h | 7 +- crypto/vm/boc.cpp | 21 +-- crypto/vm/boc.h | 10 +- crypto/vm/cells/CellUsageTree.cpp | 2 +- crypto/vm/cells/CellUsageTree.h | 4 + crypto/vm/vm.h | 5 + validator/impl/collator-impl.h | 11 ++ validator/impl/collator.cpp | 154 ++++++++++++++++++++ validator/impl/validate-query.cpp | 29 ++++ validator/impl/validate-query.hpp | 1 + 14 files changed, 512 insertions(+), 125 deletions(-) diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp index 8e8b6f7e1..f5545f8b2 100644 --- a/crypto/block/account-storage-stat.cpp +++ b/crypto/block/account-storage-stat.cpp @@ -53,51 +53,83 @@ AccountStorageStat& AccountStorageStat::operator=(AccountStorageStat&& other) { return *this; } -td::Result AccountStorageStat::add_root(const Ref& cell) { - roots_.push_back(cell); - return add_cell(cell); -} - -td::Status AccountStorageStat::remove_root(const Ref& cell) { - auto it = std::find_if(roots_.begin(), roots_.end(), - [&](const Ref& c) { return c->get_hash() == cell->get_hash(); }); - if (it == roots_.end()) { - return td::Status::Error(PSTRING() << "no such root " << cell->get_hash().to_hex()); - } - roots_.erase(it); - return remove_cell(cell); -} - td::Result AccountStorageStat::replace_roots(std::vector> new_roots) { - std::vector> old_roots = roots_; + std::erase_if(new_roots, [](const Ref& c) { return c.is_null(); }); + auto cmp = [](const Ref& c1, const Ref& c2) { return c1->get_hash() < c2->get_hash(); }; + std::sort(new_roots.begin(), new_roots.end(), cmp); + std::sort(roots_.begin(), roots_.end(), cmp); + std::vector> to_add, to_del; + std::set_difference(new_roots.begin(), new_roots.end(), roots_.begin(), roots_.end(), std::back_inserter(to_add), + cmp); + std::set_difference(roots_.begin(), roots_.end(), new_roots.begin(), new_roots.end(), std::back_inserter(to_del), + cmp); + td::uint32 max_merkle_depth = 0; - for (const Ref& root : new_roots) { - if (root.is_null()) { - continue; - } - TRY_RESULT(info, add_root(root)); + for (const Ref& root : to_add) { + TRY_RESULT(info, add_cell(root)); max_merkle_depth = std::max(max_merkle_depth, info.max_merkle_depth); } - for (const Ref& root : old_roots) { - TRY_STATUS(remove_root(root)); + for (const Ref& root : to_del) { + TRY_STATUS(remove_cell(root)); } + + roots_ = std::move(new_roots); + td::Status S = td::Status::OK(); + cache_.for_each([&](Entry& e) { + if (S.is_ok()) { + S = commit_entry(e); + } + }); + TRY_STATUS(std::move(S)); return CellInfo{max_merkle_depth}; } +void AccountStorageStat::add_hint(const td::HashSet& hint) { + td::HashSet visited; + std::function&, bool)> dfs = [&](const Ref& cell, bool is_root) { + if (!visited.insert(cell->get_hash()).second) { + return; + } + Entry& e = get_entry(cell); + e.exists = e.exists_known = true; + if (is_root) { + fetch_entry(e).ignore(); + if (e.max_merkle_depth && e.max_merkle_depth.value() != 0) { + return; + } + } + if (hint.contains(cell->get_hash())) { + bool spec; + vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); + for (unsigned i = 0; i < cs.size_refs(); ++i) { + dfs(cs.prefetch_ref(i), false); + } + } + }; + for (const Ref& root : roots_) { + dfs(root, true); + } +} + td::Result AccountStorageStat::add_cell(const Ref& cell) { Entry& e = get_entry(cell); - ++e.refcnt; - if (e.refcnt == 0) { - return td::Status::Error(PSTRING() << "cell " << cell->get_hash().to_hex() << ": refcnt overflow"); + if (!e.exists_known || e.refcnt_diff < 0) { + TRY_STATUS(fetch_entry(e)); } - if (e.refcnt != 1) { - update_dict(e); - return CellInfo{e.max_merkle_depth}; + ++e.refcnt_diff; + if (e.exists || e.refcnt_diff > 1 || (e.refcnt && e.refcnt.value() + e.refcnt_diff != 1)) { + if (!e.max_merkle_depth) { + TRY_STATUS(fetch_entry(e)); + if (!e.max_merkle_depth) { + return td::Status::Error(PSTRING() << "unexpected unknown Merkle depth of cell " << cell->get_hash()); + } + } + return CellInfo{e.max_merkle_depth.value()}; } + td::uint32 max_merkle_depth = 0; - vm::CellSlice cs{vm::NoVm{}, cell}; - ++total_cells_; - total_bits_ += cs.size(); + bool spec; + vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); for (unsigned i = 0; i < cs.size_refs(); ++i) { TRY_RESULT(info, add_cell(cs.prefetch_ref(i))); max_merkle_depth = std::max(max_merkle_depth, info.max_merkle_depth); @@ -109,64 +141,99 @@ td::Result AccountStorageStat::add_cell(const Ref< max_merkle_depth = std::min(max_merkle_depth, MERKLE_DEPTH_LIMIT); Entry& e2 = get_entry(cell); e2.max_merkle_depth = max_merkle_depth; - update_dict(e2); return CellInfo{max_merkle_depth}; } td::Status AccountStorageStat::remove_cell(const Ref& cell) { Entry& e = get_entry(cell); - if (e.refcnt == 0) { - return td::Status::Error(PSTRING() << "cell " << cell->get_hash().to_hex() << " is not in the dict"); + if (!e.exists_known) { + TRY_STATUS(fetch_entry(e)); } - --e.refcnt; - update_dict(e); - if (e.refcnt != 0) { - return td::Status::OK(); + if (!e.exists) { + return td::Status::Error(PSTRING() << "Failed to remove cell " << cell->get_hash().to_hex() + << " : does not exist in the dict"); } - vm::CellSlice cs{vm::NoVm{}, std::move(cell)}; - if (total_cells_ == 0 || total_bits_ < cs.size()) { - return td::Status::Error("total_cell/total_bits becomes negative"); + --e.refcnt_diff; + if (e.refcnt_diff < 0 && !e.refcnt) { + TRY_STATUS(fetch_entry(e)); } - --total_cells_; - total_bits_ -= cs.size(); + if (e.refcnt.value() + e.refcnt_diff != 0) { + return td::Status::OK(); + } + bool spec; + vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); for (unsigned i = 0; i < cs.size_refs(); ++i) { TRY_STATUS(remove_cell(cs.prefetch_ref(i))); } return td::Status::OK(); } -bool AccountStorageStat::Entry::serialize(vm::CellBuilder& cb) const { - return cb.store_long_bool(refcnt, 32) && cb.store_long_bool(max_merkle_depth, 2); -} - -void AccountStorageStat::Entry::fetch(Ref cs) { - if (cs.is_null()) { - refcnt = max_merkle_depth = 0; - } else { - refcnt = (td::uint32)cs.write().fetch_ulong(32); - max_merkle_depth = (td::uint32)cs.write().fetch_ulong(2); - } -} - AccountStorageStat::Entry& AccountStorageStat::get_entry(const Ref& cell) { return cache_.apply(cell->get_hash().as_slice(), [&](Entry& e) { if (e.inited) { return; } e.inited = true; - e.hash = cell->get_hash(); - e.fetch(dict_.lookup(e.hash.as_bitslice())); + e.cell = cell; }); } -void AccountStorageStat::update_dict(const Entry& e) { - if (e.refcnt == 0) { - dict_.lookup_delete(e.hash.as_bitslice()); +td::Status AccountStorageStat::fetch_entry(Entry& e) { + if (e.exists_known && e.refcnt && (!e.exists || e.max_merkle_depth)) { + return td::Status::OK(); + } + auto cs = dict_.lookup(e.cell->get_hash().as_bitslice()); + if (cs.is_null()) { + e.exists = false; + e.refcnt = 0; } else { + if (cs->size_ext() != 32 + 2) { + return td::Status::Error(PSTRING() << "invalid record for cell " << e.cell->get_hash().to_hex()); + } + e.exists = true; + e.refcnt = (td::uint32)cs.write().fetch_ulong(32); + e.max_merkle_depth = (td::uint32)cs.write().fetch_ulong(2); + if (e.refcnt.value() == 0) { + return td::Status::Error(PSTRING() << "invalid refcnt=0 for cell " << e.cell->get_hash().to_hex()); + } + } + e.exists_known = true; + return td::Status::OK(); +} + +td::Status AccountStorageStat::commit_entry(Entry& e) { + if (e.refcnt_diff == 0) { + return td::Status::OK(); + } + TRY_STATUS(fetch_entry(e)); + e.refcnt.value() += e.refcnt_diff; + e.refcnt_diff = 0; + bool spec; + if (e.refcnt.value() == 0) { + --total_cells_; + total_bits_ -= vm::load_cell_slice_special(e.cell, spec).size(); + e.exists = false; + if (dict_.lookup_delete(e.cell->get_hash().as_bitslice()).is_null()) { + return td::Status::Error(PSTRING() << "Failed to delete entry " << e.cell->get_hash().to_hex()); + } + } else { + if (!e.exists) { + ++total_cells_; + total_bits_ += vm::load_cell_slice_special(e.cell, spec).size(); + } + e.exists = true; + if (!e.max_merkle_depth) { + return td::Status::Error(PSTRING() << "Failed to store entry " << e.cell->get_hash().to_hex() + << " : unknown merkle depth"); + } vm::CellBuilder cb; - CHECK(e.serialize(cb)); - dict_.set_builder(e.hash.as_bitslice(), cb); + dict_.set_builder(e.cell->get_hash().as_bitslice(), cb); + CHECK(cb.store_long_bool(e.refcnt.value(), 32) && cb.store_long_bool(e.max_merkle_depth.value(), 2)); + if (!dict_.set_builder(e.cell->get_hash().as_bitslice(), cb)) { + return td::Status::Error(PSTRING() << "Failed to store entry " << e.cell->get_hash().to_hex()); + } } + return td::Status::OK(); } } // namespace block \ No newline at end of file diff --git a/crypto/block/account-storage-stat.h b/crypto/block/account-storage-stat.h index 894192d4f..69ac218fe 100644 --- a/crypto/block/account-storage-stat.h +++ b/crypto/block/account-storage-stat.h @@ -58,9 +58,8 @@ class AccountStorageStat { return dict_.is_empty() ? td::Bits256::zero() : td::Bits256{dict_.get_root_cell()->get_hash().bits()}; } - td::Result add_root(const Ref &cell); - td::Status remove_root(const Ref &cell); - td::Result replace_roots(std::vector> new_roots); + td::Result replace_roots(std::vector> hint); + void add_hint(const td::HashSet &visited); private: vm::Dictionary dict_; @@ -72,15 +71,14 @@ class AccountStorageStat { struct Entry { bool inited = false; - vm::Cell::Hash hash; - td::uint32 refcnt = 0; - td::uint32 max_merkle_depth = 0; - - void fetch(Ref cs); - bool serialize(vm::CellBuilder &cb) const; + Ref cell; + bool exists_known = false; + bool exists = false; + td::optional refcnt, max_merkle_depth; + td::int32 refcnt_diff = 0; vm::Cell::Hash key() const { - return hash; + return cell->get_hash(); } bool operator<(const Entry &other) const { return key() < other.key(); @@ -111,7 +109,8 @@ class AccountStorageStat { vm::CellHashTable cache_; Entry &get_entry(const Ref &cell); - void update_dict(const Entry &e); + td::Status fetch_entry(Entry &e); + td::Status commit_entry(Entry &e); static constexpr td::uint32 MERKLE_DEPTH_LIMIT = 3; }; diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index dd3b50a73..5d7f1db94 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -852,6 +852,7 @@ top_block_descr#d5 proof_for:BlockIdExt signatures:(Maybe ^BlockSignatures) // COLLATED DATA // top_block_descr_set#4ac789f3 collection:(HashmapE 96 ^TopBlockDescr) = TopBlockDescrSet; +account_storage_dict_proof#37c1e3fc proof:^Cell = AccountStorageDictProof; // // VALIDATOR MISBEHAVIOR COMPLAINTS diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 4e92b8f6c..bab126d1f 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -262,6 +262,7 @@ bool Account::unpack_storage_info(vm::CellSlice& cs) { } else { storage_dict_hash = {}; } + orig_storage_dict_hash = storage_dict_hash; if (info.due_payment->prefetch_ulong(1) == 1) { vm::CellSlice& cs2 = info.due_payment.write(); cs2.advance(1); @@ -310,8 +311,8 @@ bool Account::unpack_state(vm::CellSlice& cs) { tock = z & 1; LOG(DEBUG) << "tick=" << tick << ", tock=" << tock; } - code = state.code->prefetch_ref(); - data = state.data->prefetch_ref(); + code = orig_code = state.code->prefetch_ref(); + data = orig_data = state.data->prefetch_ref(); library = orig_library = state.library->prefetch_ref(); return true; } @@ -531,7 +532,7 @@ bool Account::init_new(ton::UnixTime now) { now_ = now; last_paid = 0; storage_used = {}; - storage_dict_hash = {}; + orig_storage_dict_hash = storage_dict_hash = {}; due_payment = td::zero_refint(); balance.set_zero(); if (my_addr_exact.is_null()) { @@ -562,6 +563,113 @@ bool Account::init_new(ton::UnixTime now) { return true; } +/** + * Removes extra currencies dict from AccountStorage. + * + * This is used for computing account storage stats. + * + * @param storage_cs AccountStorage as CellSlice. + * + * @returns AccountStorage without extra currencies as CellSlice. + */ +static td::Ref storage_without_extra_currencies(td::Ref storage_cs) { + block::gen::AccountStorage::Record rec; + if (!block::gen::csr_unpack(storage_cs, rec)) { + LOG(ERROR) << "failed to unpack AccountStorage"; + return {}; + } + if (rec.balance->size_refs() > 0) { + block::gen::CurrencyCollection::Record balance; + if (!block::gen::csr_unpack(rec.balance, balance)) { + LOG(ERROR) << "failed to unpack AccountStorage"; + return {}; + } + balance.other = vm::CellBuilder{}.store_zeroes(1).as_cellslice_ref(); + if (!block::gen::csr_pack(rec.balance, balance)) { + LOG(ERROR) << "failed to pack AccountStorage"; + return {}; + } + } + td::Ref result; + if (!block::gen::csr_pack(result, rec)) { + LOG(ERROR) << "failed to pack AccountStorage"; + return {}; + } + return result; +} + +/** + * Computes storage dict of the account from scratch. + * This requires storage_dict_hash to be set, as it guarantees that the stored storage_used was computed recently + * (in older versions it included extra currency balance, in newer versions it does not). + * + * @returns Root of the dictionary, or Error + */ +td::Result> Account::compute_account_storage_dict() const { + if (storage.is_null()) { + return td::Status::Error("cannot compute storage dict: empty storage"); + } + if (!storage_dict_hash) { + return td::Status::Error("cannot compute storage dict: storage_dict_hash is not set"); + } + AccountStorageStat stat; + auto storage_for_stat = storage_without_extra_currencies(storage); + if (storage_for_stat.is_null()) { + return td::Status::Error("cannot compute storage dict: invalid storage"); + } + TRY_STATUS(stat.replace_roots(storage_for_stat->prefetch_all_refs()).move_as_status()); + // Root of AccountStorage is not counted in AccountStorageStat + td::uint64 expected_cells = stat.get_total_cells() + 1; + td::uint64 expected_bits = stat.get_total_bits() + storage->size(); + if (expected_cells != storage_used.cells || expected_bits != storage_used.bits) { + return td::Status::Error(PSTRING() << "invalid storage_used: computed cells=" << expected_cells + << " bits=" << expected_bits << ", found cells" << storage_used.cells + << " bits=" << storage_used.bits); + } + if (storage_dict_hash.value() != stat.get_dict_hash()) { + return td::Status::Error(PSTRING() << "invalid storage dict hash: computed " << stat.get_dict_hash().to_hex() + << ", found " << storage_dict_hash.value().to_hex()); + } + return stat.get_dict_root(); +} + +/** + * Initializes account_storage_stat of the account using the existing dict_root. + * This is not strictly necessary, as the storage stat is recomputed in Transaction. + * However, it can be used to optimize cell usage. + * This requires storage_dict_hash to be set, as it guarantees that the stored storage_used was computed recently + * (in older versions it included extra currency balance, in newer versions it does not). + * + * @param dict_root Root of the storage dictionary. + * + * @returns Status of the operation. + */ +td::Status Account::init_account_storage_stat(Ref dict_root) { + if (storage.is_null()) { + if (dict_root.not_null()) { + return td::Status::Error("storage is null, but dict_root is not null"); + } + account_storage_stat = {}; + return td::Status::OK(); + } + if (!storage_dict_hash) { + return td::Status::Error("cannot init storage dict: storage_dict_hash is not set"); + } + // Root of AccountStorage is not counted in AccountStorageStat + if (storage_used.cells < 1 || storage_used.bits < storage->size()) { + return td::Status::Error(PSTRING() << "storage_used is too small: cells=" << storage_used.cells + << " bits=" << storage_used.bits << " storage_root_bits=" << storage->size()); + } + AccountStorageStat new_stat(std::move(dict_root), storage->prefetch_all_refs(), storage_used.cells - 1, + storage_used.bits - storage->size()); + if (storage_dict_hash.value() != new_stat.get_dict_hash()) { + return td::Status::Error(PSTRING() << "invalid storage dict hash: computed " << new_stat.get_dict_hash().to_hex() + << ", found " << storage_dict_hash.value().to_hex()); + } + account_storage_stat = std::move(new_stat); + return td::Status::OK(); +} + /** * Resets the split depth of the account. * @@ -1814,6 +1922,7 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { << cfg.flat_gas_limit << "]; remaining balance=" << balance.to_str(); CHECK(td::sgn(balance.grams) >= 0); } + cp.vm_loaded_cells = vm.extract_loaded_cells(); return true; } @@ -2956,6 +3065,9 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, { TD_PERF_COUNTER(transaction_storage_stat_a); td::Timer timer; + if (update_storage_stat && compute_phase) { + storage_stat.add_hint(compute_phase->vm_loaded_cells); + } TRY_RESULT(info, storage_stat.replace_roots({new_code, new_data, new_library})); if (info.max_merkle_depth > max_allowed_merkle_depth) { return td::Status::Error("too big Merkle depth"); @@ -3131,41 +3243,6 @@ bool Account::store_acc_status(vm::CellBuilder& cb, int acc_status) const { return cb.store_long_bool(v, 2); } -/** - * Removes extra currencies dict from AccountStorage. - * - * This is used for computing account storage stats. - * - * @param storage_cs AccountStorage as CellSlice. - * - * @returns AccountStorage without extra currencies as CellSlice. - */ -static td::Ref storage_without_extra_currencies(td::Ref storage_cs) { - block::gen::AccountStorage::Record rec; - if (!block::gen::csr_unpack(storage_cs, rec)) { - LOG(ERROR) << "failed to unpack AccountStorage"; - return {}; - } - if (rec.balance->size_refs() > 0) { - block::gen::CurrencyCollection::Record balance; - if (!block::gen::csr_unpack(rec.balance, balance)) { - LOG(ERROR) << "failed to unpack AccountStorage"; - return {}; - } - balance.other = vm::CellBuilder{}.store_zeroes(1).as_cellslice_ref(); - if (!block::gen::csr_pack(rec.balance, balance)) { - LOG(ERROR) << "failed to pack AccountStorage"; - return {}; - } - } - td::Ref result; - if (!block::gen::csr_pack(result, rec)) { - LOG(ERROR) << "failed to pack AccountStorage"; - return {}; - } - return result; -} - namespace transaction { /** * Computes the new state of the account. @@ -3284,6 +3361,9 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { } AccountStorageStat& stats = new_account_storage_stat.value_force(); // Don't check Merkle depth and size here - they were checked in check_state_limits + if (compute_phase) { + stats.add_hint(compute_phase->vm_loaded_cells); + } auto S = stats.replace_roots(new_storage->prefetch_all_refs()).move_as_status(); if (S.is_error()) { LOG(ERROR) << "Cannot recompute storage stats for account " << account.addr.to_hex() << ": " << S.move_as_error(); @@ -3306,6 +3386,26 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { if (!cfg.store_storage_dict_hash) { new_storage_dict_hash = {}; } + if (false) { + vm::CellStorageStat control_stats; + control_stats.add_used_storage(new_storage); + if (control_stats.bits != new_storage_used.bits || control_stats.cells != new_storage_used.cells) { + LOG(ERROR) << " [ QQQQQQ 1 Wrong storage stat " << account.workchain << ":" << account.addr.to_hex() << " " + << start_lt << " : " << new_storage_used.cells << "," << new_storage_used.bits + << " != " << control_stats.cells << "," << control_stats.bits << " ] "; + return false; + } + AccountStorageStat control_stats_2; + control_stats_2.replace_roots(new_storage->prefetch_all_refs()); + if (control_stats_2.get_total_bits() + new_storage->size() != new_storage_used.bits || + control_stats_2.get_total_cells() + 1 != new_storage_used.cells) { + LOG(ERROR) << " [ QQQQQQ 2 Wrong storage stat " << account.workchain << ":" << account.addr.to_hex() << " " + << start_lt << " : " << new_storage_used.cells << "," << new_storage_used.bits + << " != " << control_stats_2.get_total_cells() + 1 << "," + << control_stats_2.get_total_bits() + new_storage->size() << " ] "; + return false; + } + } CHECK(cb.store_long_bool(1, 1) // account$1 && cb.append_cellslice_bool(account.my_addr) // addr:MsgAddressInt diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index aa08719a8..b2f41f375 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -208,6 +208,7 @@ struct ComputePhase { Ref actions; std::string vm_log; td::optional precompiled_gas_usage; + td::HashSet vm_loaded_cells; }; struct ActionPhase { @@ -272,6 +273,7 @@ struct Account { ton::UnixTime last_paid; StorageUsed storage_used; td::optional storage_dict_hash; + td::optional orig_storage_dict_hash; td::optional account_storage_stat; block::CurrencyCollection balance; @@ -281,7 +283,8 @@ struct Account { Ref storage; // AccountStorage Ref inner_state; // StateInit ton::Bits256 state_hash; // hash of StateInit for frozen accounts - Ref code, data, library, orig_library; + Ref code, data, library; + Ref orig_code, orig_data, orig_library; std::vector transactions; Account() = default; Account(ton::WorkchainId wc, td::ConstBitPtr _addr) : workchain(wc), addr(_addr) { @@ -295,6 +298,8 @@ struct Account { bool set_address(ton::WorkchainId wc, td::ConstBitPtr new_addr); bool unpack(Ref account, ton::UnixTime now, bool special); bool init_new(ton::UnixTime now); + td::Result> compute_account_storage_dict() const; + td::Status init_account_storage_stat(Ref dict_root); bool deactivate(); bool recompute_tmp_addr(Ref& tmp_addr, int split_depth, td::ConstBitPtr orig_addr_rewrite) const; td::RefInt256 compute_storage_fees(ton::UnixTime now, const std::vector& pricing) const; diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index 72afb9988..67a443f06 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -1259,14 +1259,6 @@ bool VmStorageStat::add_storage(const CellSlice& cs) { return true; } -static td::uint64 estimate_prunned_size() { - return 41; -} - -static td::uint64 estimate_serialized_size(const Ref& cell) { - return cell->get_serialized_size() + cell->size_refs() * 3 + 3; -} - void ProofStorageStat::add_cell(const Ref& cell) { auto& status = cells_[cell->get_hash()]; if (status == c_loaded) { @@ -1290,4 +1282,17 @@ td::uint64 ProofStorageStat::estimate_proof_size() const { return proof_size_; } +ProofStorageStat::CellStatus ProofStorageStat::get_cell_status(const Cell::Hash& hash) const { + auto it = cells_.find(hash); + return it == cells_.end() ? c_none : it->second; +} + +td::uint64 ProofStorageStat::estimate_prunned_size() { + return 41; +} + +td::uint64 ProofStorageStat::estimate_serialized_size(const Ref& cell) { + return cell->get_serialized_size() + cell->size_refs() * 3 + 3; +} + } // namespace vm diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index 3a7ddc48d..64cdbc928 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -167,11 +167,17 @@ class ProofStorageStat { public: void add_cell(const Ref& cell); td::uint64 estimate_proof_size() const; - private: + enum CellStatus { c_none = 0, c_prunned = 1, c_loaded = 2 }; - td::HashMap cells_; + CellStatus get_cell_status(const Cell::Hash& hash) const; + + static td::uint64 estimate_prunned_size(); + static td::uint64 estimate_serialized_size(const Ref& cell); + + private: + td::HashMap cells_; td::uint64 proof_size_ = 0; }; diff --git a/crypto/vm/cells/CellUsageTree.cpp b/crypto/vm/cells/CellUsageTree.cpp index 410b3fcd6..3874998c2 100644 --- a/crypto/vm/cells/CellUsageTree.cpp +++ b/crypto/vm/cells/CellUsageTree.cpp @@ -112,7 +112,7 @@ void CellUsageTree::set_use_mark_for_is_loaded(bool use_mark) { } void CellUsageTree::on_load(NodeId node_id, const td::Ref& cell) { - if (nodes_[node_id].is_loaded) { + if (ignore_loads_ || nodes_[node_id].is_loaded) { return; } nodes_[node_id].is_loaded = true; diff --git a/crypto/vm/cells/CellUsageTree.h b/crypto/vm/cells/CellUsageTree.h index af0f21f53..c37b2c2d0 100644 --- a/crypto/vm/cells/CellUsageTree.h +++ b/crypto/vm/cells/CellUsageTree.h @@ -66,6 +66,9 @@ class CellUsageTree : public std::enable_shared_from_this { void set_cell_load_callback(std::function&)> f) { cell_load_callback_ = std::move(f); } + void set_ignore_loads(bool value) { + ignore_loads_ = value; + } private: struct Node { @@ -80,5 +83,6 @@ class CellUsageTree : public std::enable_shared_from_this { void on_load(NodeId node_id, const td::Ref& cell); NodeId create_node(NodeId parent); + bool ignore_loads_ = false; }; } // namespace vm diff --git a/crypto/vm/vm.h b/crypto/vm/vm.h index a171ef27e..679a72def 100644 --- a/crypto/vm/vm.h +++ b/crypto/vm/vm.h @@ -19,6 +19,7 @@ #pragma once #include "common/refcnt.hpp" +#include "td/utils/HashMap.h" #include "vm/cellslice.h" #include "vm/stack.hpp" #include "vm/vmstate.h" @@ -424,6 +425,10 @@ class VmState final : public VmStateInterface { } } + td::HashSet extract_loaded_cells() { + return std::move(loaded_cells); + } + private: void init_cregs(bool same_c3 = false, bool push_0 = true); int run_inner(); diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index ffccc4ae3..9d40c4f3c 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -220,6 +220,15 @@ class Collator final : public td::actor::Actor { std::vector neighbor_proof_builders_; std::vector> collated_roots_; + struct AccountStorageDict { + bool inited = false; + vm::MerkleProofBuilder mpb; + Ref proof_root; + size_t proof_size_estimate = 0; + bool add_to_collated_data = false; + }; + std::map account_storage_dicts_; + std::unique_ptr block_candidate; std::unique_ptr dispatch_queue_, old_dispatch_queue_; @@ -245,6 +254,7 @@ class Collator final : public td::actor::Actor { block::Account* lookup_account(td::ConstBitPtr addr) const; std::unique_ptr make_account_from(td::ConstBitPtr addr, Ref account, bool force_create); + bool init_account_storage_dict(block::Account& account); td::Result make_account(td::ConstBitPtr addr, bool force_create = false); td::actor::ActorId get_self() { return actor_id(this); @@ -344,6 +354,7 @@ class Collator final : public td::actor::Actor { bool register_dispatch_queue_op(bool force = false); bool update_account_dict_estimation(const block::transaction::Transaction& trans); bool update_min_mc_seqno(ton::BlockSeqno some_mc_seqno); + bool process_account_storage_dict(const block::Account& account); bool combine_account_transactions(); bool update_public_libraries(); bool update_account_public_libraries(Ref orig_libs, Ref final_libs, const td::Bits256& addr); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index fee6f21d1..6b7d934d3 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -2614,9 +2614,60 @@ std::unique_ptr Collator::make_account_from(td::ConstBitPtr addr return nullptr; } ptr->block_lt = start_lt; + if (!init_account_storage_dict(*ptr)) { + return nullptr; + } return ptr; } +/** + * If full collated data is enabled, initialize account storage dict and prepare MerkleProofBuilder for it + * + * @param account Account to initialize storage dict for + * @return True on success, False on failure + */ +bool Collator::init_account_storage_dict(block::Account& account) { + if (!full_collated_data_ || is_masterchain() || !account.storage_dict_hash || account.storage.is_null()) { + return true; + } + if (account.storage_used.cells < 10) { // TODO: some other threshold? + return true; + } + td::Bits256 storage_dict_hash = account.storage_dict_hash.value(); + if (storage_dict_hash.is_zero()) { + return true; + } + AccountStorageDict& dict = account_storage_dicts_[storage_dict_hash]; + if (!dict.inited) { + dict.inited = true; + // don't mark cells in account state as loaded during compute_account_storage_dict + state_usage_tree_->set_ignore_loads(true); + auto res = account.compute_account_storage_dict(); + state_usage_tree_->set_ignore_loads(false); + if (res.is_error()) { + return fatal_error(res.move_as_error_prefix(PSTRING() << "Failed to init account storage dict for " + << account.addr.to_hex() << ": ")); + } + if (res.ok().is_null()) { // Impossible if storage_dict_hash is not zero + return fatal_error(PSTRING() << "Failed to init account storage dict for " << account.addr.to_hex() + << ": dict is empty"); + } + dict.mpb = vm::MerkleProofBuilder(res.move_as_ok()); + dict.mpb.set_cell_load_callback([&](const td::Ref& cell) { + if (block_limit_status_) { + block_limit_status_->collated_data_stat.add_cell(cell); + } + }); + } + auto S = account.init_account_storage_stat(dict.mpb.root()); + if (S.is_error()) { + return fatal_error(S.move_as_error_prefix(PSTRING() << "Failed to init account storage dict for " + << account.addr.to_hex() << ": ")); + } + return true; +} + + /** * Looks up an account in the Collator's account map. * @@ -2666,12 +2717,99 @@ td::Result Collator::make_account(td::ConstBitPtr addr, bool fo return ins.first->second.get(); } +/** + * Decides whether to include storage dict proof to collated data for this account or not. + * + * @param account Account object + * + * @returns True if the operation is successful, false otherwise. + */ +bool Collator::process_account_storage_dict(const block::Account& account) { + if (!account.orig_storage_dict_hash) { + return true; + } + td::Bits256 storage_dict_hash = account.orig_storage_dict_hash.value(); + auto it = account_storage_dicts_.find(storage_dict_hash); + if (it == account_storage_dicts_.end()) { + return true; + } + CHECK(full_collated_data_ && !is_masterchain()); + AccountStorageDict& dict = it->second; + if (dict.add_to_collated_data) { + LOG(DEBUG) << "Storage dict proof of account " << account.addr.to_hex() << " : already included"; + return true; + } + + td::HashSet visited; + bool calculate_proof_size_diff = true; + td::int64 proof_size_diff = 0; + std::function&)> dfs = [&](const Ref& cell) { + if (cell.is_null() || !visited.emplace(cell->get_hash()).second) { + return; + } + auto loaded_cell = cell->load_cell().move_as_ok(); + if (calculate_proof_size_diff) { + switch (block_limit_status_->collated_data_stat.get_cell_status(cell->get_hash())) { + case vm::ProofStorageStat::c_none: + proof_size_diff += vm::ProofStorageStat::estimate_serialized_size(loaded_cell.data_cell); + break; + case vm::ProofStorageStat::c_prunned: + proof_size_diff -= vm::ProofStorageStat::estimate_prunned_size(); + proof_size_diff += vm::ProofStorageStat::estimate_serialized_size(loaded_cell.data_cell); + break; + case vm::ProofStorageStat::c_loaded: + break; + } + } + vm::CellSlice cs{std::move(loaded_cell)}; + for (unsigned i = 0; i < cs.size_refs(); ++i) { + dfs(cs.prefetch_ref(i)); + } + }; + + // Visit all cells in the original account storage to calculate collated data increase + state_usage_tree_->set_ignore_loads(true); + dfs(account.orig_code); + dfs(account.orig_data); + dfs(account.orig_library); + state_usage_tree_->set_ignore_loads(false); + + if (proof_size_diff > (td::int64)dict.proof_size_estimate) { + LOG(DEBUG) << "Storage dict proof of account " << account.addr.to_hex() + << " : account_proof_size=" << proof_size_diff << ", dict_proof_size=" << dict.proof_size_estimate + << ", include dict in collated data"; + dict.add_to_collated_data = true; + } else { + LOG(DEBUG) << "Storage dict proof of account " << account.addr.to_hex() + << " : account_proof_size=" << proof_size_diff << ", dict_proof_size=" << dict.proof_size_estimate + << ", DO NOT include dict in collated data"; + // Include account storage in collated data + calculate_proof_size_diff = false; + visited.clear(); + dfs(account.orig_code); + dfs(account.orig_data); + dfs(account.orig_library); + } + + return true; +} + /** * Combines account transactions and updates the ShardAccountBlocks and ShardAccounts. * * @returns True if the operation is successful, false otherwise. */ bool Collator::combine_account_transactions() { + for (auto& [hash, dict] : account_storage_dicts_) { + auto res = dict.mpb.extract_proof(); + if (res.is_error()) { + return fatal_error(res.move_as_error_prefix(PSTRING() << "Failed to generate proof for account storage dict " + << hash.to_hex() << ": ")); + } + dict.proof_root = res.move_as_ok(); + dict.proof_size_estimate = vm::std_boc_serialize(dict.proof_root, 31).move_as_ok().size(); + } + vm::AugmentedDictionary dict{256, block::tlb::aug_ShardAccountBlocks}; for (auto& z : accounts) { block::Account& acc = *(z.second); @@ -2755,6 +2893,9 @@ bool Collator::combine_account_transactions() { } } } + if (!process_account_storage_dict(acc)) { + return false; + } } else { if (acc.total_state->get_hash() != acc.orig_total_state->get_hash()) { return fatal_error(std::string{"total state of account "} + z.first.to_hex() + @@ -5885,6 +6026,19 @@ bool Collator::create_collated_data() { for (auto& p : proofs) { collated_roots_.push_back(std::move(p.second)); } + + // 5. Proofs for account storage dicts + for (auto& [_, dict] : account_storage_dicts_) { + if (!dict.add_to_collated_data) { + continue; + } + CHECK(dict.proof_root.not_null()); + // account_storage_dict_proof#37c1e3fc proof:^Cell = AccountStorageDictProof; + collated_roots_.push_back(vm::CellBuilder() + .store_long(block::gen::AccountStorageDictProof::cons_tag[0], 32) + .store_ref(dict.proof_root) + .finalize_novm()); + } return true; } diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 5eab8a14d..a07ef157e 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -633,6 +633,23 @@ bool ValidateQuery::extract_collated_data_from(Ref croot, int idx) { top_shard_descr_dict_ = std::make_unique(cs.prefetch_ref(), 96); return true; } + if (block::gen::t_AccountStorageDictProof.has_valid_tag(cs)) { + if (!block::gen::t_AccountStorageDictProof.validate_upto(10000, cs)) { + return reject_query("invalid AccountStorageDictProof"); + } + // account_storage_dict_proof#37c1e3fc proof:^Cell = AccountStorageDictProof; + Ref proof = cs.prefetch_ref(); + auto virt_root = vm::MerkleProof::virtualize(proof, 1); + if (virt_root.is_null()) { + return reject_query("invalid Merkle proof in AccountStorageDictProof"); + } + LOG(DEBUG) << "collated datum # " << idx << " is an AccountStorageDictProof with hash " + << virt_root->get_hash().to_hex(); + if (!virt_account_storage_dicts_.emplace(virt_root->get_hash().bits(), virt_root).second) { + return reject_query("duplicate AccountStorageDictProof"); + } + return true; + } LOG(WARNING) << "collated datum # " << idx << " has unknown type (magic " << cs.prefetch_ulong(32) << "), ignoring"; return true; } @@ -5253,6 +5270,18 @@ std::unique_ptr ValidateQuery::unpack_account(td::ConstBitPtr ad << " does not really belong to current shard"); return {}; } + if (new_acc->storage_dict_hash) { + auto it = virt_account_storage_dicts_.find(new_acc->storage_dict_hash.value()); + if (it != virt_account_storage_dicts_.end()) { + LOG(DEBUG) << "Using account storage dict proof for account " << addr.to_hex(256) + << ", hash=" << it->second->get_hash().to_hex(); + auto S = new_acc->init_account_storage_stat(it->second); + if (S.is_error()) { + reject_query(PSTRING() << "Failed to init account storage stat for account " << addr.to_hex(256), std::move(S)); + return {}; + } + } + } return new_acc; } diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 2a93c7188..e42f88548 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -170,6 +170,7 @@ class ValidateQuery : public td::actor::Actor { std::vector> collated_roots_; std::map> virt_roots_; std::unique_ptr top_shard_descr_dict_; + std::map> virt_account_storage_dicts_; Ref shard_hashes_; // from McBlockExtra Ref blk_config_params_; // from McBlockExtra From 61ef357941668b51ba7319627c91b180e317d387 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 6 Mar 2025 17:15:27 +0300 Subject: [PATCH 149/388] Request persistent state size, add missing queries to full-node-master --- tl/generate/scheme/ton_api.tl | 2 ++ tl/generate/scheme/ton_api.tlo | Bin 102484 -> 102788 bytes validator/db/archive-manager.cpp | 50 +++++++++++++++-------------- validator/db/archive-manager.hpp | 9 ++++-- validator/db/rootdb.cpp | 6 ++-- validator/db/rootdb.hpp | 4 +-- validator/full-node-master.cpp | 37 +++++++++++++++++++--- validator/full-node-master.hpp | 4 +++ validator/full-node-shard.cpp | 22 ++++++++++--- validator/full-node-shard.hpp | 2 ++ validator/interfaces/db.h | 4 +-- validator/manager-disk.cpp | 6 ++-- validator/manager-disk.hpp | 4 +-- validator/manager-hardfork.hpp | 4 +-- validator/manager.cpp | 6 ++-- validator/manager.hpp | 4 +-- validator/net/download-state.cpp | 52 ++++++++++++++++++++++++++++--- validator/net/download-state.hpp | 3 ++ validator/validator.h | 4 +-- 19 files changed, 163 insertions(+), 60 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index cfc9f3a19..14e0121fd 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -406,6 +406,7 @@ tonNode.preparedProof = tonNode.PreparedProof; tonNode.preparedProofLink = tonNode.PreparedProof; tonNode.preparedState = tonNode.PreparedState; tonNode.notFoundState = tonNode.PreparedState; +tonNode.persistentStateSize size:long = tonNode.PersistentStateSize; tonNode.prepared = tonNode.Prepared; tonNode.notFound = tonNode.Prepared; tonNode.data data:bytes = tonNode.Data; @@ -472,6 +473,7 @@ tonNode.prepareKeyBlockProofs blocks:(vector tonNode.blockIdExt) allow_partial:B tonNode.prepareBlock block:tonNode.blockIdExt = tonNode.Prepared; tonNode.prepareBlocks blocks:(vector tonNode.blockIdExt) = tonNode.Prepared; tonNode.preparePersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PreparedState; +tonNode.getPersistentStateSize block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PersistentStateSize; tonNode.prepareZeroState block:tonNode.blockIdExt = tonNode.PreparedState; tonNode.getNextKeyBlockIds block:tonNode.blockIdExt max_size:int = tonNode.KeyBlocks; tonNode.downloadNextBlockFull prev_block:tonNode.blockIdExt = tonNode.DataFull; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 96ecb7751b1ad3a31830cba56037d0037988cde3..5d9ccc2d8586fcb9da5e367372b01e3394ff8b62 100644 GIT binary patch delta 191 zcmcbzfURXQ8}Fmp`c@23puUlJwz1-+%POCwOY-yl@>5dv0#b{LGmA@7^GbqC5=&Bp zGpkY=Cm&=I-@L+Dgq?Bo=Dk*n@2jBbD8SSKlAr9b(|`rU*d8Lqn4rhlwSB7x<9r7V g-9PDWaxmM{Q%kT|$NK&7T6}-|A1THVJ;u)Mfu4->9hSFr HF|q>y03{O* diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index 8c7cde170..9b1ebf6c2 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -314,7 +314,15 @@ void ArchiveManager::register_perm_state(FileReferenceShort id) { BlockSeqno masterchain_seqno = 0; id.ref().visit(td::overloaded( [&](const fileref::PersistentStateShort &x) { masterchain_seqno = x.masterchain_seqno; }, [&](const auto &) {})); - perm_states_[{masterchain_seqno, id.hash()}] = id; + td::uint64 size; + auto r_stat = td::stat(db_root_ + "/archive/states/" + id.filename_short()); + if (r_stat.is_error()) { + LOG(WARNING) << "Cannot stat persistent state file " << id.filename_short() << " : " << r_stat.move_as_error(); + size = 0; + } else { + size = r_stat.ok().size_; + } + perm_states_[{masterchain_seqno, id.hash()}] = {.id = id, .size = size}; } void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise promise) { @@ -417,7 +425,7 @@ void ArchiveManager::get_previous_persistent_state_files( BlockSeqno mc_seqno = it->first.first; std::vector> files; while (it->first.first == mc_seqno) { - files.emplace_back(db_root_ + "/archive/states/" + it->second.filename_short(), it->second.shard()); + files.emplace_back(db_root_ + "/archive/states/" + it->second.id.filename_short(), it->second.id.shard()); if (it == perm_states_.begin()) { break; } @@ -452,15 +460,16 @@ void ArchiveManager::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt td::actor::create_actor("readfile", path, offset, max_size, 0, std::move(promise)).release(); } -void ArchiveManager::check_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { +void ArchiveManager::get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; auto hash = id.hash(); - if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) { - promise.set_result(false); + auto it = perm_states_.find({masterchain_block_id.seqno(), hash}); + if (it == perm_states_.end()) { + promise.set_error(td::Status::Error(ErrorCode::notready)); return; } - promise.set_result(true); + promise.set_result(it->second.size); } void ArchiveManager::get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, @@ -1023,15 +1032,15 @@ void ArchiveManager::persistent_state_gc(std::pair last) { int res = 0; BlockSeqno seqno = 0; - F.ref().visit(td::overloaded([&](const fileref::ZeroStateShort &) { res = 1; }, - [&](const fileref::PersistentStateShort &x) { - res = 0; - seqno = x.masterchain_seqno; - }, - [&](const auto &obj) { res = -1; })); + F.id.ref().visit(td::overloaded([&](const fileref::ZeroStateShort &) { res = 1; }, + [&](const fileref::PersistentStateShort &x) { + res = 0; + seqno = x.masterchain_seqno; + }, + [&](const auto &obj) { res = -1; })); if (res == -1) { - td::unlink(db_root_ + "/archive/states/" + F.filename_short()).ignore(); + td::unlink(db_root_ + "/archive/states/" + F.id.filename_short()).ignore(); perm_states_.erase(it); } if (res != 0) { @@ -1081,7 +1090,7 @@ void ArchiveManager::got_gc_masterchain_handle(ConstBlockHandle handle, std::pai CHECK(it != perm_states_.end()); auto &F = it->second; if (to_del) { - td::unlink(db_root_ + "/archive/states/" + F.filename_short()).ignore(); + td::unlink(db_root_ + "/archive/states/" + F.id.filename_short()).ignore(); perm_states_.erase(it); } delay_action( @@ -1202,12 +1211,7 @@ void ArchiveManager::prepare_stats(td::Promise states; for (auto &[key, file] : perm_states_) { BlockSeqno seqno = key.first; - auto r_stat = td::stat(db_root_ + "/archive/states/" + file.filename_short()); - if (r_stat.is_error()) { - LOG(WARNING) << "Cannot stat persistent state file " << file.filename_short() << " : " << r_stat.move_as_error(); - } else { - states[seqno] += r_stat.move_as_ok().size_; - } + states[seqno] += file.size; } td::StringBuilder sb; for (auto &[seqno, size] : states) { @@ -1308,7 +1312,7 @@ void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle han auto it = perm_states_.begin(); while (it != perm_states_.end()) { int res = 0; - it->second.ref().visit(td::overloaded( + it->second.id.ref().visit(td::overloaded( [&](const fileref::ZeroStateShort &x) { res = -1; }, [&](const fileref::PersistentStateShort &x) { res = x.masterchain_seqno <= masterchain_seqno ? -1 : 1; }, [&](const auto &obj) { res = 1; })); @@ -1317,7 +1321,7 @@ void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle han } else { auto it2 = it; it++; - td::unlink(db_root_ + "/archive/states/" + it2->second.filename_short()).ignore(); + td::unlink(db_root_ + "/archive/states/" + it2->second.id.filename_short()).ignore(); perm_states_.erase(it2); } } diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index d919e32ee..cd79ccc6e 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -52,7 +52,8 @@ class ArchiveManager : public td::actor::Actor { void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise); void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, td::int64 max_size, td::Promise promise); - void check_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise); + void get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise); void check_zero_state(BlockIdExt block_id, td::Promise promise); void get_previous_persistent_state_files(BlockSeqno cur_mc_seqno, td::Promise>> promise); @@ -189,7 +190,11 @@ class ArchiveManager : public td::actor::Actor { return p.key ? key_files_ : p.temp ? temp_files_ : files_; } - std::map, FileReferenceShort> perm_states_; // Mc block seqno, hash -> state + struct PermState { + FileReferenceShort id; + td::uint64 size; + }; + std::map, PermState> perm_states_; // Mc block seqno, hash -> state void load_package(PackageId seqno); void delete_package(PackageId seqno, td::Promise promise); diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 8d83e7a7d..2b370eb06 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -310,9 +310,9 @@ void RootDb::get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt mas offset, max_size, std::move(promise)); } -void RootDb::check_persistent_state_file_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { - td::actor::send_closure(archive_db_, &ArchiveManager::check_persistent_state, block_id, masterchain_block_id, +void RootDb::get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { + td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state_file_size, block_id, masterchain_block_id, std::move(promise)); } diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 52f6098e4..9e5b0b302 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -80,8 +80,8 @@ class RootDb : public Db { td::Promise promise) override; void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, td::int64 max_length, td::Promise promise) override; - void check_persistent_state_file_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override; + void get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) override; void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; void get_zero_state_file(BlockIdExt block_id, td::Promise promise) override; void check_zero_state_file_exists(BlockIdExt block_id, td::Promise promise) override; diff --git a/validator/full-node-master.cpp b/validator/full-node-master.cpp index da49f0e2e..8dc2a6815 100644 --- a/validator/full-node-master.cpp +++ b/validator/full-node-master.cpp @@ -275,20 +275,32 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_preparePersistentState &query, td::Promise promise) { - auto P = - td::PromiseCreator::lambda([SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - if (R.is_error() || !R.move_as_ok()) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { auto x = create_serialize_tl_object(); promise.set_value(std::move(x)); return; } - auto x = create_serialize_tl_object(); promise.set_value(std::move(x)); }); auto block_id = create_block_id(query.block_); auto masterchain_block_id = create_block_id(query.masterchain_block_); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::check_persistent_state_exists, block_id, + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, + masterchain_block_id, std::move(P)); +} + +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, size, std::move(R)); + promise.set_value(create_serialize_tl_object(size)); + }); + auto block_id = create_block_id(query.block_); + auto masterchain_block_id = create_block_id(query.masterchain_block_); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, masterchain_block_id, std::move(P)); } @@ -389,6 +401,21 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo ShardIdFull{masterchainId}, std::move(P)); } +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getShardArchiveInfo &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_serialize_tl_object()); + } else { + promise.set_value(create_serialize_tl_object(R.move_as_ok())); + } + }); + ShardIdFull shard_prefix = create_shard_id(query.shard_prefix_); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_id, query.masterchain_seqno_, + shard_prefix, std::move(P)); +} + void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, td::Promise promise) { td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_slice, query.archive_id_, diff --git a/validator/full-node-master.hpp b/validator/full-node-master.hpp index ce0aedd35..d6160a94e 100644 --- a/validator/full-node-master.hpp +++ b/validator/full-node-master.hpp @@ -66,6 +66,8 @@ class FullNodeMasterImpl : public FullNodeMaster { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_preparePersistentState &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadZeroState &query, @@ -80,6 +82,8 @@ class FullNodeMasterImpl : public FullNodeMaster { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getShardArchiveInfo &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, td::Promise promise); // void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareNextKeyBlockProof &query, diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index ac0eb7688..a8ddb338a 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -509,13 +509,12 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_preparePersistentState &query, td::Promise promise) { auto P = - td::PromiseCreator::lambda([SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - if (R.is_error() || !R.move_as_ok()) { + td::PromiseCreator::lambda([SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { auto x = create_serialize_tl_object(); promise.set_value(std::move(x)); return; } - auto x = create_serialize_tl_object(); promise.set_value(std::move(x)); }); @@ -523,7 +522,22 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod auto masterchain_block_id = create_block_id(query.masterchain_block_); VLOG(FULL_NODE_DEBUG) << "Got query preparePersistentState " << block_id.to_str() << " " << masterchain_block_id.to_str() << " from " << src; - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::check_persistent_state_exists, block_id, + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, + masterchain_block_id, std::move(P)); +} + +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, size, std::move(R)); + promise.set_value(create_serialize_tl_object(size)); + }); + auto block_id = create_block_id(query.block_); + auto masterchain_block_id = create_block_id(query.masterchain_block_); + VLOG(FULL_NODE_DEBUG) << "Got query getPersistentStateSize " << block_id.to_str() << " " + << masterchain_block_id.to_str() << " from " << src; + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, masterchain_block_id, std::move(P)); } diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index fb3eef769..8cbd614aa 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -123,6 +123,8 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_preparePersistentState &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadZeroState &query, diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index 29ef715b3..9a2eea56e 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -62,8 +62,8 @@ class Db : public td::actor::Actor { td::Promise promise) = 0; virtual void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, td::int64 max_length, td::Promise promise) = 0; - virtual void check_persistent_state_file_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) = 0; + virtual void get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) = 0; virtual void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) = 0; virtual void get_zero_state_file(BlockIdExt block_id, td::Promise promise) = 0; virtual void check_zero_state_file_exists(BlockIdExt block_id, td::Promise promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 62fdc4b43..8e4a4d086 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -200,9 +200,9 @@ void ValidatorManagerImpl::get_zero_state(BlockIdExt block_id, td::Promise promise) { - td::actor::send_closure(db_, &Db::check_persistent_state_file_exists, block_id, masterchain_block_id, +void ValidatorManagerImpl::get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { + td::actor::send_closure(db_, &Db::get_persistent_state_file_size, block_id, masterchain_block_id, std::move(promise)); } void ValidatorManagerImpl::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index cd06bf555..62f54b2e9 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -108,8 +108,8 @@ class ValidatorManagerImpl : public ValidatorManager { void get_block_data(BlockHandle handle, td::Promise promise) override; void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) override; void get_zero_state(BlockIdExt block_id, td::Promise promise) override; - void check_persistent_state_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override; + void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) override; void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) override; void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 0b8b9e736..486c185f7 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -128,8 +128,8 @@ class ValidatorManagerImpl : public ValidatorManager { void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) override { UNREACHABLE(); } - void check_persistent_state_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override { + void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) override { UNREACHABLE(); } void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, diff --git a/validator/manager.cpp b/validator/manager.cpp index b0ac54092..b1dc4cc9a 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -304,9 +304,9 @@ void ValidatorManagerImpl::get_zero_state(BlockIdExt block_id, td::Promise promise) { - td::actor::send_closure(db_, &Db::check_persistent_state_file_exists, block_id, masterchain_block_id, +void ValidatorManagerImpl::get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { + td::actor::send_closure(db_, &Db::get_persistent_state_file_size, block_id, masterchain_block_id, std::move(promise)); } void ValidatorManagerImpl::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, diff --git a/validator/manager.hpp b/validator/manager.hpp index 418deb350..347b8f7ef 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -377,8 +377,8 @@ class ValidatorManagerImpl : public ValidatorManager { void get_block_data(BlockHandle handle, td::Promise promise) override; void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) override; void get_zero_state(BlockIdExt block_id, td::Promise promise) override; - void check_persistent_state_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override; + void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) override; void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) override; void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, diff --git a/validator/net/download-state.cpp b/validator/net/download-state.cpp index 6735a2b5f..c481adbbf 100644 --- a/validator/net/download-state.cpp +++ b/validator/net/download-state.cpp @@ -168,6 +168,7 @@ void DownloadState::got_block_state_description(td::BufferSlice data) { }, [&, self = this](ton_api::tonNode_preparedState &f) { if (masterchain_block_id_.is_valid()) { + request_total_size(); got_block_state_part(td::BufferSlice{}, 0); return; } @@ -190,8 +191,37 @@ void DownloadState::got_block_state_description(td::BufferSlice data) { create_serialize_tl_object_suffix(std::move(query)), td::Timestamp::in(3.0), std::move(P)); } + status_.set_status(PSTRING() << block_id_.id.to_str() << " : download started"); })); - status_.set_status(PSTRING() << block_id_.id.to_str() << " : 0 bytes, 0B/s"); +} + +void DownloadState::request_total_size() { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + return; + } + auto res = fetch_tl_object(R.move_as_ok(), true); + if (res.is_error()) { + return; + } + td::actor::send_closure(SelfId, &DownloadState::got_total_size, res.ok()->size_); + }); + + td::BufferSlice query = create_serialize_tl_object( + create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_)); + if (client_.empty()) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, download_from_, local_id_, overlay_id_, + "get size", std::move(P), td::Timestamp::in(3.0), std::move(query), + FullNode::max_state_size(), rldp_); + } else { + td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "get size", + create_serialize_tl_object_suffix(std::move(query)), + td::Timestamp::in(3.0), std::move(P)); + } +} + +void DownloadState::got_total_size(td::uint64 size) { + total_size_ = size; } void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 requested_size) { @@ -203,10 +233,22 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques if (elapsed > 5.0) { prev_logged_timer_ = td::Timer(); auto speed = (td::uint64)((double)(sum_ - prev_logged_sum_) / elapsed); - LOG(WARNING) << "downloading state " << block_id_.to_str() << ": " << td::format::as_size(sum_) << " (" - << td::format::as_size(speed) << "/s)"; - status_.set_status(PSTRING() << block_id_.id.to_str() << " : " << sum_ << " bytes, " << td::format::as_size(speed) - << "/s"); + td::StringBuilder sb; + sb << td::format::as_size(sum_); + if (total_size_) { + sb << "/" << td::format::as_size(total_size_); + } + sb << " (" << td::format::as_size(speed) << "/s"; + if (total_size_) { + sb << ", " << td::StringBuilder::FixedDouble((double)sum_ / (double)total_size_ * 100.0, 2) << "%"; + if (speed > 0 && total_size_ >= sum_) { + td::uint64 rem = (total_size_ - sum_) / speed; + sb << ", " << rem << "s remaining"; + } + } + sb << ")"; + LOG(WARNING) << "downloading state " << block_id_.to_str() << " : " << sb.as_cslice(); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : " << sb.as_cslice()); prev_logged_sum_ = sum_; } diff --git a/validator/net/download-state.hpp b/validator/net/download-state.hpp index 470c54318..29854ce27 100644 --- a/validator/net/download-state.hpp +++ b/validator/net/download-state.hpp @@ -49,6 +49,8 @@ class DownloadState : public td::actor::Actor { void got_block_handle(BlockHandle handle); void got_node_to_download(adnl::AdnlNodeIdShort node); void got_block_state_description(td::BufferSlice data_description); + void request_total_size(); + void got_total_size(td::uint64 size); void got_block_state_part(td::BufferSlice data, td::uint32 requested_size); void got_block_state(td::BufferSlice data); @@ -77,6 +79,7 @@ class DownloadState : public td::actor::Actor { td::uint64 prev_logged_sum_ = 0; td::Timer prev_logged_timer_; + td::uint64 total_size_ = 0; ProcessStatus status_; }; diff --git a/validator/validator.h b/validator/validator.h index 5d6c0173c..eecb30c07 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -227,8 +227,8 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_block_data(BlockHandle handle, td::Promise promise) = 0; virtual void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) = 0; virtual void get_zero_state(BlockIdExt block_id, td::Promise promise) = 0; - virtual void check_persistent_state_exists(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) = 0; + virtual void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) = 0; virtual void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, From e0aa7f0f64a044692fb8c53a68a28cb6ff5cd9a0 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 11 Mar 2025 15:59:25 +0300 Subject: [PATCH 150/388] Improve estimating account proof size --- crypto/block/transaction.cpp | 17 +++++++----- crypto/block/transaction.h | 3 ++- validator/impl/collator-impl.h | 2 ++ validator/impl/collator.cpp | 49 +++++++++++++++++++++++++++------- 4 files changed, 54 insertions(+), 17 deletions(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index bab126d1f..8cdf261aa 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -3044,13 +3044,13 @@ static td::uint32 get_public_libraries_diff_count(const td::Ref& old_l * This function is not called for special accounts. * * @param size_limits The size limits configuration. - * @param update_storage_stat Store storage stat in the Transaction's AccountStorageStat. + * @param is_account_stat Store storage stat in the Transaction's AccountStorageStat. * * @returns A `td::Status` indicating the result of the check. * - If the state limits are within the allowed range, returns OK. * - If the state limits exceed the maximum allowed range, returns an error. */ -td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, bool update_storage_stat) { +td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, bool is_account_stat) { auto cell_equal = [](const td::Ref& a, const td::Ref& b) -> bool { return a.is_null() || b.is_null() ? a.is_null() == b.is_null() : a->get_hash() == b->get_hash(); }; @@ -3059,13 +3059,13 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, return td::Status::OK(); } AccountStorageStat storage_stat; - if (update_storage_stat && account.account_storage_stat) { + if (is_account_stat && account.account_storage_stat) { storage_stat = account.account_storage_stat.value(); } { TD_PERF_COUNTER(transaction_storage_stat_a); td::Timer timer; - if (update_storage_stat && compute_phase) { + if (is_account_stat && compute_phase) { storage_stat.add_hint(compute_phase->vm_loaded_cells); } TRY_RESULT(info, storage_stat.replace_roots({new_code, new_data, new_library})); @@ -3091,9 +3091,12 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, << size_limits.max_acc_public_libraries << ")"); } } - if (update_storage_stat) { + if (is_account_stat) { // storage_stat will be reused in compute_state() new_account_storage_stat.value_force() = std::move(storage_stat); + storage_stats_updates.push_back(new_code); + storage_stats_updates.push_back(new_data); + storage_stats_updates.push_back(new_library); } return td::Status::OK(); } @@ -3364,7 +3367,9 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { if (compute_phase) { stats.add_hint(compute_phase->vm_loaded_cells); } - auto S = stats.replace_roots(new_storage->prefetch_all_refs()).move_as_status(); + auto roots = new_storage->prefetch_all_refs(); + storage_stats_updates.insert(storage_stats_updates.end(), roots.begin(), roots.end()); + auto S = stats.replace_roots(std::move(roots)).move_as_status(); if (S.is_error()) { LOG(ERROR) << "Cannot recompute storage stats for account " << account.addr.to_hex() << ": " << S.move_as_error(); return false; diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index b2f41f375..30b2604db 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -392,6 +392,7 @@ struct Transaction { td::optional new_account_storage_stat; td::optional new_storage_dict_hash; bool gas_limit_overridden{false}; + std::vector> storage_stats_updates; Transaction(const Account& _account, int ttype, ton::LogicalTime req_start_lt, ton::UnixTime _now, Ref _inmsg = {}); bool unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* cfg); @@ -405,7 +406,7 @@ struct Transaction { bool run_precompiled_contract(const ComputePhaseConfig& cfg, precompiled::PrecompiledSmartContract& precompiled); bool prepare_compute_phase(const ComputePhaseConfig& cfg); bool prepare_action_phase(const ActionPhaseConfig& cfg); - td::Status check_state_limits(const SizeLimitsConfig& size_limits, bool update_storage_stat = true); + td::Status check_state_limits(const SizeLimitsConfig& size_limits, bool is_account_stat = true); bool prepare_bounce_phase(const ActionPhaseConfig& cfg); bool compute_state(const SerializeConfig& cfg); bool serialize(const SerializeConfig& cfg); diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 9d40c4f3c..b02fbedc3 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -226,6 +226,7 @@ class Collator final : public td::actor::Actor { Ref proof_root; size_t proof_size_estimate = 0; bool add_to_collated_data = false; + std::vector> storage_stat_updates; }; std::map account_storage_dicts_; @@ -353,6 +354,7 @@ class Collator final : public td::actor::Actor { bool register_out_msg_queue_op(bool force = false); bool register_dispatch_queue_op(bool force = false); bool update_account_dict_estimation(const block::transaction::Transaction& trans); + void update_account_storage_dict_info(const block::transaction::Transaction& trans); bool update_min_mc_seqno(ton::BlockSeqno some_mc_seqno); bool process_account_storage_dict(const block::Account& account); bool combine_account_transactions(); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 6b7d934d3..00699997c 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -2739,16 +2739,23 @@ bool Collator::process_account_storage_dict(const block::Account& account) { LOG(DEBUG) << "Storage dict proof of account " << account.addr.to_hex() << " : already included"; return true; } + if (dict.storage_stat_updates.empty()) { + LOG(DEBUG) << "Storage dict proof of account " << account.addr.to_hex() << " : not required (no storage updates)"; + return true; + } td::HashSet visited; bool calculate_proof_size_diff = true; td::int64 proof_size_diff = 0; - std::function&)> dfs = [&](const Ref& cell) { + std::function&, bool)> dfs = [&](const Ref& cell, bool from_old_state) { if (cell.is_null() || !visited.emplace(cell->get_hash()).second) { return; } auto loaded_cell = cell->load_cell().move_as_ok(); - if (calculate_proof_size_diff) { + if (!loaded_cell.tree_node.empty()) { + from_old_state = true; + } + if (calculate_proof_size_diff && from_old_state) { switch (block_limit_status_->collated_data_stat.get_cell_status(cell->get_hash())) { case vm::ProofStorageStat::c_none: proof_size_diff += vm::ProofStorageStat::estimate_serialized_size(loaded_cell.data_cell); @@ -2763,15 +2770,15 @@ bool Collator::process_account_storage_dict(const block::Account& account) { } vm::CellSlice cs{std::move(loaded_cell)}; for (unsigned i = 0; i < cs.size_refs(); ++i) { - dfs(cs.prefetch_ref(i)); + dfs(cs.prefetch_ref(i), from_old_state); } }; - // Visit all cells in the original account storage to calculate collated data increase + // Visit cells that were used in storage stat computation to calculate collated data increase state_usage_tree_->set_ignore_loads(true); - dfs(account.orig_code); - dfs(account.orig_data); - dfs(account.orig_library); + for (const auto& cell : dict.storage_stat_updates) { + dfs(cell, false); + } state_usage_tree_->set_ignore_loads(false); if (proof_size_diff > (td::int64)dict.proof_size_estimate) { @@ -2786,9 +2793,9 @@ bool Collator::process_account_storage_dict(const block::Account& account) { // Include account storage in collated data calculate_proof_size_diff = false; visited.clear(); - dfs(account.orig_code); - dfs(account.orig_data); - dfs(account.orig_library); + for (const auto& cell : dict.storage_stat_updates) { + dfs(cell, false); + } } return true; @@ -3068,6 +3075,7 @@ bool Collator::create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, t if (!update_account_dict_estimation(*trans)) { return fatal_error(-666, "cannot update account dict size estimation"); } + update_account_storage_dict_info(*trans); update_max_lt(acc->last_trans_end_lt_); block::MsgMetadata new_msg_metadata{0, acc->workchain, acc->addr, trans->start_lt}; register_new_msgs(*trans, std::move(new_msg_metadata)); @@ -3165,6 +3173,7 @@ Ref Collator::create_ordinary_transaction(Ref msg_root, fatal_error("cannot update account dict size estimation"); return {}; } + update_account_storage_dict_info(*trans); td::optional new_msg_metadata; if (external || is_special_tx) { @@ -5456,6 +5465,26 @@ bool Collator::update_account_dict_estimation(const block::transaction::Transact return true; } +/** + * Update `storage_stat_updates` for AccountStorageDict for the new transaction. + * + * @param trans Newly-created transaction. + */ +void Collator::update_account_storage_dict_info(const block::transaction::Transaction& trans) { + if (!trans.account.orig_storage_dict_hash) { + return; + } + auto it = account_storage_dicts_.find(trans.account.orig_storage_dict_hash.value()); + if (it == account_storage_dicts_.end()) { + return; + } + for (const Ref& cell : trans.storage_stats_updates) { + if (cell.not_null()) { + it->second.storage_stat_updates.push_back(cell); + } + } +} + /** * Creates a new shard state and the Merkle update. * From ad123c4879ef2ebf25eeb1fc0a058d450c9c8f36 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 13 Mar 2025 11:42:07 +0300 Subject: [PATCH 151/388] Rework estimating and generating collated data --- crypto/block/account-storage-stat.h | 12 +++ crypto/block/block.cpp | 7 +- crypto/block/block.h | 4 +- crypto/block/transaction.cpp | 35 +++------ crypto/vm/boc.cpp | 30 +++++--- crypto/vm/boc.h | 12 +-- crypto/vm/cells/CellUsageTree.h | 8 +- crypto/vm/cells/MerkleProof.h | 3 + validator/impl/collator-impl.h | 9 ++- validator/impl/collator.cpp | 113 +++++++++++++++++----------- 10 files changed, 142 insertions(+), 91 deletions(-) diff --git a/crypto/block/account-storage-stat.h b/crypto/block/account-storage-stat.h index 69ac218fe..ed3f58d82 100644 --- a/crypto/block/account-storage-stat.h +++ b/crypto/block/account-storage-stat.h @@ -115,4 +115,16 @@ class AccountStorageStat { static constexpr td::uint32 MERKLE_DEPTH_LIMIT = 3; }; +class StorageStatCalculationContext : public td::Context { + public: + explicit StorageStatCalculationContext(bool active) : active_(active) { + } + bool calculating_storage_stat() const { + return active_; + } + + private: + bool active_ = false; +}; + } // namespace block diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp index eb5640757..d06e7a40c 100644 --- a/crypto/block/block.cpp +++ b/crypto/block/block.cpp @@ -751,14 +751,13 @@ td::uint64 BlockLimitStatus::estimate_block_size(const vm::NewCellStorageStat::S } int BlockLimitStatus::classify() const { - return limits.classify(estimate_block_size(), gas_used, cur_lt, collated_data_stat.estimate_proof_size()); + return limits.classify(estimate_block_size(), gas_used, cur_lt, collated_data_size_estimate); } bool BlockLimitStatus::fits(unsigned cls) const { return cls >= ParamLimits::limits_cnt || (limits.gas.fits(cls, gas_used) && limits.lt_delta.fits(cls, cur_lt - limits.start_lt) && - limits.bytes.fits(cls, estimate_block_size()) && - limits.collated_data.fits(cls, collated_data_stat.estimate_proof_size())); + limits.bytes.fits(cls, estimate_block_size()) && limits.collated_data.fits(cls, collated_data_size_estimate)); } bool BlockLimitStatus::would_fit(unsigned cls, ton::LogicalTime end_lt, td::uint64 more_gas, @@ -766,7 +765,7 @@ bool BlockLimitStatus::would_fit(unsigned cls, ton::LogicalTime end_lt, td::uint return cls >= ParamLimits::limits_cnt || (limits.gas.fits(cls, gas_used + more_gas) && limits.lt_delta.fits(cls, std::max(cur_lt, end_lt) - limits.start_lt) && limits.bytes.fits(cls, estimate_block_size(extra)) && - limits.collated_data.fits(cls, collated_data_stat.estimate_proof_size())); + limits.collated_data.fits(cls, collated_data_size_estimate)); } // SETS: account_dict, shard_libraries_, mc_state_extra diff --git a/crypto/block/block.h b/crypto/block/block.h index c6c988cdd..65834d79e 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -280,7 +280,7 @@ struct BlockLimitStatus { td::uint64 gas_used{}; vm::NewCellStorageStat st_stat; unsigned accounts{}, transactions{}, extra_out_msgs{}; - vm::ProofStorageStat collated_data_stat; + td::uint64 collated_data_size_estimate = 0; unsigned public_library_diff{}; BlockLimitStatus(const BlockLimits& limits_, ton::LogicalTime lt = 0) : limits(limits_), cur_lt(std::max(limits_.start_lt, lt)) { @@ -292,7 +292,7 @@ struct BlockLimitStatus { gas_used = 0; extra_out_msgs = 0; public_library_diff = 0; - collated_data_stat = {}; + collated_data_size_estimate = 0; } td::uint64 estimate_block_size(const vm::NewCellStorageStat::Stat* extra = nullptr) const; int classify() const; diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 8cdf261aa..c5310933b 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -3068,6 +3068,8 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, if (is_account_stat && compute_phase) { storage_stat.add_hint(compute_phase->vm_loaded_cells); } + StorageStatCalculationContext context{is_account_stat}; + StorageStatCalculationContext::Guard guard{&context}; TRY_RESULT(info, storage_stat.replace_roots({new_code, new_data, new_library})); if (info.max_merkle_depth > max_allowed_merkle_depth) { return td::Status::Error("too big Merkle depth"); @@ -3369,10 +3371,15 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { } auto roots = new_storage->prefetch_all_refs(); storage_stats_updates.insert(storage_stats_updates.end(), roots.begin(), roots.end()); - auto S = stats.replace_roots(std::move(roots)).move_as_status(); - if (S.is_error()) { - LOG(ERROR) << "Cannot recompute storage stats for account " << account.addr.to_hex() << ": " << S.move_as_error(); - return false; + { + StorageStatCalculationContext context{true}; + StorageStatCalculationContext::Guard guard{&context}; + auto S = stats.replace_roots(std::move(roots)).move_as_status(); + if (S.is_error()) { + LOG(ERROR) << "Cannot recompute storage stats for account " << account.addr.to_hex() << ": " + << S.move_as_error(); + return false; + } } new_storage_dict_hash = stats.get_dict_hash(); // Root of AccountStorage is not counted in AccountStorageStat @@ -3391,26 +3398,6 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { if (!cfg.store_storage_dict_hash) { new_storage_dict_hash = {}; } - if (false) { - vm::CellStorageStat control_stats; - control_stats.add_used_storage(new_storage); - if (control_stats.bits != new_storage_used.bits || control_stats.cells != new_storage_used.cells) { - LOG(ERROR) << " [ QQQQQQ 1 Wrong storage stat " << account.workchain << ":" << account.addr.to_hex() << " " - << start_lt << " : " << new_storage_used.cells << "," << new_storage_used.bits - << " != " << control_stats.cells << "," << control_stats.bits << " ] "; - return false; - } - AccountStorageStat control_stats_2; - control_stats_2.replace_roots(new_storage->prefetch_all_refs()); - if (control_stats_2.get_total_bits() + new_storage->size() != new_storage_used.bits || - control_stats_2.get_total_cells() + 1 != new_storage_used.cells) { - LOG(ERROR) << " [ QQQQQQ 2 Wrong storage stat " << account.workchain << ":" << account.addr.to_hex() << " " - << start_lt << " : " << new_storage_used.cells << "," << new_storage_used.bits - << " != " << control_stats_2.get_total_cells() + 1 << "," - << control_stats_2.get_total_bits() + new_storage->size() << " ] "; - return false; - } - } CHECK(cb.store_long_bool(1, 1) // account$1 && cb.append_cellslice_bool(account.my_addr) // addr:MsgAddressInt diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index 67a443f06..073f26466 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -1259,32 +1259,44 @@ bool VmStorageStat::add_storage(const CellSlice& cs) { return true; } -void ProofStorageStat::add_cell(const Ref& cell) { - auto& status = cells_[cell->get_hash()]; +void ProofStorageStat::add_loaded_cell(const Ref& cell) { + auto& [status, size] = cells_[cell->get_hash()]; if (status == c_loaded) { return; } - if (status == c_prunned) { - proof_size_ -= estimate_prunned_size(); - } + proof_size_ -= size; status = c_loaded; - proof_size_ += estimate_serialized_size(cell); + proof_size_ += size = estimate_serialized_size(cell); for (unsigned i = 0; i < cell->size_refs(); ++i) { - auto& child_status = cells_[cell->get_ref(i)->get_hash()]; + auto& [child_status, child_size] = cells_[cell->get_ref(i)->get_hash()]; if (child_status == c_none) { child_status = c_prunned; - proof_size_ += estimate_prunned_size(); + proof_size_ += child_size = estimate_prunned_size(); + } + } +} + +void ProofStorageStat::add_loaded_cells(const ProofStorageStat& other) { + for (const auto& [hash, x] : other.cells_) { + const auto& [new_status, new_size] = x; + auto& [old_status, old_size] = cells_[hash]; + if (old_status >= new_status) { + continue; } + proof_size_ -= old_size; + old_status = new_status; + proof_size_ += old_size = new_size; } } + td::uint64 ProofStorageStat::estimate_proof_size() const { return proof_size_; } ProofStorageStat::CellStatus ProofStorageStat::get_cell_status(const Cell::Hash& hash) const { auto it = cells_.find(hash); - return it == cells_.end() ? c_none : it->second; + return it == cells_.end() ? c_none : it->second.first; } td::uint64 ProofStorageStat::estimate_prunned_size() { diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index 64cdbc928..59a984c1c 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -165,19 +165,21 @@ struct VmStorageStat { class ProofStorageStat { public: - void add_cell(const Ref& cell); + void add_loaded_cell(const Ref& cell); + void add_loaded_cells(const ProofStorageStat& other); td::uint64 estimate_proof_size() const; - enum CellStatus { - c_none = 0, c_prunned = 1, c_loaded = 2 - }; + enum CellStatus { c_none = 0, c_prunned = 1, c_loaded = 2 }; CellStatus get_cell_status(const Cell::Hash& hash) const; + bool is_loaded(const Cell::Hash& hash) const { + return get_cell_status(hash) == c_loaded; + } static td::uint64 estimate_prunned_size(); static td::uint64 estimate_serialized_size(const Ref& cell); private: - td::HashMap cells_; + td::HashMap> cells_; td::uint64 proof_size_ = 0; }; diff --git a/crypto/vm/cells/CellUsageTree.h b/crypto/vm/cells/CellUsageTree.h index c37b2c2d0..d845be10d 100644 --- a/crypto/vm/cells/CellUsageTree.h +++ b/crypto/vm/cells/CellUsageTree.h @@ -67,7 +67,11 @@ class CellUsageTree : public std::enable_shared_from_this { cell_load_callback_ = std::move(f); } void set_ignore_loads(bool value) { - ignore_loads_ = value; + if (value) { + ++ignore_loads_; + } else { + --ignore_loads_; + } } private: @@ -83,6 +87,6 @@ class CellUsageTree : public std::enable_shared_from_this { void on_load(NodeId node_id, const td::Ref& cell); NodeId create_node(NodeId parent); - bool ignore_loads_ = false; + int ignore_loads_ = 0; }; } // namespace vm diff --git a/crypto/vm/cells/MerkleProof.h b/crypto/vm/cells/MerkleProof.h index fc2cb6ebd..96a31d64e 100644 --- a/crypto/vm/cells/MerkleProof.h +++ b/crypto/vm/cells/MerkleProof.h @@ -63,6 +63,9 @@ class MerkleProofBuilder { Ref root() const { return usage_root; } + Ref original_root() const { + return orig_root; + } td::Result> extract_proof() const; bool extract_proof_to(Ref &proof_root) const; td::Result extract_proof_boc() const; diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index b02fbedc3..6e5f876ee 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -185,6 +185,7 @@ class Collator final : public td::actor::Actor { td::RefInt256 masterchain_create_fee_, basechain_create_fee_; std::unique_ptr block_limits_; std::unique_ptr block_limit_status_; + vm::ProofStorageStat collated_data_stat; int block_limit_class_ = 0; ton::LogicalTime min_new_msg_lt{std::numeric_limits::max()}; block::CurrencyCollection total_balance_, old_total_balance_, total_validator_fees_; @@ -223,8 +224,7 @@ class Collator final : public td::actor::Actor { struct AccountStorageDict { bool inited = false; vm::MerkleProofBuilder mpb; - Ref proof_root; - size_t proof_size_estimate = 0; + vm::ProofStorageStat proof_stat; bool add_to_collated_data = false; std::vector> storage_stat_updates; }; @@ -405,6 +405,11 @@ class Collator final : public td::actor::Actor { CollationStats stats_; void finalize_stats(); + + AccountStorageDict* current_tx_storage_dict_ = nullptr; + + void on_cell_loaded(const Ref& cell); + void set_current_tx_storage_dict(const block::Account& account); }; } // namespace validator diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 00699997c..29151a02d 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -944,10 +944,8 @@ void Collator::got_neighbor_msg_queue(unsigned i, Ref res) { neighbor_proof_builders_.push_back(vm::MerkleProofBuilder{res->state_root_}); state_root = neighbor_proof_builders_.back().root(); if (full_collated_data_ && !block_id.is_masterchain()) { - neighbor_proof_builders_.back().set_cell_load_callback([&](const td::Ref& cell) { - if (block_limit_status_) { - block_limit_status_->collated_data_stat.add_cell(cell); - } + neighbor_proof_builders_.back().set_cell_load_callback([&](const Ref& cell) { + on_cell_loaded(cell); }); } } @@ -1056,10 +1054,8 @@ bool Collator::unpack_merge_last_state() { // 1. prepare for creating a MerkleUpdate based on previous state state_usage_tree_ = std::make_shared(); if (full_collated_data_ && !is_masterchain()) { - state_usage_tree_->set_cell_load_callback([&](const td::Ref& cell) { - if (block_limit_status_) { - block_limit_status_->collated_data_stat.add_cell(cell); - } + state_usage_tree_->set_cell_load_callback([&](const Ref& cell) { + on_cell_loaded(cell); }); } prev_state_root_ = vm::UsageCell::create(prev_state_root_pure_, state_usage_tree_->root_ptr()); @@ -1107,10 +1103,8 @@ bool Collator::unpack_last_state() { // prepare for creating a MerkleUpdate based on previous state state_usage_tree_ = std::make_shared(); if (full_collated_data_ && !is_masterchain()) { - state_usage_tree_->set_cell_load_callback([&](const td::Ref& cell) { - if (block_limit_status_) { - block_limit_status_->collated_data_stat.add_cell(cell); - } + state_usage_tree_->set_cell_load_callback([&](const Ref& cell) { + on_cell_loaded(cell); }); } prev_state_root_ = vm::UsageCell::create(prev_state_root_pure_, state_usage_tree_->root_ptr()); @@ -1596,6 +1590,7 @@ bool Collator::init_block_limits() { } block_limits_->usage_tree = state_usage_tree_.get(); block_limit_status_ = std::make_unique(*block_limits_); + block_limit_status_->collated_data_size_estimate = collated_data_stat.estimate_proof_size(); return true; } @@ -2653,10 +2648,8 @@ bool Collator::init_account_storage_dict(block::Account& account) { << ": dict is empty"); } dict.mpb = vm::MerkleProofBuilder(res.move_as_ok()); - dict.mpb.set_cell_load_callback([&](const td::Ref& cell) { - if (block_limit_status_) { - block_limit_status_->collated_data_stat.add_cell(cell); - } + dict.mpb.set_cell_load_callback([&](const Ref& cell) { + on_cell_loaded(cell); }); } auto S = account.init_account_storage_stat(dict.mpb.root()); @@ -2756,7 +2749,7 @@ bool Collator::process_account_storage_dict(const block::Account& account) { from_old_state = true; } if (calculate_proof_size_diff && from_old_state) { - switch (block_limit_status_->collated_data_stat.get_cell_status(cell->get_hash())) { + switch (collated_data_stat.get_cell_status(cell->get_hash())) { case vm::ProofStorageStat::c_none: proof_size_diff += vm::ProofStorageStat::estimate_serialized_size(loaded_cell.data_cell); break; @@ -2781,14 +2774,16 @@ bool Collator::process_account_storage_dict(const block::Account& account) { } state_usage_tree_->set_ignore_loads(false); - if (proof_size_diff > (td::int64)dict.proof_size_estimate) { + if (proof_size_diff > (td::int64)dict.proof_stat.estimate_proof_size()) { LOG(DEBUG) << "Storage dict proof of account " << account.addr.to_hex() - << " : account_proof_size=" << proof_size_diff << ", dict_proof_size=" << dict.proof_size_estimate - << ", include dict in collated data"; + << " : account_proof_size=" << proof_size_diff + << ", dict_proof_size=" << dict.proof_stat.estimate_proof_size() << ", include dict in collated data"; dict.add_to_collated_data = true; + collated_data_stat.add_loaded_cells(dict.proof_stat); } else { LOG(DEBUG) << "Storage dict proof of account " << account.addr.to_hex() - << " : account_proof_size=" << proof_size_diff << ", dict_proof_size=" << dict.proof_size_estimate + << " : account_proof_size=" << proof_size_diff + << ", dict_proof_size=" << dict.proof_stat.estimate_proof_size() << ", DO NOT include dict in collated data"; // Include account storage in collated data calculate_proof_size_diff = false; @@ -2807,16 +2802,6 @@ bool Collator::process_account_storage_dict(const block::Account& account) { * @returns True if the operation is successful, false otherwise. */ bool Collator::combine_account_transactions() { - for (auto& [hash, dict] : account_storage_dicts_) { - auto res = dict.mpb.extract_proof(); - if (res.is_error()) { - return fatal_error(res.move_as_error_prefix(PSTRING() << "Failed to generate proof for account storage dict " - << hash.to_hex() << ": ")); - } - dict.proof_root = res.move_as_ok(); - dict.proof_size_estimate = vm::std_boc_serialize(dict.proof_root, 31).move_as_ok().size(); - } - vm::AugmentedDictionary dict{256, block::tlb::aug_ShardAccountBlocks}; for (auto& z : accounts) { block::Account& acc = *(z.second); @@ -3041,6 +3026,7 @@ bool Collator::create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, t << "last transaction time in the state of account " << workchain() << ":" << smc_addr.to_hex() << " is too large")); } + set_current_tx_storage_dict(*acc); std::unique_ptr trans = std::make_unique( *acc, mask == 2 ? block::transaction::Transaction::tr_tick : block::transaction::Transaction::tr_tock, req_start_lt, now_); @@ -3145,6 +3131,7 @@ Ref Collator::create_ordinary_transaction(Ref msg_root, if (it != last_dispatch_queue_emitted_lt_.end()) { after_lt = std::max(after_lt, it->second); } + set_current_tx_storage_dict(*acc); auto res = impl_create_ordinary_transaction(msg_root, acc, now_, start_lt, &storage_phase_cfg_, &compute_phase_cfg_, &action_phase_cfg_, &serialize_cfg_, external, after_lt); if (res.is_error()) { @@ -3900,7 +3887,7 @@ static std::string block_full_comment(const block::BlockLimitStatus& block_limit if (!block_limit_status.limits.lt_delta.fits(cls, lt_delta)) { return PSTRING() << "block_full lt_delta " << lt_delta; } - auto collated_data_bytes = block_limit_status.collated_data_stat.estimate_proof_size(); + auto collated_data_bytes = block_limit_status.collated_data_size_estimate; if (!block_limit_status.limits.collated_data.fits(cls, collated_data_bytes)) { return PSTRING() << "block_full collated_data " << collated_data_bytes; } @@ -5170,7 +5157,7 @@ bool Collator::check_block_overload() { LOG(INFO) << "block load statistics: gas=" << block_limit_status_->gas_used << " lt_delta=" << block_limit_status_->cur_lt - block_limit_status_->limits.start_lt << " size_estimate=" << block_size_estimate_ - << " collated_size_estimate=" << block_limit_status_->collated_data_stat.estimate_proof_size(); + << " collated_size_estimate=" << block_limit_status_->collated_data_size_estimate; block_limit_class_ = std::max(block_limit_class_, block_limit_status_->classify()); if (block_limit_class_ >= block::ParamLimits::cl_soft || dispatch_queue_total_limit_reached_) { std::string message = "block is overloaded "; @@ -6015,7 +6002,9 @@ bool Collator::create_collated_data() { } state_usage_tree_->set_use_mark_for_is_loaded(false); - Ref state_proof = vm::MerkleProof::generate(prev_state_root_, state_usage_tree_.get()); + Ref state_proof = vm::MerkleProof::generate(prev_state_root_, [&](const Ref& c) { + return !collated_data_stat.is_loaded(c->get_hash()); + }); if (state_proof.is_null()) { return fatal_error("cannot generate Merkle proof for previous state"); } @@ -6035,11 +6024,9 @@ bool Collator::create_collated_data() { } // 4. Proofs for message queues for (vm::MerkleProofBuilder &mpb : neighbor_proof_builders_) { - auto r_proof = mpb.extract_proof(); - if (r_proof.is_error()) { - return fatal_error(r_proof.move_as_error_prefix("cannot generate Merkle proof for neighbor: ")); - } - Ref proof = r_proof.move_as_ok(); + Ref proof = vm::MerkleProof::generate(mpb.original_root(), [&](const Ref& c) { + return !collated_data_stat.is_loaded(c->get_hash()); + }); if (proof.is_null()) { return fatal_error("cannot generate Merkle proof for neighbor"); } @@ -6061,11 +6048,15 @@ bool Collator::create_collated_data() { if (!dict.add_to_collated_data) { continue; } - CHECK(dict.proof_root.not_null()); + Ref proof = vm::MerkleProof::generate( + dict.mpb.original_root(), [&](const Ref& c) { return !collated_data_stat.is_loaded(c->get_hash()); }); + if (proof.is_null()) { + return fatal_error("cannot generate Merkle proof for neighbor"); + } // account_storage_dict_proof#37c1e3fc proof:^Cell = AccountStorageDictProof; collated_roots_.push_back(vm::CellBuilder() .store_long(block::gen::AccountStorageDictProof::cons_tag[0], 32) - .store_ref(dict.proof_root) + .store_ref(proof) .finalize_novm()); } return true; @@ -6123,7 +6114,7 @@ bool Collator::create_block_candidate() { << st.internal_refs << " " << st.external_refs << " " << block_limit_status_->accounts << " " << block_limit_status_->transactions; LOG(INFO) << "serialized collated data size " << cdata_slice.size() << " bytes (preliminary estimate was " - << block_limit_status_->collated_data_stat.estimate_proof_size() << ")"; + << block_limit_status_->collated_data_size_estimate << ")"; auto new_block_id_ext = ton::BlockIdExt{ton::BlockId{shard_, new_block_seqno}, new_block->get_hash().bits(), block::compute_file_hash(blk_slice.as_slice())}; // 3. create a BlockCandidate @@ -6371,7 +6362,7 @@ void Collator::finalize_stats() { stats_.estimated_bytes = block_limit_status_->estimate_block_size(); stats_.gas = block_limit_status_->gas_used; stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt; - stats_.estimated_collated_data_bytes = block_limit_status_->collated_data_stat.estimate_proof_size(); + stats_.estimated_collated_data_bytes = block_limit_status_->collated_data_size_estimate; stats_.cat_bytes = block_limit_status_->limits.classify_size(stats_.estimated_bytes); stats_.cat_gas = block_limit_status_->limits.classify_gas(stats_.gas); stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt); @@ -6384,6 +6375,42 @@ void Collator::finalize_stats() { stats_.time_stats = (PSTRING() << perf_log_); } +/** + * This method is called when cell from shard state is loaded. + * Only if full_collated_data is set. + * When storage stat is calculated, StorageStatCalculationContext::calculating_storage_stat() returns true. + * In this case, loaded cell is stored in a separate StorageStat for the storage dict of the account (if it exists). + * + * @param cell Loaded cell + */ +void Collator::on_cell_loaded(const Ref& cell) { + auto context = block::StorageStatCalculationContext::get(); + vm::ProofStorageStat* stat = (context && context->calculating_storage_stat() && current_tx_storage_dict_ + ? ¤t_tx_storage_dict_->proof_stat + : &collated_data_stat); + if (block_limit_status_) { + block_limit_status_->collated_data_size_estimate -= stat->estimate_proof_size(); + } + stat->add_loaded_cell(cell); + if (block_limit_status_) { + block_limit_status_->collated_data_size_estimate += stat->estimate_proof_size(); + } +} + +/** + * Initializes current_tx_storage_dict_ to be used in on_cell_loaded. + * + * @param account Account of the current transaction + */ +void Collator::set_current_tx_storage_dict(const block::Account& account) { + if (!account.orig_storage_dict_hash) { + current_tx_storage_dict_ = nullptr; + return; + } + auto it = account_storage_dicts_.find(account.orig_storage_dict_hash.value()); + current_tx_storage_dict_ = it == account_storage_dicts_.end() ? nullptr : &it->second; +} + } // namespace validator } // namespace ton From 5f9d8af6c755355e43dbb01c44d7ccf3af0a0a6f Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Thu, 13 Mar 2025 08:17:40 -0400 Subject: [PATCH 152/388] Cmake refactoring (#1551) * Ignore clangd .cache directory With the in-tree build, clangd likes to put its cache files inside of /.cache. Thus, put the directory to .gitignore. * Remove redundant cmake_minimum_required calls There is no need to repeat the same information in all subprojects, the call in the root CMakeLists.txt is more than enough. * Bump minimum CMake version to 3.16 This avoids deprecation warnings on newer CMake versions. 3.16 was chosen as a version easily available on Ubuntu 20.04 -- the oldest distributive we support. * Do not automatically add -stdlib=libc++ when sanitizers are requested In my experience, adding -stdlib=libc++ during compilation of a project with C++ dependencies which are not consistently using libc++ never works. In this particular case, modern enough stdlibc++ works just fine when compiled with Clang and sanitizers, while using libc++ results in obscure linker errors. * Do not fall back to -O2 if CMAKE_BUILD_TYPE == RelWithDebInfo * Use CMAKE_{C,CXX}_COMPILER_LAUNCHER for enabling ccache Additionally, if user has already specified either of these two variables, don't override their choices. As per documentation, RULE_LAUNCH_{COMPILE,LINK} are only supposed to be used in internal CMake code. More importantly, the previous approach appended ccache two times if CMAKE_CXX_COMPILER_LAUNCHER was provided in command line or via an environment variable. * Remove unused crypto/openssl/digest.h --- .gitignore | 1 + CMakeLists.txt | 31 ++--- adnl/CMakeLists.txt | 2 - blockchain-explorer/CMakeLists.txt | 2 - catchain/CMakeLists.txt | 2 - common/CMakeLists.txt | 2 - create-hardfork/CMakeLists.txt | 2 - crypto/CMakeLists.txt | 2 - crypto/openssl/digest.h | 151 ------------------------ dht-server/CMakeLists.txt | 2 - dht/CMakeLists.txt | 2 - emulator/CMakeLists.txt | 2 - example/android/CMakeLists.txt | 2 +- fec/CMakeLists.txt | 2 - http/CMakeLists.txt | 2 - keyring/CMakeLists.txt | 2 - keys/CMakeLists.txt | 2 - lite-client/CMakeLists.txt | 2 - memprof/CMakeLists.txt | 2 - overlay/CMakeLists.txt | 2 - rldp-http-proxy/CMakeLists.txt | 2 - rldp/CMakeLists.txt | 2 - rldp2/CMakeLists.txt | 2 - storage/CMakeLists.txt | 2 - storage/storage-daemon/CMakeLists.txt | 2 - tdactor/CMakeLists.txt | 2 - tdactor/benchmark/CMakeLists.txt | 2 - tddb/CMakeLists.txt | 2 - tdfec/CMakeLists.txt | 2 - tdfec/benchmark/CMakeLists.txt | 2 - tdnet/CMakeLists.txt | 2 - tdtl/CMakeLists.txt | 2 - tdutils/CMakeLists.txt | 2 - tdutils/generate/CMakeLists.txt | 2 - terminal/CMakeLists.txt | 2 - tl-utils/CMakeLists.txt | 2 - tl/CMakeLists.txt | 2 - tl/generate/CMakeLists.txt | 2 - tolk/CMakeLists.txt | 2 - tonlib/CMakeLists.txt | 2 - utils/CMakeLists.txt | 2 - validator-engine-console/CMakeLists.txt | 2 - validator-engine/CMakeLists.txt | 2 - validator-session/CMakeLists.txt | 2 - validator/CMakeLists.txt | 2 - validator/impl/CMakeLists.txt | 2 - 46 files changed, 19 insertions(+), 250 deletions(-) delete mode 100644 crypto/openssl/digest.h diff --git a/.gitignore b/.gitignore index e5bb366ce..ffcf93698 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ crypto/smartcont/auto/ test/regression-tests.cache/ *.swp **/*build*/ +.cache .idea .vscode .DS_Store diff --git a/CMakeLists.txt b/CMakeLists.txt index cea3fc7ec..59f7c5e08 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) +cmake_minimum_required(VERSION 3.16 FATAL_ERROR) project(TON VERSION 0.5 LANGUAGES C CXX) set(CMAKE_POSITION_INDEPENDENT_CODE ON) @@ -159,6 +159,9 @@ if (TON_USE_ROCKSDB) set(FAIL_ON_WARNINGS OFF CACHE BOOL "fail on warnings") message("Add rocksdb") add_subdirectory(third-party/rocksdb EXCLUDE_FROM_ALL) + # Broken CMake in rocksdb alters properties it has no business changing. + set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE) + set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK) endif() option(USE_COROUTINES "experimental support of coroutines" OFF) @@ -188,11 +191,12 @@ include(BuildSECP256K1) # Configure CCache if available find_program(CCACHE_FOUND ccache) -#set(CCACHE_FOUND 0) if (CCACHE_FOUND) - message(STATUS "Found ccache") - set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache) - set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache) + if (NOT DEFINED CMAKE_C_COMPILER_LAUNCHER AND NOT DEFINED CMAKE_CXX_COMPILER_LAUNCHER) + message(STATUS "Using ccache") + set(CMAKE_C_COMPILER_LAUNCHER ccache) + set(CMAKE_CXX_COMPILER_LAUNCHER ccache) + endif() else() message(STATUS "Could NOT find ccache") endif() @@ -341,6 +345,14 @@ add_cxx_compiler_flag("-Qunused-arguments") add_cxx_compiler_flag("-Wno-unused-private-field") add_cxx_compiler_flag("-Wno-redundant-move") +if (GCC OR CLANG) + if (CMAKE_BUILD_TYPE MATCHES "RelWithDebInfo") + # For historical reasons, CMake falls back to -O2 optimization level when CMAKE_BUILD_TYPE is + # set to RelWithDebInfo. + add_compile_options(-O3) + endif() +endif() + #add_cxx_compiler_flag("-Wno-unused-function") #add_cxx_compiler_flag("-Wno-unused-variable") #add_cxx_compiler_flag("-Wno-shorten-64-to-32") @@ -351,22 +363,13 @@ if (CLANG) #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") endif() if (TON_USE_ASAN) - if (CLANG) - add_cxx_compiler_flag("-stdlib=libc++") - endif() add_cxx_compiler_flag("-fsanitize=address") add_definitions(-DTD_USE_ASAN=1) endif() if (TON_USE_TSAN) - if (CLANG) - add_cxx_compiler_flag("-stdlib=libc++") - endif() add_cxx_compiler_flag("-fsanitize=thread") endif() if (TON_USE_UBSAN) - if (CLANG) - add_cxx_compiler_flag("-stdlib=libc++") - endif() add_cxx_compiler_flag("-fsanitize=undefined") endif() #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread") diff --git a/adnl/CMakeLists.txt b/adnl/CMakeLists.txt index 111c4c500..f1d1bfbfb 100644 --- a/adnl/CMakeLists.txt +++ b/adnl/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - #BEGIN internal if (NOT TON_ONLY_TONLIB) set(ADNL_HEADERS diff --git a/blockchain-explorer/CMakeLists.txt b/blockchain-explorer/CMakeLists.txt index 86cb3e740..afd304177 100644 --- a/blockchain-explorer/CMakeLists.txt +++ b/blockchain-explorer/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - option(NIX "Use \"ON\" for a static build." OFF) set(BLOCHAIN_EXPLORER_SOURCE diff --git a/catchain/CMakeLists.txt b/catchain/CMakeLists.txt index 3f766688e..6175cb86b 100644 --- a/catchain/CMakeLists.txt +++ b/catchain/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 88a3671b3..9252178b5 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(COMMON_SOURCE checksum.h errorcode.h diff --git a/create-hardfork/CMakeLists.txt b/create-hardfork/CMakeLists.txt index 3024603c2..daa7f2e73 100644 --- a/create-hardfork/CMakeLists.txt +++ b/create-hardfork/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 069083381..92ebf8044 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/crypto/openssl/digest.h b/crypto/openssl/digest.h deleted file mode 100644 index d0dfc4c08..000000000 --- a/crypto/openssl/digest.h +++ /dev/null @@ -1,151 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2019 Telegram Systems LLP -*/ -#pragma once -#include - -#include -#include - -#include "td/utils/Slice.h" - -namespace digest { -struct OpensslEVP_SHA1 { - enum { digest_bytes = 20 }; - static const EVP_MD *get_evp() { - return EVP_sha1(); - } -}; - -struct OpensslEVP_SHA256 { - enum { digest_bytes = 32 }; - static const EVP_MD *get_evp() { - return EVP_sha256(); - } -}; - -struct OpensslEVP_SHA512 { - enum { digest_bytes = 64 }; - static const EVP_MD *get_evp() { - return EVP_sha512(); - } -}; - -template -class HashCtx { - EVP_MD_CTX *ctx{nullptr}; - void init(); - void clear(); - - public: - enum { digest_bytes = H::digest_bytes }; - HashCtx() { - init(); - } - HashCtx(const void *data, std::size_t len) { - init(); - feed(data, len); - } - ~HashCtx() { - clear(); - } - void reset(); - void feed(const void *data, std::size_t len); - void feed(td::Slice slice) { - feed(slice.data(), slice.size()); - } - std::size_t extract(unsigned char buffer[digest_bytes]); - std::size_t extract(td::MutableSlice slice); - std::string extract(); -}; - -template -void HashCtx::init() { - ctx = EVP_MD_CTX_create(); - reset(); -} - -template -void HashCtx::reset() { - EVP_DigestInit_ex(ctx, H::get_evp(), 0); -} - -template -void HashCtx::clear() { - EVP_MD_CTX_destroy(ctx); - ctx = nullptr; -} - -template -void HashCtx::feed(const void *data, std::size_t len) { - EVP_DigestUpdate(ctx, data, len); -} - -template -std::size_t HashCtx::extract(unsigned char buffer[digest_bytes]) { - unsigned olen = 0; - EVP_DigestFinal_ex(ctx, buffer, &olen); - assert(olen == digest_bytes); - return olen; -} - -template -std::size_t HashCtx::extract(td::MutableSlice slice) { - return extract(slice.ubegin()); -} - -template -std::string HashCtx::extract() { - unsigned char buffer[digest_bytes]; - unsigned olen = 0; - EVP_DigestFinal_ex(ctx, buffer, &olen); - assert(olen == digest_bytes); - return std::string((char *)buffer, olen); -} - -typedef HashCtx SHA1; -typedef HashCtx SHA256; -typedef HashCtx SHA512; - -template -std::size_t hash_str(unsigned char buffer[T::digest_bytes], const void *data, std::size_t size) { - T hasher(data, size); - return hasher.extract(buffer); -} - -template -std::size_t hash_two_str(unsigned char buffer[T::digest_bytes], const void *data1, std::size_t size1, const void *data2, - std::size_t size2) { - T hasher(data1, size1); - hasher.feed(data2, size2); - return hasher.extract(buffer); -} - -template -std::string hash_str(const void *data, std::size_t size) { - T hasher(data, size); - return hasher.extract(); -} - -template -std::string hash_two_str(const void *data1, std::size_t size1, const void *data2, std::size_t size2) { - T hasher(data1, size1); - hasher.feed(data2, size2); - return hasher.extract(); -} -} // namespace digest diff --git a/dht-server/CMakeLists.txt b/dht-server/CMakeLists.txt index 6daac0334..07c93f247 100644 --- a/dht-server/CMakeLists.txt +++ b/dht-server/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/dht/CMakeLists.txt b/dht/CMakeLists.txt index 95ee70691..f8d49a2d8 100644 --- a/dht/CMakeLists.txt +++ b/dht/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/emulator/CMakeLists.txt b/emulator/CMakeLists.txt index 663c8fd26..e1bc38f52 100644 --- a/emulator/CMakeLists.txt +++ b/emulator/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - option(EMULATOR_STATIC "Build emulator as static library" OFF) set(EMULATOR_STATIC_SOURCE diff --git a/example/android/CMakeLists.txt b/example/android/CMakeLists.txt index 0101ab641..b0f998a8b 100644 --- a/example/android/CMakeLists.txt +++ b/example/android/CMakeLists.txt @@ -3,7 +3,7 @@ # Sets the minimum version of CMake required to build the native library. -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) +cmake_minimum_required(VERSION 3.16 FATAL_ERROR) project(TON_ANDROID VERSION 0.5 LANGUAGES C CXX) diff --git a/fec/CMakeLists.txt b/fec/CMakeLists.txt index 2a3056071..8549dc0e5 100644 --- a/fec/CMakeLists.txt +++ b/fec/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/http/CMakeLists.txt b/http/CMakeLists.txt index 4a3fccf82..891525446 100644 --- a/http/CMakeLists.txt +++ b/http/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(HTTP_SOURCE http.h http.cpp diff --git a/keyring/CMakeLists.txt b/keyring/CMakeLists.txt index f8f610f2f..6a5599e98 100644 --- a/keyring/CMakeLists.txt +++ b/keyring/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(KEYRING_SOURCE keyring.h keyring.hpp diff --git a/keys/CMakeLists.txt b/keys/CMakeLists.txt index e80436b7b..1b50236dd 100644 --- a/keys/CMakeLists.txt +++ b/keys/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(KEYS_SOURCE keys.cpp encryptor.cpp diff --git a/lite-client/CMakeLists.txt b/lite-client/CMakeLists.txt index b28a14e9a..03d050095 100644 --- a/lite-client/CMakeLists.txt +++ b/lite-client/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - add_library(lite-client-common STATIC lite-client-common.cpp lite-client-common.h ext-client.cpp ext-client.h query-utils.hpp query-utils.cpp) target_link_libraries(lite-client-common PUBLIC tdactor adnllite tl_api tl_lite_api tl-lite-utils ton_crypto) diff --git a/memprof/CMakeLists.txt b/memprof/CMakeLists.txt index 2ccf11dfd..677f30511 100644 --- a/memprof/CMakeLists.txt +++ b/memprof/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(MEMPROF_SOURCE memprof/memprof.cpp memprof/memprof.h diff --git a/overlay/CMakeLists.txt b/overlay/CMakeLists.txt index ab9722a60..c2c9dc617 100644 --- a/overlay/CMakeLists.txt +++ b/overlay/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/rldp-http-proxy/CMakeLists.txt b/rldp-http-proxy/CMakeLists.txt index f7e30c802..f7a9d73d9 100644 --- a/rldp-http-proxy/CMakeLists.txt +++ b/rldp-http-proxy/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - add_executable(rldp-http-proxy rldp-http-proxy.cpp DNSResolver.h DNSResolver.cpp) target_include_directories(rldp-http-proxy PUBLIC $) target_link_libraries(rldp-http-proxy PRIVATE tonhttp rldp rldp2 dht tonlib git) diff --git a/rldp/CMakeLists.txt b/rldp/CMakeLists.txt index 39e0d3ca8..688476328 100644 --- a/rldp/CMakeLists.txt +++ b/rldp/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/rldp2/CMakeLists.txt b/rldp2/CMakeLists.txt index c144ec01d..984e7bb1a 100644 --- a/rldp2/CMakeLists.txt +++ b/rldp2/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt index 30403f5e2..5a53e7398 100644 --- a/storage/CMakeLists.txt +++ b/storage/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/storage/storage-daemon/CMakeLists.txt b/storage/storage-daemon/CMakeLists.txt index cabc61439..5392d4ab9 100644 --- a/storage/storage-daemon/CMakeLists.txt +++ b/storage/storage-daemon/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - add_executable(embed-provider-code smartcont/embed-provider-code.cpp) add_custom_command( diff --git a/tdactor/CMakeLists.txt b/tdactor/CMakeLists.txt index 98b900a1e..44580dd59 100644 --- a/tdactor/CMakeLists.txt +++ b/tdactor/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - #SOURCE SETS set(TDACTOR_SOURCE td/actor/core/ActorExecutor.cpp diff --git a/tdactor/benchmark/CMakeLists.txt b/tdactor/benchmark/CMakeLists.txt index c4ff79a1b..662ae20c4 100644 --- a/tdactor/benchmark/CMakeLists.txt +++ b/tdactor/benchmark/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(BENCHMARK_SOURCE benchmark.cpp third_party/mp-queue.c diff --git a/tddb/CMakeLists.txt b/tddb/CMakeLists.txt index 55476e648..57e69403b 100644 --- a/tddb/CMakeLists.txt +++ b/tddb/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - #SOURCE SETS set(TDDB_UTILS_SOURCE td/db/utils/BlobView.cpp diff --git a/tdfec/CMakeLists.txt b/tdfec/CMakeLists.txt index 828ff90d5..97e4eb691 100644 --- a/tdfec/CMakeLists.txt +++ b/tdfec/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(TDFEC_SOURCE td/fec/raptorq/Rfc.cpp td/fec/raptorq/Rfc.h diff --git a/tdfec/benchmark/CMakeLists.txt b/tdfec/benchmark/CMakeLists.txt index ee8f72cbf..66224a6d4 100644 --- a/tdfec/benchmark/CMakeLists.txt +++ b/tdfec/benchmark/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - add_executable(benchmark-fec benchmark.cpp ) target_include_directories(benchmark-fec PUBLIC $) target_link_libraries(benchmark-fec PRIVATE tdfec) diff --git a/tdnet/CMakeLists.txt b/tdnet/CMakeLists.txt index bc00a6769..ff4ce95d0 100644 --- a/tdnet/CMakeLists.txt +++ b/tdnet/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(TDNET_SOURCE td/net/FdListener.cpp td/net/TcpListener.cpp diff --git a/tdtl/CMakeLists.txt b/tdtl/CMakeLists.txt index 482bd0f7b..0601b9220 100644 --- a/tdtl/CMakeLists.txt +++ b/tdtl/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - #SOURCE SETS set(TDTL_SOURCE td/tl/tl_config.cpp diff --git a/tdutils/CMakeLists.txt b/tdutils/CMakeLists.txt index f8c191a14..716ec322a 100644 --- a/tdutils/CMakeLists.txt +++ b/tdutils/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - option(TDUTILS_MIME_TYPE "Generate mime types conversion (gperf is required)" ON) if (WIN32) diff --git a/tdutils/generate/CMakeLists.txt b/tdutils/generate/CMakeLists.txt index 194fda391..391286066 100644 --- a/tdutils/generate/CMakeLists.txt +++ b/tdutils/generate/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - # Generates files for MIME type <-> extension conversions # DEPENDS ON: gperf grep bash/powershell diff --git a/terminal/CMakeLists.txt b/terminal/CMakeLists.txt index af51153f3..2d0c50304 100644 --- a/terminal/CMakeLists.txt +++ b/terminal/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/tl-utils/CMakeLists.txt b/tl-utils/CMakeLists.txt index d5c52d48a..709bc7cb5 100644 --- a/tl-utils/CMakeLists.txt +++ b/tl-utils/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(TL_UTILS_SOURCE common-utils.hpp tl-utils.hpp diff --git a/tl/CMakeLists.txt b/tl/CMakeLists.txt index d0760a349..168a5d73e 100644 --- a/tl/CMakeLists.txt +++ b/tl/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - add_subdirectory(generate) set_source_files_properties(${TL_TON_API} PROPERTIES GENERATED TRUE) add_library(tl_api STATIC diff --git a/tl/generate/CMakeLists.txt b/tl/generate/CMakeLists.txt index 083d39737..8ff7f79a4 100644 --- a/tl/generate/CMakeLists.txt +++ b/tl/generate/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - file(MAKE_DIRECTORY auto/tl) set(TL_TON_API diff --git a/tolk/CMakeLists.txt b/tolk/CMakeLists.txt index de4081157..c4abfec04 100644 --- a/tolk/CMakeLists.txt +++ b/tolk/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - set(TOLK_SOURCE src-file.cpp lexer.cpp diff --git a/tonlib/CMakeLists.txt b/tonlib/CMakeLists.txt index 3dbd628d3..2e3c583d4 100644 --- a/tonlib/CMakeLists.txt +++ b/tonlib/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - option(TONLIBJSON_STATIC "Build tonlibjson as static library" OFF) if (NOT OPENSSL_FOUND) diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index e9ffd2a27..99a4e92df 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/validator-engine-console/CMakeLists.txt b/validator-engine-console/CMakeLists.txt index d87d8a6f3..73f3a5877 100644 --- a/validator-engine-console/CMakeLists.txt +++ b/validator-engine-console/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - add_executable (validator-engine-console validator-engine-console.cpp validator-engine-console.h validator-engine-console-query.cpp validator-engine-console-query.h ) diff --git a/validator-engine/CMakeLists.txt b/validator-engine/CMakeLists.txt index 73949d808..693becdc8 100644 --- a/validator-engine/CMakeLists.txt +++ b/validator-engine/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/validator-session/CMakeLists.txt b/validator-session/CMakeLists.txt index 6b4e8f687..94363c79c 100644 --- a/validator-session/CMakeLists.txt +++ b/validator-session/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index 5f5544b22..f7a4ce1d1 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() diff --git a/validator/impl/CMakeLists.txt b/validator/impl/CMakeLists.txt index 978cf859a..9ab32c47b 100644 --- a/validator/impl/CMakeLists.txt +++ b/validator/impl/CMakeLists.txt @@ -1,5 +1,3 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() From 8194d1f09d9165fc7a03492dd737e75e9ab697a9 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 27 Feb 2025 19:12:12 +0300 Subject: [PATCH 153/388] [Tolk] Support binary number literals 0b1100 Along with hex 0x, 0b is also parsed. Example: 0b1010101010101010 == 43690. --- tolk-tester/tests/numbers-tests.tolk | 35 +++++++++++++++++++++++ tolk/ast-from-tokens.cpp | 21 +++++++++++++- tolk/lexer.cpp | 42 +++++++++++++++++++--------- 3 files changed, 84 insertions(+), 14 deletions(-) create mode 100644 tolk-tester/tests/numbers-tests.tolk diff --git a/tolk-tester/tests/numbers-tests.tolk b/tolk-tester/tests/numbers-tests.tolk new file mode 100644 index 000000000..2dfdee8fd --- /dev/null +++ b/tolk-tester/tests/numbers-tests.tolk @@ -0,0 +1,35 @@ +const x1 = 0xFF; +const x2 = 0xAbcdEF01234561AC; +const x3 = 0x00BEC; +const x4 = -0xD70B; + +const d1 = 255; +const d2 = 12379813738877116844; +const d3 = 003052; +const d4 = -55051; + +const b1 = 0b11111111; +const b2 = 0b1010101111001101111011110000000100100011010001010110000110101100; +const b3 = 0b00101111101100; +const b4 = -0b1101011100001011; + +fun main() { + assert(x1 == d1 && x1 == b1, 100); + assert(x2 == d2 && x2 == b2, 100); + assert(x3 == d3 && x3 == b3, 100); + assert(x4 == d4 && x4 == b4, 100); + assert(x1 + x2 + x3 == b1 + b2 + b3, 100); + assert(-x2 == -d2 && -x2 == -b2, 100); + + return ( + 0b1010101010101010+0b00 == 43690, + 0b0 == 0, + -0b1010100001010101010-0b1+0b1 == -344746, + 0b00001 == 1, + --0b00101111101100 == 3052 + ); +} + +/** +@testcase | 0 | | -1 -1 -1 -1 -1 + */ diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index fcaa11577..66e47d218 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -123,6 +123,25 @@ static AnyExprV maybe_replace_eq_null_with_isNull_check(V v return createV(v->loc, v_nullable, v->tok == tok_neq); } +// parse `123` / `0xFF` / `0b10001` to td::RefInt256 +static td::RefInt256 parse_tok_int_const(std::string_view text) { + bool bin = text[0] == '0' && text[1] == 'b'; + if (!bin) { + // this function parses decimal and hex numbers + return td::string_to_int256(static_cast(text)); + } + // parse a binary number; to make it simpler, don't allow too long numbers, it's impractical + if (text.size() > 64 + 2) { + return {}; + } + uint64_t result = 0; + for (char c : text.substr(2)) { // skip "0b" + result = (result << 1) | static_cast(c - '0'); + } + return td::make_refint(result); +} + + /* * @@ -313,7 +332,7 @@ static AnyExprV parse_expr100(Lexer& lex) { } case tok_int_const: { std::string_view orig_str = lex.cur_str(); - td::RefInt256 intval = td::string_to_int256(static_cast(orig_str)); + td::RefInt256 intval = parse_tok_int_const(orig_str); if (intval.is_null() || !intval->signed_fits_bits(257)) { lex.error("invalid integer constant"); } diff --git a/tolk/lexer.cpp b/tolk/lexer.cpp index 06913a5fc..86ae306a4 100644 --- a/tolk/lexer.cpp +++ b/tolk/lexer.cpp @@ -238,27 +238,43 @@ struct ChunkAnnotation final : ChunkLexerBase { // A number, may be a hex one. struct ChunkNumber final : ChunkLexerBase { - bool parse(Lexer* lex) const override { + static bool parse_hex_or_bin(Lexer* lex, bool bin) { const char* str_begin = lex->c_str(); - bool hex = false; - if (lex->char_at() == '0' && lex->char_at(1) == 'x') { - lex->skip_chars(2); - hex = true; - } + lex->skip_chars(2); // 0x / 0b if (lex->is_eof()) { return false; } + while (!lex->is_eof()) { char c = lex->char_at(); - if (c >= '0' && c <= '9') { - lex->skip_chars(1); - continue; - } - if (!hex) { + bool ok = bin + ? c == '0' || c == '1' + : (c >= '0' && c <= '9') || ((c | 0x20) >= 'a' && (c | 0x20) <= 'f'); + if (!ok) { break; } - c |= 0x20; - if (c < 'a' || c > 'f') { + lex->skip_chars(1); + } + + std::string_view str_val(str_begin, lex->c_str() - str_begin); + lex->add_token(tok_int_const, str_val); + return true; + } + + bool parse(Lexer* lex) const override { + if (lex->char_at() == '0') { + if (lex->char_at(1) == 'x') { + return parse_hex_or_bin(lex, false); + } + if (lex->char_at(1) == 'b') { + return parse_hex_or_bin(lex, true); + } + } + + const char* str_begin = lex->c_str(); + while (!lex->is_eof()) { + char c = lex->char_at(); + if (c < '0' || c > '9') { break; } lex->skip_chars(1); From d54e708b5719349180dea2d268feb997d8dd6c76 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Fri, 28 Feb 2025 13:02:17 +0300 Subject: [PATCH 154/388] [Tolk] Support trailing comma in various places Trailing comma is now allowed in: * tensors * tuples * function calls * function parameters Note, that `(5)` is not a tensor, it's just integer `5`. With a trailing comma `(5,)` it's still `(5)`. --- tolk-tester/tests/a6.tolk | 8 ++++---- tolk-tester/tests/allow_post_modification.tolk | 2 +- tolk-tester/tests/assignment-tests.tolk | 18 ++++++++++++++---- tolk-tester/tests/inference-tests.tolk | 9 +++++++++ tolk-tester/tests/no-spaces.tolk | 10 +++++----- tolk-tester/tests/nullable-tensors.tolk | 9 +++++++++ tolk/ast-from-tokens.cpp | 18 ++++++++++++++++++ 7 files changed, 60 insertions(+), 14 deletions(-) diff --git a/tolk-tester/tests/a6.tolk b/tolk-tester/tests/a6.tolk index 944945464..ca1737a1e 100644 --- a/tolk-tester/tests/a6.tolk +++ b/tolk-tester/tests/a6.tolk @@ -4,7 +4,7 @@ fun f(a: int, b: int, c: int, d: int, e: int, f: int): (int, int) { __expect_type(D, "int"); __expect_type(D*D, "int"); __expect_type(calc_phi, "() -> int"); - return (Dx/D,Dy/D); + return (Dx/D,Dy/D,); };;;; fun calc_phi(): int { @@ -16,7 +16,7 @@ fun calc_phi(): int { do { (p,q)=(q,p+q); } while (q <= n); //;; - return mulDivRound(p, n, q); + return mulDivRound(p, n, q,); } fun calc_sqrt2(): int { @@ -31,10 +31,10 @@ fun calc_sqrt2(): int { return mulDivRound(p, n, q); } -fun calc_root(m: int) { +fun calc_root(m: int,) { var base: int=1; repeat(70) { base *= 10; } - var (a, b, c) = (1,0,-m); + var (a, b, c) = (1,0,-m,); var (p1, q1, p2, q2) = (1, 0, 0, 1); do { var k: int=-1; diff --git a/tolk-tester/tests/allow_post_modification.tolk b/tolk-tester/tests/allow_post_modification.tolk index e20e8218e..180654e32 100644 --- a/tolk-tester/tests/allow_post_modification.tolk +++ b/tolk-tester/tests/allow_post_modification.tolk @@ -47,7 +47,7 @@ fun test_call_1(x: int): (int, int, int, int, int, int, int) { return foo1(x, x.`~inc`(x / 20), x, x = x * 2, x, x += 1, x); } -fun foo2(x1: int, x2: int, x3456: (int, int, int, int), x7: int): (int, int, int, int, int, int, int) { +fun foo2(x1: int, x2: int, x3456: (int, int, int, int), x7: int, ): (int, int, int, int, int, int, int) { var (x3: int, x4: int, x5: int, x6: int) = x3456; return (x1, x2, x3, x4, x5, x6, x7); } diff --git a/tolk-tester/tests/assignment-tests.tolk b/tolk-tester/tests/assignment-tests.tolk index bb6476521..f4d2c0d78 100644 --- a/tolk-tester/tests/assignment-tests.tolk +++ b/tolk-tester/tests/assignment-tests.tolk @@ -5,7 +5,7 @@ fun extractFromTypedTuple(params: [int]) { @method_id(101) fun test101(x: int) { - var params = [x]; + var params = [x, ]; return extractFromTypedTuple(params); } @@ -38,7 +38,7 @@ fun getTensor_1X(x: int) { } fun getTuple_12() { callOrder.tuplePush(110); - return [1, 2]; + return [1, 2, ]; } fun getTuple_1X(x: int) { callOrder.tuplePush(111); @@ -133,7 +133,7 @@ fun test108() { fun test109() { callOrder = createEmptyTuple(); var x = 0; - [getTuple_12().0, getTuple_1X(x = getIntValue5()).1, getTuple_1X(x += 10).0] = [getIntValue5(), getIntValue5(), getIntValueX(x)]; + [getTuple_12().0, getTuple_1X(x = getIntValue5()).1, getTuple_1X(x += 10).0] = [getIntValue5(), getIntValue5(), getIntValueX(x), ]; return (callOrder, x); } @@ -203,9 +203,18 @@ fun test116() { return (a, b, c, d, rhs2); } +@method_id(117) +fun test117() { + var c = [((1, ), ), ]; + __expect_type(c, "[int]"); + c.0 = 10; + c.0 = (20, ); + return c; +} + -fun main(value: int) { +fun main(value: int, ) { var (x: int?, y) = (autoInferIntNull(value), autoInferIntNull(value * 2)); if (x == null && y == null) { return null; } return x == null || y == null ? -1 : x! + y!; @@ -231,6 +240,7 @@ fun main(value: int) { @testcase | 114 | | 13 [ 1 14 ] 1 3 @testcase | 115 | | [ 101 111 ] 9 9 @testcase | 116 | | 1 2 3 4 [ 1 2 3 4 ] +@testcase | 117 | | [ 20 ] @fif_codegen diff --git a/tolk-tester/tests/inference-tests.tolk b/tolk-tester/tests/inference-tests.tolk index 5020d0ddc..e785ac5aa 100644 --- a/tolk-tester/tests/inference-tests.tolk +++ b/tolk-tester/tests/inference-tests.tolk @@ -30,6 +30,15 @@ fun test1(x: int, y: int) { __expect_type(beginCell(), "builder"); __expect_type(beginCell().endCell(), "cell"); } + __expect_type((1, 2), "(int, int)"); + __expect_type((1, 2, ), "(int, int)"); + __expect_type((1, ), "int"); + __expect_type((((1,)),), "int"); + __expect_type([1, 2], "[int, int]"); + __expect_type([1, 2, ], "[int, int]"); + __expect_type([1, ], "[int]"); + __expect_type([((1, ), ), ], "[int]"); + __expect_type([[[1, ], ], ], "[[[int]]]"); } fun test2(x: int, y: bool) { diff --git a/tolk-tester/tests/no-spaces.tolk b/tolk-tester/tests/no-spaces.tolk index 409bb342c..d63ca1411 100644 --- a/tolk-tester/tests/no-spaces.tolk +++ b/tolk-tester/tests/no-spaces.tolk @@ -3,11 +3,11 @@ const int10:int=10; fun just10(): int { return int10; } fun eq(v: int): int { return`v`; } -@method_id(101) fun `get_-1` (): int {return-1;} -@method_id(102) fun `get_--1` (): int {return--1;} -@method_id(103) fun `get_---1`(): int {return---1;} -@method_id(104) fun `get_+++1`(): int {return+++1;} -@method_id(105) fun `get_+-+1`(): int {return+-+1;} +@method_id(101,) fun `get_-1` (): int {return-1;} +@method_id(102,) fun `get_--1` (): int {return--1;} +@method_id(103,) fun `get_---1`(): int {return---1;} +@method_id(104,) fun `get_+++1`(): int {return+++1;} +@method_id(105,) fun `get_+-+1`(): int {return+-+1;} global `some()var`:int; diff --git a/tolk-tester/tests/nullable-tensors.tolk b/tolk-tester/tests/nullable-tensors.tolk index d0720273a..6de907d02 100644 --- a/tolk-tester/tests/nullable-tensors.tolk +++ b/tolk-tester/tests/nullable-tensors.tolk @@ -415,6 +415,13 @@ fun test135() { t1!.0 == null, t2!.0 == null, e1!.1.0 == null, e1!.1.1 == null, e2!.1.0 == null, e2!.1.1 == null); } +@method_id(136, ) +fun test136(x: int?, ) { + var t1 = (x, ); // it's not a tensor with 1 element + __expect_type(t1, "int?", ); + return ((t1, t1 == null, ), ); // testing trailing comma everywhere :) +} + fun main(){} @@ -466,6 +473,8 @@ fun main(){} @testcase | 133 | | 60 @testcase | 134 | | 11 21 -1 @testcase | 135 | | [ 10 ] [ (null) ] (null) 777 10 -1 (null) -1 (null) 0 777 10 -1 (null) -1 (null) 0 777 0 0 -1 0 0 -1 0 0 -1 777 0 -1 0 0 -1 0 +@testcase | 136 | 9 | 9 0 +@testcase | 136 | null | (null) -1 @fif_codegen """ diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 66e47d218..22974b1f5 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -236,6 +236,9 @@ static V parse_parameter_list(Lexer& lex) { params.push_back(parse_parameter(lex, true)); while (lex.tok() == tok_comma) { lex.next(); + if (lex.tok() == tok_clpar) { // trailing comma + break; + } params.push_back(parse_parameter(lex, false)); } } @@ -266,6 +269,9 @@ static V parse_argument_list(Lexer& lex) { args.push_back(parse_argument(lex)); while (lex.tok() == tok_comma) { lex.next(); + if (lex.tok() == tok_clpar) { // trailing comma + break; + } args.push_back(parse_argument(lex)); } } @@ -311,9 +317,15 @@ static AnyExprV parse_expr100(Lexer& lex) { std::vector items(1, first); while (lex.tok() == tok_comma) { lex.next(); + if (lex.tok() == tok_clpar) { // trailing comma + break; + } items.emplace_back(parse_expr(lex)); } lex.expect(tok_clpar, "`)`"); + if (items.size() == 1) { // we can reach here for 1 element with trailing comma: `(item, )` + return items[0]; // then just return item, not a 1-element tensor, + } // since 1-element tensors won't be type compatible with item's type return createV(loc, std::move(items)); } case tok_opbracket: { @@ -325,6 +337,9 @@ static AnyExprV parse_expr100(Lexer& lex) { std::vector items(1, parse_expr(lex)); while (lex.tok() == tok_comma) { lex.next(); + if (lex.tok() == tok_clbracket) { // trailing comma + break; + } items.emplace_back(parse_expr(lex)); } lex.expect(tok_clbracket, "`]`"); @@ -952,6 +967,9 @@ static V parse_annotation(Lexer& lex) { args.push_back(parse_expr(lex)); while (lex.tok() == tok_comma) { lex.next(); + if (lex.tok() == tok_clpar) { // trailing comma + break; + } args.push_back(parse_expr(lex)); } lex.expect(tok_clpar, "`)`"); From 49cc6d0e06b6510ce4316d2ff763b02ec00c5060 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 13 Mar 2025 18:00:34 +0400 Subject: [PATCH 155/388] [Tolk] Types `int32`, `uint64`, and fixed ints in general MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces fixed-size integer types: * intN for signed integers (0 < N ≤ 256) * uintN for unsigned integers (0 < N ≤ 257) The generic `int` type remains unchanged and can be implicitly cast to and from any `intN`. * arithmetic operations on `intN` degrade to `int` * numeric literals (like 0, 100) are just `int` * direct assignment between `intN` and `intM` is disallowed (as a probable error) There is no runtime overflow. A variable is declared as `int8` can hold any 257-bit value during execution. Overflow will occur when packing to a cell (in the future). --- tolk-tester/tests/intN-tests.tolk | 178 +++++++++++++++++++++++ tolk-tester/tests/invalid-typing-31.tolk | 10 ++ tolk-tester/tests/invalid-typing-32.tolk | 8 + tolk-tester/tests/invalid-typing-33.tolk | 10 ++ tolk-tester/tests/invalid-typing-34.tolk | 16 ++ tolk-tester/tests/nullable-tensors.tolk | 7 + tolk/pipe-ast-to-legacy.cpp | 25 +++- tolk/pipe-check-inferred-types.cpp | 8 +- tolk/pipe-infer-types-and-calls.cpp | 11 +- tolk/smart-casts-cfg.cpp | 4 + tolk/type-system.cpp | 78 +++++++++- tolk/type-system.h | 25 ++++ 12 files changed, 370 insertions(+), 10 deletions(-) create mode 100644 tolk-tester/tests/intN-tests.tolk create mode 100644 tolk-tester/tests/invalid-typing-31.tolk create mode 100644 tolk-tester/tests/invalid-typing-32.tolk create mode 100644 tolk-tester/tests/invalid-typing-33.tolk create mode 100644 tolk-tester/tests/invalid-typing-34.tolk diff --git a/tolk-tester/tests/intN-tests.tolk b/tolk-tester/tests/intN-tests.tolk new file mode 100644 index 000000000..3f4cdc015 --- /dev/null +++ b/tolk-tester/tests/intN-tests.tolk @@ -0,0 +1,178 @@ +fun takeAnyInt(a: int) { } +fun getAnyInt(): int { return 0; } +fun getNullableInt32(): int32? { return 32; } +fun getNullableVarInt32(): varint32? { return 32; } + + +fun autoInferInt8(x: int8) { + if (random()) { return x; } + else if (random()) { return 0 as int8; } + else if (random()) { return x!; } + else { return x += x; } +} + +fun autoInferUInt16Opt(x: uint16) { + if (random()) { return autoInferInt8(x as int8) as uint16; } + return null; +} + +fun autoInferInt_v1() { + if (random()) { return 0; } + else if (random()) { return 0 as uint16; } + else { return 0 as int8; } +} + +fun autoInferInt_v2() { + if (random()) { return 0 as uint1; } + else if (random()) { return 0 as int123; } + else { return 0 as int8; } +} + +@method_id(101) +fun test1(x: int8) { + var y: int8 = x; + y += x; + x *= y; + x = 1000000; + y = x; + __expect_type(x / y, "int"); + __expect_type(x & y, "int"); + __expect_type(x != y, "bool"); + if (x != y) { return -1; } + if (!(y == x)) { return -1; } + + var a1: uint8 = 0; + var a2: uint8? = 0; + var a3: uint8? = 0 as int?; + __expect_type(a1, "uint8"); + __expect_type(a2, "uint8"); + __expect_type(a3, "uint8?"); + + var z = x + y; + __expect_type(z, "int"); + return x / y; +} + +fun test2(): (uint8, uint8, uint8) { + var x: uint1 = 1; + return (1, x as uint8, x as int); +} + +fun test3(op: int32, qid: uint64) { + op = qid as int32; + + op + qid; + op = op + qid; + op = op & qid; + op += qid; + op &= qid; + if (op == qid) {} + if ((op as int32?)! == qid) {} + if (op == (qid as uint64)) {} + if ((op as uint257?)! != (qid as int256?)!) {} + __expect_type(op << qid, "int"); + + takeAnyInt(op); + op = getAnyInt(); + + __expect_type(autoInferInt8, "(int8) -> int8"); + __expect_type(autoInferUInt16Opt(0), "uint16?"); + __expect_type(autoInferInt_v1(), "int"); + __expect_type(autoInferInt_v2(), "int"); + + var amount: uint100 = 1000; + var percent: uint8 = 50; + var new = amount * percent / 100; + amount = new; +} + +@method_id(104) +fun test4(): (int32, int32?, bool, bool) { + var x = getNullableInt32(); + __expect_type(x, "int32?"); + if (x! != x! || x! != 32) { + return (0, null, false, false); + } + if (x == null) { + return (-1, null, false, false); + } + x += x; + return (x, x, x == x, x == getAnyInt()); +} + +@method_id(105) +fun test5() { + var (x: int8, y: uint16) = (1, 2); + var cell = beginCell().storeInt(x, 32).storeInt(y, 32 as int32).endCell(); + var slice = cell.beginParse(); + x = slice.loadInt(32); + y = slice.loadInt(32); + __expect_type(x & y, "int"); + return (x + y, x && y, x & y); +} + +fun test6() { + var x: int11 = ~(0 as int22); + while (x) {} + return x ? x : 0; +} + +fun test7() { + var n = getNullableVarInt32(); + __expect_type(n!, "varint32"); + __expect_type(n, "varint32?"); + if (n != null) { + __expect_type(n, "varint32"); + __expect_type(n += n, "varint32"); + __expect_type(n += n as int8, "varint32"); + } else { + n = 0; + n = 0 as varint32?; + n = 0 as varint32; + } + __expect_type(n, "varint32"); + __expect_type(n as uint32, "uint32"); + __expect_type(n as varint16, "varint16"); + __expect_type((n as varint16) as int, "int"); +} + +fun test13(x1: int?, x2: int8?, x3: int, x4: int8): (int8?, int8?, int8?, int8?, int32?, int32?) { + return (x1, x2, x3, x4, x4 as int32, x4 as int32?); +} + +@method_id(114) +fun test14(firstComponent: int8?): (int, int16)? { + return firstComponent! < 10 ? null : (firstComponent!, 2); +} + +fun assign0(mutate v: T) { v = 0; } + +fun main() { + var t = createEmptyTuple(); + t.tuplePush(1); + t.tuplePush(2); + t.tuplePush(3); + assign0(mutate t.0 as int8); + assign0(mutate t.1 as uint16); + assign0(mutate t.2 as int); + __expect_type(t.0 as int1, "int1"); + __expect_type(t.0 as uint257, "uint257"); + __expect_type(0 as int32, "int32"); + __expect_type((0 as int32) as uint64?, "uint64?"); + __expect_type(null as (int8, [uint16?])?, "(int8, [uint16?])?"); + __expect_type(10>3 ? 0 as uint8 : 0 as uint16, "int"); + return t; +} + +/** +@testcase | 0 | | [ 0 0 0 ] +@testcase | 101 | 0 | 1 +@testcase | 104 | | 64 64 -1 0 +@testcase | 105 | | 3 -1 0 +@testcase | 114 | 5 | (null) (null) 0 +@testcase | 114 | 15 | 15 2 -1 + +@fif_codegen DECLPROC assign0 +@fif_codegen DECLPROC assign0 +@fif_codegen DECLPROC assign0 +*/ diff --git a/tolk-tester/tests/invalid-typing-31.tolk b/tolk-tester/tests/invalid-typing-31.tolk new file mode 100644 index 000000000..240d5b398 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-31.tolk @@ -0,0 +1,10 @@ +fun testCantCastBoolToUIntN(x: int) { + var eq = x == 0; + eq as int8; // ok + eq as uint8; +} + +/** +@compilation_should_fail +@stderr type `bool` can not be cast to `uint8` + */ diff --git a/tolk-tester/tests/invalid-typing-32.tolk b/tolk-tester/tests/invalid-typing-32.tolk new file mode 100644 index 000000000..134a8c485 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-32.tolk @@ -0,0 +1,8 @@ +fun testUInt258DoesntExist() { + var c = 0 as uint258; +} + +/** +@compilation_should_fail +@stderr unknown type name `uint258` + */ diff --git a/tolk-tester/tests/invalid-typing-33.tolk b/tolk-tester/tests/invalid-typing-33.tolk new file mode 100644 index 000000000..d112b3606 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-33.tolk @@ -0,0 +1,10 @@ +fun testAssignBetweenDifferentIntN(op: int32, qid: uint64) { + op = qid as int32; // ok + op = qid; +} + +/** +@compilation_should_fail +@stderr can not assign `uint64` to variable of type `int32` +@stderr op = qid; + */ diff --git a/tolk-tester/tests/invalid-typing-34.tolk b/tolk-tester/tests/invalid-typing-34.tolk new file mode 100644 index 000000000..ed40fc48d --- /dev/null +++ b/tolk-tester/tests/invalid-typing-34.tolk @@ -0,0 +1,16 @@ +fun assign0(mutate x: uint8) { + x = 0; +} + +fun testCantPassDifferentIntN(x: int8) { + assign0(mutate (x as int)); // ok + assign0(mutate (x as uint8)); // ok + assign0(mutate ((x as int64) as int)); // ok + assign0(mutate x); +} + +/** +@compilation_should_fail +@stderr can not pass `int8` to `uint8` +@stderr assign0(mutate x); + */ diff --git a/tolk-tester/tests/nullable-tensors.tolk b/tolk-tester/tests/nullable-tensors.tolk index 6de907d02..6b53306b6 100644 --- a/tolk-tester/tests/nullable-tensors.tolk +++ b/tolk-tester/tests/nullable-tensors.tolk @@ -422,6 +422,12 @@ fun test136(x: int?, ) { return ((t1, t1 == null, ), ); // testing trailing comma everywhere :) } +@method_id(140) +fun test140(x: int8, y: int16) { + var t1: (int32, int)? = (x, y) as (int32, int64)?; + var t2: (int32, int)? = null as (int32, int64)?; + return (t1, t2); +} fun main(){} @@ -475,6 +481,7 @@ fun main(){} @testcase | 135 | | [ 10 ] [ (null) ] (null) 777 10 -1 (null) -1 (null) 0 777 10 -1 (null) -1 (null) 0 777 0 0 -1 0 0 -1 0 0 -1 777 0 -1 0 0 -1 0 @testcase | 136 | 9 | 9 0 @testcase | 136 | null | (null) -1 +@testcase | 140 | 8 9 | 8 9 -1 (null) (null) 0 @fif_codegen """ diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 1561aa403..4347de3e8 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -535,7 +535,30 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as()) { + return rvect; + } + // pass `int8` to `int` + // it comes from auto cast when an integer (even a literal) is assigned to intN + // to changes in rvect, intN is int at TVM level + if (target_type == TypeDataInt::create() && original_type->try_as()) { + return rvect; + } + // pass `int` to `int8` + // in code, it's probably done with `as` operator + // no changes in rvect + if (original_type == TypeDataInt::create() && target_type->try_as()) { + return rvect; + } + // pass `int8` to `int16` / `int8` to `uint8` + // in code, it's probably done with `as` operator + // no changes in rvect + if (original_type->try_as() && target_type->try_as()) { return rvect; } // pass something to `unknown` diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index bae67c5fd..5f78b9d25 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -126,7 +126,7 @@ static void handle_possible_compiler_internal_call(FunctionPtr cur_f, Vinferred_type == TypeDataInt::create(); + return v_inferred->inferred_type == TypeDataInt::create() || v_inferred->inferred_type->try_as(); } static bool expect_boolean(AnyExprV v_inferred) { @@ -145,6 +145,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { parent::visit(rhs); // all operators (+=, etc.) can work for integers (if both sides are integers) + // for intN, they are also allowed (int16 |= int8 is ok, since int16 | int8 is ok, all arithmetic is int) bool types_ok = expect_integer(lhs) && expect_integer(rhs); // bitwise operators &= |= ^= are "overloaded" for booleans also (if both sides are booleans) if (!types_ok && (v->tok == tok_set_bitwise_and || v->tok == tok_set_bitwise_or || v->tok == tok_set_bitwise_xor)) { @@ -181,8 +182,9 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { switch (v->tok) { // == != can compare both integers and booleans, (int == bool) is NOT allowed + // for intN, it also works: (int8 == int16) is ok, (int == uint32) is ok // note, that `int?` and `int?` can't be compared, since Fift `EQUAL` works with integers only - // (if to allow `int?` in the future, `==` must be expressed in a complicated Fift code considering TVM NULL) + // (if to allow `int?`/`int8?` in the future, `==` must be expressed in a complicated Fift code considering TVM NULL) case tok_eq: case tok_neq: { bool both_int = expect_integer(lhs) && expect_integer(rhs); @@ -207,6 +209,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } break; // & | ^ are "overloaded" both for integers and booleans, (int & bool) is NOT allowed + // they are allowed for intN (int16 & int32 is ok) and always "fall back" to general int case tok_bitwise_and: case tok_bitwise_or: case tok_bitwise_xor: { @@ -228,6 +231,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { break; } // others are mathematical: + * ... + // they are allowed for intN (int16 + int32 is ok) and always "fall back" to general int default: if (!expect_integer(lhs) || !expect_integer(rhs)) { fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 5fb12059f..ad1c59aeb 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -457,14 +457,13 @@ class InferTypesAndCallsAndFieldsVisitor final { FlowContext rhs_flow = std::move(after_lhs.out_flow); ExprFlow after_rhs = infer_any_expr(rhs, std::move(rhs_flow), false, lhs->inferred_type); - // almost all operators implementation is hardcoded by built-in functions `_+_` and similar + // all operators implementation is hardcoded by built-in functions `_+_` and similar std::string_view builtin_func = v->operator_name; // "+" for operator += assign_inferred_type(v, lhs); - if (!builtin_func.empty()) { - FunctionPtr builtin_sym = lookup_global_symbol("_" + static_cast(builtin_func) + "_")->try_as(); - v->mutate()->assign_fun_ref(builtin_sym); - } + + FunctionPtr builtin_sym = lookup_global_symbol("_" + static_cast(builtin_func) + "_")->try_as(); + v->mutate()->assign_fun_ref(builtin_sym); return ExprFlow(std::move(after_rhs.out_flow), used_as_condition); } @@ -530,7 +529,6 @@ class InferTypesAndCallsAndFieldsVisitor final { } else { assign_inferred_type(v, TypeDataInt::create()); } - assign_inferred_type(v, rhs); // (int & int) is int, (bool & bool) is bool break; // && || result in booleans, but building flow facts is tricky due to short-circuit case tok_logical_and: { @@ -560,6 +558,7 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(out_flow), std::move(true_flow), std::move(false_flow)); } // others are mathematical: + * ... + // they are allowed for intN (int16 + int32 is ok) and always "fall back" to general int default: flow = infer_any_expr(lhs, std::move(flow), false).out_flow; flow = infer_any_expr(rhs, std::move(flow), false).out_flow; diff --git a/tolk/smart-casts-cfg.cpp b/tolk/smart-casts-cfg.cpp index 7b86f519f..9a2f7ef4d 100644 --- a/tolk/smart-casts-cfg.cpp +++ b/tolk/smart-casts-cfg.cpp @@ -192,6 +192,10 @@ static TypePtr calculate_type_lca(TypePtr a, TypePtr b) { return TypeDataTypedTuple::create(std::move(types_lca)); } + if (a->try_as() && b->try_as()) { // cond ? int32 : int64 + return TypeDataInt::create(); + } + return nullptr; } diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index d73625c20..c88acb719 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -18,6 +18,7 @@ #include "lexer.h" #include "platform-utils.h" #include "compiler-state.h" +#include #include namespace tolk { @@ -178,6 +179,18 @@ TypePtr TypeDataTypedTuple::create(std::vector&& items) { return hash.register_unique(new TypeDataTypedTuple(hash.type_id(), hash.children_flags(), std::move(items))); } +TypePtr TypeDataIntN::create(bool is_unsigned, bool is_variadic, int n_bits) { + TypeDataTypeIdCalculation hash(1678330938771108027ULL); + hash.feed_hash(is_unsigned); + hash.feed_hash(is_variadic); + hash.feed_hash(n_bits); + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + return hash.register_unique(new TypeDataIntN(hash.type_id(), is_unsigned, is_variadic, n_bits)); +} + TypePtr TypeDataUnresolved::create(std::string&& text, SrcLocation loc) { TypeDataTypeIdCalculation hash(3680147223540048162ULL); hash.feed_string(text); @@ -240,6 +253,13 @@ std::string TypeDataTypedTuple::as_human_readable() const { return result; } +std::string TypeDataIntN::as_human_readable() const { + std::string s_int = is_variadic + ? is_unsigned ? "varuint" : "varint" + : is_unsigned ? "uint" : "int"; + return s_int + std::to_string(n_bits); +} + // -------------------------------------------- // traverse() @@ -327,6 +347,9 @@ bool TypeDataInt::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (rhs->try_as()) { + return true; + } return rhs == TypeDataNever::create(); } @@ -431,6 +454,16 @@ bool TypeDataTypedTuple::can_rhs_be_assigned(TypePtr rhs) const { return rhs == TypeDataNever::create(); } +bool TypeDataIntN::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + if (rhs == TypeDataInt::create()) { + return true; + } + return rhs == TypeDataNever::create(); // `int8` is NOT assignable to `int32` without `as` +} + bool TypeDataUnknown::can_rhs_be_assigned(TypePtr rhs) const { return true; } @@ -463,14 +496,23 @@ bool TypeDataInt::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { // `int` as `int?` return can_be_casted_with_as_operator(to_nullable->inner); } + if (cast_to->try_as()) { // `int` as `int8` / `int` as `uint2` + return true; + } return cast_to == this; } bool TypeDataBool::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (cast_to == TypeDataInt::create()) { + return true; + } if (const auto* to_nullable = cast_to->try_as()) { return can_be_casted_with_as_operator(to_nullable->inner); } - return cast_to == this || cast_to == TypeDataInt::create(); + if (const auto* to_intN = cast_to->try_as()) { + return !to_intN->is_unsigned; // `bool` as `int8` ok, `bool` as `uintN` not (true is -1) + } + return cast_to == this; } bool TypeDataCell::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -560,6 +602,16 @@ bool TypeDataTypedTuple::can_be_casted_with_as_operator(TypePtr cast_to) const { return false; } +bool TypeDataIntN::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (cast_to->try_as()) { // `int8` as `int32`, `int256` as `uint5`, anything + return true; + } + if (const auto* to_nullable = cast_to->try_as()) { // `int8` as `int32?` + return can_be_casted_with_as_operator(to_nullable->inner); + } + return cast_to == TypeDataInt::create(); +} + bool TypeDataUnknown::can_be_casted_with_as_operator(TypePtr cast_to) const { // 'unknown' can be cast to any TVM value return cast_to->get_width_on_stack() == 1; @@ -657,6 +709,16 @@ std::vector parse_nested_type_list_in_parenthesis(Lexer& lex) { return parse_nested_type_list(lex, tok_oppar, "`(`", tok_clpar, "`)` or `,`"); } +static TypePtr parse_intN(std::string_view strN, bool is_unsigned) { + int n; + auto result = std::from_chars(strN.data() + 3 + static_cast(is_unsigned), strN.data() + strN.size(), n); + bool parsed = result.ec == std::errc() && result.ptr == strN.data() + strN.size(); + if (!parsed || n <= 0 || n > 256 + static_cast(is_unsigned)) { + return nullptr; // `int1000`, maybe it's user-defined alias, let it be unresolved + } + return TypeDataIntN::create(is_unsigned, false, n); +} + static TypePtr parse_simple_type(Lexer& lex) { switch (lex.tok()) { case tok_self: @@ -681,12 +743,26 @@ static TypePtr parse_simple_type(Lexer& lex) { case 7: if (str == "builder") return TypeDataBuilder::create(); break; + case 8: + if (str == "varint16") return TypeDataIntN::create(false, true, 16); + if (str == "varint32") return TypeDataIntN::create(false, true, 32); + break; case 12: if (str == "continuation") return TypeDataContinuation::create(); break; default: break; } + if (str.starts_with("int")) { + if (TypePtr intN = parse_intN(str, false)) { + return intN; + } + } + if (str.size() > 4 && str.starts_with("uint")) { + if (TypePtr uintN = parse_intN(str, true)) { + return uintN; + } + } return TypeDataUnresolved::create(std::string(str), loc); } case tok_null: diff --git a/tolk/type-system.h b/tolk/type-system.h index 4b671e30d..b64fcfca1 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -366,6 +366,31 @@ class TypeDataTypedTuple final : public TypeData { TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; }; +/* + * `int8`, `int32`, `uint1`, `uint257`, `varint16` are TypeDataIntN. At TVM level, it's just int. + * The purpose of intN is to be used in struct fields, describing the way of serialization (n bits). + * A field `value: int32` has the TYPE `int32`, so being assigned to a variable, that variable is also `int32`. + * intN is smoothly cast from/to plain int, mathematical operators on intN also "fall back" to general int. + */ +class TypeDataIntN final : public TypeData { + TypeDataIntN(uint64_t type_id, bool is_unsigned, bool is_variadic, int n_bits) + : TypeData(type_id, 0, 1) + , is_unsigned(is_unsigned) + , is_variadic(is_variadic) + , n_bits(n_bits) {} + +public: + const bool is_unsigned; + const bool is_variadic; + const int n_bits; + + static TypePtr create(bool is_unsigned, bool is_variadic, int n_bits); + + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + /* * `unknown` is a special type, which can appear in corner cases. * The type of exception argument (which can hold any TVM value at runtime) is unknown. From 620a12d373a6ff0ef95c4d035f0b7acb4152e7ea Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 13 Mar 2025 18:01:10 +0400 Subject: [PATCH 156/388] [Tolk] Support intN and bool constants Constant evaluator has been rewritten to utilize type inferring mechanism. Now, > const a = 1 as int8; is valid, and constants can not only be int/slice, but also intN/uintN and bool. --- tolk-tester/tests/co1.tolk | 32 ++++++- tolk-tester/tests/intN-tests.tolk | 9 +- tolk-tester/tests/invalid-cyclic-1.tolk | 3 +- tolk-tester/tests/invalid-declaration-13.tolk | 2 +- tolk-tester/tests/invalid-declaration-14.tolk | 7 ++ tolk-tester/tests/invalid-declaration-15.tolk | 6 ++ tolk-tester/tests/invalid-declaration-16.tolk | 9 ++ tolk-tester/tests/invalid-typing-35.tolk | 6 ++ tolk/ast-replacer.h | 1 + tolk/ast-visitor.h | 1 + tolk/compiler-state.cpp | 4 + tolk/constant-evaluator.cpp | 94 ++++++++++--------- tolk/constant-evaluator.h | 30 +++--- tolk/pipe-ast-to-legacy.cpp | 8 +- tolk/pipe-check-inferred-types.cpp | 20 ++++ tolk/pipe-constant-folding.cpp | 9 ++ tolk/pipe-infer-types-and-calls.cpp | 41 +++++++- tolk/pipe-register-symbols.cpp | 14 +-- tolk/pipe-resolve-identifiers.cpp | 6 ++ tolk/symtable.cpp | 8 ++ tolk/symtable.h | 16 ++-- 21 files changed, 238 insertions(+), 88 deletions(-) create mode 100644 tolk-tester/tests/invalid-declaration-14.tolk create mode 100644 tolk-tester/tests/invalid-declaration-15.tolk create mode 100644 tolk-tester/tests/invalid-declaration-16.tolk create mode 100644 tolk-tester/tests/invalid-typing-35.tolk diff --git a/tolk-tester/tests/co1.tolk b/tolk-tester/tests/co1.tolk index f124e1de8..66b89190a 100644 --- a/tolk-tester/tests/co1.tolk +++ b/tolk-tester/tests/co1.tolk @@ -16,6 +16,16 @@ const str2int = 0xAABBCC; const nibbles: int = 4; +const strange_zero = (!10 as int); +const strange_minus_1: int = (!0 as int); + +const true1 = true; +const true2 = !!true; +const true3 = true1 && true2; + +const false1 = !true; +const false2 = false1 || false; + fun iget1(): int { return int1; } fun iget2(): int { return int2; } fun iget3(): int { return int1+int2; } @@ -43,6 +53,21 @@ asm "SDEQ"; fun stslicer(b: builder, s: slice): builder asm "STSLICER"; +@method_id(101) +fun test1() { + return (strange_zero, strange_minus_1); +} + +@method_id(102) +fun test2() { + return (true1, true2, true3); +} + +@method_id(103) +fun test3() { + return (false1, false2); +} + fun main() { var i1: int = iget1(); var i2: int = iget2(); @@ -66,7 +91,10 @@ fun main() { } /** -@testcase | 0 | | 0 +@testcase | 0 | | 0 +@testcase | 101 | | 0 -1 +@testcase | 102 | | -1 -1 -1 +@testcase | 103 | | 0 0 -@code_hash 61273295789179921867241079778489100375537711211918844448475493726205774530743 +@code_hash 28102194299745406750019953961984060488024870092664444642078578246708959881688 */ diff --git a/tolk-tester/tests/intN-tests.tolk b/tolk-tester/tests/intN-tests.tolk index 3f4cdc015..16aa67168 100644 --- a/tolk-tester/tests/intN-tests.tolk +++ b/tolk-tester/tests/intN-tests.tolk @@ -3,6 +3,9 @@ fun getAnyInt(): int { return 0; } fun getNullableInt32(): int32? { return 32; } fun getNullableVarInt32(): varint32? { return 32; } +const someN_u128: uint128 = 128; +const someN_i32: int32 = 20 + (12 as int32) * (1 as uint8); + fun autoInferInt8(x: int8) { if (random()) { return x; } @@ -50,7 +53,8 @@ fun test1(x: int8) { var z = x + y; __expect_type(z, "int"); - return x / y; + __expect_type(someN_u128, "uint128"); + return x / y + someN_u128; } fun test2(): (uint8, uint8, uint8) { @@ -60,6 +64,7 @@ fun test2(): (uint8, uint8, uint8) { fun test3(op: int32, qid: uint64) { op = qid as int32; + op = someN_i32; op + qid; op = op + qid; @@ -166,7 +171,7 @@ fun main() { /** @testcase | 0 | | [ 0 0 0 ] -@testcase | 101 | 0 | 1 +@testcase | 101 | 0 | 129 @testcase | 104 | | 64 64 -1 0 @testcase | 105 | | 3 -1 0 @testcase | 114 | 5 | (null) (null) 0 diff --git a/tolk-tester/tests/invalid-cyclic-1.tolk b/tolk-tester/tests/invalid-cyclic-1.tolk index c46b1640e..0f259fa68 100644 --- a/tolk-tester/tests/invalid-cyclic-1.tolk +++ b/tolk-tester/tests/invalid-cyclic-1.tolk @@ -3,6 +3,5 @@ const TWO = ONE + 1; /** @compilation_should_fail -@stderr const ONE -@stderr undefined symbol `TWO` +@stderr const `TWO` appears, directly or indirectly, in its own initializer */ diff --git a/tolk-tester/tests/invalid-declaration-13.tolk b/tolk-tester/tests/invalid-declaration-13.tolk index 758a4f21d..ada0ef5f4 100644 --- a/tolk-tester/tests/invalid-declaration-13.tolk +++ b/tolk-tester/tests/invalid-declaration-13.tolk @@ -2,6 +2,6 @@ const c: slice = 123 + 456; /** @compilation_should_fail -@stderr expression type does not match declared type +@stderr can not assign `int` to `slice` @stderr const c */ diff --git a/tolk-tester/tests/invalid-declaration-14.tolk b/tolk-tester/tests/invalid-declaration-14.tolk new file mode 100644 index 000000000..339addfb5 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-14.tolk @@ -0,0 +1,7 @@ +const ttt = 1; +const asdf = ttt; + +/** +@compilation_should_fail +@stderr generic T not expected here + */ diff --git a/tolk-tester/tests/invalid-declaration-15.tolk b/tolk-tester/tests/invalid-declaration-15.tolk new file mode 100644 index 000000000..72f393eff --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-15.tolk @@ -0,0 +1,6 @@ +const asdf = ttt; + +/** +@compilation_should_fail +@stderr undefined symbol `ttt` + */ diff --git a/tolk-tester/tests/invalid-declaration-16.tolk b/tolk-tester/tests/invalid-declaration-16.tolk new file mode 100644 index 000000000..1dae9c9b2 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-16.tolk @@ -0,0 +1,9 @@ +fun foo() { return 1; } + +const asdf = 1 + foo(); + +/** +@compilation_should_fail +@stderr not a constant expression +@stderr const asdf + */ diff --git a/tolk-tester/tests/invalid-typing-35.tolk b/tolk-tester/tests/invalid-typing-35.tolk new file mode 100644 index 000000000..b43aaef55 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-35.tolk @@ -0,0 +1,6 @@ +const asdf = 1 + ""; + +/** +@compilation_should_fail +@stderr can not apply operator `+` to `int` and `slice` + */ diff --git a/tolk/ast-replacer.h b/tolk/ast-replacer.h index 5103cc923..d1ab4c450 100644 --- a/tolk/ast-replacer.h +++ b/tolk/ast-replacer.h @@ -187,6 +187,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { const std::vector& get_all_not_builtin_functions(); +const std::vector& get_all_declared_constants(); template void replace_ast_of_all_functions() { diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h index d697aa82b..737646bb5 100644 --- a/tolk/ast-visitor.h +++ b/tolk/ast-visitor.h @@ -180,6 +180,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { const std::vector& get_all_not_builtin_functions(); +const std::vector& get_all_declared_constants(); template void visit_ast_of_all_functions() { diff --git a/tolk/compiler-state.cpp b/tolk/compiler-state.cpp index 95a7e6a59..e02d0220a 100644 --- a/tolk/compiler-state.cpp +++ b/tolk/compiler-state.cpp @@ -70,4 +70,8 @@ const std::vector& get_all_not_builtin_functions() { return G.all_functions; } +const std::vector& get_all_declared_constants() { + return G.all_constants; +} + } // namespace tolk diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp index 4d11b9223..214fb729d 100644 --- a/tolk/constant-evaluator.cpp +++ b/tolk/constant-evaluator.cpp @@ -147,6 +147,12 @@ static td::RefInt256 parse_vertex_string_const_as_int(V v) { } } +static ConstantValue parse_vertex_string_const(V v) { + return v->is_bitslice() + ? ConstantValue(parse_vertex_string_const_as_slice(v)) + : ConstantValue(parse_vertex_string_const_as_int(v)); +} + struct ConstantEvaluator { static bool is_overflow(const td::RefInt256& intval) { @@ -154,10 +160,8 @@ struct ConstantEvaluator { } static ConstantValue handle_unary_operator(V v, const ConstantValue& rhs) { - if (!rhs.is_int()) { - v->error("invalid operator, expecting integer"); - } - td::RefInt256 intval = std::get(rhs.value); + tolk_assert(rhs.is_int()); // type inferring has passed before, so it's int/bool + td::RefInt256 intval = rhs.as_int(); switch (v->tok) { case tok_minus: @@ -167,7 +171,7 @@ struct ConstantEvaluator { break; case tok_bitwise_not: intval = ~intval; - break; + break; case tok_logical_not: intval = td::make_refint(intval == 0 ? -1 : 0); break; @@ -178,15 +182,13 @@ struct ConstantEvaluator { if (is_overflow(intval)) { v->error("integer overflow"); } - return ConstantValue::from_int(std::move(intval)); + return ConstantValue(std::move(intval)); } static ConstantValue handle_binary_operator(V v, const ConstantValue& lhs, const ConstantValue& rhs) { - if (!lhs.is_int() || !rhs.is_int()) { - v->error("invalid operator, expecting integer"); - } - td::RefInt256 lhs_intval = std::get(lhs.value); - td::RefInt256 rhs_intval = std::get(rhs.value); + tolk_assert(lhs.is_int() && rhs.is_int()); // type inferring has passed before, so they are int/bool + td::RefInt256 lhs_intval = lhs.as_int(); + td::RefInt256 rhs_intval = rhs.as_int(); td::RefInt256 intval; switch (v->tok) { @@ -238,6 +240,12 @@ struct ConstantEvaluator { case tok_neq: intval = td::make_refint(lhs_intval != rhs_intval ? -1 : 0); break; + case tok_logical_and: + intval = td::make_refint(lhs_intval != 0 && rhs_intval != 0 ? -1 : 0); + break; + case tok_logical_or: + intval = td::make_refint(lhs_intval != 0 || rhs_intval != 0 ? -1 : 0); + break; default: v->error("unsupported binary operator in constant expression"); } @@ -245,32 +253,28 @@ struct ConstantEvaluator { if (is_overflow(intval)) { v->error("integer overflow"); } - return ConstantValue::from_int(std::move(intval)); + return ConstantValue(std::move(intval)); } + // `const a = 1 + b`, we met `b` static ConstantValue handle_reference(V v) { - // todo better handle "appears, directly or indirectly, in its own initializer" - std::string_view name = v->get_name(); - const Symbol* sym = lookup_global_symbol(name); - if (!sym) { - v->error("undefined symbol `" + static_cast(name) + "`"); - } - GlobalConstPtr const_ref = sym->try_as(); + GlobalConstPtr const_ref = v->sym->try_as(); if (!const_ref) { - v->error("symbol `" + static_cast(name) + "` is not a constant"); + v->error("symbol `" + static_cast(v->get_name()) + "` is not a constant"); } - if (v->has_instantiationTs()) { // SOME_CONST - v->error("constant is not a generic"); + + if (!const_ref->value.initialized()) { // maybe, `b` was already calculated + eval_and_assign_const_init_value(const_ref); // if not, dig recursively into `b` } - return {const_ref->value}; + return const_ref->value; } static ConstantValue visit(AnyExprV v) { if (auto v_int = v->try_as()) { - return ConstantValue::from_int(v_int->intval); + return ConstantValue(v_int->intval); } if (auto v_bool = v->try_as()) { - return ConstantValue::from_int(v_bool->bool_val ? -1 : 0); + return ConstantValue(v_bool->bool_val ? -1 : 0); } if (auto v_unop = v->try_as()) { return handle_unary_operator(v_unop, visit(v_unop->get_rhs())); @@ -284,34 +288,32 @@ struct ConstantEvaluator { if (auto v_par = v->try_as()) { return visit(v_par->get_expr()); } - if (v->try_as()) { - return eval_const_init_value(v); + if (auto v_cast_to = v->try_as()) { + return visit(v_cast_to->get_expr()); + } + if (auto v_string = v->try_as()) { + return parse_vertex_string_const(v_string); } v->error("not a constant expression"); } - static ConstantValue eval_const_init_value(AnyExprV init_value) { - // it init_value is incorrect, an exception is thrown - return visit(init_value); + // evaluate `const a = 2 + 3` into 5 + // type inferring has already passed, to types are correct, `const a = 1 + ""` can not occur + // recursive initializers `const a = b; const b = a` also 100% don't exist, checked on type inferring + // if init_value is not a constant expression like `const a = foo()`, an exception is thrown + static ConstantValue eval_const_init_value(GlobalConstPtr const_ref) { + return visit(const_ref->init_value); } }; -ConstantValue eval_const_init_value(AnyExprV init_value) { - // at first, handle most simple cases, not to launch heavy computation algorithm: just a number, just a string - // just `c = 1` or `c = 0xFF` - if (auto v_int = init_value->try_as()) { - return {v_int->intval}; - } - // just `c = "strval"`, probably with modifier (address, etc.) - if (auto v_string = init_value->try_as()) { - if (v_string->is_bitslice()) { - return {parse_vertex_string_const_as_slice(v_string)}; - } else { - return {parse_vertex_string_const_as_int(v_string)}; - } - } - // something more complex, like `c = anotherC` or `c = 1 << 8` - return ConstantEvaluator::eval_const_init_value(init_value); +ConstantValue eval_string_const_considering_modifier(AnyExprV v_string) { + tolk_assert(v_string->type == ast_string_const); + return parse_vertex_string_const(v_string->as()); +} + +void eval_and_assign_const_init_value(GlobalConstPtr const_ref) { + ConstantValue init_value = ConstantEvaluator::eval_const_init_value(const_ref); + const_ref->mutate()->assign_const_value(std::move(init_value)); } } // namespace tolk diff --git a/tolk/constant-evaluator.h b/tolk/constant-evaluator.h index 0f99867d8..218bd8c4b 100644 --- a/tolk/constant-evaluator.h +++ b/tolk/constant-evaluator.h @@ -22,24 +22,32 @@ namespace tolk { -struct ConstantValue { - std::variant value; +class ConstantValue { + std::variant< + td::RefInt256, // is set for int, intN, coins, bool + std::string // is set for slice, bytesN + > value; + +public: + ConstantValue() = default; // by default, not initialized + + explicit ConstantValue(int value) + : value(td::make_refint(value)) {} + explicit ConstantValue(td::RefInt256 value) + : value(std::move(value)) {} + explicit ConstantValue(std::string value) + : value(std::move(value)) {} + + bool initialized() const { return is_slice() || std::get(value).not_null(); } bool is_int() const { return std::holds_alternative(value); } bool is_slice() const { return std::holds_alternative(value); } td::RefInt256 as_int() const { return std::get(value); } const std::string& as_slice() const { return std::get(value); } - - static ConstantValue from_int(int value) { - return {td::make_refint(value)}; - } - - static ConstantValue from_int(td::RefInt256 value) { - return {std::move(value)}; - } }; -ConstantValue eval_const_init_value(AnyExprV init_value); +ConstantValue eval_string_const_considering_modifier(AnyExprV v_string); +void eval_and_assign_const_init_value(GlobalConstPtr const_ref); } // namespace tolk diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 4347de3e8..476559d35 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -644,13 +644,13 @@ std::vector pre_compile_symbol(SrcLocation loc, const Symbol* sym, Co return local_ir_idx; } if (GlobalConstPtr const_ref = sym->try_as()) { - if (const_ref->is_int_const()) { + if (const_ref->value.is_int()) { std::vector rvect = code.create_tmp_var(TypeDataInt::create(), loc, "(glob-const)"); - code.emplace_back(loc, Op::_IntConst, rvect, const_ref->as_int_const()); + code.emplace_back(loc, Op::_IntConst, rvect, const_ref->value.as_int()); return rvect; } else { std::vector rvect = code.create_tmp_var(TypeDataSlice::create(), loc, "(glob-const)"); - code.emplace_back(loc, Op::_SliceConst, rvect, const_ref->as_slice_const()); + code.emplace_back(loc, Op::_SliceConst, rvect, const_ref->value.as_slice()); return rvect; } } @@ -1013,7 +1013,7 @@ static std::vector process_int_const(V v, CodeBlob& co } static std::vector process_string_const(V v, CodeBlob& code, TypePtr target_type) { - ConstantValue value = eval_const_init_value(v); + ConstantValue value = eval_string_const_considering_modifier(v); std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(str-const)"); if (value.is_int()) { code.emplace_back(v->loc, Op::_IntConst, rvect, value.as_int()); diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 5f78b9d25..97cabd1fe 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -581,10 +581,30 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } } } + + // given `const a = 2 + 3` check types within its init_value + // so, `const a = 1 + some_slice` will fire a reasonable error + void start_visiting_constant(GlobalConstPtr const_ref) { + parent::visit(const_ref->init_value); + + // if no errors occurred, init value has correct type + // (though it may not be a valid constant expression, this would be checked after type inferring) + if (const_ref->declared_type) { // `const a: int = ...` + TypePtr inferred_type = const_ref->init_value->inferred_type; + if (!const_ref->declared_type->can_rhs_be_assigned(inferred_type)) { + throw ParseError(const_ref->loc, "can not assign " + to_string(inferred_type) + " to " + to_string(const_ref->declared_type)); + } + } + } }; void pipeline_check_inferred_types() { visit_ast_of_all_functions(); + + CheckInferredTypesVisitor visitor; + for (GlobalConstPtr const_ref : get_all_declared_constants()) { + visitor.start_visiting_constant(const_ref); + } } } // namespace tolk diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp index 9c27029b5..4b0fe9723 100644 --- a/tolk/pipe-constant-folding.cpp +++ b/tolk/pipe-constant-folding.cpp @@ -106,6 +106,15 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { }; void pipeline_constant_folding() { + // here (after type inferring) evaluate `const a = 2 + 3` into `5` + // non-constant expressions like `const a = foo()` fire an error here + for (GlobalConstPtr const_ref : get_all_declared_constants()) { + // for `const a = b`, `b` could be already calculated while calculating `a` + if (!const_ref->value.initialized()) { + eval_and_assign_const_init_value(const_ref); + } + } + replace_ast_of_all_functions(); } diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index ad1c59aeb..58d28d421 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -96,6 +96,7 @@ namespace tolk { static void infer_and_save_return_type_of_function(FunctionPtr fun_ref); +static void infer_and_save_type_of_constant(GlobalConstPtr const_ref); static TypePtr get_or_infer_return_type(FunctionPtr fun_ref) { if (!fun_ref->inferred_return_type) { @@ -677,7 +678,10 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, declared_or_smart_casted); } else if (GlobalConstPtr const_ref = v->sym->try_as()) { - assign_inferred_type(v, const_ref->is_int_const() ? TypeDataInt::create() : TypeDataSlice::create()); + if (!const_ref->inferred_type) { + infer_and_save_type_of_constant(const_ref); + } + assign_inferred_type(v, const_ref->inferred_type); } else if (GlobalVarPtr glob_ref = v->sym->try_as()) { // there are no smart casts for globals, it's a way of preventing reading one global multiple times, it costs gas @@ -1245,6 +1249,14 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_fun_full_type(fun_ref, inferred_return_type); fun_ref->mutate()->assign_is_type_inferring_done(); } + + // given `const a = 2 + 3` infer that it's int + // for `const a: int = ...` still infer all sub expressions (to be checked in a later pipe) + void start_visiting_constant(GlobalConstPtr const_ref) { + FlowContext const_flow; + infer_any_expr(const_ref->init_value, std::move(const_flow), false, const_ref->declared_type); + const_ref->mutate()->assign_inferred_type(const_ref->declared_type == nullptr ? const_ref->init_value->inferred_type : const_ref->declared_type); + } }; class LaunchInferTypesAndMethodsOnce final { @@ -1288,8 +1300,35 @@ static void infer_and_save_return_type_of_function(FunctionPtr fun_ref) { called_stack.pop_back(); } +// infer constant type "on demand" +// example: `const a = 1 + b;` +// when analyzing `a`, we need to infer what type const_ref=b has +static void infer_and_save_type_of_constant(GlobalConstPtr const_ref) { + static std::vector called_stack; + + // prevent recursion like `const a = b; const b = a` + bool contains = std::find(called_stack.begin(), called_stack.end(), const_ref) != called_stack.end(); + if (contains) { + throw ParseError(const_ref->loc, "const `" + const_ref->name + "` appears, directly or indirectly, in its own initializer"); + } + + called_stack.push_back(const_ref); + InferTypesAndCallsAndFieldsVisitor visitor; + visitor.start_visiting_constant(const_ref); + called_stack.pop_back(); +} + void pipeline_infer_types_and_calls_and_fields() { visit_ast_of_all_functions(); + + // analyze constants that weren't referenced by any function + InferTypesAndCallsAndFieldsVisitor visitor; + for (GlobalConstPtr const_ref : get_all_declared_constants()) { + if (!const_ref->inferred_type) { + visitor.start_visiting_constant(const_ref); + } + } + // (later, at constant folding, `const a = 2 + 3` will be evaluated to 5) } void pipeline_infer_types_and_calls_and_fields(FunctionPtr fun_ref) { diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index 45246d6b9..ea4cc8110 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -115,16 +115,10 @@ static const GenericsDeclaration* construct_genericTs(V v_li } static void register_constant(V v) { - ConstantValue init_value = eval_const_init_value(v->get_init_value()); - GlobalConstData* c_sym = new GlobalConstData(static_cast(v->get_identifier()->name), v->loc, v->declared_type, std::move(init_value)); - - if (v->declared_type) { - bool ok = (c_sym->is_int_const() && (v->declared_type == TypeDataInt::create())) - || (c_sym->is_slice_const() && (v->declared_type == TypeDataSlice::create())); - if (!ok) { - v->error("expression type does not match declared type"); - } - } + GlobalConstData* c_sym = new GlobalConstData(static_cast(v->get_identifier()->name), v->loc, v->declared_type, v->get_init_value()); + // init value of constant is not evaluated here + // at first, it will be type checked (in type inference pipe) + // then, at constant folding pipe, `const a = 2 + 3` will be evaluated to 5 G.symtable.add_global_const(c_sym); G.all_constants.push_back(c_sym); diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 5a735885e..2ace2e719 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -313,6 +313,11 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { current_genericTs = nullptr; cur_f = nullptr; } + + void start_visiting_constant(V v) { + // `const a = b`, resolve `b` + parent::visit(v->get_init_value()); + } }; NameAndScopeResolver AssignSymInsideFunctionVisitor::current_scope; @@ -333,6 +338,7 @@ void pipeline_resolve_identifiers_and_assign_symbols() { v_global->var_ref->mutate()->assign_resolved_type(declared_type); } else if (auto v_const = v->try_as()) { + visitor.start_visiting_constant(v_const); if (v_const->declared_type) { TypePtr declared_type = finalize_type_data(nullptr, v_const->const_ref->declared_type, nullptr); v_const->mutate()->assign_resolved_type(declared_type); diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 51dc34401..ea009957a 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -93,6 +93,14 @@ void GlobalConstData::assign_resolved_type(TypePtr declared_type) { this->declared_type = declared_type; } +void GlobalConstData::assign_inferred_type(TypePtr inferred_type) { + this->inferred_type = inferred_type; +} + +void GlobalConstData::assign_const_value(ConstantValue&& value) { + this->value = std::move(value); +} + void LocalVarData::assign_ir_idx(std::vector&& ir_idx) { this->ir_idx = std::move(ir_idx); } diff --git a/tolk/symtable.h b/tolk/symtable.h index 9419afce6..e65327fdb 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -197,23 +197,21 @@ struct GlobalVarData final : Symbol { }; struct GlobalConstData final : Symbol { + AnyExprV init_value; ConstantValue value; - TypePtr declared_type; // may be nullptr + TypePtr declared_type; // `const a: int = ...`; nullptr for `const a = ...` + TypePtr inferred_type = nullptr; // filled at type inferring pass - GlobalConstData(std::string name, SrcLocation loc, TypePtr declared_type, ConstantValue&& value) + GlobalConstData(std::string name, SrcLocation loc, TypePtr declared_type, AnyExprV init_value) : Symbol(std::move(name), loc) - , value(std::move(value)) + , init_value(init_value) , declared_type(declared_type) { } - bool is_int_const() const { return value.is_int(); } - bool is_slice_const() const { return value.is_slice(); } - - td::RefInt256 as_int_const() const { return value.as_int(); } - const std::string& as_slice_const() const { return value.as_slice(); } - GlobalConstData* mutate() const { return const_cast(this); } void assign_resolved_type(TypePtr declared_type); + void assign_inferred_type(TypePtr inferred_type); + void assign_const_value(ConstantValue&& value); }; class GlobalSymbolTable { From b086bc7ddeb92ef2e9c9ac5571b44bf5aa6d8139 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 13 Mar 2025 18:08:49 +0400 Subject: [PATCH 157/388] [Tolk] Type `coins`, function `ton("0.05")` A new compile-time function is introduced. It converts a constant floating-point string to nanotoncoins. Example: `ton("0.05")` is equal to 50000000. It requires a constant string; `ton(some_var)` is an error. Type of its return value is `coins`, not `int`. Like `intN`, it's backed by a general int at TVM level, all arithmetics (except addition) degrade to int. Type `coins` will be serialized as varint16 in the future. --- crypto/smartcont/tolk-stdlib/common.tolk | 11 +++- tolk-tester/tests/intN-tests.tolk | 78 ++++++++++++++++++++++++ tolk-tester/tests/invalid-call-12.tolk | 8 +++ tolk-tester/tests/invalid-call-13.tolk | 8 +++ tolk-tester/tests/invalid-call-14.tolk | 6 ++ tolk-tester/tests/invalid-typing-36.tolk | 12 ++++ tolk-tester/tests/invalid-typing-37.tolk | 11 ++++ tolk-tester/tests/invalid-typing-38.tolk | 10 +++ tolk/builtins.cpp | 10 +++ tolk/constant-evaluator.cpp | 77 +++++++++++++++++++++++ tolk/constant-evaluator.h | 1 + tolk/pipe-ast-to-legacy.cpp | 20 ++++++ tolk/pipe-check-inferred-types.cpp | 2 +- tolk/pipe-constant-folding.cpp | 13 ++++ tolk/pipe-infer-types-and-calls.cpp | 6 +- tolk/type-system.cpp | 34 ++++++++++- tolk/type-system.h | 18 ++++++ 17 files changed, 320 insertions(+), 5 deletions(-) create mode 100644 tolk-tester/tests/invalid-call-12.tolk create mode 100644 tolk-tester/tests/invalid-call-13.tolk create mode 100644 tolk-tester/tests/invalid-call-14.tolk create mode 100644 tolk-tester/tests/invalid-typing-36.tolk create mode 100644 tolk-tester/tests/invalid-typing-37.tolk create mode 100644 tolk-tester/tests/invalid-typing-38.tolk diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 82757c22b..9647243f9 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -54,6 +54,13 @@ fun tupleLast(self: tuple): T Mathematical primitives. */ +/// Converts a constant floating-point string to nanotoncoins. +/// Example: `ton("0.05")` is equal to 50000000. +/// Note, that `ton()` requires a constant string; `ton(some_var)` is an error +@pure +fun ton(floatString: slice): coins + builtin; + /// Computes the minimum of two integers. @pure fun min(x: int, y: int): int @@ -386,7 +393,7 @@ fun preloadBits(self: slice, len: int): slice /// Loads serialized amount of Toncoins (any unsigned integer up to `2^120 - 1`). @pure -fun loadCoins(mutate self: slice): int +fun loadCoins(mutate self: slice): coins asm( -> 1 0) "LDGRAMS"; /// Loads bool (-1 or 0) from a slice @@ -485,7 +492,7 @@ fun storeSlice(mutate self: builder, s: slice): self /// Stores amount of Toncoins into a builder. @pure -fun storeCoins(mutate self: builder, x: int): self +fun storeCoins(mutate self: builder, x: coins): self asm "STGRAMS"; /// Stores bool (-1 or 0) into a builder. diff --git a/tolk-tester/tests/intN-tests.tolk b/tolk-tester/tests/intN-tests.tolk index 16aa67168..abafdf1d7 100644 --- a/tolk-tester/tests/intN-tests.tolk +++ b/tolk-tester/tests/intN-tests.tolk @@ -5,6 +5,10 @@ fun getNullableVarInt32(): varint32? { return 32; } const someN_u128: uint128 = 128; const someN_i32: int32 = 20 + (12 as int32) * (1 as uint8); +const someN_coins = 10 as coins; + +const cost1 = ton("1.234"); +const cost2 = ton("0.05") + ton("0.001"); fun autoInferInt8(x: int8) { @@ -54,6 +58,7 @@ fun test1(x: int8) { var z = x + y; __expect_type(z, "int"); __expect_type(someN_u128, "uint128"); + __expect_type(someN_coins, "coins"); return x / y + someN_u128; } @@ -141,6 +146,65 @@ fun test7() { __expect_type((n as varint16) as int, "int"); } +fun test8(amount1: coins, amount2: coins) { + __expect_type(amount1 + 0, "coins"); + __expect_type(amount1 + amount2, "coins"); + __expect_type(amount1 += amount2, "coins"); + __expect_type(amount1 - 10, "coins"); + __expect_type(amount1 &= 1, "coins"); + __expect_type(amount1 &= (1 as coins), "coins"); + + __expect_type(amount1 & 1, "int"); + __expect_type(amount1 * 10, "int"); + __expect_type(amount1 << amount2, "int"); +} + +@method_id(109) +fun test9(c: coins, i: int8) { + __expect_type(c as int8, "int8"); + __expect_type(i as coins, "coins"); + __expect_type(c + i, "coins"); + + var amount: coins = 1000; + var percent: uint8 = 50; + var new = amount * percent / 100; + amount = new; + __expect_type(amount, "coins"); + if (!amount) { while (amount) { amount -= 1; } } + + takeAnyInt(amount); + takeAnyInt(percent); + takeAnyInt(true as int3); + takeAnyInt(amount as int3); + + return (amount, (true as int8) as coins); +} + +@method_id(110) +fun test10() { + return (ton("0.05"), ton("0.05") + 100, cost1, cost2); +} + +@method_id(111) +fun test11() { + __expect_type(ton("0"), "coins"); + __expect_type(ton("0") + ton("0"), "coins"); + __expect_type(ton("0") * 20, "int"); + return [ + ton("1"), + ton("1.0"), + ton("1.00000"), + ton("-321.123456789"), + ton("+321.123456789876"), + ton("0001.1000") + ]; +} + +fun test12() { + var a = ton("1") + ton("2"); // check absence in fif codegen + return ton("0.1"); +} + fun test13(x1: int?, x2: int8?, x3: int, x4: int8): (int8?, int8?, int8?, int8?, int32?, int32?) { return (x1, x2, x3, x4, x4 as int32, x4 as int32?); } @@ -165,6 +229,9 @@ fun main() { __expect_type(0 as int32, "int32"); __expect_type((0 as int32) as uint64?, "uint64?"); __expect_type(null as (int8, [uint16?])?, "(int8, [uint16?])?"); + __expect_type(0 as coins, "coins"); + __expect_type(someN_coins as coins?, "coins?"); + __expect_type(someN_coins as int8?, "int8?"); __expect_type(10>3 ? 0 as uint8 : 0 as uint16, "int"); return t; } @@ -174,10 +241,21 @@ fun main() { @testcase | 101 | 0 | 129 @testcase | 104 | | 64 64 -1 0 @testcase | 105 | | 3 -1 0 +@testcase | 109 | 4 4 | 500 -1 +@testcase | 110 | | 50000000 50000100 1234000000 51000000 +@testcase | 111 | | [ 1000000000 1000000000 1000000000 -321123456789 321123456789 1100000000 ] @testcase | 114 | 5 | (null) (null) 0 @testcase | 114 | 15 | 15 2 -1 @fif_codegen DECLPROC assign0 @fif_codegen DECLPROC assign0 @fif_codegen DECLPROC assign0 + +@fif_codegen +""" + test12 PROC:<{ + // + 100000000 PUSHINT // '4=100000000 + }> +""" */ diff --git a/tolk-tester/tests/invalid-call-12.tolk b/tolk-tester/tests/invalid-call-12.tolk new file mode 100644 index 000000000..a0f9bea13 --- /dev/null +++ b/tolk-tester/tests/invalid-call-12.tolk @@ -0,0 +1,8 @@ +fun main(am: slice) { + return ton(am); +} + +/** +@compilation_should_fail +@stderr function `ton` requires a constant string, like `ton("0.05")` + */ diff --git a/tolk-tester/tests/invalid-call-13.tolk b/tolk-tester/tests/invalid-call-13.tolk new file mode 100644 index 000000000..11cce6035 --- /dev/null +++ b/tolk-tester/tests/invalid-call-13.tolk @@ -0,0 +1,8 @@ +fun main() { + return ton(123); +} + +/** +@compilation_should_fail +@stderr function `ton` requires a constant string, like `ton("0.05")` + */ diff --git a/tolk-tester/tests/invalid-call-14.tolk b/tolk-tester/tests/invalid-call-14.tolk new file mode 100644 index 000000000..2d15d9fb1 --- /dev/null +++ b/tolk-tester/tests/invalid-call-14.tolk @@ -0,0 +1,6 @@ +const ss = ton("0.0.0"); + +/** +@compilation_should_fail +@stderr argument is not a valid number like "0.05" + */ diff --git a/tolk-tester/tests/invalid-typing-36.tolk b/tolk-tester/tests/invalid-typing-36.tolk new file mode 100644 index 000000000..d21099eb8 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-36.tolk @@ -0,0 +1,12 @@ +fun cantCastBoolToCoins() { + var k1 = true as int8; // ok + var k2 = true as int1; // ok + var k3 = (true as int) as coins; // ok + var k4 = true as coins; +} + +/** +@compilation_should_fail +@stderr type `bool` can not be cast to `coins` +@stderr true as coins + */ diff --git a/tolk-tester/tests/invalid-typing-37.tolk b/tolk-tester/tests/invalid-typing-37.tolk new file mode 100644 index 000000000..f55c6dd48 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-37.tolk @@ -0,0 +1,11 @@ +fun cantAssignIntNToCoins(n: int8, c: coins) { + n = c as int8; // ok + n = c as int; // ok + n = c; +} + +/** +@compilation_should_fail +@stderr can not assign `coins` to variable of type `int8` +@stderr n = c; + */ diff --git a/tolk-tester/tests/invalid-typing-38.tolk b/tolk-tester/tests/invalid-typing-38.tolk new file mode 100644 index 000000000..58208caab --- /dev/null +++ b/tolk-tester/tests/invalid-typing-38.tolk @@ -0,0 +1,10 @@ +fun cantUnifyCoinsAndUInt8(n: int8, c: coins) { + __expect_type(random() ? n : c as int8, "int8"); + __expect_type(random() ? n : c as int16, "int"); + random() ? n : c; +} + +/** +@compilation_should_fail +@stderr types of ternary branches are incompatible: `int8` and `coins` + */ diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index cb89c984c..9ef4527ae 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1071,6 +1071,13 @@ AsmOp compile_tuple_set_at(std::vector& res, std::vector& ar return exec_op("SETINDEXVAR", 2, 1); } +// fun ton(amount: slice): coins; ton("0.05") replaced by 50000000 at compile-time +AsmOp compile_int_to_ton_coins(std::vector&, std::vector&, SrcLocation) { + // all ton() invocations are constants, replaced by integers; no dynamic values allowed, no work at runtime + tolk_assert(false); + return AsmOp::Nop(); +} + // fun __isNull(X arg): bool AsmOp compile_is_null(std::vector& res, std::vector& args, SrcLocation) { tolk_assert(args.size() == 1 && res.size() == 1); @@ -1214,6 +1221,9 @@ void define_builtins() { // functions from stdlib marked as `builtin`, implemented at compiler level for optimizations // (for example, `loadInt(1)` is `1 LDI`, but `loadInt(n)` for non-constant requires it be on a stack and `LDIX`) + define_builtin_func("ton", {TypeDataUnknown::create()}, TypeDataCoins::create(), nullptr, // `unknown` to pass inferring for `ton(1)` (to fire a better error later) + compile_int_to_ton_coins, + FunctionData::flagMarkedAsPure); define_builtin_func("mulDivFloor", ParamsInt3, Int, nullptr, std::bind(compile_muldiv, _1, _2, _3, -1), FunctionData::flagMarkedAsPure); diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp index 214fb729d..5c3b3b19a 100644 --- a/tolk/constant-evaluator.cpp +++ b/tolk/constant-evaluator.cpp @@ -153,6 +153,67 @@ static ConstantValue parse_vertex_string_const(V v) { : ConstantValue(parse_vertex_string_const_as_int(v)); } +// given `ton("0.05")` evaluate it to 50000000 +static ConstantValue parse_vertex_call_to_ton_function(V v) { + tolk_assert(v->get_num_args() == 1); // checked by type inferring + AnyExprV v_arg = v->get_arg(0)->get_expr(); + + std::string_view str; + if (auto as_string = v_arg->try_as(); as_string && !as_string->modifier) { + str = as_string->str_val; + } else { + // ton(SOME_CONST) is not supported + // ton(0.05) is not supported (it can't be represented in AST even) + } + if (str.empty()) { + v->error("function `ton` requires a constant string, like `ton(\"0.05\")`"); + } + + bool is_negative = false; + size_t i = 0; + + // handle optional leading sign + if (str[0] == '-') { + is_negative = true; + i++; + } else if (str[0] == '+') { + i++; + } + + // parse "0.05" into integer part (before dot) and fractional (after) + int64_t integer_part = 0; + int64_t fractional_part = 0; + int fractional_digits = 0; + bool seen_dot = false; + + for (; i < str.size(); ++i) { + char c = str[i]; + if (c == '.') { + if (seen_dot) { + v_arg->error("argument is not a valid number like \"0.05\""); + } + seen_dot = true; + } else if (c >= '0' && c <= '9') { + if (!seen_dot) { + integer_part = integer_part * 10 + (c - '0'); + } else if (fractional_digits < 9) { + fractional_part = fractional_part * 10 + (c - '0'); + fractional_digits++; + } + } else { + v_arg->error("argument is not a valid number like \"0.05\""); + } + } + + while (fractional_digits < 9) { // after "0.05" fractional_digits is 2, scale up to 9 + fractional_part *= 10; + fractional_digits++; + } + + int64_t result = integer_part * 1000000000LL + fractional_part; + return ConstantValue(td::make_refint(is_negative ? -result : result)); +} + struct ConstantEvaluator { static bool is_overflow(const td::RefInt256& intval) { @@ -269,6 +330,14 @@ struct ConstantEvaluator { return const_ref->value; } + // `const a = ton("0.05")`, we met `ton("0.05")` + static ConstantValue handle_function_call(V v) { + if (v->fun_maybe && v->fun_maybe->is_builtin_function() && v->fun_maybe->name == "ton") { + return parse_vertex_call_to_ton_function(v); + } + v->error("not a constant expression"); + } + static ConstantValue visit(AnyExprV v) { if (auto v_int = v->try_as()) { return ConstantValue(v_int->intval); @@ -285,6 +354,9 @@ struct ConstantEvaluator { if (auto v_ref = v->try_as()) { return handle_reference(v_ref); } + if (auto v_call = v->try_as()) { + return handle_function_call(v_call); + } if (auto v_par = v->try_as()) { return visit(v_par->get_expr()); } @@ -311,6 +383,11 @@ ConstantValue eval_string_const_considering_modifier(AnyExprV v_string) { return parse_vertex_string_const(v_string->as()); } +ConstantValue eval_call_to_ton_function(AnyExprV v_call) { + tolk_assert(v_call->type == ast_function_call && v_call->as()->fun_maybe->is_builtin_function()); + return parse_vertex_call_to_ton_function(v_call->as()); +} + void eval_and_assign_const_init_value(GlobalConstPtr const_ref) { ConstantValue init_value = ConstantEvaluator::eval_const_init_value(const_ref); const_ref->mutate()->assign_const_value(std::move(init_value)); diff --git a/tolk/constant-evaluator.h b/tolk/constant-evaluator.h index 218bd8c4b..4737352ea 100644 --- a/tolk/constant-evaluator.h +++ b/tolk/constant-evaluator.h @@ -48,6 +48,7 @@ class ConstantValue { }; ConstantValue eval_string_const_considering_modifier(AnyExprV v_string); +ConstantValue eval_call_to_ton_function(AnyExprV v_call); void eval_and_assign_const_init_value(GlobalConstPtr const_ref); } // namespace tolk diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 476559d35..3d0f5dfee 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -549,18 +549,38 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as()) { return rvect; } + // pass `coins` to `int` + // same as above + if (target_type == TypeDataInt::create() && original_type == TypeDataCoins::create()) { + return rvect; + } // pass `int` to `int8` // in code, it's probably done with `as` operator // no changes in rvect if (original_type == TypeDataInt::create() && target_type->try_as()) { return rvect; } + // pass `int` to `coins` + // same as above + if (original_type == TypeDataInt::create() && target_type == TypeDataCoins::create()) { + return rvect; + } // pass `int8` to `int16` / `int8` to `uint8` // in code, it's probably done with `as` operator // no changes in rvect if (original_type->try_as() && target_type->try_as()) { return rvect; } + // pass `int8` to `coins` + // same as above + if (target_type == TypeDataCoins::create() && original_type->try_as()) { + return rvect; + } + // pass `coins` to `int8` + // same as above + if (original_type == TypeDataCoins::create() && target_type->try_as()) { + return rvect; + } // pass something to `unknown` // probably, it comes from `_ = rhs`, type of `_` is unknown, it's target_type of rhs // no changes in rvect diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 97cabd1fe..4b5abbc8b 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -126,7 +126,7 @@ static void handle_possible_compiler_internal_call(FunctionPtr cur_f, Vinferred_type == TypeDataInt::create() || v_inferred->inferred_type->try_as(); + return v_inferred->inferred_type == TypeDataInt::create() || v_inferred->inferred_type->try_as() || v_inferred->inferred_type == TypeDataCoins::create(); } static bool expect_boolean(AnyExprV v_inferred) { diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp index 4b0fe9723..bdb1763ae 100644 --- a/tolk/pipe-constant-folding.cpp +++ b/tolk/pipe-constant-folding.cpp @@ -99,6 +99,19 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { return v; } + AnyExprV replace(V v) override { + parent::replace(v); + + // replace `ton("0.05")` with 50000000 + if (v->fun_maybe && v->fun_maybe->is_builtin_function() && v->fun_maybe->name == "ton") { + ConstantValue value = eval_call_to_ton_function(v); + tolk_assert(value.is_int()); + return create_int_const(v->loc, value.as_int()); + } + + return v; + } + public: bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function(); diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 58d28d421..aa8fe64d1 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -563,7 +563,11 @@ class InferTypesAndCallsAndFieldsVisitor final { default: flow = infer_any_expr(lhs, std::move(flow), false).out_flow; flow = infer_any_expr(rhs, std::move(flow), false).out_flow; - assign_inferred_type(v, TypeDataInt::create()); + if ((v->tok == tok_plus || v->tok == tok_minus) && lhs->inferred_type == TypeDataCoins::create()) { + assign_inferred_type(v, TypeDataCoins::create()); // coins + coins = coins + } else { + assign_inferred_type(v, TypeDataInt::create()); // int8 + int8 = int, as well as other operators/types + } } if (!builtin_func.empty()) { diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index c88acb719..d8d35d890 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -84,6 +84,7 @@ TypePtr TypeDataBuilder::singleton; TypePtr TypeDataTuple::singleton; TypePtr TypeDataContinuation::singleton; TypePtr TypeDataNullLiteral::singleton; +TypePtr TypeDataCoins::singleton; TypePtr TypeDataUnknown::singleton; TypePtr TypeDataNever::singleton; TypePtr TypeDataVoid::singleton; @@ -97,6 +98,7 @@ void type_system_init() { TypeDataTuple::singleton = new TypeDataTuple; TypeDataContinuation::singleton = new TypeDataContinuation; TypeDataNullLiteral::singleton = new TypeDataNullLiteral; + TypeDataCoins::singleton = new TypeDataCoins; TypeDataUnknown::singleton = new TypeDataUnknown; TypeDataNever::singleton = new TypeDataNever; TypeDataVoid::singleton = new TypeDataVoid; @@ -350,6 +352,9 @@ bool TypeDataInt::can_rhs_be_assigned(TypePtr rhs) const { if (rhs->try_as()) { return true; } + if (rhs == TypeDataCoins::create()) { + return true; + } return rhs == TypeDataNever::create(); } @@ -464,6 +469,16 @@ bool TypeDataIntN::can_rhs_be_assigned(TypePtr rhs) const { return rhs == TypeDataNever::create(); // `int8` is NOT assignable to `int32` without `as` } +bool TypeDataCoins::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + if (rhs == TypeDataInt::create()) { + return true; + } + return rhs == TypeDataNever::create(); +} + bool TypeDataUnknown::can_rhs_be_assigned(TypePtr rhs) const { return true; } @@ -499,6 +514,9 @@ bool TypeDataInt::can_be_casted_with_as_operator(TypePtr cast_to) const { if (cast_to->try_as()) { // `int` as `int8` / `int` as `uint2` return true; } + if (cast_to == TypeDataCoins::create()) { // `int` as `coins` + return true; + } return cast_to == this; } @@ -609,7 +627,20 @@ bool TypeDataIntN::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { // `int8` as `int32?` return can_be_casted_with_as_operator(to_nullable->inner); } - return cast_to == TypeDataInt::create(); + return cast_to == TypeDataInt::create() || cast_to == TypeDataCoins::create(); +} + +bool TypeDataCoins::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (cast_to->try_as()) { // `coins` as `int8` + return true; + } + if (const auto* to_nullable = cast_to->try_as()) { // `coins` as `coins?` / `coins` as `int?` + return can_be_casted_with_as_operator(to_nullable->inner); + } + if (cast_to == TypeDataInt::create()) { + return true; + } + return cast_to == this; } bool TypeDataUnknown::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -738,6 +769,7 @@ static TypePtr parse_simple_type(Lexer& lex) { case 5: if (str == "slice") return TypeDataSlice::create(); if (str == "tuple") return TypeDataTuple::create(); + if (str == "coins") return TypeDataCoins::create(); if (str == "never") return TypeDataNever::create(); break; case 7: diff --git a/tolk/type-system.h b/tolk/type-system.h index b64fcfca1..f72cd214b 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -391,6 +391,24 @@ class TypeDataIntN final : public TypeData { bool can_be_casted_with_as_operator(TypePtr cast_to) const override; }; +/* + * `coins` is just integer at TVM level, but encoded as varint when serializing structures. + * Example: `var cost = ton("0.05")` has type `coins`. + */ +class TypeDataCoins final : public TypeData { + TypeDataCoins() : TypeData(17ULL, 0, 1) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + std::string as_human_readable() const override { return "coins"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + /* * `unknown` is a special type, which can appear in corner cases. * The type of exception argument (which can hold any TVM value at runtime) is unknown. From 78c05fd52df3923c97b994cd8f69c7a2bd9f6fe3 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 13 Mar 2025 18:09:41 +0400 Subject: [PATCH 158/388] [Tolk] Types `bytesN` and `bitsN` backed by TVM slice Their purpose is exactly like `intN`: future serialization, to specify fixed-size binary fields (like TL/B bits256). Note, that unlike intN, explicit casting is required. > fun calcHash(raw: bits512) { } > calcHash(someSlice); // error > calcHash(someSlice as bits512); // ok --- tolk-tester/tests/bytesN-tests.tolk | 48 +++++++++++++++++++ tolk-tester/tests/invalid-typing-39.tolk | 9 ++++ tolk-tester/tests/invalid-typing-40.tolk | 10 ++++ tolk-tester/tests/invalid-typing-41.tolk | 12 +++++ tolk-tester/tests/invalid-typing-42.tolk | 12 +++++ tolk-tester/tests/invalid-typing-43.tolk | 9 ++++ tolk/pipe-ast-to-legacy.cpp | 16 +++++++ tolk/type-system.cpp | 60 +++++++++++++++++++++++- tolk/type-system.h | 23 +++++++++ 9 files changed, 198 insertions(+), 1 deletion(-) create mode 100644 tolk-tester/tests/bytesN-tests.tolk create mode 100644 tolk-tester/tests/invalid-typing-39.tolk create mode 100644 tolk-tester/tests/invalid-typing-40.tolk create mode 100644 tolk-tester/tests/invalid-typing-41.tolk create mode 100644 tolk-tester/tests/invalid-typing-42.tolk create mode 100644 tolk-tester/tests/invalid-typing-43.tolk diff --git a/tolk-tester/tests/bytesN-tests.tolk b/tolk-tester/tests/bytesN-tests.tolk new file mode 100644 index 000000000..32b975bf7 --- /dev/null +++ b/tolk-tester/tests/bytesN-tests.tolk @@ -0,0 +1,48 @@ +fun takeAnySlice(slice: slice) {} +fun getSomeSlice() { return beginCell().endCell().beginParse(); } +fun getNullableBytes16(): bytes16? { return getSomeSlice() as bytes16; } + +const someBytes = "asdf" as bytes16; +const anotherBytes: bytes16 = "asdf" as bytes16; + +fun autoInferBytes16() { + if (random()) { return someBytes; } + else if (random()) { return true ? "" as bytes16 : anotherBytes; } + else { return someBytes!; } +} + +@method_id(101) +fun test1() { + var ss = beginCell().storeInt(1, 32).storeInt(2, 32).endCell().beginParse() as bits32; + __expect_type(ss, "bits32"); + __expect_type(ss as slice, "slice"); + __expect_type(ss as slice?, "slice?"); + __expect_type(ss as bytes128, "bytes128"); + __expect_type(someBytes, "bytes16"); + __expect_type(10>3 ? (null, ss as bits1) : (ss as bytes1, null), "(bytes1?, bits1?)"); + return (getRemainingBitsCount(ss as slice), loadInt(mutate ss as slice, 32), (ss as slice).loadInt(32)); +} + +fun test2(a: bytes8, b: bits64) { + a = b as bytes8; + b = a as bits64; + (a as slice).loadRef(); + (b as slice?)!.loadRef(); +} + +@method_id(103) +fun test3() { + var slice = beginCell().storeInt(1, 32).storeInt(2, 32).endCell().beginParse(); + var bq = slice as bits16?; + if (bq != null) { + return (bq as slice).loadInt(32 as uint8) + (slice = bq as slice).getRemainingBitsCount(); + } + return -1; +} + +fun main() {} + +/** +@testcase | 101 | | 64 1 2 +@testcase | 103 | | 33 + */ diff --git a/tolk-tester/tests/invalid-typing-39.tolk b/tolk-tester/tests/invalid-typing-39.tolk new file mode 100644 index 000000000..47ebb1a2f --- /dev/null +++ b/tolk-tester/tests/invalid-typing-39.tolk @@ -0,0 +1,9 @@ +fun cantUnifyBytesNAndBytesM(n: bytes8, c: bytes16) { + __expect_type(random() ? n : c as bytes8, "bytes8"); + random() ? n : c; +} + +/** +@compilation_should_fail +@stderr types of ternary branches are incompatible: `bytes8` and `bytes16` + */ diff --git a/tolk-tester/tests/invalid-typing-40.tolk b/tolk-tester/tests/invalid-typing-40.tolk new file mode 100644 index 000000000..274f51b4c --- /dev/null +++ b/tolk-tester/tests/invalid-typing-40.tolk @@ -0,0 +1,10 @@ +fun cantUnifyBytesNAndSlice(n: slice, c: bytes16) { + __expect_type(random() ? n : c as slice, "slice"); + __expect_type(random() ? n : c as slice?, "slice?"); + random() ? n : c; +} + +/** +@compilation_should_fail +@stderr types of ternary branches are incompatible: `slice` and `bytes16` + */ diff --git a/tolk-tester/tests/invalid-typing-41.tolk b/tolk-tester/tests/invalid-typing-41.tolk new file mode 100644 index 000000000..b04e8d974 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-41.tolk @@ -0,0 +1,12 @@ +fun takeAnySlice(s: slice) {} + +fun cantPassBytesNToSlice(c: bits16) { + takeAnySlice(c as slice); // ok + takeAnySlice(c); +} + +/** +@compilation_should_fail +@stderr can not pass `bits16` to `slice` +@stderr takeAnySlice(c); + */ diff --git a/tolk-tester/tests/invalid-typing-42.tolk b/tolk-tester/tests/invalid-typing-42.tolk new file mode 100644 index 000000000..d380279a9 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-42.tolk @@ -0,0 +1,12 @@ +fun getAnySlice() { return beginCell().endCell().beginParse(); } + +fun cantAssignSliceToBytesN() { + var c1: bytes16 = getAnySlice() as bytes16; // ok + var c2: bytes16 = getAnySlice(); +} + +/** +@compilation_should_fail +@stderr can not assign `slice` to variable of type `bytes16` +@stderr var c2: bytes16 = getAnySlice(); + */ diff --git a/tolk-tester/tests/invalid-typing-43.tolk b/tolk-tester/tests/invalid-typing-43.tolk new file mode 100644 index 000000000..463ba4871 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-43.tolk @@ -0,0 +1,9 @@ +fun cantAutoCastBytesNToSlice() { + var b = beginCell().storeInt(1, 32).endCell().beginParse() as bits32; + return b.loadInt(32); +} + +/** +@compilation_should_fail +@stderr can not call method for `slice` with object of type `bits32` + */ diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 3d0f5dfee..d0c252847 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -581,6 +581,22 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as()) { return rvect; } + // pass `bytes32` to `slice` + // in code, it's probably done with `as` operator + // no changes in rvect, since bytesN is slice at TVM level + if (target_type == TypeDataSlice::create() && original_type->try_as()) { + return rvect; + } + // pass `slice` to `bytes32` + // same as above + if (original_type == TypeDataSlice::create() && target_type->try_as()) { + return rvect; + } + // pass `bytes32` to `bytes64` / `bits128` to `bytes16` + // no changes in rvect + if (original_type->try_as() && target_type->try_as()) { + return rvect; + } // pass something to `unknown` // probably, it comes from `_ = rhs`, type of `_` is unknown, it's target_type of rhs // no changes in rvect diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index d8d35d890..ebcfc6fe6 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -193,6 +193,17 @@ TypePtr TypeDataIntN::create(bool is_unsigned, bool is_variadic, int n_bits) { return hash.register_unique(new TypeDataIntN(hash.type_id(), is_unsigned, is_variadic, n_bits)); } +TypePtr TypeDataBytesN::create(bool is_bits, int n_width) { + TypeDataTypeIdCalculation hash(7810988137199333041ULL); + hash.feed_hash(is_bits); + hash.feed_hash(n_width); + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + return hash.register_unique(new TypeDataBytesN(hash.type_id(), is_bits, n_width)); +} + TypePtr TypeDataUnresolved::create(std::string&& text, SrcLocation loc) { TypeDataTypeIdCalculation hash(3680147223540048162ULL); hash.feed_string(text); @@ -262,6 +273,11 @@ std::string TypeDataIntN::as_human_readable() const { return s_int + std::to_string(n_bits); } +std::string TypeDataBytesN::as_human_readable() const { + std::string s_bytes = is_bits ? "bits" : "bytes"; + return s_bytes + std::to_string(n_width); +} + // -------------------------------------------- // traverse() @@ -376,7 +392,7 @@ bool TypeDataSlice::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } - return rhs == TypeDataNever::create(); + return rhs == TypeDataNever::create(); // note, that bytesN is NOT automatically cast to slice without `as` operator } bool TypeDataBuilder::can_rhs_be_assigned(TypePtr rhs) const { @@ -469,6 +485,15 @@ bool TypeDataIntN::can_rhs_be_assigned(TypePtr rhs) const { return rhs == TypeDataNever::create(); // `int8` is NOT assignable to `int32` without `as` } +bool TypeDataBytesN::can_rhs_be_assigned(TypePtr rhs) const { + // `slice` is NOT assignable to bytesN without `as` + // `bytes32` is NOT assignable to `bytes256` and even to `bits256` without `as` + if (rhs == this) { + return true; + } + return rhs == TypeDataNever::create(); +} + bool TypeDataCoins::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; @@ -541,6 +566,9 @@ bool TypeDataCell::can_be_casted_with_as_operator(TypePtr cast_to) const { } bool TypeDataSlice::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (cast_to->try_as()) { // `slice` to `bytes32` / `slice` to `bits8` + return true; + } if (const auto* to_nullable = cast_to->try_as()) { return can_be_casted_with_as_operator(to_nullable->inner); } @@ -630,6 +658,16 @@ bool TypeDataIntN::can_be_casted_with_as_operator(TypePtr cast_to) const { return cast_to == TypeDataInt::create() || cast_to == TypeDataCoins::create(); } +bool TypeDataBytesN::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (cast_to->try_as()) { // `bytes256` as `bytes512`, `bits1` as `bytes8` + return true; + } + if (const auto* to_nullable = cast_to->try_as()) { // `bytes8` as `slice?` + return can_be_casted_with_as_operator(to_nullable->inner); + } + return cast_to == TypeDataSlice::create(); +} + bool TypeDataCoins::can_be_casted_with_as_operator(TypePtr cast_to) const { if (cast_to->try_as()) { // `coins` as `int8` return true; @@ -750,6 +788,16 @@ static TypePtr parse_intN(std::string_view strN, bool is_unsigned) { return TypeDataIntN::create(is_unsigned, false, n); } +static TypePtr parse_bytesN(std::string_view strN, bool is_bits) { + int n; + auto result = std::from_chars(strN.data() + 5 - static_cast(is_bits), strN.data() + strN.size(), n); + bool parsed = result.ec == std::errc() && result.ptr == strN.data() + strN.size(); + if (!parsed || n <= 0 || n > 1024) { + return nullptr; // `bytes9999`, maybe it's user-defined alias, let it be unresolved + } + return TypeDataBytesN::create(is_bits, n); +} + static TypePtr parse_simple_type(Lexer& lex) { switch (lex.tok()) { case tok_self: @@ -795,6 +843,16 @@ static TypePtr parse_simple_type(Lexer& lex) { return uintN; } } + if (str.size() > 4 && str.starts_with("bits")) { + if (TypePtr bitsN = parse_bytesN(str, true)) { + return bitsN; + } + } + if (str.size() > 5 && str.starts_with("bytes")) { + if (TypePtr bytesN = parse_bytesN(str, false)) { + return bytesN; + } + } return TypeDataUnresolved::create(std::string(str), loc); } case tok_null: diff --git a/tolk/type-system.h b/tolk/type-system.h index f72cd214b..07a2496ca 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -409,6 +409,29 @@ class TypeDataCoins final : public TypeData { bool can_be_casted_with_as_operator(TypePtr cast_to) const override; }; +/* + * `bytes256`, `bits512`, `bytes8` are TypeDataBytesN. At TVM level, it's just slice. + * The purpose of bytesN is to be used in struct fields, describing the way of serialization (n bytes / n bits). + * In this essence, bytesN is very similar to intN. + * Note, that unlike intN automatically cast to/from int, bytesN does NOT auto cast to slice (without `as`). + */ +class TypeDataBytesN final : public TypeData { + TypeDataBytesN(uint64_t type_id, bool is_bits, int n_width) + : TypeData(type_id, 0, 1) + , is_bits(is_bits) + , n_width(n_width) {} + +public: + const bool is_bits; + const int n_width; + + static TypePtr create(bool is_bits, int n_width); + + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + /* * `unknown` is a special type, which can appear in corner cases. * The type of exception argument (which can hold any TVM value at runtime) is unknown. From 35f45b030cf79ac13822a78e1ef803c1c73bb04b Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 13 Mar 2025 18:11:57 +0400 Subject: [PATCH 159/388] [Tolk] Tidy up the tests, separate into folders This commit reorganizes tests for the compiler. Before, they all were places in a single folder. Now, they negative tests are separated. Negative tests are named just `err-XXXX`, for quick file search in IDE. Tolk-tester can now execute tests by "contains" pattern: > tolk-tester tests catch will find "tests/**/*catch*.tolk" --- tolk-tester/tests/{a6.tolk => a-tests.tolk} | 2 +- tolk-tester/tests/a6_1.tolk | 22 --- tolk-tester/tests/a6_5.tolk | 26 ---- tolk-tester/tests/a7.tolk | 24 --- ...tion.tolk => allow-post-modification.tolk} | 0 ...{asm_arg_order.tolk => asm-arg-order.tolk} | 0 tolk-tester/tests/c2.tolk | 28 ---- tolk-tester/tests/c2_1.tolk | 14 -- tolk-tester/tests/code_after_ifelse.tolk | 41 ----- ...heck_demo.tolk => codegen-check-demo.tolk} | 0 .../tests/{co1.tolk => constants-tests.tolk} | 0 .../{if_stmt.tolk => if-else-tests.tolk} | 48 ++++++ tolk-tester/tests/inference-tests.tolk | 12 +- tolk-tester/tests/inline-tests.tolk | 140 ++++++++++++++++++ tolk-tester/tests/inline_big.tolk | 62 -------- tolk-tester/tests/inline_if.tolk | 28 ---- tolk-tester/tests/inline_loops.tolk | 48 ------ .../err-1084.tolk} | 0 .../err-1153.tolk} | 0 .../err-1209.tolk} | 0 .../err-1361.tolk} | 0 .../err-1449.tolk} | 0 .../err-1505.tolk} | 0 .../err-1601.tolk} | 0 .../err-1683.tolk} | 0 .../err-1701.tolk} | 0 .../err-1790.tolk} | 0 .../err-1794.tolk} | 0 .../err-1805.tolk} | 0 .../err-1842.tolk} | 0 .../err-1918.tolk} | 2 +- .../err-1940.tolk} | 0 tolk-tester/tests/invalid-no-import-1.tolk | 8 - .../err-4002.tolk} | 0 .../err-4061.tolk} | 0 .../err-4064.tolk} | 0 .../err-4080.tolk} | 0 .../err-4083.tolk} | 0 .../err-4104.tolk} | 0 .../err-4127.tolk} | 0 .../err-4195.tolk} | 0 .../err-4266.tolk} | 0 .../err-4377.tolk} | 0 .../err-4381.tolk} | 0 .../err-4385.tolk} | 0 .../err-4413.tolk} | 0 .../err-4416.tolk} | 0 .../err-4473.tolk} | 0 .../err-4506.tolk} | 0 .../err-4510.tolk} | 0 .../err-4520.tolk} | 0 .../err-4542.tolk} | 0 .../err-4553.tolk} | 0 .../err-4571.tolk} | 0 .../err-4604.tolk} | 0 .../err-4639.tolk} | 0 .../err-4705.tolk} | 0 .../err-4728.tolk} | 0 .../err-4751.tolk} | 0 .../err-4804.tolk} | 0 .../err-4819.tolk} | 0 .../err-4828.tolk} | 0 .../err-4838.tolk} | 0 .../err-4841.tolk} | 0 .../err-4851.tolk} | 0 .../err-4880.tolk} | 0 .../err-4899.tolk} | 0 .../err-4912.tolk} | 0 .../err-4941.tolk} | 0 .../err-4951.tolk} | 0 .../err-4956.tolk} | 0 .../err-2055.tolk} | 0 .../err-2188.tolk} | 0 .../tests/invalid-symbol/err-2205.tolk | 8 + .../err-2255.tolk} | 0 .../err-2374.tolk} | 0 .../err-2463.tolk} | 0 .../err-2501.tolk} | 0 .../err-2520.tolk} | 0 .../err-2539.tolk} | 4 +- .../err-2649.tolk} | 0 .../err-2673.tolk} | 0 .../err-2715.tolk} | 0 .../err-2750.tolk} | 0 .../err-2804.tolk} | 0 .../err-2853.tolk} | 0 .../err-2914.tolk} | 0 .../err-2980.tolk} | 4 +- .../err-3001.tolk} | 0 .../err-3015.tolk} | 0 .../err-3035.tolk} | 0 .../err-3041.tolk} | 0 .../err-3044.tolk} | 0 .../err-3059.tolk} | 0 .../err-3102.tolk} | 0 .../err-3164.tolk} | 0 .../err-3183.tolk} | 0 .../err-3197.tolk} | 0 .../err-3205.tolk} | 0 .../err-3274.tolk} | 0 .../err-3295.tolk} | 0 .../err-3390.tolk} | 0 .../err-3407.tolk} | 0 .../err-3472.tolk} | 0 .../err-3527.tolk} | 0 .../err-3551.tolk} | 0 .../err-3586.tolk} | 0 .../err-3588.tolk} | 0 .../err-3603.tolk} | 0 .../tests/invalid-syntax/err-3618.tolk | 6 + .../err-3714.tolk} | 0 .../err-3719.tolk} | 0 .../err-3781.tolk} | 0 .../err-3819.tolk} | 0 .../err-3851.tolk} | 0 .../err-3853.tolk} | 0 .../err-3898.tolk} | 0 .../err-3910.tolk} | 0 .../err-3967.tolk} | 0 .../err-3992.tolk} | 0 .../err-6039.tolk} | 0 .../err-6040.tolk} | 0 .../err-6041.tolk} | 0 .../err-6080.tolk} | 0 .../err-6085.tolk} | 0 .../err-6087.tolk} | 0 .../err-6095.tolk} | 0 .../err-6114.tolk} | 0 .../err-6130.tolk} | 0 .../err-6134.tolk} | 0 .../err-6135.tolk} | 0 .../err-6148.tolk} | 0 .../err-6188.tolk} | 0 .../err-6200.tolk} | 0 .../err-6204.tolk} | 0 .../err-6220.tolk} | 0 .../err-6249.tolk} | 0 .../err-6257.tolk} | 0 .../err-6288.tolk} | 0 .../err-6337.tolk} | 0 .../err-6368.tolk} | 0 .../err-6369.tolk} | 0 .../err-6370.tolk} | 0 .../err-6386.tolk} | 0 .../err-6391.tolk} | 0 .../err-6393.tolk} | 0 .../err-6403.tolk} | 0 .../err-6407.tolk} | 0 .../err-6450.tolk} | 0 .../err-6496.tolk} | 0 .../err-6499.tolk} | 0 .../err-6509.tolk} | 0 .../err-6519.tolk} | 0 .../err-6533.tolk} | 0 .../err-6553.tolk} | 0 .../err-6580.tolk} | 0 .../err-6600.tolk} | 0 .../err-6613.tolk} | 0 .../err-6619.tolk} | 0 .../err-6629.tolk} | 0 .../err-6637.tolk} | 0 .../err-6711.tolk} | 0 .../err-6713.tolk} | 0 .../err-6731.tolk} | 0 .../err-6734.tolk} | 0 .../err-6737.tolk} | 0 .../err-6743.tolk} | 0 .../err-6803.tolk} | 0 .../err-6810.tolk} | 0 .../err-6820.tolk} | 0 .../err-6839.tolk} | 0 .../err-6840.tolk} | 0 .../err-6841.tolk} | 0 .../err-6844.tolk} | 0 .../err-6881.tolk} | 0 .../err-6901.tolk} | 0 .../err-6903.tolk} | 0 .../err-6921.tolk} | 0 .../err-6925.tolk} | 0 .../err-6937.tolk} | 0 .../err-6981.tolk} | 0 .../err-6992.tolk} | 0 tolk-tester/tests/method_id.tolk | 17 --- tolk-tester/tests/some-tests-1.tolk | 127 ++++++++++++++++ .../tests/{a10.tolk => some-tests-2.tolk} | 0 .../tests/{w2.tolk => some-tests-3.tolk} | 0 .../tests/{s1.tolk => strings-tests.tolk} | 0 .../{try-func.tolk => try-catch-tests.tolk} | 0 ...t_loops.tolk => unbalanced-ret-loops.tolk} | 0 tolk-tester/tests/unbalanced-ret.tolk | 73 +++++++++ tolk-tester/tests/unbalanced_ret.tolk | 17 --- tolk-tester/tests/unbalanced_ret_inline.tolk | 19 --- tolk-tester/tests/unbalanced_ret_nested.tolk | 37 ----- tolk-tester/tests/use-before-declare.tolk | 15 ++ .../{var-apply.tolk => var-apply-tests.tolk} | 30 +++- tolk-tester/tests/w1.tolk | 14 -- tolk-tester/tests/w6.tolk | 19 --- tolk-tester/tests/w7.tolk | 26 ---- tolk-tester/tests/w9.tolk | 14 -- .../unreachable-1.tolk | 0 .../unreachable-2.tolk | 0 .../unreachable-3.tolk | 0 .../unreachable-4.tolk | 0 .../{ => warnings-not-errors}/warnings-1.tolk | 0 .../{ => warnings-not-errors}/warnings-2.tolk | 0 tolk-tester/tolk-tester.js | 46 +++--- tolk-tester/tolk-tester.py | 63 ++++---- 207 files changed, 527 insertions(+), 517 deletions(-) rename tolk-tester/tests/{a6.tolk => a-tests.tolk} (98%) delete mode 100644 tolk-tester/tests/a6_1.tolk delete mode 100644 tolk-tester/tests/a6_5.tolk delete mode 100644 tolk-tester/tests/a7.tolk rename tolk-tester/tests/{allow_post_modification.tolk => allow-post-modification.tolk} (100%) rename tolk-tester/tests/{asm_arg_order.tolk => asm-arg-order.tolk} (100%) delete mode 100644 tolk-tester/tests/c2.tolk delete mode 100644 tolk-tester/tests/c2_1.tolk delete mode 100644 tolk-tester/tests/code_after_ifelse.tolk rename tolk-tester/tests/{codegen_check_demo.tolk => codegen-check-demo.tolk} (100%) rename tolk-tester/tests/{co1.tolk => constants-tests.tolk} (100%) rename tolk-tester/tests/{if_stmt.tolk => if-else-tests.tolk} (59%) create mode 100644 tolk-tester/tests/inline-tests.tolk delete mode 100644 tolk-tester/tests/inline_big.tolk delete mode 100644 tolk-tester/tests/inline_if.tolk delete mode 100644 tolk-tester/tests/inline_loops.tolk rename tolk-tester/tests/{invalid-declaration-8.tolk => invalid-declaration/err-1084.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-16.tolk => invalid-declaration/err-1153.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-1.tolk => invalid-declaration/err-1209.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-10.tolk => invalid-declaration/err-1361.tolk} (100%) rename tolk-tester/tests/{invalid-get-method-2.tolk => invalid-declaration/err-1449.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-13.tolk => invalid-declaration/err-1505.tolk} (100%) rename tolk-tester/tests/{invalid-self-6.tolk => invalid-declaration/err-1601.tolk} (100%) rename tolk-tester/tests/{invalid-redefinition-1.tolk => invalid-declaration/err-1683.tolk} (100%) rename tolk-tester/tests/{invalid-builtin-1.tolk => invalid-declaration/err-1701.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-11.tolk => invalid-declaration/err-1790.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-6.tolk => invalid-declaration/err-1794.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-9.tolk => invalid-declaration/err-1805.tolk} (100%) rename tolk-tester/tests/{invalid-cyclic-1.tolk => invalid-declaration/err-1842.tolk} (100%) rename tolk-tester/tests/{invalid-redefinition-2.tolk => invalid-declaration/err-1918.tolk} (78%) rename tolk-tester/tests/{invalid-get-method-1.tolk => invalid-declaration/err-1940.tolk} (100%) delete mode 100644 tolk-tester/tests/invalid-no-import-1.tolk rename tolk-tester/tests/{invalid-call-5.tolk => invalid-semantics/err-4002.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-5.tolk => invalid-semantics/err-4061.tolk} (100%) rename tolk-tester/tests/{invalid-pure-1.tolk => invalid-semantics/err-4064.tolk} (100%) rename tolk-tester/tests/{invalid-call-4.tolk => invalid-semantics/err-4080.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-16.tolk => invalid-semantics/err-4083.tolk} (100%) rename tolk-tester/tests/{invalid-generics-6.tolk => invalid-semantics/err-4104.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-14.tolk => invalid-semantics/err-4127.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-18.tolk => invalid-semantics/err-4195.tolk} (100%) rename tolk-tester/tests/{invalid-generics-10.tolk => invalid-semantics/err-4266.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-7.tolk => invalid-semantics/err-4377.tolk} (100%) rename tolk-tester/tests/{invalid-typing-13.tolk => invalid-semantics/err-4381.tolk} (100%) rename tolk-tester/tests/{invalid-pure-3.tolk => invalid-semantics/err-4385.tolk} (100%) rename tolk-tester/tests/{invalid-pure-2.tolk => invalid-semantics/err-4413.tolk} (100%) rename tolk-tester/tests/{invalid-generics-5.tolk => invalid-semantics/err-4416.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-11.tolk => invalid-semantics/err-4473.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-10.tolk => invalid-semantics/err-4506.tolk} (100%) rename tolk-tester/tests/{invalid-generics-11.tolk => invalid-semantics/err-4510.tolk} (100%) rename tolk-tester/tests/{invalid-call-8.tolk => invalid-semantics/err-4520.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-17.tolk => invalid-semantics/err-4542.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-4.tolk => invalid-semantics/err-4553.tolk} (100%) rename tolk-tester/tests/{invalid-syntax-3.tolk => invalid-semantics/err-4571.tolk} (100%) rename tolk-tester/tests/{invalid-generics-9.tolk => invalid-semantics/err-4604.tolk} (100%) rename tolk-tester/tests/{invalid-syntax-6.tolk => invalid-semantics/err-4639.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-12.tolk => invalid-semantics/err-4705.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-1.tolk => invalid-semantics/err-4728.tolk} (100%) rename tolk-tester/tests/{invalid-call-3.tolk => invalid-semantics/err-4751.tolk} (100%) rename tolk-tester/tests/{invalid-call-11.tolk => invalid-semantics/err-4804.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-3.tolk => invalid-semantics/err-4819.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-15.tolk => invalid-semantics/err-4828.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-6.tolk => invalid-semantics/err-4838.tolk} (100%) rename tolk-tester/tests/{invalid-call-6.tolk => invalid-semantics/err-4841.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-9.tolk => invalid-semantics/err-4851.tolk} (100%) rename tolk-tester/tests/{invalid-generics-1.tolk => invalid-semantics/err-4880.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-14.tolk => invalid-semantics/err-4899.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-2.tolk => invalid-semantics/err-4912.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-19.tolk => invalid-semantics/err-4941.tolk} (100%) rename tolk-tester/tests/{invalid-call-9.tolk => invalid-semantics/err-4951.tolk} (100%) rename tolk-tester/tests/{invalid-call-2.tolk => invalid-semantics/err-4956.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-8.tolk => invalid-symbol/err-2055.tolk} (100%) rename tolk-tester/tests/{invalid-call-7.tolk => invalid-symbol/err-2188.tolk} (100%) create mode 100644 tolk-tester/tests/invalid-symbol/err-2205.tolk rename tolk-tester/tests/{invalid-typing-32.tolk => invalid-symbol/err-2255.tolk} (100%) rename tolk-tester/tests/{invalid-symbol-1.tolk => invalid-symbol/err-2374.tolk} (100%) rename tolk-tester/tests/{invalid-symbol-2.tolk => invalid-symbol/err-2463.tolk} (100%) rename tolk-tester/tests/{invalid-redefinition-6.tolk => invalid-symbol/err-2501.tolk} (100%) rename tolk-tester/tests/{invalid-self-7.tolk => invalid-symbol/err-2520.tolk} (100%) rename tolk-tester/tests/{invalid-no-import-2.tolk => invalid-symbol/err-2539.tolk} (66%) rename tolk-tester/tests/{invalid-redefinition-3.tolk => invalid-symbol/err-2649.tolk} (100%) rename tolk-tester/tests/{invalid-self-3.tolk => invalid-symbol/err-2673.tolk} (100%) rename tolk-tester/tests/{invalid-redefinition-5.tolk => invalid-symbol/err-2715.tolk} (100%) rename tolk-tester/tests/{invalid-typing-1.tolk => invalid-symbol/err-2750.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-15.tolk => invalid-symbol/err-2804.tolk} (100%) rename tolk-tester/tests/{invalid-redefinition-4.tolk => invalid-symbol/err-2853.tolk} (100%) rename tolk-tester/tests/{invalid-call-1.tolk => invalid-symbol/err-2914.tolk} (100%) rename tolk-tester/tests/{invalid-import.tolk => invalid-symbol/err-2980.tolk} (73%) rename tolk-tester/tests/{invalid-tolk-version.tolk => invalid-syntax/err-3001.tolk} (100%) rename tolk-tester/tests/{invalid-call-13.tolk => invalid-syntax/err-3015.tolk} (100%) rename tolk-tester/tests/{invalid-bitwise-4.tolk => invalid-syntax/err-3035.tolk} (100%) rename tolk-tester/tests/{invalid-nopar-1.tolk => invalid-syntax/err-3041.tolk} (100%) rename tolk-tester/tests/{invalid-shift-1.tolk => invalid-syntax/err-3044.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-4.tolk => invalid-syntax/err-3059.tolk} (100%) rename tolk-tester/tests/{invalid-bitwise-3.tolk => invalid-syntax/err-3102.tolk} (100%) rename tolk-tester/tests/{invalid-catch-1.tolk => invalid-syntax/err-3164.tolk} (100%) rename tolk-tester/tests/{invalid-nopar-2.tolk => invalid-syntax/err-3183.tolk} (100%) rename tolk-tester/tests/{invalid-nopar-4.tolk => invalid-syntax/err-3197.tolk} (100%) rename tolk-tester/tests/{invalid-syntax-7.tolk => invalid-syntax/err-3205.tolk} (100%) rename tolk-tester/tests/{invalid-nopar-3.tolk => invalid-syntax/err-3274.tolk} (100%) rename tolk-tester/tests/{invalid-const-1.tolk => invalid-syntax/err-3295.tolk} (100%) rename tolk-tester/tests/{invalid-cmt-old.tolk => invalid-syntax/err-3390.tolk} (100%) rename tolk-tester/tests/{invalid-cmt-nested.tolk => invalid-syntax/err-3407.tolk} (100%) rename tolk-tester/tests/{invalid-call-14.tolk => invalid-syntax/err-3472.tolk} (100%) rename tolk-tester/tests/{invalid-syntax-5.tolk => invalid-syntax/err-3527.tolk} (100%) rename tolk-tester/tests/{invalid-syntax-1.tolk => invalid-syntax/err-3551.tolk} (100%) rename tolk-tester/tests/{invalid-catch-2.tolk => invalid-syntax/err-3586.tolk} (100%) rename tolk-tester/tests/{invalid-bitwise-5.tolk => invalid-syntax/err-3588.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-7.tolk => invalid-syntax/err-3603.tolk} (100%) create mode 100644 tolk-tester/tests/invalid-syntax/err-3618.tolk rename tolk-tester/tests/{invalid-call-12.tolk => invalid-syntax/err-3714.tolk} (100%) rename tolk-tester/tests/{invalid-syntax-4.tolk => invalid-syntax/err-3719.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-5.tolk => invalid-syntax/err-3781.tolk} (100%) rename tolk-tester/tests/{invalid-bitwise-1.tolk => invalid-syntax/err-3819.tolk} (100%) rename tolk-tester/tests/{invalid-bitwise-7.tolk => invalid-syntax/err-3851.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-2.tolk => invalid-syntax/err-3853.tolk} (100%) rename tolk-tester/tests/{invalid-syntax-2.tolk => invalid-syntax/err-3898.tolk} (100%) rename tolk-tester/tests/{invalid-bitwise-2.tolk => invalid-syntax/err-3910.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-3.tolk => invalid-syntax/err-3967.tolk} (100%) rename tolk-tester/tests/{invalid-bitwise-6.tolk => invalid-syntax/err-3992.tolk} (100%) rename tolk-tester/tests/{invalid-assign-1.tolk => invalid-typing/err-6039.tolk} (100%) rename tolk-tester/tests/{invalid-assign-3.tolk => invalid-typing/err-6040.tolk} (100%) rename tolk-tester/tests/{invalid-call-10.tolk => invalid-typing/err-6041.tolk} (100%) rename tolk-tester/tests/{invalid-typing-18.tolk => invalid-typing/err-6080.tolk} (100%) rename tolk-tester/tests/{invalid-generics-14.tolk => invalid-typing/err-6085.tolk} (100%) rename tolk-tester/tests/{invalid-typing-43.tolk => invalid-typing/err-6087.tolk} (100%) rename tolk-tester/tests/{invalid-generics-4.tolk => invalid-typing/err-6095.tolk} (100%) rename tolk-tester/tests/{invalid-typing-20.tolk => invalid-typing/err-6114.tolk} (100%) rename tolk-tester/tests/{invalid-typing-44.tolk => invalid-typing/err-6130.tolk} (100%) rename tolk-tester/tests/{invalid-typing-4.tolk => invalid-typing/err-6134.tolk} (100%) rename tolk-tester/tests/{invalid-typing-37.tolk => invalid-typing/err-6135.tolk} (100%) rename tolk-tester/tests/{invalid-typing-25.tolk => invalid-typing/err-6148.tolk} (100%) rename tolk-tester/tests/{invalid-typing-10.tolk => invalid-typing/err-6188.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-13.tolk => invalid-typing/err-6200.tolk} (100%) rename tolk-tester/tests/{invalid-typing-38.tolk => invalid-typing/err-6204.tolk} (100%) rename tolk-tester/tests/{invalid-typing-24.tolk => invalid-typing/err-6220.tolk} (100%) rename tolk-tester/tests/{invalid-typing-40.tolk => invalid-typing/err-6249.tolk} (100%) rename tolk-tester/tests/{invalid-self-5.tolk => invalid-typing/err-6257.tolk} (100%) rename tolk-tester/tests/{invalid-self-1.tolk => invalid-typing/err-6288.tolk} (100%) rename tolk-tester/tests/{invalid-typing-16.tolk => invalid-typing/err-6337.tolk} (100%) rename tolk-tester/tests/{invalid-typing-45.tolk => invalid-typing/err-6368.tolk} (100%) rename tolk-tester/tests/{invalid-generics-3.tolk => invalid-typing/err-6369.tolk} (100%) rename tolk-tester/tests/{invalid-typing-23.tolk => invalid-typing/err-6370.tolk} (100%) rename tolk-tester/tests/{invalid-generics-2.tolk => invalid-typing/err-6386.tolk} (100%) rename tolk-tester/tests/{invalid-mutate-20.tolk => invalid-typing/err-6391.tolk} (100%) rename tolk-tester/tests/{invalid-typing-9.tolk => invalid-typing/err-6393.tolk} (100%) rename tolk-tester/tests/{invalid-typing-34.tolk => invalid-typing/err-6403.tolk} (100%) rename tolk-tester/tests/{invalid-never-1.tolk => invalid-typing/err-6407.tolk} (100%) rename tolk-tester/tests/{invalid-typing-2.tolk => invalid-typing/err-6450.tolk} (100%) rename tolk-tester/tests/{invalid-typing-29.tolk => invalid-typing/err-6496.tolk} (100%) rename tolk-tester/tests/{invalid-typing-22.tolk => invalid-typing/err-6499.tolk} (100%) rename tolk-tester/tests/{invalid-generics-13.tolk => invalid-typing/err-6509.tolk} (100%) rename tolk-tester/tests/{invalid-typing-5.tolk => invalid-typing/err-6519.tolk} (100%) rename tolk-tester/tests/{invalid-self-2.tolk => invalid-typing/err-6533.tolk} (100%) rename tolk-tester/tests/{invalid-typing-26.tolk => invalid-typing/err-6553.tolk} (100%) rename tolk-tester/tests/{invalid-typing-21.tolk => invalid-typing/err-6580.tolk} (100%) rename tolk-tester/tests/{invalid-typing-8.tolk => invalid-typing/err-6600.tolk} (100%) rename tolk-tester/tests/{invalid-typing-31.tolk => invalid-typing/err-6613.tolk} (100%) rename tolk-tester/tests/{invalid-typing-35.tolk => invalid-typing/err-6619.tolk} (100%) rename tolk-tester/tests/{invalid-generics-8.tolk => invalid-typing/err-6629.tolk} (100%) rename tolk-tester/tests/{invalid-typing-14.tolk => invalid-typing/err-6637.tolk} (100%) rename tolk-tester/tests/{invalid-typing-11.tolk => invalid-typing/err-6711.tolk} (100%) rename tolk-tester/tests/{invalid-typing-27.tolk => invalid-typing/err-6713.tolk} (100%) rename tolk-tester/tests/{invalid-assign-2.tolk => invalid-typing/err-6731.tolk} (100%) rename tolk-tester/tests/{invalid-typing-12.tolk => invalid-typing/err-6734.tolk} (100%) rename tolk-tester/tests/{invalid-typing-33.tolk => invalid-typing/err-6737.tolk} (100%) rename tolk-tester/tests/{invalid-typing-6.tolk => invalid-typing/err-6743.tolk} (100%) rename tolk-tester/tests/{invalid-typing-41.tolk => invalid-typing/err-6803.tolk} (100%) rename tolk-tester/tests/{invalid-typing-36.tolk => invalid-typing/err-6810.tolk} (100%) rename tolk-tester/tests/{invalid-typing-39.tolk => invalid-typing/err-6820.tolk} (100%) rename tolk-tester/tests/{invalid-typing-3.tolk => invalid-typing/err-6839.tolk} (100%) rename tolk-tester/tests/{invalid-typing-15.tolk => invalid-typing/err-6840.tolk} (100%) rename tolk-tester/tests/{invalid-typing-28.tolk => invalid-typing/err-6841.tolk} (100%) rename tolk-tester/tests/{invalid-self-4.tolk => invalid-typing/err-6844.tolk} (100%) rename tolk-tester/tests/{invalid-typing-17.tolk => invalid-typing/err-6881.tolk} (100%) rename tolk-tester/tests/{invalid-typing-7.tolk => invalid-typing/err-6901.tolk} (100%) rename tolk-tester/tests/{invalid-typing-19.tolk => invalid-typing/err-6903.tolk} (100%) rename tolk-tester/tests/{invalid-generics-7.tolk => invalid-typing/err-6921.tolk} (100%) rename tolk-tester/tests/{invalid-declaration-12.tolk => invalid-typing/err-6925.tolk} (100%) rename tolk-tester/tests/{invalid-typing-42.tolk => invalid-typing/err-6937.tolk} (100%) rename tolk-tester/tests/{invalid-generics-12.tolk => invalid-typing/err-6981.tolk} (100%) rename tolk-tester/tests/{invalid-typing-30.tolk => invalid-typing/err-6992.tolk} (100%) delete mode 100644 tolk-tester/tests/method_id.tolk create mode 100644 tolk-tester/tests/some-tests-1.tolk rename tolk-tester/tests/{a10.tolk => some-tests-2.tolk} (100%) rename tolk-tester/tests/{w2.tolk => some-tests-3.tolk} (100%) rename tolk-tester/tests/{s1.tolk => strings-tests.tolk} (100%) rename tolk-tester/tests/{try-func.tolk => try-catch-tests.tolk} (100%) rename tolk-tester/tests/{unbalanced_ret_loops.tolk => unbalanced-ret-loops.tolk} (100%) create mode 100644 tolk-tester/tests/unbalanced-ret.tolk delete mode 100644 tolk-tester/tests/unbalanced_ret.tolk delete mode 100644 tolk-tester/tests/unbalanced_ret_inline.tolk delete mode 100644 tolk-tester/tests/unbalanced_ret_nested.tolk rename tolk-tester/tests/{var-apply.tolk => var-apply-tests.tolk} (85%) delete mode 100644 tolk-tester/tests/w1.tolk delete mode 100644 tolk-tester/tests/w6.tolk delete mode 100644 tolk-tester/tests/w7.tolk delete mode 100644 tolk-tester/tests/w9.tolk rename tolk-tester/tests/{ => warnings-not-errors}/unreachable-1.tolk (100%) rename tolk-tester/tests/{ => warnings-not-errors}/unreachable-2.tolk (100%) rename tolk-tester/tests/{ => warnings-not-errors}/unreachable-3.tolk (100%) rename tolk-tester/tests/{ => warnings-not-errors}/unreachable-4.tolk (100%) rename tolk-tester/tests/{ => warnings-not-errors}/warnings-1.tolk (100%) rename tolk-tester/tests/{ => warnings-not-errors}/warnings-2.tolk (100%) diff --git a/tolk-tester/tests/a6.tolk b/tolk-tester/tests/a-tests.tolk similarity index 98% rename from tolk-tester/tests/a6.tolk rename to tolk-tester/tests/a-tests.tolk index ca1737a1e..65bff5ad1 100644 --- a/tolk-tester/tests/a6.tolk +++ b/tolk-tester/tests/a-tests.tolk @@ -76,7 +76,7 @@ fun main(): int { } /** - method_id | in | out + method_id | in | out @testcase | 0 | | 31415926535897932384626433832795028841971693993751058209749445923078164 @code_hash 84337043972311674339187056298873613816389434478842780265748859098303774481976 diff --git a/tolk-tester/tests/a6_1.tolk b/tolk-tester/tests/a6_1.tolk deleted file mode 100644 index 8079972b1..000000000 --- a/tolk-tester/tests/a6_1.tolk +++ /dev/null @@ -1,22 +0,0 @@ -fun main(a: int, b: int, c: int, d: int, e: int, f: int): (int, int) { - var D: int = a * d - b * c; - var Dx: int = e * d - b * f; - var Dy: int = a * f - e * c; - return (Dx / D, Dy / D); -} - -@method_id(101) -fun testDivMod(x: int, y: int) { - return (divMod(x, y), modDiv(x, y), mulDivMod(x, y, 10)); -} - -/** - method_id | in | out -@testcase | 0 | 1 1 1 -1 10 6 | 8 2 -@testcase | 0 | 817 -31 624 -241 132272 272276 | 132 -788 -@testcase | 0 | -886 562 498 -212 -36452 -68958 | -505 -861 -@testcase | 0 | 448 -433 -444 792 150012 -356232 | -218 -572 -@testcase | 0 | -40 -821 433 -734 -721629 -741724 | -206 889 -@testcase | 0 | -261 -98 -494 868 -166153 733738 | 263 995 -@testcase | 101 | 112 3 | 37 1 1 37 33 6 -*/ diff --git a/tolk-tester/tests/a6_5.tolk b/tolk-tester/tests/a6_5.tolk deleted file mode 100644 index 43fd59c5a..000000000 --- a/tolk-tester/tests/a6_5.tolk +++ /dev/null @@ -1,26 +0,0 @@ -@deprecated -fun twice(f: int -> int, x: int) { - return f (f (x)); -} - -fun sqr(x: int) { - return x * x; -} - -fun main(x: int): int { - var f = sqr; - return twice(f, x) * f(x); -} - -@method_id(4) -fun pow6(x: int): int { - return twice(sqr, x) * sqr(x); -} - -/** - method_id | in | out -@testcase | 0 | 3 | 729 -@testcase | 0 | 10 | 1000000 -@testcase | 4 | 3 | 729 -@testcase | 4 | 10 | 1000000 -*/ diff --git a/tolk-tester/tests/a7.tolk b/tolk-tester/tests/a7.tolk deleted file mode 100644 index 1c0ae2eb3..000000000 --- a/tolk-tester/tests/a7.tolk +++ /dev/null @@ -1,24 +0,0 @@ -fun main() { } -@method_id(1) -fun steps(x: int): int { - var n = 0; - while (x > 1) { - n += 1; - if (x & 1) { - x = 3 * x + 1; - } else { - x >>= 1; - } - } - return n; -} - -/** - method_id | in | out -@testcase | 1 | 1 | 0 -@testcase | 1 | 2 | 1 -@testcase | 1 | 5 | 5 -@testcase | 1 | 19 | 20 -@testcase | 1 | 27 | 111 -@testcase | 1 | 100 | 25 -*/ diff --git a/tolk-tester/tests/allow_post_modification.tolk b/tolk-tester/tests/allow-post-modification.tolk similarity index 100% rename from tolk-tester/tests/allow_post_modification.tolk rename to tolk-tester/tests/allow-post-modification.tolk diff --git a/tolk-tester/tests/asm_arg_order.tolk b/tolk-tester/tests/asm-arg-order.tolk similarity index 100% rename from tolk-tester/tests/asm_arg_order.tolk rename to tolk-tester/tests/asm-arg-order.tolk diff --git a/tolk-tester/tests/c2.tolk b/tolk-tester/tests/c2.tolk deleted file mode 100644 index bcbc6c938..000000000 --- a/tolk-tester/tests/c2.tolk +++ /dev/null @@ -1,28 +0,0 @@ -global op: (int, int) -> int; - -fun check_assoc(a: int, b: int, c: int): bool { - return op(op(a, b), c) == op(a, op(b, c)); -} - -fun unnamed_args(_: int, _: slice, _: int) { - return true; -} - -fun main(x: int, y: int, z: int): bool? { - op = `_+_`; - if (0) { return null; } - return check_assoc(x, y, z); -} - -@method_id(101) -fun test101(x: int, z: int) { - return unnamed_args(x, "asdf", z); -} - -/** - method_id | in | out -@testcase | 0 | 2 3 9 | -1 -@testcase | 0 | 11 22 44 | -1 -@testcase | 0 | -1 -10 -20 | -1 -@testcase | 101 | 1 10 | -1 -*/ diff --git a/tolk-tester/tests/c2_1.tolk b/tolk-tester/tests/c2_1.tolk deleted file mode 100644 index ef1e589ad..000000000 --- a/tolk-tester/tests/c2_1.tolk +++ /dev/null @@ -1,14 +0,0 @@ -fun check_assoc(op: (int, int) -> int, a: int, b: int, c: int) { - return op(op(a, b), c) == op(a, op(b, c)); -} - -fun main(x: int, y: int, z: int): bool { - return check_assoc(`_+_`, x, y, z); -} - -/** - method_id | in | out -@testcase | 0 | 2 3 9 | -1 -@testcase | 0 | 11 22 44 | -1 -@testcase | 0 | -1 -10 -20 | -1 -*/ diff --git a/tolk-tester/tests/code_after_ifelse.tolk b/tolk-tester/tests/code_after_ifelse.tolk deleted file mode 100644 index 6a16262f8..000000000 --- a/tolk-tester/tests/code_after_ifelse.tolk +++ /dev/null @@ -1,41 +0,0 @@ -fun elseif(cond: int) { - if (cond > 0) { - throw(cond); - } -} - -@inline -@method_id(101) -fun foo(x: int): int { - if (x==1) { - return 111; - } else { - x *= 2; - } - return x + 1; -} - -fun main(x: int): (int, int) { - return (foo(x), 222); -} - -@method_id(102) -fun test2(x: int) { - try { - if (x < 0) { return -1; } - elseif (x); - } catch(excNo) { - return excNo * 1000; - } - return 0; -} - -/** - method_id | in | out -@testcase | 0 | 1 | 111 222 -@testcase | 0 | 3 | 7 222 -@testcase | 101 | 1 | 111 -@testcase | 101 | 3 | 7 -@testcase | 102 | -5 | -1 -@testcase | 102 | 5 | 5000 -*/ diff --git a/tolk-tester/tests/codegen_check_demo.tolk b/tolk-tester/tests/codegen-check-demo.tolk similarity index 100% rename from tolk-tester/tests/codegen_check_demo.tolk rename to tolk-tester/tests/codegen-check-demo.tolk diff --git a/tolk-tester/tests/co1.tolk b/tolk-tester/tests/constants-tests.tolk similarity index 100% rename from tolk-tester/tests/co1.tolk rename to tolk-tester/tests/constants-tests.tolk diff --git a/tolk-tester/tests/if_stmt.tolk b/tolk-tester/tests/if-else-tests.tolk similarity index 59% rename from tolk-tester/tests/if_stmt.tolk rename to tolk-tester/tests/if-else-tests.tolk index 0f78a5162..0f1d31f94 100644 --- a/tolk-tester/tests/if_stmt.tolk +++ b/tolk-tester/tests/if-else-tests.tolk @@ -31,6 +31,48 @@ fun test3(x: int) { return -1; } +fun elseif(cond: int) { + if (cond > 0) { + throw(cond); + } +} + +@inline +@method_id(104) +fun test4(x: int): int { + if (x==1) { + return 111; + } else { + x *= 2; + } + return x + 1; +} + +@method_id(105) +fun test5(x: int): (int, int) { + return (test4(x), 222); +} + +@method_id(106) +fun test6(x: int) { + try { + if (x < 0) { return -1; } + elseif (x); + } catch(excNo) { + return excNo * 1000; + } + return 0; +} + +fun doNothing(): void {} + +@method_id(107) +fun test7() { + if (random()) { return doNothing(); } + // here we test that no "missing return" error +} + + fun main() { } @@ -48,6 +90,12 @@ fun main() { @testcase | 103 | 20 | 20 @testcase | 103 | 40 | 50 @testcase | 103 | 50 | -1 +@testcase | 104 | 1 | 111 +@testcase | 104 | 3 | 7 +@testcase | 105 | 1 | 111 222 +@testcase | 105 | 3 | 7 222 +@testcase | 106 | -5 | -1 +@testcase | 106 | 5 | 5000 @fif_codegen """ diff --git a/tolk-tester/tests/inference-tests.tolk b/tolk-tester/tests/inference-tests.tolk index e785ac5aa..463d4a7b7 100644 --- a/tolk-tester/tests/inference-tests.tolk +++ b/tolk-tester/tests/inference-tests.tolk @@ -95,6 +95,15 @@ fun test7() { // __expect_type(eq<(int, slice)>, "(int, slice) -> (int, slice)"); } +fun unnamed_args(_: int, _: slice, _: int) { + return true; +} + +@method_id(108) +fun test8(x: int, z: int) { + return unnamed_args(x, "asdf", z); +} + fun alwaysThrows(): never { throw 123; } fun alwaysThrowsNotAnnotated() { throw 123; } fun alwaysThrowsNotAnnotated2() { alwaysThrows(); } @@ -112,5 +121,6 @@ fun main() { } /** -@testcase | 0 | | 0 +@testcase | 0 | | 0 +@testcase | 108 | 1 10 | -1 */ diff --git a/tolk-tester/tests/inline-tests.tolk b/tolk-tester/tests/inline-tests.tolk new file mode 100644 index 000000000..13add753e --- /dev/null +++ b/tolk-tester/tests/inline-tests.tolk @@ -0,0 +1,140 @@ +fun foo1(x: int): int { + if (x == 1) { + return 1; + } + return 2; +} + +@inline +fun foo2(x: int): int { + if (x == 1) { + return 11; + } + return 22; +} + +@inline_ref +fun foo3(x: int): int { + if (x == 1) { + return 111; + } + return 222; +} + +@method_id(101) +fun test1(x: int): (int, int, int) { + return (foo1(x)+1, foo2(x)+1, foo3(x)+1); +} + +global g: int; + +@inline +fun foo_repeat() { + g = 1; + repeat(5) { + g *= 2; + } +} + +@inline +fun foo_until(): int { + g = 1; + var i: int = 0; + do { + g *= 2; + i += 1; + } while (i < 8); + return i; +} + +@inline +fun foo_while(): int { + g = 1; + var i: int = 0; + while (i < 10) { + g *= 2; + i += 1; + } + return i; +} + +@method_id(102) +fun test2() { + foo_repeat(); + var x: int = g; + foo_until(); + var y: int = g; + foo_while(); + var z: int = g; + return (x, y, z); +} + +@inline +fun foo_big(x: int): int { + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + return x; +} + +@method_id(103) +fun test3(x: int): int { + return foo_big(x) * 10 + 5; +} + +fun main() {} + +/** + method_id | in | out +@testcase | 101 | 1 | 2 12 112 +@testcase | 101 | 2 | 3 23 223 +@testcase | 102 | | 32 256 1024 +@testcase | 103 | 9 | 9111111111111111111111111111111111111111111111111115 +*/ diff --git a/tolk-tester/tests/inline_big.tolk b/tolk-tester/tests/inline_big.tolk deleted file mode 100644 index be014eb5d..000000000 --- a/tolk-tester/tests/inline_big.tolk +++ /dev/null @@ -1,62 +0,0 @@ -@inline -fun foo(x: int): int { - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - return x; -} - -fun main(x: int): int { - return foo(x) * 10 + 5; -} -/** - method_id | in | out -@testcase | 0 | 9 | 9111111111111111111111111111111111111111111111111115 -*/ diff --git a/tolk-tester/tests/inline_if.tolk b/tolk-tester/tests/inline_if.tolk deleted file mode 100644 index 9f1fa8c12..000000000 --- a/tolk-tester/tests/inline_if.tolk +++ /dev/null @@ -1,28 +0,0 @@ -fun foo1(x: int): int { - if (x == 1) { - return 1; - } - return 2; -} -@inline -fun foo2(x: int): int { - if (x == 1) { - return 11; - } - return 22; -} -@inline_ref -fun foo3(x: int): int { - if (x == 1) { - return 111; - } - return 222; -} -fun main(x: int): (int, int, int) { - return (foo1(x)+1, foo2(x)+1, foo3(x)+1); -} -/** - method_id | in | out -@testcase | 0 | 1 | 2 12 112 -@testcase | 0 | 2 | 3 23 223 -*/ diff --git a/tolk-tester/tests/inline_loops.tolk b/tolk-tester/tests/inline_loops.tolk deleted file mode 100644 index eba595a5e..000000000 --- a/tolk-tester/tests/inline_loops.tolk +++ /dev/null @@ -1,48 +0,0 @@ -global g: int; - -@inline -fun foo_repeat() { - g = 1; - repeat(5) { - g *= 2; - } -} - -@inline -fun foo_until(): int { - g = 1; - var i: int = 0; - do { - g *= 2; - i += 1; - } while (i < 8); - return i; -} - -@inline -fun foo_while(): int { - g = 1; - var i: int = 0; - while (i < 10) { - g *= 2; - i += 1; - } - return i; -} - -fun main() { - foo_repeat(); - var x: int = g; - foo_until(); - var y: int = g; - foo_while(); - var z: int = g; - return (x, y, z); -} - -/** - method_id | in | out -@testcase | 0 | | 32 256 1024 - -@code_hash 102749806552989901976653997041637095139193406161777448419603700344770997608788 -*/ diff --git a/tolk-tester/tests/invalid-declaration-8.tolk b/tolk-tester/tests/invalid-declaration/err-1084.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-8.tolk rename to tolk-tester/tests/invalid-declaration/err-1084.tolk diff --git a/tolk-tester/tests/invalid-declaration-16.tolk b/tolk-tester/tests/invalid-declaration/err-1153.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-16.tolk rename to tolk-tester/tests/invalid-declaration/err-1153.tolk diff --git a/tolk-tester/tests/invalid-declaration-1.tolk b/tolk-tester/tests/invalid-declaration/err-1209.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-1.tolk rename to tolk-tester/tests/invalid-declaration/err-1209.tolk diff --git a/tolk-tester/tests/invalid-declaration-10.tolk b/tolk-tester/tests/invalid-declaration/err-1361.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-10.tolk rename to tolk-tester/tests/invalid-declaration/err-1361.tolk diff --git a/tolk-tester/tests/invalid-get-method-2.tolk b/tolk-tester/tests/invalid-declaration/err-1449.tolk similarity index 100% rename from tolk-tester/tests/invalid-get-method-2.tolk rename to tolk-tester/tests/invalid-declaration/err-1449.tolk diff --git a/tolk-tester/tests/invalid-mutate-13.tolk b/tolk-tester/tests/invalid-declaration/err-1505.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-13.tolk rename to tolk-tester/tests/invalid-declaration/err-1505.tolk diff --git a/tolk-tester/tests/invalid-self-6.tolk b/tolk-tester/tests/invalid-declaration/err-1601.tolk similarity index 100% rename from tolk-tester/tests/invalid-self-6.tolk rename to tolk-tester/tests/invalid-declaration/err-1601.tolk diff --git a/tolk-tester/tests/invalid-redefinition-1.tolk b/tolk-tester/tests/invalid-declaration/err-1683.tolk similarity index 100% rename from tolk-tester/tests/invalid-redefinition-1.tolk rename to tolk-tester/tests/invalid-declaration/err-1683.tolk diff --git a/tolk-tester/tests/invalid-builtin-1.tolk b/tolk-tester/tests/invalid-declaration/err-1701.tolk similarity index 100% rename from tolk-tester/tests/invalid-builtin-1.tolk rename to tolk-tester/tests/invalid-declaration/err-1701.tolk diff --git a/tolk-tester/tests/invalid-declaration-11.tolk b/tolk-tester/tests/invalid-declaration/err-1790.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-11.tolk rename to tolk-tester/tests/invalid-declaration/err-1790.tolk diff --git a/tolk-tester/tests/invalid-declaration-6.tolk b/tolk-tester/tests/invalid-declaration/err-1794.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-6.tolk rename to tolk-tester/tests/invalid-declaration/err-1794.tolk diff --git a/tolk-tester/tests/invalid-declaration-9.tolk b/tolk-tester/tests/invalid-declaration/err-1805.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-9.tolk rename to tolk-tester/tests/invalid-declaration/err-1805.tolk diff --git a/tolk-tester/tests/invalid-cyclic-1.tolk b/tolk-tester/tests/invalid-declaration/err-1842.tolk similarity index 100% rename from tolk-tester/tests/invalid-cyclic-1.tolk rename to tolk-tester/tests/invalid-declaration/err-1842.tolk diff --git a/tolk-tester/tests/invalid-redefinition-2.tolk b/tolk-tester/tests/invalid-declaration/err-1918.tolk similarity index 78% rename from tolk-tester/tests/invalid-redefinition-2.tolk rename to tolk-tester/tests/invalid-declaration/err-1918.tolk index 3a300dc2e..ae801de26 100644 --- a/tolk-tester/tests/invalid-redefinition-2.tolk +++ b/tolk-tester/tests/invalid-declaration/err-1918.tolk @@ -8,5 +8,5 @@ fun hello(): int { @compilation_should_fail @stderr fun hello() @stderr redefinition of symbol, previous was at -@stderr invalid-redefinition-2.tolk:1:1 +@stderr err-1918.tolk:1:1 */ diff --git a/tolk-tester/tests/invalid-get-method-1.tolk b/tolk-tester/tests/invalid-declaration/err-1940.tolk similarity index 100% rename from tolk-tester/tests/invalid-get-method-1.tolk rename to tolk-tester/tests/invalid-declaration/err-1940.tolk diff --git a/tolk-tester/tests/invalid-no-import-1.tolk b/tolk-tester/tests/invalid-no-import-1.tolk deleted file mode 100644 index 89f879a36..000000000 --- a/tolk-tester/tests/invalid-no-import-1.tolk +++ /dev/null @@ -1,8 +0,0 @@ -import "imports/some-math.tolk"; -import "imports/invalid-no-import.tolk"; - -/** -@compilation_should_fail -@stderr imports/invalid-no-import.tolk:2:13 -@stderr Using a non-imported symbol `someAdd` - */ diff --git a/tolk-tester/tests/invalid-call-5.tolk b/tolk-tester/tests/invalid-semantics/err-4002.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-5.tolk rename to tolk-tester/tests/invalid-semantics/err-4002.tolk diff --git a/tolk-tester/tests/invalid-mutate-5.tolk b/tolk-tester/tests/invalid-semantics/err-4061.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-5.tolk rename to tolk-tester/tests/invalid-semantics/err-4061.tolk diff --git a/tolk-tester/tests/invalid-pure-1.tolk b/tolk-tester/tests/invalid-semantics/err-4064.tolk similarity index 100% rename from tolk-tester/tests/invalid-pure-1.tolk rename to tolk-tester/tests/invalid-semantics/err-4064.tolk diff --git a/tolk-tester/tests/invalid-call-4.tolk b/tolk-tester/tests/invalid-semantics/err-4080.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-4.tolk rename to tolk-tester/tests/invalid-semantics/err-4080.tolk diff --git a/tolk-tester/tests/invalid-mutate-16.tolk b/tolk-tester/tests/invalid-semantics/err-4083.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-16.tolk rename to tolk-tester/tests/invalid-semantics/err-4083.tolk diff --git a/tolk-tester/tests/invalid-generics-6.tolk b/tolk-tester/tests/invalid-semantics/err-4104.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-6.tolk rename to tolk-tester/tests/invalid-semantics/err-4104.tolk diff --git a/tolk-tester/tests/invalid-declaration-14.tolk b/tolk-tester/tests/invalid-semantics/err-4127.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-14.tolk rename to tolk-tester/tests/invalid-semantics/err-4127.tolk diff --git a/tolk-tester/tests/invalid-mutate-18.tolk b/tolk-tester/tests/invalid-semantics/err-4195.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-18.tolk rename to tolk-tester/tests/invalid-semantics/err-4195.tolk diff --git a/tolk-tester/tests/invalid-generics-10.tolk b/tolk-tester/tests/invalid-semantics/err-4266.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-10.tolk rename to tolk-tester/tests/invalid-semantics/err-4266.tolk diff --git a/tolk-tester/tests/invalid-mutate-7.tolk b/tolk-tester/tests/invalid-semantics/err-4377.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-7.tolk rename to tolk-tester/tests/invalid-semantics/err-4377.tolk diff --git a/tolk-tester/tests/invalid-typing-13.tolk b/tolk-tester/tests/invalid-semantics/err-4381.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-13.tolk rename to tolk-tester/tests/invalid-semantics/err-4381.tolk diff --git a/tolk-tester/tests/invalid-pure-3.tolk b/tolk-tester/tests/invalid-semantics/err-4385.tolk similarity index 100% rename from tolk-tester/tests/invalid-pure-3.tolk rename to tolk-tester/tests/invalid-semantics/err-4385.tolk diff --git a/tolk-tester/tests/invalid-pure-2.tolk b/tolk-tester/tests/invalid-semantics/err-4413.tolk similarity index 100% rename from tolk-tester/tests/invalid-pure-2.tolk rename to tolk-tester/tests/invalid-semantics/err-4413.tolk diff --git a/tolk-tester/tests/invalid-generics-5.tolk b/tolk-tester/tests/invalid-semantics/err-4416.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-5.tolk rename to tolk-tester/tests/invalid-semantics/err-4416.tolk diff --git a/tolk-tester/tests/invalid-mutate-11.tolk b/tolk-tester/tests/invalid-semantics/err-4473.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-11.tolk rename to tolk-tester/tests/invalid-semantics/err-4473.tolk diff --git a/tolk-tester/tests/invalid-mutate-10.tolk b/tolk-tester/tests/invalid-semantics/err-4506.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-10.tolk rename to tolk-tester/tests/invalid-semantics/err-4506.tolk diff --git a/tolk-tester/tests/invalid-generics-11.tolk b/tolk-tester/tests/invalid-semantics/err-4510.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-11.tolk rename to tolk-tester/tests/invalid-semantics/err-4510.tolk diff --git a/tolk-tester/tests/invalid-call-8.tolk b/tolk-tester/tests/invalid-semantics/err-4520.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-8.tolk rename to tolk-tester/tests/invalid-semantics/err-4520.tolk diff --git a/tolk-tester/tests/invalid-mutate-17.tolk b/tolk-tester/tests/invalid-semantics/err-4542.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-17.tolk rename to tolk-tester/tests/invalid-semantics/err-4542.tolk diff --git a/tolk-tester/tests/invalid-mutate-4.tolk b/tolk-tester/tests/invalid-semantics/err-4553.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-4.tolk rename to tolk-tester/tests/invalid-semantics/err-4553.tolk diff --git a/tolk-tester/tests/invalid-syntax-3.tolk b/tolk-tester/tests/invalid-semantics/err-4571.tolk similarity index 100% rename from tolk-tester/tests/invalid-syntax-3.tolk rename to tolk-tester/tests/invalid-semantics/err-4571.tolk diff --git a/tolk-tester/tests/invalid-generics-9.tolk b/tolk-tester/tests/invalid-semantics/err-4604.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-9.tolk rename to tolk-tester/tests/invalid-semantics/err-4604.tolk diff --git a/tolk-tester/tests/invalid-syntax-6.tolk b/tolk-tester/tests/invalid-semantics/err-4639.tolk similarity index 100% rename from tolk-tester/tests/invalid-syntax-6.tolk rename to tolk-tester/tests/invalid-semantics/err-4639.tolk diff --git a/tolk-tester/tests/invalid-mutate-12.tolk b/tolk-tester/tests/invalid-semantics/err-4705.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-12.tolk rename to tolk-tester/tests/invalid-semantics/err-4705.tolk diff --git a/tolk-tester/tests/invalid-mutate-1.tolk b/tolk-tester/tests/invalid-semantics/err-4728.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-1.tolk rename to tolk-tester/tests/invalid-semantics/err-4728.tolk diff --git a/tolk-tester/tests/invalid-call-3.tolk b/tolk-tester/tests/invalid-semantics/err-4751.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-3.tolk rename to tolk-tester/tests/invalid-semantics/err-4751.tolk diff --git a/tolk-tester/tests/invalid-call-11.tolk b/tolk-tester/tests/invalid-semantics/err-4804.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-11.tolk rename to tolk-tester/tests/invalid-semantics/err-4804.tolk diff --git a/tolk-tester/tests/invalid-mutate-3.tolk b/tolk-tester/tests/invalid-semantics/err-4819.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-3.tolk rename to tolk-tester/tests/invalid-semantics/err-4819.tolk diff --git a/tolk-tester/tests/invalid-mutate-15.tolk b/tolk-tester/tests/invalid-semantics/err-4828.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-15.tolk rename to tolk-tester/tests/invalid-semantics/err-4828.tolk diff --git a/tolk-tester/tests/invalid-mutate-6.tolk b/tolk-tester/tests/invalid-semantics/err-4838.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-6.tolk rename to tolk-tester/tests/invalid-semantics/err-4838.tolk diff --git a/tolk-tester/tests/invalid-call-6.tolk b/tolk-tester/tests/invalid-semantics/err-4841.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-6.tolk rename to tolk-tester/tests/invalid-semantics/err-4841.tolk diff --git a/tolk-tester/tests/invalid-mutate-9.tolk b/tolk-tester/tests/invalid-semantics/err-4851.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-9.tolk rename to tolk-tester/tests/invalid-semantics/err-4851.tolk diff --git a/tolk-tester/tests/invalid-generics-1.tolk b/tolk-tester/tests/invalid-semantics/err-4880.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-1.tolk rename to tolk-tester/tests/invalid-semantics/err-4880.tolk diff --git a/tolk-tester/tests/invalid-mutate-14.tolk b/tolk-tester/tests/invalid-semantics/err-4899.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-14.tolk rename to tolk-tester/tests/invalid-semantics/err-4899.tolk diff --git a/tolk-tester/tests/invalid-mutate-2.tolk b/tolk-tester/tests/invalid-semantics/err-4912.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-2.tolk rename to tolk-tester/tests/invalid-semantics/err-4912.tolk diff --git a/tolk-tester/tests/invalid-mutate-19.tolk b/tolk-tester/tests/invalid-semantics/err-4941.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-19.tolk rename to tolk-tester/tests/invalid-semantics/err-4941.tolk diff --git a/tolk-tester/tests/invalid-call-9.tolk b/tolk-tester/tests/invalid-semantics/err-4951.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-9.tolk rename to tolk-tester/tests/invalid-semantics/err-4951.tolk diff --git a/tolk-tester/tests/invalid-call-2.tolk b/tolk-tester/tests/invalid-semantics/err-4956.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-2.tolk rename to tolk-tester/tests/invalid-semantics/err-4956.tolk diff --git a/tolk-tester/tests/invalid-mutate-8.tolk b/tolk-tester/tests/invalid-symbol/err-2055.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-8.tolk rename to tolk-tester/tests/invalid-symbol/err-2055.tolk diff --git a/tolk-tester/tests/invalid-call-7.tolk b/tolk-tester/tests/invalid-symbol/err-2188.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-7.tolk rename to tolk-tester/tests/invalid-symbol/err-2188.tolk diff --git a/tolk-tester/tests/invalid-symbol/err-2205.tolk b/tolk-tester/tests/invalid-symbol/err-2205.tolk new file mode 100644 index 000000000..3c9d72204 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2205.tolk @@ -0,0 +1,8 @@ +import "../imports/some-math.tolk"; +import "../imports/invalid-no-import.tolk"; + +/** +@compilation_should_fail +@stderr ../imports/invalid-no-import.tolk:2:13 +@stderr Using a non-imported symbol `someAdd` + */ diff --git a/tolk-tester/tests/invalid-typing-32.tolk b/tolk-tester/tests/invalid-symbol/err-2255.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-32.tolk rename to tolk-tester/tests/invalid-symbol/err-2255.tolk diff --git a/tolk-tester/tests/invalid-symbol-1.tolk b/tolk-tester/tests/invalid-symbol/err-2374.tolk similarity index 100% rename from tolk-tester/tests/invalid-symbol-1.tolk rename to tolk-tester/tests/invalid-symbol/err-2374.tolk diff --git a/tolk-tester/tests/invalid-symbol-2.tolk b/tolk-tester/tests/invalid-symbol/err-2463.tolk similarity index 100% rename from tolk-tester/tests/invalid-symbol-2.tolk rename to tolk-tester/tests/invalid-symbol/err-2463.tolk diff --git a/tolk-tester/tests/invalid-redefinition-6.tolk b/tolk-tester/tests/invalid-symbol/err-2501.tolk similarity index 100% rename from tolk-tester/tests/invalid-redefinition-6.tolk rename to tolk-tester/tests/invalid-symbol/err-2501.tolk diff --git a/tolk-tester/tests/invalid-self-7.tolk b/tolk-tester/tests/invalid-symbol/err-2520.tolk similarity index 100% rename from tolk-tester/tests/invalid-self-7.tolk rename to tolk-tester/tests/invalid-symbol/err-2520.tolk diff --git a/tolk-tester/tests/invalid-no-import-2.tolk b/tolk-tester/tests/invalid-symbol/err-2539.tolk similarity index 66% rename from tolk-tester/tests/invalid-no-import-2.tolk rename to tolk-tester/tests/invalid-symbol/err-2539.tolk index d78346b90..f7c66c258 100644 --- a/tolk-tester/tests/invalid-no-import-2.tolk +++ b/tolk-tester/tests/invalid-symbol/err-2539.tolk @@ -1,9 +1,9 @@ import "@stdlib/tvm-dicts" -import "imports/use-dicts-err.tolk" +import "../imports/use-dicts-err.tolk" /** @compilation_should_fail -@stderr imports/use-dicts-err.tolk:2:22 +@stderr ../imports/use-dicts-err.tolk:2:22 @stderr Using a non-imported symbol `createEmptyDict` @stderr Forgot to import "@stdlib/tvm-dicts"? */ diff --git a/tolk-tester/tests/invalid-redefinition-3.tolk b/tolk-tester/tests/invalid-symbol/err-2649.tolk similarity index 100% rename from tolk-tester/tests/invalid-redefinition-3.tolk rename to tolk-tester/tests/invalid-symbol/err-2649.tolk diff --git a/tolk-tester/tests/invalid-self-3.tolk b/tolk-tester/tests/invalid-symbol/err-2673.tolk similarity index 100% rename from tolk-tester/tests/invalid-self-3.tolk rename to tolk-tester/tests/invalid-symbol/err-2673.tolk diff --git a/tolk-tester/tests/invalid-redefinition-5.tolk b/tolk-tester/tests/invalid-symbol/err-2715.tolk similarity index 100% rename from tolk-tester/tests/invalid-redefinition-5.tolk rename to tolk-tester/tests/invalid-symbol/err-2715.tolk diff --git a/tolk-tester/tests/invalid-typing-1.tolk b/tolk-tester/tests/invalid-symbol/err-2750.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-1.tolk rename to tolk-tester/tests/invalid-symbol/err-2750.tolk diff --git a/tolk-tester/tests/invalid-declaration-15.tolk b/tolk-tester/tests/invalid-symbol/err-2804.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-15.tolk rename to tolk-tester/tests/invalid-symbol/err-2804.tolk diff --git a/tolk-tester/tests/invalid-redefinition-4.tolk b/tolk-tester/tests/invalid-symbol/err-2853.tolk similarity index 100% rename from tolk-tester/tests/invalid-redefinition-4.tolk rename to tolk-tester/tests/invalid-symbol/err-2853.tolk diff --git a/tolk-tester/tests/invalid-call-1.tolk b/tolk-tester/tests/invalid-symbol/err-2914.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-1.tolk rename to tolk-tester/tests/invalid-symbol/err-2914.tolk diff --git a/tolk-tester/tests/invalid-import.tolk b/tolk-tester/tests/invalid-symbol/err-2980.tolk similarity index 73% rename from tolk-tester/tests/invalid-import.tolk rename to tolk-tester/tests/invalid-symbol/err-2980.tolk index 416764b62..de29cc657 100644 --- a/tolk-tester/tests/invalid-import.tolk +++ b/tolk-tester/tests/invalid-symbol/err-2980.tolk @@ -4,8 +4,8 @@ /** @compilation_should_fail -On Linux/Mac, `realpath()` returns an error, and the error message is `cannot find file` +On Linux/Mac, `realpath()` returns an error, and the error message is "cannot find file" On Windows, it fails after, on reading, with a message "cannot open file" -@stderr invalid-import.tolk:2:7: error: Failed to import: cannot +@stderr err-2980.tolk:2:7: error: Failed to import: cannot @stderr import "unexisting.tolk"; */ diff --git a/tolk-tester/tests/invalid-tolk-version.tolk b/tolk-tester/tests/invalid-syntax/err-3001.tolk similarity index 100% rename from tolk-tester/tests/invalid-tolk-version.tolk rename to tolk-tester/tests/invalid-syntax/err-3001.tolk diff --git a/tolk-tester/tests/invalid-call-13.tolk b/tolk-tester/tests/invalid-syntax/err-3015.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-13.tolk rename to tolk-tester/tests/invalid-syntax/err-3015.tolk diff --git a/tolk-tester/tests/invalid-bitwise-4.tolk b/tolk-tester/tests/invalid-syntax/err-3035.tolk similarity index 100% rename from tolk-tester/tests/invalid-bitwise-4.tolk rename to tolk-tester/tests/invalid-syntax/err-3035.tolk diff --git a/tolk-tester/tests/invalid-nopar-1.tolk b/tolk-tester/tests/invalid-syntax/err-3041.tolk similarity index 100% rename from tolk-tester/tests/invalid-nopar-1.tolk rename to tolk-tester/tests/invalid-syntax/err-3041.tolk diff --git a/tolk-tester/tests/invalid-shift-1.tolk b/tolk-tester/tests/invalid-syntax/err-3044.tolk similarity index 100% rename from tolk-tester/tests/invalid-shift-1.tolk rename to tolk-tester/tests/invalid-syntax/err-3044.tolk diff --git a/tolk-tester/tests/invalid-declaration-4.tolk b/tolk-tester/tests/invalid-syntax/err-3059.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-4.tolk rename to tolk-tester/tests/invalid-syntax/err-3059.tolk diff --git a/tolk-tester/tests/invalid-bitwise-3.tolk b/tolk-tester/tests/invalid-syntax/err-3102.tolk similarity index 100% rename from tolk-tester/tests/invalid-bitwise-3.tolk rename to tolk-tester/tests/invalid-syntax/err-3102.tolk diff --git a/tolk-tester/tests/invalid-catch-1.tolk b/tolk-tester/tests/invalid-syntax/err-3164.tolk similarity index 100% rename from tolk-tester/tests/invalid-catch-1.tolk rename to tolk-tester/tests/invalid-syntax/err-3164.tolk diff --git a/tolk-tester/tests/invalid-nopar-2.tolk b/tolk-tester/tests/invalid-syntax/err-3183.tolk similarity index 100% rename from tolk-tester/tests/invalid-nopar-2.tolk rename to tolk-tester/tests/invalid-syntax/err-3183.tolk diff --git a/tolk-tester/tests/invalid-nopar-4.tolk b/tolk-tester/tests/invalid-syntax/err-3197.tolk similarity index 100% rename from tolk-tester/tests/invalid-nopar-4.tolk rename to tolk-tester/tests/invalid-syntax/err-3197.tolk diff --git a/tolk-tester/tests/invalid-syntax-7.tolk b/tolk-tester/tests/invalid-syntax/err-3205.tolk similarity index 100% rename from tolk-tester/tests/invalid-syntax-7.tolk rename to tolk-tester/tests/invalid-syntax/err-3205.tolk diff --git a/tolk-tester/tests/invalid-nopar-3.tolk b/tolk-tester/tests/invalid-syntax/err-3274.tolk similarity index 100% rename from tolk-tester/tests/invalid-nopar-3.tolk rename to tolk-tester/tests/invalid-syntax/err-3274.tolk diff --git a/tolk-tester/tests/invalid-const-1.tolk b/tolk-tester/tests/invalid-syntax/err-3295.tolk similarity index 100% rename from tolk-tester/tests/invalid-const-1.tolk rename to tolk-tester/tests/invalid-syntax/err-3295.tolk diff --git a/tolk-tester/tests/invalid-cmt-old.tolk b/tolk-tester/tests/invalid-syntax/err-3390.tolk similarity index 100% rename from tolk-tester/tests/invalid-cmt-old.tolk rename to tolk-tester/tests/invalid-syntax/err-3390.tolk diff --git a/tolk-tester/tests/invalid-cmt-nested.tolk b/tolk-tester/tests/invalid-syntax/err-3407.tolk similarity index 100% rename from tolk-tester/tests/invalid-cmt-nested.tolk rename to tolk-tester/tests/invalid-syntax/err-3407.tolk diff --git a/tolk-tester/tests/invalid-call-14.tolk b/tolk-tester/tests/invalid-syntax/err-3472.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-14.tolk rename to tolk-tester/tests/invalid-syntax/err-3472.tolk diff --git a/tolk-tester/tests/invalid-syntax-5.tolk b/tolk-tester/tests/invalid-syntax/err-3527.tolk similarity index 100% rename from tolk-tester/tests/invalid-syntax-5.tolk rename to tolk-tester/tests/invalid-syntax/err-3527.tolk diff --git a/tolk-tester/tests/invalid-syntax-1.tolk b/tolk-tester/tests/invalid-syntax/err-3551.tolk similarity index 100% rename from tolk-tester/tests/invalid-syntax-1.tolk rename to tolk-tester/tests/invalid-syntax/err-3551.tolk diff --git a/tolk-tester/tests/invalid-catch-2.tolk b/tolk-tester/tests/invalid-syntax/err-3586.tolk similarity index 100% rename from tolk-tester/tests/invalid-catch-2.tolk rename to tolk-tester/tests/invalid-syntax/err-3586.tolk diff --git a/tolk-tester/tests/invalid-bitwise-5.tolk b/tolk-tester/tests/invalid-syntax/err-3588.tolk similarity index 100% rename from tolk-tester/tests/invalid-bitwise-5.tolk rename to tolk-tester/tests/invalid-syntax/err-3588.tolk diff --git a/tolk-tester/tests/invalid-declaration-7.tolk b/tolk-tester/tests/invalid-syntax/err-3603.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-7.tolk rename to tolk-tester/tests/invalid-syntax/err-3603.tolk diff --git a/tolk-tester/tests/invalid-syntax/err-3618.tolk b/tolk-tester/tests/invalid-syntax/err-3618.tolk new file mode 100644 index 000000000..7751c4a95 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3618.tolk @@ -0,0 +1,6 @@ +const x = 0b0012; + +/** +@compilation_should_fail +@stderr expected `;`, got `2` + */ diff --git a/tolk-tester/tests/invalid-call-12.tolk b/tolk-tester/tests/invalid-syntax/err-3714.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-12.tolk rename to tolk-tester/tests/invalid-syntax/err-3714.tolk diff --git a/tolk-tester/tests/invalid-syntax-4.tolk b/tolk-tester/tests/invalid-syntax/err-3719.tolk similarity index 100% rename from tolk-tester/tests/invalid-syntax-4.tolk rename to tolk-tester/tests/invalid-syntax/err-3719.tolk diff --git a/tolk-tester/tests/invalid-declaration-5.tolk b/tolk-tester/tests/invalid-syntax/err-3781.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-5.tolk rename to tolk-tester/tests/invalid-syntax/err-3781.tolk diff --git a/tolk-tester/tests/invalid-bitwise-1.tolk b/tolk-tester/tests/invalid-syntax/err-3819.tolk similarity index 100% rename from tolk-tester/tests/invalid-bitwise-1.tolk rename to tolk-tester/tests/invalid-syntax/err-3819.tolk diff --git a/tolk-tester/tests/invalid-bitwise-7.tolk b/tolk-tester/tests/invalid-syntax/err-3851.tolk similarity index 100% rename from tolk-tester/tests/invalid-bitwise-7.tolk rename to tolk-tester/tests/invalid-syntax/err-3851.tolk diff --git a/tolk-tester/tests/invalid-declaration-2.tolk b/tolk-tester/tests/invalid-syntax/err-3853.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-2.tolk rename to tolk-tester/tests/invalid-syntax/err-3853.tolk diff --git a/tolk-tester/tests/invalid-syntax-2.tolk b/tolk-tester/tests/invalid-syntax/err-3898.tolk similarity index 100% rename from tolk-tester/tests/invalid-syntax-2.tolk rename to tolk-tester/tests/invalid-syntax/err-3898.tolk diff --git a/tolk-tester/tests/invalid-bitwise-2.tolk b/tolk-tester/tests/invalid-syntax/err-3910.tolk similarity index 100% rename from tolk-tester/tests/invalid-bitwise-2.tolk rename to tolk-tester/tests/invalid-syntax/err-3910.tolk diff --git a/tolk-tester/tests/invalid-declaration-3.tolk b/tolk-tester/tests/invalid-syntax/err-3967.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-3.tolk rename to tolk-tester/tests/invalid-syntax/err-3967.tolk diff --git a/tolk-tester/tests/invalid-bitwise-6.tolk b/tolk-tester/tests/invalid-syntax/err-3992.tolk similarity index 100% rename from tolk-tester/tests/invalid-bitwise-6.tolk rename to tolk-tester/tests/invalid-syntax/err-3992.tolk diff --git a/tolk-tester/tests/invalid-assign-1.tolk b/tolk-tester/tests/invalid-typing/err-6039.tolk similarity index 100% rename from tolk-tester/tests/invalid-assign-1.tolk rename to tolk-tester/tests/invalid-typing/err-6039.tolk diff --git a/tolk-tester/tests/invalid-assign-3.tolk b/tolk-tester/tests/invalid-typing/err-6040.tolk similarity index 100% rename from tolk-tester/tests/invalid-assign-3.tolk rename to tolk-tester/tests/invalid-typing/err-6040.tolk diff --git a/tolk-tester/tests/invalid-call-10.tolk b/tolk-tester/tests/invalid-typing/err-6041.tolk similarity index 100% rename from tolk-tester/tests/invalid-call-10.tolk rename to tolk-tester/tests/invalid-typing/err-6041.tolk diff --git a/tolk-tester/tests/invalid-typing-18.tolk b/tolk-tester/tests/invalid-typing/err-6080.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-18.tolk rename to tolk-tester/tests/invalid-typing/err-6080.tolk diff --git a/tolk-tester/tests/invalid-generics-14.tolk b/tolk-tester/tests/invalid-typing/err-6085.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-14.tolk rename to tolk-tester/tests/invalid-typing/err-6085.tolk diff --git a/tolk-tester/tests/invalid-typing-43.tolk b/tolk-tester/tests/invalid-typing/err-6087.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-43.tolk rename to tolk-tester/tests/invalid-typing/err-6087.tolk diff --git a/tolk-tester/tests/invalid-generics-4.tolk b/tolk-tester/tests/invalid-typing/err-6095.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-4.tolk rename to tolk-tester/tests/invalid-typing/err-6095.tolk diff --git a/tolk-tester/tests/invalid-typing-20.tolk b/tolk-tester/tests/invalid-typing/err-6114.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-20.tolk rename to tolk-tester/tests/invalid-typing/err-6114.tolk diff --git a/tolk-tester/tests/invalid-typing-44.tolk b/tolk-tester/tests/invalid-typing/err-6130.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-44.tolk rename to tolk-tester/tests/invalid-typing/err-6130.tolk diff --git a/tolk-tester/tests/invalid-typing-4.tolk b/tolk-tester/tests/invalid-typing/err-6134.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-4.tolk rename to tolk-tester/tests/invalid-typing/err-6134.tolk diff --git a/tolk-tester/tests/invalid-typing-37.tolk b/tolk-tester/tests/invalid-typing/err-6135.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-37.tolk rename to tolk-tester/tests/invalid-typing/err-6135.tolk diff --git a/tolk-tester/tests/invalid-typing-25.tolk b/tolk-tester/tests/invalid-typing/err-6148.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-25.tolk rename to tolk-tester/tests/invalid-typing/err-6148.tolk diff --git a/tolk-tester/tests/invalid-typing-10.tolk b/tolk-tester/tests/invalid-typing/err-6188.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-10.tolk rename to tolk-tester/tests/invalid-typing/err-6188.tolk diff --git a/tolk-tester/tests/invalid-declaration-13.tolk b/tolk-tester/tests/invalid-typing/err-6200.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-13.tolk rename to tolk-tester/tests/invalid-typing/err-6200.tolk diff --git a/tolk-tester/tests/invalid-typing-38.tolk b/tolk-tester/tests/invalid-typing/err-6204.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-38.tolk rename to tolk-tester/tests/invalid-typing/err-6204.tolk diff --git a/tolk-tester/tests/invalid-typing-24.tolk b/tolk-tester/tests/invalid-typing/err-6220.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-24.tolk rename to tolk-tester/tests/invalid-typing/err-6220.tolk diff --git a/tolk-tester/tests/invalid-typing-40.tolk b/tolk-tester/tests/invalid-typing/err-6249.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-40.tolk rename to tolk-tester/tests/invalid-typing/err-6249.tolk diff --git a/tolk-tester/tests/invalid-self-5.tolk b/tolk-tester/tests/invalid-typing/err-6257.tolk similarity index 100% rename from tolk-tester/tests/invalid-self-5.tolk rename to tolk-tester/tests/invalid-typing/err-6257.tolk diff --git a/tolk-tester/tests/invalid-self-1.tolk b/tolk-tester/tests/invalid-typing/err-6288.tolk similarity index 100% rename from tolk-tester/tests/invalid-self-1.tolk rename to tolk-tester/tests/invalid-typing/err-6288.tolk diff --git a/tolk-tester/tests/invalid-typing-16.tolk b/tolk-tester/tests/invalid-typing/err-6337.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-16.tolk rename to tolk-tester/tests/invalid-typing/err-6337.tolk diff --git a/tolk-tester/tests/invalid-typing-45.tolk b/tolk-tester/tests/invalid-typing/err-6368.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-45.tolk rename to tolk-tester/tests/invalid-typing/err-6368.tolk diff --git a/tolk-tester/tests/invalid-generics-3.tolk b/tolk-tester/tests/invalid-typing/err-6369.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-3.tolk rename to tolk-tester/tests/invalid-typing/err-6369.tolk diff --git a/tolk-tester/tests/invalid-typing-23.tolk b/tolk-tester/tests/invalid-typing/err-6370.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-23.tolk rename to tolk-tester/tests/invalid-typing/err-6370.tolk diff --git a/tolk-tester/tests/invalid-generics-2.tolk b/tolk-tester/tests/invalid-typing/err-6386.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-2.tolk rename to tolk-tester/tests/invalid-typing/err-6386.tolk diff --git a/tolk-tester/tests/invalid-mutate-20.tolk b/tolk-tester/tests/invalid-typing/err-6391.tolk similarity index 100% rename from tolk-tester/tests/invalid-mutate-20.tolk rename to tolk-tester/tests/invalid-typing/err-6391.tolk diff --git a/tolk-tester/tests/invalid-typing-9.tolk b/tolk-tester/tests/invalid-typing/err-6393.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-9.tolk rename to tolk-tester/tests/invalid-typing/err-6393.tolk diff --git a/tolk-tester/tests/invalid-typing-34.tolk b/tolk-tester/tests/invalid-typing/err-6403.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-34.tolk rename to tolk-tester/tests/invalid-typing/err-6403.tolk diff --git a/tolk-tester/tests/invalid-never-1.tolk b/tolk-tester/tests/invalid-typing/err-6407.tolk similarity index 100% rename from tolk-tester/tests/invalid-never-1.tolk rename to tolk-tester/tests/invalid-typing/err-6407.tolk diff --git a/tolk-tester/tests/invalid-typing-2.tolk b/tolk-tester/tests/invalid-typing/err-6450.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-2.tolk rename to tolk-tester/tests/invalid-typing/err-6450.tolk diff --git a/tolk-tester/tests/invalid-typing-29.tolk b/tolk-tester/tests/invalid-typing/err-6496.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-29.tolk rename to tolk-tester/tests/invalid-typing/err-6496.tolk diff --git a/tolk-tester/tests/invalid-typing-22.tolk b/tolk-tester/tests/invalid-typing/err-6499.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-22.tolk rename to tolk-tester/tests/invalid-typing/err-6499.tolk diff --git a/tolk-tester/tests/invalid-generics-13.tolk b/tolk-tester/tests/invalid-typing/err-6509.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-13.tolk rename to tolk-tester/tests/invalid-typing/err-6509.tolk diff --git a/tolk-tester/tests/invalid-typing-5.tolk b/tolk-tester/tests/invalid-typing/err-6519.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-5.tolk rename to tolk-tester/tests/invalid-typing/err-6519.tolk diff --git a/tolk-tester/tests/invalid-self-2.tolk b/tolk-tester/tests/invalid-typing/err-6533.tolk similarity index 100% rename from tolk-tester/tests/invalid-self-2.tolk rename to tolk-tester/tests/invalid-typing/err-6533.tolk diff --git a/tolk-tester/tests/invalid-typing-26.tolk b/tolk-tester/tests/invalid-typing/err-6553.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-26.tolk rename to tolk-tester/tests/invalid-typing/err-6553.tolk diff --git a/tolk-tester/tests/invalid-typing-21.tolk b/tolk-tester/tests/invalid-typing/err-6580.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-21.tolk rename to tolk-tester/tests/invalid-typing/err-6580.tolk diff --git a/tolk-tester/tests/invalid-typing-8.tolk b/tolk-tester/tests/invalid-typing/err-6600.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-8.tolk rename to tolk-tester/tests/invalid-typing/err-6600.tolk diff --git a/tolk-tester/tests/invalid-typing-31.tolk b/tolk-tester/tests/invalid-typing/err-6613.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-31.tolk rename to tolk-tester/tests/invalid-typing/err-6613.tolk diff --git a/tolk-tester/tests/invalid-typing-35.tolk b/tolk-tester/tests/invalid-typing/err-6619.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-35.tolk rename to tolk-tester/tests/invalid-typing/err-6619.tolk diff --git a/tolk-tester/tests/invalid-generics-8.tolk b/tolk-tester/tests/invalid-typing/err-6629.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-8.tolk rename to tolk-tester/tests/invalid-typing/err-6629.tolk diff --git a/tolk-tester/tests/invalid-typing-14.tolk b/tolk-tester/tests/invalid-typing/err-6637.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-14.tolk rename to tolk-tester/tests/invalid-typing/err-6637.tolk diff --git a/tolk-tester/tests/invalid-typing-11.tolk b/tolk-tester/tests/invalid-typing/err-6711.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-11.tolk rename to tolk-tester/tests/invalid-typing/err-6711.tolk diff --git a/tolk-tester/tests/invalid-typing-27.tolk b/tolk-tester/tests/invalid-typing/err-6713.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-27.tolk rename to tolk-tester/tests/invalid-typing/err-6713.tolk diff --git a/tolk-tester/tests/invalid-assign-2.tolk b/tolk-tester/tests/invalid-typing/err-6731.tolk similarity index 100% rename from tolk-tester/tests/invalid-assign-2.tolk rename to tolk-tester/tests/invalid-typing/err-6731.tolk diff --git a/tolk-tester/tests/invalid-typing-12.tolk b/tolk-tester/tests/invalid-typing/err-6734.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-12.tolk rename to tolk-tester/tests/invalid-typing/err-6734.tolk diff --git a/tolk-tester/tests/invalid-typing-33.tolk b/tolk-tester/tests/invalid-typing/err-6737.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-33.tolk rename to tolk-tester/tests/invalid-typing/err-6737.tolk diff --git a/tolk-tester/tests/invalid-typing-6.tolk b/tolk-tester/tests/invalid-typing/err-6743.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-6.tolk rename to tolk-tester/tests/invalid-typing/err-6743.tolk diff --git a/tolk-tester/tests/invalid-typing-41.tolk b/tolk-tester/tests/invalid-typing/err-6803.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-41.tolk rename to tolk-tester/tests/invalid-typing/err-6803.tolk diff --git a/tolk-tester/tests/invalid-typing-36.tolk b/tolk-tester/tests/invalid-typing/err-6810.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-36.tolk rename to tolk-tester/tests/invalid-typing/err-6810.tolk diff --git a/tolk-tester/tests/invalid-typing-39.tolk b/tolk-tester/tests/invalid-typing/err-6820.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-39.tolk rename to tolk-tester/tests/invalid-typing/err-6820.tolk diff --git a/tolk-tester/tests/invalid-typing-3.tolk b/tolk-tester/tests/invalid-typing/err-6839.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-3.tolk rename to tolk-tester/tests/invalid-typing/err-6839.tolk diff --git a/tolk-tester/tests/invalid-typing-15.tolk b/tolk-tester/tests/invalid-typing/err-6840.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-15.tolk rename to tolk-tester/tests/invalid-typing/err-6840.tolk diff --git a/tolk-tester/tests/invalid-typing-28.tolk b/tolk-tester/tests/invalid-typing/err-6841.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-28.tolk rename to tolk-tester/tests/invalid-typing/err-6841.tolk diff --git a/tolk-tester/tests/invalid-self-4.tolk b/tolk-tester/tests/invalid-typing/err-6844.tolk similarity index 100% rename from tolk-tester/tests/invalid-self-4.tolk rename to tolk-tester/tests/invalid-typing/err-6844.tolk diff --git a/tolk-tester/tests/invalid-typing-17.tolk b/tolk-tester/tests/invalid-typing/err-6881.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-17.tolk rename to tolk-tester/tests/invalid-typing/err-6881.tolk diff --git a/tolk-tester/tests/invalid-typing-7.tolk b/tolk-tester/tests/invalid-typing/err-6901.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-7.tolk rename to tolk-tester/tests/invalid-typing/err-6901.tolk diff --git a/tolk-tester/tests/invalid-typing-19.tolk b/tolk-tester/tests/invalid-typing/err-6903.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-19.tolk rename to tolk-tester/tests/invalid-typing/err-6903.tolk diff --git a/tolk-tester/tests/invalid-generics-7.tolk b/tolk-tester/tests/invalid-typing/err-6921.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-7.tolk rename to tolk-tester/tests/invalid-typing/err-6921.tolk diff --git a/tolk-tester/tests/invalid-declaration-12.tolk b/tolk-tester/tests/invalid-typing/err-6925.tolk similarity index 100% rename from tolk-tester/tests/invalid-declaration-12.tolk rename to tolk-tester/tests/invalid-typing/err-6925.tolk diff --git a/tolk-tester/tests/invalid-typing-42.tolk b/tolk-tester/tests/invalid-typing/err-6937.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-42.tolk rename to tolk-tester/tests/invalid-typing/err-6937.tolk diff --git a/tolk-tester/tests/invalid-generics-12.tolk b/tolk-tester/tests/invalid-typing/err-6981.tolk similarity index 100% rename from tolk-tester/tests/invalid-generics-12.tolk rename to tolk-tester/tests/invalid-typing/err-6981.tolk diff --git a/tolk-tester/tests/invalid-typing-30.tolk b/tolk-tester/tests/invalid-typing/err-6992.tolk similarity index 100% rename from tolk-tester/tests/invalid-typing-30.tolk rename to tolk-tester/tests/invalid-typing/err-6992.tolk diff --git a/tolk-tester/tests/method_id.tolk b/tolk-tester/tests/method_id.tolk deleted file mode 100644 index e7e70d245..000000000 --- a/tolk-tester/tests/method_id.tolk +++ /dev/null @@ -1,17 +0,0 @@ -@method_id(1) -fun foo1(): int { return 111; } -@method_id(3) -fun foo2(): int { return 222; } -@method_id(10) -fun foo3(): int { return 333; } -@method_id(11) -fun slice(slice: slice): slice { return slice; } -fun main(): int { return 999; } - -/** - method_id | in | out -@testcase | 1 | | 111 -@testcase | 3 | | 222 -@testcase | 10 | | 333 -@testcase | 0 | | 999 -*/ diff --git a/tolk-tester/tests/some-tests-1.tolk b/tolk-tester/tests/some-tests-1.tolk new file mode 100644 index 000000000..42bdc773a --- /dev/null +++ b/tolk-tester/tests/some-tests-1.tolk @@ -0,0 +1,127 @@ +fun main(a: int, b: int, c: int, d: int, e: int, f: int): (int, int) { + var D: int = a * d - b * c; + var Dx: int = e * d - b * f; + var Dy: int = a * f - e * c; + return (Dx / D, Dy / D); +} + +@method_id(101) +fun testDivMod(x: int, y: int) { + return (divMod(x, y), modDiv(x, y), mulDivMod(x, y, 10)); +} + +@deprecated +fun twice(f: (int) -> int, x: int) { + return f (f (x)); +} + +fun sqr(x: int) { + return x * x; +} + +@method_id(102) +fun testCallVar(x: int): int { + var f = sqr; + return twice(f, x) * f(x); +} + +@method_id(103) +fun testCallDirect(x: int): int { + return twice(sqr, x) * sqr(x); +} + +@method_id(104) +fun testLoop(x: int): int { + var n = 0; + while (x > 1) { + n += 1; + if (x & 1) { + x = 3 * x + 1; + } else { + x >>= 1; + } + } + return n; +} + +@method_id(105) +fun test5(id: int): (int, int) { + if (id > 0) { + if (id > 10) { + return (2 * id, 3 * id); + } + } + return (5, 6); +} + +@method_id(106) +fun test6(x: int): int { + var i: int = 0; + do { + i = i + 1; + if (i > 5) { + return 1; + } + var f: bool = (i * i == 64); + } while (!f); + return -1; +} + +@method_id(107) +fun test7(y: int): int { + var x: int = 1; + if (y > 0) { + return 1; + } + return x > 0 ? -1 : 0; +} + +@method_id(108) +fun test8(y: int): int { + if (y > 0) { + return 1; + } + return 2; +} + +@method_id(109) +fun test9(s: int) { + var (z, t) = (17, s); + while (z > 0) { + t = s; + z -= 1; + } + return ~ t; +} + + +/** + method_id | in | out +@testcase | 0 | 1 1 1 -1 10 6 | 8 2 +@testcase | 0 | 817 -31 624 -241 132272 272276 | 132 -788 +@testcase | 0 | -886 562 498 -212 -36452 -68958 | -505 -861 +@testcase | 0 | 448 -433 -444 792 150012 -356232 | -218 -572 +@testcase | 0 | -40 -821 433 -734 -721629 -741724 | -206 889 +@testcase | 0 | -261 -98 -494 868 -166153 733738 | 263 995 +@testcase | 101 | 112 3 | 37 1 1 37 33 6 +@testcase | 102 | 3 | 729 +@testcase | 102 | 10 | 1000000 +@testcase | 103 | 3 | 729 +@testcase | 103 | 10 | 1000000 +@testcase | 104 | 1 | 0 +@testcase | 104 | 2 | 1 +@testcase | 104 | 5 | 5 +@testcase | 104 | 19 | 20 +@testcase | 104 | 27 | 111 +@testcase | 104 | 100 | 25 +@testcase | 105 | 0 | 5 6 +@testcase | 105 | 4 | 5 6 +@testcase | 105 | 11 | 22 33 +@testcase | 106 | 0 | 1 +@testcase | 107 | 10 | 1 +@testcase | 107 | -5 | -1 +@testcase | 108 | 10 | 1 +@testcase | 108 | -5 | 2 +@testcase | 109 | 1 | -2 +@testcase | 109 | 5 | -6 +*/ diff --git a/tolk-tester/tests/a10.tolk b/tolk-tester/tests/some-tests-2.tolk similarity index 100% rename from tolk-tester/tests/a10.tolk rename to tolk-tester/tests/some-tests-2.tolk diff --git a/tolk-tester/tests/w2.tolk b/tolk-tester/tests/some-tests-3.tolk similarity index 100% rename from tolk-tester/tests/w2.tolk rename to tolk-tester/tests/some-tests-3.tolk diff --git a/tolk-tester/tests/s1.tolk b/tolk-tester/tests/strings-tests.tolk similarity index 100% rename from tolk-tester/tests/s1.tolk rename to tolk-tester/tests/strings-tests.tolk diff --git a/tolk-tester/tests/try-func.tolk b/tolk-tester/tests/try-catch-tests.tolk similarity index 100% rename from tolk-tester/tests/try-func.tolk rename to tolk-tester/tests/try-catch-tests.tolk diff --git a/tolk-tester/tests/unbalanced_ret_loops.tolk b/tolk-tester/tests/unbalanced-ret-loops.tolk similarity index 100% rename from tolk-tester/tests/unbalanced_ret_loops.tolk rename to tolk-tester/tests/unbalanced-ret-loops.tolk diff --git a/tolk-tester/tests/unbalanced-ret.tolk b/tolk-tester/tests/unbalanced-ret.tolk new file mode 100644 index 000000000..eca5dd2a5 --- /dev/null +++ b/tolk-tester/tests/unbalanced-ret.tolk @@ -0,0 +1,73 @@ +fun main(x: int): (int, int) { + var y: int = 5; + if (x < 0) { + x *= 2; + y += 1; + if (x == -10) { + return (111, 0); + } + } + return (x + 1, y); +} + +@inline +fun foo1(x: int): int { + if (x < 0) { + x *= 2; + if (x == -10) { + return 111; + } + } + return x + 1; +} + +@method_id(101) +fun test1(x: int): int { + return foo1(x) * 10; +} + +fun foo2(y: int): int { + if (y < 0) { + y *= 2; + if (y == -10) { + return 111; + } + } + return y + 1; +} + +fun bar2(x: int, y: int): (int, int) { + if (x < 0) { + y = foo2(y); + x *= 2; + if (x == -10) { + return (111, y); + } + } + return (x + 1, y); +} + +@method_id(102) +fun test2(x: int, y: int): (int, int) { + (x, y) = bar2(x, y); + return (x, y * 10); +} + +/** + method_id | in | out +@testcase | 0 | 10 | 11 5 +@testcase | 0 | -5 | 111 0 +@testcase | 0 | -4 | -7 6 +@testcase | 101 | 10 | 110 +@testcase | 101 | -5 | 1110 +@testcase | 101 | -4 | -70 +@testcase | 102 | 3 3 | 4 30 +@testcase | 102 | 3 -5 | 4 -50 +@testcase | 102 | 3 -4 | 4 -40 +@testcase | 102 | -5 3 | 111 40 +@testcase | 102 | -5 -5 | 111 1110 +@testcase | 102 | -5 -4 | 111 -70 +@testcase | 102 | -4 3 | -7 40 +@testcase | 102 | -4 -5 | -7 1110 +@testcase | 102 | -4 -4 | -7 -70 +*/ diff --git a/tolk-tester/tests/unbalanced_ret.tolk b/tolk-tester/tests/unbalanced_ret.tolk deleted file mode 100644 index 6cf42643a..000000000 --- a/tolk-tester/tests/unbalanced_ret.tolk +++ /dev/null @@ -1,17 +0,0 @@ -fun main(x: int): (int, int) { - var y: int = 5; - if (x < 0) { - x *= 2; - y += 1; - if (x == -10) { - return (111, 0); - } - } - return (x + 1, y); -} -/** - method_id | in | out -@testcase | 0 | 10 | 11 5 -@testcase | 0 | -5 | 111 0 -@testcase | 0 | -4 | -7 6 -*/ diff --git a/tolk-tester/tests/unbalanced_ret_inline.tolk b/tolk-tester/tests/unbalanced_ret_inline.tolk deleted file mode 100644 index 4e24fbd8f..000000000 --- a/tolk-tester/tests/unbalanced_ret_inline.tolk +++ /dev/null @@ -1,19 +0,0 @@ -@inline -fun foo(x: int): int { - if (x < 0) { - x *= 2; - if (x == -10) { - return 111; - } - } - return x + 1; -} -fun main(x: int): int { - return foo(x) * 10; -} -/** - method_id | in | out -@testcase | 0 | 10 | 110 -@testcase | 0 | -5 | 1110 -@testcase | 0 | -4 | -70 -*/ diff --git a/tolk-tester/tests/unbalanced_ret_nested.tolk b/tolk-tester/tests/unbalanced_ret_nested.tolk deleted file mode 100644 index 05e609240..000000000 --- a/tolk-tester/tests/unbalanced_ret_nested.tolk +++ /dev/null @@ -1,37 +0,0 @@ -fun foo(y: int): int { - if (y < 0) { - y *= 2; - if (y == -10) { - return 111; - } - } - return y + 1; -} -fun bar(x: int, y: int): (int, int) { - if (x < 0) { - y = foo(y); - x *= 2; - if (x == -10) { - return (111, y); - } - } - return (x + 1, y); -} -fun main(x: int, y: int): (int, int) { - (x, y) = bar(x, y); - return (x, y * 10); -} -/** - method_id | in | out -@testcase | 0 | 3 3 | 4 30 -@testcase | 0 | 3 -5 | 4 -50 -@testcase | 0 | 3 -4 | 4 -40 -@testcase | 0 | -5 3 | 111 40 -@testcase | 0 | -5 -5 | 111 1110 -@testcase | 0 | -5 -4 | 111 -70 -@testcase | 0 | -4 3 | -7 40 -@testcase | 0 | -4 -5 | -7 1110 -@testcase | 0 | -4 -4 | -7 -70 - -@code_hash 68625253347714662162648433047986779710161195298061582217368558479961252943991 -*/ diff --git a/tolk-tester/tests/use-before-declare.tolk b/tolk-tester/tests/use-before-declare.tolk index 2a0e0e7ff..9c5f06349 100644 --- a/tolk-tester/tests/use-before-declare.tolk +++ b/tolk-tester/tests/use-before-declare.tolk @@ -33,9 +33,16 @@ fun test1(): int { return demo_var + demo_slice; } +fun test2() { + return second; +} + global demo_slice: slice; const demo_20: int = 20; +const second = first + 1; +const first = 1; + /** @testcase | 0 | | 34 @@ -46,4 +53,12 @@ const demo_20: int = 20; 30 PUSHINT // '10 }> """ + +@fif_codegen +""" + test2 PROC:<{ + // + 2 PUSHINT // '0=2 + }> +""" */ diff --git a/tolk-tester/tests/var-apply.tolk b/tolk-tester/tests/var-apply-tests.tolk similarity index 85% rename from tolk-tester/tests/var-apply.tolk rename to tolk-tester/tests/var-apply-tests.tolk index d189430fb..96b4d134d 100644 --- a/tolk-tester/tests/var-apply.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -175,6 +175,28 @@ fun testVarsModificationInsideVarCall(x: int) { return x > 3 ? cb(x, x += 5, eq(x *= x), x, eq(x)) : null; } +fun check_assoc_1(op: (int, int) -> int, a: int, b: int, c: int) { + return op(op(a, b), c) == op(a, op(b, c)); +} + +@method_id(111) +fun testApplyNativePlus(x: int, y: int, z: int): bool { + return check_assoc_1(`_+_`, x, y, z); +} + +global op: (int, int) -> int; + +fun check_assoc_2(a: int, b: int, c: int): bool { + return op(op(a, b), c) == op(a, op(b, c)); +} + +@method_id(112) +fun testApplyGlobalVar(x: int, y: int, z: int): bool? { + op = `_+_`; + if (0) { return null; } + return check_assoc_2(x, y, z); +} + fun main() {} /** @@ -189,4 +211,10 @@ fun main() {} @testcase | 109 | | 0 3 0 7 @testcase | 110 | 5 | 5 10 100 100 100 -1 @testcase | 110 | 0 | (null) (null) (null) (null) (null) 0 - */ +@testcase | 111 | 2 3 9 | -1 +@testcase | 111 | 11 22 44 | -1 +@testcase | 111 | -1 -10 -20 | -1 +@testcase | 112 | 2 3 9 | -1 +@testcase | 112 | 11 22 44 | -1 +@testcase | 112 | -1 -10 -20 | -1 +*/ diff --git a/tolk-tester/tests/w1.tolk b/tolk-tester/tests/w1.tolk deleted file mode 100644 index eb06bec67..000000000 --- a/tolk-tester/tests/w1.tolk +++ /dev/null @@ -1,14 +0,0 @@ -fun main(id: int): (int, int) { - if (id > 0) { - if (id > 10) { - return (2 * id, 3 * id); - } - } - return (5, 6); -} -/** - method_id | in | out -@testcase | 0 | 0 | 5 6 -@testcase | 0 | 4 | 5 6 -@testcase | 0 | 11 | 22 33 -*/ diff --git a/tolk-tester/tests/w6.tolk b/tolk-tester/tests/w6.tolk deleted file mode 100644 index 489ffa8cc..000000000 --- a/tolk-tester/tests/w6.tolk +++ /dev/null @@ -1,19 +0,0 @@ -fun main(x: int): int { - var i: int = 0; - // int f = false; - do { - i = i + 1; - if (i > 5) { - return 1; - } - var f: bool = (i * i == 64); - } while (!f); - return -1; -} - -/** - method_id | in | out -@testcase | 0 | 0 | 1 - -@code_hash 36599880583276393028571473830850694081778552118303309411432666239740650614479 -*/ diff --git a/tolk-tester/tests/w7.tolk b/tolk-tester/tests/w7.tolk deleted file mode 100644 index 3d68c775b..000000000 --- a/tolk-tester/tests/w7.tolk +++ /dev/null @@ -1,26 +0,0 @@ -@method_id(1) -fun test(y: int): int { - var x: int = 1; - if (y > 0) { - return 1; - } - return x > 0 ? -1 : 0; -} - -@method_id(2) -fun f(y: int): int { - if (y > 0) { - return 1; - } - return 2; -} - -fun main() { } - -/** - method_id | in | out -@testcase | 1 | 10 | 1 -@testcase | 1 | -5 | -1 -@testcase | 2 | 10 | 1 -@testcase | 2 | -5 | 2 -*/ diff --git a/tolk-tester/tests/w9.tolk b/tolk-tester/tests/w9.tolk deleted file mode 100644 index b88dc736e..000000000 --- a/tolk-tester/tests/w9.tolk +++ /dev/null @@ -1,14 +0,0 @@ -fun main(s: int) { - var (z, t) = (17, s); - while (z > 0) { - t = s; - z -= 1; - } - return ~ t; -} - -/** - method_id | in | out -@testcase | 0 | 1 | -2 -@testcase | 0 | 5 | -6 -*/ diff --git a/tolk-tester/tests/unreachable-1.tolk b/tolk-tester/tests/warnings-not-errors/unreachable-1.tolk similarity index 100% rename from tolk-tester/tests/unreachable-1.tolk rename to tolk-tester/tests/warnings-not-errors/unreachable-1.tolk diff --git a/tolk-tester/tests/unreachable-2.tolk b/tolk-tester/tests/warnings-not-errors/unreachable-2.tolk similarity index 100% rename from tolk-tester/tests/unreachable-2.tolk rename to tolk-tester/tests/warnings-not-errors/unreachable-2.tolk diff --git a/tolk-tester/tests/unreachable-3.tolk b/tolk-tester/tests/warnings-not-errors/unreachable-3.tolk similarity index 100% rename from tolk-tester/tests/unreachable-3.tolk rename to tolk-tester/tests/warnings-not-errors/unreachable-3.tolk diff --git a/tolk-tester/tests/unreachable-4.tolk b/tolk-tester/tests/warnings-not-errors/unreachable-4.tolk similarity index 100% rename from tolk-tester/tests/unreachable-4.tolk rename to tolk-tester/tests/warnings-not-errors/unreachable-4.tolk diff --git a/tolk-tester/tests/warnings-1.tolk b/tolk-tester/tests/warnings-not-errors/warnings-1.tolk similarity index 100% rename from tolk-tester/tests/warnings-1.tolk rename to tolk-tester/tests/warnings-not-errors/warnings-1.tolk diff --git a/tolk-tester/tests/warnings-2.tolk b/tolk-tester/tests/warnings-not-errors/warnings-2.tolk similarity index 100% rename from tolk-tester/tests/warnings-2.tolk rename to tolk-tester/tests/warnings-not-errors/warnings-2.tolk diff --git a/tolk-tester/tolk-tester.js b/tolk-tester/tolk-tester.js index c7e710214..0aaf0738f 100644 --- a/tolk-tester/tolk-tester.js +++ b/tolk-tester/tolk-tester.js @@ -1,4 +1,4 @@ -// Usage: `node tolk-tester.js tests_dir` OR `node tolk-tester.js test_file.tolk` +// Usage: `node tolk-tester.js tests_dir` OR `node tolk-tester.js tests_dir file_pattern` // from current dir, providing some env (see getenv() calls). // This is a JS version of tolk-tester.py to test Tolk compiled to WASM. // Don't forget to keep it identical to Python version! @@ -32,32 +32,38 @@ const TMP_DIR = os.tmpdir() class CmdLineOptions { constructor(/**string[]*/ argv) { - if (argv.length !== 3) { - print("Usage: node tolk-tester.js tests_dir OR node tolk-tester.js test_file.tolk") + if (argv.length < 3) { + print("Usage: node tolk-tester.js tests_dir [file_pattern]") process.exit(1) } - if (!fs.existsSync(argv[2])) { - print(`Input '${argv[2]}' doesn't exist`) + if (!fs.existsSync(argv[2]) || !fs.lstatSync(argv[2]).isDirectory()) { + print(`Directory '${argv[2]}' doesn't exist`) process.exit(1) } - if (fs.lstatSync(argv[2]).isDirectory()) { - this.tests_dir = argv[2] - this.test_file = null - } else { - this.tests_dir = path.dirname(argv[2]) - this.test_file = argv[2] - } + this.tests_dir = argv[2] + this.file_pattern = argv[3] } /** @return {string[]} */ find_tests() { - if (this.test_file) // an option to run (debug) a single test - return [this.test_file] + let all_test_files = [] + let all_children_of_tests_dir = fs.readdirSync(this.tests_dir) + all_children_of_tests_dir.sort() + for (let f of all_children_of_tests_dir) + if (f.endsWith(".tolk")) + all_test_files.push(path.join(this.tests_dir, f)) + for (let f of all_children_of_tests_dir) + if (!f.endsWith(".tolk") && f !== "imports") { + let subdir = path.join(this.tests_dir, f) + let children_of_subdir = fs.readdirSync(subdir) + children_of_subdir.sort() + all_test_files.push(...children_of_subdir.map(f => path.join(subdir, f))) + } - let tests = fs.readdirSync(this.tests_dir).filter(f => f.endsWith('.tolk') || f.endsWith('.ton')) - tests.sort() - return tests.map(f => path.join(this.tests_dir, f)) + if (this.file_pattern) + all_test_files = all_test_files.filter(f => f.includes(this.file_pattern)) + return all_test_files } } @@ -126,7 +132,7 @@ class TolkTestCaseInputOutput { check(/**string[]*/ stdout_lines, /**number*/ line_idx) { if (stdout_lines[line_idx] !== this.expected_output) - throw new CompareOutputError(`error on case #${line_idx + 1} (${this.method_id} | ${this.input}): expected '${this.expected_output}', found '${stdout_lines[line_idx]}'`, stdout_lines.join("\n")) + throw new CompareOutputError(`error on case #${line_idx + 1} (${this.method_id} | ${this.input}):\n expect: ${this.expected_output}\n actual: ${stdout_lines[line_idx]}`, stdout_lines.join("\n")) } } @@ -400,7 +406,7 @@ class TolkTestFile { async function run_all_tests(/**string[]*/ tests) { for (let ti = 0; ti < tests.length; ++ti) { let tolk_filename = tests[ti] - print(`Running test ${ti + 1}/${tests.length}: ${tolk_filename}`) + print(`Running test ${ti + 1}/${tests.length}: ${path.basename(tolk_filename)}`) let artifacts_folder = path.join(TMP_DIR, tolk_filename) let testcase = new TolkTestFile(tolk_filename, artifacts_folder) @@ -413,7 +419,7 @@ async function run_all_tests(/**string[]*/ tests) { fs.rmSync(artifacts_folder, {recursive: true}) if (testcase.compilation_should_fail) - print(" OK, compilation failed as it should") + print(" OK, stderr match") else print(` OK, ${testcase.input_output.length} cases`) } catch (e) { diff --git a/tolk-tester/tolk-tester.py b/tolk-tester/tolk-tester.py index 0b3c774ca..3b4fb5b7f 100644 --- a/tolk-tester/tolk-tester.py +++ b/tolk-tester/tolk-tester.py @@ -1,4 +1,4 @@ -# Usage: `tolk-tester.py tests_dir` OR `tolk-tester.py test_file.tolk` +# Usage: `tolk-tester.py tests_dir` OR `tolk-tester.py tests_dir file_pattern` # from current dir, providing some env (see getenv() calls). # Every .tolk file should provide /* testcase description in a comment */, consider tests/ folder. # @@ -36,27 +36,33 @@ def getenv(name, default=None): class CmdLineOptions: def __init__(self, argv: List[str]): - if len(argv) != 2: - print("Usage: tolk-tester.py tests_dir OR tolk-tester.py test_file.tolk", file=sys.stderr) + if len(argv) < 2: + print("Usage: tolk-tester.py tests_dir [file_pattern]", file=sys.stderr) exit(1) - if not os.path.exists(argv[1]): - print("Input '%s' doesn't exist" % argv[1], file=sys.stderr) + if not os.path.isdir(argv[1]): + print("Directory '%s' doesn't exist" % argv[1], file=sys.stderr) exit(1) - if os.path.isdir(argv[1]): - self.tests_dir = argv[1] - self.test_file = None - else: - self.tests_dir = os.path.dirname(argv[1]) - self.test_file = argv[1] + self.tests_dir = argv[1] + self.file_pattern = argv[2] if len(argv) > 2 else None def find_tests(self) -> List[str]: - if self.test_file is not None: # an option to run (debug) a single test - return [self.test_file] - - tests = [f for f in os.listdir(self.tests_dir) if f.endswith(".tolk") or f.endswith(".ton")] - tests.sort() - return [os.path.join(self.tests_dir, f) for f in tests] + all_test_files: List[str] = [] + all_children_of_tests_dir = os.listdir(self.tests_dir) + all_children_of_tests_dir.sort() + for f in all_children_of_tests_dir: + if f.endswith(".tolk"): + all_test_files.append(os.path.join(self.tests_dir, f)) + for f in all_children_of_tests_dir: + if not f.endswith(".tolk") and f != "imports": + subdir = os.path.join(self.tests_dir, f) + children_of_subdir = os.listdir(subdir) + children_of_subdir.sort() + all_test_files += [os.path.join(subdir, f) for f in children_of_subdir] + + if self.file_pattern is not None: + all_test_files = [f for f in all_test_files if f.find(self.file_pattern) != -1] + return all_test_files class ParseInputError(Exception): @@ -101,6 +107,7 @@ class TolkTestCaseInputOutput: """ reJustNumber = re.compile(r"[-+]?\d+") reMathExpr = re.compile(r"[0x123456789()+\-*/<>]+") + reGasUsed = re.compile(r"gas:\sused=(\d+)") def __init__(self, method_id_str: str, input_str: str, output_str: str): processed_inputs = [] @@ -122,7 +129,7 @@ def __init__(self, method_id_str: str, input_str: str, output_str: str): def check(self, stdout_lines: List[str], line_idx: int): if stdout_lines[line_idx] != self.expected_output: - raise CompareOutputError("error on case #%d (%d | %s): expected '%s', found '%s'" % (line_idx + 1, self.method_id, self.input, self.expected_output, stdout_lines[line_idx]), "\n".join(stdout_lines)) + raise CompareOutputError("error on case #%d (%d | %s):\n expect: %s\n actual: %s" % (line_idx + 1, self.method_id, self.input, self.expected_output, stdout_lines[line_idx]), "\n".join(stdout_lines)) class TolkTestCaseStderr: @@ -331,7 +338,7 @@ def run_and_check(self): should_include.check(stderr) if exit_code != 0 and self.compilation_should_fail: - return + return 0 if exit_code != 0 and not self.compilation_should_fail: raise TolkCompilationFailedError("tolk exit_code = %d" % exit_code, stderr) @@ -350,6 +357,7 @@ def run_and_check(self): if exit_code != 0: raise FiftExecutionFailedError("fift exit_code = %d" % exit_code, stderr) + gas_used = sum(map(int, TolkTestCaseInputOutput.reGasUsed.findall(stderr))) stdout_lines = [x.strip() for x in stdout.split("\n")] stdout_lines = [x for x in stdout_lines if x != ""] fif_code_hash = None @@ -372,11 +380,14 @@ def run_and_check(self): if self.expected_hash is not None: self.expected_hash.check(fif_code_hash) + return gas_used + def run_all_tests(tests: List[str]): + total_gas_used = 0 for ti in range(len(tests)): tolk_filename = tests[ti] - print("Running test %d/%d: %s" % (ti + 1, len(tests), tolk_filename), file=sys.stderr) + print("Running test %d/%d: %s" % (ti + 1, len(tests), os.path.basename(tolk_filename)), file=sys.stderr) artifacts_folder = os.path.join(TMP_DIR, tolk_filename) testcase = TolkTestFile(tolk_filename, artifacts_folder) @@ -384,13 +395,14 @@ def run_all_tests(tests: List[str]): if not os.path.exists(artifacts_folder): os.makedirs(artifacts_folder) testcase.parse_input_from_tolk_file() - testcase.run_and_check() + gas_used = testcase.run_and_check() shutil.rmtree(artifacts_folder) + total_gas_used += gas_used if testcase.compilation_should_fail: - print(" OK, compilation failed as it should", file=sys.stderr) + print(" OK, stderr match", file=sys.stderr) else: - print(" OK, %d cases" % len(testcase.input_output), file=sys.stderr) + print(" OK, %d cases, gas %d" % (len(testcase.input_output), gas_used), file=sys.stderr) except ParseInputError as e: print(" Error parsing input (cur line #%d):" % (testcase.line_idx + 1), e, file=sys.stderr) exit(2) @@ -423,9 +435,10 @@ def run_all_tests(tests: List[str]): print(" Mismatch in code hash:", e, file=sys.stderr) print(" Was compiled to:", testcase.get_compiled_fif_filename(), file=sys.stderr) exit(2) + return total_gas_used tests = CmdLineOptions(sys.argv).find_tests() print("Found", len(tests), "tests", file=sys.stderr) -run_all_tests(tests) -print("Done, %d tests" % len(tests), file=sys.stderr) +total_gas_used = run_all_tests(tests) +print("Done, %d tests, gas %d" % (len(tests), total_gas_used), file=sys.stderr) From 888cb43319f4e54c656b85df6ff14f5b73c8bf68 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 14 Mar 2025 10:34:50 +0300 Subject: [PATCH 160/388] Fix computing storage stat --- crypto/block/transaction.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 4e92b8f6c..33d0f7314 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -3284,7 +3284,7 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { } AccountStorageStat& stats = new_account_storage_stat.value_force(); // Don't check Merkle depth and size here - they were checked in check_state_limits - auto S = stats.replace_roots(new_storage->prefetch_all_refs()).move_as_status(); + auto S = stats.replace_roots(new_storage_for_stat->prefetch_all_refs()).move_as_status(); if (S.is_error()) { LOG(ERROR) << "Cannot recompute storage stats for account " << account.addr.to_hex() << ": " << S.move_as_error(); return false; @@ -3292,7 +3292,7 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { new_storage_dict_hash = stats.get_dict_hash(); // Root of AccountStorage is not counted in AccountStorageStat new_storage_used.cells = stats.get_total_cells() + 1; - new_storage_used.bits = stats.get_total_bits() + new_storage->size(); + new_storage_used.bits = stats.get_total_bits() + new_storage_for_stat->size(); if (timer.elapsed() > 0.1) { LOG(INFO) << "Compute used storage (2) took " << timer.elapsed() << "s"; } From f12b63f3b8d2977c4a062a8d2573a1b6e78a58f8 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 14 Mar 2025 10:36:02 +0300 Subject: [PATCH 161/388] Optimize AccountStorageStat cleanup --- crypto/block/account-storage-stat.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp index f5545f8b2..bdd68b1fd 100644 --- a/crypto/block/account-storage-stat.cpp +++ b/crypto/block/account-storage-stat.cpp @@ -55,6 +55,14 @@ AccountStorageStat& AccountStorageStat::operator=(AccountStorageStat&& other) { td::Result AccountStorageStat::replace_roots(std::vector> new_roots) { std::erase_if(new_roots, [](const Ref& c) { return c.is_null(); }); + if (new_roots.empty()) { + roots_.clear(); + total_bits_ = total_cells_ = 0; + dict_ = vm::Dictionary{256}; + cache_ = {}; + return CellInfo{}; + } + auto cmp = [](const Ref& c1, const Ref& c2) { return c1->get_hash() < c2->get_hash(); }; std::sort(new_roots.begin(), new_roots.end(), cmp); std::sort(roots_.begin(), roots_.end(), cmp); From 9f8c3dabff295644c796dcd580ed4ff0a6c309b0 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 14 Mar 2025 11:59:35 +0300 Subject: [PATCH 162/388] Don't store zero dict hash --- crypto/block/transaction.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 33d0f7314..7dbc5249f 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -3303,7 +3303,7 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { new_storage_dict_hash = account.storage_dict_hash; new_account_storage_stat = account.account_storage_stat; } - if (!cfg.store_storage_dict_hash) { + if (!cfg.store_storage_dict_hash || (new_storage_dict_hash && new_storage_dict_hash.value().is_zero())) { new_storage_dict_hash = {}; } From 21b8f2a1f6e04f095022ff19b45b7ff896c80e56 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 14 Mar 2025 13:30:20 +0300 Subject: [PATCH 163/388] Change collated data serialize mode to 2 --- validator-session/candidate-serializer.cpp | 11 +++++++---- validator-session/candidate-serializer.h | 6 ++++-- validator-session/validator-session.cpp | 3 ++- validator/collation-manager.cpp | 15 +++++++++------ validator/collation-manager.hpp | 8 ++++---- validator/collator-node.cpp | 6 +++--- validator/collator-node.hpp | 2 +- validator/impl/collator.cpp | 5 +++-- validator/validator-group.cpp | 2 +- 9 files changed, 34 insertions(+), 24 deletions(-) diff --git a/validator-session/candidate-serializer.cpp b/validator-session/candidate-serializer.cpp index 4442504ed..2fc7d7a5f 100644 --- a/validator-session/candidate-serializer.cpp +++ b/validator-session/candidate-serializer.cpp @@ -35,7 +35,8 @@ td::Result serialize_candidate(const tl_object_ptr> deserialize_candidate(td::Slice data, bool compression_enabled, - int max_decompressed_data_size) { + int max_decompressed_data_size, + int proto_version) { if (!compression_enabled) { return fetch_tl_object(data, true); } @@ -43,7 +44,7 @@ td::Result> deserialize_candi if (f->decompressed_size_ > max_decompressed_data_size) { return td::Status::Error("decompressed size is too big"); } - TRY_RESULT(p, decompress_candidate_data(f->data_, f->decompressed_size_)); + TRY_RESULT(p, decompress_candidate_data(f->data_, f->decompressed_size_, proto_version)); return create_tl_object(f->src_, f->round_, f->root_hash_, std::move(p.first), std::move(p.second)); } @@ -68,7 +69,8 @@ td::Result compress_candidate_data(td::Slice block, td::Slice c } td::Result> decompress_candidate_data(td::Slice compressed, - int decompressed_size) { + int decompressed_size, + int proto_version) { TRY_RESULT(decompressed, td::lz4_decompress(compressed, decompressed_size)); if (decompressed.size() != (size_t)decompressed_size) { return td::Status::Error("decompressed size mismatch"); @@ -79,7 +81,8 @@ td::Result> decompress_candidate_dat } TRY_RESULT(block_data, vm::std_boc_serialize(roots[0], 31)); roots.erase(roots.begin()); - TRY_RESULT(collated_data, vm::std_boc_serialize_multi(std::move(roots), 31)); + int collated_data_mode = proto_version >= 5 ? 2 : 31; + TRY_RESULT(collated_data, vm::std_boc_serialize_multi(std::move(roots), collated_data_mode)); LOG(DEBUG) << "Decompressing block candidate: " << compressed.size() << " -> " << block_data.size() + collated_data.size(); return std::make_pair(std::move(block_data), std::move(collated_data)); diff --git a/validator-session/candidate-serializer.h b/validator-session/candidate-serializer.h index e88376cd7..7cc77f0be 100644 --- a/validator-session/candidate-serializer.h +++ b/validator-session/candidate-serializer.h @@ -24,11 +24,13 @@ td::Result serialize_candidate(const tl_object_ptr> deserialize_candidate(td::Slice data, bool compression_enabled, - int max_decompressed_data_size); + int max_decompressed_data_size, + int proto_version); td::Result compress_candidate_data(td::Slice block, td::Slice collated_data, size_t& decompressed_size); td::Result> decompress_candidate_data(td::Slice compressed, - int decompressed_size); + int decompressed_size, + int proto_version); } // namespace ton::validatorsession diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index e18dd1eb5..79233d699 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -229,7 +229,8 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice td::Timer deserialize_timer; auto R = deserialize_candidate(data, compress_block_candidates_, - description().opts().max_block_size + description().opts().max_collated_data_size + 1024); + description().opts().max_block_size + description().opts().max_collated_data_size + 1024, + description().opts().proto_version); double deserialize_time = deserialize_timer.elapsed(); if (R.is_error()) { VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice()) diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index 2b06ad482..27af220df 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -35,7 +35,7 @@ void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_mastercha std::vector prev, Ed25519_PublicKey creator, BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, td::CancellationToken cancellation_token, - td::Promise promise) { + td::Promise promise, int proto_version) { if (shard.is_masterchain()) { run_collate_query( shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), @@ -46,14 +46,16 @@ void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_mastercha return; } collate_shard_block(shard, min_masterchain_block_id, std::move(prev), creator, priority, std::move(validator_set), - max_answer_size, std::move(cancellation_token), std::move(promise), td::Timestamp::in(10.0)); + max_answer_size, std::move(cancellation_token), std::move(promise), td::Timestamp::in(10.0), + proto_version); } void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, Ed25519_PublicKey creator, BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, td::CancellationToken cancellation_token, - td::Promise promise, td::Timestamp timeout) { + td::Promise promise, td::Timestamp timeout, + int proto_version) { TRY_STATUS_PROMISE(promise, cancellation_token.check()); ShardInfo* s = select_shard_info(shard); if (s == nullptr) { @@ -144,7 +146,7 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas [=, promise = std::move(promise)]() mutable { td::actor::send_closure(SelfId, &CollationManager::collate_shard_block, shard, min_masterchain_block_id, prev, creator, priority, validator_set, max_answer_size, cancellation_token, - std::move(promise), timeout); + std::move(promise), timeout, proto_version); }, retry_at); }; @@ -170,8 +172,9 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas return; } TRY_RESULT_PROMISE(P, f, fetch_tl_object(data, true)); - TRY_RESULT_PROMISE(P, candidate, - CollatorNode::deserialize_candidate(std::move(f), td::narrow_cast(max_answer_size))); + TRY_RESULT_PROMISE( + P, candidate, + CollatorNode::deserialize_candidate(std::move(f), td::narrow_cast(max_answer_size), proto_version)); if (candidate.pubkey.as_bits256() != creator.as_bits256()) { P.set_error(td::Status::Error("collate query: block candidate source mismatch")); return; diff --git a/validator/collation-manager.hpp b/validator/collation-manager.hpp index 72cc02e5a..048d30812 100644 --- a/validator/collation-manager.hpp +++ b/validator/collation-manager.hpp @@ -35,9 +35,9 @@ class CollationManager : public td::actor::Actor { void alarm() override; void collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, - Ed25519_PublicKey creator, BlockCandidatePriority priority, - td::Ref validator_set, td::uint64 max_answer_size, - td::CancellationToken cancellation_token, td::Promise promise); + Ed25519_PublicKey creator, BlockCandidatePriority priority, td::Ref validator_set, + td::uint64 max_answer_size, td::CancellationToken cancellation_token, + td::Promise promise, int proto_version); void update_options(td::Ref opts); @@ -56,7 +56,7 @@ class CollationManager : public td::actor::Actor { Ed25519_PublicKey creator, BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, td::CancellationToken cancellation_token, td::Promise promise, - td::Timestamp timeout); + td::Timestamp timeout, int proto_version); void update_collators_list(const CollatorsList& collators_list); diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index ff84456bd..238eb8921 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -640,7 +640,7 @@ tl_object_ptr CollatorNode::serialize_candidate } td::Result CollatorNode::deserialize_candidate(tl_object_ptr f, - int max_decompressed_data_size) { + int max_decompressed_data_size, int proto_version) { td::Result res; ton_api::downcast_call(*f, td::overloaded( [&](ton_api::collatorNode_candidate& c) { @@ -663,8 +663,8 @@ td::Result CollatorNode::deserialize_candidate(tl_object_ptr max_decompressed_data_size) { return td::Status::Error("decompressed size is too big"); } - TRY_RESULT( - p, validatorsession::decompress_candidate_data(c.data_, c.decompressed_size_)); + TRY_RESULT(p, validatorsession::decompress_candidate_data( + c.data_, c.decompressed_size_, proto_version)); auto collated_data_hash = td::sha256_bits256(p.second); auto key = ton::PublicKey{c.source_}; if (!key.is_ed25519()) { diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp index 5aab24635..604338f0b 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node.hpp @@ -103,7 +103,7 @@ class CollatorNode : public td::actor::Actor { public: static tl_object_ptr serialize_candidate(const BlockCandidate& block, bool compress); static td::Result deserialize_candidate(tl_object_ptr f, - int max_decompressed_data_size); + int max_decompressed_data_size, int proto_version); }; } // namespace ton::validator diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index fee6f21d1..362af889c 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -5901,6 +5901,7 @@ bool Collator::create_collated_data() { * @returns True if the block candidate was created successfully, false otherwise. */ bool Collator::create_block_candidate() { + auto consensus_config = config_->get_consensus_config(); // 1. serialize block LOG(INFO) << "serializing new Block"; vm::BagOfCells boc; @@ -5926,7 +5927,8 @@ bool Collator::create_block_candidate() { if (res.is_error()) { return fatal_error(res.move_as_error()); } - auto cdata_res = boc_collated.serialize_to_slice(31); + int cdata_serialize_mode = consensus_config.proto_version >= 5 ? 2 : 31; + auto cdata_res = boc_collated.serialize_to_slice(cdata_serialize_mode); if (cdata_res.is_error()) { LOG(ERROR) << "cannot serialize collated data"; return fatal_error(cdata_res.move_as_error()); @@ -5985,7 +5987,6 @@ bool Collator::create_block_candidate() { } // 3.1 check block and collated data size - auto consensus_config = config_->get_consensus_config(); if (block_candidate->data.size() > consensus_config.max_block_size) { return fatal_error(PSTRING() << "block size (" << block_candidate->data.size() << ") exceeds the limit in consensus config (" << consensus_config.max_block_size diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 88c0535f9..ed063cb6d 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -70,7 +70,7 @@ void ValidatorGroup::generate_block_candidate(validatorsession::BlockSourceInfo td::actor::send_closure(collation_manager_, &CollationManager::collate_block, shard_, min_masterchain_block_id_, prev_block_ids_, Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, source_info.priority, validator_set_, max_answer_size, - cancellation_token_source_.get_cancellation_token(), std::move(P)); + cancellation_token_source_.get_cancellation_token(), std::move(P), config_.proto_version); } void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo source_info, From 1cb420163f825a8d00f63b7ba6a2036eedaefb72 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 17 Mar 2025 12:03:51 +0300 Subject: [PATCH 164/388] Ping neighbours in private overlays, show ping time in overlay stats --- catchain/catchain-receiver.cpp | 1 + overlay/overlay-peers.cpp | 48 +++++++++++++----- overlay/overlay.cpp | 27 ++++++++-- overlay/overlay.hpp | 23 ++++++--- overlay/overlays.h | 1 + tl/generate/scheme/ton_api.tl | 6 ++- tl/generate/scheme/ton_api.tlo | Bin 102788 -> 103024 bytes .../validator-engine-console-query.cpp | 12 ++++- validator/full-node-private-overlay.cpp | 1 + 9 files changed, 92 insertions(+), 27 deletions(-) diff --git a/catchain/catchain-receiver.cpp b/catchain/catchain-receiver.cpp index a6160383c..b663cfc06 100644 --- a/catchain/catchain-receiver.cpp +++ b/catchain/catchain-receiver.cpp @@ -528,6 +528,7 @@ void CatChainReceiverImpl::start_up() { } overlay::OverlayOptions overlay_options; overlay_options.broadcast_speed_multiplier_ = opts_.broadcast_speed_multiplier; + overlay_options.private_ping_peers_ = true; td::actor::send_closure(overlay_manager_, &overlay::Overlays::create_private_overlay_ex, get_source(local_idx_)->get_adnl_id(), overlay_full_id_.clone(), std::move(ids), make_callback(), overlay::OverlayPrivacyRules{0, 0, std::move(root_keys)}, diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index 7def4a2d3..923f0fb12 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -247,12 +247,16 @@ void OverlayImpl::add_peers(const tl_object_ptr &nodes } } -void OverlayImpl::on_ping_result(adnl::AdnlNodeIdShort peer, bool success) { - if (overlay_type_ == OverlayType::FixedMemberList) { +void OverlayImpl::on_ping_result(adnl::AdnlNodeIdShort peer, bool success, double store_ping_time) { + if (overlay_type_ == OverlayType::FixedMemberList && (!success || store_ping_time < 0.0)) { return; } if (OverlayPeer *p = peer_list_.peers_.get(peer)) { p->on_ping_result(success); + if (store_ping_time >= 0.0 && success) { + p->last_ping_at = td::Timestamp::now(); + p->last_ping_time = store_ping_time; + } if (p->is_alive()) { peer_list_.bad_peers_.erase(peer); } else { @@ -261,9 +265,9 @@ void OverlayImpl::on_ping_result(adnl::AdnlNodeIdShort peer, bool success) { } } -void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::Result R) { +void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::Result R, double elapsed) { CHECK(overlay_type_ != OverlayType::FixedMemberList); - on_ping_result(src, R.is_ok()); + on_ping_result(src, R.is_ok(), elapsed); if (R.is_error()) { VLOG(OVERLAY_NOTICE) << this << ": failed getRandomPeers query: " << R.move_as_error(); return; @@ -278,9 +282,9 @@ void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::Result R) { +void OverlayImpl::receive_random_peers_v2(adnl::AdnlNodeIdShort src, td::Result R, double elapsed) { CHECK(overlay_type_ != OverlayType::FixedMemberList); - on_ping_result(src, R.is_ok()); + on_ping_result(src, R.is_ok(), elapsed); if (R.is_error()) { VLOG(OVERLAY_NOTICE) << this << ": failed getRandomPeersV2 query: " << R.move_as_error(); return; @@ -318,9 +322,9 @@ void OverlayImpl::send_random_peers_cont(adnl::AdnlNodeIdShort src, OverlayNode auto Q = create_tl_object(std::move(vec)); promise.set_value(serialize_tl_object(Q, true)); } else { - auto P = - td::PromiseCreator::lambda([SelfId = actor_id(this), src, oid = print_id()](td::Result res) { - td::actor::send_closure(SelfId, &OverlayImpl::receive_random_peers, src, std::move(res)); + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), src, timer = td::Timer()](td::Result res) { + td::actor::send_closure(SelfId, &OverlayImpl::receive_random_peers, src, std::move(res), timer.elapsed()); }); auto Q = create_tl_object(create_tl_object(std::move(vec))); @@ -367,9 +371,9 @@ void OverlayImpl::send_random_peers_v2_cont(adnl::AdnlNodeIdShort src, OverlayNo auto Q = create_tl_object(std::move(vec)); promise.set_value(serialize_tl_object(Q, true)); } else { - auto P = - td::PromiseCreator::lambda([SelfId = actor_id(this), src, oid = print_id()](td::Result res) { - td::actor::send_closure(SelfId, &OverlayImpl::receive_random_peers_v2, src, std::move(res)); + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), src, timer = td::Timer()](td::Result res) { + td::actor::send_closure(SelfId, &OverlayImpl::receive_random_peers_v2, src, std::move(res), timer.elapsed()); }); auto Q = create_tl_object(create_tl_object(std::move(vec))); @@ -392,6 +396,26 @@ void OverlayImpl::send_random_peers_v2(adnl::AdnlNodeIdShort src, td::Promise R) { + if (R.is_error()) { + VLOG(OVERLAY_INFO) << oid << " ping to " << peer << " failed : " << R.move_as_error(); + return; + } + td::actor::send_closure(SelfId, &OverlayImpl::receive_pong, peer, timer.elapsed()); + }); + td::actor::send_closure(manager_, &OverlayManager::send_query, peer, local_id_, overlay_id_, "overlay ping", + std::move(P), td::Timestamp::in(5.0), create_serialize_tl_object()); + } +} + +void OverlayImpl::receive_pong(adnl::AdnlNodeIdShort peer, double elapsed) { + on_ping_result(peer, true, elapsed); +} + void OverlayImpl::update_neighbours(td::uint32 nodes_to_change) { if (peer_list_.peers_.size() == 0) { return; diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index 30a40b1cf..cad18bedc 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -134,6 +134,11 @@ void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getR } } +void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_ping &query, + td::Promise promise) { + promise.set_value(create_serialize_tl_object()); +} + void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcast &query, td::Promise promise) { auto it = broadcasts_.find(query.hash_); @@ -319,7 +324,7 @@ void OverlayImpl::alarm() { } } } else { - VLOG(OVERLAY_WARNING) << "meber certificate ist invalid, valid_until=" + VLOG(OVERLAY_WARNING) << "member certificate ist invalid, valid_until=" << peer_list_.local_cert_is_valid_until_.at_unix(); } if (next_dht_query_ && next_dht_query_.is_in_past() && overlay_type_ == OverlayType::Public) { @@ -356,8 +361,19 @@ void OverlayImpl::alarm() { } alarm_timestamp() = td::Timestamp::in(1.0); } else { - update_neighbours(0); - alarm_timestamp() = td::Timestamp::in(60.0 + td::Random::fast(0, 100) * 0.6); + if (update_neighbours_at_.is_in_past()) { + update_neighbours(0); + update_neighbours_at_ = td::Timestamp::in(60.0 + td::Random::fast(0, 100) * 0.6); + } + if (opts_.private_ping_peers_) { + if (private_ping_peers_at_.is_in_past()) { + ping_random_peers(); + private_ping_peers_at_ = td::Timestamp::in(td::Random::fast(30.0, 50.0)); + } + alarm_timestamp().relax(private_ping_peers_at_); + } + alarm_timestamp().relax(update_neighbours_at_); + alarm_timestamp().relax(update_throughput_at_); } } @@ -483,7 +499,7 @@ void OverlayImpl::send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::Bu void OverlayImpl::send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) { if (!has_valid_membership_certificate()) { - VLOG(OVERLAY_WARNING) << "meber certificate is invalid, valid_until=" + VLOG(OVERLAY_WARNING) << "member certificate is invalid, valid_until=" << peer_list_.local_cert_is_valid_until_.at_unix(); return; } @@ -721,6 +737,9 @@ void OverlayImpl::get_stats(td::Promiseis_alive_ = peer.is_alive(); node_obj->node_flags_ = peer.get_node()->flags(); + node_obj->last_ping_at_ = (peer.last_ping_at ? peer.last_ping_at.at_unix() : -1.0); + node_obj->last_ping_time_ = peer.last_ping_time; + res->nodes_.push_back(std::move(node_obj)); }); diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 41a04dec2..7727f5f1f 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -105,11 +105,11 @@ class OverlayPeer { void on_ping_result(bool success) { if (success) { missed_pings_ = 0; - last_ping_at_ = td::Timestamp::now(); + last_receive_at_ = td::Timestamp::now(); is_alive_ = true; } else { ++missed_pings_; - if (missed_pings_ >= 3 && last_ping_at_.is_in_past(td::Timestamp::in(-15.0))) { + if (missed_pings_ >= 3 && last_receive_at_.is_in_past(td::Timestamp::in(-15.0))) { is_alive_ = false; } } @@ -149,6 +149,9 @@ class OverlayPeer { td::string ip_addr_str = "undefined"; + td::Timestamp last_ping_at = td::Timestamp::never(); + double last_ping_time = -1.0; + private: OverlayNode node_; adnl::AdnlNodeIdShort id_; @@ -157,7 +160,7 @@ class OverlayPeer { size_t missed_pings_ = 0; bool is_alive_ = true; bool is_permanent_member_ = false; - td::Timestamp last_ping_at_ = td::Timestamp::now(); + td::Timestamp last_receive_at_ = td::Timestamp::now(); }; class OverlayImpl : public Overlay { @@ -195,13 +198,15 @@ class OverlayImpl : public Overlay { alarm_timestamp() = td::Timestamp::in(1); } - void on_ping_result(adnl::AdnlNodeIdShort peer, bool success); - void receive_random_peers(adnl::AdnlNodeIdShort src, td::Result R); - void receive_random_peers_v2(adnl::AdnlNodeIdShort src, td::Result R); + void on_ping_result(adnl::AdnlNodeIdShort peer, bool success, double store_ping_time = -1.0); + void receive_random_peers(adnl::AdnlNodeIdShort src, td::Result R, double elapsed); + void receive_random_peers_v2(adnl::AdnlNodeIdShort src, td::Result R, double elapsed); void send_random_peers(adnl::AdnlNodeIdShort dst, td::Promise promise); void send_random_peers_v2(adnl::AdnlNodeIdShort dst, td::Promise promise); void send_random_peers_cont(adnl::AdnlNodeIdShort dst, OverlayNode node, td::Promise promise); void send_random_peers_v2_cont(adnl::AdnlNodeIdShort dst, OverlayNode node, td::Promise promise); + void ping_random_peers(); + void receive_pong(adnl::AdnlNodeIdShort peer, double elapsed); void get_overlay_random_peers(td::uint32 max_peers, td::Promise> promise) override; void set_privacy_rules(OverlayPrivacyRules rules) override; void add_certificate(PublicKeyHash key, std::shared_ptr cert) override { @@ -335,6 +340,7 @@ class OverlayImpl : public Overlay { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeersV2 &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_ping &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcast &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcastList &query, @@ -390,9 +396,10 @@ class OverlayImpl : public Overlay { td::Timestamp next_dht_query_ = td::Timestamp::in(1.0); td::Timestamp next_dht_store_query_ = td::Timestamp::in(1.0); td::Timestamp update_db_at_; - td::Timestamp update_throughput_at_; - td::Timestamp update_neighbours_at_; + td::Timestamp update_throughput_at_ = td::Timestamp::now(); + td::Timestamp update_neighbours_at_ = td::Timestamp::now(); td::Timestamp last_throughput_update_; + td::Timestamp private_ping_peers_at_ = td::Timestamp::now(); std::unique_ptr callback_; diff --git a/overlay/overlays.h b/overlay/overlays.h index 5eb63b13f..972e48cce 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -270,6 +270,7 @@ struct OverlayOptions { td::uint32 propagate_broadcast_to_ = 5; td::uint32 default_permanent_members_flags_ = 0; double broadcast_speed_multiplier_ = 1.0; + bool private_ping_peers_ = false; }; class Overlays : public td::actor::Actor { diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 14e0121fd..580d74861 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -234,6 +234,8 @@ overlay.nodeV2 id:PublicKey overlay:int256 flags:int version:int signature:bytes overlay.nodes nodes:(vector overlay.node) = overlay.Nodes; overlay.nodesV2 nodes:(vector overlay.NodeV2) = overlay.NodesV2; +overlay.pong = overlay.Pong; + overlay.messageExtra flags:# certificate:flags.0?overlay.MemberCertificate = overlay.MessageExtra; overlay.message overlay:int256 = overlay.Message; overlay.messageWithExtra overlay:int256 extra:overlay.messageExtra = overlay.Message; @@ -270,6 +272,7 @@ overlay.broadcastNotFound = overlay.Broadcast; overlay.getRandomPeers peers:overlay.nodes = overlay.Nodes; overlay.getRandomPeersV2 peers:overlay.NodesV2 = overlay.NodesV2; +overlay.ping = overlay.Pong; overlay.query overlay:int256 = True; overlay.queryWithExtra overlay:int256 extra:overlay.messageExtra = True; @@ -714,7 +717,8 @@ engine.validator.overlayStatsTraffic t_out_bytes:long t_in_bytes:long t_out_pckt engine.validator.overlayStatsNode adnl_id:int256 ip_addr:string is_neighbour:Bool is_alive:Bool node_flags:int bdcst_errors:int fec_bdcst_errors:int last_in_query:int last_out_query:int - traffic:engine.validator.overlayStatsTraffic traffic_responses:engine.validator.overlayStatsTraffic = engine.validator.OverlayStatsNode; + traffic:engine.validator.overlayStatsTraffic traffic_responses:engine.validator.overlayStatsTraffic + last_ping_at:double last_ping_time:double = engine.validator.OverlayStatsNode; engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 scope:string nodes:(vector engine.validator.overlayStatsNode) stats:(vector engine.validator.oneStat) diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 5d9ccc2d8586fcb9da5e367372b01e3394ff8b62..ae1785ebb566de1406a4792679287455daaccfc3 100644 GIT binary patch delta 271 zcmZo!%=Tdk8}Fmp`c@23ps|tHU5a(_s--s-C(BC7ZZ4Aw;^1N7C`jkYFH0@TNvxbC zsId8jl8yx9md#(S`YTvKN+;*+Fqph!^Bf2#Vv7nVNc840As^n!6W^(CUbCf!Q9%0H zeNCR6#Nv|pg3P@1_{2%6GW>i9eo1C->SX^o(aAM?Il$&@HreK*q=Rg=UO|3dI?$aU zF_7UPCl@)h>2TP#9R!JOF1c~1fU$dfpeN&RWu3+?Jee>@>w&DqqjS4ME~ASI04f+_ ACIA2c delta 136 zcmeycgso*U8}Fmp`c@23puUmUU5fRR%khhflVzo3H%85lqcZXOfz;oiJ)%VI{JB4;)o4%@baAmPm$ bw)rS+Ho1ABfU#>ks~6*MY`ARoI diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index d11100194..ab214c3f7 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -920,6 +920,10 @@ td::Status GetOverlaysStatsQuery::receive(td::BufferSlice data) { << "\n"; sb << " is_neighbour: " << n->is_neighbour_ << " is_alive: " << n->is_alive_ << " node_flags: " << n->node_flags_ << "\n"; + if (n->last_ping_time_ >= 0.0) { + sb << " last_ping_at: " << (td::uint32)n->last_ping_at_ << " (" << time_to_human((td::uint32)n->last_ping_at_) + << ") last_ping_time: " << n->last_ping_time_ << "\n"; + } print_traffic("throughput", " ", n->traffic_); print_traffic("throughput (responses only)", " ", n->traffic_responses_); } @@ -985,8 +989,12 @@ td::Status GetOverlaysStatsJsonQuery::receive(td::BufferSlice data) { << ",\n \"last_in_query_unix\": " << n->last_in_query_ << ",\n \"last_in_query_human\": \"" << time_to_human(n->last_in_query_) << "\",\n" << " \"last_out_query_unix\": " << n->last_out_query_ << ",\n \"last_out_query_human\": \"" - << time_to_human(n->last_out_query_) << "\",\n" - << "\n "; + << time_to_human(n->last_out_query_) << "\",\n"; + if (n->last_ping_time_ >= 0.0) { + sb << " \"last_ping_at\": " << (td::uint32)n->last_ping_at_ << ", \"last_ping_at_human\": \"" + << time_to_human((td::uint32)n->last_ping_at_) << "\", \"last_ping_time\": " << n->last_ping_time_ << ",\n"; + } + sb << "\n "; print_traffic("throughput", n->traffic_); sb << ",\n "; print_traffic("throughput_responses", n->traffic_responses_); diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp index f86323fc0..1fb9c1507 100644 --- a/validator/full-node-private-overlay.cpp +++ b/validator/full-node-private-overlay.cpp @@ -266,6 +266,7 @@ void FullNodePrivateBlockOverlay::init() { {}}; overlay::OverlayOptions overlay_options; overlay_options.broadcast_speed_multiplier_ = opts_.private_broadcast_speed_multiplier_; + overlay_options.private_ping_peers_ = true; td::actor::send_closure(overlays_, &overlay::Overlays::create_private_overlay_ex, local_id_, overlay_id_full_.clone(), nodes_, std::make_unique(actor_id(this)), rules, R"({ "type": "private-blocks" })", overlay_options); From 3740efdb75c39aa83a0917e1080e6e0a3656063f Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 13 Mar 2025 18:25:08 +0400 Subject: [PATCH 165/388] [Tolk] Replace `"..."c` postfixes with `stringCrc32("...")` functions With the introduction function `ton("0.05")`, I decided to remove old FunC-style string postfixes "..."c, "..."H, etc. in favor of a clearer and more flexible approach. "..."c -> stringCrc32("...") no way -> stringCrc16("...") "..."H -> stringSha256("...") "..."h -> stringSha256_32("...") "..."a -> stringAddressToSlice("...") "..."s -> stringHexToSlice("...") "..."u -> stringToBase256("...") These functions are compile-time only. They can be used in constant initialization. --- crypto/smartcont/tolk-stdlib/common.tolk | 54 +++++- tolk-tester/tests/constants-tests.tolk | 2 +- tolk-tester/tests/inference-tests.tolk | 2 +- tolk-tester/tests/parse-address.tolk | 194 ++++++++++---------- tolk-tester/tests/strings-tests.tolk | 41 +++-- tolk/ast-from-tokens.cpp | 11 +- tolk/ast-replicator.h | 2 +- tolk/ast-stringifier.h | 6 +- tolk/ast.cpp | 4 + tolk/ast.h | 19 +- tolk/builtins.cpp | 34 +++- tolk/constant-evaluator.cpp | 217 ++++++++++++----------- tolk/constant-evaluator.h | 4 +- tolk/lexer.cpp | 11 +- tolk/lexer.h | 1 - tolk/pipe-ast-to-legacy.cpp | 8 +- tolk/pipe-constant-folding.cpp | 34 +++- tolk/pipe-infer-types-and-calls.cpp | 7 +- tolk/symtable.h | 2 + 19 files changed, 382 insertions(+), 271 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 9647243f9..d5290976a 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -194,6 +194,42 @@ fun commitContractDataAndActions(): void Signature checks, hashing, cryptography. */ +/// Compile-time function that calculates crc32 of a constant string. +/// Example: `const op = stringCrc32("some_str")` = 4013618352 = 0xEF3AF4B0 +/// Note: stringCrc32(slice_var) does not work! It accepts a constant string and works at compile-time. +@pure +fun stringCrc32(constString: slice): int + builtin; + +/// Compile-time function that calculates crc16 (XMODEM) of a constant string. +/// Example: `const op = stringCrc16("some_str")` = 53407 = 0xD09F +/// Note: stringCrc16(slice_var) does not work! It accepts a constant string and works at compile-time. +@pure +fun stringCrc16(constString: slice): int + builtin; + +/// Compile-time function that calculates sha256 of a constant string and returns 256-bit integer. +/// Example: `const hash = stringSha256("some_crypto_key")` +/// Note: it's a compile-time function, `stringSha256(slice_var)` does not work. +/// Use `sliceBitsHash` or `sliceHash` (declared below) to hash a slice without/with its refs at runtime. +@pure +fun stringSha256(constString: slice): int + builtin; + +/// Compile-time function that calculates sha256 of a constant string and takes the first 32 bits. +/// Example: `const minihash = stringSha256_32("some_crypto_key")` +/// Note: stringSha256_32(slice_var) does not work! It accepts a constant string and works at compile-time. +@pure +fun stringSha256_32(constString: slice): int + builtin; + +/// Compile-time function that takes N-chars ascii string and interprets it as a number in base 256. +/// Example: `const value = stringToBase256("AB")` = 16706 (65*256 + 66) +/// Note: stringToBase256(slice_var) does not work! It accepts a constant string and works at compile-time. +@pure +fun stringToBase256(constString: slice): int + builtin; + /// Computes the representation hash of a `cell` [c] and returns it as a 256-bit unsigned integer `x`. /// Useful for signing and checking signatures of arbitrary entities represented by a tree of cells. @pure @@ -210,7 +246,7 @@ fun sliceHash(s: slice): int /// Computes sha256 of the data bits of `slice` [s]. If the bit length of `s` is not divisible by eight, /// throws a cell underflow exception. The hash value is returned as a 256-bit unsigned integer `x`. @pure -fun stringHash(s: slice): int +fun sliceBitsHash(s: slice): int asm "SHA256U"; /// Checks the Ed25519-`signature` of a `hash` (a 256-bit unsigned integer, usually computed as the hash of some data) @@ -340,6 +376,22 @@ fun debugDumpStack(): void When you _preload_ some data, you just get the result without mutating the slice. */ +/// Compile-time function that converts a constant string to TL-encoded MsgAddressInt (TVM slice). +/// Example: stringAddressToSlice("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5") +/// Example: stringAddressToSlice("0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8") +/// Note: it's a compile-time function, `stringAddressToSlice(slice_var)` does not work. +/// Use `parseStandardAddress` to decode a slice at runtime into workchain and hash. +@pure +fun stringAddressToSlice(constStringAddress: slice): slice + builtin; + +/// Compile-time function that converts a constant hex-encoded string to N/2 bytes. +/// Example: `const v = stringHexToSlice("abcdef")` = slice with 3 bytes `[ 0xAB, 0xCD, 0xEF ]` +/// Note: stringHexToSlice(slice_var) does not work! It accepts a constant string and works at compile-time. +@pure +fun stringHexToSlice(constStringBytesHex: slice): slice + builtin; + /// Converts a `cell` [c] into a `slice`. Notice that [c] must be either an ordinary cell, /// or an exotic cell (see [TVM.pdf](https://ton-blockchain.github.io/docs/tvm.pdf), 3.1.2) /// which is automatically loaded to yield an ordinary cell `c'`, converted into a `slice` afterwards. diff --git a/tolk-tester/tests/constants-tests.tolk b/tolk-tester/tests/constants-tests.tolk index 66b89190a..e145306ea 100644 --- a/tolk-tester/tests/constants-tests.tolk +++ b/tolk-tester/tests/constants-tests.tolk @@ -7,7 +7,7 @@ const int111: int = 111; const int1r = int1; const str1 = "const1"; -const str2 = "aabbcc"s; +const str2 = stringHexToSlice("aabbcc"); const str2r: slice = str2; diff --git a/tolk-tester/tests/inference-tests.tolk b/tolk-tester/tests/inference-tests.tolk index 463d4a7b7..436848317 100644 --- a/tolk-tester/tests/inference-tests.tolk +++ b/tolk-tester/tests/inference-tests.tolk @@ -5,7 +5,7 @@ fun eq(value: X): X { return value; } fun test1(x: int, y: int) { __expect_type(0, "int"); - __expect_type("0"c, "int"); + __expect_type(stringCrc32("0"), "int"); __expect_type(x, "int"); __expect_type(x + y, "int"); __expect_type(x * y, "int"); diff --git a/tolk-tester/tests/parse-address.tolk b/tolk-tester/tests/parse-address.tolk index 385aa3b53..64ca5d67e 100644 --- a/tolk-tester/tests/parse-address.tolk +++ b/tolk-tester/tests/parse-address.tolk @@ -1,5 +1,5 @@ -const cc1 = "0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e"a; -const cc2 = "EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"a; +const cc1 = stringAddressToSlice("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e"); +const cc2 = stringAddressToSlice("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"); fun verifyAddr(addr: slice, workchain: int, number: int) { assert (addr.getRemainingBitsCount() == 3 + 8 + 256) throw 112; @@ -9,101 +9,101 @@ fun verifyAddr(addr: slice, workchain: int, number: int) { } fun main() { - verifyAddr("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"a, 255, 23158417847463239084714197001737581570653996933128112807891516801582625927987); - verifyAddr("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"a, 0, 0); - verifyAddr("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"a, 0, 65607996509792174074532427555986248720836864382484024657400295821210434460432); - verifyAddr("UQCOgxbCOjOLH_cEuQdGgS23zBM5SrQQepMFedjK-oixYbis"a, 0, 64460038539088394980732229180523693489682583665805557562964506609821558550881); - verifyAddr("EQDa4VOnTYlLvDJ0gZjNYm5PXfSmmtL6Vs6A_CZEtXCNICq_"a, 0, 99002318936150612861744867526221033858534811876886359650897405270877291973920); - verifyAddr("Ef8BtXO9bcTMXjg9bgivKh4lhJmZWQPP6_rb9vfjlTP5FJtM"a, 255, 772910975127952880303441415761050161913031788763061162001556772893733681428); - verifyAddr("Ef89xh-uy860-mCcvS8zcAUs8bApmxLGygDLEKjUk5RL-311"a, 255, 27941138149036269893630478666581900122707382189183906805784676408403709676539); - verifyAddr("Ef_vA6yRfmt2P4UHnxlrQUZFcBnKux8mL2eMqBgpeMFPorr4"a, 255, 108109262375472472702582493362335418330829651067377177643099076957184687427490); - verifyAddr("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi"a, 255, 18502444830824300068094395885436326119386947594392869497312068745716154912158); - verifyAddr("Ef_fvrd0hBoVJUxoi7wH173Zk8NPiyVvxh5IoYSjEYZbOhsu"a, 255, 101202732337223525952216789200341489000836292542250083765062769181728788863802); - verifyAddr("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf"a, 255, 46952625717497919357580310066854892621799390294920450816077086267929711460872); - verifyAddr("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA"a, 255, 48545777798729612074233611768739897492467685225150339217043102685589809464695); - verifyAddr("Ef9LynHHKgBxY6-l-W_dWN-CtGT2_ji5rN3EzOI-p9zWEfq6"a, 255, 34281152017620085319078796986198022632548048219136747083019177301186013091345); - verifyAddr("Ef9hMd78gzSiVsK0zz0AHtEja8x1UoB_NDZMjn-l86NQK_2Y"a, 255, 43962460814164090767878334494257755557842170134382045184921495822637115592747); - verifyAddr("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6"a, 255, 23557057702048801338698514499604413540742716310574705490458593067566768087319); - verifyAddr("Ef_fdIbThooPs4_r2DE_Z6ZsWycJdHLnsuKAJHTcbaZaipez"a, 255, 101071650030310556115830521522496708686577365303530257137459798093298869361290); - verifyAddr("Ef_lva0qEiZhWrrZJl-IJxyCcTQmmTo71fIWyQ31HxJ8NurV"a, 255, 103914771557158282349484109182290824591675204108148026180964788916630125182006); - verifyAddr("Ef8sMGKypw006AeRYqimLjmY2Ufp-SHk8C0ZJBNgVBlzw_Nr"a, 255, 19987255184378161380023126214650814972824352533523055905552702178965809886147); - verifyAddr("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff+W72r5gqPrHF"a, 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); - verifyAddr("EQCaSCHVak-jIc9ANutTAfHpZNM3YdGky7yaDzsTrg0WhFlm"a, 0, 69783625181781015447914682554083924798054947959007050695795761257887453484676); - verifyAddr("EQBS9U3AfD15fGmOtRMXQAxcPVBwNuItfLcDni9fkbTyyNX0"a, 0, 37523067738561024305547433298623118197038688994386001017161816416175242146504); - verifyAddr("EQBiMNL9qNWMAkJHuM0BFneYcuHL17kzS4pswpaEO-NGWrFG"a, 0, 44412924025649114419413541526870954696667907029239618728289150652715284776538); - verifyAddr("EQAUzE-Nef80O9dLZy91HfPiOb6EEQ8YqyWKyIU-KeaYLNUi"a, 0, 9407242825041766837311851458322335726136775042891143504070507665010681354284); - verifyAddr("EQD-nhrinjv0B4LTgr0dRHTHwH1MOsgGhKBXJZd7vESMZUf1"a, 0, 115166810931401616117484448645661180241548402534908005320733783571353775148133); - verifyAddr("EQAVD3Fni9I6j8XeSIl-wAGBEhqhame6OtAY0GScKT0D9X6f"a, 0, 9525855215156855607080079714361451576383963668563135377495902959388099150837); - verifyAddr("EQC6ACq3VANZjqfRBy7JMHkpLwqQ9qyYJsCIGx1mYbQgxaKw"a, 0, 84130484652351964071210477536969520113177637645401392541565606610268614566085); - verifyAddr("EQCIJLNFIko5CvpKn9oAkrDgLocDOoD4vwmHxNx_fsG_LkwW"a, 0, 61579391178099797614367237687950512448308156724136883899001108680249616482094); - verifyAddr("EQCe4AYIBce1pAk2qJJPSs1OzyZRlKjkfq8zuC8D7erv6DUP"a, 0, 71861245445432818728925844931259040612664802586395398157190478191760507596776); - verifyAddr("EQCtrtTXEAoSpoERmiqOnICe9LHxn2N89N4BH9qdHlrG-U0i"a, 0, 78559023162479717496981724991265882229440558807791659796411897368395464230649); - verifyAddr("EQBBlraAps0OZaB9Q8ePQn2wVAaL1G411A-dNppyWe3X3GIT"a, 0, 29666621803903557832193058147214384979915773445007872807927344851911086823388); - verifyAddr("EQBiASqUqaVizrozLRbszkWC2kETbkhpO2qniDVDPPg2_0W8"a, 0, 44328719889509369519441680467651025944540360433148852643949783408843779749631); - verifyAddr("EQBu2Q1EO8gIoNA1qoGWnHUudKfmqlKEDTQE-DxN-_4sdg14"a, 0, 50137910719490808065414827264266674858051167131188257457782826342827836714102); - verifyAddr("EQA5bvxWd5-q2vJUVqR9AlbEIfdFysLR0PXGgVlBf8x5hWuF"a, 0, 25977927117604457079092522008276392864656238504700352770597256138254994667909); - verifyAddr("EQBguMSHjFv5bfoOdshr3ruS9ymSZzhRKMovoNrxGxZXvmee"a, 0, 43748489720571123896506696370504498290006245978262404519821633796370658121662); - verifyAddr("EQAxL0oF1-zNgimPKthbDnYS4xj94rHtfNRN7_Pd1r2LNNv3"a, 0, 22246882279393590648219842750911786376759362211171398419754426796438233910068); - verifyAddr("EQANX1uRKGZfyPIwEaIXrR0ZOqadct5q10dvKxWIxx7SQqzW"a, 0, 6048549475100840191738010856156544571222758030966479209409932714701987172930); - verifyAddr("EQBitdFDoU5DWSjfKq7AsO29RIwAnBzzvcVVSn5ekQoB9Liv"a, 0, 44647902768175374073183447303109856895983123510911038181495138821771906122228); - verifyAddr("EQBgbux7VSjqJHP7ByRK1q4QuVZbpSCesNgvz5qad3lfXX_B"a, 0, 43618018778298854282398238948198420936771670943015013768514626213000552996701); - verifyAddr("EQDisBd8U7M3CEOZ8gcWCdetdmJi3AI31zIT5qBwOdmUbsxY"a, 0, 102533830955233207294921564956803510155400341370448300698800842506363763004526); - verifyAddr("EQAZpn_eynVlf7Ii2d6jP_p1URPrdF9F3S7DiudQyelkjzwE"a, 0, 11602000355550451044739442929923326898313570892134000961608306166632391730319); - verifyAddr("EQDE0HBgfkOiqHezLtExBGTvOs8eitthHQosBjW3BmDy1y2K"a, 0, 89021598108837008984355105304701054698583123510131754065320641619941010764503); - verifyAddr("EQDyT36zktBN9PVWvZ1joRxhIfEUgCPt4F2isa-enUA_d6CP"a, 0, 109600164736599393471831241268953938618560132398692391517933933264745646800759); - verifyAddr("EQDSMUGwt25IQd3_yHjI03F71G8Kp2GMaMEv2TiWoTKbsyRH"a, 0, 95072727086440754059372943502908629555499501854161516009430039520728770059187); - verifyAddr("EQAgK1EcrvEuL9sCtoj3cNhVNOuf3lo5GIPE2gn1fwZZYB3j"a, 0, 14550545393206146289454646242321274637527057595221202748348667645886114191712); - verifyAddr("EQCDKqL5w_6MD-Z7AOButu-uR-ZJTsgNU1fu464hn9grY81U"a, 0, 59328315557704100696483472039557119625141880163887490602190749720459366378339); - verifyAddr("EQB1aVMyFBhnlYXmQjsma0S63kvxKU7ccZKFNCFTwX7ASPv4"a, 0, 53106696421104300082516512931084483581353095629408473618166869610568148238408); - verifyAddr("EQBbjrXHoxDyh1ZYGBdBoQgLaScxW6pZR1hEhJC8BqF-5Kgq"a, 0, 41412616102566803060532874463898939692666425753852274254609049615175463829220); - verifyAddr("EQC-QeZ13QP0lszxNKt380fCWuaV94vwC/bfuqmrlg1/fJPA"a, 0, 86055876869280374285292827775555707420719385459150221433115419095878595346300); - verifyAddr("EQAiUwpF27vXCngqNhf_TQ5E_06ah0G4zuSrnfU7CLLaht5H"a, 0, 15525356059048115813946213102829493539706126913595626308144289257869196581510); - verifyAddr("EQBqiVjmhe2iVGmgOSDO1FGjSiz_AMtb1w7lLEiP4XIF_SFy"a, 0, 48187833566271418625754761625661652107159264793429628379411792200127405491709); - verifyAddr("EQDmwvaK2d_SbaPdpOM60effPWeKsksgDVwFPEyxuftM396K"a, 0, 104376425077737068747642645125299653296942252727305637929378053253273342397663); - verifyAddr("EQDWtPZZgF7wvIMUHZQojuD3utiuivsW7WslRJ33dgv-5yc8"a, 0, 97114682311034709685427168495629428400170984047839002197324103884924936519399); - verifyAddr("EQAA7z0JI0JKqbN-1uENKz9JrxIO5ZRY-ehMeg9fPncx50Ck"a, 0, 422697701361909095759185681783393186844038628935759044330165207027374567911); - verifyAddr("EQBVUHRoCq6coQYUwOAhGSoAmQ6Mpm7dFlDYon6HMgWV8Ftr"a, 0, 38588743302295548905191533977469452945717219128199196974980570837505276220912); - verifyAddr("EQCTdvDCf0bA5dOPI1-44tB2ZfNcMGiklzvg27TovgDEqM6E"a, 0, 66700138358140658950710678965721715920748906761125730971082529064117803730088); - verifyAddr("EQBDBKE5WGKIlnoi3OOzw7vkKKIX55eWjPvgxJWwek8AyL2J"a, 0, 30313140970524770883308749215942283658935592719811899513010665548955593408712); - verifyAddr("EQAvCSyLCo21GrqLAifdov4WkOxuGQCjMRxgF1cXSaNzLHZe"a, 0, 21274912932379789207153885262858116665851037273450532982121514600400844714796); - verifyAddr("EQCsLpDeHB2qpRbmsCb_0xmsYVNx1NgeYrvHGT1TDrHkDgL4"a, 0, 77880084760844670718511036557364450189323549135231036747480176919181282894862); - verifyAddr("EQCTQ8kPwyX92r48gCIL_pLN_RcQT9ghZygnmDTYkOkuW_j5"a, 0, 66609755171046741472463433430016629628609840960137226492652665879987546041947); - verifyAddr("EQCTrFRSHt-tfk7WxK9ZHQmqLcgxXxTK7wGfCEbqgY2W9Mcx"a, 0, 66794468397542182731534559853537484892417154018190804733043974345563210356468); - verifyAddr("EQCv28y49GdaLncoclv0ISdDlMUY_cxDPGNWFCPT8t4GuqUJ"a, 0, 79543100951881731989812212377176715376834409392116644269458867858071577560762); - verifyAddr("EQCVL-k6deDR56Z8pcb0Btg0lGfaivOGfdDCD1vvyRsyL9vS"a, 0, 67479265933941008511790471646564661743401752930295407567346938670637286896175); - verifyAddr("EQD6t2dXDjZxF1DqunKF-8dEWivJdliY_0FYiCXnthuqnDCa"a, 0, 113402258385556889021060606279033166272577193563727959698596277924908309916316); - verifyAddr("EQDE98XNzXiPq7VnbJJ2M4-Ht3tX_OWR0xUTTnDC8NObLmyU"a, 0, 89091094739778473356272490822716056624384395256388481953562551087642791090990); - verifyAddr("EQDfeRDE1TDhwt478CDR0Q7MDwqcTUhfjqyTT59mgoAaF6f7"a, 0, 101079669463449311486034260688909914923832300293253430457119371423825321269783); - verifyAddr("EQDijcEyUKa-QgCbeGlggQk1uBtt2ZRHyW4Y4gB4R6MN6RLW"a, 0, 102473162609487797404330889623966425536887610061087715571345738626121871855081); - verifyAddr("EQDOtFOt41skbjBkZF89oYXpoDECjlxIzD-ShWAOYyzuxqLA"a, 0, 93495056812773926196963707371665512785268729004579280701087533371037976424134); - verifyAddr("EQDuJKSFWU7AYqH6KLFfAbYvMuz346eWmJvG6_2NYE42_B4T"a, 0, 107715199938628393100813870735031557263256555616273999363057194279168054802172); - verifyAddr("EQDwGu4vFv1e3wn8min_iy7OPJXegOYTFQ5bZFZ5a5ZPiBpX"a, 0, 108602665568837301744601989570019709742180613578164394799026726718721456754568); - verifyAddr("EQC4G2ph6AS_mD_-cIv4aIYm1z5jAgCW_TTDEr72ygXOP2X-"a, 0, 83274003234732023403481554420859495155084746906198543572711543697320249249343); - verifyAddr("EQDpUkyAa6lZ12P3ZB2PL_rmWwI1I55BU4kxw_rssFL5dswA"a, 0, 105534303174146507629736518862713754948570412188900908177600861330298381728118); - verifyAddr("EQDoIA20MF1qEcSPtROdCu5ukGx9dVjgMeJh1oQ4A4cf_Jif"a, 0, 104993214557977037193613824776415934089204193426692473563548548423424814817276); - verifyAddr("EQDpUkyAa6lZ12P3ZB2PL_rmWwI1I55BU4kxw_rssFL5dswA"a, 0, 105534303174146507629736518862713754948570412188900908177600861330298381728118); - verifyAddr("EQClLO4EnZ_rTyV1GVpWy53pLgWJRki5c4ZzuM_1O_ClBkO9"a, 0, 74711004027159342540251007601464500186374346239921204216319145006974068892934); - verifyAddr("EQDmkj65Ab_m0aZaW8IpKw4kYqIgITw_HRstYEkVQ6NIYCyW"a, 0, 104290347741656803921830951060768893809692975574470790497562993373950614128736); - verifyAddr("EQCqNTwAYUNhPFS0RgqZoTLGJcQQxbAJ7csUo4YO3_TONLab"a, 0, 76987241268612358571638783428744566580605181728938801022059780105627411729972); - verifyAddr("EQCL3DmCynaRK7-vsfeNmd4Jj-UxAIHPvA4qS2xwaL6UpLbF"a, 0, 63260589232981964910240894899061676480139492286430589202252472895352724165796); - verifyAddr("EQDbU1SVEjBE73oUqgAoM9gDcShUkM5EC2PgoCjuwVUKo-Ee"a, 0, 99203745911752606845646497420891218522647962685916739950275357890977532807843); - verifyAddr("EQD02VdcF4TDbCKLLhZJ39NQTu6aWq2LnLjp0oXqbNu_BANK"a, 0, 110748343802097970709980079967961144373090790244250392237586606542170934198020); - verifyAddr("EQBynBO23ywHy_CgarY9NK9FTz0yDsG82PtcbSTQgGoXwiuA"a, 0, 51839428943991432793039248316067731096592274748149794482308513726460953499586); - verifyAddr("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"a, 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); - verifyAddr("EQAUTbQiM522Y_XJ_T98QPhPhTmb4nV--VSPiha8kC6kRfPO"a, 0, 9183547432069678364603018431103042146626948674383548774683927217595824907333); - verifyAddr("EQBlqsm144Dq6SjbPI4jjZvA1hqTIP3CvHovbIfW_t-SCALE"a, 0, 45985353862647206060987594732861817093328871106941773337270673759241903247880); - verifyAddr("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"a, 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); - verifyAddr("kQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPgpP"a, 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); - verifyAddr("kf-Dfdg-YQXaR2Q97gZJ4fGBtmV1DHOU1y1RPyyZZtRy_Ikh"a, 255, 59475331506450494976393625198911249698879029820580340449086829444312920781564); - verifyAddr("0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"a, 0, 37304138005561100291416421295333982606153966175434134130332440738068913455320); - verifyAddr("0:0000000000000000000000000000000000000000000000000000000000000000"a, 0, 0); - verifyAddr("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff"a, 0, 115792089237316195423570985008687907853269984665640564039457584007913129639935); - verifyAddr("0:zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"a, 0, 23158417847463239084714197001737581570653996933128112807891516801582625927987); - verifyAddr("0:0000000000000000000000000000000000000000000000000000000000000000"a, 0, 0); - verifyAddr("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"a, 1, 37304138005561100291416421295333982606153966175434134130332440738068913455320); - verifyAddr("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"a, 9, 37304138005561100291416421295333982606153966175434134130332440738068913455320); - verifyAddr("99:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"a, 99, 37304138005561100291416421295333982606153966175434134130332440738068913455320); - verifyAddr("-1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"a, 255, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr(stringAddressToSlice("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"), 255, 23158417847463239084714197001737581570653996933128112807891516801582625927987); + verifyAddr(stringAddressToSlice("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"), 0, 0); + verifyAddr(stringAddressToSlice("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"), 0, 65607996509792174074532427555986248720836864382484024657400295821210434460432); + verifyAddr(stringAddressToSlice("UQCOgxbCOjOLH_cEuQdGgS23zBM5SrQQepMFedjK-oixYbis"), 0, 64460038539088394980732229180523693489682583665805557562964506609821558550881); + verifyAddr(stringAddressToSlice("EQDa4VOnTYlLvDJ0gZjNYm5PXfSmmtL6Vs6A_CZEtXCNICq_"), 0, 99002318936150612861744867526221033858534811876886359650897405270877291973920); + verifyAddr(stringAddressToSlice("Ef8BtXO9bcTMXjg9bgivKh4lhJmZWQPP6_rb9vfjlTP5FJtM"), 255, 772910975127952880303441415761050161913031788763061162001556772893733681428); + verifyAddr(stringAddressToSlice("Ef89xh-uy860-mCcvS8zcAUs8bApmxLGygDLEKjUk5RL-311"), 255, 27941138149036269893630478666581900122707382189183906805784676408403709676539); + verifyAddr(stringAddressToSlice("Ef_vA6yRfmt2P4UHnxlrQUZFcBnKux8mL2eMqBgpeMFPorr4"), 255, 108109262375472472702582493362335418330829651067377177643099076957184687427490); + verifyAddr(stringAddressToSlice("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi"), 255, 18502444830824300068094395885436326119386947594392869497312068745716154912158); + verifyAddr(stringAddressToSlice("Ef_fvrd0hBoVJUxoi7wH173Zk8NPiyVvxh5IoYSjEYZbOhsu"), 255, 101202732337223525952216789200341489000836292542250083765062769181728788863802); + verifyAddr(stringAddressToSlice("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf"), 255, 46952625717497919357580310066854892621799390294920450816077086267929711460872); + verifyAddr(stringAddressToSlice("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA"), 255, 48545777798729612074233611768739897492467685225150339217043102685589809464695); + verifyAddr(stringAddressToSlice("Ef9LynHHKgBxY6-l-W_dWN-CtGT2_ji5rN3EzOI-p9zWEfq6"), 255, 34281152017620085319078796986198022632548048219136747083019177301186013091345); + verifyAddr(stringAddressToSlice("Ef9hMd78gzSiVsK0zz0AHtEja8x1UoB_NDZMjn-l86NQK_2Y"), 255, 43962460814164090767878334494257755557842170134382045184921495822637115592747); + verifyAddr(stringAddressToSlice("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6"), 255, 23557057702048801338698514499604413540742716310574705490458593067566768087319); + verifyAddr(stringAddressToSlice("Ef_fdIbThooPs4_r2DE_Z6ZsWycJdHLnsuKAJHTcbaZaipez"), 255, 101071650030310556115830521522496708686577365303530257137459798093298869361290); + verifyAddr(stringAddressToSlice("Ef_lva0qEiZhWrrZJl-IJxyCcTQmmTo71fIWyQ31HxJ8NurV"), 255, 103914771557158282349484109182290824591675204108148026180964788916630125182006); + verifyAddr(stringAddressToSlice("Ef8sMGKypw006AeRYqimLjmY2Ufp-SHk8C0ZJBNgVBlzw_Nr"), 255, 19987255184378161380023126214650814972824352533523055905552702178965809886147); + verifyAddr(stringAddressToSlice("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff+W72r5gqPrHF"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr(stringAddressToSlice("EQCaSCHVak-jIc9ANutTAfHpZNM3YdGky7yaDzsTrg0WhFlm"), 0, 69783625181781015447914682554083924798054947959007050695795761257887453484676); + verifyAddr(stringAddressToSlice("EQBS9U3AfD15fGmOtRMXQAxcPVBwNuItfLcDni9fkbTyyNX0"), 0, 37523067738561024305547433298623118197038688994386001017161816416175242146504); + verifyAddr(stringAddressToSlice("EQBiMNL9qNWMAkJHuM0BFneYcuHL17kzS4pswpaEO-NGWrFG"), 0, 44412924025649114419413541526870954696667907029239618728289150652715284776538); + verifyAddr(stringAddressToSlice("EQAUzE-Nef80O9dLZy91HfPiOb6EEQ8YqyWKyIU-KeaYLNUi"), 0, 9407242825041766837311851458322335726136775042891143504070507665010681354284); + verifyAddr(stringAddressToSlice("EQD-nhrinjv0B4LTgr0dRHTHwH1MOsgGhKBXJZd7vESMZUf1"), 0, 115166810931401616117484448645661180241548402534908005320733783571353775148133); + verifyAddr(stringAddressToSlice("EQAVD3Fni9I6j8XeSIl-wAGBEhqhame6OtAY0GScKT0D9X6f"), 0, 9525855215156855607080079714361451576383963668563135377495902959388099150837); + verifyAddr(stringAddressToSlice("EQC6ACq3VANZjqfRBy7JMHkpLwqQ9qyYJsCIGx1mYbQgxaKw"), 0, 84130484652351964071210477536969520113177637645401392541565606610268614566085); + verifyAddr(stringAddressToSlice("EQCIJLNFIko5CvpKn9oAkrDgLocDOoD4vwmHxNx_fsG_LkwW"), 0, 61579391178099797614367237687950512448308156724136883899001108680249616482094); + verifyAddr(stringAddressToSlice("EQCe4AYIBce1pAk2qJJPSs1OzyZRlKjkfq8zuC8D7erv6DUP"), 0, 71861245445432818728925844931259040612664802586395398157190478191760507596776); + verifyAddr(stringAddressToSlice("EQCtrtTXEAoSpoERmiqOnICe9LHxn2N89N4BH9qdHlrG-U0i"), 0, 78559023162479717496981724991265882229440558807791659796411897368395464230649); + verifyAddr(stringAddressToSlice("EQBBlraAps0OZaB9Q8ePQn2wVAaL1G411A-dNppyWe3X3GIT"), 0, 29666621803903557832193058147214384979915773445007872807927344851911086823388); + verifyAddr(stringAddressToSlice("EQBiASqUqaVizrozLRbszkWC2kETbkhpO2qniDVDPPg2_0W8"), 0, 44328719889509369519441680467651025944540360433148852643949783408843779749631); + verifyAddr(stringAddressToSlice("EQBu2Q1EO8gIoNA1qoGWnHUudKfmqlKEDTQE-DxN-_4sdg14"), 0, 50137910719490808065414827264266674858051167131188257457782826342827836714102); + verifyAddr(stringAddressToSlice("EQA5bvxWd5-q2vJUVqR9AlbEIfdFysLR0PXGgVlBf8x5hWuF"), 0, 25977927117604457079092522008276392864656238504700352770597256138254994667909); + verifyAddr(stringAddressToSlice("EQBguMSHjFv5bfoOdshr3ruS9ymSZzhRKMovoNrxGxZXvmee"), 0, 43748489720571123896506696370504498290006245978262404519821633796370658121662); + verifyAddr(stringAddressToSlice("EQAxL0oF1-zNgimPKthbDnYS4xj94rHtfNRN7_Pd1r2LNNv3"), 0, 22246882279393590648219842750911786376759362211171398419754426796438233910068); + verifyAddr(stringAddressToSlice("EQANX1uRKGZfyPIwEaIXrR0ZOqadct5q10dvKxWIxx7SQqzW"), 0, 6048549475100840191738010856156544571222758030966479209409932714701987172930); + verifyAddr(stringAddressToSlice("EQBitdFDoU5DWSjfKq7AsO29RIwAnBzzvcVVSn5ekQoB9Liv"), 0, 44647902768175374073183447303109856895983123510911038181495138821771906122228); + verifyAddr(stringAddressToSlice("EQBgbux7VSjqJHP7ByRK1q4QuVZbpSCesNgvz5qad3lfXX_B"), 0, 43618018778298854282398238948198420936771670943015013768514626213000552996701); + verifyAddr(stringAddressToSlice("EQDisBd8U7M3CEOZ8gcWCdetdmJi3AI31zIT5qBwOdmUbsxY"), 0, 102533830955233207294921564956803510155400341370448300698800842506363763004526); + verifyAddr(stringAddressToSlice("EQAZpn_eynVlf7Ii2d6jP_p1URPrdF9F3S7DiudQyelkjzwE"), 0, 11602000355550451044739442929923326898313570892134000961608306166632391730319); + verifyAddr(stringAddressToSlice("EQDE0HBgfkOiqHezLtExBGTvOs8eitthHQosBjW3BmDy1y2K"), 0, 89021598108837008984355105304701054698583123510131754065320641619941010764503); + verifyAddr(stringAddressToSlice("EQDyT36zktBN9PVWvZ1joRxhIfEUgCPt4F2isa-enUA_d6CP"), 0, 109600164736599393471831241268953938618560132398692391517933933264745646800759); + verifyAddr(stringAddressToSlice("EQDSMUGwt25IQd3_yHjI03F71G8Kp2GMaMEv2TiWoTKbsyRH"), 0, 95072727086440754059372943502908629555499501854161516009430039520728770059187); + verifyAddr(stringAddressToSlice("EQAgK1EcrvEuL9sCtoj3cNhVNOuf3lo5GIPE2gn1fwZZYB3j"), 0, 14550545393206146289454646242321274637527057595221202748348667645886114191712); + verifyAddr(stringAddressToSlice("EQCDKqL5w_6MD-Z7AOButu-uR-ZJTsgNU1fu464hn9grY81U"), 0, 59328315557704100696483472039557119625141880163887490602190749720459366378339); + verifyAddr(stringAddressToSlice("EQB1aVMyFBhnlYXmQjsma0S63kvxKU7ccZKFNCFTwX7ASPv4"), 0, 53106696421104300082516512931084483581353095629408473618166869610568148238408); + verifyAddr(stringAddressToSlice("EQBbjrXHoxDyh1ZYGBdBoQgLaScxW6pZR1hEhJC8BqF-5Kgq"), 0, 41412616102566803060532874463898939692666425753852274254609049615175463829220); + verifyAddr(stringAddressToSlice("EQC-QeZ13QP0lszxNKt380fCWuaV94vwC/bfuqmrlg1/fJPA"), 0, 86055876869280374285292827775555707420719385459150221433115419095878595346300); + verifyAddr(stringAddressToSlice("EQAiUwpF27vXCngqNhf_TQ5E_06ah0G4zuSrnfU7CLLaht5H"), 0, 15525356059048115813946213102829493539706126913595626308144289257869196581510); + verifyAddr(stringAddressToSlice("EQBqiVjmhe2iVGmgOSDO1FGjSiz_AMtb1w7lLEiP4XIF_SFy"), 0, 48187833566271418625754761625661652107159264793429628379411792200127405491709); + verifyAddr(stringAddressToSlice("EQDmwvaK2d_SbaPdpOM60effPWeKsksgDVwFPEyxuftM396K"), 0, 104376425077737068747642645125299653296942252727305637929378053253273342397663); + verifyAddr(stringAddressToSlice("EQDWtPZZgF7wvIMUHZQojuD3utiuivsW7WslRJ33dgv-5yc8"), 0, 97114682311034709685427168495629428400170984047839002197324103884924936519399); + verifyAddr(stringAddressToSlice("EQAA7z0JI0JKqbN-1uENKz9JrxIO5ZRY-ehMeg9fPncx50Ck"), 0, 422697701361909095759185681783393186844038628935759044330165207027374567911); + verifyAddr(stringAddressToSlice("EQBVUHRoCq6coQYUwOAhGSoAmQ6Mpm7dFlDYon6HMgWV8Ftr"), 0, 38588743302295548905191533977469452945717219128199196974980570837505276220912); + verifyAddr(stringAddressToSlice("EQCTdvDCf0bA5dOPI1-44tB2ZfNcMGiklzvg27TovgDEqM6E"), 0, 66700138358140658950710678965721715920748906761125730971082529064117803730088); + verifyAddr(stringAddressToSlice("EQBDBKE5WGKIlnoi3OOzw7vkKKIX55eWjPvgxJWwek8AyL2J"), 0, 30313140970524770883308749215942283658935592719811899513010665548955593408712); + verifyAddr(stringAddressToSlice("EQAvCSyLCo21GrqLAifdov4WkOxuGQCjMRxgF1cXSaNzLHZe"), 0, 21274912932379789207153885262858116665851037273450532982121514600400844714796); + verifyAddr(stringAddressToSlice("EQCsLpDeHB2qpRbmsCb_0xmsYVNx1NgeYrvHGT1TDrHkDgL4"), 0, 77880084760844670718511036557364450189323549135231036747480176919181282894862); + verifyAddr(stringAddressToSlice("EQCTQ8kPwyX92r48gCIL_pLN_RcQT9ghZygnmDTYkOkuW_j5"), 0, 66609755171046741472463433430016629628609840960137226492652665879987546041947); + verifyAddr(stringAddressToSlice("EQCTrFRSHt-tfk7WxK9ZHQmqLcgxXxTK7wGfCEbqgY2W9Mcx"), 0, 66794468397542182731534559853537484892417154018190804733043974345563210356468); + verifyAddr(stringAddressToSlice("EQCv28y49GdaLncoclv0ISdDlMUY_cxDPGNWFCPT8t4GuqUJ"), 0, 79543100951881731989812212377176715376834409392116644269458867858071577560762); + verifyAddr(stringAddressToSlice("EQCVL-k6deDR56Z8pcb0Btg0lGfaivOGfdDCD1vvyRsyL9vS"), 0, 67479265933941008511790471646564661743401752930295407567346938670637286896175); + verifyAddr(stringAddressToSlice("EQD6t2dXDjZxF1DqunKF-8dEWivJdliY_0FYiCXnthuqnDCa"), 0, 113402258385556889021060606279033166272577193563727959698596277924908309916316); + verifyAddr(stringAddressToSlice("EQDE98XNzXiPq7VnbJJ2M4-Ht3tX_OWR0xUTTnDC8NObLmyU"), 0, 89091094739778473356272490822716056624384395256388481953562551087642791090990); + verifyAddr(stringAddressToSlice("EQDfeRDE1TDhwt478CDR0Q7MDwqcTUhfjqyTT59mgoAaF6f7"), 0, 101079669463449311486034260688909914923832300293253430457119371423825321269783); + verifyAddr(stringAddressToSlice("EQDijcEyUKa-QgCbeGlggQk1uBtt2ZRHyW4Y4gB4R6MN6RLW"), 0, 102473162609487797404330889623966425536887610061087715571345738626121871855081); + verifyAddr(stringAddressToSlice("EQDOtFOt41skbjBkZF89oYXpoDECjlxIzD-ShWAOYyzuxqLA"), 0, 93495056812773926196963707371665512785268729004579280701087533371037976424134); + verifyAddr(stringAddressToSlice("EQDuJKSFWU7AYqH6KLFfAbYvMuz346eWmJvG6_2NYE42_B4T"), 0, 107715199938628393100813870735031557263256555616273999363057194279168054802172); + verifyAddr(stringAddressToSlice("EQDwGu4vFv1e3wn8min_iy7OPJXegOYTFQ5bZFZ5a5ZPiBpX"), 0, 108602665568837301744601989570019709742180613578164394799026726718721456754568); + verifyAddr(stringAddressToSlice("EQC4G2ph6AS_mD_-cIv4aIYm1z5jAgCW_TTDEr72ygXOP2X-"), 0, 83274003234732023403481554420859495155084746906198543572711543697320249249343); + verifyAddr(stringAddressToSlice("EQDpUkyAa6lZ12P3ZB2PL_rmWwI1I55BU4kxw_rssFL5dswA"), 0, 105534303174146507629736518862713754948570412188900908177600861330298381728118); + verifyAddr(stringAddressToSlice("EQDoIA20MF1qEcSPtROdCu5ukGx9dVjgMeJh1oQ4A4cf_Jif"), 0, 104993214557977037193613824776415934089204193426692473563548548423424814817276); + verifyAddr(stringAddressToSlice("EQDpUkyAa6lZ12P3ZB2PL_rmWwI1I55BU4kxw_rssFL5dswA"), 0, 105534303174146507629736518862713754948570412188900908177600861330298381728118); + verifyAddr(stringAddressToSlice("EQClLO4EnZ_rTyV1GVpWy53pLgWJRki5c4ZzuM_1O_ClBkO9"), 0, 74711004027159342540251007601464500186374346239921204216319145006974068892934); + verifyAddr(stringAddressToSlice("EQDmkj65Ab_m0aZaW8IpKw4kYqIgITw_HRstYEkVQ6NIYCyW"), 0, 104290347741656803921830951060768893809692975574470790497562993373950614128736); + verifyAddr(stringAddressToSlice("EQCqNTwAYUNhPFS0RgqZoTLGJcQQxbAJ7csUo4YO3_TONLab"), 0, 76987241268612358571638783428744566580605181728938801022059780105627411729972); + verifyAddr(stringAddressToSlice("EQCL3DmCynaRK7-vsfeNmd4Jj-UxAIHPvA4qS2xwaL6UpLbF"), 0, 63260589232981964910240894899061676480139492286430589202252472895352724165796); + verifyAddr(stringAddressToSlice("EQDbU1SVEjBE73oUqgAoM9gDcShUkM5EC2PgoCjuwVUKo-Ee"), 0, 99203745911752606845646497420891218522647962685916739950275357890977532807843); + verifyAddr(stringAddressToSlice("EQD02VdcF4TDbCKLLhZJ39NQTu6aWq2LnLjp0oXqbNu_BANK"), 0, 110748343802097970709980079967961144373090790244250392237586606542170934198020); + verifyAddr(stringAddressToSlice("EQBynBO23ywHy_CgarY9NK9FTz0yDsG82PtcbSTQgGoXwiuA"), 0, 51839428943991432793039248316067731096592274748149794482308513726460953499586); + verifyAddr(stringAddressToSlice("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr(stringAddressToSlice("EQAUTbQiM522Y_XJ_T98QPhPhTmb4nV--VSPiha8kC6kRfPO"), 0, 9183547432069678364603018431103042146626948674383548774683927217595824907333); + verifyAddr(stringAddressToSlice("EQBlqsm144Dq6SjbPI4jjZvA1hqTIP3CvHovbIfW_t-SCALE"), 0, 45985353862647206060987594732861817093328871106941773337270673759241903247880); + verifyAddr(stringAddressToSlice("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr(stringAddressToSlice("kQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPgpP"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr(stringAddressToSlice("kf-Dfdg-YQXaR2Q97gZJ4fGBtmV1DHOU1y1RPyyZZtRy_Ikh"), 255, 59475331506450494976393625198911249698879029820580340449086829444312920781564); + verifyAddr(stringAddressToSlice("0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 0, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr(stringAddressToSlice("0:0000000000000000000000000000000000000000000000000000000000000000"), 0, 0); + verifyAddr(stringAddressToSlice("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff"), 0, 115792089237316195423570985008687907853269984665640564039457584007913129639935); + verifyAddr(stringAddressToSlice("0:zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"), 0, 23158417847463239084714197001737581570653996933128112807891516801582625927987); + verifyAddr(stringAddressToSlice("0:0000000000000000000000000000000000000000000000000000000000000000"), 0, 0); + verifyAddr(stringAddressToSlice("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 1, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr(stringAddressToSlice("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 9, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr(stringAddressToSlice("99:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 99, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr(stringAddressToSlice("-1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 255, 37304138005561100291416421295333982606153966175434134130332440738068913455320); return cc1.isSliceBitsEqual(cc2); } diff --git a/tolk-tester/tests/strings-tests.tolk b/tolk-tester/tests/strings-tests.tolk index c7c4f6946..af8c6edeb 100644 --- a/tolk-tester/tests/strings-tests.tolk +++ b/tolk-tester/tests/strings-tests.tolk @@ -3,27 +3,31 @@ get ascii_slice(): slice { } get raw_slice(): slice { - return "abcdef"s; + return stringHexToSlice("abcdef"); } get addr_slice(): slice { - return "Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"a; + return stringAddressToSlice("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"); } get string_hex(): int { - return "ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"u; + return stringToBase256("ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"); } get fun string_minihash(): int { // 'get' and 'get fun' both possible - return "transfer(slice, int)"h; + return stringSha256_32("transfer(slice, int)"); } get fun string_maxihash(): int { - return "transfer(slice, int)"H; + return stringSha256("transfer(slice, int)"); } get fun string_crc32(): int { - return "transfer(slice, int)"c; + return stringCrc32("transfer(slice, int)"); +} + +get fun string_crc16(): int { + return stringCrc16("transfer(slice, int)"); } @pure @@ -35,6 +39,15 @@ asm "ENDC" "CTOS"; fun sdeq(s1: slice, s2: slice): int asm "SDEQ"; +fun calcSliceBytes(slice: slice): tuple { + var t = createEmptyTuple(); + var n = 0; + while (n = slice.getRemainingBitsCount()) { + t.tuplePush(slice.loadUint(min(8, n))); + } + return t; +} + fun main() { var s_ascii: slice = ascii_slice(); var s_raw: slice = raw_slice(); @@ -42,7 +55,8 @@ fun main() { var i_hex: int = string_hex(); var i_mini: int = string_minihash(); var i_maxi: int = string_maxihash(); - var i_crc: int = string_crc32(); + var i_crc32: int = string_crc32(); + var i_crc16: int = string_crc16(); assert(sdeq(s_ascii, newc().storeUint(0x737472696E67, 12 * 4).endcs())) throw 101; assert(sdeq(s_raw, newc().storeUint(0xABCDEF, 6 * 4).endcs())) throw 102; assert(sdeq(s_addr, newc().storeUint(4, 3).storeInt(-1, 8) @@ -50,12 +64,19 @@ fun main() { assert(i_hex == 0x4142434445464748494A4B4C4D4E4F505152535455565758595A303132333435) throw 104; assert(i_mini == 0x7a62e8a8) throw 105; assert(i_maxi == 0x7a62e8a8ebac41bd6de16c65e7be363bc2d2cbc6a0873778dead4795c13db979) throw 106; - assert(i_crc == 2235694568) throw 107; + assert(i_crc32 == 2235694568 && i_crc32 == 0x8541fde8) throw 107; + assert(i_crc16 == 52550 && i_crc16 == 0xCD46) throw 108; return 0; } +@method_id(101) +fun test1() { + return calcSliceBytes("ABCD"); +} + /** -@testcase | 0 | | 0 +@testcase | 0 | | 0 +@testcase | 101 | | [ 65 66 67 68 ] -@code_hash 13830542019509784148027107880226447201604257839069192762244575629978154217223 +@code_hash 38184847030631877916087987911699475358017315230885358090110033079289166112584 */ diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 22974b1f5..94ef848c1 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -357,12 +357,7 @@ static AnyExprV parse_expr100(Lexer& lex) { case tok_string_const: { std::string_view str_val = lex.cur_str(); lex.next(); - char modifier = 0; - if (lex.tok() == tok_string_modifier) { - modifier = lex.cur_str()[0]; - lex.next(); - } - return createV(loc, str_val, modifier); + return createV(loc, str_val); } case tok_underscore: { lex.next(); @@ -927,7 +922,7 @@ static AnyV parse_asm_func_body(Lexer& lex, V param_list) { lex.check(tok_string_const, "\"ASM COMMAND\""); while (lex.tok() == tok_string_const) { std::string_view asm_command = lex.cur_str(); - asm_commands.push_back(createV(lex.cur_location(), asm_command, 0)); + asm_commands.push_back(createV(lex.cur_location(), asm_command)); lex.next(); } lex.expect(tok_semicolon, "`;`"); @@ -1142,7 +1137,7 @@ static AnyV parse_import_directive(Lexer& lex) { if (rel_filename.empty()) { lex.error("imported file name is an empty string"); } - auto v_str = createV(lex.cur_location(), rel_filename, 0); + auto v_str = createV(lex.cur_location(), rel_filename); lex.next(); return createV(loc, v_str); // semicolon is not necessary } diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index 16bbbeb8d..bf48c2389 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -80,7 +80,7 @@ class ASTReplicatorFunction : public ASTReplicator { return createV(v->loc, v->intval, v->orig_str); } virtual V clone(V v) { - return createV(v->loc, v->str_val, v->modifier); + return createV(v->loc, v->str_val); } virtual V clone(V v) { return createV(v->loc, v->bool_val); diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index a7f260de8..07f03745c 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -136,11 +136,7 @@ class ASTStringifier final : public ASTVisitor { case ast_int_const: return static_cast(v->as()->orig_str); case ast_string_const: - if (char modifier = v->as()->modifier) { - return "\"" + static_cast(v->as()->str_val) + "\"" + std::string(1, modifier); - } else { - return "\"" + static_cast(v->as()->str_val) + "\""; - } + return "\"" + static_cast(v->as()->str_val) + "\""; case ast_bool_const: return v->as()->bool_val ? "true" : "false"; case ast_dot_access: { diff --git a/tolk/ast.cpp b/tolk/ast.cpp index 26eaacd59..ac93d21c3 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -126,6 +126,10 @@ void Vertex::assign_sym(const Symbol* sym) { this->sym = sym; } +void Vertex::assign_literal_value(ConstantValue&& literal_value) { + this->literal_value = std::move(literal_value); +} + void Vertex::assign_fun_ref(FunctionPtr fun_ref) { this->fun_maybe = fun_ref; } diff --git a/tolk/ast.h b/tolk/ast.h index 9b7c5d1a5..385e1ff89 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -458,25 +458,18 @@ struct Vertex final : ASTExprLeaf { template<> // ast_string_const is a string literal in double quotes or """ when multiline -// examples: "asdf" / "Ef8zMz..."a / "to_calc_crc32_from"c -// an optional modifier specifies how a string is parsed (probably, like an integer) +// examples: "asdf" / "LTIME" (in asm body) / stringCrc32("asdf") (as an argument) // note, that TVM doesn't have strings, it has only slices, so "hello" has type slice struct Vertex final : ASTExprLeaf { std::string_view str_val; - char modifier; + ConstantValue literal_value; // value of type `slice`, calculated after type inferring, at constants evaluation - bool is_bitslice() const { - char m = modifier; - return m == 0 || m == 's' || m == 'a'; - } - bool is_intval() const { - char m = modifier; - return m == 'u' || m == 'h' || m == 'H' || m == 'c'; - } + Vertex* mutate() const { return const_cast(this); } + void assign_literal_value(ConstantValue&& literal_value); - Vertex(SrcLocation loc, std::string_view str_val, char modifier) + Vertex(SrcLocation loc, std::string_view str_val) : ASTExprLeaf(ast_string_const, loc) - , str_val(str_val), modifier(modifier) {} + , str_val(str_val) {} }; template<> diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 9ef4527ae..75ead3458 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1072,7 +1072,8 @@ AsmOp compile_tuple_set_at(std::vector& res, std::vector& ar } // fun ton(amount: slice): coins; ton("0.05") replaced by 50000000 at compile-time -AsmOp compile_int_to_ton_coins(std::vector&, std::vector&, SrcLocation) { +// same for stringCrc32(constString: slice) and others +AsmOp compile_time_only_function(std::vector&, std::vector&, SrcLocation) { // all ton() invocations are constants, replaced by integers; no dynamic values allowed, no work at runtime tolk_assert(false); return AsmOp::Nop(); @@ -1219,11 +1220,36 @@ void define_builtins() { compile_throw_if_unless, 0); + // compile-time only functions, evaluated essentially at compile-time, no runtime implementation + // they are placed in stdlib and marked as `builtin` + // note their parameter being `unknown`: in order to `ton(1)` pass type inferring but fire a more gentle error later + define_builtin_func("ton", {TypeDataUnknown::create()}, TypeDataCoins::create(), nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + define_builtin_func("stringCrc32", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + define_builtin_func("stringCrc16", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + define_builtin_func("stringSha256", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + define_builtin_func("stringSha256_32", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + define_builtin_func("stringToBase256", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + define_builtin_func("stringAddressToSlice", {TypeDataUnknown::create()}, TypeDataSlice::create(), nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + define_builtin_func("stringHexToSlice", {TypeDataUnknown::create()}, TypeDataSlice::create(), nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + // functions from stdlib marked as `builtin`, implemented at compiler level for optimizations // (for example, `loadInt(1)` is `1 LDI`, but `loadInt(n)` for non-constant requires it be on a stack and `LDIX`) - define_builtin_func("ton", {TypeDataUnknown::create()}, TypeDataCoins::create(), nullptr, // `unknown` to pass inferring for `ton(1)` (to fire a better error later) - compile_int_to_ton_coins, - FunctionData::flagMarkedAsPure); define_builtin_func("mulDivFloor", ParamsInt3, Int, nullptr, std::bind(compile_muldiv, _1, _2, _3, -1), FunctionData::flagMarkedAsPure); diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp index 5c3b3b19a..7daebc1da 100644 --- a/tolk/constant-evaluator.cpp +++ b/tolk/constant-evaluator.cpp @@ -24,6 +24,14 @@ namespace tolk { +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_const_string_required(SrcLocation loc, std::string_view f_name, std::string_view example_arg) { + std::string name = static_cast(f_name); + std::string example = static_cast(example_arg); + throw ParseError(loc, "function `" + name + "` requires a constant string, like `" + name + "(\"" + example + "\")`"); +} + + // parse address like "EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5" // based on unpack_std_smc_addr() from block.cpp // (which is not included to avoid linking with ton_crypto) @@ -44,10 +52,10 @@ static bool parse_friendly_address(const char packed[48], ton::WorkchainId& work // parse address like "0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8" // based on StdAddress::parse_addr() from block.cpp // (which is not included to avoid linking with ton_crypto) -static bool parse_raw_address(const std::string& acc_string, int& workchain, ton::StdSmcAddress& addr) { +static bool parse_raw_address(std::string_view acc_string, int& workchain, ton::StdSmcAddress& addr) { size_t pos = acc_string.find(':'); if (pos != std::string::npos) { - td::Result r_wc = td::to_integer_safe(acc_string.substr(0, pos)); + td::Result r_wc = td::to_integer_safe(td::Slice(acc_string.data(), pos)); if (r_wc.is_error()) { return false; } @@ -82,93 +90,8 @@ static bool parse_raw_address(const std::string& acc_string, int& workchain, ton return true; } - -static std::string parse_vertex_string_const_as_slice(V v) { - std::string str = static_cast(v->str_val); - switch (v->modifier) { - case 0: { - return td::hex_encode(str); - } - case 's': { - unsigned char buff[128]; - long bits = td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.data(), str.data() + str.size()); - if (bits < 0) { - v->error("invalid hex bitstring constant '" + str + "'"); - } - return str; - } - case 'a': { // MsgAddress - ton::WorkchainId workchain; - ton::StdSmcAddress addr; - bool correct = (str.size() == 48 && parse_friendly_address(str.data(), workchain, addr)) || - (str.size() != 48 && parse_raw_address(str, workchain, addr)); - if (!correct) { - v->error("invalid standard address '" + str + "'"); - } - if (workchain < -128 || workchain >= 128) { - v->error("anycast addresses not supported"); - } - - unsigned char data[3 + 8 + 256]; // addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; - td::bitstring::bits_store_long_top(data, 0, static_cast(4) << (64 - 3), 3); - td::bitstring::bits_store_long_top(data, 3, static_cast(workchain) << (64 - 8), 8); - td::bitstring::bits_memcpy(data, 3 + 8, addr.bits().ptr, 0, ton::StdSmcAddress::size()); - return td::BitSlice{data, sizeof(data)}.to_hex(); - } - default: - tolk_assert(false); - } -} - -static td::RefInt256 parse_vertex_string_const_as_int(V v) { - std::string str = static_cast(v->str_val); - switch (v->modifier) { - case 'u': { - td::RefInt256 intval = td::hex_string_to_int256(td::hex_encode(str)); - if (str.empty()) { - v->error("empty integer ascii-constant"); - } - if (intval.is_null()) { - v->error("too long integer ascii-constant"); - } - return intval; - } - case 'h': - case 'H': { - unsigned char hash[32]; - digest::hash_str(hash, str.data(), str.size()); - return td::bits_to_refint(hash, (v->modifier == 'h') ? 32 : 256, false); - } - case 'c': { - return td::make_refint(td::crc32(td::Slice{str})); - } - default: - tolk_assert(false); - } -} - -static ConstantValue parse_vertex_string_const(V v) { - return v->is_bitslice() - ? ConstantValue(parse_vertex_string_const_as_slice(v)) - : ConstantValue(parse_vertex_string_const_as_int(v)); -} - -// given `ton("0.05")` evaluate it to 50000000 -static ConstantValue parse_vertex_call_to_ton_function(V v) { - tolk_assert(v->get_num_args() == 1); // checked by type inferring - AnyExprV v_arg = v->get_arg(0)->get_expr(); - - std::string_view str; - if (auto as_string = v_arg->try_as(); as_string && !as_string->modifier) { - str = as_string->str_val; - } else { - // ton(SOME_CONST) is not supported - // ton(0.05) is not supported (it can't be represented in AST even) - } - if (str.empty()) { - v->error("function `ton` requires a constant string, like `ton(\"0.05\")`"); - } - +// internal helper: for `ton("0.05")`, parse string literal "0.05" to 50000000 +static td::RefInt256 parse_nanotons_as_floating_string(SrcLocation loc, std::string_view str) { bool is_negative = false; size_t i = 0; @@ -190,7 +113,7 @@ static ConstantValue parse_vertex_call_to_ton_function(V v) { char c = str[i]; if (c == '.') { if (seen_dot) { - v_arg->error("argument is not a valid number like \"0.05\""); + throw ParseError(loc, "argument is not a valid number like \"0.05\""); } seen_dot = true; } else if (c >= '0' && c <= '9') { @@ -201,7 +124,7 @@ static ConstantValue parse_vertex_call_to_ton_function(V v) { fractional_digits++; } } else { - v_arg->error("argument is not a valid number like \"0.05\""); + throw ParseError(loc, "argument is not a valid number like \"0.05\""); } } @@ -211,7 +134,99 @@ static ConstantValue parse_vertex_call_to_ton_function(V v) { } int64_t result = integer_part * 1000000000LL + fractional_part; - return ConstantValue(td::make_refint(is_negative ? -result : result)); + return td::make_refint(is_negative ? -result : result); +} + +static ConstantValue parse_vertex_string_const(V v) { + td::Slice str_slice = td::Slice(v->str_val.data(), v->str_val.size()); + return ConstantValue(td::hex_encode(str_slice)); +} + +// given `ton("0.05")` evaluate it to 50000000 +// given `stringCrc32("some_str")` evaluate it +// etc. +// currently, all compile-time functions accept 1 argument, a literal string +static ConstantValue parse_vertex_call_to_compile_time_function(V v, std::string_view f_name) { + tolk_assert(v->get_num_args() == 1); // checked by type inferring + AnyExprV v_arg = v->get_arg(0)->get_expr(); + + std::string_view str; + if (auto as_string = v_arg->try_as()) { + str = as_string->str_val; + } else { + // ton(SOME_CONST) is not supported + // ton(0.05) is not supported (it can't be represented in AST even) + // stringCrc32(SOME_CONST) / stringCrc32(some_var) also, it's compile-time literal-only + } + if (str.empty()) { + fire_error_const_string_required(v->loc, f_name, f_name == "ton" ? "0.05" : "some_str"); + } + + if (f_name == "ton") { + return ConstantValue(parse_nanotons_as_floating_string(v_arg->loc, str)); + } + + if (f_name == "stringCrc32") { // previously, postfix "..."c + return ConstantValue(td::make_refint(td::crc32(td::Slice{str.data(), str.size()}))); + } + + if (f_name == "stringCrc16") { // previously, there was no postfix in FunC, no way to calc at compile-time + return ConstantValue(td::make_refint(td::crc16(td::Slice{str.data(), str.size()}))); + } + + if (f_name == "stringSha256") { // previously, postfix "..."H + unsigned char hash[32]; + digest::hash_str(hash, str.data(), str.size()); + return ConstantValue(td::bits_to_refint(hash, 256, false)); + } + + if (f_name == "stringSha256_32") { // previously, postfix "..."h + unsigned char hash[32]; + digest::hash_str(hash, str.data(), str.size()); + return ConstantValue(td::bits_to_refint(hash, 32, false)); + } + + if (f_name == "stringAddressToSlice") { // previously, postfix "..."a + ton::WorkchainId workchain; + ton::StdSmcAddress addr; + bool correct = (str.size() == 48 && parse_friendly_address(str.data(), workchain, addr)) || + (str.size() != 48 && parse_raw_address(str, workchain, addr)); + if (!correct) { + v_arg->error("invalid standard address"); + } + if (workchain < -128 || workchain >= 128) { + v_arg->error("anycast addresses not supported"); + } + + unsigned char data[3 + 8 + 256]; // addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; + td::bitstring::bits_store_long_top(data, 0, static_cast(4) << (64 - 3), 3); + td::bitstring::bits_store_long_top(data, 3, static_cast(workchain) << (64 - 8), 8); + td::bitstring::bits_memcpy(data, 3 + 8, addr.bits().ptr, 0, ton::StdSmcAddress::size()); + return ConstantValue(td::BitSlice{data, sizeof(data)}.to_hex()); + } + + if (f_name == "stringHexToSlice") { // previously, postfix "..."s + unsigned char buff[128]; + long bits = td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.data(), str.data() + str.size()); + if (bits < 0) { + v_arg->error("invalid hex bitstring constant"); + } + return ConstantValue(static_cast(str)); + } + + if (f_name == "stringToBase256") { // previously, postfix "..."u + td::RefInt256 intval = td::hex_string_to_int256(td::hex_encode(td::Slice(str.data(), str.size()))); + if (str.empty()) { + v_arg->error("empty integer ascii-constant"); + } + if (intval.is_null()) { + v_arg->error("too long integer ascii-constant"); + } + return ConstantValue(std::move(intval)); + } + + tolk_assert(false); + return ConstantValue(); } @@ -332,8 +347,8 @@ struct ConstantEvaluator { // `const a = ton("0.05")`, we met `ton("0.05")` static ConstantValue handle_function_call(V v) { - if (v->fun_maybe && v->fun_maybe->is_builtin_function() && v->fun_maybe->name == "ton") { - return parse_vertex_call_to_ton_function(v); + if (v->fun_maybe && v->fun_maybe->is_compile_time_only()) { + return parse_vertex_call_to_compile_time_function(v, v->fun_maybe->name); } v->error("not a constant expression"); } @@ -378,14 +393,16 @@ struct ConstantEvaluator { } }; -ConstantValue eval_string_const_considering_modifier(AnyExprV v_string) { - tolk_assert(v_string->type == ast_string_const); - return parse_vertex_string_const(v_string->as()); +ConstantValue eval_string_const_standalone(AnyExprV v_string) { + auto v = v_string->try_as(); + tolk_assert(v); + return parse_vertex_string_const(v); } -ConstantValue eval_call_to_ton_function(AnyExprV v_call) { - tolk_assert(v_call->type == ast_function_call && v_call->as()->fun_maybe->is_builtin_function()); - return parse_vertex_call_to_ton_function(v_call->as()); +ConstantValue eval_call_to_compile_time_function(AnyExprV v_call) { + auto v = v_call->try_as(); + tolk_assert(v && v->fun_maybe->is_compile_time_only()); + return parse_vertex_call_to_compile_time_function(v, v->fun_maybe->name); } void eval_and_assign_const_init_value(GlobalConstPtr const_ref) { diff --git a/tolk/constant-evaluator.h b/tolk/constant-evaluator.h index 4737352ea..b065e6f49 100644 --- a/tolk/constant-evaluator.h +++ b/tolk/constant-evaluator.h @@ -47,8 +47,8 @@ class ConstantValue { const std::string& as_slice() const { return std::get(value); } }; -ConstantValue eval_string_const_considering_modifier(AnyExprV v_string); -ConstantValue eval_call_to_ton_function(AnyExprV v_call); +ConstantValue eval_string_const_standalone(AnyExprV v_string); +ConstantValue eval_call_to_compile_time_function(AnyExprV v_call); void eval_and_assign_const_init_value(GlobalConstPtr const_ref); } // namespace tolk diff --git a/tolk/lexer.cpp b/tolk/lexer.cpp index 86ae306a4..961be1674 100644 --- a/tolk/lexer.cpp +++ b/tolk/lexer.cpp @@ -23,7 +23,6 @@ namespace tolk { // By 'chunk' in lexer I mean a token or a list of tokens parsed simultaneously. // E.g., when we meet "str", ChunkString is called, it emits tok_string. -// E.g., when we meet "str"x, ChunkString emits not only tok_string, but tok_string_modifier. // E.g., when we meet //, ChunkInlineComment is called, it emits nothing (just skips a line). // We store all valid chunks lexers in a prefix tree (LexingTrie), see below. struct ChunkLexerBase { @@ -170,8 +169,8 @@ struct ChunkMultilineComment final : ChunkLexerBase { // A string, starting from " // Note, that there are no escape symbols inside: the purpose of strings in Tolk just doesn't need it. -// After a closing quote, a string modifier may be present, like "Ef8zMzMzMzMzMzMzMzMzMzM0vF"a. -// If present, it emits a separate tok_string_modifier. +// In FunC, a string might have ended with a modifier like `"..."c` +// It's not valid in Tolk, valid is `stringCrc32("...")` struct ChunkString final : ChunkLexerBase { bool parse(Lexer* lex) const override { const char* str_begin = lex->c_str(); @@ -187,12 +186,6 @@ struct ChunkString final : ChunkLexerBase { lex->skip_chars(1); lex->add_token(tok_string_const, str_val); - if (std::isalpha(lex->char_at())) { - std::string_view modifier_val(lex->c_str(), 1); - lex->skip_chars(1); - lex->add_token(tok_string_modifier, modifier_val); - } - return true; } }; diff --git a/tolk/lexer.h b/tolk/lexer.h index 58bc3640a..34c792ebc 100644 --- a/tolk/lexer.h +++ b/tolk/lexer.h @@ -48,7 +48,6 @@ enum TokenType { tok_int_const, tok_string_const, - tok_string_modifier, tok_true, tok_false, tok_null, diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index d0c252847..b26c9d2c4 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -1049,13 +1049,9 @@ static std::vector process_int_const(V v, CodeBlob& co } static std::vector process_string_const(V v, CodeBlob& code, TypePtr target_type) { - ConstantValue value = eval_string_const_considering_modifier(v); + tolk_assert(v->literal_value.is_slice()); std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(str-const)"); - if (value.is_int()) { - code.emplace_back(v->loc, Op::_IntConst, rvect, value.as_int()); - } else { - code.emplace_back(v->loc, Op::_SliceConst, rvect, value.as_slice()); - } + code.emplace_back(v->loc, Op::_SliceConst, rvect, v->literal_value.as_slice()); return transition_to_target_type(std::move(rvect), code, target_type, v); } diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp index bdb1763ae..13ecece51 100644 --- a/tolk/pipe-constant-folding.cpp +++ b/tolk/pipe-constant-folding.cpp @@ -57,6 +57,14 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { return inner; } + static V create_string_const(SrcLocation loc, ConstantValue&& literal_value) { + auto v_string = createV(loc, literal_value.as_slice()); + v_string->assign_inferred_type(TypeDataSlice::create()); + v_string->assign_literal_value(std::move(literal_value)); + v_string->assign_rvalue_true(); + return v_string; + } + AnyExprV replace(V v) override { parent::replace(v); @@ -102,17 +110,31 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { AnyExprV replace(V v) override { parent::replace(v); - // replace `ton("0.05")` with 50000000 - if (v->fun_maybe && v->fun_maybe->is_builtin_function() && v->fun_maybe->name == "ton") { - ConstantValue value = eval_call_to_ton_function(v); - tolk_assert(value.is_int()); - return create_int_const(v->loc, value.as_int()); + // replace `ton("0.05")` with 50000000 / `stringCrc32("some_str")` with calculated value / etc. + if (v->fun_maybe && v->fun_maybe->is_compile_time_only()) { + ConstantValue value = eval_call_to_compile_time_function(v); + if (value.is_int()) { + return create_int_const(v->loc, value.as_int()); + } + if (value.is_slice()) { + return create_string_const(v->loc, std::move(value)); + } + tolk_assert(false); } return v; } -public: + AnyExprV replace(V v) override { + // when "some_str" occurs as a standalone constant (not inside `stringCrc32("some_str")`, + // it's actually a slice + ConstantValue literal_value = eval_string_const_standalone(v); + v->mutate()->assign_literal_value(std::move(literal_value)); + + return v; + } + + public: bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function(); } diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index aa8fe64d1..9db82f576 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -280,12 +280,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } static ExprFlow infer_string_const(V v, FlowContext&& flow, bool used_as_condition) { - if (v->is_bitslice()) { - assign_inferred_type(v, TypeDataSlice::create()); - } else { - assign_inferred_type(v, TypeDataInt::create()); - } - + assign_inferred_type(v, TypeDataSlice::create()); return ExprFlow(std::move(flow), used_as_condition); } diff --git a/tolk/symtable.h b/tolk/symtable.h index e65327fdb..e78eef6e8 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -103,6 +103,7 @@ struct FunctionData final : Symbol { flagAcceptsSelf = 512, // is a member function (has `self` first parameter) flagReturnsSelf = 1024, // return type is `self` (returns the mutated 1st argument), calls can be chainable flagReallyUsed = 2048, // calculated via dfs from used functions; declared but unused functions are not codegenerated + flagCompileTimeOnly = 4096, // calculated only at compile-time for constant arguments: `ton("0.05")`, `stringCrc32`, and others }; int method_id = EMPTY_METHOD_ID; @@ -163,6 +164,7 @@ struct FunctionData final : Symbol { bool does_return_self() const { return flags & flagReturnsSelf; } bool does_mutate_self() const { return (flags & flagAcceptsSelf) && parameters[0].is_mutate_parameter(); } bool is_really_used() const { return flags & flagReallyUsed; } + bool is_compile_time_only() const { return flags & flagCompileTimeOnly; } bool does_need_codegen() const; From 6ba2355ca0a5a00c5fbd71d76b0bd0836cc624b0 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 13 Mar 2025 20:20:16 +0400 Subject: [PATCH 166/388] [Tolk] Bump version to v0.10 --- crypto/smartcont/tolk-stdlib/common.tolk | 2 +- crypto/smartcont/tolk-stdlib/gas-payments.tolk | 2 +- crypto/smartcont/tolk-stdlib/lisp-lists.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-dicts.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk | 2 +- tolk/tolk-version.h | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index d5290976a..0075764b6 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -1,7 +1,7 @@ // Standard library for Tolk (LGPL licence). // It contains common functions that are available out of the box, the user doesn't have to import anything. // More specific functions are required to be imported explicitly, like "@stdlib/tvm-dicts". -tolk 0.9 +tolk 0.10 /** Tuple manipulation primitives. diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk index 9873ca94d..11e3db7ff 100644 --- a/crypto/smartcont/tolk-stdlib/gas-payments.tolk +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.9 +tolk 0.10 /** Gas and payment related primitives. diff --git a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk index e63438b51..fd6d2c894 100644 --- a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk +++ b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.9 +tolk 0.10 /** Lisp-style lists are nested 2-elements tuples: `(1, (2, (3, null)))` represents list `[1, 2, 3]`. diff --git a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk index ee2056874..d4babd1ef 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.9 +tolk 0.10 /** Dictionaries are represented as `cell` data type (cells can store anything, dicts in particular). diff --git a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk index 136eaa4a1..48bacec6c 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.9 +tolk 0.10 /// Usually `c3` has a continuation initialized by the whole code of the contract. It is used for function calls. /// The primitive returns the current value of `c3`. diff --git a/tolk/tolk-version.h b/tolk/tolk-version.h index bbea63ffd..be161a393 100644 --- a/tolk/tolk-version.h +++ b/tolk/tolk-version.h @@ -18,6 +18,6 @@ namespace tolk { -constexpr const char* TOLK_VERSION = "0.9.0"; +constexpr const char* TOLK_VERSION = "0.10.0"; } // namespace tolk From f3d67d57751688120ae77ef5b0b1e7a5a37b9e9f Mon Sep 17 00:00:00 2001 From: neodix42 Date: Wed, 19 Mar 2025 12:57:42 +0400 Subject: [PATCH 167/388] force clang-16 usage when compiling ubuntu portable shared libraries (#1560) --- assembly/native/build-ubuntu-portable-libs.sh | 4 ++-- assembly/native/build-ubuntu-portable.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/assembly/native/build-ubuntu-portable-libs.sh b/assembly/native/build-ubuntu-portable-libs.sh index 2f0a1ba4d..b634ce0c4 100644 --- a/assembly/native/build-ubuntu-portable-libs.sh +++ b/assembly/native/build-ubuntu-portable-libs.sh @@ -21,8 +21,8 @@ else rm -rf .ninja* CMakeCache.txt fi -export CC=$(which clang) -export CXX=$(which clang++) +export CC=$(which clang-16) +export CXX=$(which clang++-16) export CCACHE_DISABLE=1 if [ ! -d "lz4" ]; then diff --git a/assembly/native/build-ubuntu-portable.sh b/assembly/native/build-ubuntu-portable.sh index 16e77ac8d..00b34f832 100644 --- a/assembly/native/build-ubuntu-portable.sh +++ b/assembly/native/build-ubuntu-portable.sh @@ -24,8 +24,8 @@ else rm -rf .ninja* CMakeCache.txt fi -export CC=$(which clang) -export CXX=$(which clang++) +export CC=$(which clang-16) +export CXX=$(which clang++-16) export CCACHE_DISABLE=1 if [ ! -d "lz4" ]; then From e3a8a770abdc599833a8e8e4609689bb4d76443b Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 19 Mar 2025 13:45:42 +0300 Subject: [PATCH 168/388] Fix null reference error in ValidatorManagerImpl::prepare_stats (#1561) --- validator/manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/manager.cpp b/validator/manager.cpp index b0ac54092..00b923c23 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2891,7 +2891,7 @@ void ValidatorManagerImpl::prepare_stats(td::Promiseget_state_serializer_enabled(); - if (is_validator() && last_masterchain_state_->get_global_id() == -239) { + if (is_validator() && last_masterchain_state_.not_null() && last_masterchain_state_->get_global_id() == -239) { serializer_enabled = false; } vec.emplace_back("stateserializerenabled", serializer_enabled ? "true" : "false"); From 79aad724cf453db30717767375a0492a63f351b0 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 20 Mar 2025 19:10:57 +0300 Subject: [PATCH 169/388] Add missing stop() to ArchiveImporter --- validator/import-db-slice.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/validator/import-db-slice.cpp b/validator/import-db-slice.cpp index 06573d347..3e1392bef 100644 --- a/validator/import-db-slice.cpp +++ b/validator/import-db-slice.cpp @@ -513,6 +513,7 @@ void ArchiveImporter::abort_query(td::Status error) { td::unlink(f).ignore(); } promise_.set_error(std::move(error)); + stop(); return; } LOG(INFO) << "Archive import: " << error; From 31b351c82accac0fd6f4213b2973559b71943a98 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Thu, 20 Mar 2025 19:37:27 -0400 Subject: [PATCH 170/388] Implement SHA256 hasher using the deprecated SHA256_* function family SHA256_* family avoids a (gigantic) overhead of OpenSSL's EVP interface. Locally, this gives 6% performance boost for block verification. --- crypto/openssl/digest.hpp | 57 +++++++++++++++++++++++++++++++++++- crypto/vm/cells/DataCell.cpp | 18 +++++------- 2 files changed, 64 insertions(+), 11 deletions(-) diff --git a/crypto/openssl/digest.hpp b/crypto/openssl/digest.hpp index 2adeef1d7..3e36f7e99 100644 --- a/crypto/openssl/digest.hpp +++ b/crypto/openssl/digest.hpp @@ -21,6 +21,7 @@ #include #include +#include #include "td/utils/Slice.h" @@ -124,9 +125,63 @@ std::string HashCtx::extract() { } typedef HashCtx SHA1; -typedef HashCtx SHA256; typedef HashCtx SHA512; +struct SHA256Tag {}; + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + +template <> +struct HashCtx { + public: + enum { digest_bytes = 32 }; + + HashCtx() { + SHA256_Init(&ctx_); + } + + HashCtx(const void *data, std::size_t len) : HashCtx() { + feed(data, len); + } + + void reset() { + SHA256_Init(&ctx_); + } + + void feed(const void *data, std::size_t len) { + SHA256_Update(&ctx_, data, len); + } + + void feed(td::Slice slice) { + feed(slice.data(), slice.size()); + } + + std::size_t extract(unsigned char buffer[digest_bytes]) { + SHA256_Final(buffer, &ctx_); + return digest_bytes; + } + + std::size_t extract(td::MutableSlice slice) { + CHECK(slice.size() == digest_bytes); + SHA256_Final(slice.ubegin(), &ctx_); + return digest_bytes; + } + + std::string extract() { + unsigned char buffer[digest_bytes]; + extract(buffer); + return std::string((char *)buffer, digest_bytes); + } + + private: + SHA256_CTX ctx_; +}; + +#pragma GCC diagnostic pop + +typedef HashCtx SHA256; + template std::size_t hash_str(unsigned char buffer[T::digest_bytes], const void *data, std::size_t size) { T hasher(data, size); diff --git a/crypto/vm/cells/DataCell.cpp b/crypto/vm/cells/DataCell.cpp index 4dd301616..e5f21b013 100644 --- a/crypto/vm/cells/DataCell.cpp +++ b/crypto/vm/cells/DataCell.cpp @@ -268,18 +268,16 @@ td::Result> DataCell::create(td::ConstBitPtr data, unsigned bits, tmp[0] = info.d1(level_mask.apply(level_i)); tmp[1] = info.d2(); - static TD_THREAD_LOCAL digest::SHA256* hasher; - td::init_thread_local(hasher); - hasher->reset(); + digest::SHA256 hasher; - hasher->feed(td::Slice(tmp, 2)); + hasher.feed(td::Slice(tmp, 2)); if (hash_i == hash_i_offset) { DCHECK(level_i == 0 || type == SpecialType::PrunnedBranch); - hasher->feed(td::Slice(data_ptr, (bits + 7) >> 3)); + hasher.feed(td::Slice(data_ptr, (bits + 7) >> 3)); } else { DCHECK(level_i != 0 && type != SpecialType::PrunnedBranch); - hasher->feed(hashes_ptr[hash_i - hash_i_offset - 1].as_slice()); + hasher.feed(hashes_ptr[hash_i - hash_i_offset - 1].as_slice()); } auto dest_i = hash_i - hash_i_offset; @@ -297,7 +295,7 @@ td::Result> DataCell::create(td::ConstBitPtr data, unsigned bits, // add depth into hash td::uint8 child_depth_buf[depth_bytes]; store_depth(child_depth_buf, child_depth); - hasher->feed(td::Slice(child_depth_buf, depth_bytes)); + hasher.feed(td::Slice(child_depth_buf, depth_bytes)); depth = std::max(depth, child_depth); } @@ -312,12 +310,12 @@ td::Result> DataCell::create(td::ConstBitPtr data, unsigned bits, // children hash for (int i = 0; i < info.refs_count_; i++) { if (type == SpecialType::MerkleProof || type == SpecialType::MerkleUpdate) { - hasher->feed(refs_ptr[i]->get_hash(level_i + 1).as_slice()); + hasher.feed(refs_ptr[i]->get_hash(level_i + 1).as_slice()); } else { - hasher->feed(refs_ptr[i]->get_hash(level_i).as_slice()); + hasher.feed(refs_ptr[i]->get_hash(level_i).as_slice()); } } - auto extracted_size = hasher->extract(hashes_ptr[dest_i].as_slice()); + auto extracted_size = hasher.extract(hashes_ptr[dest_i].as_slice()); DCHECK(extracted_size == hash_bytes); } From 513a60fec5374ef19f3018d4ef81d25ef149a48b Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Sat, 22 Mar 2025 13:51:15 +0300 Subject: [PATCH 171/388] Cache validator sets for create_shard_state --- crypto/block/mc-config.cpp | 71 ++++++++++++++++++++++++++++++-- crypto/block/mc-config.h | 4 +- lite-client/lite-client.h | 2 +- validator/impl/shard.cpp | 4 +- validator/impl/validator-set.hpp | 2 +- 5 files changed, 73 insertions(+), 10 deletions(-) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 0f019b068..7b4f326f8 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -38,6 +38,7 @@ #include "openssl/digest.hpp" #include #include +#include namespace block { using namespace std::literals::string_literals; @@ -253,7 +254,7 @@ td::Status Config::unpack() { } config_dict = std::make_unique(config_root, 32); if (mode & needValidatorSet) { - auto vset_res = unpack_validator_set(get_config_param(35, 34)); + auto vset_res = unpack_validator_set(get_config_param(35, 34), true); if (vset_res.is_error()) { return vset_res.move_as_error(); } @@ -486,15 +487,74 @@ td::Result Config::unpack_workchain_list(Ref root) { return std::move(pair.first); } -td::Result> Config::unpack_validator_set(Ref vset_root) { +class ValidatorSetCache { +public: + ValidatorSetCache() { + cache_.reserve(MAX_CACHE_SIZE + 1); + } + + std::shared_ptr get(const vm::CellHash& hash) { + std::lock_guard lock{mutex_}; + auto it = cache_.find(hash); + if (it == cache_.end()) { + return {}; + } + auto entry = it->second.get(); + entry->remove(); + lru_.put(entry); + return entry->value; + } + + void set(const vm::CellHash& hash, std::shared_ptr vset) { + std::lock_guard lock{mutex_}; + std::unique_ptr& entry = cache_[hash]; + if (entry == nullptr) { + entry = std::make_unique(hash, std::move(vset)); + } else { + entry->value = std::move(vset); + entry->remove(); + } + lru_.put(entry.get()); + if (cache_.size() > MAX_CACHE_SIZE) { + auto to_remove = (CacheEntry*)lru_.get(); + CHECK(to_remove); + to_remove->remove(); + cache_.erase(to_remove->key); + } + } + +private: + std::mutex mutex_; + + struct CacheEntry : td::ListNode { + explicit CacheEntry(vm::CellHash key, std::shared_ptr value) : key(key), value(std::move(value)) { + } + vm::CellHash key; + std::shared_ptr value; + }; + td::HashMap> cache_; + td::ListNode lru_; + + static constexpr size_t MAX_CACHE_SIZE = 100; +}; + +td::Result> Config::unpack_validator_set(Ref vset_root, bool use_cache) { if (vset_root.is_null()) { return td::Status::Error("validator set is absent"); } + static ValidatorSetCache cache; + if (use_cache) { + auto result = cache.get(vset_root->get_hash()); + if (result) { + return result; + } + } + gen::ValidatorSet::Record_validators_ext rec; Ref dict_root; if (!tlb::unpack_cell(vset_root, rec)) { gen::ValidatorSet::Record_validators rec0; - if (!tlb::unpack_cell(std::move(vset_root), rec0)) { + if (!tlb::unpack_cell(vset_root, rec0)) { return td::Status::Error("validator set is invalid"); } rec.utime_since = rec0.utime_since; @@ -515,7 +575,7 @@ td::Result> Config::unpack_validator_set(Ref(rec.utime_since, rec.utime_until, rec.total, rec.main); + auto ptr = std::make_shared(rec.utime_since, rec.utime_until, rec.total, rec.main); for (int i = 0; i < rec.total; i++) { key_buffer.store_ulong(i); auto descr_cs = dict.lookup(key_buffer.bits(), 16); @@ -548,6 +608,9 @@ td::Result> Config::unpack_validator_set(Reftotal_weight) { return td::Status::Error("validator set declares incorrect total weight"); } + if (use_cache) { + cache.set(vset_root->get_hash(), ptr); + } return std::move(ptr); } diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 98e6a26df..a44522f11 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -559,7 +559,7 @@ class Config { td::BitArray<256> config_addr; Ref config_root; std::unique_ptr config_dict; - std::unique_ptr cur_validators_; + std::shared_ptr cur_validators_; std::unique_ptr workchains_dict_; WorkchainSet workchains_; int version_{-1}; @@ -623,7 +623,7 @@ class Config { bool set_block_id_ext(const ton::BlockIdExt& block_id_ext); td::Result> get_special_smartcontracts(bool without_config = false) const; bool is_special_smartcontract(const ton::StdSmcAddress& addr) const; - static td::Result> unpack_validator_set(Ref valset_root); + static td::Result> unpack_validator_set(Ref valset_root, bool use_cache = false); td::Result> get_storage_prices() const; static td::Result do_get_one_storage_prices(vm::CellSlice cs); td::Result get_gas_limits_prices(bool is_masterchain = false) const; diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index 721d2b20d..3284f255d 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -143,7 +143,7 @@ class TestNode : public td::actor::Actor { ton::LogicalTime end_lt{0}; ton::Bits256 vset_hash; Ref vset_root; - std::unique_ptr vset; + std::shared_ptr vset; std::map vset_map; int special_idx{-1}; std::pair created_total, created_special; diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index 0ab216f71..d2d402dd6 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -390,12 +390,12 @@ td::Status MasterchainStateQ::mc_reinit() { auto cv_root = config_->get_config_param(35, 34); if (cv_root.not_null()) { - TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(cv_root))); + TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(cv_root), true)); cur_validators_ = std::move(validators); } auto nv_root = config_->get_config_param(37, 36); if (nv_root.not_null()) { - TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(nv_root))); + TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(nv_root), true)); next_validators_ = std::move(validators); } diff --git a/validator/impl/validator-set.hpp b/validator/impl/validator-set.hpp index 951ca4b71..81c0bd518 100644 --- a/validator/impl/validator-set.hpp +++ b/validator/impl/validator-set.hpp @@ -74,7 +74,7 @@ class ValidatorSetCompute { private: const block::Config* config_{nullptr}; - std::unique_ptr cur_validators_, next_validators_; + std::shared_ptr cur_validators_, next_validators_; td::Ref compute_validator_set(ShardIdFull shard, const block::ValidatorSet& vset, UnixTime time, CatchainSeqno seqno) const; }; From b304371f538e489affdb580e53df50ce4b058079 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Sat, 22 Mar 2025 13:56:52 +0300 Subject: [PATCH 172/388] Permanent celldb, improve importing archive slices, new flags for validator-engine --- crypto/vm/db/CellStorage.cpp | 33 +- crypto/vm/db/CellStorage.h | 3 +- validator-engine/validator-engine.cpp | 30 +- validator-engine/validator-engine.hpp | 12 + validator/CMakeLists.txt | 4 + validator/db/celldb.cpp | 204 +++++- validator/db/celldb.hpp | 8 + .../permanent-celldb-utils.cpp | 83 +++ .../permanent-celldb/permanent-celldb-utils.h | 37 + validator/db/rootdb.cpp | 35 + validator/db/rootdb.hpp | 4 + validator/downloaders/wait-block-state.cpp | 28 +- validator/downloaders/wait-block-state.hpp | 11 +- validator/import-db-slice-local.cpp | 635 ++++++++++++++++++ validator/import-db-slice-local.hpp | 103 +++ validator/import-db-slice.cpp | 2 +- validator/interfaces/db.h | 4 + validator/interfaces/validator-manager.h | 4 + validator/manager-disk.cpp | 14 +- validator/manager-disk.hpp | 4 + validator/manager-hardfork.cpp | 4 +- validator/manager-hardfork.hpp | 7 + validator/manager.cpp | 61 +- validator/manager.hpp | 3 + validator/validator-options.cpp | 15 +- validator/validator-options.hpp | 18 +- validator/validator.h | 6 +- 27 files changed, 1304 insertions(+), 68 deletions(-) create mode 100644 validator/db/permanent-celldb/permanent-celldb-utils.cpp create mode 100644 validator/db/permanent-celldb/permanent-celldb-utils.h create mode 100644 validator/import-db-slice-local.cpp create mode 100644 validator/import-db-slice-local.hpp diff --git a/crypto/vm/db/CellStorage.cpp b/crypto/vm/db/CellStorage.cpp index a07d85e87..0c8ba179a 100644 --- a/crypto/vm/db/CellStorage.cpp +++ b/crypto/vm/db/CellStorage.cpp @@ -32,8 +32,9 @@ namespace { class RefcntCellStorer { public: - RefcntCellStorer(td::int32 refcnt, const td::Ref &cell, bool as_boc) - : refcnt_(refcnt), cell_(cell), as_boc_(as_boc) { + RefcntCellStorer(td::int32 refcnt, const td::Ref &cell, bool as_boc, int max_level = vm::Cell::max_level) + : refcnt_(refcnt), cell_(cell), as_boc_(as_boc), max_level_(max_level) { + CHECK(!as_boc_ || max_level_ == vm::Cell::max_level); } template @@ -51,10 +52,25 @@ class RefcntCellStorer { CHECK(refcnt_ > 0); store(refcnt_, storer); CHECK(cell_.not_null()) - store(*cell_, storer); + if (max_level_ == vm::Cell::max_level) { + store(*cell_, storer); + } else { + auto level_mask = cell_->get_level_mask().apply(max_level_); + storer.template store_binary( + static_cast(cell_->get_refs_cnt() + 8 * cell_->is_special() + 32 * level_mask.get_mask())); + auto d2 = static_cast((cell_->get_bits() / 8) * 2); + if ((cell_->get_bits() & 7) != 0) { + d2 = static_cast(d2 + 1); + } + storer.template store_binary(d2); + storer.store_slice(td::Slice(cell_->get_data(), (cell_->get_bits() + 7) / 8)); + } + for (unsigned i = 0; i < cell_->size_refs(); i++) { auto cell = cell_->get_ref(i); - auto level_mask = cell->get_level_mask(); + auto level_mask = + cell->get_level_mask().apply(max_level_ + (cell_->special_type() == CellTraits::SpecialType::MerkleProof || + cell_->special_type() == CellTraits::SpecialType::MerkleUpdate)); auto level = level_mask.get_level(); td::uint8 x = static_cast(level_mask.get_mask()); storer.store_slice(td::Slice(&x, 1)); @@ -79,6 +95,7 @@ class RefcntCellStorer { td::int32 refcnt_; td::Ref cell_; bool as_boc_; + int max_level_; }; class RefcntCellParser { @@ -120,13 +137,13 @@ class RefcntCellParser { Ref refs[Cell::max_refs]; for (int i = 0; i < info.refs_cnt; i++) { if (data.size() < 1) { - return td::Status::Error("Not enought data"); + return td::Status::Error("Not enough data"); } Cell::LevelMask level_mask(data[0]); auto n = level_mask.get_hashes_count(); auto end_offset = 1 + n * (Cell::hash_bytes + Cell::depth_bytes); if (data.size() < end_offset) { - return td::Status::Error("Not enought data"); + return td::Status::Error("Not enough data"); } TRY_RESULT(ext_cell, ext_cell_creator.ext_cell(level_mask, data.substr(1, n * Cell::hash_bytes), @@ -221,8 +238,8 @@ td::Status CellStorer::erase(td::Slice hash) { return kv_.erase(hash); } -std::string CellStorer::serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc) { - return td::serialize(RefcntCellStorer(refcnt, cell, as_boc)); +std::string CellStorer::serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc, int max_level) { + return td::serialize(RefcntCellStorer(refcnt, cell, as_boc, max_level)); } td::Status CellStorer::set(td::int32 refcnt, const td::Ref &cell, bool as_boc) { diff --git a/crypto/vm/db/CellStorage.h b/crypto/vm/db/CellStorage.h index ca32a8007..fe1d6b250 100644 --- a/crypto/vm/db/CellStorage.h +++ b/crypto/vm/db/CellStorage.h @@ -71,7 +71,8 @@ class CellStorer { static void merge_refcnt_diffs(std::string &left, td::Slice right); static std::string serialize_refcnt_diffs(td::int32 refcnt_diff); - static std::string serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc); + static std::string serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc, + int max_level = vm::Cell::max_level); struct Diff { enum Type { Set, Erase, Merge } type{Set}; diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index cbcc3ab1f..273ae3ba3 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1488,6 +1488,8 @@ td::Status ValidatorEngine::load_global_config() { if (catchain_max_block_delay_slow_) { validator_options_.write().set_catchain_max_block_delay_slow(catchain_max_block_delay_slow_.value()); } + validator_options_.write().set_permanent_celldb(permanent_celldb_); + validator_options_.write().set_initial_sync_disabled(skip_key_sync_); std::vector h; for (auto &x : conf.validator_->hardforks_) { @@ -1515,16 +1517,23 @@ td::Status ValidatorEngine::load_global_config() { void ValidatorEngine::set_shard_check_function() { if (!not_all_shards_) { - validator_options_.write().set_shard_check_function([](ton::ShardIdFull shard) -> bool { return true; }); + validator_options_.write().set_shard_check_function( + [sync_shards_upto = sync_shards_upto_](ton::ShardIdFull shard, ton::BlockSeqno mc_seqno) -> bool { + return shard.is_masterchain() || !sync_shards_upto || mc_seqno <= sync_shards_upto.value(); + }); } else { std::vector shards = {ton::ShardIdFull(ton::masterchainId)}; - for (const auto& s : config_.shards_to_monitor) { + for (const auto &s : config_.shards_to_monitor) { shards.push_back(s); } std::sort(shards.begin(), shards.end()); shards.erase(std::unique(shards.begin(), shards.end()), shards.end()); validator_options_.write().set_shard_check_function( - [shards = std::move(shards)](ton::ShardIdFull shard) -> bool { + [shards = std::move(shards), sync_shards_upto = sync_shards_upto_](ton::ShardIdFull shard, + ton::BlockSeqno mc_seqno) -> bool { + if (!shard.is_masterchain() && sync_shards_upto && mc_seqno > sync_shards_upto.value()) { + return false; + } for (auto s : shards) { if (shard_intersects(shard, s)) { return true; @@ -4613,6 +4622,21 @@ int main(int argc, char *argv[]) { [&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_broadcast_speed_multiplier_private, v); }); return td::Status::OK(); }); + p.add_option( + '\0', "permanent-celldb", + "disable garbage collection in CellDb. This improves performance on archival nodes (once enabled, this option " + "cannot be disabled)", + [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_permanent_celldb, true); }); }); + p.add_option('\0', "skip-key-sync", + "don't select the best persistent state on initial sync, start on init_block from global config", [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_skip_key_sync, true); }); + }); + p.add_checked_option( + '\0', "sync-shards-upto", "stop syncing shards on this masterchain seqno", [&](td::Slice s) -> td::Status { + TRY_RESULT(v, td::to_integer_safe(s)); + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_sync_shards_upto, v); }); + return td::Status::OK(); + }); auto S = p.run(argc, argv); if (S.is_error()) { LOG(ERROR) << "failed to parse options: " << S.move_as_error(); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 5a1db7f3f..dceb3503a 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -233,6 +233,9 @@ class ValidatorEngine : public td::actor::Actor { double broadcast_speed_multiplier_catchain_ = 1.0; double broadcast_speed_multiplier_public_ = 1.0; double broadcast_speed_multiplier_private_ = 1.0; + bool permanent_celldb_ = false; + bool skip_key_sync_ = false; + td::optional sync_shards_upto_; std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; @@ -345,6 +348,15 @@ class ValidatorEngine : public td::actor::Actor { void set_broadcast_speed_multiplier_private(double value) { broadcast_speed_multiplier_private_ = value; } + void set_permanent_celldb(bool value) { + permanent_celldb_ = value; + } + void set_skip_key_sync(bool value) { + skip_key_sync_ = value; + } + void set_sync_shards_upto(ton::BlockSeqno seqno) { + sync_shards_upto_ = seqno; + } void start_up() override; ValidatorEngine() { diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index f7a4ce1d1..9fb0d190f 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -14,6 +14,8 @@ set(VALIDATOR_DB_SOURCE db/archive-slice.hpp db/celldb.cpp db/celldb.hpp + db/permanent-celldb/permanent-celldb-utils.cpp + db/permanent-celldb/permanent-celldb-utils.h db/files-async.hpp db/fileref.hpp db/fileref.cpp @@ -54,6 +56,7 @@ set(VALIDATOR_HEADERS invariants.hpp import-db-slice.hpp + import-db-slice-local.hpp queue-size-counter.hpp validator-telemetry.hpp @@ -74,6 +77,7 @@ set(VALIDATOR_SOURCE block-handle.cpp get-next-key-blocks.cpp import-db-slice.cpp + import-db-slice-local.cpp shard-client.cpp state-serializer.cpp token-manager.cpp diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 90c659cc4..65a60af3f 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -27,6 +27,9 @@ #include "ton/ton-tl.hpp" #include "ton/ton-io.hpp" #include "common/delay.h" +#include "block/block-auto.h" +#include "permanent-celldb/permanent-celldb-utils.h" +#include "td/actor/MultiPromise.h" #include #include @@ -324,6 +327,30 @@ void CellDbIn::start_up() { last_deleted_mc_state_ = r_value.move_as_ok(); } } + { + std::string key = "opts.permanent_mode", value; + auto R = boc_->meta_get(td::as_slice(key), value); + R.ensure(); + bool stored_permanent_mode = R.ok() == td::KeyValue::GetStatus::Ok; + permanent_mode_ = stored_permanent_mode || opts_->get_permanent_celldb(); + if (permanent_mode_) { + LOG(WARNING) << "Celldb is in permanent mode"; + if (!stored_permanent_mode) { + cell_db_->begin_write_batch().ensure(); + value = "1"; + vm::CellStorer stor{*cell_db_}; + boc_->meta_set(td::as_slice(key), td::as_slice(value)); + boc_->commit(stor).ensure(); + cell_db_->commit_write_batch().ensure(); + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + } + } + } + cell_db_statistics_.permanent_mode_ = permanent_mode_; + LOG_IF(FATAL, permanent_mode_ && opts_->get_celldb_in_memory()) + << "celldb permanent_mode and in_memory_mode are not compatible"; + } } void CellDbIn::load_cell(RootHash hash, td::Promise> promise) { @@ -359,7 +386,9 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi auto R = get_block(key_hash); // duplicate if (R.is_ok()) { - promise.set_result(boc_->load_cell(cell->get_hash().as_slice())); + delay_action([cell = boc_->load_cell(cell->get_hash().as_slice()), + promise = std::move(promise)]() mutable { promise.set_result(std::move(cell)); }, + td::Timestamp::now()); return; } @@ -370,7 +399,7 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi cell = std::move(cell)](td::Result Res) mutable { Res.ensure(); timer_prepare.pause(); - td::actor::send_lambda( + td::actor::send_lambda_later( SelfId, [=, this, timer = std::move(timer), promise = std::move(promise), cell = std::move(cell)]() mutable { TD_PERF_COUNTER(celldb_store_cell); auto empty = get_empty_key_hash(); @@ -407,7 +436,9 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); } - promise.set_result(boc_->load_cell(cell->get_hash().as_slice())); + delay_action([cell = boc_->load_cell(cell->get_hash().as_slice()), + promise = std::move(promise)]() mutable { promise.set_result(std::move(cell)); }, + td::Timestamp::now()); if (!opts_->get_disable_rocksdb_stats()) { cell_db_statistics_.store_cell_time_.insert(timer.elapsed() * 1e6); cell_db_statistics_.store_cell_prepare_time_.insert(timer_prepare.elapsed() * 1e6); @@ -430,6 +461,149 @@ void CellDbIn::get_cell_db_reader(td::Promise> promise.set_result(boc_->get_cell_db_reader()); } +void CellDbIn::store_block_state_permanent(td::Ref block, td::Promise> promise) { + if (!permanent_mode_) { + promise.set_error(td::Status::Error("celldb is not in permanent mode")); + return; + } + if (db_busy_) { + action_queue_.push( + [self = this, block = std::move(block), promise = std::move(promise)](td::Result R) mutable { + R.ensure(); + self->store_block_state_permanent(std::move(block), std::move(promise)); + }); + return; + } + auto key_hash = get_key_hash(block->block_id()); + auto R = get_block(key_hash); + // duplicate + if (R.is_ok()) { + delay_action([cell = boc_->load_cell(R.ok().root_hash.as_slice()), + promise = std::move(promise)]() mutable { promise.set_result(std::move(cell)); }, + td::Timestamp::now()); + return; + } + store_block_state_permanent_bulk( + {block}, [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_STATUS_PROMISE(promise, R.move_as_status()); + block::gen::Block::Record rec; + if (!block::gen::unpack_cell(block->root_cell(), rec)) { + promise.set_error(td::Status::Error("cannot unpack Block record")); + return; + } + bool spec; + vm::CellSlice update_cs = vm::load_cell_slice_special(rec.state_update, spec); + if (update_cs.special_type() != vm::CellTraits::SpecialType::MerkleUpdate) { + promise.set_error(td::Status::Error("invalid Merkle update in block")); + return; + } + td::Ref new_state_root = update_cs.prefetch_ref(1); + RootHash state_root_hash = new_state_root->get_hash(0).bits(); + td::actor::send_closure(SelfId, &CellDbIn::load_cell, state_root_hash, std::move(promise)); + }); +} + +void CellDbIn::store_block_state_permanent_bulk(std::vector> blocks, td::Promise promise) { + if (!permanent_mode_) { + promise.set_error(td::Status::Error("celldb is not in permanent mode")); + return; + } + if (db_busy_) { + action_queue_.push( + [self = this, blocks = std::move(blocks), promise = std::move(promise)](td::Result R) mutable { + R.ensure(); + self->store_block_state_permanent_bulk(std::move(blocks), std::move(promise)); + }); + return; + } + td::PerfWarningTimer timer{"storecellbulk", 0.1}; + td::Timer timer_prepare; + std::map> new_blocks; + for (auto& block : blocks) { + BlockIdExt block_id = block->block_id(); + if (new_blocks.contains(block_id)) { + continue; + } + if (get_block(get_key_hash(block_id)).is_ok()) { + continue; + } + new_blocks[block_id] = std::move(block); + } + if (new_blocks.empty()) { + promise.set_value(td::Unit{}); + return; + } + for (auto& [block_id, block] : new_blocks) { + std::vector prev; + BlockIdExt mc_blkid; + bool after_split; + TRY_STATUS_PROMISE(promise, + block::unpack_block_prev_blk_try(block->root_cell(), block_id, prev, mc_blkid, after_split)); + for (const BlockIdExt& prev_id : prev) { + if (!new_blocks.contains(prev_id) && get_block(get_key_hash(prev_id)).is_error()) { + promise.set_error(td::Status::Error("cannot store block state: previous block is not in db")); + return; + } + } + } + db_busy_ = true; + calculate_permanent_celldb_update( + new_blocks, async_executor, + [this, SelfId = actor_id(this), timer = std::move(timer), timer_prepare = std::move(timer_prepare), + promise = std::move(promise)](td::Result> R) mutable { + TRY_RESULT_PROMISE(promise, updates, std::move(R)); + td::actor::send_lambda_later( + SelfId, [=, this, timer = std::move(timer), timer_prepare = std::move(timer_prepare), + updates = std::move(updates), promise = std::move(promise)]() mutable { + TD_PERF_COUNTER(celldb_store_cell_multi); + timer_prepare.pause(); + td::Timer timer_write; + vm::CellStorer stor{*cell_db_}; + cell_db_->begin_write_batch().ensure(); + + for (auto& update : updates) { + for (auto& [k, v] : update.to_store) { + cell_db_->set(k.as_slice(), v).ensure(); + } + } + + CHECK(!updates.empty()); + auto empty = get_empty_key_hash(); + auto E = get_block(empty).move_as_ok(); + for (size_t i = 0; i < updates.size(); ++i) { + KeyHash prev = i == 0 ? empty : get_key_hash(updates[i - 1].block_id); + KeyHash next = i + 1 == updates.size() ? E.next : get_key_hash(updates[i + 1].block_id); + DbEntry entry{updates[i].block_id, prev, next, updates[i].state_root_hash}; + set_block(get_key_hash(updates[i].block_id), std::move(entry)); + } + E.next = get_key_hash(updates[0].block_id); + if (E.prev == empty) { + E.prev = get_key_hash(updates.back().block_id); + } + set_block(empty, std::move(E)); + + boc_->commit(stor).ensure(); // Save meta + cell_db_->commit_write_batch().ensure(); + timer_write.pause(); + + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); + } + + if (!opts_->get_disable_rocksdb_stats()) { + cell_db_statistics_.store_cell_time_.insert(timer.elapsed() * 1e6); + cell_db_statistics_.store_cell_prepare_time_.insert(timer_prepare.elapsed() * 1e6); + cell_db_statistics_.store_cell_write_time_.insert(timer_write.elapsed() * 1e6); + cell_db_statistics_.store_cell_bulk_queries_++; + cell_db_statistics_.store_cell_bulk_total_blocks_ += updates.size(); + } + release_db(); + promise.set_result(td::Unit()); + }); + }); +} + std::vector> CellDbIn::prepare_stats() { TD_PERF_COUNTER(celldb_prepare_stats); auto r_boc_stats = boc_->get_stats(); @@ -497,6 +671,7 @@ void CellDbIn::flush_db_stats() { } td::RocksDb::reset_statistics(statistics_); cell_db_statistics_.clear(); + cell_db_statistics_.permanent_mode_ = permanent_mode_; } void CellDbIn::alarm() { @@ -515,6 +690,10 @@ void CellDbIn::alarm() { << " queue_size=" << cells_to_migrate_.size(); migration_stats_ = {}; } + if (permanent_mode_) { + skip_gc(); + return; + } auto E = get_block(get_empty_key_hash()).move_as_ok(); auto N = get_block(E.next).move_as_ok(); if (N.is_empty()) { @@ -568,6 +747,7 @@ void CellDbIn::gc_cont2(BlockHandle handle) { }); return; } + CHECK(!permanent_mode_); td::PerfWarningTimer timer{"gccell", 0.1}; td::PerfWarningTimer timer_all{"gccell_all", 0.05}; @@ -610,7 +790,7 @@ void CellDbIn::gc_cont2(BlockHandle handle) { P = std::move(P), N = std::move(N), cell = std::move(cell), timer = std::move(timer), timer_all = std::move(timer_all), handle](td::Result R) mutable { R.ensure(); - td::actor::send_lambda(SelfId, [this, timer_boc = std::move(timer_boc), F = std::move(F), key_hash, + td::actor::send_lambda_later(SelfId, [this, timer_boc = std::move(timer_boc), F = std::move(F), key_hash, P = std::move(P), N = std::move(N), cell = std::move(cell), timer = std::move(timer), timer_all = std::move(timer_all), handle]() mutable { TD_PERF_COUNTER(celldb_gc_cell); @@ -710,6 +890,9 @@ void CellDbIn::set_block(KeyHash key_hash, DbEntry e) { } void CellDbIn::migrate_cell(td::Bits256 hash) { + if (permanent_mode_) { + return; + } cells_to_migrate_.insert(hash); if (!migration_active_) { migration_active_ = true; @@ -823,6 +1006,14 @@ void CellDb::store_cell(BlockIdExt block_id, td::Ref cell, td::Promise td::actor::send_closure(cell_db_, &CellDbIn::store_cell, block_id, std::move(cell), std::move(promise)); } +void CellDb::store_block_state_permanent(td::Ref block, td::Promise> promise) { + td::actor::send_closure(cell_db_, &CellDbIn::store_block_state_permanent, std::move(block), std::move(promise)); +} + +void CellDb::store_block_state_permanent_bulk(std::vector> blocks, td::Promise promise) { + td::actor::send_closure(cell_db_, &CellDbIn::store_block_state_permanent_bulk, std::move(blocks), std::move(promise)); +} + void CellDb::get_cell_db_reader(td::Promise> promise) { td::actor::send_closure(cell_db_, &CellDbIn::get_cell_db_reader, std::move(promise)); } @@ -859,9 +1050,14 @@ td::BufferSlice CellDbIn::DbEntry::release() { std::vector> CellDbIn::CellDbStatistics::prepare_stats() { std::vector> stats; + stats.emplace_back("permanent_mode", PSTRING() << permanent_mode_); stats.emplace_back("store_cell.micros", PSTRING() << store_cell_time_.to_string()); stats.emplace_back("store_cell.prepare.micros", PSTRING() << store_cell_prepare_time_.to_string()); stats.emplace_back("store_cell.write.micros", PSTRING() << store_cell_write_time_.to_string()); + if (permanent_mode_) { + stats.emplace_back("store_cell.bulk.queries", PSTRING() << store_cell_bulk_queries_); + stats.emplace_back("store_cell.bulk.total_blocks", PSTRING() << store_cell_bulk_total_blocks_); + } stats.emplace_back("gc_cell.micros", PSTRING() << gc_cell_time_.to_string()); stats.emplace_back("total_time.micros", PSTRING() << (td::Timestamp::now().at() - stats_start_time_.at()) * 1e6); stats.emplace_back("in_memory", PSTRING() << bool(in_memory_load_time_)); diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index 1e1ccddab..f70f6aef8 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -66,6 +66,8 @@ class CellDbIn : public CellDbBase { void load_cell(RootHash hash, td::Promise> promise); void store_cell(BlockIdExt block_id, td::Ref cell, td::Promise> promise); void get_cell_db_reader(td::Promise> promise); + void store_block_state_permanent(td::Ref block, td::Promise> promise); + void store_block_state_permanent_bulk(std::vector> blocks, td::Promise promise); void migrate_cell(td::Bits256 hash); @@ -137,9 +139,12 @@ class CellDbIn : public CellDbBase { std::unique_ptr migration_stats_; struct CellDbStatistics { + bool permanent_mode_; PercentileStats store_cell_time_; PercentileStats store_cell_prepare_time_; PercentileStats store_cell_write_time_; + size_t store_cell_bulk_queries_ = 0; + size_t store_cell_bulk_total_blocks_ = 0; PercentileStats gc_cell_time_; td::Timestamp stats_start_time_ = td::Timestamp::now(); std::optional in_memory_load_time_; @@ -156,6 +161,7 @@ class CellDbIn : public CellDbBase { CellDbStatistics cell_db_statistics_; td::Timestamp statistics_flush_at_ = td::Timestamp::never(); BlockSeqno last_deleted_mc_state_ = 0; + bool permanent_mode_ = false; bool db_busy_ = false; std::queue> action_queue_; @@ -188,6 +194,8 @@ class CellDb : public CellDbBase { void prepare_stats(td::Promise>> promise); void load_cell(RootHash hash, td::Promise> promise); void store_cell(BlockIdExt block_id, td::Ref cell, td::Promise> promise); + void store_block_state_permanent(td::Ref block, td::Promise> promise); + void store_block_state_permanent_bulk(std::vector> blocks, td::Promise promise); void update_snapshot(std::unique_ptr snapshot) { CHECK(!opts_->get_celldb_in_memory()); if (!started_) { diff --git a/validator/db/permanent-celldb/permanent-celldb-utils.cpp b/validator/db/permanent-celldb/permanent-celldb-utils.cpp new file mode 100644 index 000000000..b1d19ceb4 --- /dev/null +++ b/validator/db/permanent-celldb/permanent-celldb-utils.cpp @@ -0,0 +1,83 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "permanent-celldb-utils.h" +#include "block/block-auto.h" +#include "td/actor/MultiPromise.h" +#include "td/utils/HashMap.h" +#include "vm/db/CellStorage.h" + +namespace ton::validator { + +void calculate_permanent_celldb_update(const std::map>& blocks, + std::shared_ptr executor, + td::Promise> promise) { + td::MultiPromise mp; + auto ig = mp.init_guard(); + auto updates = std::make_shared>(); + updates->reserve(blocks.size()); + for (auto& [_, block] : blocks) { + executor->execute_async([block = block, updates, executor, + promise = std::make_shared>(ig.get_promise())]() mutable { + block::gen::Block::Record rec; + if (!block::gen::unpack_cell(block->root_cell(), rec)) { + promise->set_error(td::Status::Error("cannot unpack Block record")); + return; + } + bool spec; + vm::CellSlice update_cs = vm::load_cell_slice_special(rec.state_update, spec); + if (update_cs.special_type() != vm::CellTraits::SpecialType::MerkleUpdate) { + promise->set_error(td::Status::Error("invalid Merkle update in block")); + return; + } + td::Ref new_state_root = update_cs.prefetch_ref(1); + td::HashMap visited; + PermanentCellDbUpdate update{.block_id = block->block_id(), + .state_root_hash = new_state_root->get_hash(0).bits()}; + std::function&, int)> dfs = [&](const td::Ref& cell, int merkle_depth) { + int& vis = visited[cell->get_hash()]; + if (vis & (1 << merkle_depth)) { + return; + } + vis |= (1 << merkle_depth); + vm::CellSlice cs{vm::NoVm(), cell}; + if (cs.special_type() == vm::CellTraits::SpecialType::PrunnedBranch && cell->get_level() == merkle_depth + 1) { + return; + } + update.to_store.emplace_back( + cell->get_hash(merkle_depth), + vm::CellStorer::serialize_value(1 << 29, cell->load_cell().move_as_ok().data_cell, false, merkle_depth)); + merkle_depth = cs.child_merkle_depth(merkle_depth); + for (unsigned i = 0; i < cs.size_refs(); ++i) { + dfs(cs.prefetch_ref(i), merkle_depth); + } + }; + dfs(new_state_root, 0); + executor->execute_sync( + [update = std::move(update), updates = std::move(updates), promise = std::move(promise)]() mutable { + updates->push_back(std::move(update)); + promise->set_result(td::Unit()); + }); + }); + } + + ig.add_promise([updates, promise = std::move(promise)](td::Result R) mutable { + TRY_STATUS_PROMISE(promise, R.move_as_status()); + promise.set_value(std::move(*updates)); + }); +} + +} // namespace ton::validator diff --git a/validator/db/permanent-celldb/permanent-celldb-utils.h b/validator/db/permanent-celldb/permanent-celldb-utils.h new file mode 100644 index 000000000..fad2ddfcd --- /dev/null +++ b/validator/db/permanent-celldb/permanent-celldb-utils.h @@ -0,0 +1,37 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "vm/cells.h" +#include "ton/ton-types.h" +#include "interfaces/block.h" +#include "vm/db/DynamicBagOfCellsDb.h" + +#include +#include + +namespace ton::validator { + +struct PermanentCellDbUpdate { + BlockIdExt block_id; + RootHash state_root_hash; + std::vector> to_store; +}; +void calculate_permanent_celldb_update(const std::map>& blocks, + std::shared_ptr executor, + td::Promise> promise); + +} // namespace ton::validator diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 8d83e7a7d..7213e8d9a 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -259,6 +259,41 @@ void RootDb::store_block_state(BlockHandle handle, td::Ref state, } } +void RootDb::store_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) { + if (handle->id() != block->block_id()) { + promise.set_error(td::Status::Error("block id mismatch")); + return; + } + if (handle->moved_to_archive() || handle->inited_state_boc()) { + get_block_state(handle, std::move(promise)); + return; + } + auto P = td::PromiseCreator::lambda( + [b = archive_db_.get(), handle, promise = std::move(promise)](td::Result> R) mutable { + TRY_RESULT_PROMISE(promise, root, std::move(R)); + handle->set_state_root_hash(root->get_hash().bits()); + handle->set_state_boc(); + + auto S = create_shard_state(handle->id(), std::move(root)); + S.ensure(); + + auto P = td::PromiseCreator::lambda( + [promise = std::move(promise), state = S.move_as_ok()](td::Result R) mutable { + R.ensure(); + promise.set_value(std::move(state)); + }); + + td::actor::send_closure(b, &ArchiveManager::update_handle, std::move(handle), std::move(P)); + }); + td::actor::send_closure(cell_db_, &CellDb::store_block_state_permanent, std::move(block), std::move(P)); +} + +void RootDb::store_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) { + td::actor::send_closure(cell_db_, &CellDb::store_block_state_permanent_bulk, std::move(blocks), std::move(promise)); +} + void RootDb::get_block_state(ConstBlockHandle handle, td::Promise> promise) { if (handle->inited_state_boc()) { if (handle->deleted_state_boc()) { diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 52f6098e4..5e59e1a95 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -62,6 +62,10 @@ class RootDb : public Db { void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; + void store_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) override; + void store_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) override; void get_block_state(ConstBlockHandle handle, td::Promise> promise) override; void get_cell_db_reader(td::Promise> promise) override; diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index c80e7d896..6d05e654e 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -68,6 +68,8 @@ void WaitBlockState::start() { return; } bool inited_proof = handle_->id().is_masterchain() ? handle_->inited_proof() : handle_->inited_proof_link(); + bool allow_download = + last_masterchain_state_.is_null() || opts_->need_monitor(handle_->id().shard_full(), last_masterchain_state_); if (handle_->received_state() && inited_proof) { reading_from_db_ = true; @@ -81,8 +83,8 @@ void WaitBlockState::start() { td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle_, std::move(P)); } else if (handle_->id().id.seqno == 0 && next_static_file_attempt_.is_in_past()) { next_static_file_attempt_ = td::Timestamp::in(60.0); - // id.file_hash contrains correct file hash of zero state - // => if file with this sha256 is found it is garanteed to be correct + // id.file_hash contains correct file hash of zero state + // => if file with this sha256 is found it is guaranteed to be correct // => if it is not, this error is permanent auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), id = handle_->id()](td::Result R) { if (R.is_error()) { @@ -108,7 +110,7 @@ void WaitBlockState::start() { }); td::actor::send_closure(manager_, &ValidatorManager::send_get_zero_state_request, handle_->id(), priority_, std::move(P)); - } else if (check_persistent_state_desc() && !handle_->received_state()) { + } else if (check_persistent_state_desc() && !handle_->received_state() && allow_download) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { LOG(WARNING) << "failed to get persistent state: " << R.move_as_error(); @@ -122,7 +124,11 @@ void WaitBlockState::start() { timeout_, std::move(P)) .release(); } else if (!handle_->inited_prev() || (!handle_->inited_proof() && !handle_->inited_proof_link())) { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result R) { + if (!allow_download) { + abort_query(td::Status::Error(PSTRING() << "not monitoring shard " << handle_->id().shard_full())); + return; + } + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link); }, td::Timestamp::in(0.1)); @@ -148,6 +154,10 @@ void WaitBlockState::start() { td::actor::send_closure(manager_, &ValidatorManager::wait_prev_block_state, handle_, priority_, timeout_, std::move(P)); } else if (handle_->id().is_masterchain() && !handle_->inited_proof()) { + if (!allow_download) { + abort_query(td::Status::Error(PSTRING() << "not monitoring shard " << handle_->id().shard_full())); + return; + } auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result R) { if (R.is_error()) { delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof); }, @@ -161,6 +171,10 @@ void WaitBlockState::start() { td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_request, handle_->id(), priority_, std::move(P)); } else if (block_.is_null()) { + if (!allow_download && !handle_->received()) { + abort_query(td::Status::Error(PSTRING() << "not monitoring shard " << handle_->id().shard_full())); + return; + } auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &WaitBlockState::failed_to_get_block_data, @@ -245,6 +259,12 @@ void WaitBlockState::got_block_data(td::Ref data) { } void WaitBlockState::apply() { + if (opts_->get_permanent_celldb()) { + td::actor::send_closure(manager_, &ValidatorManager::set_block_state_from_data, handle_, block_, + std::move(promise_)); + stop(); + return; + } TD_PERF_COUNTER(apply_block_to_state); td::PerfWarningTimer t{"applyblocktostate", 0.1}; auto S = prev_state_.write().apply_block(handle_->id(), block_); diff --git a/validator/downloaders/wait-block-state.hpp b/validator/downloaders/wait-block-state.hpp index 6a14d909f..a9317381c 100644 --- a/validator/downloaders/wait-block-state.hpp +++ b/validator/downloaders/wait-block-state.hpp @@ -26,18 +26,21 @@ namespace validator { class WaitBlockState : public td::actor::Actor { public: - WaitBlockState(BlockHandle handle, td::uint32 priority, td::actor::ActorId manager, + WaitBlockState(BlockHandle handle, td::uint32 priority, td::Ref opts, + td::Ref last_masterchain_state, td::actor::ActorId manager, td::Timestamp timeout, td::Promise> promise, td::Ref persistent_state_desc = {}) : handle_(std::move(handle)) , priority_(priority) + , opts_(opts) + , last_masterchain_state_(last_masterchain_state) , manager_(manager) , timeout_(timeout) , promise_(std::move(promise)) , persistent_state_desc_(std::move(persistent_state_desc)) , perf_timer_("waitstate", 1.0, [manager](double duration) { - send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitstate", duration); - }) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitstate", duration); + }) { } void abort_query(td::Status reason); @@ -89,6 +92,8 @@ class WaitBlockState : public td::actor::Actor { td::uint32 priority_; + td::Ref opts_; + td::Ref last_masterchain_state_; td::actor::ActorId manager_; td::Timestamp timeout_; td::Promise> promise_; diff --git a/validator/import-db-slice-local.cpp b/validator/import-db-slice-local.cpp new file mode 100644 index 000000000..a16a26c63 --- /dev/null +++ b/validator/import-db-slice-local.cpp @@ -0,0 +1,635 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "import-db-slice-local.hpp" + +#include "validator/db/fileref.hpp" +#include "td/utils/overloaded.h" +#include "validator/fabric.h" +#include "td/actor/MultiPromise.h" +#include "common/checksum.h" +#include "td/utils/port/path.h" +#include "ton/ton-io.hpp" +#include "downloaders/download-state.hpp" +#include "block/block-auto.h" + +#include + +namespace ton { + +namespace validator { + +ArchiveImporterLocal::ArchiveImporterLocal(std::string db_root, td::Ref state, + BlockSeqno shard_client_seqno, td::Ref opts, + td::actor::ActorId manager, + std::vector to_import_files, + td::Promise> promise) + : db_root_(std::move(db_root)) + , last_masterchain_state_(std::move(state)) + , shard_client_seqno_(shard_client_seqno) + , opts_(std::move(opts)) + , manager_(manager) + , to_import_files_(std::move(to_import_files)) + , promise_(std::move(promise)) { +} + +void ArchiveImporterLocal::start_up() { + LOG(WARNING) << "Importing archive for masterchain seqno #" << shard_client_seqno_ + 1 << " from disk"; + for (const std::string &path : to_import_files_) { + LOG(INFO) << "Importing file from disk " << path; + td::Status S = process_package(path); + if (S.is_error()) { + LOG(WARNING) << "Error processing package " << path << ": " << S; + } + } + + process_masterchain_blocks(); +} + +td::Status ArchiveImporterLocal::process_package(std::string path) { + LOG(DEBUG) << "Processing package " << path; + TRY_RESULT(p, Package::open(path, false, false)); + auto package = std::make_shared(std::move(p)); + + td::Status S = td::Status::OK(); + package->iterate([&](std::string filename, td::BufferSlice data, td::uint64) -> bool { + auto F = FileReference::create(filename); + if (F.is_error()) { + S = F.move_as_error(); + return false; + } + auto f = F.move_as_ok(); + + BlockIdExt block_id; + bool is_proof = false; + bool ignore = true; + + f.ref().visit(td::overloaded( + [&](const fileref::Proof &p) { + block_id = p.block_id; + ignore = !block_id.is_masterchain(); + is_proof = true; + }, + [&](const fileref::ProofLink &p) { + block_id = p.block_id; + ignore = block_id.is_masterchain(); + is_proof = true; + }, + [&](const fileref::Block &p) { + block_id = p.block_id; + ignore = false; + is_proof = false; + }, + [&](const auto &) { ignore = true; })); + + if (ignore || block_id.is_masterchain() && block_id.seqno() <= last_masterchain_state_->get_seqno()) { + return true; + } + + if (is_proof) { + if (block_id.is_masterchain()) { + auto R = create_proof(block_id, std::move(data)); + if (R.is_error()) { + S = R.move_as_error(); + return false; + } + blocks_[block_id].proof = R.move_as_ok(); + } else { + auto R = create_proof_link(block_id, std::move(data)); + if (R.is_error()) { + S = R.move_as_error(); + return false; + } + blocks_[block_id].proof_link = R.move_as_ok(); + } + } else { + if (td::sha256_bits256(data) != block_id.file_hash) { + S = td::Status::Error(ErrorCode::protoviolation, "bad block file hash"); + return false; + } + auto R = create_block(block_id, std::move(data)); + if (R.is_error()) { + S = R.move_as_error(); + return false; + } + blocks_[block_id].block = R.move_as_ok(); + } + if (block_id.is_masterchain()) { + masterchain_blocks_[block_id.seqno()] = block_id; + } + return true; + }); + return S; +} + +void ArchiveImporterLocal::process_masterchain_blocks() { + if (masterchain_blocks_.empty()) { + LOG(INFO) << "No masterchain blocks in the archive"; + checked_masterchain_proofs(); + return; + } + + if (masterchain_blocks_.begin()->first != last_masterchain_state_->get_seqno() + 1) { + abort_query(td::Status::Error(ErrorCode::notready, PSTRING() << "expected masterchain seqno " + << last_masterchain_state_->get_seqno() + 1 + << ", found " << masterchain_blocks_.begin()->first)); + return; + } + { + BlockSeqno expected_seqno = last_masterchain_state_->get_seqno() + 1; + for (auto &[seqno, _] : masterchain_blocks_) { + if (seqno != expected_seqno) { + abort_query( + td::Status::Error(ErrorCode::protoviolation, "non-consequential masterchain blocks in the archive")); + return; + } + ++expected_seqno; + } + } + BlockInfo &first_block = blocks_[masterchain_blocks_.begin()->second]; + if (first_block.proof.is_null()) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block proof")); + return; + } + if (first_block.block.is_null()) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block data")); + return; + } + block::gen::Block::Record rec; + block::gen::BlockInfo::Record info; + if (!(block::gen::unpack_cell(first_block.block->root_cell(), rec) && block::gen::unpack_cell(rec.info, info))) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "cannot unpack masterchain block info")); + return; + } + if (info.key_block) { + import_first_key_block(); + return; + } + + process_masterchain_blocks_cont(); +} + +void ArchiveImporterLocal::import_first_key_block() { + BlockIdExt block_id = masterchain_blocks_.begin()->second; + BlockInfo &first_block = blocks_[block_id]; + LOG(INFO) << "First block in archive is key block : " << block_id.id.to_str(); + + auto P = + td::PromiseCreator::lambda([SelfId = actor_id(this), prev_block_id = last_masterchain_state_->get_block_id()]( + td::Result R) mutable { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + return; + } + auto handle = R.move_as_ok(); + CHECK(!handle->merge_before()); + if (handle->one_prev(true) != prev_block_id) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, + td::Status::Error(ErrorCode::protoviolation, "prev block mismatch")); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::checked_key_block_proof, std::move(handle)); + }); + + run_check_proof_query(block_id, first_block.proof, manager_, td::Timestamp::in(10.0), std::move(P), + last_masterchain_state_, opts_->is_hardfork(block_id)); +} + +void ArchiveImporterLocal::checked_key_block_proof(BlockHandle handle) { + BlockIdExt block_id = masterchain_blocks_.begin()->second; + CHECK(block_id == handle->id()); + BlockInfo &first_block = blocks_[block_id]; + run_apply_block_query( + handle->id(), first_block.block, handle->id(), manager_, td::Timestamp::in(600.0), + [SelfId = actor_id(this), manager = manager_, handle](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure( + manager, &ValidatorManager::get_shard_state_from_db, handle, [=](td::Result> R2) { + if (R2.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R2.move_as_error()); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::applied_key_block, + td::Ref{R2.move_as_ok()}); + }); + }); +} + +void ArchiveImporterLocal::applied_key_block(td::Ref state) { + CHECK(state->get_block_id() == masterchain_blocks_.begin()->second); + last_masterchain_state_ = state; + imported_any_ = true; + masterchain_blocks_.erase(masterchain_blocks_.begin()); + blocks_.erase(state->get_block_id()); + LOG(INFO) << "Imported key block " << state->get_block_id().id.to_str(); + if (masterchain_blocks_.empty()) { + LOG(INFO) << "No more masterchain blocks in the archive"; + checked_masterchain_proofs(); + return; + } + process_masterchain_blocks_cont(); +} + +void ArchiveImporterLocal::process_masterchain_blocks_cont() { + LOG(INFO) << "Importing masterchain blocks from " << masterchain_blocks_.begin()->first << " to " + << masterchain_blocks_.rbegin()->first; + + td::MultiPromise mp; + auto ig = mp.init_guard(); + + BlockIdExt prev_block_id = last_masterchain_state_->get_block_id(); + for (auto &[_, block_id] : masterchain_blocks_) { + auto &info = blocks_[block_id]; + info.import = true; + if (info.proof.is_null()) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block proof")); + return; + } + if (info.block.is_null()) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block data")); + return; + } + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), prev_block_id, promise = ig.get_promise()](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + CHECK(!handle->merge_before()); + if (handle->one_prev(true) != prev_block_id) { + promise.set_error(td::Status::Error(ErrorCode::protoviolation, "prev block mismatch")); + return; + } + promise.set_result(td::Unit()); + }); + run_check_proof_query(block_id, info.proof, manager_, td::Timestamp::in(10.0), std::move(P), + last_masterchain_state_, opts_->is_hardfork(block_id)); + prev_block_id = block_id; + } + ig.add_promise([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + } else { + LOG(INFO) << "Checked proofs for masterchain blocks"; + td::actor::send_closure(SelfId, &ArchiveImporterLocal::checked_masterchain_proofs); + } + }); +} + +void ArchiveImporterLocal::checked_masterchain_proofs() { + if (shard_client_seqno_ == last_masterchain_state_->get_seqno()) { + got_shard_client_state(last_masterchain_state_); + } else { + CHECK(shard_client_seqno_ < last_masterchain_state_->get_seqno()); + BlockIdExt block_id; + if (!last_masterchain_state_->get_old_mc_block_id(shard_client_seqno_, block_id)) { + abort_query(td::Status::Error("failed to get shard client block id")); + return; + } + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db_short, block_id, + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, + R.move_as_error_prefix("failed to get shard client state: ")); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::got_shard_client_state, + td::Ref{R.move_as_ok()}); + }); + } +} + +void ArchiveImporterLocal::got_shard_client_state(td::Ref state) { + CHECK(state->get_seqno() == shard_client_seqno_); + LOG(DEBUG) << "got_shard_client_state " << shard_client_seqno_; + shard_client_state_ = state; + new_shard_client_seqno_ = shard_client_seqno_; + current_shard_client_seqno_ = shard_client_seqno_; + for (auto &shard : state->get_shards()) { + visited_shard_blocks_.insert(shard->top_block_id()); + } + try_advance_shard_client_seqno(); +} + +void ArchiveImporterLocal::try_advance_shard_client_seqno() { + BlockSeqno seqno = new_shard_client_seqno_ + 1; + auto it = masterchain_blocks_.find(seqno); + if (it != masterchain_blocks_.end()) { + try_advance_shard_client_seqno_cont(blocks_[it->second].block); + return; + } + if (seqno > last_masterchain_state_->get_seqno()) { + processed_shard_blocks(); + return; + } + BlockIdExt block_id; + if (!last_masterchain_state_->get_old_mc_block_id(seqno, block_id)) { + abort_query(td::Status::Error("failed to get old mc block id")); + return; + } + td::actor::send_closure(manager_, &ValidatorManager::get_block_data_from_db_short, block_id, + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, + R.move_as_error_prefix("failed to get block data: ")); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::try_advance_shard_client_seqno_cont, + R.move_as_ok()); + }); +} + +void ArchiveImporterLocal::try_advance_shard_client_seqno_cont(td::Ref mc_block) { + CHECK(mc_block.not_null()); + CHECK(mc_block->block_id().seqno() == new_shard_client_seqno_ + 1); + LOG(DEBUG) << "try_advance_shard_client_seqno " << new_shard_client_seqno_ + 1; + + block::gen::Block::Record rec; + block::gen::BlockExtra::Record extra; + block::gen::McBlockExtra::Record mc_extra; + CHECK(block::gen::unpack_cell(mc_block->root_cell(), rec) && block::gen::unpack_cell(rec.extra, extra) && + block::gen::unpack_cell(extra.custom->prefetch_ref(), mc_extra)); + auto shard_config = std::make_unique(mc_extra.shard_hashes->prefetch_ref()); + + std::vector blocks_to_import; + std::function dfs = [&](const BlockIdExt &block_id) -> td::Status { + if (visited_shard_blocks_.contains(block_id)) { + return td::Status::OK(); + } + if (block_id.seqno() == 0) { + new_zerostates_.insert(block_id); + return td::Status::OK(); + } + visited_shard_blocks_.insert(block_id); + auto &info = blocks_[block_id]; + if (info.block.is_null()) { + return td::Status::Error(PSTRING() << "no shard block " << block_id.to_str()); + } + blocks_to_import.push_back(block_id); + + std::vector prev; + BlockIdExt mc_blkid; + bool after_split; + TRY_STATUS(block::unpack_block_prev_blk_try(info.block->root_cell(), block_id, prev, mc_blkid, after_split)); + for (const BlockIdExt &prev_block_id : prev) { + TRY_STATUS(dfs(prev_block_id)); + } + + return td::Status::OK(); + }; + td::Status S = td::Status::OK(); + std::vector top_shard_blocks; + shard_config->process_shard_hashes([&](block::McShardHash &shard) { + if (!opts_->need_monitor(shard.shard(), shard_client_state_)) { + return 0; + } + S = dfs(shard.top_block_id()); + top_shard_blocks.push_back(shard.top_block_id()); + if (S.is_error()) { + return -1; + } + return 0; + }); + if (S.is_error()) { + LOG(DEBUG) << "Cannot advance shard client seqno to " << new_shard_client_seqno_ + 1 << " : " << S; + processed_shard_blocks(); + return; + } + shard_configs_[mc_block->block_id().seqno()] = {mc_block->block_id(), std::move(top_shard_blocks)}; + ++new_shard_client_seqno_; + LOG(DEBUG) << "Advancing shard client seqno to " << new_shard_client_seqno_; + for (const BlockIdExt &block_id : blocks_to_import) { + blocks_[block_id].import = true; + } + td::actor::send_closure(actor_id(this), &ArchiveImporterLocal::try_advance_shard_client_seqno); +} + +void ArchiveImporterLocal::processed_shard_blocks() { + if (new_shard_client_seqno_ == shard_client_seqno_) { + LOG(INFO) << "No new shard blocks"; + } else { + LOG(INFO) << "New shard client seqno = " << new_shard_client_seqno_; + } + + td::MultiPromise mp; + auto ig = mp.init_guard(); + for (const BlockIdExt &block_id : new_zerostates_) { + LOG(INFO) << "Downloading zerostate " << block_id.to_str(); + td::actor::create_actor( + "downloadstate", block_id, shard_client_state_->get_block_id(), 2, manager_, td::Timestamp::in(3600), + ig.get_promise().wrap([](td::Ref &&) { return td::Unit(); })) + .release(); + } + ig.add_promise([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::store_data); + } + }); +} + +void ArchiveImporterLocal::store_data() { + td::MultiPromise mp; + auto ig = mp.init_guard(); + + if (opts_->get_permanent_celldb()) { + std::vector> blocks; + for (auto &[_, info] : blocks_) { + if (info.import) { + blocks.push_back(info.block); + } + } + td::actor::send_closure(manager_, &ValidatorManager::set_block_state_from_data_preliminary, std::move(blocks), + ig.get_promise()); + } + for (auto &[block_id, info] : blocks_) { + if (info.import) { + td::actor::send_closure( + manager_, &ValidatorManager::get_block_handle, block_id, true, + [promise = ig.get_promise(), block = info.block, manager = manager_](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + td::actor::send_closure(manager, &ValidatorManager::set_block_data, handle, std::move(block), + std::move(promise)); + }); + if (info.proof_link.not_null()) { + run_check_proof_link_query(block_id, info.proof_link, manager_, td::Timestamp::in(60.0), + ig.get_promise().wrap([](BlockHandle &&) { return td::Unit(); })); + } + } + } + + ig.add_promise([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::apply_next_masterchain_block); + } + }); +} + +void ArchiveImporterLocal::apply_next_masterchain_block() { + auto it = masterchain_blocks_.find(last_masterchain_state_->get_seqno() + 1); + if (it == masterchain_blocks_.end()) { + LOG(INFO) << "Applied masterchain blocks, last seqno = " << last_masterchain_state_->get_seqno(); + apply_shard_blocks(); + return; + } + BlockIdExt block_id = it->second; + LOG(DEBUG) << "Applying masterchain block " << block_id.to_str(); + BlockInfo &info = blocks_[block_id]; + run_apply_block_query(block_id, info.block, block_id, manager_, td::Timestamp::in(600.0), + [=, SelfId = actor_id(this), manager = manager_](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure( + manager, &ValidatorManager::get_shard_state_from_db_short, block_id, + [=](td::Result> R2) { + if (R2.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, + R2.move_as_error()); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::applied_next_masterchain_block, + td::Ref{R2.move_as_ok()}); + }); + }); +} + +void ArchiveImporterLocal::applied_next_masterchain_block(td::Ref state) { + last_masterchain_state_ = state; + imported_any_ = true; + LOG(DEBUG) << "Applied masterchain block " << state->get_block_id().to_str(); + apply_next_masterchain_block(); +} + +void ArchiveImporterLocal::apply_shard_blocks() { + if (current_shard_client_seqno_ == new_shard_client_seqno_) { + finish_query(); + return; + } + auto it = shard_configs_.find(current_shard_client_seqno_ + 1); + if (it == shard_configs_.end()) { + abort_query(td::Status::Error("no shard config for the next shard client seqno")); + return; + } + + td::MultiPromise mp; + auto ig = mp.init_guard(); + BlockIdExt mc_block_id = it->second.first; + LOG(DEBUG) << "Applying top shard blocks from " << current_shard_client_seqno_ + 1; + for (const BlockIdExt &block_id : it->second.second) { + apply_shard_block(block_id, mc_block_id, ig.get_promise()); + } + + ig.add_promise([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::applied_shard_blocks); + }); +} + +void ArchiveImporterLocal::applied_shard_blocks() { + LOG(DEBUG) << "Applied top shard blocks from " << current_shard_client_seqno_ + 1; + ++current_shard_client_seqno_; + imported_any_ = true; + apply_shard_blocks(); +} + +void ArchiveImporterLocal::apply_shard_block(BlockIdExt block_id, BlockIdExt mc_block_id, + td::Promise promise) { + td::actor::send_closure( + manager_, &ValidatorManager::get_block_handle, block_id, true, + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + R.ensure(); + td::actor::send_closure(SelfId, &ArchiveImporterLocal::apply_shard_block_cont1, R.move_as_ok(), mc_block_id, + std::move(promise)); + }); +} + +void ArchiveImporterLocal::apply_shard_block_cont1(BlockHandle handle, BlockIdExt mc_block_id, + td::Promise promise) { + if (handle->is_applied()) { + promise.set_value(td::Unit()); + return; + } + + promise = [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_STATUS_PROMISE(promise, R.move_as_status()); + td::actor::send_closure(SelfId, &ArchiveImporterLocal::apply_shard_block_cont2, handle, mc_block_id, + std::move(promise)); + }; + + if (!handle->merge_before() && handle->one_prev(true).shard_full() == handle->id().shard_full()) { + apply_shard_block(handle->one_prev(true), mc_block_id, std::move(promise)); + } else { + td::MultiPromise mp; + auto ig = mp.init_guard(); + ig.add_promise(std::move(promise)); + check_shard_block_applied(handle->one_prev(true), ig.get_promise()); + if (handle->merge_before()) { + check_shard_block_applied(handle->one_prev(false), ig.get_promise()); + } + } +} + +void ArchiveImporterLocal::apply_shard_block_cont2(BlockHandle handle, BlockIdExt mc_block_id, + td::Promise promise) { + td::Ref block = blocks_[handle->id()].block; + CHECK(block.not_null()); + LOG(DEBUG) << "Applying shard block " << handle->id().to_str(); + run_apply_block_query(handle->id(), std::move(block), mc_block_id, manager_, td::Timestamp::in(600.0), + std::move(promise)); +} + +void ArchiveImporterLocal::check_shard_block_applied(BlockIdExt block_id, td::Promise promise) { + td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, block_id, false, + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + if (!handle->is_applied()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "not applied")); + } else { + promise.set_value(td::Unit()); + } + }); +} + +void ArchiveImporterLocal::abort_query(td::Status error) { + if (!imported_any_) { + LOG(ERROR) << "Archive import: " << error; + promise_.set_error(std::move(error)); + stop(); + } else { + LOG(WARNING) << "Archive import: " << error; + finish_query(); + } +} + +void ArchiveImporterLocal::finish_query() { + LOG(WARNING) << "Imported archive in " << timer_.elapsed() << "s : mc_seqno=" << last_masterchain_state_->get_seqno() + << " shard_seqno=" << current_shard_client_seqno_; + promise_.set_value({last_masterchain_state_->get_seqno(), + std::min(last_masterchain_state_->get_seqno(), current_shard_client_seqno_)}); + stop(); +} + +} // namespace validator + +} // namespace ton diff --git a/validator/import-db-slice-local.hpp b/validator/import-db-slice-local.hpp new file mode 100644 index 000000000..778e4eed6 --- /dev/null +++ b/validator/import-db-slice-local.hpp @@ -0,0 +1,103 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "td/actor/actor.h" +#include "td/utils/port/path.h" +#include "validator/interfaces/validator-manager.h" +#include "validator/db/package.hpp" + +namespace ton { + +namespace validator { + +class ArchiveImporterLocal : public td::actor::Actor { + public: + ArchiveImporterLocal(std::string db_root, td::Ref state, BlockSeqno shard_client_seqno, + td::Ref opts, td::actor::ActorId manager, + std::vector to_import_files, + td::Promise> promise); + void start_up() override; + + void abort_query(td::Status error); + void finish_query(); + + td::Status process_package(std::string path); + + void process_masterchain_blocks(); + void process_masterchain_blocks_cont(); + + void import_first_key_block(); + void checked_key_block_proof(BlockHandle handle); + void applied_key_block(td::Ref state); + + void checked_masterchain_proofs(); + void got_shard_client_state(td::Ref state); + + void try_advance_shard_client_seqno(); + void try_advance_shard_client_seqno_cont(td::Ref mc_block); + + void processed_shard_blocks(); + void store_data(); + void apply_next_masterchain_block(); + void applied_next_masterchain_block(td::Ref state); + + void apply_shard_blocks(); + void applied_shard_blocks(); + + void apply_shard_block(BlockIdExt block_id, BlockIdExt mc_block_id, td::Promise promise); + void apply_shard_block_cont1(BlockHandle handle, BlockIdExt mc_block_id, td::Promise promise); + void apply_shard_block_cont2(BlockHandle handle, BlockIdExt mc_block_id, td::Promise promise); + void check_shard_block_applied(BlockIdExt block_id, td::Promise promise); + + private: + std::string db_root_; + td::Ref last_masterchain_state_; + BlockSeqno shard_client_seqno_; + + td::Ref opts_; + + td::actor::ActorId manager_; + + std::vector to_import_files_; + td::Promise> promise_; + + struct BlockInfo { + td::Ref block; + td::Ref proof; + td::Ref proof_link; + bool import = false; + }; + std::map blocks_; + std::map masterchain_blocks_; + + td::Ref shard_client_state_; + BlockSeqno new_shard_client_seqno_; + BlockSeqno current_shard_client_seqno_; + std::set visited_shard_blocks_; + std::set new_zerostates_; + + std::map>> shard_configs_; + + bool imported_any_ = false; + + td::Timer timer_; +}; + +} // namespace validator + +} // namespace ton diff --git a/validator/import-db-slice.cpp b/validator/import-db-slice.cpp index 06573d347..64861a8bb 100644 --- a/validator/import-db-slice.cpp +++ b/validator/import-db-slice.cpp @@ -85,7 +85,7 @@ void ArchiveImporter::downloaded_mc_archive(std::string path) { void ArchiveImporter::processed_mc_archive() { if (masterchain_blocks_.empty()) { - LOG(DEBUG) << "No masterhchain blocks in archive"; + LOG(DEBUG) << "No masterchain blocks in archive"; last_masterchain_seqno_ = last_masterchain_state_->get_seqno(); checked_all_masterchain_blocks(); return; diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index 29ef715b3..3a26871e1 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -50,6 +50,10 @@ class Db : public td::actor::Actor { virtual void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) = 0; + virtual void store_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) = 0; + virtual void store_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) = 0; virtual void get_block_state(ConstBlockHandle handle, td::Promise> promise) = 0; virtual void get_cell_db_reader(td::Promise> promise) = 0; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 00fb77e1e..c311cc08e 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -70,6 +70,10 @@ class ValidatorManager : public ValidatorManagerInterface { } virtual void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) = 0; + virtual void set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) = 0; + virtual void set_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) = 0; virtual void get_cell_db_reader(td::Promise> promise) = 0; virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 62fdc4b43..a487556c4 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -321,8 +321,8 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle->id(), std::move(R)); }); - auto id = td::actor::create_actor("waitstate", handle, 0, actor_id(this), td::Timestamp::in(10.0), - std::move(P)) + auto id = td::actor::create_actor("waitstate", handle, 0, opts_, last_masterchain_state_, + actor_id(this), td::Timestamp::in(10.0), std::move(P)) .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); @@ -686,6 +686,16 @@ void ValidatorManagerImpl::set_block_state(BlockHandle handle, td::Ref block, + td::Promise> promise) { + td::actor::send_closure(db_, &Db::store_block_state_from_data, handle, block, std::move(promise)); +} + +void ValidatorManagerImpl::set_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) { + td::actor::send_closure(db_, &Db::store_block_state_from_data_preliminary, std::move(blocks), std::move(promise)); +} + void ValidatorManagerImpl::get_cell_db_reader(td::Promise> promise) { td::actor::send_closure(db_, &Db::get_cell_db_reader, std::move(promise)); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index cd06bf555..3c516ef15 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -148,6 +148,10 @@ class ValidatorManagerImpl : public ValidatorManager { void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; + void set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) override; + void set_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) override; void get_cell_db_reader(td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) override; diff --git a/validator/manager-hardfork.cpp b/validator/manager-hardfork.cpp index 91c598aa2..c3d6267be 100644 --- a/validator/manager-hardfork.cpp +++ b/validator/manager-hardfork.cpp @@ -171,8 +171,8 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle->id(), std::move(R)); }); - auto id = td::actor::create_actor("waitstate", handle, 0, actor_id(this), td::Timestamp::in(10.0), - std::move(P)) + auto id = td::actor::create_actor("waitstate", handle, 0, opts_, last_masterchain_state_, + actor_id(this), td::Timestamp::in(10.0), std::move(P)) .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 0b8b9e736..b28b29685 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -174,6 +174,13 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { UNREACHABLE(); } + void set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) override { + UNREACHABLE(); + } + void set_block_state_from_data_preliminary(std::vector> blocks, td::Promise promise) { + UNREACHABLE(); + } void get_cell_db_reader(td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) override { diff --git a/validator/manager.cpp b/validator/manager.cpp index 00b923c23..1d75b523a 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -31,6 +31,7 @@ #include "state-serializer.hpp" #include "get-next-key-blocks.h" #include "import-db-slice.hpp" +#include "import-db-slice-local.hpp" #include "auto/tl/lite_api.h" #include "tl-utils/lite-utils.hpp" @@ -685,10 +686,6 @@ void ValidatorManagerImpl::run_ext_query(td::BufferSlice data, td::Promise> promise) { - if (last_masterchain_state_.not_null() && !opts_->need_monitor(handle->id().shard_full(), last_masterchain_state_)) { - return promise.set_error( - td::Status::Error(PSTRING() << "not monitoring shard " << handle->id().shard_full().to_str())); - } auto it0 = block_state_cache_.find(handle->id()); if (it0 != block_state_cache_.end()) { it0->second.ttl_ = td::Timestamp::in(30.0); @@ -700,10 +697,11 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); }); - auto id = td::actor::create_actor("waitstate", handle, priority, actor_id(this), - td::Timestamp::at(timeout.at() + 10.0), std::move(P), - get_block_persistent_state_to_download(handle->id())) - .release(); + auto id = + td::actor::create_actor("waitstate", handle, priority, opts_, last_masterchain_state_, + actor_id(this), td::Timestamp::at(timeout.at() + 10.0), std::move(P), + get_block_persistent_state_to_download(handle->id())) + .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); } @@ -762,10 +760,6 @@ void ValidatorManagerImpl::wait_block_data_short(BlockIdExt block_id, td::uint32 void ValidatorManagerImpl::wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) { - if (last_masterchain_state_.not_null() && !opts_->need_monitor(left_id.shard_full(), last_masterchain_state_)) { - return promise.set_error( - td::Status::Error(PSTRING() << "not monitoring shard " << left_id.shard_full().to_str())); - } td::actor::create_actor("merge", left_id, right_id, priority, actor_id(this), timeout, std::move(promise)) .release(); @@ -1149,10 +1143,10 @@ void ValidatorManagerImpl::finished_wait_state(BlockHandle handle, td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); }); - auto id = - td::actor::create_actor("waitstate", handle, X.second, actor_id(this), X.first, - std::move(P), get_block_persistent_state_to_download(handle->id())) - .release(); + auto id = td::actor::create_actor("waitstate", handle, X.second, opts_, last_masterchain_state_, + actor_id(this), X.first, std::move(P), + get_block_persistent_state_to_download(handle->id())) + .release(); it->second.actor_ = id; return; } @@ -1211,6 +1205,16 @@ void ValidatorManagerImpl::set_block_state(BlockHandle handle, td::Ref block, + td::Promise> promise) { + td::actor::send_closure(db_, &Db::store_block_state_from_data, handle, block, std::move(promise)); +} + +void ValidatorManagerImpl::set_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) { + td::actor::send_closure(db_, &Db::store_block_state_from_data_preliminary, std::move(blocks), std::move(promise)); +} + void ValidatorManagerImpl::get_cell_db_reader(td::Promise> promise) { td::actor::send_closure(db_, &Db::get_cell_db_reader, std::move(promise)); } @@ -1688,6 +1692,7 @@ void ValidatorManagerImpl::start_up() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::started, R.move_as_ok()); }); + size_t to_import_files = 0; auto to_import_dir = db_root_ + "/import"; auto S = td::WalkPath::run(to_import_dir, [&](td::CSlice cfname, td::WalkPath::Type t) -> void { auto fname = td::Slice(cfname); @@ -1707,26 +1712,30 @@ void ValidatorManagerImpl::start_up() { } fname = fname.substr(8); - while (fname.size() > 1 && fname[0] == '0') { - fname.remove_prefix(1); - } auto i = fname.find('.'); if (i == td::Slice::npos) { return; } fname = fname.substr(0, i); + while (fname.size() > 1 && fname[0] == '0') { + fname.remove_prefix(1); + } auto v = td::to_integer_safe(fname); if (v.is_error()) { return; } auto seqno = v.move_as_ok(); - LOG(INFO) << "found archive slice '" << cfname << "' for seqno " << seqno; + LOG(DEBUG) << "found archive slice '" << cfname << "' for seqno " << seqno; to_import_[seqno].push_back(cfname.str()); + ++to_import_files; } }); if (S.is_error()) { LOG(INFO) << "failed to load blocks from import dir: " << S; } + if (to_import_files > 0) { + LOG(INFO) << "found " << to_import_files << " files to import"; + } validator_manager_init(opts_, actor_id(this), db_.get(), std::move(P)); @@ -1909,9 +1918,15 @@ void ValidatorManagerImpl::download_next_archive() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::checked_archive_slice, R.ok().first, R.ok().second); } }); - td::actor::create_actor("archiveimport", db_root_, last_masterchain_state_, seqno, opts_, - actor_id(this), std::move(to_import_files), std::move(P)) - .release(); + if (to_import_files.empty()) { + td::actor::create_actor("archiveimport", db_root_, last_masterchain_state_, seqno, opts_, + actor_id(this), std::move(to_import_files), std::move(P)) + .release(); + } else { + td::actor::create_actor("archiveimport", db_root_, last_masterchain_state_, seqno, opts_, + actor_id(this), std::move(to_import_files), std::move(P)) + .release(); + } } void ValidatorManagerImpl::checked_archive_slice(BlockSeqno new_last_mc_seqno, BlockSeqno new_shard_client_seqno) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 418deb350..850baf47b 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -407,6 +407,9 @@ class ValidatorManagerImpl : public ValidatorManager { void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; + void set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) override; + void set_block_state_from_data_preliminary(std::vector> blocks, td::Promise promise) override; void get_cell_db_reader(td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) override; diff --git a/validator/validator-options.cpp b/validator/validator-options.cpp index cb26fe44d..a87562377 100644 --- a/validator/validator-options.cpp +++ b/validator/validator-options.cpp @@ -24,14 +24,13 @@ namespace ton { namespace validator { -td::Ref ValidatorManagerOptions::create( - BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard, bool allow_blockchain_init, - double sync_blocks_before, double block_ttl, double state_ttl, double max_mempool_num, - double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) { - return td::make_ref(zero_block_id, init_block_id, std::move(check_shard), - allow_blockchain_init, sync_blocks_before, block_ttl, state_ttl, - max_mempool_num, +td::Ref ValidatorManagerOptions::create(BlockIdExt zero_block_id, BlockIdExt init_block_id, + bool allow_blockchain_init, double sync_blocks_before, + double block_ttl, double state_ttl, + double max_mempool_num, double archive_ttl, + double key_proof_ttl, bool initial_sync_disabled) { + return td::make_ref(zero_block_id, init_block_id, allow_blockchain_init, + sync_blocks_before, block_ttl, state_ttl, max_mempool_num, archive_ttl, key_proof_ttl, initial_sync_disabled); } diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 45b8d7ec2..def897995 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -34,7 +34,8 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { } bool need_monitor(ShardIdFull shard, const td::Ref& state) const override { td::uint32 min_split = state->monitor_min_split_depth(shard.workchain); - return check_shard_((td::uint32)shard.pfx_len() <= min_split ? shard : shard_prefix(shard, min_split)); + return check_shard_((td::uint32)shard.pfx_len() <= min_split ? shard : shard_prefix(shard, min_split), + state->get_seqno()); } bool allow_blockchain_init() const override { return allow_blockchain_init_; @@ -160,6 +161,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { double get_catchain_broadcast_speed_multiplier() const override { return catchain_broadcast_speed_multipliers_; } + bool get_permanent_celldb() const override { + return permanent_celldb_; + } void set_zero_block_id(BlockIdExt block_id) override { zero_block_id_ = block_id; @@ -167,7 +171,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_init_block_id(BlockIdExt block_id) override { init_block_id_ = block_id; } - void set_shard_check_function(std::function check_shard) override { + void set_shard_check_function(std::function check_shard) override { check_shard_ = std::move(check_shard); } void set_allow_blockchain_init(bool value) override { @@ -261,18 +265,19 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_catchain_broadcast_speed_multiplier(double value) override { catchain_broadcast_speed_multipliers_ = value; } + void set_permanent_celldb(bool value) override { + permanent_celldb_ = value; + } ValidatorManagerOptionsImpl *make_copy() const override { return new ValidatorManagerOptionsImpl(*this); } - ValidatorManagerOptionsImpl(BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard, bool allow_blockchain_init, + ValidatorManagerOptionsImpl(BlockIdExt zero_block_id, BlockIdExt init_block_id, bool allow_blockchain_init, double sync_blocks_before, double block_ttl, double state_ttl, double max_mempool_num, double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) : zero_block_id_(zero_block_id) , init_block_id_(init_block_id) - , check_shard_(std::move(check_shard)) , allow_blockchain_init_(allow_blockchain_init) , sync_blocks_before_(sync_blocks_before) , block_ttl_(block_ttl) @@ -286,7 +291,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { private: BlockIdExt zero_block_id_; BlockIdExt init_block_id_; - std::function check_shard_; + std::function check_shard_ = [](ShardIdFull, BlockSeqno) { return true; }; bool allow_blockchain_init_; double sync_blocks_before_; double block_ttl_; @@ -316,6 +321,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { td::Ref collator_options_{true}; bool fast_state_serializer_enabled_ = false; double catchain_broadcast_speed_multipliers_; + bool permanent_celldb_ = false; }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 66795cece..4631bf7ad 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -118,10 +118,11 @@ struct ValidatorManagerOptions : public td::CntObject { virtual td::Ref get_collator_options() const = 0; virtual bool get_fast_state_serializer_enabled() const = 0; virtual double get_catchain_broadcast_speed_multiplier() const = 0; + virtual bool get_permanent_celldb() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; - virtual void set_shard_check_function(std::function check_shard) = 0; + virtual void set_shard_check_function(std::function check_shard) = 0; virtual void set_allow_blockchain_init(bool value) = 0; virtual void set_sync_blocks_before(double value) = 0; virtual void set_block_ttl(double value) = 0; @@ -152,11 +153,10 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_collator_options(td::Ref value) = 0; virtual void set_fast_state_serializer_enabled(bool value) = 0; virtual void set_catchain_broadcast_speed_multiplier(double value) = 0; + virtual void set_permanent_celldb(bool value) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, - - std::function check_shard = [](ShardIdFull) { return true; }, bool allow_blockchain_init = false, double sync_blocks_before = 3600, double block_ttl = 86400, double state_ttl = 86400, double archive_ttl = 86400 * 7, double key_proof_ttl = 86400 * 3650, double max_mempool_num = 999999, bool initial_sync_disabled = false); From 949ae86038af8cca3dcbce845bf02daa3ede36ad Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Sat, 22 Mar 2025 13:57:03 +0300 Subject: [PATCH 173/388] Fix persistent state lookup --- validator/net/download-state.cpp | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/validator/net/download-state.cpp b/validator/net/download-state.cpp index 6735a2b5f..550c8882e 100644 --- a/validator/net/download-state.cpp +++ b/validator/net/download-state.cpp @@ -73,16 +73,20 @@ void DownloadState::start_up() { status_ = ProcessStatus(validator_manager_, "process.download_state_net"); alarm_timestamp() = timeout_; - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state, block_id_, - masterchain_block_id_, - [SelfId = actor_id(this), block_id = block_id_](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &DownloadState::get_block_handle); - } else { - LOG(WARNING) << "got block state from disk: " << block_id.to_str(); - td::actor::send_closure(SelfId, &DownloadState::got_block_state, R.move_as_ok()); - } - }); + td::Promise P = [SelfId = actor_id(this), block_id = block_id_](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &DownloadState::get_block_handle); + } else { + LOG(WARNING) << "got block state from disk: " << block_id.to_str(); + td::actor::send_closure(SelfId, &DownloadState::got_block_state, R.move_as_ok()); + } + }; + if (block_id_.seqno() == 0) { + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_zero_state, block_id_, std::move(P)); + } else { + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state, block_id_, + masterchain_block_id_, std::move(P)); + } } void DownloadState::get_block_handle() { From 71ddef2de02d7568127d33286861f48333a2d041 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Sat, 22 Mar 2025 13:57:32 +0300 Subject: [PATCH 174/388] Tool for generating liteserver desc for global config --- utils/CMakeLists.txt | 3 + utils/prepare-ls-slice-config.cpp | 266 ++++++++++++++++++++++++++++++ 2 files changed, 269 insertions(+) create mode 100644 utils/prepare-ls-slice-config.cpp diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 99a4e92df..0afdf7389 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -22,4 +22,7 @@ target_include_directories(pack-viewer PUBLIC $. +*/ +#include "td/utils/filesystem.h" +#include "td/actor/actor.h" +#include "td/actor/MultiPromise.h" +#include "td/utils/OptionParser.h" +#include "td/utils/port/path.h" +#include "td/utils/port/signals.h" +#include "td/utils/port/IPAddress.h" +#include "td/utils/Random.h" +#include "td/utils/FileLog.h" +#include "git.h" +#include "auto/tl/ton_api.h" +#include "auto/tl/lite_api.h" +#include "tl-utils/lite-utils.hpp" +#include "auto/tl/ton_api_json.h" +#include "adnl/adnl.h" +#include "lite-client/ext-client.h" +#include "ton/lite-tl.hpp" + +#include "td/utils/overloaded.h" + +#include +#include "td/utils/tl_storers.h" +#include "vm/boc.h" +#include "vm/cells/MerkleProof.h" + +#include +#include "block/block-auto.h" +#include "block/mc-config.h" + +using namespace ton; + +std::string global_config_file; +td::optional start_mc_seqno, end_mc_seqno; +std::vector shards; + +class PrepareLsSliceConfig : public td::actor::Actor { + public: + void start_up() override { + if (start_mc_seqno && end_mc_seqno && start_mc_seqno.value() > end_mc_seqno.value()) { + LOG(FATAL) << "from-seqno is greater than to-seqno"; + } + + if (!start_mc_seqno && !end_mc_seqno) { + auto slice = create_tl_object(); + if (shards.empty()) { + slice->shards_.push_back(create_tl_shard_id(ShardIdFull{basechainId, shardIdAll})); + } else { + for (const ShardIdFull& shard : shards) { + if (!shard.is_masterchain()) { + slice->shards_.push_back(create_tl_shard_id(shard)); + } + } + } + print_result(*slice); + return; + } + + auto gc_s = td::read_file(global_config_file).move_as_ok(); + auto gc_j = td::json_decode(gc_s.as_slice()).move_as_ok(); + ton_api::liteclient_config_global gc; + ton_api::from_json(gc, gc_j.get_object()).ensure(); + auto r_servers = liteclient::LiteServerConfig::parse_global_config(gc); + r_servers.ensure(); + client_ = liteclient::ExtClient::create(r_servers.move_as_ok(), nullptr); + + slice_timed_ = create_tl_object(); + ++pending_; + request_shards_info(start_mc_seqno, true); + request_shards_info(end_mc_seqno, false); + dec_pending(); + } + + template + static td::BufferSlice create_query(Args&&... args) { + Type object(std::forward(args)...); + return create_serialize_tl_object(serialize_tl_object(&object, true)); + } + + template + static tl_object_ptr parse_response(const td::Result& R) { + R.ensure(); + auto err = fetch_tl_object(R.ok(), true); + if (err.is_ok()) { + LOG(FATAL) << "liteserver error: " << err.ok()->message_; + } + auto res = fetch_tl_object(R.ok(), true); + res.ensure(); + return res.move_as_ok(); + } + + void request_shards_info(td::optional seqno, bool is_start) { + if (!seqno) { + return; + } + ++pending_; + td::actor::send_closure( + client_, &liteclient::ExtClient::send_query, "q", + create_query( + 1, create_tl_object(masterchainId, shardIdAll, seqno.value()), 0, 0), + td::Timestamp::in(5.0), [=, client = client_.get(), SelfId = actor_id(this)](td::Result R) { + auto mc_header = parse_response(std::move(R)); + auto block_id = create_block_id(mc_header->id_); + td::actor::send_closure( + client, &liteclient::ExtClient::send_query, "q", + create_query(create_tl_lite_block_id(block_id)), + td::Timestamp::in(5.0), [=, mc_header = std::move(mc_header)](td::Result R) mutable { + auto shards_info = parse_response(std::move(R)); + td::actor::send_closure(SelfId, &PrepareLsSliceConfig::got_shards_info, std::move(mc_header), + std::move(shards_info), is_start); + }); + }); + } + + static tl_object_ptr parse_header(const lite_api::liteServer_blockHeader& obj, + bool is_start) { + auto res = create_tl_object(); + + BlockIdExt block_id = create_block_id(obj.id_); + res->shard_id_ = create_tl_shard_id(block_id.shard_full()); + res->seqno_ = block_id.seqno(); + + auto root = vm::std_boc_deserialize(obj.header_proof_).move_as_ok(); + root = vm::MerkleProof::virtualize(root, 1); + block::gen::Block::Record blk; + block::gen::BlockInfo::Record info; + CHECK(tlb::unpack_cell(root, blk) && tlb::unpack_cell(blk.info, info)); + res->utime_ = info.gen_utime; + res->lt_ = (is_start ? info.start_lt : info.end_lt); + + return res; + } + + void got_shards_info(tl_object_ptr mc_header, + tl_object_ptr shards_info, bool is_start) { + (is_start ? slice_timed_->shards_from_ : slice_timed_->shards_to_).push_back(parse_header(*mc_header, is_start)); + + auto root = vm::std_boc_deserialize(shards_info->data_).move_as_ok(); + block::ShardConfig sh_conf; + CHECK(sh_conf.unpack(vm::load_cell_slice_ref(root))); + auto ids = sh_conf.get_shard_hash_ids(true); + for (auto id : ids) { + BlockIdExt block_id = sh_conf.get_shard_hash(ton::ShardIdFull(id))->top_block_id(); + bool ok = shards.empty(); + for (const auto& our_shard : shards) { + if (shard_intersects(our_shard, block_id.shard_full())) { + ok = true; + break; + } + } + if (ok) { + ++pending_; + td::actor::send_closure( + client_, &liteclient::ExtClient::send_query, "q", + create_query(create_tl_lite_block_id(block_id), 0xffff), + td::Timestamp::in(5.0), [=, SelfId = actor_id(this)](td::Result R) mutable { + auto header = parse_response(std::move(R)); + td::actor::send_closure(SelfId, &PrepareLsSliceConfig::got_block_header, std::move(header), is_start); + }); + } + } + + dec_pending(); + } + + void got_block_header(tl_object_ptr header, bool is_start) { + (is_start ? slice_timed_->shards_from_ : slice_timed_->shards_to_).push_back(parse_header(*header, is_start)); + dec_pending(); + } + + void print_result(const ton_api::liteserver_descV2_Slice& result) { + auto s = td::json_encode(td::ToJson(result), true); + std::cout << s << "\n"; + std::cout.flush(); + exit(0); + } + + private: + td::actor::ActorOwn client_; + tl_object_ptr slice_timed_; + size_t pending_ = 0; + + void dec_pending() { + --pending_; + if (pending_ == 0) { + auto cmp = [](const tl_object_ptr& a, + const tl_object_ptr& b) { + return create_shard_id(a->shard_id_) < create_shard_id(b->shard_id_); + }; + std::sort(slice_timed_->shards_from_.begin(), slice_timed_->shards_from_.end(), cmp); + std::sort(slice_timed_->shards_to_.begin(), slice_timed_->shards_to_.end(), cmp); + print_result(*slice_timed_); + } + } +}; + +int main(int argc, char* argv[]) { + SET_VERBOSITY_LEVEL(verbosity_INFO); + + td::unique_ptr logger_; + SCOPE_EXIT { + td::log_interface = td::default_log_interface; + }; + + td::OptionParser p; + p.set_description( + "Generate liteserver.descV2.Slice for global-config.json from given shards and masterchain seqnos\n"); + p.add_option('v', "verbosity", "set verbosity level", [&](td::Slice arg) { + int v = VERBOSITY_NAME(FATAL) + (td::to_integer(arg)); + SET_VERBOSITY_LEVEL(v); + }); + p.add_option('V', "version", "show build information", [&]() { + std::cout << "prepare-ls-slice-config build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::exit(0); + }); + p.add_option('h', "help", "print help", [&]() { + char b[10240]; + td::StringBuilder sb(td::MutableSlice{b, 10000}); + sb << p; + std::cout << sb.as_cslice().c_str(); + std::exit(2); + }); + p.add_option('C', "global-config", "global TON configuration file (used to fetch shard configuration)", + [&](td::Slice arg) { global_config_file = arg.str(); }); + p.add_checked_option('f', "from-seqno", "starting masterchain seqno (default: none)", + [&](td::Slice arg) -> td::Status { + TRY_RESULT_ASSIGN(start_mc_seqno, td::to_integer_safe(arg)) + return td::Status::OK(); + }); + p.add_checked_option('t', "to-seqno", "ending masterchain seqno (default: none)", [&](td::Slice arg) -> td::Status { + TRY_RESULT_ASSIGN(end_mc_seqno, td::to_integer_safe(arg)) + return td::Status::OK(); + }); + p.add_checked_option('s', "shard", "shard in format 0:8000000000000000 (default: all shards)", + [&](td::Slice arg) -> td::Status { + TRY_RESULT(shard, ShardIdFull::parse(arg)); + if (!shard.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard " << arg); + } + shards.push_back(shard); + return td::Status::OK(); + }); + + p.run(argc, argv).ensure(); + td::actor::Scheduler scheduler({3}); + + scheduler.run_in_context([&] { td::actor::create_actor("main").release(); }); + while (scheduler.run(1)) { + } +} From ab4888702aa0f7844e90826f060f715ee7cc31ee Mon Sep 17 00:00:00 2001 From: k-dimentional tree <98183693+kdimentionaltree@users.noreply.github.com> Date: Mon, 24 Mar 2025 16:23:07 +0300 Subject: [PATCH 175/388] Added normalized hash of external-in message and increased stack depth for tonlib (#1557) * Update stack_limit * Added external-in normalized hash and increased stack depth for tonlib * Removed debug logs --- tl/generate/scheme/tonlib_api.tl | 2 +- tl/generate/scheme/tonlib_api.tlo | Bin 34344 -> 34380 bytes tonlib/tonlib/TonlibClient.cpp | 44 ++++++++++++++++++++++++++++-- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/tl/generate/scheme/tonlib_api.tl b/tl/generate/scheme/tonlib_api.tl index 31ca6fd40..e7d8cb59c 100644 --- a/tl/generate/scheme/tonlib_api.tl +++ b/tl/generate/scheme/tonlib_api.tl @@ -57,7 +57,7 @@ raw.message hash:bytes source:accountAddress destination:accountAddress value:in raw.transaction address:accountAddress utime:int53 data:bytes transaction_id:internal.transactionId fee:int64 storage_fee:int64 other_fee:int64 in_msg:raw.message out_msgs:vector = raw.Transaction; raw.transactions transactions:vector previous_transaction_id:internal.transactionId = raw.Transactions; -raw.extMessageInfo hash:bytes = raw.ExtMessageInfo; +raw.extMessageInfo hash:bytes hash_norm:bytes = raw.ExtMessageInfo; pchan.config alice_public_key:string alice_address:accountAddress bob_public_key:string bob_address:accountAddress init_timeout:int32 close_timeout:int32 channel_id:int64 = pchan.Config; diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index 10b9ed8db207f9ad7b39322937bafc107802d319..9b366407e367642cb707b63d0f93161ff34b1bf1 100644 GIT binary patch delta 106 zcmZ3{!*r&HX+sMy%VYiX8k;+K=QzlKSVBdK<$9?VCBCV}#fj;uo_T5c3?T8z6GCMs v8`$$qPEcgw;mk-Z&WO*;FUnwETN*ra=p}w65rI~;>7e+&%CsJ29Ws8V?sWR blO^r7A?j!NEinMICOgza7@JG#Z?FLXZZIFV diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index d73e715c9..ae3189ba8 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -3312,9 +3312,47 @@ td::Status TonlibClient::do_request(const tonlib_api::raw_sendMessageReturnHash& td::Promise>&& promise) { TRY_RESULT_PREFIX(body, vm::std_boc_deserialize(request.body_), TonlibError::InvalidBagOfCells("body")); auto hash = body->get_hash().as_slice().str(); + + // compute hash normalized + block::gen::Message::Record message; + if (!tlb::type_unpack_cell(body, block::gen::t_Message_Any, message)) { + return td::Status::Error("Failed to unpack Message"); + } + auto tag = block::gen::CommonMsgInfo().get_tag(*message.info); + if (tag != block::gen::CommonMsgInfo::ext_in_msg_info) { + return td::Status::Error("CommonMsgInfo tag is not ext_in_msg_info"); + } + block::gen::CommonMsgInfo::Record_ext_in_msg_info msg_info; + if (!tlb::csr_unpack(message.info, msg_info)) { + return td::Status::Error("Failed to unpack CommonMsgInfo::ext_in_msg_info"); + } + + td::Ref body_norm; + auto body_cs = message.body.write(); + if (body_cs.fetch_long(1) == 1) { + body_norm = body_cs.fetch_ref(); + } else { + body_norm = vm::CellBuilder().append_cellslice(body_cs).finalize(); + } + + auto cb = vm::CellBuilder(); + bool status = + cb.store_long_bool(2, 2) && // message$_ -> info:CommonMsgInfo -> ext_in_msg_info$10 + cb.store_long_bool(0, 2) && // message$_ -> info:CommonMsgInfo -> src:MsgAddressExt -> addr_none$00 + cb.append_cellslice_bool(msg_info.dest) && // message$_ -> info:CommonMsgInfo -> dest:MsgAddressInt + cb.store_long_bool(0, 4) && // message$_ -> info:CommonMsgInfo -> import_fee:Grams -> 0 + cb.store_long_bool(0, 1) && // message$_ -> init:(Maybe (Either StateInit ^StateInit)) -> nothing$0 + cb.store_long_bool(1, 1) && // message$_ -> body:(Either X ^X) -> right$1 + cb.store_ref_bool(body_norm); + + if (!status) { + return td::Status::Error("Failed to build normalized message"); + } + auto hash_norm = cb.finalize()->get_hash().as_slice().str(); + make_request(int_api::SendMessage{std::move(body)}, - promise.wrap([hash = std::move(hash)](auto res) { - return tonlib_api::make_object(std::move(hash)); + promise.wrap([hash = std::move(hash), hash_norm = std::move(hash_norm)](auto res) { + return tonlib_api::make_object(std::move(hash), std::move(hash_norm)); })); return td::Status::OK(); } @@ -4510,7 +4548,7 @@ auto to_tonlib_api(const vm::StackEntry& entry, int& limit) -> td::Result& stack) -> td::Result>> { - int stack_limit = 1000; + int stack_limit = 8000; std::vector> tl_stack; for (auto& entry: stack->as_span()) { TRY_RESULT(tl_entry, to_tonlib_api(entry, --stack_limit)); From 48f5657d221a52866f4167fd49adf92a3d5affed Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 25 Mar 2025 12:48:23 +0300 Subject: [PATCH 176/388] Import proof for initial block --- validator/downloaders/download-state.cpp | 16 +++++- validator/downloaders/download-state.hpp | 1 + validator/interfaces/validator-manager.h | 5 ++ validator/manager-init.cpp | 27 +++++----- validator/manager.cpp | 63 ++++++++++++++++++++++++ validator/manager.hpp | 4 ++ 6 files changed, 103 insertions(+), 13 deletions(-) diff --git a/validator/downloaders/download-state.cpp b/validator/downloaders/download-state.cpp index 8473cb228..a6de05893 100644 --- a/validator/downloaders/download-state.cpp +++ b/validator/downloaders/download-state.cpp @@ -72,7 +72,22 @@ void DownloadShardState::download_state() { checked_proof_link(); return; } + status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading proof"); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id = block_id_](td::Result R) { + if (R.is_error()) { + LOG(DEBUG) << "Cannot get proof link from import: " << R.move_as_error(); + td::actor::send_closure(SelfId, &DownloadShardState::download_proof_link); + } else { + LOG(INFO) << "Got proof link for " << block_id.to_str() << " from import"; + td::actor::send_closure(SelfId, &DownloadShardState::downloaded_proof_link, R.move_as_ok()); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::get_block_proof_link_from_import, block_id_, + masterchain_block_id_, std::move(P)); +} + +void DownloadShardState::download_proof_link() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { fail_handler(SelfId, R.move_as_error()); @@ -82,7 +97,6 @@ void DownloadShardState::download_state() { }); td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_link_request, block_id_, priority_, std::move(P)); - status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading proof"); } void DownloadShardState::downloaded_proof_link(td::BufferSlice data) { diff --git a/validator/downloaders/download-state.hpp b/validator/downloaders/download-state.hpp index bde80aae1..9b3d7fa5e 100644 --- a/validator/downloaders/download-state.hpp +++ b/validator/downloaders/download-state.hpp @@ -38,6 +38,7 @@ class DownloadShardState : public td::actor::Actor { void checked_proof_link(); void download_state(); + void download_proof_link(); void download_zero_state(); void downloaded_zero_state(td::BufferSlice data); diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index c311cc08e..3cb2a2c45 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -156,6 +156,11 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) = 0; + virtual void get_block_proof_link_from_import(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { + promise.set_error(td::Status::Error("not supported")); + } + virtual void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_shard_client_state(bool from_db, td::Promise promise) = 0; diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index 6f304680b..3b57eb9b8 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -59,7 +59,7 @@ void ValidatorManagerMasterchainReiniter::got_masterchain_handle(BlockHandle han handle_ = std::move(handle); key_blocks_.push_back(handle_); - if (opts_->initial_sync_disabled()) { + if (opts_->initial_sync_disabled() && handle_->id().seqno() == 0) { status_.set_status(PSTRING() << "downloading masterchain state " << handle_->id().seqno()); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); @@ -94,22 +94,21 @@ void ValidatorManagerMasterchainReiniter::download_proof_link() { td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::downloaded_proof_link, R.move_as_ok()); } }); - td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_link_request, handle_->id(), 2, + td::actor::send_closure(manager_, &ValidatorManager::get_block_proof_link_from_import, handle_->id(), handle_->id(), std::move(P)); } } -void ValidatorManagerMasterchainReiniter::downloaded_proof_link(td::BufferSlice proof) { - auto pp = create_proof_link(handle_->id(), std::move(proof)); - if (pp.is_error()) { - LOG(WARNING) << "bad proof link: " << pp.move_as_error(); +void ValidatorManagerMasterchainReiniter::downloaded_proof_link(td::BufferSlice data) { + auto r_proof = create_proof(handle_->id(), std::move(data)); + if (r_proof.is_error()) { + LOG(WARNING) << "bad proof link: " << r_proof.move_as_error(); download_proof_link(); return; } + auto proof = r_proof.move_as_ok(); - auto proof_link = pp.move_as_ok(); - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), db = db_, proof_link](td::Result R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), db = db_, proof](td::Result R) { if (R.is_error()) { LOG(WARNING) << "downloaded proof link failed: " << R.move_as_error(); td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_proof_link); @@ -118,11 +117,11 @@ void ValidatorManagerMasterchainReiniter::downloaded_proof_link(td::BufferSlice R.ensure(); td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::try_download_key_blocks, false); }); - td::actor::send_closure(db, &Db::add_key_block_proof_link, proof_link, std::move(P)); + td::actor::send_closure(db, &Db::add_key_block_proof_link, proof, std::move(P)); } }); - - run_check_proof_link_query(handle_->id(), proof_link, manager_, td::Timestamp::in(60.0), std::move(P)); + run_check_proof_query(handle_->id(), proof, manager_, td::Timestamp::in(60.0), std::move(P), + /* skip_check_signatures = */ true); } void ValidatorManagerMasterchainReiniter::downloaded_zero_state() { @@ -130,6 +129,10 @@ void ValidatorManagerMasterchainReiniter::downloaded_zero_state() { } void ValidatorManagerMasterchainReiniter::try_download_key_blocks(bool try_start) { + if (opts_->initial_sync_disabled()) { + download_masterchain_state(); + return; + } if (!download_new_key_blocks_until_) { if (opts_->allow_blockchain_init()) { download_new_key_blocks_until_ = td::Timestamp::in(60.0); diff --git a/validator/manager.cpp b/validator/manager.cpp index 1d75b523a..680f949ec 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -43,6 +43,7 @@ #include "td/utils/JsonBuilder.h" #include "common/delay.h" +#include "db/fileref.hpp" #include "td/utils/filesystem.h" #include "validator/stats-merger.h" @@ -1671,6 +1672,67 @@ void ValidatorManagerImpl::send_download_archive_request(BlockSeqno mc_seqno, Sh callback_->download_archive(mc_seqno, shard_prefix, std::move(tmp_dir), timeout, std::move(promise)); } +void ValidatorManagerImpl::get_block_proof_link_from_import(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { + auto it = to_import_all_.upper_bound(masterchain_block_id.seqno() + 1); + while (true) { + if (it == to_import_all_.begin()) { + promise.set_error(td::Status::Error("proof not found")); + return; + } + --it; + bool stop = false; + for (const std::string &path : it->second) { + td::BufferSlice result; + auto r_package = Package::open(path, false, false); + if (r_package.is_error()) { + LOG(WARNING) << "Cannot open package " << path << " : " << r_package.move_as_error(); + continue; + } + auto package = r_package.move_as_ok(); + package.iterate([&](std::string filename, td::BufferSlice data, td::uint64) -> bool { + auto F = FileReference::create(filename); + if (F.is_error()) { + return true; + } + auto f = F.move_as_ok(); + BlockIdExt id; + bool is_proof = false; + f.ref().visit(td::overloaded( + [&](const fileref::Block &p) { + id = p.block_id; + is_proof = false; + }, + [&](const fileref::Proof &p) { + id = p.block_id; + is_proof = true; + }, + [&](const fileref::ProofLink &p) { + id = p.block_id; + is_proof = true; + }, + [&](const auto &) {})); + if (is_proof && id == block_id) { + result = std::move(data); + return false; + } + if (shard_intersects(id.shard_full(), block_id.shard_full()) && id.seqno() < block_id.seqno()) { + stop = true; + } + return true; + }); + if (!result.empty()) { + promise.set_result(std::move(result)); + return; + } + } + if (block_id.is_masterchain() || stop) { + promise.set_error(td::Status::Error("proof not found")); + return; + } + } +} + void ValidatorManagerImpl::start_up() { db_ = create_db_actor(actor_id(this), db_root_, opts_); actor_stats_ = td::actor::create_actor("actor_stats"); @@ -1730,6 +1792,7 @@ void ValidatorManagerImpl::start_up() { ++to_import_files; } }); + to_import_all_ = to_import_; if (S.is_error()) { LOG(INFO) << "failed to load blocks from import dir: " << S; } diff --git a/validator/manager.hpp b/validator/manager.hpp index 850baf47b..0e26e014f 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -521,6 +521,9 @@ class ValidatorManagerImpl : public ValidatorManager { void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) override; + void get_block_proof_link_from_import(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) override; + void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; @@ -706,6 +709,7 @@ class ValidatorManagerImpl : public ValidatorManager { td::actor::ActorOwn serializer_; std::map> to_import_; + std::map> to_import_all_; private: std::unique_ptr callback_; From 0eeb565ad35c4b5ce1029b3d41c88a175c419ab0 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 27 Mar 2025 13:09:29 +0300 Subject: [PATCH 177/388] Perf timer for archive import --- validator/import-db-slice-local.cpp | 8 ++++++-- validator/import-db-slice-local.hpp | 2 +- validator/import-db-slice.cpp | 6 +++++- validator/import-db-slice.hpp | 2 ++ 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/validator/import-db-slice-local.cpp b/validator/import-db-slice-local.cpp index a16a26c63..38edbb10e 100644 --- a/validator/import-db-slice-local.cpp +++ b/validator/import-db-slice-local.cpp @@ -43,7 +43,10 @@ ArchiveImporterLocal::ArchiveImporterLocal(std::string db_root, td::Refget_seqno(), std::min(last_masterchain_state_->get_seqno(), shard_client_seqno_)}); } diff --git a/validator/import-db-slice.hpp b/validator/import-db-slice.hpp index 04f22642d..4cf6ce40a 100644 --- a/validator/import-db-slice.hpp +++ b/validator/import-db-slice.hpp @@ -91,6 +91,8 @@ class ArchiveImporter : public td::actor::Actor { bool imported_any_ = false; bool have_shard_blocks_ = false; std::vector files_to_cleanup_; + + td::PerfWarningTimer perf_timer_; }; } // namespace validator From 9e2664095df129c15069f31b77f3f5b45ead8524 Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Thu, 27 Mar 2025 08:21:21 -0400 Subject: [PATCH 178/388] Remove unused check for SSE4.2 (#1578) --- CMakeLists.txt | 50 -------------------------------------------------- 1 file changed, 50 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 59f7c5e08..dd193d8aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,56 +29,6 @@ if (TON_REAL_BINARY_DIR STREQUAL TON_REAL_SOURCE_DIR) message(FATAL_ERROR "In-source build failed.") endif() -# HAVE_SSE42 for crc32c and rocksdb -include(CheckCXXSourceCompiles) -# Check for SSE4.2 support in the compiler. -set(OLD_CMAKE_REQURED_FLAGS ${CMAKE_REQUIRED_FLAGS}) -if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} /arch:AVX") -else(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -msse4.2") -endif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") -check_cxx_source_compiles(" -#if defined(_MSC_VER) -#include -#else // !defined(_MSC_VER) -#include -#include -#endif // defined(_MSC_VER) - -int main() { - _mm_crc32_u8(0, 0); _mm_crc32_u32(0, 0); -#if defined(_M_X64) || defined(__x86_64__) - _mm_crc32_u64(0, 0); -#endif // defined(_M_X64) || defined(__x86_64__) - return 0; -} -" CRC32C_HAVE_SSE42) -set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQURED_FLAGS}) - -if(NOT MSVC) - set(CMAKE_REQUIRED_FLAGS "-msse4.2 -mpclmul") -endif() -CHECK_CXX_SOURCE_COMPILES(" -#include -#include -#include -int main() { - volatile uint32_t x = _mm_crc32_u32(0, 0); - const auto a = _mm_set_epi64x(0, 0); - const auto b = _mm_set_epi64x(0, 0); - const auto c = _mm_clmulepi64_si128(a, b, 0x00); - auto d = _mm_cvtsi128_si64(c); -} -" ROCKSDB_HAVE_SSE42) -unset(CMAKE_REQUIRED_FLAGS) - -if (ROCKSDB_HAVE_SSE42 AND CRC32C_HAVE_SSE42) - set(HAVE_SSE42 TRUE) -else() - set(HAVE_SSE42 FALSE) -endif() - set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED TRUE) set(CMAKE_CXX_EXTENSIONS FALSE) From 1166a40857e9c720f87380fc875e947a8d906863 Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Thu, 27 Mar 2025 08:22:33 -0400 Subject: [PATCH 179/388] Add a CMake option to disable git watcher (#1579) Whenever git watcher decides that the git state has changed, it spends multiple seconds trying to retrieve information about this changed state. This is incredibly annoying as it slows down edit-compile-run cycle for no reason. --- git_watcher.cmake | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/git_watcher.cmake b/git_watcher.cmake index 78e57ba14..5993916c4 100644 --- a/git_watcher.cmake +++ b/git_watcher.cmake @@ -315,4 +315,14 @@ function(Main) endfunction() # And off we go... -Main() +option(ENABLE_GIT_WATCHER "Use git tree information in the version strings" ON) + +if (ENABLE_GIT_WATCHER) + Main() +else() + add_custom_target(check_git) + set(GIT_RETRIEVED_STATE false) + set(GIT_IS_DIRTY false) + set(GIT_COMMIT_BODY "\"\"") + configure_file("${PRE_CONFIGURE_FILE}" "${POST_CONFIGURE_FILE}" @ONLY) +endif() From 070e27d3c6f32837c9b2b8add64e427f9287695e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 28 Mar 2025 10:54:13 +0300 Subject: [PATCH 180/388] Dictionary::multiset; optimize account storage stat --- crypto/block/account-storage-stat.cpp | 202 +++++++++++++------- crypto/block/account-storage-stat.h | 32 +++- crypto/common/bitstring.cpp | 3 + crypto/vm/dict.cpp | 254 ++++++++++++++++++++++++++ crypto/vm/dict.h | 4 +- 5 files changed, 421 insertions(+), 74 deletions(-) diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp index 8e8b6f7e1..918d2ac00 100644 --- a/crypto/block/account-storage-stat.cpp +++ b/crypto/block/account-storage-stat.cpp @@ -53,51 +53,95 @@ AccountStorageStat& AccountStorageStat::operator=(AccountStorageStat&& other) { return *this; } -td::Result AccountStorageStat::add_root(const Ref& cell) { - roots_.push_back(cell); - return add_cell(cell); -} - -td::Status AccountStorageStat::remove_root(const Ref& cell) { - auto it = std::find_if(roots_.begin(), roots_.end(), - [&](const Ref& c) { return c->get_hash() == cell->get_hash(); }); - if (it == roots_.end()) { - return td::Status::Error(PSTRING() << "no such root " << cell->get_hash().to_hex()); +td::Result AccountStorageStat::replace_roots(std::vector> new_roots) { + std::erase_if(new_roots, [](const Ref& c) { return c.is_null(); }); + if (new_roots.empty()) { + roots_.clear(); + total_bits_ = total_cells_ = 0; + dict_ = vm::Dictionary{256}; + cache_ = {}; + return CellInfo{}; } - roots_.erase(it); - return remove_cell(cell); -} -td::Result AccountStorageStat::replace_roots(std::vector> new_roots) { - std::vector> old_roots = roots_; + auto cmp = [](const Ref& c1, const Ref& c2) { return c1->get_hash() < c2->get_hash(); }; + std::sort(new_roots.begin(), new_roots.end(), cmp); + std::sort(roots_.begin(), roots_.end(), cmp); + std::vector> to_add, to_del; + std::set_difference(new_roots.begin(), new_roots.end(), roots_.begin(), roots_.end(), std::back_inserter(to_add), + cmp); + std::set_difference(roots_.begin(), roots_.end(), new_roots.begin(), new_roots.end(), std::back_inserter(to_del), + cmp); + td::uint32 max_merkle_depth = 0; - for (const Ref& root : new_roots) { - if (root.is_null()) { - continue; - } - TRY_RESULT(info, add_root(root)); + for (const Ref& root : to_add) { + TRY_RESULT(info, add_cell(root)); max_merkle_depth = std::max(max_merkle_depth, info.max_merkle_depth); } - for (const Ref& root : old_roots) { - TRY_STATUS(remove_root(root)); + for (const Ref& root : to_del) { + TRY_STATUS(remove_cell(root)); + } + + roots_ = std::move(new_roots); + td::Status S = td::Status::OK(); + std::vector>> values; + cache_.for_each([&](Entry& e) { + if (S.is_ok()) { + S = commit_entry(e, values); + } + }); + TRY_STATUS(std::move(S)); + if (!dict_.multiset(values)) { + return td::Status::Error("failed to update dictionary"); } return CellInfo{max_merkle_depth}; } +void AccountStorageStat::add_hint(const td::HashSet& hint) { + td::HashSet visited; + std::function&, bool)> dfs = [&](const Ref& cell, bool is_root) { + if (!visited.insert(cell->get_hash()).second) { + return; + } + Entry& e = get_entry(cell); + e.exists = e.exists_known = true; + if (is_root) { + fetch_entry(e).ignore(); + if (e.max_merkle_depth && e.max_merkle_depth.value() != 0) { + return; + } + } + if (hint.contains(cell->get_hash())) { + bool spec; + vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); + for (unsigned i = 0; i < cs.size_refs(); ++i) { + dfs(cs.prefetch_ref(i), false); + } + } + }; + for (const Ref& root : roots_) { + dfs(root, true); + } +} + td::Result AccountStorageStat::add_cell(const Ref& cell) { Entry& e = get_entry(cell); - ++e.refcnt; - if (e.refcnt == 0) { - return td::Status::Error(PSTRING() << "cell " << cell->get_hash().to_hex() << ": refcnt overflow"); + if (!e.exists_known || e.refcnt_diff < 0) { + TRY_STATUS(fetch_entry(e)); } - if (e.refcnt != 1) { - update_dict(e); - return CellInfo{e.max_merkle_depth}; + ++e.refcnt_diff; + if (e.exists || e.refcnt_diff > 1 || (e.refcnt && e.refcnt.value() + e.refcnt_diff != 1)) { + if (!e.max_merkle_depth) { + TRY_STATUS(fetch_entry(e)); + if (!e.max_merkle_depth) { + return td::Status::Error(PSTRING() << "unexpected unknown Merkle depth of cell " << cell->get_hash()); + } + } + return CellInfo{e.max_merkle_depth.value()}; } + td::uint32 max_merkle_depth = 0; - vm::CellSlice cs{vm::NoVm{}, cell}; - ++total_cells_; - total_bits_ += cs.size(); + bool spec; + vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); for (unsigned i = 0; i < cs.size_refs(); ++i) { TRY_RESULT(info, add_cell(cs.prefetch_ref(i))); max_merkle_depth = std::max(max_merkle_depth, info.max_merkle_depth); @@ -109,64 +153,96 @@ td::Result AccountStorageStat::add_cell(const Ref< max_merkle_depth = std::min(max_merkle_depth, MERKLE_DEPTH_LIMIT); Entry& e2 = get_entry(cell); e2.max_merkle_depth = max_merkle_depth; - update_dict(e2); return CellInfo{max_merkle_depth}; } td::Status AccountStorageStat::remove_cell(const Ref& cell) { Entry& e = get_entry(cell); - if (e.refcnt == 0) { - return td::Status::Error(PSTRING() << "cell " << cell->get_hash().to_hex() << " is not in the dict"); + if (!e.exists_known) { + TRY_STATUS(fetch_entry(e)); } - --e.refcnt; - update_dict(e); - if (e.refcnt != 0) { - return td::Status::OK(); + if (!e.exists) { + return td::Status::Error(PSTRING() << "Failed to remove cell " << cell->get_hash().to_hex() + << " : does not exist in the dict"); } - vm::CellSlice cs{vm::NoVm{}, std::move(cell)}; - if (total_cells_ == 0 || total_bits_ < cs.size()) { - return td::Status::Error("total_cell/total_bits becomes negative"); + --e.refcnt_diff; + if (e.refcnt_diff < 0 && !e.refcnt) { + TRY_STATUS(fetch_entry(e)); + } + if (e.refcnt.value() + e.refcnt_diff != 0) { + return td::Status::OK(); } - --total_cells_; - total_bits_ -= cs.size(); + bool spec; + vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); for (unsigned i = 0; i < cs.size_refs(); ++i) { TRY_STATUS(remove_cell(cs.prefetch_ref(i))); } return td::Status::OK(); } -bool AccountStorageStat::Entry::serialize(vm::CellBuilder& cb) const { - return cb.store_long_bool(refcnt, 32) && cb.store_long_bool(max_merkle_depth, 2); -} - -void AccountStorageStat::Entry::fetch(Ref cs) { - if (cs.is_null()) { - refcnt = max_merkle_depth = 0; - } else { - refcnt = (td::uint32)cs.write().fetch_ulong(32); - max_merkle_depth = (td::uint32)cs.write().fetch_ulong(2); - } -} - AccountStorageStat::Entry& AccountStorageStat::get_entry(const Ref& cell) { return cache_.apply(cell->get_hash().as_slice(), [&](Entry& e) { if (e.inited) { return; } e.inited = true; + e.cell = cell; e.hash = cell->get_hash(); - e.fetch(dict_.lookup(e.hash.as_bitslice())); }); } -void AccountStorageStat::update_dict(const Entry& e) { - if (e.refcnt == 0) { - dict_.lookup_delete(e.hash.as_bitslice()); +td::Status AccountStorageStat::fetch_entry(Entry& e) { + if (e.exists_known && e.refcnt && (!e.exists || e.max_merkle_depth)) { + return td::Status::OK(); + } + auto cs = dict_.lookup(e.hash.as_bitslice()); + if (cs.is_null()) { + e.exists = false; + e.refcnt = 0; + } else { + if (cs->size_ext() != 32 + 2) { + return td::Status::Error(PSTRING() << "invalid record for cell " << e.hash.to_hex()); + } + e.exists = true; + e.refcnt = (td::uint32)cs.write().fetch_ulong(32); + e.max_merkle_depth = (td::uint32)cs.write().fetch_ulong(2); + if (e.refcnt.value() == 0) { + return td::Status::Error(PSTRING() << "invalid refcnt=0 for cell " << e.hash.to_hex()); + } + } + e.exists_known = true; + return td::Status::OK(); +} + +td::Status AccountStorageStat::commit_entry(Entry& e, + std::vector>>& values) { + if (e.refcnt_diff == 0) { + return td::Status::OK(); + } + TRY_STATUS(fetch_entry(e)); + e.refcnt.value() += e.refcnt_diff; + e.refcnt_diff = 0; + bool spec; + if (e.refcnt.value() == 0) { + --total_cells_; + total_bits_ -= vm::load_cell_slice_special(e.cell, spec).size(); + e.exists = false; + values.emplace_back(e.hash.bits(), td::Ref{}); } else { - vm::CellBuilder cb; - CHECK(e.serialize(cb)); - dict_.set_builder(e.hash.as_bitslice(), cb); + if (!e.exists) { + ++total_cells_; + total_bits_ += vm::load_cell_slice_special(e.cell, spec).size(); + } + e.exists = true; + if (!e.max_merkle_depth) { + return td::Status::Error(PSTRING() << "Failed to store entry " << e.hash.to_hex() << " : unknown merkle depth"); + } + td::Ref cbr{true}; + auto& cb = cbr.write(); + CHECK(cb.store_long_bool(e.refcnt.value(), 32) && cb.store_long_bool(e.max_merkle_depth.value(), 2)); + values.emplace_back(e.hash.bits(), std::move(cbr)); } + return td::Status::OK(); } } // namespace block \ No newline at end of file diff --git a/crypto/block/account-storage-stat.h b/crypto/block/account-storage-stat.h index 894192d4f..83eb3af5a 100644 --- a/crypto/block/account-storage-stat.h +++ b/crypto/block/account-storage-stat.h @@ -58,9 +58,8 @@ class AccountStorageStat { return dict_.is_empty() ? td::Bits256::zero() : td::Bits256{dict_.get_root_cell()->get_hash().bits()}; } - td::Result add_root(const Ref &cell); - td::Status remove_root(const Ref &cell); - td::Result replace_roots(std::vector> new_roots); + td::Result replace_roots(std::vector> hint); + void add_hint(const td::HashSet &visited); private: vm::Dictionary dict_; @@ -72,12 +71,12 @@ class AccountStorageStat { struct Entry { bool inited = false; - vm::Cell::Hash hash; - td::uint32 refcnt = 0; - td::uint32 max_merkle_depth = 0; - - void fetch(Ref cs); - bool serialize(vm::CellBuilder &cb) const; + vm::CellHash hash; + Ref cell; + bool exists_known = false; + bool exists = false; + td::optional refcnt, max_merkle_depth; + td::int32 refcnt_diff = 0; vm::Cell::Hash key() const { return hash; @@ -111,9 +110,22 @@ class AccountStorageStat { vm::CellHashTable cache_; Entry &get_entry(const Ref &cell); - void update_dict(const Entry &e); + td::Status fetch_entry(Entry &e); + td::Status commit_entry(Entry &e, std::vector>> &values); static constexpr td::uint32 MERKLE_DEPTH_LIMIT = 3; }; +class StorageStatCalculationContext : public td::Context { + public: + explicit StorageStatCalculationContext(bool active) : active_(active) { + } + bool calculating_storage_stat() const { + return active_; + } + + private: + bool active_ = false; +}; + } // namespace block diff --git a/crypto/common/bitstring.cpp b/crypto/common/bitstring.cpp index 52e57c9a8..3a6f33119 100644 --- a/crypto/common/bitstring.cpp +++ b/crypto/common/bitstring.cpp @@ -347,6 +347,9 @@ std::size_t bits_memscan_rev(ConstBitPtr bs, std::size_t bit_count, bool cmp_to) int bits_memcmp(const unsigned char* bs1, int bs1_offs, const unsigned char* bs2, int bs2_offs, std::size_t bit_count, std::size_t* same_upto) { if (!bit_count) { + if (same_upto) { + *same_upto = 0; + } return 0; } bs1 += (bs1_offs >> 3); diff --git a/crypto/vm/dict.cpp b/crypto/vm/dict.cpp index 41f9c3396..e1e683fe3 100644 --- a/crypto/vm/dict.cpp +++ b/crypto/vm/dict.cpp @@ -1772,6 +1772,260 @@ void Dictionary::map(const simple_map_func_t& simple_map_func) { map(map_func); } +bool Dictionary::multiset(td::MutableSpan>> new_values) { + force_validate(); + auto cmp = [&](const std::pair>& a, + const std::pair>& b) { + return td::bitstring::bits_memcmp(a.first, b.first, key_bits) < 0; + }; + if (!std::is_sorted(new_values.begin(), new_values.end(), cmp)) { + std::sort(new_values.begin(), new_values.end(), cmp); + } + for (size_t i = 0; i + 1 < new_values.size(); ++i) { + if (td::bitstring::bits_memcmp(new_values[i].first, new_values[i + 1].first, key_bits) == 0) { + return false; + } + } + unsigned char key_buffer[max_key_bytes]; + try { + Ref root = dict_multiset(get_root_cell(), new_values, key_buffer, key_bits, key_bits, 0); + set_root_cell(std::move(root)); + return true; + } catch (CombineError) { + return false; + } +} + +static Ref dict_build(td::Span>> values, int total_key_len, + int prefix_len) { + if (values.empty()) { + return {}; + } + if (values.size() == 1) { + if (values[0].second.is_null()) { + throw CombineError{}; + } + CellBuilder cb; + append_dict_label(cb, values[0].first + prefix_len, total_key_len - prefix_len, total_key_len - prefix_len); + if (!cb.append_builder_bool(values[0].second)) { + throw VmError{Excno::cell_ov, "cannot store new value into a dictionary cell"}; + } + return cb.finalize(); + } + size_t common_prefix_len; + td::bitstring::bits_memcmp(values.front().first + prefix_len, values.back().first + prefix_len, + total_key_len - prefix_len, &common_prefix_len); + CHECK(prefix_len + common_prefix_len < total_key_len); + size_t idx = 0; + while (values[idx].first[prefix_len + common_prefix_len] == 0) { + ++idx; + } + Ref left = dict_build(values.substr(0, idx), total_key_len, prefix_len + common_prefix_len + 1); + Ref right = dict_build(values.substr(idx), total_key_len, prefix_len + common_prefix_len + 1); + CellBuilder cb; + append_dict_label(cb, values[0].first + prefix_len, common_prefix_len, total_key_len - prefix_len); + if (!(cb.store_ref_bool(std::move(left)) && cb.store_ref_bool(std::move(right)))) { + throw VmError{Excno::dict_err, "cannot store branch references into a dictionary fork cell"}; + } + return cb.finalize(); +} + +// Based on DictionaryFixed::dict_combine_with, but dict2 is replaced with a list of values, mode is false +Ref Dictionary::dict_multiset(Ref dict1, td::Span>> values2, + td::BitPtr key_buffer, int n, int total_key_len, int skip1) { + int prefix_len = total_key_len - n; + for (auto& [k, _] : values2) { + CHECK(td::bitstring::bits_memcmp(k, key_buffer - prefix_len, prefix_len) == 0); + } + if (dict1.is_null()) { + return dict_build(values2, total_key_len, prefix_len); + } + if (values2.empty()) { + assert(!skip1); + return dict1; + } + size_t common_prefix_len; + td::bitstring::bits_memcmp(values2.front().first + prefix_len, values2.back().first + prefix_len, + total_key_len - prefix_len, &common_prefix_len); + assert(prefix_len + common_prefix_len < total_key_len || values2.size() == 1); + // both dictionaries non-empty + // skip1: remove that much first bits from all keys in dictionary dict1 (its keys are actually n + skip1 bits long) + // resulting dictionary will have n-bit keys + LabelParser label1{dict1, n + skip1, LabelParser::chk_all}; + int l1 = label1.l_bits - skip1, l2 = (int)common_prefix_len; + assert(l1 >= 0 && l2 >= 0); + assert(!skip1 || label1.common_prefix_len(key_buffer - skip1, skip1) == skip1); + int c = label1.common_prefix_len(values2[0].first + prefix_len - skip1, skip1 + l2) - skip1; + label1.extract_label_to(key_buffer - skip1); + assert(c >= 0 && c <= l1 && c <= l2); + if (c < l1 && c < l2) { + // the two dictionaries have disjoint keys + CellBuilder cb; + append_dict_label(cb, key_buffer + c + 1, l1 - c - 1, n - c - 1); + if (!cell_builder_add_slice_bool(cb, *label1.remainder)) { + throw VmError{Excno::cell_ov, "cannot prune label of an old dictionary cell while merging dictionaries"}; + } + label1.remainder.clear(); + dict1 = cb.finalize(); + // cb.reset(); // included into finalize(); + // now dict1 has been "pruned" -- first skip1+c+1 bits removed from its root egde label + Ref dict2 = dict_build(values2, total_key_len, prefix_len + c + 1); + if (!values2[0].first[prefix_len + c]) { + std::swap(dict1, dict2); + } + // put dict1 into the left tree (with smaller labels), dict2 into the right tree + append_dict_label(cb, key_buffer, c, n); + if (!(cb.store_ref_bool(std::move(dict1)) && cb.store_ref_bool(std::move(dict2)))) { + throw VmError{Excno::dict_err, "cannot store branch references into a dictionary fork cell"}; + } + return cb.finalize(); + } + + auto combine_func = [&](CellBuilder& cb, const Ref& cb2) -> bool { + if (cb2.is_null()) { + return false; + } + cb.append_builder(*cb2); + return true; + }; + size_t idx = 0; + while (prefix_len + common_prefix_len < total_key_len && idx < values2.size() && + values2[idx].first[prefix_len + common_prefix_len] == 0) { + ++idx; + } + auto values2_left = values2.substr(0, idx); + auto values2_right = values2.substr(idx); + + if (c == l1 && c == l2) { + // funny enough, the non-skipped parts of labels of l1 and l2 match + CellBuilder cb; + append_dict_label(cb, key_buffer, c, n); + if (c == n) { + // our two dictionaries are in fact leafs with matching edge labels (keys) + if (!combine_func(cb, values2[0].second)) { + // alas, the two values did not combine, this key will be absent from resulting dictionary + return {}; + } + return cb.finalize(); + } + assert(c < n); + key_buffer += c + 1; + key_buffer[-1] = 0; + // combine left subtrees + auto c1 = dict_multiset(label1.remainder->prefetch_ref(0), values2_left, key_buffer, n - c - 1, total_key_len, 0); + key_buffer[-1] = 1; + // combine right subtrees + auto c2 = dict_multiset(label1.remainder->prefetch_ref(1), values2_right, key_buffer, n - c - 1, total_key_len, 0); + label1.remainder.clear(); + // c1 and c2 are merged left and right children of dict1 and dict2 + if (!c1.is_null() && !c2.is_null()) { + // both children non-empty, simply put them into the new node + if (!(cb.store_ref_bool(std::move(c1)) && cb.store_ref_bool(std::move(c2)))) { + throw VmError{Excno::dict_err, "cannot store branch references into a dictionary fork cell"}; + } + return cb.finalize(); + } + if (c1.is_null() && c2.is_null()) { + return {}; // both children empty, resulting dictionary also empty + } + // exactly one of c1 and c2 is non-empty, have to merge labels + bool sw = c1.is_null(); + key_buffer[-1] = sw; + if (sw) { + c1 = std::move(c2); + } + LabelParser label3{std::move(c1), n - c - 1, LabelParser::chk_all}; + label3.extract_label_to(key_buffer); + key_buffer -= c + 1; + // store combined label for the new edge + cb.reset(); + append_dict_label(cb, key_buffer, c + 1 + label3.l_bits, n); + // store payload + if (!cell_builder_add_slice_bool(cb, *label3.remainder)) { + throw VmError{Excno::cell_ov, "cannot change label of an old dictionary cell while merging edges"}; + } + return cb.finalize(); + } + if (c == l1) { + assert(c < l2); + dict1.clear(); + // children of root node of dict1 + auto c1 = label1.remainder->prefetch_ref(0); + auto c2 = label1.remainder->prefetch_ref(1); + label1.remainder.clear(); + // have to merge dict2 with one of the children of dict1 + td::bitstring::bits_memcpy(key_buffer, values2[0].first + prefix_len, l2); // dict2 has longer label, extract it + bool sw = key_buffer[c]; + if (!sw) { + // merge c1 with dict2 + c1 = dict_multiset(std::move(c1), values2, key_buffer + c + 1, n - c - 1, total_key_len, 0); + } else { + // merge c2 with dict2 + c2 = dict_multiset(std::move(c2), values2, key_buffer + c + 1, n - c - 1, total_key_len, 0); + } + if (!c1.is_null() && !c2.is_null()) { + CellBuilder cb; + append_dict_label(cb, key_buffer, c, n); + if (!(cb.store_ref_bool(std::move(c1)) && cb.store_ref_bool(std::move(c2)))) { + throw VmError{Excno::dict_err, "cannot store branch references into a dictionary fork cell"}; + } + return cb.finalize(); + } + // one of children is empty, have to merge root edges + key_buffer[c] = !sw; + if (!sw) { + std::swap(c1, c2); + } + assert(!c1.is_null() && c2.is_null()); + LabelParser label3{std::move(c1), n - c - 1, LabelParser::chk_all}; + label3.extract_label_to(key_buffer + c + 1); + CellBuilder cb; + append_dict_label(cb, key_buffer, c + 1 + label3.l_bits, n); + // store payload + if (!cell_builder_add_slice_bool(cb, *label3.remainder)) { + throw VmError{Excno::cell_ov, "cannot change label of an old dictionary cell while merging edges"}; + } + return cb.finalize(); + } else { + assert(c == l2 && c < l1); + // have to merge dict1 with one of the children of dict2 + bool sw = key_buffer[c]; + Ref c1, c2; + if (!sw) { + // merge dict1 with c1 + c1 = dict_multiset(std::move(dict1), values2_left, key_buffer + c + 1, n - c - 1, total_key_len, skip1 + c + 1); + c2 = dict_build(values2_right, total_key_len, prefix_len + l2 + 1); + } else { + // merge dict1 with c2 + c2 = dict_multiset(std::move(dict1), values2_right, key_buffer + c + 1, n - c - 1, total_key_len, skip1 + c + 1); + c1 = dict_build(values2_left, total_key_len, prefix_len + l2 + 1); + } + if (!c1.is_null() && !c2.is_null()) { + CellBuilder cb; + append_dict_label(cb, key_buffer, c, n); + if (!(cb.store_ref_bool(std::move(c1)) && cb.store_ref_bool(std::move(c2)))) { + throw VmError{Excno::dict_err, "cannot store branch references into a dictionary fork cell"}; + } + return cb.finalize(); + } + // one of children is empty, have to merge root edges + key_buffer[c] = !sw; + if (!sw) { + std::swap(c1, c2); + } + assert(!c1.is_null() && c2.is_null()); + LabelParser label3{std::move(c1), n - c - 1, LabelParser::chk_all}; + label3.extract_label_to(key_buffer + c + 1); + CellBuilder cb; + append_dict_label(cb, key_buffer, c + 1 + label3.l_bits, n); + // store payload + if (!cell_builder_add_slice_bool(cb, *label3.remainder)) { + throw VmError{Excno::cell_ov, "cannot change label of an old dictionary cell while merging edges"}; + } + return cb.finalize(); + } +} + // mode: +1 = forbid empty dict1 with non-empty dict2 // +2 = forbid empty dict2 with non-empty dict1 Ref DictionaryFixed::dict_combine_with(Ref dict1, Ref dict2, td::BitPtr key_buffer, int n, diff --git a/crypto/vm/dict.h b/crypto/vm/dict.h index c4044963f..e22b90946 100644 --- a/crypto/vm/dict.h +++ b/crypto/vm/dict.h @@ -527,13 +527,15 @@ class Dictionary final : public DictionaryFixed { auto range(bool rev = false, bool sgnd = false) { return dict_range(*this, rev, sgnd); } + bool multiset(td::MutableSpan>> new_values); private: bool check_fork(CellSlice& cs, Ref c1, Ref c2, int n) const override { return cs.empty_ext(); } static Ref extract_value_ref(Ref cs); - std::pair, int> dict_filter(Ref dict, td::BitPtr key, int n, const filter_func_t& check_leaf) const; + static Ref dict_multiset(Ref dict1, td::Span>> values2, + td::BitPtr key_buffer, int n, int total_key_len, int skip1); }; class PrefixDictionary final : public DictionaryBase { From ad3c05938dd7b5a1b77444e545ef039853bd3377 Mon Sep 17 00:00:00 2001 From: leopardracer <136604165+leopardracer@users.noreply.github.com> Date: Fri, 28 Mar 2025 11:19:45 +0200 Subject: [PATCH 181/388] fix: typos in comments (#1580) * Update collator.cpp * Update Stack.fif * Update parse-func.cpp --- crypto/fift/lib/Stack.fif | 2 +- crypto/func/parse-func.cpp | 2 +- validator/impl/collator.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crypto/fift/lib/Stack.fif b/crypto/fift/lib/Stack.fif index c7ef6f8e5..6efc0a060 100644 --- a/crypto/fift/lib/Stack.fif +++ b/crypto/fift/lib/Stack.fif @@ -1,4 +1,4 @@ -library Stack // advanced stack manupulation library +library Stack // advanced stack manipulation library "Lists.fif" include // S(a b c - a c 2 a b) would compile to code performing the requested stack manipulation diff --git a/crypto/func/parse-func.cpp b/crypto/func/parse-func.cpp index 4fc1cece5..179cc9bb4 100644 --- a/crypto/func/parse-func.cpp +++ b/crypto/func/parse-func.cpp @@ -288,7 +288,7 @@ void parse_const_decl(Lexer& lex) { code.emplace_back(loc, Op::_Import, std::vector()); auto tmp_vars = x->pre_compile(code); code.emplace_back(loc, Op::_Return, std::move(tmp_vars)); - code.emplace_back(loc, Op::_Nop); // This is neccessary to prevent SIGSEGV! + code.emplace_back(loc, Op::_Nop); // This is necessary to prevent SIGSEGV! // It is REQUIRED to execute "optimizations" as in func.cpp code.simplify_var_types(); code.prune_unreachable_code(); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 2a6d7a2b2..805240b3e 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -5723,7 +5723,7 @@ void Collator::return_block_candidate(td::Result saved) { * @returns Result indicating the success or failure of the registration. * - If the external message is invalid, returns an error. * - If the external message has been previously rejected, returns an error - * - If the external message has been previuosly registered and accepted, returns false. + * - If the external message has been previously registered and accepted, returns false. * - Otherwise returns true. */ td::Result Collator::register_external_message_cell(Ref ext_msg, const ExtMessage::Hash& ext_hash, From 57772a9bfce29aa857b2d7393441f13b68049984 Mon Sep 17 00:00:00 2001 From: Stanislav-Povolotsky Date: Fri, 28 Mar 2025 10:30:39 +0100 Subject: [PATCH 182/388] Fix TonlibClient.cpp (#1534) --- tonlib/tonlib/TonlibClient.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index ae3189ba8..b137ee769 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -268,7 +268,7 @@ td::Result> add_extra_currencies(const td::Ref &e1, block::CurrencyCollection c1{td::zero_refint(), e1}; block::CurrencyCollection c2{td::zero_refint(), e2}; TRY_RESULT_ASSIGN(c1, TRY_VM(td::Result{c1 + c2})); - if (c1.is_valid()) { + if (!c1.is_valid()) { return td::Status::Error("Failed to add extra currencies"); } return c1.extra; From 353c621d015306081762b5f7acec44871a1f43bf Mon Sep 17 00:00:00 2001 From: neodix42 Date: Sat, 29 Mar 2025 17:53:54 +0400 Subject: [PATCH 183/388] Improve github actions utilizing github cache and ccache (#1582) * improve github actions by utilizing github cache and ccache. * download libmicrohttpd from ftp * minor fix * no cache * no cache 2 * no cache test 3 * with cache test 1 * with cache test 2 * test without gh cache * test with gh cache --- .../build-ton-linux-android-tonlib.yml | 8 ++ .../build-ton-linux-arm64-appimage.yml | 49 +++++++++- .../build-ton-linux-arm64-shared.yml | 38 ++++++-- .../build-ton-linux-x86-64-appimage.yml | 48 +++++++++- .../build-ton-linux-x86-64-shared.yml | 38 ++++++-- .../build-ton-macos-13-x86-64-portable.yml | 34 ++++++- .../build-ton-macos-14-arm64-portable.yml | 34 ++++++- .../build-ton-macos-15-arm64-shared.yml | 30 ++++-- .../build-ton-macos-arm64-shared.yml | 30 ++++-- .../build-ton-macos-x86-64-shared.yml | 30 ++++-- .../workflows/build-ton-wasm-emscripten.yml | 22 ++++- .github/workflows/ton-x86-64-windows.yml | 17 ++++ assembly/native/build-macos-portable.sh | 89 ++++++++++-------- assembly/native/build-macos-shared.sh | 25 +++-- assembly/native/build-ubuntu-appimages.sh | 28 ++++-- assembly/native/build-ubuntu-portable-libs.sh | 91 +++++++++++-------- assembly/native/build-ubuntu-portable.sh | 83 +++++++++-------- assembly/native/build-ubuntu-shared.sh | 37 ++++---- assembly/native/build-windows-2019.bat | 21 ++--- assembly/native/build-windows.bat | 21 ++--- assembly/wasm/fift-func-wasm-build-ubuntu.sh | 83 ++++++++--------- 21 files changed, 596 insertions(+), 260 deletions(-) diff --git a/.github/workflows/build-ton-linux-android-tonlib.yml b/.github/workflows/build-ton-linux-android-tonlib.yml index b1ef52818..5cd74e363 100644 --- a/.github/workflows/build-ton-linux-android-tonlib.yml +++ b/.github/workflows/build-ton-linux-android-tonlib.yml @@ -18,6 +18,14 @@ jobs: sudo apt-get install -y build-essential git cmake ninja-build automake libtool texinfo autoconf libgflags-dev \ zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev \ libtool autoconf libsodium-dev libsecp256k1-dev liblz4-dev + + + - name: Cache Android NDK + id: cache-android-ndk + uses: actions/cache@v4 + with: + path: android-ndk-r25b + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-cache-android-ndk-${{ hashFiles('**/assembly/android/build-android-tonlib.sh') }} - name: Build TON run: | diff --git a/.github/workflows/build-ton-linux-arm64-appimage.yml b/.github/workflows/build-ton-linux-arm64-appimage.yml index d464d8a2a..289cd2a7e 100644 --- a/.github/workflows/build-ton-linux-arm64-appimage.yml +++ b/.github/workflows/build-ton-linux-arm64-appimage.yml @@ -12,17 +12,45 @@ jobs: with: submodules: 'recursive' + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + - name: Install system libraries run: | sudo apt update - sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev + sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache sudo apt remove libgsl-dev + mkdir ~/.ccache 3pp - name: Install clang-16 run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 16 all + sudo ./llvm.sh 16 clang + + - name: Cache 3pp + id: cache-3pp + uses: actions/cache@v4 + with: + path: 3pp + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-3pp-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} + + - name: Cache OpenSSL + id: cache-openssl + uses: actions/cache@v4 + with: + path: openssl_3 + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-openssl_3-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} + + - name: Restore cache TON + uses: actions/cache/restore@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-portable-ccache - name: Build TON run: | @@ -30,7 +58,14 @@ jobs: git submodule update cp assembly/native/build-ubuntu-appimages.sh . chmod +x build-ubuntu-appimages.sh - ./build-ubuntu-appimages.sh -a + ./build-ubuntu-appimages.sh -a -c + ccache -sp + + - name: Save cache TON + uses: actions/cache/save@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} - name: Make AppImages run: | @@ -41,12 +76,18 @@ jobs: ./create-appimages.sh aarch64 rm -rf artifacts + - name: Save/Restore cache TON libs + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-portable-libs-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-portable-libs-ccache - name: Build TON libs run: | cp assembly/native/build-ubuntu-portable-libs.sh . chmod +x build-ubuntu-portable-libs.sh - ./build-ubuntu-portable-libs.sh -a + ./build-ubuntu-portable-libs.sh -a -c cp ./artifacts/libtonlibjson.so appimages/artifacts/ cp ./artifacts/libemulator.so appimages/artifacts/ diff --git a/.github/workflows/build-ton-linux-arm64-shared.yml b/.github/workflows/build-ton-linux-arm64-shared.yml index 6433df0b5..540fffece 100644 --- a/.github/workflows/build-ton-linux-arm64-shared.yml +++ b/.github/workflows/build-ton-linux-arm64-shared.yml @@ -16,17 +16,39 @@ jobs: with: submodules: 'recursive' + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + - name: Install system libraries run: | sudo apt-get update - sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev + sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev ccache + mkdir ~/.ccache - if: matrix.os != 'ubuntu-24.04-arm' name: Install llvm-16 run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 16 all + sudo ./llvm.sh 16 clang + + - name: Cache OpenSSL + id: cache-openssl + uses: actions/cache@v4 + with: + path: openssl_3 + key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.os }}-openssl_3-${{ hashFiles('**/assembly/native/build-ubuntu-shared.sh') }} + + - name: Cache TON test + id: cache-ton + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.os }}-shared-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.os }}-shared-ccache - name: Build TON run: | @@ -34,10 +56,10 @@ jobs: git submodule update cp assembly/native/build-ubuntu-shared.sh . chmod +x build-ubuntu-shared.sh - ./build-ubuntu-shared.sh -t -a + ./build-ubuntu-shared.sh -t -c + ccache -sp - - name: Upload artifacts - uses: actions/upload-artifact@master - with: - name: ton-binaries-${{ matrix.os }} - path: artifacts + - name: Run Tests + run: | + cd build + ctest --output-on-failure --timeout 1800 diff --git a/.github/workflows/build-ton-linux-x86-64-appimage.yml b/.github/workflows/build-ton-linux-x86-64-appimage.yml index 4f78ece9d..0f5645a3a 100644 --- a/.github/workflows/build-ton-linux-x86-64-appimage.yml +++ b/.github/workflows/build-ton-linux-x86-64-appimage.yml @@ -12,11 +12,18 @@ jobs: with: submodules: 'recursive' + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + - name: Install system libraries run: | sudo apt update - sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev + sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache sudo apt remove libgsl-dev + mkdir ~/.ccache 3pp - name: Install gcc-11 g++-11 run: | @@ -28,7 +35,27 @@ jobs: run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 16 all + sudo ./llvm.sh 16 clang + + - name: Cache 3pp + id: cache-3pp + uses: actions/cache@v4 + with: + path: 3pp + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-3pp-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} + + - name: Cache OpenSSL + id: cache-openssl + uses: actions/cache@v4 + with: + path: openssl_3 + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-openssl_3-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} + + - name: Restore cache TON + uses: actions/cache/restore@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-portable-ccache - name: Build TON run: | @@ -36,7 +63,14 @@ jobs: git submodule update cp assembly/native/build-ubuntu-appimages.sh . chmod +x build-ubuntu-appimages.sh - ./build-ubuntu-appimages.sh -a + ./build-ubuntu-appimages.sh -a -c + ccache -sp + + - name: Save cache TON + uses: actions/cache/save@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} - name: Make AppImages run: | @@ -47,12 +81,18 @@ jobs: ./create-appimages.sh x86_64 rm -rf artifacts + - name: Save/Restore cache TON libs + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-portable-libs-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-portable-libs-ccache - name: Build TON libs run: | cp assembly/native/build-ubuntu-portable-libs.sh . chmod +x build-ubuntu-portable-libs.sh - ./build-ubuntu-portable-libs.sh -a + ./build-ubuntu-portable-libs.sh -a -c cp ./artifacts/libtonlibjson.so appimages/artifacts/ cp ./artifacts/libemulator.so appimages/artifacts/ diff --git a/.github/workflows/build-ton-linux-x86-64-shared.yml b/.github/workflows/build-ton-linux-x86-64-shared.yml index bf78f7df1..37eddf369 100644 --- a/.github/workflows/build-ton-linux-x86-64-shared.yml +++ b/.github/workflows/build-ton-linux-x86-64-shared.yml @@ -16,12 +16,20 @@ jobs: with: submodules: 'recursive' + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + - name: Install system libraries run: | sudo apt-get update - sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev + sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev ccache + mkdir ~/.ccache - if: matrix.os == 'ubuntu-20.04' + name: Install gcc-11 run: | sudo apt install -y manpages-dev software-properties-common sudo add-apt-repository ppa:ubuntu-toolchain-r/test @@ -31,7 +39,21 @@ jobs: run: | wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh - sudo ./llvm.sh 16 all + sudo ./llvm.sh 16 clang + + - name: Cache OpenSSL + id: cache-openssl + uses: actions/cache@v4 + with: + path: openssl_3 + key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.os }}-openssl_3-${{ hashFiles('**/assembly/native/build-ubuntu-shared.sh') }} + + - name: Restore cache TON + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.os }}-shared-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.os }}-shared-ccache - name: Build TON run: | @@ -39,10 +61,10 @@ jobs: git submodule update cp assembly/native/build-ubuntu-shared.sh . chmod +x build-ubuntu-shared.sh - ./build-ubuntu-shared.sh -t -a + ./build-ubuntu-shared.sh -t -c + ccache -sp - - name: Upload artifacts - uses: actions/upload-artifact@master - with: - name: ton-binaries-${{ matrix.os }} - path: artifacts + - name: Run Tests + run: | + cd build + ctest --output-on-failure --timeout 1800 diff --git a/.github/workflows/build-ton-macos-13-x86-64-portable.yml b/.github/workflows/build-ton-macos-13-x86-64-portable.yml index 5e50a4680..6a6a6a255 100644 --- a/.github/workflows/build-ton-macos-13-x86-64-portable.yml +++ b/.github/workflows/build-ton-macos-13-x86-64-portable.yml @@ -12,13 +12,45 @@ jobs: with: submodules: 'recursive' + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + + - name: Create directories + run: | + mkdir -p ~/.ccache + mkdir -p 3pp + + - name: Cache 3pp + id: cache-3pp + uses: actions/cache@v4 + with: + path: 3pp + key: ${{ runner.os }}-${{ runner.arch }}-13-3pp-${{ hashFiles('**/assembly/native/build-macos-portable.sh') }} + + - name: Cache TON test + id: cache-ton + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-13-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-13-portable-ccache + - name: Build TON run: | git submodule sync --recursive git submodule update cp assembly/native/build-macos-portable.sh . chmod +x build-macos-portable.sh - ./build-macos-portable.sh -t -a + ./build-macos-portable.sh -t -a -c -o 13.0 + ccache -sp + + - name: Run Tests + run: | + cd build + ctest --output-on-failure --timeout 1800 - name: Upload artifacts uses: actions/upload-artifact@master diff --git a/.github/workflows/build-ton-macos-14-arm64-portable.yml b/.github/workflows/build-ton-macos-14-arm64-portable.yml index 8eb3af70a..995ff1c40 100644 --- a/.github/workflows/build-ton-macos-14-arm64-portable.yml +++ b/.github/workflows/build-ton-macos-14-arm64-portable.yml @@ -12,13 +12,45 @@ jobs: with: submodules: 'recursive' + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + + - name: Create directories + run: | + mkdir -p ~/.ccache + mkdir -p 3pp + + - name: Cache 3pp + id: cache-3pp + uses: actions/cache@v4 + with: + path: 3pp + key: ${{ runner.os }}-${{ runner.arch }}-14-3pp-${{ hashFiles('**/assembly/native/build-macos-portable.sh') }} + + - name: Cache TON + id: cache-ton + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-14-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-14-portable-ccache + - name: Build TON run: | git submodule sync --recursive git submodule update cp assembly/native/build-macos-portable.sh . chmod +x build-macos-portable.sh - ./build-macos-portable.sh -t -a + ./build-macos-portable.sh -t -a -c -o 14.0 + ccache -sp + + - name: Run Tests + run: | + cd build + ctest --output-on-failure --timeout 1800 - name: Upload artifacts uses: actions/upload-artifact@master diff --git a/.github/workflows/build-ton-macos-15-arm64-shared.yml b/.github/workflows/build-ton-macos-15-arm64-shared.yml index 00d8f6392..1423b4eb0 100644 --- a/.github/workflows/build-ton-macos-15-arm64-shared.yml +++ b/.github/workflows/build-ton-macos-15-arm64-shared.yml @@ -12,16 +12,34 @@ jobs: with: submodules: 'recursive' + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + + - name: Create ~/.ccache + run: | + mkdir -p ~/.ccache + + - name: Cache TON + id: cache-ton + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-15-shared-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-15-shared-ccache + - name: Build TON run: | git submodule sync --recursive git submodule update cp assembly/native/build-macos-shared.sh . chmod +x build-macos-shared.sh - ./build-macos-shared.sh -t -a + ./build-macos-shared.sh -t -c -o 15.0 + ccache -sp - - name: Upload artifacts - uses: actions/upload-artifact@master - with: - name: ton-binaries-macos-15 - path: artifacts + - name: Run Tests + run: | + cd build + ctest --output-on-failure --timeout 1800 diff --git a/.github/workflows/build-ton-macos-arm64-shared.yml b/.github/workflows/build-ton-macos-arm64-shared.yml index 7481f9ff6..5ef9266af 100644 --- a/.github/workflows/build-ton-macos-arm64-shared.yml +++ b/.github/workflows/build-ton-macos-arm64-shared.yml @@ -12,16 +12,34 @@ jobs: with: submodules: 'recursive' + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + + - name: Create ~/.ccache + run: | + mkdir -p ~/.ccache + + - name: Cache TON + id: cache-ton + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-14-shared-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-14-shared-ccache + - name: Build TON run: | git submodule sync --recursive git submodule update cp assembly/native/build-macos-shared.sh . chmod +x build-macos-shared.sh - ./build-macos-shared.sh -t -a + ./build-macos-shared.sh -t -c -o 14.0 + ccache -sp - - name: Upload artifacts - uses: actions/upload-artifact@master - with: - name: ton-binaries-macos-14 - path: artifacts + - name: Run Tests + run: | + cd build + ctest --output-on-failure --timeout 1800 diff --git a/.github/workflows/build-ton-macos-x86-64-shared.yml b/.github/workflows/build-ton-macos-x86-64-shared.yml index 6a69b2e35..6200af663 100644 --- a/.github/workflows/build-ton-macos-x86-64-shared.yml +++ b/.github/workflows/build-ton-macos-x86-64-shared.yml @@ -12,16 +12,34 @@ jobs: with: submodules: 'recursive' + - name: Date Stamp + shell: bash + id: date-stamp + run: | + echo "timestamp=$(date -u "+%Y%m%d%H%M_%S")" >> "$GITHUB_OUTPUT" + + - name: Create ~/.ccache + run: | + mkdir -p ~/.ccache + + - name: Cache TON test + id: cache-ton + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ${{ runner.os }}-${{ runner.arch }}-13-shared-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-13-shared-ccache + - name: Build TON run: | git submodule sync --recursive git submodule update cp assembly/native/build-macos-shared.sh . chmod +x build-macos-shared.sh - ./build-macos-shared.sh -t -a + ./build-macos-shared.sh -t -c + ccache -sp - - name: Upload artifacts - uses: actions/upload-artifact@master - with: - name: ton-binaries-macos-13 - path: artifacts + - name: Run Tests + run: | + cd build + ctest --output-on-failure --timeout 1800 diff --git a/.github/workflows/build-ton-wasm-emscripten.yml b/.github/workflows/build-ton-wasm-emscripten.yml index bac0cf984..f9f861148 100644 --- a/.github/workflows/build-ton-wasm-emscripten.yml +++ b/.github/workflows/build-ton-wasm-emscripten.yml @@ -16,6 +16,27 @@ jobs: run: | sudo apt-get update sudo apt-get install -y build-essential git openssl cmake ninja-build zlib1g-dev libssl-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev + mkdir -p 3pp_emscripten + + - name: Install clang-16 + run: | + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 16 clang + + - name: Cache OpenSSL + id: cache-openssl + uses: actions/cache@v4 + with: + path: openssl_3 + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-openssl_3-${{ hashFiles('**/assembly/wasm/fift-func-wasm-build-ubuntu.sh') }} + + - name: Cache 3pp Emscripten + id: cache-3pp-emscripten + uses: actions/cache@v4 + with: + path: 3pp_emscripten + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-3pp_em-${{ hashFiles('**/assembly/wasm/fift-func-wasm-build-ubuntu.sh') }} - name: Build TON WASM artifacts run: | @@ -43,7 +64,6 @@ jobs: cp artifacts/funcfiftlib.js func-js/node_modules/@ton-community/func-js-bin/dist/funcfiftlib.js npx func-js stdlib.fc intrinsics.fc --fift ./output.f - - name: Upload artifacts uses: actions/upload-artifact@master with: diff --git a/.github/workflows/ton-x86-64-windows.yml b/.github/workflows/ton-x86-64-windows.yml index baaad778b..b930d25a9 100644 --- a/.github/workflows/ton-x86-64-windows.yml +++ b/.github/workflows/ton-x86-64-windows.yml @@ -21,6 +21,17 @@ jobs: with: submodules: 'recursive' + - name: Create directories + run: | + mkdir third_libs + + - name: Cache 3pp + id: cache-3pp + uses: actions/cache@v4 + with: + path: third_libs + key: ${{ runner.os }}-${{ runner.arch }}-windows-2019-3pp-${{ hashFiles('**/assembly/native/build-windows-2019.bat') }} + - name: Build TON run: | git submodule sync --recursive @@ -28,6 +39,12 @@ jobs: copy assembly\native\build-windows-github-2019.bat . copy assembly\native\build-windows-2019.bat . build-windows-github-2019.bat Enterprise + ccache -sp + +# - name: Run Tests +# run: | +# cd build +# ctest -C Release --output-on-failure -E "test-bigint" --timeout 1800 - name: Upload artifacts uses: actions/upload-artifact@master diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index b785339e8..81ac507b7 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -2,14 +2,16 @@ with_tests=false with_artifacts=false -OSX_TARGET=10.15 +with_ccache=false +OSX_TARGET=11.0 -while getopts 'tao:' flag; do +while getopts 'taco:' flag; do case "${flag}" in t) with_tests=true ;; a) with_artifacts=true ;; o) OSX_TARGET=${OPTARG} ;; + c) with_ccache=true ;; *) break ;; esac @@ -25,8 +27,19 @@ fi export NONINTERACTIVE=1 brew install ninja pkg-config automake libtool autoconf texinfo +export PATH=/usr/local/opt/ccache/libexec:$PATH brew install llvm@16 +if [ "$with_ccache" = true ]; then + brew install ccache + mkdir -p ~/.ccache + export CCACHE_DIR=~/.ccache + ccache -M 0 + test $? -eq 0 || { echo "ccache not installed"; exit 1; } +else + export CCACHE_DISABLE=1 +fi + if [ -f /opt/homebrew/opt/llvm@16/bin/clang ]; then export CC=/opt/homebrew/opt/llvm@16/bin/clang @@ -35,77 +48,79 @@ else export CC=/usr/local/opt/llvm@16/bin/clang export CXX=/usr/local/opt/llvm@16/bin/clang++ fi -export CCACHE_DISABLE=1 -if [ ! -d "lz4" ]; then -git clone https://github.com/lz4/lz4.git -cd lz4 +if [ ! -d "../3pp/lz4" ]; then +mkdir -p ../3pp +git clone https://github.com/lz4/lz4.git ../3pp/lz4 +cd ../3pp/lz4 lz4Path=`pwd` git checkout v1.9.4 -make -j12 +make -j$(nproc) test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } -cd .. +cd ../../build # ./lib/liblz4.a # ./lib else - lz4Path=$(pwd)/lz4 + lz4Path=$(pwd)/../3pp/lz4 echo "Using compiled lz4" fi -if [ ! -d "libsodium" ]; then +if [ ! -d "../3pp/libsodium" ]; then export LIBSODIUM_FULL_BUILD=1 - git clone https://github.com/jedisct1/libsodium.git - cd libsodium + git clone https://github.com/jedisct1/libsodium.git ../3pp/libsodium + cd ../3pp/libsodium sodiumPath=`pwd` git checkout 1.0.18 ./autogen.sh ./configure --with-pic --enable-static - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile libsodium"; exit 1; } - cd .. + cd ../../build else - sodiumPath=$(pwd)/libsodium + sodiumPath=$(pwd)/../3pp/libsodium echo "Using compiled libsodium" fi -if [ ! -d "openssl_3" ]; then - git clone https://github.com/openssl/openssl openssl_3 - cd openssl_3 +if [ ! -d "../3pp/openssl_3" ]; then + git clone https://github.com/openssl/openssl ../3pp/openssl_3 + cd ../3pp/openssl_3 opensslPath=`pwd` git checkout openssl-3.1.4 ./config - make build_libs -j12 + make build_libs -j$(nproc) test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } - cd .. + cd ../../build else - opensslPath=$(pwd)/openssl_3 + opensslPath=$(pwd)/../3pp/openssl_3 echo "Using compiled openssl_3" fi -if [ ! -d "zlib" ]; then - git clone https://github.com/madler/zlib.git - cd zlib +if [ ! -d "../3pp/zlib" ]; then + git clone https://github.com/madler/zlib.git ../3pp/zlib + cd ../3pp/zlib zlibPath=`pwd` ./configure --static - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile zlib"; exit 1; } - cd .. + cd ../../build else - zlibPath=$(pwd)/zlib + zlibPath=$(pwd)/../3pp/zlib echo "Using compiled zlib" fi -if [ ! -d "libmicrohttpd" ]; then - git clone https://git.gnunet.org/libmicrohttpd.git - cd libmicrohttpd +if [ ! -d "../3pp/libmicrohttpd" ]; then + mkdir -p ../3pp/libmicrohttpd + wget -O ../3pp/libmicrohttpd/libmicrohttpd-1.0.1.tar.gz https://ftpmirror.gnu.org/libmicrohttpd/libmicrohttpd-1.0.1.tar.gz + cd ../3pp/libmicrohttpd/ + tar xf libmicrohttpd-1.0.1.tar.gz + cd libmicrohttpd-1.0.1 libmicrohttpdPath=`pwd` - ./autogen.sh ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile libmicrohttpd"; exit 1; } - cd .. + cd ../../../build else - libmicrohttpdPath=$(pwd)/libmicrohttpd + libmicrohttpdPath=$(pwd)/../3pp/libmicrohttpd/libmicrohttpd-1.0.1 echo "Using compiled libmicrohttpd" fi @@ -181,9 +196,3 @@ if [ "$with_artifacts" = true ]; then rsync -r crypto/fift/lib artifacts/ chmod -R +x artifacts/* fi - -if [ "$with_tests" = true ]; then - cd build -# ctest --output-on-failure -E "test-catchain|test-actors" - ctest --output-on-failure -fi diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh index 5c4a5fe07..dbb4e7d29 100644 --- a/assembly/native/build-macos-shared.sh +++ b/assembly/native/build-macos-shared.sh @@ -2,14 +2,16 @@ with_tests=false with_artifacts=false -OSX_TARGET=10.15 +with_ccache=false +OSX_TARGET=11.0 -while getopts 'tao:' flag; do +while getopts 'taco:' flag; do case "${flag}" in t) with_tests=true ;; a) with_artifacts=true ;; o) OSX_TARGET=${OPTARG} ;; + c) with_ccache=true ;; *) break ;; esac @@ -25,8 +27,19 @@ fi export NONINTERACTIVE=1 brew install ninja libsodium libmicrohttpd pkg-config automake libtool autoconf gnutls +export PATH=/usr/local/opt/ccache/libexec:$PATH brew install llvm@16 +if [ "$with_ccache" = true ]; then + brew install ccache + mkdir -p ~/.ccache + export CCACHE_DIR=~/.ccache + ccache -M 0 + test $? -eq 0 || { echo "ccache not installed"; exit 1; } +else + export CCACHE_DISABLE=1 +fi + if [ -f /opt/homebrew/opt/llvm@16/bin/clang ]; then export CC=/opt/homebrew/opt/llvm@16/bin/clang export CXX=/opt/homebrew/opt/llvm@16/bin/clang++ @@ -34,14 +47,13 @@ else export CC=/usr/local/opt/llvm@16/bin/clang export CXX=/usr/local/opt/llvm@16/bin/clang++ fi -export CCACHE_DISABLE=1 if [ ! -d "lz4" ]; then git clone https://github.com/lz4/lz4 cd lz4 lz4Path=`pwd` git checkout v1.9.4 - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } cd .. else @@ -110,8 +122,3 @@ if [ "$with_artifacts" = true ]; then chmod -R +x artifacts/* fi -if [ "$with_tests" = true ]; then - cd build -# ctest --output-on-failure -E "test-catchain|test-actors" - ctest --output-on-failure --timeout 1800 -fi diff --git a/assembly/native/build-ubuntu-appimages.sh b/assembly/native/build-ubuntu-appimages.sh index 4e63234d9..458ca9d0b 100644 --- a/assembly/native/build-ubuntu-appimages.sh +++ b/assembly/native/build-ubuntu-appimages.sh @@ -2,17 +2,27 @@ with_tests=false with_artifacts=false +with_ccache=false - -while getopts 'ta' flag; do +while getopts 'tac' flag; do case "${flag}" in t) with_tests=true ;; a) with_artifacts=true ;; + c) with_ccache=true ;; *) break ;; esac done +if [ "$with_ccache" = true ]; then + mkdir -p ~/.ccache + export CCACHE_DIR=~/.ccache + ccache -M 0 + test $? -eq 0 || { echo "ccache not installed"; exit 1; } +else + export CCACHE_DISABLE=1 +fi + if [ ! -d "build" ]; then mkdir build cd build @@ -23,19 +33,18 @@ fi export CC=$(which clang-16) export CXX=$(which clang++-16) -export CCACHE_DISABLE=1 -if [ ! -d "openssl_3" ]; then - git clone https://github.com/openssl/openssl openssl_3 - cd openssl_3 +if [ ! -d "../openssl_3" ]; then + git clone https://github.com/openssl/openssl ../openssl_3 + cd ../openssl_3 opensslPath=`pwd` git checkout openssl-3.1.4 ./config - make build_libs -j12 + make build_libs -j$(nproc) test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } - cd .. + cd ../build else - opensslPath=$(pwd)/openssl_3 + opensslPath=$(pwd)/../openssl_3 echo "Using compiled openssl_3" fi @@ -104,6 +113,5 @@ fi if [ "$with_tests" = true ]; then cd build -# ctest --output-on-failure -E "test-catchain|test-actors|test-smartcont|test-adnl|test-validator-session-state|test-dht|test-rldp" ctest --output-on-failure --timeout 1800 fi diff --git a/assembly/native/build-ubuntu-portable-libs.sh b/assembly/native/build-ubuntu-portable-libs.sh index b634ce0c4..c7bffd40e 100644 --- a/assembly/native/build-ubuntu-portable-libs.sh +++ b/assembly/native/build-ubuntu-portable-libs.sh @@ -1,18 +1,29 @@ #/bin/bash #sudo apt-get update -#sudo apt-get install -y build-essential git cmake ninja-build automake libtool texinfo autoconf libc++-dev libc++abi-dev +#sudo apt-get install -y build-essential git cmake ninja-build automake libtool texinfo autoconf libc++-dev libc++abi-dev ccache with_artifacts=false +with_ccache=false -while getopts 'ta' flag; do +while getopts 'tac' flag; do case "${flag}" in a) with_artifacts=true ;; + c) with_ccache=true ;; *) break ;; esac done +if [ "$with_ccache" = true ]; then + mkdir -p ~/.ccache + export CCACHE_DIR=~/.ccache + ccache -M 0 + test $? -eq 0 || { echo "ccache not installed"; exit 1; } +else + export CCACHE_DISABLE=1 +fi + if [ ! -d "build" ]; then mkdir build cd build @@ -23,77 +34,78 @@ fi export CC=$(which clang-16) export CXX=$(which clang++-16) -export CCACHE_DISABLE=1 -if [ ! -d "lz4" ]; then -git clone https://github.com/lz4/lz4.git -cd lz4 +if [ ! -d "../3pp/lz4" ]; then +mkdir -p ../3pp +git clone https://github.com/lz4/lz4.git ../3pp/lz4 +cd ../3pp/lz4 lz4Path=`pwd` git checkout v1.9.4 -CFLAGS="-fPIC" make -j12 +CFLAGS="-fPIC" make -j$(nproc) test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } -cd .. -# ./lib/liblz4.a -# ./lib +cd ../../build else - lz4Path=$(pwd)/lz4 + lz4Path=$(pwd)/../3pp/lz4 echo "Using compiled lz4" fi -if [ ! -d "libsodium" ]; then +if [ ! -d "../3pp/libsodium" ]; then export LIBSODIUM_FULL_BUILD=1 - git clone https://github.com/jedisct1/libsodium.git - cd libsodium + mkdir -p ../3pp/libsodium + wget -O ../3pp/libsodium/libsodium-1.0.18.tar.gz https://github.com/jedisct1/libsodium/releases/download/1.0.18-RELEASE/libsodium-1.0.18.tar.gz + cd ../3pp/libsodium + tar xf libsodium-1.0.18.tar.gz + cd libsodium-1.0.18 sodiumPath=`pwd` - git checkout 1.0.18 - ./autogen.sh ./configure --with-pic --enable-static - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile libsodium"; exit 1; } - cd .. + cd ../../../build else - sodiumPath=$(pwd)/libsodium + sodiumPath=$(pwd)/../3pp/libsodium/libsodium-1.0.18 echo "Using compiled libsodium" fi -if [ ! -d "openssl_3" ]; then - git clone https://github.com/openssl/openssl openssl_3 - cd openssl_3 +if [ ! -d "../3pp/openssl_3" ]; then + git clone https://github.com/openssl/openssl ../3pp/openssl_3 + cd ../3pp/openssl_3 opensslPath=`pwd` git checkout openssl-3.1.4 ./config - make build_libs -j12 + make build_libs -j$(nproc) test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } - cd .. + cd ../../build else - opensslPath=$(pwd)/openssl_3 + opensslPath=$(pwd)/../3pp/openssl_3 echo "Using compiled openssl_3" fi -if [ ! -d "zlib" ]; then - git clone https://github.com/madler/zlib.git - cd zlib +if [ ! -d "../3pp/zlib" ]; then + git clone https://github.com/madler/zlib.git ../3pp/zlib + cd ../3pp/zlib zlibPath=`pwd` ./configure --static - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile zlib"; exit 1; } - cd .. + cd ../../build else - zlibPath=$(pwd)/zlib + zlibPath=$(pwd)/../3pp/zlib echo "Using compiled zlib" fi -if [ ! -d "libmicrohttpd" ]; then - git clone https://git.gnunet.org/libmicrohttpd.git - cd libmicrohttpd +if [ ! -d "../3pp/libmicrohttpd" ]; then + mkdir -p ../3pp/libmicrohttpd + wget -O ../3pp/libmicrohttpd/libmicrohttpd-1.0.1.tar.gz https://ftpmirror.gnu.org/libmicrohttpd/libmicrohttpd-1.0.1.tar.gz + cd ../3pp/libmicrohttpd/ + tar xf libmicrohttpd-1.0.1.tar.gz + cd libmicrohttpd-1.0.1 libmicrohttpdPath=`pwd` - ./autogen.sh ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile libmicrohttpd"; exit 1; } - cd .. + cd ../../../build else - libmicrohttpdPath=$(pwd)/libmicrohttpd + libmicrohttpdPath=$(pwd)/../3pp/libmicrohttpd/libmicrohttpd-1.0.1 echo "Using compiled libmicrohttpd" fi @@ -117,11 +129,10 @@ cmake -GNinja .. \ -DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a - test $? -eq 0 || { echo "Can't configure ton"; exit 1; } ninja tonlibjson emulator -test $? -eq 0 || { echo "Can't compile ton"; exit 1; } +test $? -eq 0 || { echo "Can't compile tonlibjson and emulator"; exit 1; } cd .. diff --git a/assembly/native/build-ubuntu-portable.sh b/assembly/native/build-ubuntu-portable.sh index 00b34f832..f208148c9 100644 --- a/assembly/native/build-ubuntu-portable.sh +++ b/assembly/native/build-ubuntu-portable.sh @@ -6,16 +6,26 @@ with_tests=false with_artifacts=false - -while getopts 'ta' flag; do +while getopts 'tac' flag; do case "${flag}" in t) with_tests=true ;; a) with_artifacts=true ;; + c) with_ccache=true ;; *) break ;; esac done +if [ "$with_ccache" = true ]; then + mkdir -p ~/.ccache + export CCACHE_DIR=~/.ccache + ccache -M 0 + test $? -eq 0 || { echo "ccache not installed"; exit 1; } +else + export CCACHE_DISABLE=1 + rm -rf ~/.ccache +fi + if [ ! -d "build" ]; then mkdir build cd build @@ -26,77 +36,77 @@ fi export CC=$(which clang-16) export CXX=$(which clang++-16) -export CCACHE_DISABLE=1 -if [ ! -d "lz4" ]; then -git clone https://github.com/lz4/lz4.git -cd lz4 +if [ ! -d "../3pp/lz4" ]; then +mkdir -p ../3pp +git clone https://github.com/lz4/lz4.git ../3pp/lz4 +cd ../3pp/lz4 lz4Path=`pwd` git checkout v1.9.4 -CFLAGS="-fPIC" make -j12 +CFLAGS="-fPIC" make -j$(nproc) test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } -cd .. -# ./lib/liblz4.a -# ./lib +cd ../../build else - lz4Path=$(pwd)/lz4 + lz4Path=$(pwd)/../3pp/lz4 echo "Using compiled lz4" fi -if [ ! -d "libsodium" ]; then +if [ ! -d "../3pp/libsodium" ]; then export LIBSODIUM_FULL_BUILD=1 - git clone https://github.com/jedisct1/libsodium.git - cd libsodium + git clone https://github.com/jedisct1/libsodium.git ../3pp/libsodium + cd ../3pp/libsodium sodiumPath=`pwd` git checkout 1.0.18 ./autogen.sh ./configure --with-pic --enable-static - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile libsodium"; exit 1; } - cd .. + cd ../../build else - sodiumPath=$(pwd)/libsodium + sodiumPath=$(pwd)/../3pp/libsodium echo "Using compiled libsodium" fi -if [ ! -d "openssl_3" ]; then - git clone https://github.com/openssl/openssl openssl_3 - cd openssl_3 +if [ ! -d "../3pp/openssl_3" ]; then + git clone https://github.com/openssl/openssl ../3pp/openssl_3 + cd ../3pp/openssl_3 opensslPath=`pwd` git checkout openssl-3.1.4 ./config - make build_libs -j12 + make build_libs -j$(nproc) test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } - cd .. + cd ../../build else - opensslPath=$(pwd)/openssl_3 + opensslPath=$(pwd)/../3pp/openssl_3 echo "Using compiled openssl_3" fi -if [ ! -d "zlib" ]; then - git clone https://github.com/madler/zlib.git - cd zlib +if [ ! -d "../3pp/zlib" ]; then + git clone https://github.com/madler/zlib.git ../3pp/zlib + cd ../3pp/zlib zlibPath=`pwd` ./configure --static - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile zlib"; exit 1; } - cd .. + cd ../../build else - zlibPath=$(pwd)/zlib + zlibPath=$(pwd)/../3pp/zlib echo "Using compiled zlib" fi -if [ ! -d "libmicrohttpd" ]; then - git clone https://git.gnunet.org/libmicrohttpd.git - cd libmicrohttpd +if [ ! -d "../3pp/libmicrohttpd" ]; then + mkdir -p ../3pp/libmicrohttpd + wget -O ../3pp/libmicrohttpd/libmicrohttpd-1.0.1.tar.gz https://ftpmirror.gnu.org/libmicrohttpd/libmicrohttpd-1.0.1.tar.gz + cd ../3pp/libmicrohttpd/ + tar xf libmicrohttpd-1.0.1.tar.gz + cd libmicrohttpd-1.0.1 libmicrohttpdPath=`pwd` - ./autogen.sh ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic - make -j12 + make -j$(nproc) test $? -eq 0 || { echo "Can't compile libmicrohttpd"; exit 1; } - cd .. + cd ../../../build else - libmicrohttpdPath=$(pwd)/libmicrohttpd + libmicrohttpdPath=$(pwd)/../3pp/libmicrohttpd/libmicrohttpd-1.0.1 echo "Using compiled libmicrohttpd" fi @@ -120,7 +130,6 @@ cmake -GNinja .. \ -DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a - test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then diff --git a/assembly/native/build-ubuntu-shared.sh b/assembly/native/build-ubuntu-shared.sh index 49cc8e1ea..68ab6aba6 100644 --- a/assembly/native/build-ubuntu-shared.sh +++ b/assembly/native/build-ubuntu-shared.sh @@ -1,21 +1,31 @@ #/bin/bash #sudo apt-get update -#sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev +#sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev ccache with_tests=false with_artifacts=false +with_ccache=false - -while getopts 'ta' flag; do +while getopts 'tac' flag; do case "${flag}" in t) with_tests=true ;; a) with_artifacts=true ;; + c) with_ccache=true ;; *) break ;; esac done +if [ "$with_ccache" = true ]; then + mkdir -p ~/.ccache + export CCACHE_DIR=~/.ccache + ccache -M 0 + test $? -eq 0 || { echo "ccache not installed"; exit 1; } +else + export CCACHE_DISABLE=1 +fi + if [ ! -d "build" ]; then mkdir build cd build @@ -26,19 +36,19 @@ fi export CC=$(which clang-16) export CXX=$(which clang++-16) -export CCACHE_DISABLE=1 -if [ ! -d "openssl_3" ]; then - git clone https://github.com/openssl/openssl openssl_3 - cd openssl_3 + +if [ ! -d "../openssl_3" ]; then + git clone https://github.com/openssl/openssl ../openssl_3 + cd ../openssl_3 opensslPath=`pwd` git checkout openssl-3.1.4 ./config - make build_libs -j12 + make build_libs -j$(nproc) test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } - cd .. + cd ../build else - opensslPath=$(pwd)/openssl_3 + opensslPath=$(pwd)/../openssl_3 echo "Using compiled openssl_3" fi @@ -78,6 +88,7 @@ ldd ./validator-engine/validator-engine || exit 1 cd .. + if [ "$with_artifacts" = true ]; then rm -rf artifacts mkdir artifacts @@ -94,9 +105,3 @@ if [ "$with_artifacts" = true ]; then cp -R crypto/fift/lib artifacts chmod -R +x artifacts/* fi - -if [ "$with_tests" = true ]; then - cd build -# ctest --output-on-failure -E "test-catchain|test-actors|test-smartcont|test-adnl|test-validator-session-state|test-dht|test-rldp" - ctest --output-on-failure --timeout 1800 -fi diff --git a/assembly/native/build-windows-2019.bat b/assembly/native/build-windows-2019.bat index 844c09fcd..0b96978ca 100644 --- a/assembly/native/build-windows-2019.bat +++ b/assembly/native/build-windows-2019.bat @@ -26,6 +26,13 @@ IF %errorlevel% NEQ 0 ( exit /b %errorlevel% ) +echo Installing ccache... +choco install -y ccache +IF %errorlevel% NEQ 0 ( + echo Can't install ccache + exit /b %errorlevel% +) + echo Installing nasm... choco install -y nasm where nasm @@ -35,7 +42,9 @@ IF %errorlevel% NEQ 0 ( exit /b %errorlevel% ) -mkdir third_libs +if not exist "third_libs" ( + mkdir "third_libs" +) cd third_libs set third_libs=%cd% @@ -161,16 +170,6 @@ IF %errorlevel% NEQ 0 ( exit /b %errorlevel% ) -IF "%1"=="-t" ( - echo Running tests... -REM ctest -C Release --output-on-failure -E "test-catchain|test-actors|test-validator-session-state" - ctest -C Release --output-on-failure -E "test-bigint" --timeout 1800 - IF %errorlevel% NEQ 0 ( - echo Some tests failed - exit /b %errorlevel% - ) -) - echo Strip and copy artifacts cd .. echo where strip diff --git a/assembly/native/build-windows.bat b/assembly/native/build-windows.bat index 68f83c394..c51314078 100644 --- a/assembly/native/build-windows.bat +++ b/assembly/native/build-windows.bat @@ -26,6 +26,13 @@ IF %errorlevel% NEQ 0 ( exit /b %errorlevel% ) +echo Installing ccache... +choco install -y ccache +IF %errorlevel% NEQ 0 ( + echo Can't install ccache + exit /b %errorlevel% +) + echo Installing nasm... choco install -y nasm where nasm @@ -35,7 +42,9 @@ IF %errorlevel% NEQ 0 ( exit /b %errorlevel% ) -mkdir third_libs +if not exist "third_libs" ( + mkdir "third_libs" +) cd third_libs set third_libs=%cd% @@ -161,16 +170,6 @@ IF %errorlevel% NEQ 0 ( exit /b %errorlevel% ) -IF "%1"=="-t" ( - echo Running tests... -REM ctest -C Release --output-on-failure -E "test-catchain|test-actors|test-validator-session-state" - ctest -C Release --output-on-failure -E "test-bigint" --timeout 1800 - IF %errorlevel% NEQ 0 ( - echo Some tests failed - exit /b %errorlevel% - ) -) - echo Strip and copy artifacts cd .. echo where strip diff --git a/assembly/wasm/fift-func-wasm-build-ubuntu.sh b/assembly/wasm/fift-func-wasm-build-ubuntu.sh index a463c02aa..a80430c1d 100644 --- a/assembly/wasm/fift-func-wasm-build-ubuntu.sh +++ b/assembly/wasm/fift-func-wasm-build-ubuntu.sh @@ -2,7 +2,7 @@ # sudo apt update # sudo apt install -y build-essential git make cmake ninja-build clang libgflags-dev zlib1g-dev libssl-dev \ # libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev python3-pip \ -# nodejs libsodium-dev automake libtool libjemalloc-dev +# nodejs libsodium-dev automake libtool libjemalloc-dev ccache # wget https://apt.llvm.org/llvm.sh # chmod +x llvm.sh @@ -30,19 +30,18 @@ if [ "$scratch_new" = true ]; then rm -rf openssl zlib lz4 emsdk libsodium build openssl_em fi - -if [ ! -d "openssl" ]; then - git clone https://github.com/openssl/openssl.git - cp -r openssl openssl_em - cd openssl +if [ ! -d "openssl_3" ]; then + git clone https://github.com/openssl/openssl openssl_3 + cd openssl_3 + opensslPath=`pwd` git checkout openssl-3.1.4 ./config - make -j16 - OPENSSL_DIR=`pwd` + make build_libs -j$(nproc) + test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } cd .. else - OPENSSL_DIR=`pwd`/openssl - echo Using compiled openssl at $OPENSSL_DIR + opensslPath=$(pwd)/openssl_3 + echo "Using compiled openssl_3" fi if [ ! -d "build" ]; then @@ -50,9 +49,9 @@ if [ ! -d "build" ]; then cd build cmake -GNinja -DTON_USE_JEMALLOC=ON .. \ -DCMAKE_BUILD_TYPE=Release \ - -DOPENSSL_ROOT_DIR=$OPENSSL_DIR \ - -DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include \ - -DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.so + -DOPENSSL_ROOT_DIR=$opensslPath \ + -DOPENSSL_INCLUDE_DIR=$opensslPath/include \ + -DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.so test $? -eq 0 || { echo "Can't configure TON build"; exit 1; } ninja fift smc-envelope @@ -82,61 +81,63 @@ export CCACHE_DISABLE=1 cd .. -if [ ! -f "openssl_em/openssl_em" ]; then - cd openssl_em +if [ ! -f "3pp_emscripten/openssl_em/openssl_em" ]; then + mkdir -p 3pp_emscripten + git clone https://github.com/openssl/openssl 3pp_emscripten/openssl_em + cd 3pp_emscripten/openssl_em emconfigure ./Configure linux-generic32 no-shared no-dso no-engine no-unit-test no-tests no-fuzz-afl no-fuzz-libfuzzer sed -i 's/CROSS_COMPILE=.*/CROSS_COMPILE=/g' Makefile sed -i 's/-ldl//g' Makefile sed -i 's/-O3/-Os/g' Makefile emmake make depend - emmake make -j16 + emmake make -j$(nproc) test $? -eq 0 || { echo "Can't compile OpenSSL with emmake "; exit 1; } - OPENSSL_DIR=`pwd` + opensslPath=`pwd` touch openssl_em - cd .. + cd ../.. else - OPENSSL_DIR=`pwd`/openssl_em - echo Using compiled with empscripten openssl at $OPENSSL_DIR + opensslPath=`pwd`/3pp_emscripten/openssl_em + echo Using compiled with empscripten openssl at $opensslPath fi -if [ ! -d "zlib" ]; then - git clone https://github.com/madler/zlib.git - cd zlib +if [ ! -d "3pp_emscripten/zlib" ]; then + git clone https://github.com/madler/zlib.git 3pp_emscripten/zlib + cd 3pp_emscripten/zlib git checkout v1.3.1 ZLIB_DIR=`pwd` emconfigure ./configure --static - emmake make -j16 + emmake make -j$(nproc) test $? -eq 0 || { echo "Can't compile zlib with emmake "; exit 1; } - cd .. + cd ../.. else - ZLIB_DIR=`pwd`/zlib + ZLIB_DIR=`pwd`/3pp_emscripten/zlib echo Using compiled zlib with emscripten at $ZLIB_DIR fi -if [ ! -d "lz4" ]; then - git clone https://github.com/lz4/lz4.git - cd lz4 +if [ ! -d "3pp_emscripten/lz4" ]; then + git clone https://github.com/lz4/lz4.git 3pp_emscripten/lz4 + cd 3pp_emscripten/lz4 git checkout v1.9.4 LZ4_DIR=`pwd` - emmake make -j16 + emmake make -j$(nproc) test $? -eq 0 || { echo "Can't compile lz4 with emmake "; exit 1; } - cd .. + cd ../.. else - LZ4_DIR=`pwd`/lz4 + LZ4_DIR=`pwd`/3pp_emscripten/lz4 echo Using compiled lz4 with emscripten at $LZ4_DIR fi -if [ ! -d "libsodium" ]; then - git clone https://github.com/jedisct1/libsodium - cd libsodium +if [ ! -d "3pp_emscripten/libsodium" ]; then + git clone https://github.com/jedisct1/libsodium 3pp_emscripten/libsodium + cd 3pp_emscripten/libsodium git checkout 1.0.18-RELEASE SODIUM_DIR=`pwd` emconfigure ./configure --disable-ssp - emmake make -j16 + emmake make -j$(nproc) test $? -eq 0 || { echo "Can't compile libsodium with emmake "; exit 1; } - cd .. + cd ../.. else - SODIUM_DIR=`pwd`/libsodium + SODIUM_DIR=`pwd`/3pp_emscripten/libsodium echo Using compiled libsodium with emscripten at $SODIUM_DIR fi @@ -150,8 +151,8 @@ emcmake cmake -DUSE_EMSCRIPTEN=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAK -DLZ4_LIBRARIES=$LZ4_DIR/lib/liblz4.a \ -DLZ4_INCLUDE_DIRS=$LZ4_DIR/lib \ -DOPENSSL_FOUND=1 \ --DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include \ --DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.a \ +-DOPENSSL_INCLUDE_DIR=$opensslPath/include \ +-DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.a \ -DCMAKE_TOOLCHAIN_FILE=$EMSDK_DIR/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake \ -DCMAKE_CXX_FLAGS="-sUSE_ZLIB=1" \ -DSODIUM_FOUND=1 \ @@ -163,7 +164,7 @@ emcmake cmake -DUSE_EMSCRIPTEN=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAK test $? -eq 0 || { echo "Can't configure TON with emmake "; exit 1; } cp -R ../crypto/smartcont ../crypto/fift/lib crypto -emmake make -j16 funcfiftlib func fift tlbc emulator-emscripten +emmake make -j$(nproc) funcfiftlib func fift tlbc emulator-emscripten test $? -eq 0 || { echo "Can't compile TON with emmake "; exit 1; } From 84082e79f7a8327d90ac23d7d594b8bbe661d05c Mon Sep 17 00:00:00 2001 From: birydrad Date: Sat, 29 Mar 2025 17:55:09 +0400 Subject: [PATCH 184/388] celldb: version 2 (#1463) * celldb: version 2 - thread safe cache - parallel commit - multiple optimizations - support of key-value merge operations - improved tests and benchmarks - in-memory version won't read from key value after start - uses vector in-memory table now - use rocksdb::WALRecoveryMode::kTolerateCorruptedTailRecords - do not silently ignore errors during recovery * celldb: add test for load nonexisting cell, test thread safeness of CellUsageTree, fixes --------- Co-authored-by: birydrad <> --- CMakeLists.txt | 1 + crypto/CMakeLists.txt | 3 +- crypto/test/test-db.cpp | 982 ++++++++++++--- crypto/vm/cells/Cell.h | 1 + crypto/vm/cells/DataCell.cpp | 5 +- crypto/vm/cells/DataCell.h | 8 +- crypto/vm/cells/ExtCell.h | 20 + crypto/vm/cells/PrunnedCell.h | 4 + crypto/vm/cells/UsageCell.h | 3 + crypto/vm/cells/VirtualCell.h | 3 + crypto/vm/db/CellHashTable.h | 13 +- crypto/vm/db/CellStorage.cpp | 85 ++ crypto/vm/db/CellStorage.h | 23 + crypto/vm/db/DynamicBagOfCellsDb.cpp | 85 +- crypto/vm/db/DynamicBagOfCellsDb.h | 71 +- crypto/vm/db/DynamicBagOfCellsDbV2.cpp | 1511 ++++++++++++++++++++++++ crypto/vm/db/InMemoryBagOfCellsDb.cpp | 180 ++- crypto/vm/db/StaticBagOfCellsDb.cpp | 18 +- tddb/td/db/KeyValue.h | 53 +- tddb/td/db/MemoryKeyValue.cpp | 82 +- tddb/td/db/MemoryKeyValue.h | 36 +- tddb/td/db/RocksDb.cpp | 123 +- tddb/td/db/RocksDb.h | 26 +- tdutils/td/utils/MpmcQueue.h | 4 +- tdutils/td/utils/Status.h | 7 + tdutils/td/utils/ThreadSafeCounter.h | 60 +- validator-engine/validator-engine.cpp | 12 +- validator-engine/validator-engine.hpp | 4 + validator/db/celldb.cpp | 218 +++- validator/db/celldb.hpp | 9 +- validator/validator-options.hpp | 7 + validator/validator.h | 2 + 32 files changed, 3343 insertions(+), 316 deletions(-) create mode 100644 crypto/vm/db/DynamicBagOfCellsDbV2.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index dd193d8aa..2c65d66a9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -106,6 +106,7 @@ if (TON_USE_ROCKSDB) set(WITH_GFLAGS OFF CACHE BOOL "build with GFlags") set(WITH_TESTS OFF CACHE BOOL "build with tests") set(WITH_TOOLS OFF CACHE BOOL "build with tools") + set(USE_RTTI ON CACHE BOOL "use rtti") set(FAIL_ON_WARNINGS OFF CACHE BOOL "fail on warnings") message("Add rocksdb") add_subdirectory(third-party/rocksdb EXCLUDE_FROM_ALL) diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 92ebf8044..2988f501f 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -142,6 +142,7 @@ set(TON_CRYPTO_SOURCE set(TON_DB_SOURCE vm/db/DynamicBagOfCellsDb.cpp + vm/db/DynamicBagOfCellsDbV2.cpp vm/db/CellStorage.cpp vm/db/TonDb.cpp @@ -539,7 +540,7 @@ target_include_directories(create-state PUBLIC $ #include #include +#include #include #include "openssl/digest.hpp" +#include "storage/db.h" +#include "td/utils/VectorQueue.h" #include "vm/dict.h" -#include #include #include #include -#include +#include -namespace vm { -class ThreadExecutor : public DynamicBagOfCellsDb::AsyncExecutor { +#include +#include +#include + +#include "td/actor/actor.h" +#include "td/utils/overloaded.h" + + +class ActorExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor { public: - explicit ThreadExecutor(size_t threads_n) { - for (size_t i = 0; i < threads_n; ++i) { - threads_.emplace_back([this]() { - while (true) { - auto task = pop_task(); - if (!task) { - break; - } - CHECK(generation_.load() % 2 == 1); - task(); - } - }); - } + ActorExecutor(size_t tn) : tn_(tn) { + scheduler_.run_in_context([&] { worker_ = td::actor::create_actor("Worker"); }); + thread_ = td::thread([this]() { scheduler_.run(); }); } - - ~ThreadExecutor() override { - for (size_t i = 0; i < threads_.size(); ++i) { - push_task({}); + ~ActorExecutor() { + scheduler_.run_in_context_external([&] { send_closure(worker_, &Worker::close); }); + thread_.join(); + } + std::string describe() const override { + return PSTRING() << "ActorExecutor(tn=" << tn_ << ")"; + } + class Worker : public td::actor::Actor { + public: + void close() { + td::actor::core::SchedulerContext::get()->stop(); + stop(); } - for (auto &t : threads_) { - t.join(); + void execute_sync(std::function f) { + f(); } - } + }; void execute_async(std::function f) override { - push_task(std::move(f)); + class Runner : public td::actor::Actor { + public: + explicit Runner(std::function f) : f_(std::move(f)) { + } + void start_up() override { + f_(); + stop(); + } + + private: + std::function f_; + }; + auto context = td::actor::SchedulerContext::get(); + if (context) { + td::actor::create_actor("executeasync", std::move(f)).release(); + } else { + scheduler_.run_in_context_external( + [&] { td::actor::create_actor("executeasync", std::move(f)).release(); }); + } } void execute_sync(std::function f) override { - auto x = generation_.load(); - std::scoped_lock lock(sync_mutex_); - CHECK(x == generation_); - CHECK(generation_.load() % 2 == 1); - f(); - CHECK(generation_.load() % 2 == 1); - } - void inc_generation() { - generation_.fetch_add(1); + auto context = td::actor::SchedulerContext::get(); + if (context) { + td::actor::send_closure(worker_, &Worker::execute_sync, std::move(f)); + } else { + scheduler_.run_in_context_external( + [&] { td::actor::send_closure(worker_, &Worker::execute_sync, std::move(f)); }); + } } private: - std::atomic generation_{0}; - std::queue, size_t>> queue_; - std::mutex queue_mutex_; - std::condition_variable cv_; - std::mutex sync_mutex_; - std::vector threads_; - - std::function pop_task() { - std::unique_lock lock(queue_mutex_); - cv_.wait(lock, [&] { return !queue_.empty(); }); - CHECK(!queue_.empty()); - auto task = std::move(queue_.front()); - queue_.pop(); - CHECK(task.second == generation_); - return task.first; - } - - void push_task(std::function task) { - { - std::scoped_lock lock(queue_mutex_); - queue_.emplace(std::move(task), generation_.load()); - } - cv_.notify_one(); - } + size_t tn_; + td::actor::Scheduler scheduler_{{tn_}, false, td::actor::Scheduler::Paused}; + td::actor::ActorOwn worker_; + td::thread thread_; }; +namespace vm { std::vector do_get_serialization_modes() { std::vector res; for (int i = 0; i < 32; i++) { @@ -324,6 +323,34 @@ TEST(Cell, sha_benchmark_threaded) { bench_threaded([n]() { return BenchSha256(n); }); } } +class BenchTasks : public td::Benchmark { + public: + explicit BenchTasks(size_t tn) : tn_(tn) { + } + + std::string get_description() const override { + return PSTRING() << "bench_tasks(threads_n=" << tn_ << ")"; + } + + void run(int n) override { + ActorExecutor executor(tn_); + for (int i = 0; i < n; i++) { + std::latch latch(tn_); + for (size_t j = 0; j < tn_; j++) { + executor.execute_async([&]() { latch.count_down(); }); + } + latch.wait(); + } + } + + private: + size_t tn_{}; +}; +TEST(Bench, Tasks) { + for (size_t tn : {1, 4, 16}) { + bench(BenchTasks(tn)); + } +} std::string serialize_boc(Ref cell, int mode = 31) { CHECK(cell.not_null()); @@ -437,6 +464,8 @@ class CellExplorer { cs_ = {}; break; } + default: + UNREACHABLE(); } } @@ -474,6 +503,8 @@ class CellExplorer { case op.ReadCellSlice: log_ << "read slice " << op.children_mask << "\n"; break; + default: + UNREACHABLE(); } } void log_cell(const Ref &cell) { @@ -627,7 +658,9 @@ TEST(Cell, MerkleProof) { auto exploration2 = CellExplorer::explore(usage_cell, exploration.ops); ASSERT_EQ(exploration.log, exploration2.log); - auto is_prunned = [&](const Ref &cell) { return exploration.visited.count(cell->get_hash()) == 0; }; + auto is_prunned = [&](const Ref &cell_to_check) { + return exploration.visited.count(cell_to_check->get_hash()) == 0; + }; auto proof = MerkleProof::generate(cell, is_prunned); // CellBuilder::virtualize(proof, 1); //ASSERT_EQ(1u, proof->get_level()); @@ -706,7 +739,7 @@ TEST(Cell, MerkleProofCombine) { check(proof_union_fast); } { - auto cell = MerkleProof::virtualize(proof12, 1); + cell = MerkleProof::virtualize(proof12, 1); auto usage_tree = std::make_shared(); auto usage_cell = UsageCell::create(cell, usage_tree->root_ptr()); @@ -927,7 +960,6 @@ TEST(TonDb, InMemoryDynamicBocSimple) { auto before = counter(); SCOPE_EXIT { LOG_CHECK(before == counter()) << before << " vs " << counter(); - ; }; td::Random::Xorshift128plus rnd{123}; auto kv = std::make_shared(); @@ -963,26 +995,193 @@ TEST(TonDb, InMemoryDynamicBocSimple) { int VERBOSITY_NAME(boc) = VERBOSITY_NAME(DEBUG) + 10; +struct CellMerger : td::Merger { + void merge_value_and_update(std::string &value, td::Slice update) override { + return CellStorer::merge_value_and_refcnt_diff(value, update); + } + void merge_update_and_update(std::string &left_update, td::Slice right_update) override { + LOG(ERROR) << "update_and_update"; + UNREACHABLE(); + return CellStorer::merge_refcnt_diffs(left_update, right_update); + } +}; +struct CompactionFilterEraseEmptyValues : public rocksdb::CompactionFilter { + bool Filter(int level, const rocksdb::Slice & /*key*/, const rocksdb::Slice &existing_value, std::string *new_value, + bool *value_changed) const override { + return existing_value.empty(); + } + bool FilterMergeOperand(int, const rocksdb::Slice & /*key*/, const rocksdb::Slice &operand) const override { + return operand.empty(); + } + + // Name of the compaction filter + const char *Name() const override { + return "CompactionFilterEraseEmptyValues"; + } +}; +auto to_td(rocksdb::Slice value) -> td::Slice { + return td::Slice(value.data(), value.size()); +} + +struct MergeOperatorAddCellRefcnt : public rocksdb::MergeOperator { + const char *Name() const override { + return "MergeOperatorAddCellRefcnt"; + } + bool FullMergeV2(const MergeOperationInput &merge_in, MergeOperationOutput *merge_out) const override { + CHECK(merge_in.existing_value); + auto &value = *merge_in.existing_value; + CHECK(merge_in.operand_list.size() >= 1); + td::Slice diff; + std::string diff_buf; + if (merge_in.operand_list.size() == 1) { + diff = to_td(merge_in.operand_list[0]); + } else { + diff_buf = merge_in.operand_list[0].ToString(); + for (size_t i = 1; i < merge_in.operand_list.size(); ++i) { + CellStorer::merge_refcnt_diffs(diff_buf, to_td(merge_in.operand_list[i])); + } + diff = diff_buf; + } + + merge_out->new_value = value.ToString(); + CellStorer::merge_value_and_refcnt_diff(merge_out->new_value, diff); + return true; + } + bool PartialMerge(const rocksdb::Slice & /*key*/, const rocksdb::Slice &left, const rocksdb::Slice &right, + std::string *new_value, rocksdb::Logger *logger) const override { + *new_value = left.ToString(); + CellStorer::merge_refcnt_diffs(*new_value, to_td(right)); + return true; + } +}; + +struct DB { + std::unique_ptr dboc; + std::shared_ptr kv; + void reset_loader() { + dboc->set_loader(std::make_unique(kv->snapshot())); + } +}; struct BocOptions { - std::shared_ptr async_executor; - std::optional o_in_memory; + using AsyncExecutor = DynamicBagOfCellsDb::AsyncExecutor; + + using CreateInMemoryOptions = DynamicBagOfCellsDb::CreateInMemoryOptions; + using CreateV1Options = DynamicBagOfCellsDb::CreateV1Options; + using CreateV2Options = DynamicBagOfCellsDb::CreateV2Options; + + std::shared_ptr async_executor; + struct KvOptions { + enum KvType { InMemory, RocksDb } kv_type{InMemory}; + bool experimental{false}; + bool no_transactions{false}; + size_t cache_size{0}; + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const KvOptions &kv_options) { + if (kv_options.kv_type == KvType::InMemory) { + return sb << "InMemory{}"; + } + return sb << "RockDb{cache_size=" << kv_options.cache_size << ", no_transactions=" << kv_options.no_transactions + << ", experimental=" << kv_options.experimental << "}"; + } + }; + KvOptions kv_options; + std::variant options; td::uint64 seed{123}; - auto create_dboc(td::KeyValueReader *kv, std::optional o_root_n) { - if (o_in_memory) { - auto res = DynamicBagOfCellsDb::create_in_memory(kv, *o_in_memory); - auto stats = res->get_stats().move_as_ok(); - if (o_root_n) { - ASSERT_EQ(*o_root_n, stats.roots_total_count); + std::shared_ptr create_kv(std::shared_ptr old_key_value, bool no_reads = false) { + if (kv_options.kv_type == KvOptions::InMemory) { + if (old_key_value) { + return old_key_value; + } + return std::make_shared(std::make_shared()); + } else if (kv_options.kv_type == KvOptions::RocksDb) { + auto merge_operator = std::make_shared(); + static const CompactionFilterEraseEmptyValues compaction_filter; + CHECK(!old_key_value || old_key_value.use_count() == 1); + std::string db_path = "test_celldb"; + if (old_key_value) { + //LOG(ERROR) << "Reload rocksdb"; + old_key_value.reset(); + } else { + //LOG(ERROR) << "New rocksdb"; + td::RocksDb::destroy(db_path).ensure(); + } + auto db_options = td::RocksDbOptions{ + .block_cache = {}, + .merge_operator = merge_operator, + .compaction_filter = &compaction_filter, + .experimental = kv_options.experimental, + .no_reads = no_reads, + .no_transactions = kv_options.no_transactions, + .use_direct_reads = true, + .no_block_cache = true, + }; + if (kv_options.cache_size != 0) { + db_options.no_block_cache = false; + db_options.block_cache = rocksdb::NewLRUCache(kv_options.cache_size); } - VLOG(boc) << "reset roots_n=" << stats.roots_total_count << " cells_n=" << stats.cells_total_count; - return res; + return std::make_shared(td::RocksDb::open(db_path, std::move(db_options)).move_as_ok()); + } else { + UNREACHABLE(); + } + } + void check_kv_is_empty(KeyValue &kv) { + if (kv_options.kv_type == KvOptions::InMemory) { + ASSERT_EQ(0u, kv.count("").move_as_ok()); + return; } - return DynamicBagOfCellsDb::create(); + + size_t non_empty_values = 0; + kv.for_each([&](auto key, auto value) { + non_empty_values += !value.empty(); + return td::Status::OK(); + }); + if (non_empty_values != 0) { + kv.for_each([&](auto key, auto value) { + LOG(ERROR) << "Key: " << td::hex_encode(key) << " Value: " << td::hex_encode(value); + std::string x; + LOG(ERROR) << int(kv.get(key, x).move_as_ok()); + return td::Status::OK(); + }); + } + ASSERT_EQ(0u, non_empty_values); + } + + [[nodiscard]] auto create_db(DB db, std::optional o_root_n) { + auto old_boc = std::move(db.dboc); + auto old_kv = std::move(db.kv); + old_boc.reset(); + using ResT = DB; + return std::visit(td::overloaded( + [&](CreateV1Options &) -> ResT { + auto new_kv = create_kv(std::move(old_kv)); + auto res = DynamicBagOfCellsDb::create(); + res->set_loader(std::make_unique(new_kv->snapshot())); + return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; + }, + [&](CreateV2Options &options) -> ResT { + auto new_kv = create_kv(std::move(old_kv)); + auto res = DynamicBagOfCellsDb::create_v2(options); + res->set_loader(std::make_unique(new_kv->snapshot())); + return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; + }, + [&](CreateInMemoryOptions &options) -> ResT { + auto read_kv = create_kv(std::move(old_kv), false); + auto res = DynamicBagOfCellsDb::create_in_memory(read_kv.get(), options); + auto new_kv = create_kv(std::move(read_kv), true); + res->set_loader(std::make_unique(new_kv->snapshot())); + auto stats = res->get_stats().move_as_ok(); + if (o_root_n) { + ASSERT_EQ(*o_root_n, stats.roots_total_count); + } + VLOG(boc) << "reset roots_n=" << stats.roots_total_count + << " cells_n=" << stats.cells_total_count; + return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; + }), + options); }; void prepare_commit(DynamicBagOfCellsDb &dboc) { + td::PerfWarningTimer warning_timer("test_db_prepare_commit"); if (async_executor) { - async_executor->inc_generation(); std::latch latch(1); td::Result res; async_executor->execute_sync([&] { @@ -993,70 +1192,175 @@ struct BocOptions { }); latch.wait(); async_executor->execute_sync([&] {}); - async_executor->inc_generation(); + res.ensure(); } else { dboc.prepare_commit(); } } + enum CacheAction { ResetCache, KeepCache }; + void write_commit(DynamicBagOfCellsDb &dboc, std::shared_ptr kv, CacheAction action) { + td::PerfWarningTimer warning_timer("test_db_write_commit"); + kv->begin_write_batch().ensure(); + CellStorer cell_storer(*kv); + { + td::PerfWarningTimer timer("test_db_commit"); + dboc.commit(cell_storer).ensure(); + } + { + td::PerfWarningTimer timer("test_db_commit_write_batch"); + kv->commit_write_batch().ensure(); + } + switch (action) { + case ResetCache: { + td::PerfWarningTimer timer("test_db_reset_cache"); + dboc.set_loader(std::make_unique(kv->snapshot())); + break; + } + case KeepCache: + break; + } + } + + void commit(DB &db, CacheAction action = ResetCache) { + prepare_commit(*db.dboc); + write_commit(*db.dboc, db.kv, action); + } + + std::string description() const { + td::StringBuilder sb; + + sb << "DBOC(type="; + std::visit(td::overloaded([&](const CreateV1Options &) { sb << "V1"; }, + [&](const CreateV2Options &options) { + sb << "V2(concurrency=" << options.extra_threads + 1; + if (options.executor) { + sb << ", executor=" << options.executor->describe(); + } else { + sb << ", executor=threads"; + } + sb << ")"; + }, + [&](const CreateInMemoryOptions &options) { + sb << "InMemory(use_arena=" << options.use_arena + << ", less_memory=" << options.use_less_memory_during_creation << ")"; + }), + options); + sb << kv_options; + if (async_executor) { + sb << ", executor=" << async_executor->describe(); + } + sb << ")"; + + return sb.as_cslice().str(); + } }; template -void with_all_boc_options(F &&f, size_t tests_n = 500) { +void with_all_boc_options(F &&f, size_t tests_n, bool single_thread = false) { LOG(INFO) << "Test dynamic boc"; auto counter = [] { return td::NamedThreadSafeCounter::get_default().get_counter("DataCell").sum(); }; + std::map>> benches; auto run = [&](BocOptions options) { - LOG(INFO) << "\t" << (options.o_in_memory ? "in memory" : "on disk") << (options.async_executor ? " async" : ""); - if (options.o_in_memory) { - LOG(INFO) << "\t\tuse_arena=" << options.o_in_memory->use_arena - << " less_memory=" << options.o_in_memory->use_less_memory_during_creation; - } + auto description = options.description(); + LOG(INFO) << "Running " << description; + auto start = td::Timestamp::now(); + DynamicBagOfCellsDb::Stats stats; + auto o_in_memory = std::get_if(&options.options); for (td::uint32 i = 0; i < tests_n; i++) { auto before = counter(); + options.seed = i == 0 ? 123 : i; - f(options); + auto stats_diff = f(options); + stats.apply_diff(stats_diff); + auto after = counter(); - LOG_CHECK((options.o_in_memory && options.o_in_memory->use_arena) || before == after) - << before << " vs " << after; + LOG_CHECK((o_in_memory && o_in_memory->use_arena) || before == after) << before << " vs " << after; } + LOG(INFO) << "\ttook " << td::Timestamp::now().at() - start.at() << " seconds"; + LOG(INFO) << stats; + for (auto &[key, value] : stats.named_stats.stats_int) { + if (td::begins_with(key, "bench_")) { + benches[key].emplace_back(value, description); + } + } + }; + + // NB: use .experimental to play with different RocksDb parameters + // Note, that new benchmark are necessary to fully understand the effect of different RocksDb options + std::vector kv_options_list = { + // BocOptions::KvOptions{.kv_type = BocOptions::KvOptions::InMemory}, + // BocOptions::KvOptions{.kv_type = BocOptions::KvOptions::RocksDb, .experimental = false, .cache_size = 0}, + BocOptions::KvOptions{ + .kv_type = BocOptions::KvOptions::RocksDb, .experimental = false, .cache_size = size_t{128 << 20}}, }; - run({.async_executor = std::make_shared(4)}); - run({}); - for (auto use_arena : {false, true}) { - for (auto less_memory : {false, true}) { - run({.o_in_memory = - DynamicBagOfCellsDb::CreateInMemoryOptions{.extra_threads = std::thread::hardware_concurrency(), - .verbose = false, - .use_arena = use_arena, - .use_less_memory_during_creation = less_memory}}); + std::vector has_executor_options = {false, true}; + for (auto kv_options : kv_options_list) { + for (bool has_executor : has_executor_options) { + std::shared_ptr executor; + if (has_executor) { + executor = std::make_shared( + 4); // 4 - to compare V1 and V2, because V1 has parallel_load = 4 by default + } + // V2 - 4 threads + run({.async_executor = executor, + .kv_options = kv_options, + .options = DynamicBagOfCellsDb::CreateV2Options{ + .extra_threads = 3, .executor = executor, .cache_ttl_max = 5}}); + + // V1 + run({.async_executor = executor, .kv_options = kv_options, .options = DynamicBagOfCellsDb::CreateV1Options{}}); + + // V2 - one thread + run({.async_executor = executor, + .kv_options = kv_options, + .options = + DynamicBagOfCellsDb::CreateV2Options{.extra_threads = 0, .executor = executor, .cache_ttl_max = 5}}); + + // InMemory + for (auto use_arena : {false, true}) { + for (auto less_memory : {false, true}) { + run({.async_executor = executor, + .kv_options = kv_options, + .options = + DynamicBagOfCellsDb::CreateInMemoryOptions{.extra_threads = std::thread::hardware_concurrency(), + .verbose = false, + .use_arena = use_arena, + .use_less_memory_during_creation = less_memory}}); + } + } + } + } + + for (auto &[name, v] : benches) { + std::sort(v.begin(), v.end()); + LOG(INFO) << "Bench " << name; + for (auto &[t, name] : v) { + LOG(INFO) << "\t" << name << " " << double(t) / 1000 << "s"; } } } -void test_dynamic_boc(BocOptions options) { - auto counter = [] { return td::NamedThreadSafeCounter::get_default().get_counter("DataCell").sum(); }; - auto before = counter(); - SCOPE_EXIT { - LOG_CHECK((options.o_in_memory && options.o_in_memory->use_arena) || before == counter()) - << before << " vs " << counter(); - }; +DynamicBagOfCellsDb::Stats test_dynamic_boc(BocOptions options) { + DynamicBagOfCellsDb::Stats stats; td::Random::Xorshift128plus rnd{options.seed}; std::string old_root_hash; std::string old_root_serialization; - auto kv = std::make_shared(); - auto create_dboc = [&]() { + DB db; + auto reload_db = [&]() { auto roots_n = old_root_hash.empty() ? 0 : 1; - return options.create_dboc(kv.get(), roots_n); + db = options.create_db(std::move(db), roots_n); }; - auto dboc = create_dboc(); - dboc->set_loader(std::make_unique(kv)); + reload_db(); for (int t = 1000; t >= 0; t--) { if (rnd() % 10 == 0) { - dboc = create_dboc(); + reload_db(); } - dboc->set_loader(std::make_unique(kv)); + db.dboc->load_cell(vm::CellHash{}.as_slice()).ensure_error(); + + db.reset_loader(); Ref old_root; if (!old_root_hash.empty()) { - old_root = dboc->load_cell(old_root_hash).move_as_ok(); + old_root = db.dboc->load_cell(old_root_hash).move_as_ok(); auto serialization = serialize_boc(old_root); ASSERT_EQ(old_root_serialization, serialization); } @@ -1071,47 +1375,61 @@ void test_dynamic_boc(BocOptions options) { ->get_root_cell(0) .move_as_ok(); - dboc->dec(old_root); + db.dboc->dec(old_root); if (t != 0) { - dboc->inc(cell); - } - dboc->prepare_commit().ensure(); - { - CellStorer cell_storer(*kv); - dboc->commit(cell_storer).ensure(); + db.dboc->inc(cell); } + options.commit(db, BocOptions::ResetCache); } - ASSERT_EQ(0u, kv->count("").ok()); + options.check_kv_is_empty(*db.kv); + + stats.named_stats.apply_diff(db.kv->get_usage_stats().to_named_stats()); + return stats; } TEST(TonDb, DynamicBoc) { with_all_boc_options(test_dynamic_boc, 1); }; -void test_dynamic_boc2(BocOptions options) { +DynamicBagOfCellsDb::Stats test_dynamic_boc2(BocOptions options) { td::Random::Xorshift128plus rnd{options.seed}; + DynamicBagOfCellsDb::Stats stats; - int total_roots = rnd.fast(1, !rnd.fast(0, 10) * 100 + 10); + int total_roots = rnd.fast(1, !rnd.fast(0, 30) * 100 + 10); int max_roots = rnd.fast(1, 20); + int max_cells = 20; + + // VERBOSITY_NAME(boc) = 1; + // LOG(WARNING) << "====================================================\n\n"; + // max_roots = 2; + // total_roots = 2; + // max_cells = 2; + + auto meta_key = [](size_t i) { return PSTRING() << "meta." << i; }; + std::array meta; + int last_commit_at = 0; int first_root_id = 0; int last_root_id = 0; - auto kv = std::make_shared(); - auto create_dboc = [&](td::int64 root_n) { return options.create_dboc(kv.get(), root_n); }; - auto dboc = create_dboc(0); - dboc->set_loader(std::make_unique(kv)); + DB db; + auto reload_db = [&](td::int64 root_n) { db = options.create_db(std::move(db), root_n); }; + reload_db(0); auto counter = [] { return td::NamedThreadSafeCounter::get_default().get_counter("DataCell").sum(); }; auto before = counter(); - SCOPE_EXIT{ - // LOG_CHECK((options.o_in_memory && options.o_in_memory->use_arena) || before == counter()) - // << before << " vs " << counter(); + SCOPE_EXIT { + bool skip_check = false; + if (std::holds_alternative(options.options) && + std::get(options.options).use_arena) { + skip_check = true; + } + LOG_IF(FATAL, !(skip_check || before == counter())) << before << " vs " << counter(); }; std::vector> roots(max_roots); std::vector root_hashes(max_roots); auto add_root = [&](Ref root) { - dboc->inc(root); + db.dboc->inc(root); root_hashes[last_root_id % max_roots] = (root->get_hash().as_slice().str()); roots[last_root_id % max_roots] = root; last_root_id++; @@ -1124,9 +1442,9 @@ void test_dynamic_boc2(BocOptions options) { VLOG(boc) << " from db"; auto from_root_hash = root_hashes[root_id % max_roots]; if (rnd() % 2 == 0) { - from_root = dboc->load_root(from_root_hash).move_as_ok(); + from_root = db.dboc->load_root(from_root_hash).move_as_ok(); } else { - from_root = dboc->load_cell(from_root_hash).move_as_ok(); + from_root = db.dboc->load_cell(from_root_hash).move_as_ok(); } } else { VLOG(boc) << "FROM MEMORY"; @@ -1147,31 +1465,69 @@ void test_dynamic_boc2(BocOptions options) { from_root = get_root(rnd.fast(first_root_id, last_root_id - 1)); } VLOG(boc) << " ..."; - auto new_root = gen_random_cell(rnd.fast(1, 20), from_root, rnd); - root_cnt[new_root->get_hash()]++; - add_root(std::move(new_root)); + auto new_root_cell = gen_random_cell(rnd.fast(1, max_cells), from_root, rnd); + root_cnt[new_root_cell->get_hash()]++; + add_root(std::move(new_root_cell)); VLOG(boc) << " OK"; }; - auto commit = [&] { - VLOG(boc) << "commit"; - //rnd.fast(0, 1); - options.prepare_commit(*dboc); - { - CellStorer cell_storer(*kv); - dboc->commit(cell_storer); + td::UsageStats commit_stats{}; + auto commit = [&](bool finish = false) { + for (size_t i = 0; i < meta.size(); i++) { + std::string value; + auto status = db.dboc->meta_get(meta_key(i), value).move_as_ok(); + if (status == KeyValue::GetStatus::Ok) { + ASSERT_EQ(value, meta[i]); + ASSERT_TRUE(!meta[i].empty()); + } else { + ASSERT_TRUE(meta[i].empty()); + } + + if (meta[i].empty()) { + if (!finish && rnd() % 2 == 0) { + meta[i] = td::to_string(rnd()); + db.dboc->meta_set(meta_key(i), meta[i]); + VLOG(boc) << "meta set " << meta_key(i) << " " << meta[i]; + } + } else { + auto f = finish ? 1 : rnd() % 3; + if (f == 0) { + meta[i] = td::to_string(rnd()); + db.dboc->meta_set(meta_key(i), meta[i]); + VLOG(boc) << "meta set " << meta_key(i) << " " << meta[i]; + } else if (f == 1) { + meta[i] = ""; + db.dboc->meta_erase(meta_key(i)); + VLOG(boc) << "meta erase " << meta_key(i); + } + } } - dboc->set_loader(std::make_unique(kv)); + + VLOG(boc) << "before commit cells_in_db=" << db.kv->count(""); + //rnd.fast(0, 1); + auto stats_before = db.kv->get_usage_stats(); + options.commit(db, BocOptions::ResetCache); + auto stats_after = db.kv->get_usage_stats(); + commit_stats = commit_stats + stats_after - stats_before; + VLOG(boc) << "after commit cells_in_db=" << db.kv->count(""); + + // db.reset_loader(); for (int i = last_commit_at; i < last_root_id; i++) { roots[i % max_roots].clear(); } last_commit_at = last_root_id; }; - auto reset = [&] { + auto reset = [&](bool force_full = false) { VLOG(boc) << "reset"; commit(); - dboc = create_dboc(td::int64(root_cnt.size())); - dboc->set_loader(std::make_unique(kv)); + if (rnd() % 3 == 0 || force_full) { + // very slow for rocksdb + auto r_stats = db.dboc->get_stats(); + if (r_stats.is_ok()) { + stats.apply_diff(r_stats.ok()); + } + reload_db(root_cnt.size()); + } }; auto delete_root = [&] { @@ -1187,22 +1543,28 @@ void test_dynamic_boc2(BocOptions options) { root_cnt.erase(it); } - dboc->dec(std::move(old_root)); + db.dboc->dec(std::move(old_root)); first_root_id++; VLOG(boc) << " OK"; }; td::RandomSteps steps({{new_root, 10}, {delete_root, 9}, {commit, 2}, {reset, 1}}); while (first_root_id != total_roots) { - VLOG(boc) << first_root_id << " " << last_root_id << " " << kv->count("").ok(); + VLOG(boc) << first_root_id << " " << last_root_id; // << " " << db.kv->count("").ok(); steps.step(rnd); } - commit(); - ASSERT_EQ(0u, kv->count("").ok()); + commit(true); + options.check_kv_is_empty(*db.kv); + + // auto stats = kv->get_usage_stats(); + // LOG(ERROR) << "total: " << stats; + reset(true); + stats.named_stats.apply_diff(db.kv->get_usage_stats().to_named_stats()); + return stats; } TEST(TonDb, DynamicBoc2) { - with_all_boc_options(test_dynamic_boc2); + with_all_boc_options(test_dynamic_boc2, 50); } template @@ -1341,6 +1703,10 @@ class CompactArray { size_t size() const { return size_; } + void reset() { + size_ = 0; + root_ = {}; + } Ref merkle_proof(std::vector keys) { std::set hashes; @@ -1435,6 +1801,283 @@ class FastCompactArray { std::vector v_; }; +struct BocTestHelper { + public: + BocTestHelper() = default; + BocTestHelper(td::int64 seed) : rnd_(seed) { + } + + CompactArray create_array(size_t size, td::uint64 max_value) { + std::vector v(size); + td::Random::Xorshift128plus rnd{123}; + for (auto &x : v) { + x = rnd() % max_value; + } + return CompactArray(v); + } + + private: + td::Random::Xorshift128plus rnd_{123}; +}; + +DynamicBagOfCellsDb::Stats bench_dboc_get_and_set(BocOptions options) { + BocTestHelper helper(options.seed); + size_t n = 1 << 20; + size_t max_value = 1 << 26; + auto arr = helper.create_array(n, max_value); + + // auto kv = std::make_shared(); + td::Slice db_path = "compact_array_db"; + td::RocksDb::destroy(db_path).ensure(); + + DB db = options.create_db({}, {}); + DynamicBagOfCellsDb::Stats stats; + + td::Timer total_timer; + + auto bench = [&](td::Slice desc, auto &&f) { + auto before = db.dboc->get_stats().move_as_ok(); + td::Timer timer; + LOG(ERROR) << "Benchmarking " << desc; + f(); + stats.named_stats.stats_int[desc.str()] = td::int64(timer.elapsed() * 1000); + LOG(ERROR) << "Benchmarking " << desc << " done: " << timer.elapsed() << "s\n"; + auto after = db.dboc->get_stats().move_as_ok(); + after.named_stats.subtract_diff(before.named_stats); + LOG(ERROR) << after; + }; + + td::VectorQueue roots; + // Save array in db + bench(PSLICE() << "bench_inc_large_db(n=" << n << ")", [&] { + db.dboc->inc(arr.root()); + roots.push(arr.root()->get_hash()); + options.commit(db, BocOptions::ResetCache); + }); + bench("bench_compactify", [&] { + auto status = dynamic_cast(*db.kv).raw_db()->CompactRange({}, nullptr, nullptr); + LOG_IF(FATAL, !status.ok()) << status.ToString(); + }); + db = options.create_db(std::move(db), {}); + + bench(PSLICE() << "bench_inc_large_existed_db(n=" << n << ")", [&] { + db.dboc->inc(arr.root()); + roots.push(arr.root()->get_hash()); + options.commit(db, BocOptions::ResetCache); + }); + + td::Random::Xorshift128plus rnd{123}; + while (false) { + auto hash = arr.root()->get_hash(); + arr = CompactArray{n, db.dboc->load_root(hash.as_slice()).move_as_ok()}; + td::Timer timer; + for (size_t i = 0; i < 10000; i++) { + auto pos = rnd() % n; + arr.get(pos); + } + LOG(ERROR) << timer.elapsed() << "s\n"; + db.reset_loader(); + } + + for (auto p : + std::vector>{{10000, 0}, {10000, 5}, {5000, 5000}, {5, 10000}, {0, 10000}}) { + auto get_n = p.first; + auto set_n = p.second; + auto hash = arr.root()->get_hash(); + arr = CompactArray{n, db.dboc->load_root(hash.as_slice()).move_as_ok()}; + bench(PSTRING() << "bench_changes(get_n=" << get_n << ", set_n=" << set_n << ")", [&] { + for (size_t i = 0; i < get_n; i++) { + auto pos = rnd() % n; + arr.get(pos); + } + for (size_t i = 0; i < set_n; i++) { + auto pos = rnd() % n; + auto value = rnd() % max_value; + arr.set(pos, value); + } + }); + bench(PSTRING() << "bench_commit(get_n=" << get_n << ", set_n=" << set_n << ")", [&] { + db.dboc->inc(arr.root()); + roots.push(arr.root()->get_hash()); + options.commit(db, BocOptions::ResetCache); + }); + } + arr.reset(); + + bench(PSLICE() << "bench_dec_some_roots()", [&] { + while (roots.size() > 1) { + auto hash = roots.pop(); + auto cell = db.dboc->load_cell(hash.as_slice()).move_as_ok(); + db.dboc->dec(cell); + } + options.commit(db, BocOptions::ResetCache); + }); + + db = options.create_db(std::move(db), {}); + + bench(PSLICE() << "bench_dec_large_root(n=" << n << ")", [&] { + while (!roots.empty()) { + auto hash = roots.pop(); + auto cell = db.dboc->load_cell(hash.as_slice()).move_as_ok(); + db.dboc->dec(cell); + + /* + do { + auto cell = db.dboc->load_cell(hash.as_slice()).move_as_ok(); + db.dboc->dec(cell); + cell = {}; + options.prepare_commit(*db.dboc); + //db.dboc->prepare_commit().ensure(); + db.reset_loader(); + db = options.create_db(std::move(db), {}); + } while (true); + */ + } + options.commit(db, BocOptions::ResetCache); + }); + stats.named_stats.stats_int["bench_total"] = td::int64(total_timer.elapsed() * 1000); + + return stats; +} + +TEST(TonDb, BenchDynamicBocGetAndSet) { + with_all_boc_options(bench_dboc_get_and_set, 1); +} + +TEST(TonDb, DynamicBocIncSimple) { + auto kv = std::make_shared(std::make_shared()); + auto db = DynamicBagOfCellsDb::create_v2({.extra_threads = 0}); + db->set_loader(std::make_unique(kv)); + + td::Random::Xorshift128plus rnd(123); + size_t size = 4; + std::vector values(size); + for (auto &v : values) { + //v = rnd() % 2; + v = rnd(); + } + // 1. Create large dictionary and store it in db + auto arr_ptr = std::make_unique(values); + auto &arr = *arr_ptr; + td::VectorQueue queue; + auto push = [&]() { + //LOG(ERROR) << "PUSH ROOT"; + auto begin_stats = kv->get_usage_stats(); + db->inc(arr.root()); + queue.push(arr.root()->get_hash()); + vm::CellStorer cell_storer(*kv); + db->commit(cell_storer); + auto end_stats = kv->get_usage_stats(); + LOG(ERROR) << end_stats - begin_stats; + db->set_loader(std::make_unique(kv)); + auto hash = arr.root()->get_hash(); + arr = CompactArray{size, db->load_root(hash.as_slice()).move_as_ok()}; + //LOG(ERROR) << "CELLS IN DB: " << kv->count("").move_as_ok(); + }; + auto pop = [&]() { + if (queue.empty()) { + return; + } + //LOG(ERROR) << "POP ROOT"; + auto begin_stats = kv->get_usage_stats(); + auto cell = db->load_cell(queue.pop().as_slice()).move_as_ok(); + db->dec(cell); + vm::CellStorer cell_storer(*kv); + db->commit(cell_storer); + auto end_stats = kv->get_usage_stats(); + db->set_loader(std::make_unique(kv)); + //LOG(ERROR) << end_stats - begin_stats; + //LOG(ERROR) << "CELLS IN DB: " << kv->count("").move_as_ok(); + }; + auto upd = [&] { + for (int i = 0; i < 20; i++) { + auto pos = rnd.fast(0, td::narrow_cast(size) - 1); + if (rnd() % 2) { + auto value = rnd() % 2; + arr.set(pos, value); + } else { + arr.get(pos); + } + } + }; + + //LOG(ERROR) << "Created compact array"; + push(); + pop(); + //CHECK(kv->count("").move_as_ok() == 0); + + // 2. Lets change first 20 keys and read last 20 keys + /* + for (size_t i = 0; i < 20 && i < size; i++) { + arr.set(i, rnd()); + } + */ + //arr.set(0, rnd()); + arr.set(size - 1, rnd()); + for (size_t i = 0; i < 20 && i < size; i++) { + arr.get(size - i - 1); + } + + // 3. And now commit diff with stats + push(); + push(); + upd(); + upd(); + push(); + push(); + upd(); + pop(); + pop(); + upd(); + push(); + push(); + while (!queue.empty()) { + pop(); + } + LOG(ERROR) << "CELLS IN DB: " << kv->count("").move_as_ok(); +} + +class BenchCellStorerMergeRefcntDiffs : public td::Benchmark { + public: + std::string get_description() const override { + return PSTRING() << "bench_cells_storer_merge_refcnt_diffs"; + } + + void run(int n) override { + auto cell = vm::CellBuilder().store_bytes(std::string(32, 'A')).finalize(); + auto left_update = CellStorer::serialize_refcnt_diffs(1); + auto right_update = CellStorer::serialize_refcnt_diffs(1); + for (int i = 0; i < n; i++) { + CellStorer::merge_refcnt_diffs(left_update, right_update); + } + } + + private: + size_t tn_{}; +}; +class BenchCellStorerMergeValueAndRefcntDiff : public td::Benchmark { + public: + std::string get_description() const override { + return PSTRING() << "bench_cells_storer_merge_value_and_refcnt_diffs"; + } + + void run(int n) override { + auto cell = vm::CellBuilder().store_bytes(std::string(32, 'A')).finalize(); + auto value = CellStorer::serialize_value(10, cell, false); + auto update = CellStorer::serialize_refcnt_diffs(1); + for (int i = 0; i < n; i++) { + CellStorer::merge_value_and_refcnt_diff(value, update); + } + } + + private: + size_t tn_{}; +}; +TEST(Bench, CellStorerMerge) { + bench(BenchCellStorerMergeRefcntDiffs()); + bench(BenchCellStorerMergeValueAndRefcntDiff()); +} + TEST(Cell, BocHands) { serialize_boc(CellBuilder{}.store_bytes("AAAAAAAA").finalize()); auto a = CellBuilder{}.store_bytes("abcd").store_ref(CellBuilder{}.store_bytes("???").finalize()).finalize(); @@ -2262,7 +2905,37 @@ TEST(TonDb, BocRespectsUsageCell) { ASSERT_STREQ(serialization, serialization_of_virtualized_cell); } -void test_dynamic_boc_respectes_usage_cell(vm::BocOptions options) { +TEST(UsageTree, ThreadSafe) { + size_t test_n = 100; + td::Random::Xorshift128plus rnd(123); + for (size_t test_i = 0; test_i < test_n; test_i++) { + auto cell = vm::gen_random_cell(rnd.fast(2, 100), rnd, false); + auto usage_tree = std::make_shared(); + auto usage_cell = vm::UsageCell::create(cell, usage_tree->root_ptr()); + std::ptrdiff_t threads_n = 1; // TODO: when CellUsageTree is thread safe, change it to 4 + auto barrier = std::barrier{threads_n}; + std::vector threads; + std::vector explorations(threads_n); + for (std::ptrdiff_t i = 0; i < threads_n; i++) { + threads.emplace_back([&, i = i]() { + barrier.arrive_and_wait(); + explorations[i] = vm::CellExplorer::random_explore(usage_cell, rnd); + }); + } + for (auto &thread : threads) { + thread.join(); + } + auto proof = vm::MerkleProof::generate(cell, usage_tree.get()); + auto virtualized_proof = vm::MerkleProof::virtualize(proof, 1); + for (auto &exploration : explorations) { + auto new_exploration = vm::CellExplorer::explore(virtualized_proof, exploration.ops); + ASSERT_EQ(exploration.log, new_exploration.log); + } + } +} + +/* +vm::DynamicBagOfCellsDb::Stats test_dynamic_boc_respects_usage_cell(vm::BocOptions options) { td::Random::Xorshift128plus rnd(options.seed); auto cell = vm::gen_random_cell(20, rnd, true); auto usage_tree = std::make_shared(); @@ -2283,11 +2956,14 @@ void test_dynamic_boc_respectes_usage_cell(vm::BocOptions options) { auto serialization_of_virtualized_cell = serialize_boc(virtualized_proof); auto serialization = serialize_boc(cell); ASSERT_STREQ(serialization, serialization_of_virtualized_cell); + vm::DynamicBagOfCellsDb::Stats stats; + return stats; } TEST(TonDb, DynamicBocRespectsUsageCell) { - vm::with_all_boc_options(test_dynamic_boc_respectes_usage_cell, 20); + vm::with_all_boc_options(test_dynamic_boc_respects_usage_cell, 20, true); } +*/ TEST(TonDb, LargeBocSerializer) { td::Random::Xorshift128plus rnd{123}; diff --git a/crypto/vm/cells/Cell.h b/crypto/vm/cells/Cell.h index a75371dbb..e2b47ffc0 100644 --- a/crypto/vm/cells/Cell.h +++ b/crypto/vm/cells/Cell.h @@ -55,6 +55,7 @@ class Cell : public CellTraits { } // load interface + virtual td::Status set_data_cell(Ref &&data_cell) const = 0; virtual td::Result load_cell() const = 0; virtual Ref virtualize(VirtualizationParameters virt) const; virtual td::uint32 get_virtualization() const = 0; diff --git a/crypto/vm/cells/DataCell.cpp b/crypto/vm/cells/DataCell.cpp index e5f21b013..5134123d8 100644 --- a/crypto/vm/cells/DataCell.cpp +++ b/crypto/vm/cells/DataCell.cpp @@ -36,7 +36,8 @@ struct ArenaAllocator { T* obj = new (ptr) T(std::forward(args)...); return std::unique_ptr(obj); } -private: + + private: td::MutableSlice alloc_batch() { size_t batch_size = 1 << 20; auto batch = std::make_unique(batch_size); @@ -53,7 +54,7 @@ struct ArenaAllocator { return res; } }; -} +} // namespace std::unique_ptr DataCell::create_empty_data_cell(Info info) { if (use_arena) { ArenaAllocator allocator; diff --git a/crypto/vm/cells/DataCell.h b/crypto/vm/cells/DataCell.h index 6d3c845fc..b39ee1d4b 100644 --- a/crypto/vm/cells/DataCell.h +++ b/crypto/vm/cells/DataCell.h @@ -31,6 +31,9 @@ class DataCell : public Cell { static thread_local bool use_arena; DataCell(const DataCell& other) = delete; + DataCell(DataCell&& other) = delete; + DataCell& operator=(const DataCell& other) = delete; + DataCell& operator=(DataCell&& other) = delete; ~DataCell() override; static void store_depth(td::uint8* dest, td::uint16 depth) { @@ -126,6 +129,10 @@ class DataCell : public Cell { explicit DataCell(Info info); public: + td::Status set_data_cell(Ref&& data_cell) const override { + CHECK(get_hash() == data_cell->get_hash()); + return td::Status::OK(); + } td::Result load_cell() const override { return LoadedCell{Ref{this}, {}, {}}; } @@ -228,4 +235,3 @@ inline CellHash as_cell_hash(const Ref& cell) { } } // namespace vm - diff --git a/crypto/vm/cells/ExtCell.h b/crypto/vm/cells/ExtCell.h index 401bb0483..dbbd8575b 100644 --- a/crypto/vm/cells/ExtCell.h +++ b/crypto/vm/cells/ExtCell.h @@ -65,6 +65,9 @@ class ExtCell : public Cell { bool is_loaded() const override { return CellView(this)->is_loaded(); } + Ref> get_prunned_cell() const { + return prunned_cell_.load(); + } private: mutable td::AtomicRef data_cell_; @@ -112,6 +115,23 @@ class ExtCell : public Cell { return CellView(this)->get_depth(level); } + td::Status set_data_cell(Ref&& new_data_cell) const override { + auto prunned_cell = prunned_cell_.load(); + if (prunned_cell.is_null()) { + auto old_data_cell = data_cell_.get_unsafe(); + DCHECK(old_data_cell); + TRY_STATUS(old_data_cell->check_equals_unloaded(new_data_cell)); + return td::Status::OK(); + } + + TRY_STATUS(prunned_cell->check_equals_unloaded(new_data_cell)); + if (data_cell_.store_if_empty(new_data_cell)) { + prunned_cell_.store({}); + get_thread_safe_counter_unloaded().add(-1); + } + return td::Status::OK(); + } + td::Result> load_data_cell() const { auto data_cell = data_cell_.get_unsafe(); if (data_cell) { diff --git a/crypto/vm/cells/PrunnedCell.h b/crypto/vm/cells/PrunnedCell.h index a58b245cc..6e8b77093 100644 --- a/crypto/vm/cells/PrunnedCell.h +++ b/crypto/vm/cells/PrunnedCell.h @@ -142,6 +142,10 @@ class PrunnedCell : public Cell { return info_.get_depth(get_storage())[get_level_mask().apply(level).get_hash_i()]; } + td::Status set_data_cell(Ref &&data_cell) const override { + return td::Status::OK(); + } + td::Result load_cell() const override { return td::Status::Error("Can't load prunned branch"); } diff --git a/crypto/vm/cells/UsageCell.h b/crypto/vm/cells/UsageCell.h index 3e6e88982..978b91f76 100644 --- a/crypto/vm/cells/UsageCell.h +++ b/crypto/vm/cells/UsageCell.h @@ -36,6 +36,9 @@ class UsageCell : public Cell { return Ref{true, std::move(cell), std::move(tree_node), PrivateTag{}}; } + td::Status set_data_cell(Ref &&data_cell) const override { + return cell_->set_data_cell(std::move(data_cell)); + } // load interface td::Result load_cell() const override { TRY_RESULT(loaded_cell, cell_->load_cell()); diff --git a/crypto/vm/cells/VirtualCell.h b/crypto/vm/cells/VirtualCell.h index 02abc1c88..a75bdf9de 100644 --- a/crypto/vm/cells/VirtualCell.h +++ b/crypto/vm/cells/VirtualCell.h @@ -37,6 +37,9 @@ class VirtualCell : public Cell { } // load interface + td::Status set_data_cell(Ref &&data_cell) const override { + return cell_->set_data_cell(std::move(data_cell)); + } td::Result load_cell() const override { TRY_RESULT(loaded_cell, cell_->load_cell()); loaded_cell.virt = loaded_cell.virt.apply(virt_); diff --git a/crypto/vm/db/CellHashTable.h b/crypto/vm/db/CellHashTable.h index 522c987be..a38980638 100644 --- a/crypto/vm/db/CellHashTable.h +++ b/crypto/vm/db/CellHashTable.h @@ -40,6 +40,17 @@ class CellHashTable { return res; } + template + std::pair emplace(td::Slice hash, ArgsT &&...args) { + auto it = set_.find(hash); + if (it != set_.end()) { + return std::pair(const_cast(*it), false); + } + auto res = set_.emplace(std::forward(args)...); + CHECK(res.second); + return std::pair(const_cast(*res.first), res.second); + } + template void for_each(F &&f) { for (auto &info : set_) { @@ -64,7 +75,7 @@ class CellHashTable { size_t size() const { return set_.size(); } - InfoT* get_if_exists(td::Slice hash) { + InfoT *get_if_exists(td::Slice hash) { auto it = set_.find(hash); if (it != set_.end()) { return &const_cast(*it); diff --git a/crypto/vm/db/CellStorage.cpp b/crypto/vm/db/CellStorage.cpp index 06df461ef..a07d85e87 100644 --- a/crypto/vm/db/CellStorage.cpp +++ b/crypto/vm/db/CellStorage.cpp @@ -17,14 +17,19 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "vm/db/CellStorage.h" + +#include "td/utils/Parser.h" #include "vm/db/DynamicBagOfCellsDb.h" #include "vm/boc.h" #include "td/utils/base64.h" #include "td/utils/tl_parsers.h" #include "td/utils/tl_helpers.h" +#include + namespace vm { namespace { + class RefcntCellStorer { public: RefcntCellStorer(td::int32 refcnt, const td::Ref &cell, bool as_boc) @@ -43,7 +48,9 @@ class RefcntCellStorer { storer.store_slice(data); return; } + CHECK(refcnt_ > 0); store(refcnt_, storer); + CHECK(cell_.not_null()) store(*cell_, storer); for (unsigned i = 0; i < cell_->size_refs(); i++) { auto cell = cell_->get_ref(i); @@ -91,6 +98,7 @@ class RefcntCellParser { stored_boc_ = true; parse(refcnt, parser); } + CHECK(refcnt > 0); if (!need_data_) { return; } @@ -159,6 +167,9 @@ td::Result CellLoader::load(td::Slice hash, bool need_da DCHECK(get_status == KeyValue::GetStatus::NotFound); return LoadResult{}; } + if (serialized.empty()) { + return LoadResult{}; + } TRY_RESULT(res, load(hash, serialized, need_data, ext_cell_creator)); if (on_load_callback_) { on_load_callback_(res); @@ -198,6 +209,7 @@ td::Result CellLoader::load_refcnt(td::Slice hash) { if (res.refcnt_ == -1) { parse(res.refcnt_, parser); } + CHECK(res.refcnt_ > 0); TRY_STATUS(parser.get_status()); return res; } @@ -216,4 +228,77 @@ std::string CellStorer::serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc) { return kv_.set(cell->get_hash().as_slice(), serialize_value(refcnt, cell, as_boc)); } + +td::Status CellStorer::merge(td::Slice hash, td::int32 refcnt_diff) { + return kv_.merge(hash, serialize_refcnt_diffs(refcnt_diff)); +} + +void CellStorer::merge_value_and_refcnt_diff(std::string &left, td::Slice right) { + if (right.empty()) { + return; + } + CHECK(left.size() > 4); + CHECK(right.size() == 4); + + td::int32 left_refcnt = td::as(left.data()); + size_t shift = 0; + if (left_refcnt == -1) { + CHECK(left.size() >= 8); + left_refcnt = td::as(left.data() + 4); + shift = 4; + } + td::int32 right_refcnt_diff = td::as(right.data()); + td::int32 new_refcnt = left_refcnt + right_refcnt_diff; + CHECK(new_refcnt > 0); + td::as(left.data() + shift) = new_refcnt; +} +void CellStorer::merge_refcnt_diffs(std::string &left, td::Slice right) { + if (right.empty()) { + return; + } + if (left.empty()) { + left = right.str(); + return; + } + CHECK(left.size() == 4); + CHECK(right.size() == 4); + td::int32 left_refcnt_diff = td::as(left.data()); + td::int32 right_refcnt_diff = td::as(right.data()); + td::int32 total_refcnt_diff = left_refcnt_diff + right_refcnt_diff; + td::as(left.data()) = total_refcnt_diff; +} + +std::string CellStorer::serialize_refcnt_diffs(td::int32 refcnt_diff) { + TD_PERF_COUNTER(cell_store_refcnt_diff); + std::string s(4, 0); + td::as(s.data()) = refcnt_diff; + return s; +} + +td::Status CellStorer::apply_diff(const Diff &diff) { + switch (diff.type) { + case Diff::Set: + return kv_.set(diff.key.as_slice(), diff.value); + case Diff::Erase: + return kv_.erase(diff.key.as_slice()); + case Diff::Merge: + return kv_.merge(diff.key.as_slice(), diff.value); + default: + UNREACHABLE(); + } +} +td::Status CellStorer::apply_meta_diff(const MetaDiff &diff) { + switch (diff.type) { + case MetaDiff::Set: + CHECK(diff.key.size() != CellTraits::hash_bytes); + CHECK(!diff.value.empty()); + return kv_.set(diff.key, diff.value); + case MetaDiff::Erase: + CHECK(diff.key.size() != CellTraits::hash_bytes); + CHECK(diff.value.empty()); + return kv_.erase(diff.key); + default: + UNREACHABLE(); + } +} } // namespace vm diff --git a/crypto/vm/db/CellStorage.h b/crypto/vm/db/CellStorage.h index cabd7fdcb..ca32a8007 100644 --- a/crypto/vm/db/CellStorage.h +++ b/crypto/vm/db/CellStorage.h @@ -51,6 +51,9 @@ class CellLoader { td::Result load(td::Slice hash, bool need_data, ExtCellCreator &ext_cell_creator); static td::Result load(td::Slice hash, td::Slice value, bool need_data, ExtCellCreator &ext_cell_creator); td::Result load_refcnt(td::Slice hash); // This only loads refcnt_, cell_ == null + KeyValueReader &key_value_reader() const { + return *reader_; + } private: std::shared_ptr reader_; @@ -62,8 +65,28 @@ class CellStorer { CellStorer(KeyValue &kv); td::Status erase(td::Slice hash); td::Status set(td::int32 refcnt, const td::Ref &cell, bool as_boc); + td::Status merge(td::Slice hash, td::int32 refcnt_diff); + + static void merge_value_and_refcnt_diff(std::string &value, td::Slice right); + static void merge_refcnt_diffs(std::string &left, td::Slice right); + static std::string serialize_refcnt_diffs(td::int32 refcnt_diff); + static std::string serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc); + struct Diff { + enum Type { Set, Erase, Merge } type{Set}; + CellHash key; + std::string value{}; + }; + td::Status apply_diff(const Diff &diff); + + struct MetaDiff { + enum Type { Set, Erase } type{Set}; + std::string key; + std::string value{}; + }; + td::Status apply_meta_diff(const MetaDiff &diff); + private: KeyValue &kv_; }; diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp index 093037583..d6731b039 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDb.cpp @@ -66,19 +66,27 @@ struct CellInfo { struct Eq { using is_transparent = void; // Pred to use - bool operator()(const CellInfo &info, const CellInfo &other_info) const { return info.key() == other_info.key();} - bool operator()(const CellInfo &info, td::Slice hash) const { return info.key().as_slice() == hash;} - bool operator()(td::Slice hash, const CellInfo &info) const { return info.key().as_slice() == hash;} - + bool operator()(const CellInfo &info, const CellInfo &other_info) const { + return info.key() == other_info.key(); + } + bool operator()(const CellInfo &info, td::Slice hash) const { + return info.key().as_slice() == hash; + } + bool operator()(td::Slice hash, const CellInfo &info) const { + return info.key().as_slice() == hash; + } }; struct Hash { using is_transparent = void; // Pred to use using transparent_key_equal = Eq; - size_t operator()(td::Slice hash) const { return cell_hash_slice_hash(hash); } - size_t operator()(const CellInfo &info) const { return cell_hash_slice_hash(info.key().as_slice());} + size_t operator()(td::Slice hash) const { + return cell_hash_slice_hash(hash); + } + size_t operator()(const CellInfo &info) const { + return cell_hash_slice_hash(info.key().as_slice()); + } }; }; - bool operator<(const CellInfo &a, td::Slice b) { return a.key().as_slice() < b; } @@ -99,6 +107,36 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { return get_cell_info_lazy(level_mask, hash, depth).cell; } + td::Result>> meta_get_all(size_t max_count) const override { + std::vector> result; + auto s = loader_->key_value_reader().for_each_in_range("desc", "desd", + [&](const td::Slice &key, const td::Slice &value) { + if (result.size() >= max_count) { + return td::Status::Error("COUNT_LIMIT"); + } + if (td::begins_with(key, "desc") && key.size() != 32) { + result.emplace_back(key.str(), value.str()); + } + return td::Status::OK(); + }); + if (s.message() == "COUNT_LIMIT") { + s = td::Status::OK(); + } + TRY_STATUS(std::move(s)); + return result; + } + td::Result meta_get(td::Slice key, std::string &value) override { + return loader_->key_value_reader().get(key, value); + } + td::Status meta_set(td::Slice key, td::Slice value) override { + meta_diffs_.push_back( + CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Set, .key = key.str(), .value = value.str()}); + return td::Status::OK(); + } + td::Status meta_erase(td::Slice key) override { + meta_diffs_.push_back(CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Erase, .key = key.str()}); + return td::Status::OK(); + } td::Result> load_cell(td::Slice hash) override { auto info = hash_table_.get_if_exists(hash); if (info && info->sync_with_db) { @@ -198,21 +236,29 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat if (is_prepared_for_commit()) { return td::Status::OK(); } + td::PerfWarningTimer timer_dfs_new_cells_in_db("dfs_new_cells_in_db"); for (auto &new_cell : to_inc_) { auto &new_cell_info = get_cell_info(new_cell); dfs_new_cells_in_db(new_cell_info); } + timer_dfs_new_cells_in_db.reset(); + td::PerfWarningTimer timer_dfs_new_cells("dfs_new_cells"); for (auto &new_cell : to_inc_) { auto &new_cell_info = get_cell_info(new_cell); dfs_new_cells(new_cell_info); } + timer_dfs_new_cells.reset(); + td::PerfWarningTimer timer_dfs_old_cells("dfs_old_cells"); for (auto &old_cell : to_dec_) { auto &old_cell_info = get_cell_info(old_cell); dfs_old_cells(old_cell_info); } + timer_dfs_old_cells.reset(); + td::PerfWarningTimer timer_save_diff_prepare("save_diff_prepare"); save_diff_prepare(); + timer_save_diff_prepare.reset(); to_inc_.clear(); to_dec_.clear(); @@ -222,6 +268,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat td::Status commit(CellStorer &storer) override { prepare_commit(); + td::PerfWarningTimer times_save_diff("save diff", 0.01); save_diff(storer); // Some elements are erased from hash table, to keep it small. // Hash table is no longer represents the difference between the loader and @@ -249,7 +296,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat celldb_compress_depth_ = value; } - vm::ExtCellCreator& as_ext_cell_creator() override { + vm::ExtCellCreator &as_ext_cell_creator() override { return *this; } @@ -259,6 +306,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat std::vector> to_dec_; CellHashTable hash_table_; std::vector visited_; + std::vector meta_diffs_; Stats stats_diff_; td::uint32 celldb_compress_depth_{0}; @@ -269,8 +317,9 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat class SimpleExtCellCreator : public ExtCellCreator { public: - explicit SimpleExtCellCreator(std::shared_ptr cell_db_reader) : - cell_db_reader_(std::move(cell_db_reader)) {} + explicit SimpleExtCellCreator(std::shared_ptr cell_db_reader) + : cell_db_reader_(std::move(cell_db_reader)) { + } td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { TRY_RESULT(ext_cell, DynamicBocExtCell::create(PrunnedCellInfo{level_mask, hash, depth}, @@ -279,7 +328,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat return std::move(ext_cell); } - std::vector>& get_created_cells() { + std::vector> &get_created_cells() { return created_cells_; } @@ -382,8 +431,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat } bool not_in_db = false; - for_each( - info, [¬_in_db, this](auto &child_info) { not_in_db |= !dfs_new_cells_in_db(child_info); }, false); + for_each(info, [¬_in_db, this](auto &child_info) { not_in_db |= !dfs_new_cells_in_db(child_info); }, false); if (not_in_db) { CHECK(!info.in_db); @@ -441,6 +489,10 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat for (auto info_ptr : visited_) { save_cell(*info_ptr, storer); } + for (auto meta_diff : meta_diffs_) { + storer.apply_meta_diff(meta_diff); + } + meta_diffs_.clear(); visited_.clear(); } @@ -558,6 +610,8 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat } auto res = r_res.move_as_ok(); if (res.status != CellLoader::LoadResult::Ok) { + LOG_CHECK(info.cell.not_null()) << "Trying to load nonexistent cell from db " + << CellHash::from_slice(hash).to_hex(); break; } info.cell = std::move(res.cell()); @@ -651,7 +705,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat CellHashTable cells_; - std::queue load_queue_; + std::queue load_queue_; td::uint32 active_load_ = 0; td::uint32 max_parallel_load_ = 4; }; @@ -814,11 +868,10 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat pca_state_->promise_.set_result(td::Unit()); pca_state_ = {}; } - }; } // namespace -std::unique_ptr DynamicBagOfCellsDb::create() { +std::unique_ptr DynamicBagOfCellsDb::create(CreateV1Options) { return std::make_unique(); } } // namespace vm diff --git a/crypto/vm/db/DynamicBagOfCellsDb.h b/crypto/vm/db/DynamicBagOfCellsDb.h index 62864ad97..82028f3fe 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.h +++ b/crypto/vm/db/DynamicBagOfCellsDb.h @@ -17,6 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "td/db/KeyValue.h" #include "vm/cells.h" #include "td/utils/Slice.h" @@ -49,13 +50,23 @@ class CellDbReader { class DynamicBagOfCellsDb { public: virtual ~DynamicBagOfCellsDb() = default; + + virtual td::Result>> meta_get_all(size_t max_count) const = 0; + virtual td::Result meta_get(td::Slice key, std::string &value) = 0; + virtual td::Status meta_set(td::Slice key, td::Slice value) = 0; + virtual td::Status meta_erase(td::Slice key) = 0; + virtual td::Result> load_cell(td::Slice hash) = 0; virtual td::Result> load_root(td::Slice hash) = 0; virtual td::Result> load_root_thread_safe(td::Slice hash) const = 0; + virtual td::Result>> load_known_roots() const { + return std::vector>(); + } struct Stats { td::int64 roots_total_count{0}; td::int64 cells_total_count{0}; td::int64 cells_total_size{0}; + td::NamedStats named_stats; std::vector> custom_stats; void apply_diff(const Stats &diff) { roots_total_count += diff.roots_total_count; @@ -64,6 +75,20 @@ class DynamicBagOfCellsDb { CHECK(roots_total_count >= 0); CHECK(cells_total_count >= 0); CHECK(cells_total_size >= 0); + named_stats.apply_diff(diff.named_stats); + } + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const Stats &stats) { + sb << "STATS\n"; + for (auto &p : stats.custom_stats) { + sb << "\t" << p.first << "\t" << p.second << "\n"; + } + for (auto &p : stats.named_stats.stats_int) { + sb << "\t" << p.first << "\t" << p.second << "\n"; + } + for (auto &p : stats.named_stats.stats_str) { + sb << "\t" << p.first << "\t" << p.second << "\n"; + } + return sb; } }; virtual void inc(const Ref &old_root) = 0; @@ -72,7 +97,7 @@ class DynamicBagOfCellsDb { virtual td::Status prepare_commit() = 0; virtual Stats get_stats_diff() = 0; virtual td::Result get_stats() { - return td::Status::Error("Not implemented"); + return Stats{}; } virtual td::Status commit(CellStorer &) = 0; virtual std::shared_ptr get_cell_db_reader() = 0; @@ -83,25 +108,49 @@ class DynamicBagOfCellsDb { virtual void set_celldb_compress_depth(td::uint32 value) = 0; virtual vm::ExtCellCreator &as_ext_cell_creator() = 0; - static std::unique_ptr create(); + class AsyncExecutor { + public: + virtual ~AsyncExecutor() { + } + virtual void execute_async(std::function f) = 0; + virtual void execute_sync(std::function f) = 0; + virtual std::string describe() const { + return "AsyncExecutor"; + } + }; + + struct CreateV1Options { + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const CreateV1Options &options) { + return sb << "V1{}"; + } + }; + static std::unique_ptr create(CreateV1Options = {}); + + struct CreateV2Options { + size_t extra_threads{std::thread::hardware_concurrency()}; + std::shared_ptr executor{}; + size_t cache_ttl_max{2000}; + size_t cache_size_max{1000000}; + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const CreateV2Options &options) { + return sb << "V2{extra_threads=" << options.extra_threads << ", cache_ttl_max=" << options.cache_ttl_max + << ", cache_size_max=" << options.cache_size_max << "}"; + } + }; + static std::unique_ptr create_v2(CreateV2Options options); struct CreateInMemoryOptions { size_t extra_threads{std::thread::hardware_concurrency()}; bool verbose{true}; - // Allocated DataCels will never be deleted + // Allocated DataCells will never be deleted bool use_arena{false}; // Almost no overhead in memory during creation, but will scan database twice bool use_less_memory_during_creation{true}; - }; - static std::unique_ptr create_in_memory(td::KeyValueReader *kv, CreateInMemoryOptions options); - - class AsyncExecutor { - public: - virtual ~AsyncExecutor() { + friend td::StringBuilder &operator<<(td::StringBuilder &sb, const CreateInMemoryOptions &options) { + return sb << "InMemory{extra_threads=" << options.extra_threads << ", use_arena=" << options.use_arena + << ", use_less_memory_during_creation=" << options.use_less_memory_during_creation << "}"; } - virtual void execute_async(std::function f) = 0; - virtual void execute_sync(std::function f) = 0; }; + static std::unique_ptr create_in_memory(td::KeyValueReader *kv, CreateInMemoryOptions options); virtual void load_cell_async(td::Slice hash, std::shared_ptr executor, td::Promise> promise) = 0; diff --git a/crypto/vm/db/DynamicBagOfCellsDbV2.cpp b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp new file mode 100644 index 000000000..eff74e214 --- /dev/null +++ b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp @@ -0,0 +1,1511 @@ +#include "vm/db/DynamicBagOfCellsDb.h" +#include "vm/db/CellStorage.h" +#include "vm/db/CellHashTable.h" + +#include "vm/cells/ExtCell.h" + +#include "td/utils/base64.h" +#include "td/utils/format.h" +#include "td/utils/ThreadSafeCounter.h" +#include "td/utils/misc.h" +#include "validator/validator.h" + +#include "vm/cellslice.h" + +#include + +namespace vm { +namespace { + +// Very stupid Vector/MpmcQueue +template +struct TsVector { + TsVector() { + first_block_size_ = 64; + blocks_[0].data.resize(first_block_size_); + blocks_[0].is_ready = true; + } + TsVector(std::vector base) { + first_block_size_ = base.size(); + blocks_[0].data = std::move(base); + blocks_[0].is_ready = true; + } + struct Block { + std::mutex mutex; + std::atomic is_ready{false}; + std::vector data; + }; + T &at(size_t i) { + td::uint64 j = i / first_block_size_; + td::int32 hb = 63 - td::count_leading_zeroes64(j); // hb = -1 if j=0, else hb>=0 + + // If j=0, hb<0, so hb>>31 = -1 => mask=0 + // If j>0, hb>=0, so hb>>31=0 => mask=~0 (all ones) + td::uint64 mask = ~(td::uint64)(hb >> 31); + + size_t block_i = hb + 1; + uint64_t shift = hb & 63ULL; + uint64_t start = ((1ULL << shift) * first_block_size_) & mask; + size_t pos_in_block = i - start; + auto &block = blocks_[block_i]; + if (block.is_ready.load(std::memory_order_acquire)) { + return block.data.at(pos_in_block); + } + + std::unique_lock lock(block.mutex); + if (block.is_ready.load(std::memory_order_acquire)) { + return block.data.at(pos_in_block); + } + block.resize(start); + block.is_ready.store(true, std::memory_order_release); + return block.data.at(pos_in_block); + } + template + void push_back(S &&value) { + at(end_.fetch_add(1, std::memory_order_relaxed)) = std::forward(value); + } + T pop_front() { + auto pos = begin_.fetch_add(1, std::memory_order_relaxed); + while (pos >= end_.load(std::memory_order_acquire)) { + // This may (or may not) use too much CPU + td::this_thread::yield(); + } + return std::move(at(pos)); + } + size_t size() const { + return end_.load(); + } + + std::array blocks_; + size_t first_block_size_{0}; + std::atomic begin_{0}; + std::atomic end_{0}; +}; +struct CellInfo; + +class CellDbReaderExt; +struct DynamicBocExtCellExtra { + std::shared_ptr reader; +}; + +class DynamicBocCellLoader { + public: + static td::Result> load_data_cell(const ExtCell &cell, + const DynamicBocExtCellExtra &extra); +}; +using DynamicBocExtCell = ExtCell; + +class CellDbReaderExt : public CellDbReader { + public: + virtual td::Result> load_ext_cell(Ref cell) = 0; +}; + +td::Result> DynamicBocCellLoader::load_data_cell(const DynamicBocExtCell &cell, + const DynamicBocExtCellExtra &extra) { + return extra.reader->load_ext_cell(Ref(&cell)); +} + +#define S(x) \ + td::NamedThreadSafeCounter::CounterRef x { \ + nc.get_counter(#x) \ + } + +struct CacheStats { + td::NamedThreadSafeCounter nc; + S(load_cell_ext); + S(load_cell_ext_cache_hits); + S(load_cell_sync); + S(load_cell_sync_cache_hits); + S(load_cell_async); + S(load_cell_async_cache_hits); + S(ext_cells); + S(ext_cells_load); + S(ext_cells_load_cache_hits); + + S(kv_read_found); + S(kv_read_not_found); + + S(sync_with_db); + S(sync_with_db_only_ref); + S(load_cell_no_cache); +}; + +struct CommitStats { + td::NamedThreadSafeCounter nc; + + S(to_inc); + S(to_dec); + + S(gather_new_cells_calls); + S(gather_new_cells_calls_it); + S(update_parents_calls); + S(update_parents_calls_it); + S(dec_calls); + S(dec_calls_it); + + S(new_cells); + S(new_cells_leaves); + + S(new_cells_loaded_not_in_db); + S(new_cells_loaded_in_db); + S(new_cells_not_in_db_fast); + + S(dec_loaded); + S(dec_to_zero); + + S(changes_loaded); + + // new diff logic + S(diff_zero); + S(diff_full); + S(diff_erase); + S(diff_ref_cnt); + + // old full data logic + S(inc_save); + S(inc_save_full); + S(inc_save_only_ref_cnt); + S(inc_new_cell); + S(inc_just_ref_cnt); + + S(dec_save); + S(dec_save_full); + S(dec_save_only_refcnt); + S(dec_save_erase); + S(dec_erase_cell); + S(dec_just_ref_cnt); +}; + +template +struct AtomicPod { + T load() const { + while (true) { + if (auto res = try_read_stable()) { + return res->second; + } + } + } + + template + std::pair update(F &&f) { + while (true) { + auto res = try_read_stable(); + if (!res) { + continue; + } + auto [before, old_data] = *res; + + auto o_new_data = f(old_data); + if (!o_new_data) { + return {old_data, false}; + } + + if (!lock_.compare_exchange_weak(before, before + 1, std::memory_order_acq_rel, std::memory_order_relaxed)) { + continue; + } + + data_ = *o_new_data; // relaxed store inside lock + lock_.fetch_add(1, std::memory_order_release); + return {*o_new_data, true}; + } + } + + private: + mutable std::atomic lock_{0}; + T data_{}; + + std::optional> try_read_stable() const { + auto before = lock_.load(std::memory_order_acquire); + if (before % 2 == 1) { + return std::nullopt; + } + T temp = data_; // relaxed read is ok, checked by versioning + auto after = lock_.load(std::memory_order_acquire); + if (after != before) { + return std::nullopt; + } + return std::make_pair(before, temp); + } +}; + +struct InDbInfo { + std::vector parents; + std::atomic pending_children{0}; + std::atomic maybe_in_db{true}; + std::atomic visited_in_gather_new_cells{false}; +}; +td::StringBuilder &operator<<(td::StringBuilder &sb, const InDbInfo &info) { + sb << "mb_in_db:" << info.maybe_in_db.load() << " chld_n:" << info.pending_children + << " prnt_n:" << info.parents.size(); + return sb; +} +struct CellInfo { + struct State { + // db_ref_cnt and in_db are correct + bool sync_with_db{false}; + + // ignore if sync_with_db is false + td::int32 db_ref_cnt{0}; + td::int32 db_refcnt_fixup{0}; + + // if true - cell is definitely in db + // if false - we know that cell is not in db only is sync_with_db=true + bool in_db{false}; + + // diff to be applied + }; + AtomicPod state; + std::atomic ref_cnt_diff{0}; + + std::atomic visited{false}; + td::unique_ptr in_db_info_ptr; + std::mutex mutex; + + // Could be AtomicRef, but is am not sure that it is worth it + const Ref cell; + + explicit CellInfo(Ref cell) : cell(std::move(cell)) { + } + + InDbInfo &in_db_info() { + return *in_db_info_ptr; + } + const InDbInfo &in_db_info() const { + return *in_db_info_ptr; + } + InDbInfo &in_db_info_create() { // NOT thread safe + if (!in_db_info_ptr) { + in_db_info_ptr = td::make_unique(); + } + return in_db_info(); + } + InDbInfo &in_db_info_create(CellInfo *parent) { // Thread Safe + std::unique_lock lock(mutex); + if (!in_db_info_ptr) { + in_db_info_ptr = td::make_unique(); + } + auto &res = *in_db_info_ptr; + if (parent != nullptr) { + res.parents.emplace_back(parent); + } + lock.unlock(); + return res; + } + void in_db_info_destroy() { + in_db_info_ptr = nullptr; + } + td::int32 inc_ref_cnt() { + return ref_cnt_diff.fetch_add(1, std::memory_order_relaxed) + 1; + } + td::int32 dec_ref_cnt() { + return ref_cnt_diff.fetch_sub(1, std::memory_order_relaxed) - 1; + } + td::int32 get_ref_cnt_diff() const { + return ref_cnt_diff.load(std::memory_order_relaxed); + } + + void set_not_in_db() { + state.update([&](State state) -> std::optional { + if (state.sync_with_db) { + CHECK(state.db_ref_cnt == 0); + CHECK(!state.in_db); + return {}; + } + state.sync_with_db = true; + state.in_db = false; + state.db_ref_cnt = 0; + return state; + }); + } + void set_in_db() { + state.update([&](State state) -> std::optional { + if (state.sync_with_db) { + //LOG_CHECK(state.in_db) << *this; + return {}; + } + state.in_db = true; + return state; + }); + } + void synced_with_db(td::int32 db_ref_cnt) { + state.update([&](State state) -> std::optional { + if (state.sync_with_db) { + CHECK(state.in_db); + CHECK(state.db_ref_cnt == db_ref_cnt); + return {}; + } + state.in_db = true; + state.db_ref_cnt = db_ref_cnt; + return state; + }); + } + bool visit() { + return !visited.exchange(true); + } + void on_written_to_db() { + auto diff = ref_cnt_diff.exchange(0); + state.update([&](State state) -> std::optional { + if (diff == 0) { + return {}; + } + if (state.sync_with_db) { + state.db_ref_cnt += diff; + CHECK(state.db_ref_cnt >= 0); + state.in_db = state.db_ref_cnt > 0; + } else { + CHECK(diff > 0); + state.in_db = true; + state.db_refcnt_fixup += diff; + } + return state; + }); + } + + td::Result> get_data_cell() { + TRY_RESULT(loaded_cell, cell->load_cell()); + return loaded_cell.data_cell; + } + Cell::Hash key() const { + return cell->get_hash(); + } + bool operator<(const CellInfo &other) const { + return key() < other.key(); + } + + struct Eq { + using is_transparent = void; // Pred to use + bool operator()(const CellInfo &info, const CellInfo &other_info) const { + return info.key() == other_info.key(); + } + bool operator()(const CellInfo &info, td::Slice hash) const { + return info.key().as_slice() == hash; + } + bool operator()(td::Slice hash, const CellInfo &info) const { + return info.key().as_slice() == hash; + } + }; + struct Hash { + using is_transparent = void; // Pred to use + using transparent_key_equal = Eq; + size_t operator()(td::Slice hash) const { + return cell_hash_slice_hash(hash); + } + size_t operator()(const CellInfo &info) const { + return cell_hash_slice_hash(info.key().as_slice()); + } + }; +}; +td::StringBuilder &operator<<(td::StringBuilder &sb, const CellInfo &info) { + if (info.cell->is_loaded()) { + auto data_cell = info.cell->load_cell().move_as_ok().data_cell; + vm::CellSlice cs(vm::NoVm{}, data_cell); + sb << data_cell->get_hash().to_hex().substr(0, 8) << " refs:" << data_cell->size_refs() + << " data:" << cs.data_bits().to_hex(cs.size()) << " data_ptr=" << data_cell.get() << " data_ref_cnt(" + << data_cell->get_refcnt() << ")"; + } else { + sb << info.cell->get_hash().to_hex().substr(0, 8); + } + auto state = info.state.load(); + sb << " " << &info; + sb << "\n\tin_db=" << state.in_db << " sync_with_db=" << state.sync_with_db + << " ref_cnt_diff=" << info.get_ref_cnt_diff() << " db_ref_cnt=" << state.db_ref_cnt + << " db_ref_cnt_fixup=" << state.db_refcnt_fixup; + if (state.sync_with_db) { + sb << " REFS(" << info.get_ref_cnt_diff() + state.db_ref_cnt << ")"; + } + if (info.in_db_info_ptr) { + sb << " " << info.in_db_info(); + } + sb << " visited=" << info.visited.load(); + return sb; +} + +struct ExecutorOptions { + size_t extra_threads_n{0}; + std::shared_ptr async_executor; +}; +template +class ExecutorImpl { + public: + ExecutorImpl(ExecutorOptions options) : options_(options) { + } + ExecutorOptions options_; + using InputData = std::vector>; + using OutputData = std::vector>; + struct InputChunk { + td::Span infos; + size_t begin{}; + size_t end{}; + }; + + template + OutputData process(const InputData &data, const F &process_task_f) { + if (options_.extra_threads_n > 0) { + return process_parallel(data, process_task_f); + } else { + return process_sequential(data, process_task_f); + } + } + template + struct SingleThreadWorker { + const F &process_task_f; + mutable std::vector results{}; + void add_task(InputT input) const { + process_task_f(input, *this); + } + void add_result(OutputT output) const { + results.push_back(output); + } + }; + template + OutputData process_sequential(const InputData &data, const F &process_task_f) { + auto w = SingleThreadWorker{process_task_f}; + for (auto &chunk : data) { + for (auto &info : chunk) { + process_task_f(info, w); + } + } + + return {std::move(w.results)}; + } + + template + struct Shared; + + template + struct Worker { + size_t worker_i{}; + std::shared_ptr> shared; + + void add_task(InputT input) const { + shared->delay_or_process_task(input, *this); + } + void add_result(OutputT value) const { + shared->add_result(value, worker_i); + } + void loop() const { + shared->loop(*this); + } + }; + + template + struct Shared { + Shared(size_t workers_n, const InputData &input_data, const ProcessTaskF &process_task_f) + : input_chunks(prepare_input_chunks(input_data)) + , workers_n(workers_n) + , input_size(input_chunks.empty() ? 0 : input_chunks.back().end) + , batch_size(std::clamp(input_size / workers_n / 4, size_t(1), size_t(128))) + , process_task_f(process_task_f) { + } + + const std::vector input_chunks; + + const size_t workers_n{0}; + const size_t input_size{0}; + const size_t batch_size{128}; + + const ProcessTaskF &process_task_f; + + // Position in input + std::atomic next_input_i{0}; + + // Shared queue + // Probably a simpler queue would also work fine + td::MpmcQueue mpmc_queue{workers_n}; + using Waiter = td::MpmcSleepyWaiter; + Waiter waiter; + std::atomic mpmc_queue_size{workers_n}; // guard + + // Output vectors + struct ThreadData { + std::vector output; + char pad[TD_CONCURRENCY_PAD - sizeof(output)]; + }; + std::vector thread_data{workers_n}; + + auto prepare_input_chunks(const InputData &input_data) { + std::vector chunks; + for (auto &chunk : input_data) { + size_t prev_end = chunks.empty() ? 0 : chunks.back().end; + chunks.push_back({.infos = td::as_span(chunk), .begin = prev_end, .end = prev_end + chunk.size()}); + } + return chunks; + } + + void delay_or_process_task(InputT input, const Worker &worker) { + // if there is enough tasks in queue, we continue recursion + if (mpmc_queue_size.load(std::memory_order_acquire) > 256) { + process_task_f(input, worker); + } else { + mpmc_queue_size.fetch_add(1, std::memory_order_acq_rel); + mpmc_queue.push(input, worker.worker_i); + waiter.notify(); + } + } + + void add_result(OutputT result, size_t worker_i) { + thread_data[worker_i].output.push_back(std::move(result)); + } + + void process_initial_input(const Worker &worker) { + size_t input_chunk_i = 0; + while (true) { + auto begin_i = next_input_i.fetch_add(batch_size, std::memory_order_relaxed); + auto end_i = begin_i + batch_size; + if (begin_i >= input_size) { + break; + } + for (size_t i = begin_i; i < end_i && i < input_size; i++) { + while (input_chunks[input_chunk_i].end <= i) { + input_chunk_i++; + } + auto offset = i - input_chunks[input_chunk_i].begin; + auto task = input_chunks[input_chunk_i].infos[offset]; + process_task_f(task, worker); + } + } + } + + void on_processed_task_from_queue(size_t worker_i) { + if (mpmc_queue_size.fetch_add(-1, std::memory_order_acq_rel) == 1) { + for (size_t i = 0; i < workers_n; i++) { + mpmc_queue.push(nullptr, worker_i); + waiter.notify(); + } + } + } + + void process_queue(const Worker &worker) { + on_processed_task_from_queue(worker.worker_i); + + Waiter::Slot slot; + waiter.init_slot(slot, td::narrow_cast(worker.worker_i)); + + while (true) { + InputT input{}; + if (mpmc_queue.try_pop(input, worker.worker_i)) { + waiter.stop_wait(slot); + if (!input) { + break; + } + process_task_f(input, worker); + on_processed_task_from_queue(worker.worker_i); + } else { + waiter.wait(slot); + } + } + } + void loop(const Worker &worker) { + process_initial_input(worker); + process_queue(worker); + } + void finish() const { + CHECK(mpmc_queue_size == 0); + } + }; + + template + OutputData process_parallel(const InputData &input_data, const F &process_task_f) { + const size_t workers_n = options_.extra_threads_n + 1; + auto shared = std::make_shared>(workers_n, input_data, process_task_f); + + CHECK(workers_n >= 1); + for (size_t i = 0; i < workers_n; i++) { + auto to_run = [worker = Worker{.worker_i = i, .shared = shared}] { worker.loop(); }; + + if (i + 1 == workers_n) { + to_run(); + } else if (options_.async_executor) { + options_.async_executor->execute_async(std::move(to_run)); + } else { + // NB: td::thread, NOT std::thread + td::thread(std::move(to_run)).detach(); + } + } + shared->finish(); + return td::transform(shared->thread_data, [](auto &&x) { return std::move(x.output); }); + } +}; +struct Executor { + Executor(ExecutorOptions options = {}) : options_(options) { + } + + template + auto operator()(const std::vector> &data, const F &process_task_f) { + return ExecutorImpl(options_).process(data, process_task_f); + } + + private: + ExecutorOptions options_; +}; + +// Thread safe storage for CellInfo +// Will be used by everybody as shared cache. Yes there is some overhead, but it don't want to create other hash table +struct CellInfoStorage { + public: + // All methods are thead safe + // All CellInfo pointers lives as long as CellInfoStorage + + // returns CellInfo, only if it is already exists + CellInfo *get_cell_info(td::Slice hash) { + return lock(hash)->hash_table.get_if_exists(hash); + } + + CellInfo &create_cell_info_from_db(Ref data_cell, td::int32 ref_cnt) { + auto &info = create_cell_info_from_data_cell(std::move(data_cell)); + info.synced_with_db(ref_cnt); + return info; + } + + // Creates CellInfo from data_cell, or updates existing CellInfo if is not yet loaded + CellInfo &create_cell_info_from_data_cell(Ref cell) { + CHECK(cell.not_null()); + CHECK(cell->is_loaded()); + + auto hash = cell->get_hash(); + auto [info, created] = lock(hash.as_slice())->hash_table.emplace(hash.as_slice(), std::move(cell)); + + if (!created) { + info.cell->set_data_cell(std::move(cell)); + } + return info; + } + + // Creates CellInfo from cell. If cell is loaded, it will be used to rewrite or udpate current cell + CellInfo &create_cell_info(Ref cell, CellDbReaderExt *from_reader, CacheStats &stats) { + if (cell->is_loaded()) { + return create_cell_info_from_data_cell(cell->load_cell().move_as_ok().data_cell); + } + + bool our_ext_cell = false; + auto ext_cell = dynamic_cast(cell.get()); + if (ext_cell) { + auto prunned_cell = ext_cell->get_prunned_cell(); + if (prunned_cell.not_null()) { + our_ext_cell = prunned_cell->get_extra().reader.get() == from_reader; + } + our_ext_cell = true; + } else if (!cell->is_loaded()) { + // if we cached cell from OTHER db is good idea to drop it ASAP + force_drop_cache_.store(true, std::memory_order_relaxed); + } + + auto hash = cell->get_hash(); + auto [info, created] = lock(hash.as_slice())->hash_table.emplace(hash.as_slice(), std::move(cell)); + if (our_ext_cell) { + stats.ext_cells_load.inc(); + if (info.cell->is_loaded()) { + stats.ext_cells_load_cache_hits.inc(); + } + info.set_in_db(); + } + return info; + } + + void dump() { + LOG(ERROR) << "===========BEGIN DUMP==========="; + for (auto &bucket : buckets_) { + std::lock_guard guard(bucket.mutex); + bucket.hash_table.for_each([&](auto &info) { LOG(INFO) << info; }); + } + LOG(ERROR) << "===========END DUMP==========="; + } + + size_t cache_size() { + size_t res = 0; + for (auto &bucket : buckets_) { + std::lock_guard guard(bucket.mutex); + res += bucket.hash_table.size(); + } + return res; + } + bool force_drop_cache() { + return force_drop_cache_.load(std::memory_order_relaxed); + } + + private: + struct Bucket { + std::mutex mutex; + CellHashTable hash_table; + }; + constexpr static size_t buckets_n = 8192; + std::array bucket_; + + struct Unlock { + void operator()(Bucket *bucket) const { + bucket->mutex.unlock(); + } + }; + std::array buckets_{}; + std::atomic force_drop_cache_{false}; + + std::unique_ptr lock(Bucket &bucket) { + bucket.mutex.lock(); + return std::unique_ptr(&bucket); + } + std::unique_ptr lock(td::Slice key) { + auto hash = td::as(key.substr(16, 8).ubegin()); + auto bucket_i = hash % buckets_n; + return lock(buckets_[bucket_i]); + } +}; + +class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { + public: + explicit DynamicBagOfCellsDbImplV2(CreateV2Options options) : options_(options) { + get_thread_safe_counter().inc(); + // LOG(ERROR) << "Constructor called for DynamicBagOfCellsDbImplV2"; + } + ~DynamicBagOfCellsDbImplV2() { + // LOG(ERROR) << "Destructor called for DynamicBagOfCellsDbImplV2"; + get_thread_safe_counter().add(-1); + + if (cell_db_reader_) { + cell_db_reader_->drop_cache(); + } + } + + td::Result>> meta_get_all(size_t max_count) const override { + CHECK(meta_db_fixup_.empty()); + std::vector> result; + auto s = cell_db_reader_->key_value_reader().for_each_in_range( + "desc", "desd", [&](const td::Slice &key, const td::Slice &value) { + if (result.size() >= max_count) { + return td::Status::Error("COUNT_LIMIT"); + } + if (td::begins_with(key, "desc") && key.size() != 32) { + result.emplace_back(key.str(), value.str()); + } + return td::Status::OK(); + }); + if (s.message() == "COUNT_LIMIT") { + s = td::Status::OK(); + } + TRY_STATUS(std::move(s)); + return result; + } + td::Result meta_get(td::Slice key, std::string &value) override { + auto it = meta_db_fixup_.find(key); + if (it != meta_db_fixup_.end()) { + if (it->second.empty()) { + return KeyValue::GetStatus::NotFound; + } + value = it->second; + return KeyValue::GetStatus::Ok; + } + return cell_db_reader_->key_value_reader().get(key, value); + } + td::Status meta_set(td::Slice key, td::Slice value) override { + meta_diffs_.push_back( + CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Set, .key = key.str(), .value = value.str()}); + return td::Status::OK(); + } + td::Status meta_erase(td::Slice key) override { + meta_diffs_.push_back(CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Erase, .key = key.str()}); + return td::Status::OK(); + } + td::Result> load_cell(td::Slice hash) override { + CHECK(cell_db_reader_); + return cell_db_reader_->load_cell(hash); + } + td::Result> load_root(td::Slice hash) override { + return load_cell(hash); + } + td::Result> load_root_thread_safe(td::Slice hash) const override { + // TODO: it is better to use AtomicRef, or atomic shared pointer + // But to use AtomicRef we need a little refactoring + // And std::atomic> is still unsupported by clang + std::unique_lock lock(atomic_cell_db_reader_mutex_); + auto reader = atomic_cell_db_reader_; + lock.unlock(); + if (!reader) { + return td::Status::Error("Empty reader"); + } + return reader->load_cell(hash); + } + void load_cell_async(td::Slice hash, std::shared_ptr executor, + td::Promise> promise) override { + CHECK(cell_db_reader_); + return cell_db_reader_->load_cell_async(hash, std::move(executor), std::move(promise)); + } + void prepare_commit_async(std::shared_ptr executor, td::Promise promise) override { + auto promise_ptr = std::make_shared>(std::move(promise)); + executor->execute_async([this, promise_ptr = std::move(promise_ptr)] { + prepare_commit(); + promise_ptr->set_value(td::Unit()); + }); + } + + void inc(const Ref &cell) override { + if (cell.is_null()) { + return; + } + if (cell->get_virtualization() != 0) { + return; + } + to_inc_.push_back(cell); + } + void dec(const Ref &cell) override { + if (cell.is_null()) { + return; + } + if (cell->get_virtualization() != 0) { + return; + } + to_dec_.push_back(cell); + } + + bool is_prepared_for_commit() { + return to_inc_.empty() && to_dec_.empty(); + } + + Stats get_stats_diff() override { + return {}; + } + + td::Status prepare_commit() override { + if (is_prepared_for_commit()) { + return td::Status::OK(); + } + // NB: we don't use options.executor, because it is prone to deadlocks. We need extra_threads_n threads + // available for blocking + Executor executor{{.extra_threads_n = options_.extra_threads, .async_executor = {}}}; + // calculate in_db for all vertices reachable from to_inc_ roots + // - for ext cells we already know they are in db + // - calculate in_db up from leaves + // - if at least one child is not in db, then the cell is definitely not in db + // - so in best case only leaves will be loaded from db + // - this is optional step. All other logic must work in any case + // - only already loaded cells are loaded from db + + stats_.to_inc.add(to_inc_.size()); + stats_.to_dec.add(to_dec_.size()); + + std::vector> visited_cells; + auto add_visited_cells = [&](std::vector> new_visited_cells) { + for (auto &x : new_visited_cells) { + visited_cells.push_back(std::move(x)); + } + }; + + std::vector> new_cells_leaves; + { + td::PerfWarningTimer timer("celldb_v2: gather_new_cells"); + std::vector prepared_to_inc; + std::vector visited_roots; + for (auto &cell : to_inc_) { + auto &info = cell_db_reader_->cell_info(cell); + if (info.inc_ref_cnt() == 1 && info.visit()) { + visited_roots.push_back(&info); + } + if (info.state.load().in_db) { + continue; + } + auto &in_db_info = info.in_db_info_create(nullptr); + if (!in_db_info.visited_in_gather_new_cells.exchange(true)) { + prepared_to_inc.push_back(&info); + } + } + new_cells_leaves = + executor({std::move(prepared_to_inc)}, [&](CellInfo *info, auto &worker) { gather_new_cells(info, worker); }); + visited_cells.push_back(std::move(visited_roots)); + } + + // LOG(WARNING) << "new_cells_leaves: " << new_cells_leaves.size(); + { + td::PerfWarningTimer timer("celldb_v2: update_parents"); + add_visited_cells( + executor({std::move(new_cells_leaves)}, [&](CellInfo *info, auto &worker) { update_parents(info, worker); })); + } + { + td::PerfWarningTimer timer("dec"); + std::vector prepared_to_dec; + for (auto &cell : to_dec_) { + auto &info = cell_db_reader_->cell_info(cell); + prepared_to_dec.push_back(&info); + } + add_visited_cells( + executor({std::move(prepared_to_dec)}, [&](CellInfo *info, auto &worker) { dec_cell(info, worker); })); + } + + td::PerfWarningTimer timer_serialize("celldb_v2: save_diff_serialize", 0.01); + // LOG(INFO) << "threads_n = " << options_.extra_threads + 1; + diff_chunks_ = executor.operator()( + visited_cells, [&](CellInfo *info, auto &worker) { serialize_diff(info, worker); }); + timer_serialize.reset(); + + { + td::PerfWarningTimer timer("celldb_v2: clear"); + to_inc_.clear(); + to_dec_.clear(); + } + + //cell_db_reader_->dump(); + return td::Status::OK(); + } + + td::Status commit(CellStorer &storer) override { + prepare_commit(); + save_diff(storer); + // We DON'T delete entries from cache, so cache actually represents diff with snapshot in reader + // But we don't want took keep old snapshot forever + LOG_IF(ERROR, dbg) << "clear cell_db_reader"; + //cell_db_reader_->dump(); + //TODO: call drop_cache reliably via rtti + + constexpr bool always_drop_cache = false; + if (always_drop_cache) { + td::PerfWarningTimer timer("celldb_v2: reset reader"); + cell_db_reader_->drop_cache(); + cache_stats_.apply_diff(cell_db_reader_->get_stats()); + cache_stats_.stats_int["commits"] += 1; + cell_db_reader_ = {}; + // keep atomic reader, to it will be reused + } + return td::Status::OK(); + } + + std::shared_ptr get_cell_db_reader() override { + CHECK(cell_db_reader_); + return cell_db_reader_; + } + + td::Status set_loader(std::unique_ptr loader) override { + if (cell_db_reader_) { + auto cache_size = cell_db_reader_->cache_size(); + bool force_drop_cache = cell_db_reader_->force_drop_cache(); + if (loader && cache_size < options_.cache_size_max && cell_db_reader_ttl_ < options_.cache_ttl_max && + !force_drop_cache) { + // keep cache + cell_db_reader_ttl_++; + return td::Status::OK(); + } + + td::PerfWarningTimer timer(PSTRING() << "celldb_v2: reset reader, TTL=" << cell_db_reader_ttl_ << "/" + << options_.cache_ttl_max << ", cache_size=" << cache_size + << ", force_drop_cache=" << force_drop_cache); + cache_stats_.apply_diff(cell_db_reader_->get_stats()); + cell_db_reader_->drop_cache(); + cell_db_reader_ = {}; + meta_db_fixup_ = {}; + cell_db_reader_ttl_ = 0; + } + + if (loader) { + cell_db_reader_ = std::make_shared(std::move(loader)); + cell_db_reader_ttl_ = 0; + } + + { + std::lock_guard guard(atomic_cell_db_reader_mutex_); + atomic_cell_db_reader_ = cell_db_reader_; + } + return td::Status::OK(); + } + + void set_celldb_compress_depth(td::uint32 value) override { + CHECK(value == 0); + } + + vm::ExtCellCreator &as_ext_cell_creator() override { + CHECK(cell_db_reader_); + return *cell_db_reader_; + } + td::Result get_stats() override { + auto ps = stats_.nc.get_stats().with_prefix("storage_"); + ps.apply_diff(cache_stats_.with_prefix("cache_cum_")); + if (cell_db_reader_) { + ps.apply_diff(cell_db_reader_->get_stats().with_prefix("cache_now_")); + ps.apply_diff(cell_db_reader_->get_stats().with_prefix("cache_cum_")); + } + Stats res; + res.named_stats = std::move(ps); + res.named_stats.stats_int["cache.size"] = cell_db_reader_ ? cell_db_reader_->cache_size() : 0; + res.named_stats.stats_int["cache.size_max"] = options_.cache_size_max; + res.named_stats.stats_int["cache.ttl"] = cell_db_reader_ttl_; + res.named_stats.stats_int["cache.ttl_max"] = options_.cache_ttl_max; + return res; + } + + private: + static td::NamedThreadSafeCounter::CounterRef get_thread_safe_counter() { + static auto res = td::NamedThreadSafeCounter::get_default().get_counter("DynamicBagOfCellsDb"); + return res; + } + + class CellDbReaderImpl : public CellDbReaderExt, + public ExtCellCreator, + public std::enable_shared_from_this { + public: + explicit CellDbReaderImpl(std::unique_ptr cell_loader) : cell_loader_(std::move(cell_loader)) { + } + + size_t cache_size() const { + // NOT thread safe + if (internal_storage_) { + return internal_storage_->cache_size(); + } + return 0; + } + bool force_drop_cache() const { + // NOT thread safe + if (internal_storage_) { + return internal_storage_->force_drop_cache(); + } + return false; + } + void drop_cache() { + // NOT thread safe + internal_storage_.reset(); + } + + td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { + // thread safe function + stats_.ext_cells.inc(); + TRY_RESULT(ext_cell, DynamicBocExtCell::create(PrunnedCellInfo{level_mask, hash, depth}, + DynamicBocExtCellExtra{shared_from_this()})); + + return ext_cell; + } + CellInfo *register_ext_cell_inner(Ref ext_cell, CellInfoStorage &storage) { + auto &info = storage.create_cell_info(std::move(ext_cell), this, stats_); + return &info; + } + + void load_cell_async(td::Slice hash, std::shared_ptr executor, td::Promise> promise) { + // thread safe function + stats_.load_cell_async.inc(); + auto maybe_cell = load_cell_fast_path(hash, false, nullptr); + if (maybe_cell.not_null()) { + stats_.load_cell_async_cache_hits.inc(); + return promise.set_value(std::move(maybe_cell)); + } + auto promise_ptr = std::make_shared>>(std::move(promise)); + + executor->execute_async( + [self = shared_from_this(), promise_ptr = std::move(promise_ptr), hash = CellHash::from_slice(hash)]() { + promise_ptr->set_result(self->load_cell(hash.as_slice())); + }); + } + + td::Result> load_cell(td::Slice hash) override { + // thread safe function + stats_.load_cell_sync.inc(); + bool loaded{false}; + auto maybe_cell = load_cell_fast_path(hash, true, &loaded); + if (maybe_cell.not_null()) { + if (!loaded) { + stats_.load_cell_sync_cache_hits.inc(); + } + return maybe_cell; + } + return load_cell_slow_path(hash); + } + + td::Result> load_ext_cell(Ref ext_cell) override { + // thread safe function. + // Called by external cell + stats_.load_cell_ext.inc(); + auto storage = weak_storage_.lock(); + if (!storage) { + TRY_RESULT(load_result, load_cell_no_cache(ext_cell->get_hash().as_slice())); + return load_result.cell_; + } + // we delayed registering ext cell till this moment + auto cell_info = register_ext_cell_inner(std::move(ext_cell), *storage); + + CHECK(cell_info != nullptr); // currently all ext_cells are registered in cache + if (!cell_info->cell->is_loaded()) { + sync_with_db(*cell_info, true); + CHECK(cell_info->cell->is_loaded()); // critical, better to fail + } else { + stats_.load_cell_ext_cache_hits.inc(); + } + return cell_info->cell->load_cell().move_as_ok().data_cell; + } + + CellInfo &cell_info(Ref cell) { + // thread safe function, but called only by DB + CHECK(internal_storage_) + return internal_storage_->create_cell_info(std::move(cell), this, stats_); + } + + std::pair sync_with_db(CellInfo &info, bool need_data) { + // thread safe function, but called only by DB + auto effective_need_data = need_data; + if (info.cell->is_loaded()) { + effective_need_data = false; + } + return info.state.update([&](CellInfo::State state) -> std::optional { + if (state.sync_with_db) { + return {}; + } + stats_.sync_with_db.inc(); + if (!effective_need_data) { + stats_.sync_with_db_only_ref.inc(); + } + auto load_result = + cell_loader_->load(info.cell->get_hash().as_slice(), effective_need_data, *this).move_as_ok(); + + state.sync_with_db = true; + if (load_result.status == CellLoader::LoadResult::NotFound) { + CHECK(state.in_db == false); + CHECK(state.db_ref_cnt == 0); + stats_.kv_read_not_found.inc(); + return state; + } + stats_.kv_read_found.inc(); + + state.in_db = true; + state.db_ref_cnt = load_result.refcnt() + state.db_refcnt_fixup; + if (load_result.cell().not_null()) { + info.cell->set_data_cell(std::move(load_result.cell())); + } + CHECK(!need_data || info.cell->is_loaded()); + return state; + }); + } + + void dump() { + internal_storage_->dump(); + } + + td::NamedStats get_stats() const { + return stats_.nc.get_stats(); + } + td::KeyValueReader &key_value_reader() { + return cell_loader_->key_value_reader(); + } + + private: + static td::NamedThreadSafeCounter::CounterRef get_thread_safe_counter() { + static auto res = td::NamedThreadSafeCounter::get_default().get_counter("DynamicBagOfCellsDbLoader"); + return res; + } + std::shared_ptr internal_storage_{std::make_shared()}; + std::weak_ptr weak_storage_{internal_storage_}; + std::unique_ptr cell_loader_; + CacheStats stats_; + + Ref load_cell_fast_path(td::Slice hash, bool may_block, bool *loaded) { + auto storage = weak_storage_.lock(); + if (!storage) { + return {}; + } + auto cell_info = storage->get_cell_info(hash); + if (cell_info != nullptr) { + if (!cell_info->cell->is_loaded()) { + if (may_block) { + if (loaded) { + *loaded = true; + } + CHECK(cell_info->state.load().in_db); + sync_with_db(*cell_info, true); + CHECK(cell_info->cell->is_loaded()); + } else { + return {}; + } + } + return cell_info->cell->load_cell().move_as_ok().data_cell; + } + return {}; + } + td::Result load_cell_no_cache(td::Slice hash) { + stats_.load_cell_no_cache.inc(); + TRY_RESULT(load_result, cell_loader_->load(hash, true, *this)); + if (load_result.status == CellLoader::LoadResult::NotFound) { + stats_.kv_read_not_found.inc(); + return td::Status::Error("Cell load failed: not in db"); + } + stats_.kv_read_found.inc(); + return load_result; + } + td::Result> load_cell_slow_path(td::Slice hash) { + TRY_RESULT(load_result, load_cell_no_cache(hash)); + auto storage = weak_storage_.lock(); + if (!storage) { + return load_result.cell_; + } + auto &cell_info = storage->create_cell_info_from_db(std::move(load_result.cell()), load_result.refcnt()); + return cell_info.cell->load_cell().move_as_ok().data_cell; + } + }; + + CreateV2Options options_; + std::vector> to_inc_; + std::vector> to_dec_; + std::vector> diff_chunks_; + std::vector meta_diffs_; + std::map> meta_db_fixup_; + + mutable std::mutex atomic_cell_db_reader_mutex_; + std::shared_ptr atomic_cell_db_reader_; + + std::shared_ptr cell_db_reader_; + size_t cell_db_reader_ttl_{0}; + td::NamedStats cache_stats_; + CommitStats stats_; + bool dbg{false}; + + template + void gather_new_cells(CellInfo *info, WorkerT &worker) { + stats_.gather_new_cells_calls.inc(); + do { + // invariant: info is not in DB; with created in_db_info + // we enter into each root only once + stats_.gather_new_cells_calls_it.inc(); + stats_.new_cells.inc(); + auto &in_db_info = info->in_db_info(); + + CellSlice cs(vm::NoVm{}, info->cell); // ensure cell is loaded + CellInfo *prev_child_info = nullptr; + while (cs.have_refs()) { + auto *child_info = &cell_db_reader_->cell_info(cs.fetch_ref()); + auto child_state = child_info->state.load(); + + if (child_state.in_db) { + LOG_IF(INFO, dbg) << "gather_new_cells: IN DB\n\tchld: " << *child_info; + continue; + } + + auto &child_in_db_info = child_info->in_db_info_create(info); + in_db_info.pending_children.fetch_add(1, std::memory_order_relaxed); + + if (child_in_db_info.visited_in_gather_new_cells.exchange(true)) { + continue; + } + + if (prev_child_info != nullptr) { + worker.add_task(prev_child_info); + } + prev_child_info = child_info; + } + LOG_IF(INFO, dbg) << "gather_new_cells: NOT IN DB\n\t" << *info; + if (in_db_info.pending_children.load(std::memory_order_relaxed) == 0) { + worker.add_result(info); + stats_.new_cells_leaves.inc(); + LOG_IF(WARNING, dbg) << "gather_new_cells: ADD LEAVE\n\t" << *info; + } + info = prev_child_info; + } while (info != nullptr); + } + + template + void update_parents(CellInfo *info, const WorkerT &worker) { + stats_.update_parents_calls.inc(); + size_t it = 0; + do { + stats_.update_parents_calls_it.inc(); + it++; + //LOG(INFO) << "update_parents: it=" << it << "\n\t"; + auto &in_db_info = info->in_db_info(); + bool in_db = false; + if (in_db_info.maybe_in_db.load(std::memory_order_relaxed)) { + auto [state, loaded] = cell_db_reader_->sync_with_db(*info, false); + in_db = state.in_db; + if (in_db) { + stats_.new_cells_loaded_in_db.inc(); + } else { + stats_.new_cells_loaded_not_in_db.inc(); + } + } else { + stats_.new_cells_not_in_db_fast.inc(); + info->set_not_in_db(); + } + LOG_IF(INFO, dbg) << "update_parents: it=" << it << "\n\t" << *info; + + CellInfo *prev_parent{nullptr}; + for (auto &parent : in_db_info.parents) { + auto &parent_in_db_info = parent->in_db_info(); + if (!in_db) { + parent_in_db_info.maybe_in_db.store(false, std::memory_order_relaxed); + } + if (parent_in_db_info.pending_children.fetch_sub(1, std::memory_order_release) == 1) { + if (prev_parent) { + worker.add_task(prev_parent); + } + prev_parent = parent; + } + } + if (!in_db) { + CellSlice cs(vm::NoVm{}, info->cell); + while (cs.have_refs()) { + auto child = cs.fetch_ref(); + auto &child_info = cell_db_reader_->cell_info(std::move(child)); + if (child_info.inc_ref_cnt() == 1 && child_info.visit()) { + worker.add_result(&child_info); + } + } + } + info->in_db_info_destroy(); + info = prev_parent; + } while (info); + } + + template + void dec_cell(CellInfo *info, WorkerT &worker) { + stats_.dec_calls.inc(); + + while (true) { + stats_.dec_calls_it.inc(); + if (info->visit()) { + worker.add_result(info); + } + auto ref_cnt_diff = info->dec_ref_cnt(); + if (ref_cnt_diff > 0) { + LOG_IF(INFO, dbg) << "NOT DEC" + << "\n\t" << info; + break; + } + auto state = info->state.load(); + if (ref_cnt_diff == 0 && state.in_db) { + LOG_IF(INFO, dbg) << "NOT DEC (in_db) " + << "\n\t" << info; + break; + } + if (!state.sync_with_db) { + state = cell_db_reader_->sync_with_db(*info, true).first; + stats_.dec_loaded.inc(); + CHECK(ref_cnt_diff == 0 || state.in_db); + } + auto ref_cnt = state.db_ref_cnt + ref_cnt_diff; + if (ref_cnt > 0) { + LOG_IF(INFO, dbg) << "DEC " << ref_cnt << "\n\t" << info; + } else { + LOG_IF(ERROR, dbg) << "DEC " << ref_cnt << "\n\t" << info; + } + CHECK(ref_cnt >= 0); + if (ref_cnt > 0) { + break; + } + stats_.dec_to_zero.inc(); + CellSlice cs(vm::NoVm{}, info->cell); + if (!cs.have_refs()) { + break; + } + while (cs.size_refs() > 1) { + worker.add_task(&cell_db_reader_->cell_info(cs.fetch_ref())); + } + info = &cell_db_reader_->cell_info(cs.fetch_ref()); + } + } + + template + void serialize_diff(CellInfo *info, Worker &worker) { + info->visited.store(false, std::memory_order_relaxed); + auto ref_cnt_diff = info->get_ref_cnt_diff(); + if (ref_cnt_diff == 0) { + stats_.diff_zero.inc(); + return; + } + + bool merge_supported = true; + if (merge_supported) { + auto state = info->state.load(); + if (ref_cnt_diff < 0) { + CHECK(state.sync_with_db); + /* + if (state.db_ref_cnt + ref_cnt_diff == 0) { + LOG(ERROR) << "DEC ERASE " << info->cell->get_hash().to_hex(); + } else { + LOG(ERROR) << "DEC MERGE " << info->cell->get_hash().to_hex() << *info; + } + */ + } + if (ref_cnt_diff < 0 && state.sync_with_db && state.db_ref_cnt + ref_cnt_diff == 0) { + // Erase is better than Merge+CompactionFilter + // So I see no reason for CompactionFilter at all + worker.add_result({.type = CellStorer::Diff::Erase, .key = info->cell->get_hash()}); + stats_.diff_erase.inc(); + } else { + bool with_data = ref_cnt_diff > 0 && !state.in_db; + if (with_data) { + CHECK(state.sync_with_db); + auto data_cell = info->cell->load_cell().move_as_ok().data_cell; + stats_.diff_full.inc(); + worker.add_result({.type = CellStorer::Diff::Set, + .key = info->cell->get_hash(), + .value = CellStorer::serialize_value(ref_cnt_diff + state.db_ref_cnt, data_cell, false)}); + } else { + stats_.diff_ref_cnt.inc(); + worker.add_result({.type = CellStorer::Diff::Merge, + .key = info->cell->get_hash(), + .value = CellStorer::serialize_refcnt_diffs(ref_cnt_diff)}); + } + } + info->on_written_to_db(); + return; + } + + auto state = info->state.load(); + if (!state.sync_with_db) { + stats_.changes_loaded.inc(); + state = cell_db_reader_->sync_with_db(*info, true).first; + } + CHECK(state.sync_with_db); + auto new_ref_cnt = ref_cnt_diff + state.db_ref_cnt; + + if (ref_cnt_diff < 0) { + stats_.dec_save.inc(); + if (new_ref_cnt == 0) { + stats_.dec_erase_cell.inc(); + + LOG_IF(ERROR, dbg) << "DEC ERASE " << *info; + worker.add_result({.type = CellStorer::Diff::Erase, .key = info->cell->get_hash()}); + stats_.dec_save_erase.inc(); + } else { + stats_.dec_just_ref_cnt.inc(); + + LOG_IF(ERROR, dbg) << "DEC REFCNT " << *info; + CHECK(info->cell->is_loaded()); + worker.add_result( + {.type = CellStorer::Diff::Set, + .key = info->cell->get_hash(), + .value = CellStorer::serialize_value(new_ref_cnt, info->cell->load_cell().move_as_ok().data_cell, false)}); + stats_.dec_save_full.inc(); + } + } else { + stats_.inc_save.inc(); + CHECK(info->cell->is_loaded()); + if (state.db_ref_cnt == 0) { + stats_.inc_new_cell.inc(); + LOG_IF(ERROR, dbg) << "INC CREATE " << *info; + } else { + stats_.inc_just_ref_cnt.inc(); + LOG_IF(ERROR, dbg) << "INC REFCNT " << *info; + } + + worker.add_result( + {.type = CellStorer::Diff::Set, + .key = info->cell->get_hash(), + .value = CellStorer::serialize_value(new_ref_cnt, info->cell->load_cell().move_as_ok().data_cell, false)}); + stats_.inc_save_full.inc(); + } + } + + void save_diff(CellStorer &storer) { + td::PerfWarningTimer timer("celldb_v2: save_diff"); + td::PerfWarningTimer timer_store_to_db("celldb_v2: save_diff_store_to_db", 0.01); + // Have no idea hot to parallelize this in case of rocksdb + for (auto &diffs : diff_chunks_) { + for (auto &diff : diffs) { + storer.apply_diff(diff).ensure(); + } + } + for (auto &meta_diff : meta_diffs_) { + meta_db_fixup_[meta_diff.key] = meta_diff.value; + storer.apply_meta_diff(meta_diff).ensure(); + } + timer_store_to_db.reset(); + td::PerfWarningTimer timer_clear("celldb_v2: save_diff_clear"); + diff_chunks_.clear(); + meta_diffs_.clear(); + timer_clear.reset(); + } +}; +} // namespace + +std::unique_ptr DynamicBagOfCellsDb::create_v2(CreateV2Options options) { + return std::make_unique(options); +} +} // namespace vm diff --git a/crypto/vm/db/InMemoryBagOfCellsDb.cpp b/crypto/vm/db/InMemoryBagOfCellsDb.cpp index 03cad0934..e43cfde4e 100644 --- a/crypto/vm/db/InMemoryBagOfCellsDb.cpp +++ b/crypto/vm/db/InMemoryBagOfCellsDb.cpp @@ -413,6 +413,7 @@ class CellStorage { size_t dense_ht_size = 0; size_t new_ht_size = 0; for_each_bucket(0, [&](auto bucket_id, CellBucket &bucket) { + // TODO: this leads to CE when use_dense_hash_map == false dense_ht_capacity += bucket.infos_.dense_ht_values_.size(); dense_ht_size += bucket.infos_.dense_ht_size_; new_ht_capacity += bucket.infos_.new_ht_.bucket_count(); @@ -468,6 +469,14 @@ class CellStorage { } return td::Status::Error("not found"); } + td::Result>> load_known_roots_local() const { + auto lock = local_access_.lock(); + std::vector> result; + for (auto &root : roots_) { + result.emplace_back(root); + } + return result; + } td::Result> load_root_shared(const CellHash &hash) const { std::lock_guard lock(root_mutex_); if (auto it = roots_.find(hash); it != roots_.end()) { @@ -620,7 +629,7 @@ class CellStorage { sb << "\n\t" << key << "=" << value; } LOG_IF(ERROR, desc_count != 0 && desc_count != stats.roots_total_count + 1) - << "desc<> keys count is " << desc_count << " wich is different from roots count " << stats.roots_total_count; + << "desc<> keys count is " << desc_count << " which is different from roots count " << stats.roots_total_count; LOG_IF(WARNING, verbose) << P << "done in " << full_timer.elapsed() << "\n\troots_count=" << stats.roots_total_count << "\n\t" << desc_count << "\n\tcells_count=" << stats.cells_total_count @@ -757,15 +766,84 @@ class CellStorage { } }; +class MetaStorage { + public: + explicit MetaStorage(std::vector> values) + : meta_(std::move_iterator(values.begin()), std::move_iterator(values.end())) { + for (auto &p : meta_) { + CHECK(p.first.size() != CellTraits::hash_bytes); + } + } + std::vector> meta_get_all(size_t max_count) const { + std::vector> res; + for (const auto &[k, v] : meta_) { + if (res.size() >= max_count) { + break; + } + res.emplace_back(k, v); + } + return res; + } + KeyValue::GetStatus meta_get(td::Slice key, std::string &value) const { + auto lock = local_access_.lock(); + auto it = meta_.find(key.str()); + if (it == meta_.end()) { + return KeyValue::GetStatus::NotFound; + } + value = it->second; + return KeyValue::GetStatus::Ok; + } + void meta_set(td::Slice key, td::Slice value) { + auto lock = local_access_.lock(); + meta_[key.str()] = value.str(); + meta_diffs_.push_back( + CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Set, .key = key.str(), .value = value.str()}); + } + void meta_erase(td::Slice key) { + auto lock = local_access_.lock(); + meta_.erase(key.str()); + meta_diffs_.push_back(CellStorer::MetaDiff{.type = CellStorer::MetaDiff::Erase, .key = key.str()}); + } + std::vector extract_diffs() { + auto lock = local_access_.lock(); + return std::move(meta_diffs_); + } + + private: + mutable UniqueAccess local_access_; + std::unordered_map meta_; + std::vector meta_diffs_; +}; + class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { public: - explicit InMemoryBagOfCellsDb(td::unique_ptr storage) : storage_(std::move(storage)) { + explicit InMemoryBagOfCellsDb(td::unique_ptr storage, td::unique_ptr meta_storage) + : storage_(std::move(storage)), meta_storage_(std::move(meta_storage)) { + } + + td::Result>> meta_get_all(size_t max_count) const override { + return meta_storage_->meta_get_all(max_count); + } + td::Result meta_get(td::Slice key, std::string &value) override { + CHECK(key.size() != CellTraits::hash_bytes); + return meta_storage_->meta_get(key, value); + } + td::Status meta_set(td::Slice key, td::Slice value) override { + meta_storage_->meta_set(key, value); + return td::Status::OK(); + } + td::Status meta_erase(td::Slice key) override { + meta_storage_->meta_erase(key); + return td::Status::OK(); } td::Result> load_cell(td::Slice hash) override { return storage_->load_cell(CellHash::from_slice(hash)); } + td::Result>> load_known_roots() const override { + return storage_->load_known_roots_local(); + } td::Result> load_root(td::Slice hash) override { return storage_->load_root_local(CellHash::from_slice(hash)); } @@ -798,29 +876,37 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { TRY_STATUS(prepare_commit()); } + td::PerfWarningTimer times_save_diff("save diff"); Stats diff; CHECK(to_dec_.empty()); - for (auto &it : info_) { - auto &info = it.second; + for (auto &info : info_) { if (info.diff_refcnt == 0) { continue; } auto refcnt = td::narrow_cast(static_cast(info.db_refcnt) + info.diff_refcnt); - CHECK(refcnt >= 0); + LOG_CHECK(refcnt >= 0) << info.db_refcnt << " + " << info.diff_refcnt; if (refcnt > 0) { - cell_storer.set(refcnt, info.cell, false); + if (info.db_refcnt == 0) { + TRY_STATUS(cell_storer.set(refcnt, info.cell, false)); + } else { + TRY_STATUS(cell_storer.merge(info.cell->get_hash().as_slice(), info.diff_refcnt)); + } storage_->set(refcnt, info.cell); if (info.db_refcnt == 0) { diff.cells_total_count++; diff.cells_total_size += static_cast(info.cell->get_storage_size()); } } else { - cell_storer.erase(info.cell->get_hash().as_slice()); + TRY_STATUS(cell_storer.erase(info.cell->get_hash().as_slice())); storage_->erase(info.cell->get_hash()); diff.cells_total_count--; diff.cells_total_size -= static_cast(info.cell->get_storage_size()); } } + auto meta_diffs = meta_storage_->extract_diffs(); + for (const auto &meta_diff : meta_diffs) { + TRY_STATUS(cell_storer.apply_meta_diff(meta_diff)); + } storage_->apply_stats_diff(diff); info_ = {}; return td::Status::OK(); @@ -872,13 +958,39 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { private: td::unique_ptr storage_; + td::unique_ptr meta_storage_; struct Info { - td::int32 db_refcnt{0}; - td::int32 diff_refcnt{0}; + mutable td::int32 db_refcnt{0}; + mutable td::int32 diff_refcnt{0}; Ref cell; + vm::CellHash key() const { + return cell->get_hash(); + } + struct Eq { + using is_transparent = void; // Pred to use + bool operator()(const Info &info, const Info &other_info) const { + return info.key() == other_info.key(); + } + bool operator()(const Info &info, td::Slice hash) const { + return info.key().as_slice() == hash; + } + bool operator()(td::Slice hash, const Info &info) const { + return info.key().as_slice() == hash; + } + }; + struct Hash { + using is_transparent = void; // Pred to use + using transparent_key_equal = Eq; + size_t operator()(td::Slice hash) const { + return cell_hash_slice_hash(hash); + } + size_t operator()(const Info &info) const { + return cell_hash_slice_hash(info.key().as_slice()); + } + }; }; - td::HashMap info_; + td::HashSet info_; std::unique_ptr loader_; std::vector> to_inc_; @@ -886,13 +998,13 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { Ref do_inc(Ref cell) { auto cell_hash = cell->get_hash(); - if (auto it = info_.find(cell_hash); it != info_.end()) { - CHECK(it->second.diff_refcnt != std::numeric_limits::max()); - it->second.diff_refcnt++; - return it->second.cell; + if (auto it = info_.find(cell_hash.as_slice()); it != info_.end()) { + CHECK(it->diff_refcnt != std::numeric_limits::max()); + it->diff_refcnt++; + return it->cell; } if (auto o_info = storage_->get_info(cell_hash)) { - info_.emplace(cell_hash, Info{.db_refcnt = o_info->db_refcnt, .diff_refcnt = 1, .cell = o_info->cell}); + info_.emplace(Info{.db_refcnt = o_info->db_refcnt, .diff_refcnt = 1, .cell = o_info->cell}); return std::move(o_info->cell); } @@ -905,21 +1017,21 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { } auto res = cb.finalize(cs.is_special()); CHECK(res->get_hash() == cell_hash); - info_.emplace(cell_hash, Info{.db_refcnt = 0, .diff_refcnt = 1, .cell = res}); + info_.emplace(Info{.db_refcnt = 0, .diff_refcnt = 1, .cell = res}); return res; } void do_dec(Ref cell) { auto cell_hash = cell->get_hash(); - auto it = info_.find(cell_hash); + auto it = info_.find(cell_hash.as_slice()); if (it != info_.end()) { - CHECK(it->second.diff_refcnt != std::numeric_limits::min()); - --it->second.diff_refcnt; + CHECK(it->diff_refcnt != std::numeric_limits::min()); + --it->diff_refcnt; } else { auto info = *storage_->get_info(cell_hash); - it = info_.emplace(cell_hash, Info{.db_refcnt = info.db_refcnt, .diff_refcnt = -1, .cell = info.cell}).first; + it = info_.emplace(Info{.db_refcnt = info.db_refcnt, .diff_refcnt = -1, .cell = info.cell}).first; } - if (it->second.diff_refcnt + it->second.db_refcnt != 0) { + if (it->diff_refcnt + it->db_refcnt != 0) { return; } CellSlice cs(NoVm{}, std::move(cell)); @@ -936,7 +1048,8 @@ std::unique_ptr DynamicBagOfCellsDb::create_in_memory(td::K if (kv == nullptr) { LOG_IF(WARNING, options.verbose) << "Create empty in-memory cells database (no key value is given)"; auto storage = CellStorage::build(options, [](auto, auto, auto) { return std::make_pair(0, 0); }); - return std::make_unique(std::move(storage)); + auto meta_storage = td::make_unique(std::vector>{}); + return std::make_unique(std::move(storage), std::move(meta_storage)); } std::vector keys; @@ -962,6 +1075,9 @@ std::unique_ptr DynamicBagOfCellsDb::create_in_memory(td::K local_desc_count++; return td::Status::OK(); } + if (key.size() != 32) { + return td::Status::OK(); + } auto r_res = CellLoader::load(key, value.str(), true, pc_creator); if (r_res.is_error()) { LOG(ERROR) << r_res.error() << " at " << td::format::escaped(key); @@ -983,6 +1099,24 @@ std::unique_ptr DynamicBagOfCellsDb::create_in_memory(td::K }; auto storage = CellStorage::build(options, parallel_scan_cells); - return std::make_unique(std::move(storage)); + + std::vector> meta; + // NB: it scans 1/(2^32) of the database which is not much + kv->for_each_in_range("desc", "desd", [&meta](td::Slice key, td::Slice value) { + if (key.size() != 32) { + meta.emplace_back(key.str(), value.str()); + } + return td::Status::OK(); + }); + // this is for tests mostly. desc* keys are expected to correspond to roots + kv->for_each_in_range("meta", "metb", [&meta](td::Slice key, td::Slice value) { + if (key.size() != 32) { + meta.emplace_back(key.str(), value.str()); + } + return td::Status::OK(); + }); + auto meta_storage = td::make_unique(std::move(meta)); + + return std::make_unique(std::move(storage), std::move(meta_storage)); } } // namespace vm diff --git a/crypto/vm/db/StaticBagOfCellsDb.cpp b/crypto/vm/db/StaticBagOfCellsDb.cpp index 80dbfbf0b..c65d2624f 100644 --- a/crypto/vm/db/StaticBagOfCellsDb.cpp +++ b/crypto/vm/db/StaticBagOfCellsDb.cpp @@ -40,6 +40,9 @@ class RootCell : public Cell { struct PrivateTag {}; public: + td::Status set_data_cell(Ref &&data_cell) const override { + return cell_->set_data_cell(std::move(data_cell)); + } td::Result load_cell() const override { return cell_->load_cell(); } @@ -94,11 +97,11 @@ class DataCellCacheNoop { class DataCellCacheMutex { public: Ref store(int idx, Ref cell) { - auto lock = cells_rw_mutex_.lock_write(); + std::lock_guard lock(mutex_); return cells_.emplace(idx, std::move(cell)).first->second; } Ref load(int idx) { - auto lock = cells_rw_mutex_.lock_read(); + std::lock_guard lock(mutex_); auto it = cells_.find(idx); if (it != cells_.end()) { return it->second; @@ -106,12 +109,13 @@ class DataCellCacheMutex { return {}; } void clear() { - auto guard = cells_rw_mutex_.lock_write(); + std::lock_guard lock(mutex_); cells_.clear(); } private: - td::RwMutex cells_rw_mutex_; + std::mutex mutex_; + // NB: in case of high contention, one should use multiple buckets with per bucket mutexes td::HashMap> cells_; }; @@ -246,7 +250,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { BagOfCells::Info info_; std::mutex index_i_mutex_; - td::RwMutex index_data_rw_mutex_; + std::mutex index_mutex_; std::string index_data_; std::atomic index_i_{0}; size_t index_offset_{0}; @@ -319,7 +323,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { info_.index_offset + (td::int64)idx * info_.offset_byte_size)); offset_view = new_offset_view; } else { - guard = index_data_rw_mutex_.lock_read().move_as_ok(); + std::lock_guard guard(index_mutex_); offset_view = td::Slice(index_data_).substr((td::int64)idx * info_.offset_byte_size, info_.offset_byte_size); } @@ -432,7 +436,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { } td::uint8 tmp[8]; info_.write_offset(tmp, index_offset_); - auto guard = index_data_rw_mutex_.lock_write(); + std::lock_guard guard(index_mutex_); index_data_.append(reinterpret_cast(tmp), info_.offset_byte_size); } return td::Status::OK(); diff --git a/tddb/td/db/KeyValue.h b/tddb/td/db/KeyValue.h index 12c3a4f8d..c3f83919b 100644 --- a/tddb/td/db/KeyValue.h +++ b/tddb/td/db/KeyValue.h @@ -20,19 +20,51 @@ #include "td/utils/Status.h" #include "td/utils/Time.h" #include "td/utils/logging.h" +#include "td/utils/ThreadSafeCounter.h" #include namespace td { +struct UsageStats { + size_t get_count{}; + size_t get_found_count{}; + size_t get_not_found_count{}; + size_t set_count{}; + UsageStats operator+(const UsageStats& other) const { + return UsageStats{.get_count = get_count + other.get_count, + .get_found_count = get_found_count + other.get_found_count, + .get_not_found_count = get_not_found_count + other.get_not_found_count, + .set_count = set_count + other.set_count}; + } + UsageStats operator-(const UsageStats& other) const { + return UsageStats{.get_count = get_count - other.get_count, + .get_found_count = get_found_count - other.get_found_count, + .get_not_found_count = get_not_found_count - other.get_not_found_count, + .set_count = set_count - other.set_count}; + } + NamedStats to_named_stats() const { + NamedStats ns; + ns.stats_int["usage_get_count"] += get_count; + ns.stats_int["usage_get_found_count"] += get_found_count; + ns.stats_int["usage_get_not_found_count"] += get_not_found_count; + ns.stats_int["usage_set_count"] += set_count; + return ns; + } +}; +inline td::StringBuilder& operator<<(td::StringBuilder& sb, const UsageStats& stats) { + sb << "get: " << stats.get_count << ", +" << stats.get_found_count << ", -" << stats.get_not_found_count; + return sb; +} + class KeyValueReader { public: virtual ~KeyValueReader() = default; enum class GetStatus : int32 { Ok, NotFound }; - virtual Result get(Slice key, std::string &value) = 0; + virtual Result get(Slice key, std::string& value) = 0; virtual Result count(Slice prefix) = 0; virtual Status for_each(std::function f) { return Status::Error("for_each is not supported"); } - virtual Status for_each_in_range (Slice begin, Slice end, std::function f) { + virtual Status for_each_in_range(Slice begin, Slice end, std::function f) { return td::Status::Error("foreach_range is not supported"); } }; @@ -42,7 +74,7 @@ class PrefixedKeyValueReader : public KeyValueReader { PrefixedKeyValueReader(std::shared_ptr reader, Slice prefix) : reader_(std::move(reader)), prefix_(prefix.str()) { } - Result get(Slice key, std::string &value) override { + Result get(Slice key, std::string& value) override { return reader_->get(PSLICE() << prefix_ << key, value); } Result count(Slice prefix) override { @@ -54,14 +86,16 @@ class PrefixedKeyValueReader : public KeyValueReader { std::string prefix_; }; -class KeyValueUtils { - public: -}; - class KeyValue : public KeyValueReader { public: virtual Status set(Slice key, Slice value) = 0; virtual Status erase(Slice key) = 0; + virtual Status merge(Slice key, Slice value) { + return Status::Error("merge is not supported"); + } + virtual Status run_gc() { + return Status::OK(); + } virtual Status begin_write_batch() = 0; virtual Status commit_write_batch() = 0; @@ -80,12 +114,15 @@ class KeyValue : public KeyValueReader { virtual Status flush() { return Status::OK(); } + virtual UsageStats get_usage_stats() { + return {}; + } }; class PrefixedKeyValue : public KeyValue { public: PrefixedKeyValue(std::shared_ptr kv, Slice prefix) : kv_(std::move(kv)), prefix_(prefix.str()) { } - Result get(Slice key, std::string &value) override { + Result get(Slice key, std::string& value) override { return kv_->get(PSLICE() << prefix_ << key, value); } Result count(Slice prefix) override { diff --git a/tddb/td/db/MemoryKeyValue.cpp b/tddb/td/db/MemoryKeyValue.cpp index 080133602..7105f72b9 100644 --- a/tddb/td/db/MemoryKeyValue.cpp +++ b/tddb/td/db/MemoryKeyValue.cpp @@ -22,57 +22,99 @@ namespace td { Result MemoryKeyValue::get(Slice key, std::string &value) { - auto it = map_.find(key); - if (it == map_.end()) { + auto bucket = lock(key); + auto &map = bucket->map; + + usage_stats_.get_count++; + auto it = map.find(key); + if (it == map.end()) { + usage_stats_.get_not_found_count++; return GetStatus::NotFound; } value = it->second; + usage_stats_.get_found_count++; return GetStatus::Ok; } +std::unique_ptr MemoryKeyValue::lock(td::Slice key) { + auto bucket_id = std::hash()(std::string_view(key.data(), key.size())) % buckets_.size(); + return lock(buckets_[bucket_id]); +} + Status MemoryKeyValue::for_each(std::function f) { - for (auto &it : map_) { - TRY_STATUS(f(it.first, it.second)); + for (auto &unlocked_bucket : buckets_) { + auto bucket = lock(unlocked_bucket); + for (auto &it : bucket->map) { + TRY_STATUS(f(it.first, it.second)); + } } return Status::OK(); } Status MemoryKeyValue::for_each_in_range(Slice begin, Slice end, std::function f) { - for (auto it = map_.lower_bound(begin); it != map_.end(); it++) { - if (it->first < end) { - TRY_STATUS(f(it->first, it->second)); - } else { - break; + for (auto &unlocked_bucket : buckets_) { + auto bucket = lock(unlocked_bucket); + auto &map = bucket->map; + for (auto it = map.lower_bound(begin); it != map.end(); it++) { + if (it->first < end) { + TRY_STATUS(f(it->first, it->second)); + } else { + break; + } } } return Status::OK(); } Status MemoryKeyValue::set(Slice key, Slice value) { - map_[key.str()] = value.str(); + auto bucket = lock(key); + auto &map = bucket->map; + + usage_stats_.set_count++; + map[key.str()] = value.str(); return Status::OK(); } +Status MemoryKeyValue::merge(Slice key, Slice update) { + CHECK(merger_); + auto bucket = lock(key); + auto &map = bucket->map; + auto &value = map[key.str()]; + merger_->merge_value_and_update(value, update); + if (value.empty()) { + map.erase(key.str()); + } + return td::Status::OK(); +} Status MemoryKeyValue::erase(Slice key) { - auto it = map_.find(key); - if (it != map_.end()) { - map_.erase(it); + auto bucket = lock(key); + auto &map = bucket->map; + auto it = map.find(key); + if (it != map.end()) { + map.erase(it); } return Status::OK(); } Result MemoryKeyValue::count(Slice prefix) { size_t res = 0; - for (auto it = map_.lower_bound(prefix); it != map_.end(); it++) { - if (Slice(it->first).truncate(prefix.size()) != prefix) { - break; + for (auto &unlocked_bucket : buckets_) { + auto bucket = lock(unlocked_bucket); + auto &map = bucket->map; + for (auto it = map.lower_bound(prefix); it != map.end(); it++) { + if (Slice(it->first).truncate(prefix.size()) != prefix) { + break; + } + res++; } - res++; } return res; } std::unique_ptr MemoryKeyValue::snapshot() { auto res = std::make_unique(); - res->map_ = map_; + for (size_t i = 0; i < buckets_.size(); i++) { + auto bucket = lock(buckets_[i]); + res->buckets_[i].map = bucket->map; + } return std::move(res); } @@ -80,10 +122,10 @@ std::string MemoryKeyValue::stats() const { return PSTRING() << "MemoryKeyValueStats{" << tag("get_count", get_count_) << "}"; } Status MemoryKeyValue::begin_write_batch() { - UNREACHABLE(); + return Status::OK(); } Status MemoryKeyValue::commit_write_batch() { - UNREACHABLE(); + return Status::OK(); } Status MemoryKeyValue::abort_write_batch() { UNREACHABLE(); diff --git a/tddb/td/db/MemoryKeyValue.h b/tddb/td/db/MemoryKeyValue.h index f0b5faa08..cf896095d 100644 --- a/tddb/td/db/MemoryKeyValue.h +++ b/tddb/td/db/MemoryKeyValue.h @@ -22,12 +22,22 @@ #include namespace td { + +struct Merger { + virtual ~Merger() = default; + virtual void merge_value_and_update(std::string &value, Slice update) = 0; + virtual void merge_update_and_update(std::string &left_update, Slice right_update) = 0; +}; class MemoryKeyValue : public KeyValue { public: - Result get(Slice key, std::string &value) override; + MemoryKeyValue() = default; + MemoryKeyValue(std::shared_ptr merger) : merger_(std::move(merger)) { + } + Result get(Slice key, std::string& value) override; Status for_each(std::function f) override; Status for_each_in_range(Slice begin, Slice end, std::function f) override; Status set(Slice key, Slice value) override; + Status merge(Slice key, Slice value) override; Status erase(Slice key) override; Result count(Slice prefix) override; @@ -43,8 +53,30 @@ class MemoryKeyValue : public KeyValue { std::string stats() const override; + UsageStats get_usage_stats() override { + return usage_stats_; + } + private: - std::map> map_; + static constexpr size_t buckets_n = 64; + struct Bucket { + std::mutex mutex; + std::map> map; + }; + struct Unlock { + void operator()(Bucket* bucket) const { + bucket->mutex.unlock(); + } + }; + std::array buckets_{}; int64 get_count_{0}; + UsageStats usage_stats_{}; + std::shared_ptr merger_; + + std::unique_ptr lock(Bucket& bucket) { + bucket.mutex.lock(); + return std::unique_ptr(&bucket); + } + std::unique_ptr lock(td::Slice key); }; } // namespace td diff --git a/tddb/td/db/RocksDb.cpp b/tddb/td/db/RocksDb.cpp index f1aa64a5d..b56f3b145 100644 --- a/tddb/td/db/RocksDb.cpp +++ b/tddb/td/db/RocksDb.cpp @@ -24,10 +24,13 @@ #include "rocksdb/write_batch.h" #include "rocksdb/utilities/optimistic_transaction_db.h" #include "rocksdb/utilities/transaction.h" +#include "td/utils/misc.h" + +#include namespace td { namespace { -static Status from_rocksdb(rocksdb::Status status) { +static Status from_rocksdb(const rocksdb::Status &status) { if (status.ok()) { return Status::OK(); } @@ -56,62 +59,83 @@ RocksDb::~RocksDb() { } RocksDb RocksDb::clone() const { + if (transaction_db_) { + return RocksDb{transaction_db_, options_}; + } return RocksDb{db_, options_}; } Result RocksDb::open(std::string path, RocksDbOptions options) { - rocksdb::OptimisticTransactionDB *db; - { - rocksdb::Options db_options; + rocksdb::Options db_options; + db_options.merge_operator = options.merge_operator; + db_options.compaction_filter = options.compaction_filter; - static auto default_cache = rocksdb::NewLRUCache(1 << 30); - if (!options.no_block_cache && options.block_cache == nullptr) { - options.block_cache = default_cache; - } + static auto default_cache = rocksdb::NewLRUCache(1 << 30); + if (!options.no_block_cache && options.block_cache == nullptr) { + options.block_cache = default_cache; + } - rocksdb::BlockBasedTableOptions table_options; - if (options.no_block_cache) { - table_options.no_block_cache = true; - } else { - table_options.block_cache = options.block_cache; - } - db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); - - db_options.use_direct_reads = options.use_direct_reads; - db_options.manual_wal_flush = true; - db_options.create_if_missing = true; - db_options.max_background_compactions = 4; - db_options.max_background_flushes = 2; - db_options.bytes_per_sync = 1 << 20; - db_options.writable_file_max_buffer_size = 2 << 14; - db_options.statistics = options.statistics; - db_options.max_log_file_size = 100 << 20; - db_options.keep_log_file_num = 1; - rocksdb::OptimisticTransactionDBOptions occ_options; - occ_options.validate_policy = rocksdb::OccValidationPolicy::kValidateSerial; + rocksdb::BlockBasedTableOptions table_options; + if (options.no_block_cache) { + table_options.no_block_cache = true; + } else { + table_options.block_cache = options.block_cache; + } + db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); + + // table_options.block_align = true; + if (options.no_reads) { + db_options.memtable_factory.reset(new rocksdb::VectorRepFactory()); + db_options.allow_concurrent_memtable_write = false; + } + + db_options.wal_recovery_mode = rocksdb::WALRecoveryMode::kTolerateCorruptedTailRecords; + db_options.use_direct_reads = options.use_direct_reads; + db_options.manual_wal_flush = true; + db_options.create_if_missing = true; + db_options.max_background_compactions = 4; + db_options.max_background_flushes = 2; + db_options.bytes_per_sync = 1 << 20; + db_options.writable_file_max_buffer_size = 2 << 14; + db_options.statistics = options.statistics; + db_options.max_log_file_size = 100 << 20; + db_options.keep_log_file_num = 1; + + if (options.experimental) { + // Place your experimental options here + } + + if (options.no_transactions) { + rocksdb::DB *db{nullptr}; + TRY_STATUS(from_rocksdb(rocksdb::DB::Open(db_options, std::move(path), &db))); + return RocksDb(std::shared_ptr(db), std::move(options)); + } else { + rocksdb::OptimisticTransactionDB *db{nullptr}; rocksdb::ColumnFamilyOptions cf_options(db_options); std::vector column_families; column_families.push_back(rocksdb::ColumnFamilyDescriptor(rocksdb::kDefaultColumnFamilyName, cf_options)); std::vector handles; + rocksdb::OptimisticTransactionDBOptions occ_options; + occ_options.validate_policy = rocksdb::OccValidationPolicy::kValidateSerial; TRY_STATUS(from_rocksdb(rocksdb::OptimisticTransactionDB::Open(db_options, occ_options, std::move(path), column_families, &handles, &db))); CHECK(handles.size() == 1); // i can delete the handle since DBImpl is always holding a reference to // default column family delete handles[0]; + return RocksDb(std::shared_ptr(db), std::move(options)); } - return RocksDb(std::shared_ptr(db), std::move(options)); } std::shared_ptr RocksDb::create_statistics() { return rocksdb::CreateDBStatistics(); } -std::string RocksDb::statistics_to_string(const std::shared_ptr statistics) { +std::string RocksDb::statistics_to_string(const std::shared_ptr &statistics) { return statistics->ToString(); } -void RocksDb::reset_statistics(const std::shared_ptr statistics) { +void RocksDb::reset_statistics(const std::shared_ptr &statistics) { statistics->Reset(); } @@ -133,7 +157,9 @@ std::string RocksDb::stats() const { } Result RocksDb::get(Slice key, std::string &value) { - //LOG(ERROR) << "GET"; + if (options_.no_reads) { + return td::Status::Error("trying to read from write-only database"); + } rocksdb::Status status; if (snapshot_) { rocksdb::ReadOptions options; @@ -162,6 +188,18 @@ Status RocksDb::set(Slice key, Slice value) { } return from_rocksdb(db_->Put({}, to_rocksdb(key), to_rocksdb(value))); } +Status RocksDb::merge(Slice key, Slice value) { + if (write_batch_) { + return from_rocksdb(write_batch_->Merge(to_rocksdb(key), to_rocksdb(value))); + } + if (transaction_) { + return from_rocksdb(transaction_->Merge(to_rocksdb(key), to_rocksdb(value))); + } + return from_rocksdb(db_->Merge({}, to_rocksdb(key), to_rocksdb(value))); +} +Status RocksDb::run_gc() { + return from_rocksdb(db_->CompactRange({}, nullptr, nullptr)); +} Status RocksDb::erase(Slice key) { if (write_batch_) { @@ -174,7 +212,11 @@ Status RocksDb::erase(Slice key) { } Result RocksDb::count(Slice prefix) { + if (options_.no_reads) { + return td::Status::Error("trying to read from write-only database"); + } rocksdb::ReadOptions options; + options.auto_prefix_mode = true; options.snapshot = snapshot_.get(); std::unique_ptr iterator; if (snapshot_ || !transaction_) { @@ -197,7 +239,11 @@ Result RocksDb::count(Slice prefix) { } Status RocksDb::for_each(std::function f) { + if (options_.no_reads) { + return td::Status::Error("trying to read from write-only database"); + } rocksdb::ReadOptions options; + options.auto_prefix_mode = true; options.snapshot = snapshot_.get(); std::unique_ptr iterator; if (snapshot_ || !transaction_) { @@ -219,7 +265,11 @@ Status RocksDb::for_each(std::function f) { } Status RocksDb::for_each_in_range(Slice begin, Slice end, std::function f) { + if (options_.no_reads) { + return td::Status::Error("trying to read from write-only database"); + } rocksdb::ReadOptions options; + options.auto_prefix_mode = true; options.snapshot = snapshot_.get(); std::unique_ptr iterator; if (snapshot_ || !transaction_) { @@ -252,9 +302,10 @@ Status RocksDb::begin_write_batch() { Status RocksDb::begin_transaction() { CHECK(!write_batch_); + CHECK(transaction_db_); rocksdb::WriteOptions options; options.sync = true; - transaction_.reset(db_->BeginTransaction(options, {})); + transaction_.reset(transaction_db_->BeginTransaction(options, {})); return Status::OK(); } @@ -307,7 +358,11 @@ Status RocksDb::end_snapshot() { } RocksDb::RocksDb(std::shared_ptr db, RocksDbOptions options) - : db_(std::move(db)), options_(options) { + : transaction_db_{db}, db_(std::move(db)), options_(std::move(options)) { +} + +RocksDb::RocksDb(std::shared_ptr db, RocksDbOptions options) + : db_(std::move(db)), options_(std::move(options)) { } void RocksDbSnapshotStatistics::begin_snapshot(const rocksdb::Snapshot *snapshot) { diff --git a/tddb/td/db/RocksDb.h b/tddb/td/db/RocksDb.h index 499a33281..d24a20dd7 100644 --- a/tddb/td/db/RocksDb.h +++ b/tddb/td/db/RocksDb.h @@ -36,12 +36,16 @@ #include namespace rocksdb { +class DB; +class Comparator; class Cache; class OptimisticTransactionDB; class Transaction; class WriteBatch; class Snapshot; class Statistics; +class MergeOperator; +class CompactionFilter; } // namespace rocksdb namespace td { @@ -61,6 +65,14 @@ struct RocksDbOptions { std::shared_ptr statistics = nullptr; std::shared_ptr block_cache; // Default - one 1G cache for all RocksDb std::shared_ptr snapshot_statistics = nullptr; + + std::shared_ptr merge_operator = nullptr; + const rocksdb::CompactionFilter *compaction_filter = nullptr; + + bool experimental = false; + bool no_reads = false; + bool no_transactions = false; + bool use_direct_reads = false; bool no_block_cache = false; }; @@ -73,10 +85,12 @@ class RocksDb : public KeyValue { Result get(Slice key, std::string &value) override; Status set(Slice key, Slice value) override; + Status merge(Slice key, Slice value) override; Status erase(Slice key) override; + Status run_gc() override; Result count(Slice prefix) override; Status for_each(std::function f) override; - Status for_each_in_range (Slice begin, Slice end, std::function f) override; + Status for_each_in_range(Slice begin, Slice end, std::function f) override; Status begin_write_batch() override; Status commit_write_batch() override; @@ -94,8 +108,8 @@ class RocksDb : public KeyValue { std::string stats() const override; static std::shared_ptr create_statistics(); - static std::string statistics_to_string(const std::shared_ptr statistics); - static void reset_statistics(const std::shared_ptr statistics); + static std::string statistics_to_string(const std::shared_ptr &statistics); + static void reset_statistics(const std::shared_ptr &statistics); static std::shared_ptr create_cache(size_t capacity); @@ -103,12 +117,13 @@ class RocksDb : public KeyValue { RocksDb &operator=(RocksDb &&); ~RocksDb(); - std::shared_ptr raw_db() const { + std::shared_ptr raw_db() const { return db_; }; private: - std::shared_ptr db_; + std::shared_ptr transaction_db_; + std::shared_ptr db_; RocksDbOptions options_; std::unique_ptr transaction_; @@ -123,5 +138,6 @@ class RocksDb : public KeyValue { std::unique_ptr snapshot_; explicit RocksDb(std::shared_ptr db, RocksDbOptions options); + explicit RocksDb(std::shared_ptr db, RocksDbOptions options); }; } // namespace td diff --git a/tdutils/td/utils/MpmcQueue.h b/tdutils/td/utils/MpmcQueue.h index e6504e358..1a5f8fa36 100644 --- a/tdutils/td/utils/MpmcQueue.h +++ b/tdutils/td/utils/MpmcQueue.h @@ -414,7 +414,9 @@ class MpmcQueue { while (true) { auto node = hazard_pointers_.protect(thread_id, 0, read_pos_); auto &block = node->block; - if (block.write_pos <= block.read_pos && node->next.load(std::memory_order_relaxed) == nullptr) { + auto read_pos = block.read_pos.load(); + auto write_pos = block.write_pos.load(); + if (write_pos <= read_pos && node->next.load(std::memory_order_relaxed) == nullptr) { return false; } auto pos = block.read_pos++; diff --git a/tdutils/td/utils/Status.h b/tdutils/td/utils/Status.h index cff808143..f75de466a 100644 --- a/tdutils/td/utils/Status.h +++ b/tdutils/td/utils/Status.h @@ -619,6 +619,13 @@ inline Result::Result(Status &&status) : status_(std::move(status)) { inline StringBuilder &operator<<(StringBuilder &string_builder, const Status &status) { return status.print(string_builder); } +template +StringBuilder &operator<<(StringBuilder &sb, const Result &result) { + if (result.is_ok()) { + return sb << "Ok{" << result.ok() << "}"; + } + return sb << result.error(); +} namespace detail { diff --git a/tdutils/td/utils/ThreadSafeCounter.h b/tdutils/td/utils/ThreadSafeCounter.h index aa976b2fb..46dc16bf7 100644 --- a/tdutils/td/utils/ThreadSafeCounter.h +++ b/tdutils/td/utils/ThreadSafeCounter.h @@ -19,6 +19,7 @@ #pragma once +#include "port/thread.h" #include "td/utils/common.h" #include "td/utils/Slice.h" #include "td/utils/StringBuilder.h" @@ -26,6 +27,7 @@ #include #include +#include #include namespace td { @@ -69,6 +71,50 @@ class ThreadSafeCounter { ThreadSafeMultiCounter<1> counter_; }; +struct NamedStats { + std::map stats_int; + std::map stats_str; + + NamedStats with_suffix(const std::string &suffix) const { + NamedStats res; + for (auto &p : stats_int) { + res.stats_int[p.first + suffix] = p.second; + } + for (auto &p : stats_str) { + res.stats_str[p.first + suffix] = p.second; + } + return res; + } + NamedStats with_prefix(const std::string &prefix) const { + NamedStats res; + for (auto &p : stats_int) { + res.stats_int[prefix + p.first] = p.second; + } + for (auto &p : stats_str) { + res.stats_str[prefix + p.first] = p.second; + } + return res; + } + void apply_diff(const NamedStats &other) { + for (auto &p : other.stats_int) { + stats_int[p.first] += p.second; + } + for (auto &p : other.stats_str) { + stats_str[p.first] = p.second; + } + } + void subtract_diff(const NamedStats &other) { + for (auto &p : other.stats_int) { + stats_int[p.first] -= p.second; + } + } + NamedStats combine_with(const NamedStats &other) const { + NamedStats res = *this; + res.apply_diff(other); + return res; + } +}; + class NamedThreadSafeCounter { static constexpr int N = 128; using Counter = ThreadSafeMultiCounter; @@ -79,6 +125,9 @@ class NamedThreadSafeCounter { CounterRef() = default; CounterRef(size_t index, Counter *counter) : index_(index), counter_(counter) { } + void inc() { + add(1); + } void add(int64 diff) { counter_->add(index_, diff); } @@ -119,6 +168,11 @@ class NamedThreadSafeCounter { f(names_[i], counter_.sum(i)); } } + NamedStats get_stats() const { + NamedStats res; + for_each([&](Slice name, int64 cnt) { res.stats_int.emplace(name.str(), cnt); }); + return res; + } void clear() { std::unique_lock guard(mutex_); @@ -181,11 +235,11 @@ struct NamedPerfCounter { } // namespace td -#define TD_PERF_COUNTER(name) \ +#define TD_PERF_COUNTER(name) \ static auto perf_##name = td::NamedPerfCounter::get_default().get_counter(td::Slice(#name)); \ auto scoped_perf_##name = td::NamedPerfCounter::ScopedPerfCounterRef{.perf_counter = perf_##name}; -#define TD_PERF_COUNTER_SINCE(name, since) \ +#define TD_PERF_COUNTER_SINCE(name, since) \ static auto perf_##name = td::NamedPerfCounter::get_default().get_counter(td::Slice(#name)); \ - auto scoped_perf_##name = \ + auto scoped_perf_##name = \ td::NamedPerfCounter::ScopedPerfCounterRef{.perf_counter = perf_##name, .started_at_ticks = since}; diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 2ea04e183..cbcc3ab1f 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1414,6 +1414,9 @@ td::Status ValidatorEngine::load_global_config() { if (zero_state.root_hash.is_zero() || zero_state.file_hash.is_zero()) { return td::Status::Error(ton::ErrorCode::error, "[validator] section contains incomplete [zero_state]"); } + if (celldb_in_memory_ && celldb_v2_) { + return td::Status::Error(ton::ErrorCode::error, "at most one of --celldb-in-memory --celldb-v2 could be used"); + } ton::BlockIdExt init_block; if (!conf.validator_->init_block_) { @@ -1461,11 +1464,12 @@ td::Status ValidatorEngine::load_global_config() { if (!session_logs_file_.empty()) { validator_options_.write().set_session_logs_file(session_logs_file_); } - if (celldb_in_memory_) { + if (celldb_in_memory_ || celldb_v2_) { celldb_compress_depth_ = 0; } validator_options_.write().set_celldb_compress_depth(celldb_compress_depth_); validator_options_.write().set_celldb_in_memory(celldb_in_memory_); + validator_options_.write().set_celldb_v2(celldb_v2_); validator_options_.write().set_max_open_archive_files(max_open_archive_files_); validator_options_.write().set_archive_preload_period(archive_preload_period_); validator_options_.write().set_disable_rocksdb_stats(disable_rocksdb_stats_); @@ -4526,6 +4530,12 @@ int main(int argc, char *argv[]) { [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_in_memory, true); }); }); + p.add_option( + '\0', "celldb-v2", + "use new version off celldb", + [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_v2, true); }); + }); p.add_checked_option( '\0', "catchain-max-block-delay", "delay before creating a new catchain block, in seconds (default: 0.4)", [&](td::Slice s) -> td::Status { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index e0dc91f13..5a1db7f3f 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -218,6 +218,7 @@ class ValidatorEngine : public td::actor::Actor { bool celldb_direct_io_ = false; bool celldb_preload_all_ = false; bool celldb_in_memory_ = false; + bool celldb_v2_ = false; td::optional catchain_max_block_delay_, catchain_max_block_delay_slow_; bool read_config_ = false; bool started_keyring_ = false; @@ -311,6 +312,9 @@ class ValidatorEngine : public td::actor::Actor { void set_celldb_in_memory(bool value) { celldb_in_memory_ = value; } + void set_celldb_v2(bool value) { + celldb_v2_ = value; + } void set_catchain_max_block_delay(double value) { catchain_max_block_delay_ = value; } diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index e86a373d1..90c659cc4 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -28,6 +28,9 @@ #include "ton/ton-io.hpp" #include "common/delay.h" +#include +#include + namespace ton { namespace validator { @@ -73,6 +76,101 @@ CellDbIn::CellDbIn(td::actor::ActorId root_db, td::actor::ActorId td::Slice { + return td::Slice(value.data(), value.size()); + } + bool FullMergeV2(const MergeOperationInput& merge_in, MergeOperationOutput* merge_out) const override { + CHECK(merge_in.existing_value); + auto& value = *merge_in.existing_value; + CHECK(merge_in.operand_list.size() >= 1); + td::Slice diff; + std::string diff_buf; + if (merge_in.operand_list.size() == 1) { + diff = to_td(merge_in.operand_list[0]); + } else { + diff_buf = merge_in.operand_list[0].ToString(); + for (size_t i = 1; i < merge_in.operand_list.size(); ++i) { + vm::CellStorer::merge_refcnt_diffs(diff_buf, to_td(merge_in.operand_list[i])); + } + diff = diff_buf; + } + + merge_out->new_value = value.ToString(); + vm::CellStorer::merge_value_and_refcnt_diff(merge_out->new_value, diff); + return true; + } + bool PartialMerge(const rocksdb::Slice& /*key*/, const rocksdb::Slice& left, const rocksdb::Slice& right, + std::string* new_value, rocksdb::Logger* logger) const override { + *new_value = left.ToString(); + vm::CellStorer::merge_refcnt_diffs(*new_value, to_td(right)); + return true; + } +}; + +void CellDbIn::validate_meta() { + LOG(INFO) << "Validating metadata\n"; + size_t max_meta_keys_loaded = opts_->get_celldb_in_memory() ? std::numeric_limits::max() : 10000; + auto meta = boc_->meta_get_all(max_meta_keys_loaded).move_as_ok(); + bool partial_check = meta.size() == max_meta_keys_loaded; + if (partial_check) { + LOG(ERROR) << "Too much metadata in the database, do only partial check"; + } + size_t missing_roots = 0; + size_t unknown_roots = 0; + std::set root_hashes; + for (auto [k, v] : meta) { + if (k == "desczero") { + continue; + } + auto obj = fetch_tl_object(td::BufferSlice{v}, true); + obj.ensure(); + auto entry = DbEntry{obj.move_as_ok()}; + root_hashes.insert(vm::CellHash::from_slice(entry.root_hash.as_slice())); + auto cell = boc_->load_cell(entry.root_hash.as_slice()); + missing_roots += cell.is_error(); + LOG_IF(ERROR, cell.is_error()) << "Cannot load root from meta: " << entry.block_id.to_str() << " " << cell.error(); + } + + // load_known_roots is only supported by InMemory database, so it is ok to check all known roots here + auto known_roots = boc_->load_known_roots().move_as_ok(); + for (auto& root : known_roots) { + block::gen::ShardStateUnsplit::Record info; + block::gen::OutMsgQueueInfo::Record qinfo; + block::ShardId shard; + if (!(tlb::unpack_cell(root, info) && shard.deserialize(info.shard_id.write()) && + tlb::unpack_cell(info.out_msg_queue_info, qinfo))) { + LOG(FATAL) << "cannot create ShardDescr from a root in celldb"; + } + if (!partial_check && !root_hashes.contains(root->get_hash())) { + unknown_roots++; + LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no; + constexpr bool delete_unknown_roots = false; + if (delete_unknown_roots) { + vm::CellStorer stor{*cell_db_}; + cell_db_->begin_write_batch().ensure(); + boc_->dec(root); + boc_->commit(stor).ensure(); + cell_db_->commit_write_batch().ensure(); + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + } + LOG(ERROR) << "Unknown root" << ShardIdFull(shard).to_str() << ":" << info.seq_no << " REMOVED"; + } + } + } + + LOG_IF(ERROR, missing_roots != 0) << "Missing root hashes: " << missing_roots; + LOG_IF(ERROR, unknown_roots != 0) << "Unknown roots: " << unknown_roots; + + LOG_IF(FATAL, missing_roots != 0) << "Missing root hashes: " << missing_roots; + LOG_IF(FATAL, unknown_roots != 0) << "Unknown roots: " << unknown_roots; + LOG(INFO) << "Validating metadata: OK\n"; +} + void CellDbIn::start_up() { on_load_callback_ = [actor = std::make_shared>( td::actor::create_actor("celldbmigration", actor_id(this))), @@ -96,44 +194,101 @@ void CellDbIn::start_up() { db_options.snapshot_statistics = snapshot_statistics_; } db_options.statistics = statistics_; - if (opts_->get_celldb_cache_size()) { - db_options.block_cache = td::RocksDb::create_cache(opts_->get_celldb_cache_size().value()); - LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(opts_->get_celldb_cache_size().value()); + auto o_celldb_cache_size = opts_->get_celldb_cache_size(); + + std::optional boc_in_memory_options; + std::optional boc_v1_options; + std::optional boc_v2_options; + + if (opts_->get_celldb_v2()) { + boc_v2_options = vm::DynamicBagOfCellsDb::CreateV2Options{ + .extra_threads = std::clamp(std::thread::hardware_concurrency() / 2, 1u, 8u), + .executor = {}, + .cache_ttl_max = 2000, + .cache_size_max = 1000000}; + size_t min_rocksdb_cache = std::max(size_t{1} << 30, boc_v2_options->cache_size_max * 5000); + if (!o_celldb_cache_size || o_celldb_cache_size.value() < min_rocksdb_cache) { + LOG(WARNING) << "Increase CellDb block cache size to " << td::format::as_size(min_rocksdb_cache) << " from " + << td::format::as_size(o_celldb_cache_size.value()); + o_celldb_cache_size = min_rocksdb_cache; + } + LOG(WARNING) << "Using V2 DynamicBagOfCells with options " << *boc_v2_options; + } else if (opts_->get_celldb_in_memory()) { + // default options + boc_in_memory_options = vm::DynamicBagOfCellsDb::CreateInMemoryOptions{ + .extra_threads = std::thread::hardware_concurrency(), + .verbose = true, + .use_arena = false, + .use_less_memory_during_creation = true, + }; + LOG(WARNING) << "Using InMemory DynamicBagOfCells with options " << *boc_v2_options; + } else { + boc_v1_options = vm::DynamicBagOfCellsDb::CreateV1Options{}; + LOG(WARNING) << "Using V1 DynamicBagOfCells with options " << *boc_v1_options; + } + + if (o_celldb_cache_size) { + db_options.block_cache = td::RocksDb::create_cache(o_celldb_cache_size.value()); + LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(o_celldb_cache_size.value()); } db_options.use_direct_reads = opts_->get_celldb_direct_io(); + // NB: from now on we MUST use this merge operator + // Only V2 and InMemory BoC actually use them, but it still should be kept for V1, + // to handle updates written by V2 or InMemory BoCs + db_options.merge_operator = std::make_shared(); + if (opts_->get_celldb_in_memory()) { td::RocksDbOptions read_db_options; read_db_options.use_direct_reads = true; read_db_options.no_block_cache = true; read_db_options.block_cache = {}; + read_db_options.merge_operator = std::make_shared(); LOG(WARNING) << "Loading all cells in memory (because of --celldb-in-memory)"; td::Timer timer; auto read_cell_db = std::make_shared(td::RocksDb::open(path_, std::move(read_db_options)).move_as_ok()); - boc_ = vm::DynamicBagOfCellsDb::create_in_memory(read_cell_db.get(), {}); + boc_ = vm::DynamicBagOfCellsDb::create_in_memory(read_cell_db.get(), *boc_in_memory_options); in_memory_load_time_ = timer.elapsed(); - td::actor::send_closure(parent_, &CellDb::set_in_memory_boc, boc_); + + // no reads will be allowed from rocksdb, only writes + db_options.no_reads = true; } auto rocks_db = std::make_shared(td::RocksDb::open(path_, std::move(db_options)).move_as_ok()); rocks_db_ = rocks_db->raw_db(); cell_db_ = std::move(rocks_db); if (!opts_->get_celldb_in_memory()) { - boc_ = vm::DynamicBagOfCellsDb::create(); + if (opts_->get_celldb_v2()) { + boc_ = vm::DynamicBagOfCellsDb::create_v2(*boc_v2_options); + } else { + boc_ = vm::DynamicBagOfCellsDb::create(*boc_v1_options); + } boc_->set_celldb_compress_depth(opts_->get_celldb_compress_depth()); boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); - td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); } + validate_meta(); + alarm_timestamp() = td::Timestamp::in(10.0); auto empty = get_empty_key_hash(); if (get_block(empty).is_error()) { DbEntry e{get_empty_key(), empty, empty, RootHash::zero()}; + vm::CellStorer stor{*cell_db_}; cell_db_->begin_write_batch().ensure(); set_block(empty, std::move(e)); + boc_->commit(stor); cell_db_->commit_write_batch().ensure(); + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + } + } + + if (opts_->get_celldb_v2() || opts_->get_celldb_in_memory()) { + send_closure(parent_, &CellDb::set_thread_safe_boc, boc_); + } else { + send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); } if (opts_->get_celldb_preload_all()) { @@ -161,7 +316,7 @@ void CellDbIn::start_up() { { std::string key = "stats.last_deleted_mc_seqno", value; - auto R = cell_db_->get(td::as_slice(key), value); + auto R = boc_->meta_get(td::as_slice(key), value); R.ensure(); if (R.ok() == td::KeyValue::GetStatus::Ok) { auto r_value = td::to_integer_safe(value); @@ -240,10 +395,10 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi td::Timer timer_write; vm::CellStorer stor{*cell_db_}; cell_db_->begin_write_batch().ensure(); - boc_->commit(stor).ensure(); set_block(get_empty_key_hash(), std::move(E)); set_block(D.prev, std::move(P)); set_block(key_hash, std::move(D)); + boc_->commit(stor).ensure(); cell_db_->commit_write_batch().ensure(); timer_write.pause(); @@ -266,11 +421,10 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi void CellDbIn::get_cell_db_reader(td::Promise> promise) { if (db_busy_) { - action_queue_.push( - [self = this, promise = std::move(promise)](td::Result R) mutable { - R.ensure(); - self->get_cell_db_reader(std::move(promise)); - }); + action_queue_.push([self = this, promise = std::move(promise)](td::Result R) mutable { + R.ensure(); + self->get_cell_db_reader(std::move(promise)); + }); return; } promise.set_result(boc_->get_cell_db_reader()); @@ -440,9 +594,16 @@ void CellDbIn::gc_cont2(BlockHandle handle) { timer_get_keys.reset(); td::PerfWarningTimer timer_boc{"gccell_boc", 0.05}; - auto cell = boc_->load_cell(F.root_hash.as_slice()).move_as_ok(); + auto r_cell = boc_->load_cell(F.root_hash.as_slice()); + td::Ref cell; + if (r_cell.is_ok()) { + cell = r_cell.move_as_ok(); + boc_->dec(cell); + LOG(ERROR) << "GC of " << handle->id().to_str(); + } else { + LOG(ERROR) << "GC of UNKNOWN root: " << handle->id().to_str(); + } - boc_->dec(cell); db_busy_ = true; boc_->prepare_commit_async( async_executor, [this, SelfId = actor_id(this), timer_boc = std::move(timer_boc), F = std::move(F), key_hash, @@ -458,17 +619,19 @@ void CellDbIn::gc_cont2(BlockHandle handle) { td::PerfWarningTimer timer_write_batch{"gccell_write_batch", 0.05}; cell_db_->begin_write_batch().ensure(); - boc_->commit(stor).ensure(); - cell_db_->erase(get_key(key_hash)).ensure(); + boc_->meta_erase(get_key(key_hash)).ensure(); set_block(F.prev, std::move(P)); set_block(F.next, std::move(N)); if (handle->id().is_masterchain()) { last_deleted_mc_state_ = handle->id().seqno(); std::string key = "stats.last_deleted_mc_seqno", value = td::to_string(last_deleted_mc_state_); - cell_db_->set(td::as_slice(key), td::as_slice(value)); + boc_->meta_set(td::as_slice(key), td::as_slice(value)); } + + boc_->commit(stor).ensure(); cell_db_->commit_write_batch().ensure(); + alarm_timestamp() = td::Timestamp::now(); timer_write_batch.reset(); @@ -530,7 +693,7 @@ CellDbIn::KeyHash CellDbIn::get_empty_key_hash() { td::Result CellDbIn::get_block(KeyHash key_hash) { const auto key = get_key(key_hash); std::string value; - auto R = cell_db_->get(td::as_slice(key), value); + auto R = boc_->meta_get(td::as_slice(key), value); R.ensure(); auto S = R.move_as_ok(); if (S == td::KeyValue::GetStatus::NotFound) { @@ -543,7 +706,7 @@ td::Result CellDbIn::get_block(KeyHash key_hash) { void CellDbIn::set_block(KeyHash key_hash, DbEntry e) { const auto key = get_key(key_hash); - cell_db_->set(td::as_slice(key), e.release()).ensure(); + boc_->meta_set(td::as_slice(key), e.release()); } void CellDbIn::migrate_cell(td::Bits256 hash) { @@ -631,12 +794,14 @@ void CellDb::alarm() { } void CellDb::load_cell(RootHash hash, td::Promise> promise) { - if (in_memory_boc_) { - auto result = in_memory_boc_->load_root_thread_safe(hash.as_slice()); + if (thread_safe_boc_) { + auto result = thread_safe_boc_->load_root_thread_safe(hash.as_slice()); if (result.is_ok()) { return async_apply("load_cell_result", std::move(promise), std::move(result)); } else { LOG(ERROR) << "load_root_thread_safe failed - this is suspicious"; + send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise)); + return; } } if (!started_) { @@ -710,6 +875,13 @@ std::vector> CellDbIn::CellDbStatistics::pre for (auto& [key, value] : boc_stats_->custom_stats) { stats.emplace_back(key, value); } + + for (auto& [key, value] : boc_stats_->named_stats.stats_str) { + stats.emplace_back(key, value); + } + for (auto& [key, value] : boc_stats_->named_stats.stats_int) { + stats.emplace_back(key, td::to_string(value)); + } } return stats; } diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index 5639b9748..1e1ccddab 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -74,6 +74,7 @@ class CellDbIn : public CellDbBase { CellDbIn(td::actor::ActorId root_db, td::actor::ActorId parent, std::string path, td::Ref opts); + void validate_meta(); void start_up() override; void alarm() override; @@ -195,13 +196,13 @@ class CellDb : public CellDbBase { started_ = true; boc_->set_loader(std::make_unique(std::move(snapshot), on_load_callback_)).ensure(); } - void set_in_memory_boc(std::shared_ptr in_memory_boc) { - CHECK(opts_->get_celldb_in_memory()); + void set_thread_safe_boc(std::shared_ptr thread_safe_boc) { + CHECK(opts_->get_celldb_in_memory() || opts_->get_celldb_v2()); if (!started_) { alarm(); } started_ = true; - in_memory_boc_ = std::move(in_memory_boc); + thread_safe_boc_ = std::move(thread_safe_boc); } void get_cell_db_reader(td::Promise> promise); @@ -219,7 +220,7 @@ class CellDb : public CellDbBase { td::actor::ActorOwn cell_db_; std::unique_ptr boc_; - std::shared_ptr in_memory_boc_; + std::shared_ptr thread_safe_boc_; bool started_ = false; std::vector> prepared_stats_{{"started", "false"}}; diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index ace6b1066..45b8d7ec2 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -139,6 +139,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool get_celldb_in_memory() const override { return celldb_in_memory_; } + bool get_celldb_v2() const override { + return celldb_v2_; + } td::optional get_catchain_max_block_delay() const override { return catchain_max_block_delay_; } @@ -237,6 +240,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_celldb_in_memory(bool value) override { celldb_in_memory_ = value; } + void set_celldb_v2(bool value) override { + celldb_v2_ = value; + } void set_catchain_max_block_delay(double value) override { catchain_max_block_delay_ = value; } @@ -304,6 +310,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool celldb_direct_io_ = false; bool celldb_preload_all_ = false; bool celldb_in_memory_ = false; + bool celldb_v2_ = false; td::optional catchain_max_block_delay_, catchain_max_block_delay_slow_; bool state_serializer_enabled_ = true; td::Ref collator_options_{true}; diff --git a/validator/validator.h b/validator/validator.h index eecb30c07..9e1dba7db 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -104,6 +104,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual std::string get_session_logs_file() const = 0; virtual td::uint32 get_celldb_compress_depth() const = 0; virtual bool get_celldb_in_memory() const = 0; + virtual bool get_celldb_v2() const = 0; virtual size_t get_max_open_archive_files() const = 0; virtual double get_archive_preload_period() const = 0; virtual bool get_disable_rocksdb_stats() const = 0; @@ -144,6 +145,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_celldb_direct_io(bool value) = 0; virtual void set_celldb_preload_all(bool value) = 0; virtual void set_celldb_in_memory(bool value) = 0; + virtual void set_celldb_v2(bool value) = 0; virtual void set_catchain_max_block_delay(double value) = 0; virtual void set_catchain_max_block_delay_slow(double value) = 0; virtual void set_state_serializer_enabled(bool value) = 0; From 81d24492fa732c2a992224adc44ae059067eadcb Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Sun, 30 Mar 2025 09:12:42 -0400 Subject: [PATCH 185/388] Optimize layout of and simplify vm::DataCell (#1570) * Add missing includes to td/utils/ThreadSafeCounter.h This allows the file to be included first in other headers. * Optimize layout of and simplify vm::DataCell This is a complete rewrite of vm::DataCell initially undertaken to make optimizer life easier when working with DataCell. In particular, the commit makes references to child cells reside at the known offset from `this` of the data cell, removes a bunch of unneeded complexity from the class definition itself (like `Info` struct), and uses placement new to allocate class with a trailer instead of relying on template trickery from `CellWithStorage.h`. Additionally, DataCell verification is rewritten from scratch and collected into a nice `CellChecker` class to make it easier to follow through and reason about both for humans and compilers. All in all, this is 3.5% time speedup on the verification benchmark (which is somewhat hard to attribute to a particular change) and average 5.5% memory consumption improvement (most likely coming from better packing of the fields). * Store variable-sized data in `char trailer_[]` in vm::PrunnedCell This is similar to how it is done in DataCell. * Remove now unused vm/cells/CellWithStorage.h * Make DynamicBagOfCellsDb::validate_bucket_a agnostic to use_arena option This prevents test failures if we actually start destructing (but not freeing) arena-allocated cells. --- crypto/CMakeLists.txt | 1 - crypto/vm/cells/CellBuilder.cpp | 5 +- crypto/vm/cells/CellWithStorage.h | 105 ----- crypto/vm/cells/DataCell.cpp | 578 ++++++++++++++------------ crypto/vm/cells/DataCell.h | 248 +++++------ crypto/vm/cells/PrunnedCell.h | 50 ++- crypto/vm/db/InMemoryBagOfCellsDb.cpp | 20 +- crypto/vm/db/StaticBagOfCellsDb.cpp | 1 - tdutils/td/utils/ThreadSafeCounter.h | 4 +- 9 files changed, 463 insertions(+), 549 deletions(-) delete mode 100644 crypto/vm/cells/CellWithStorage.h diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 2988f501f..defe2b58e 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -70,7 +70,6 @@ set(TON_CRYPTO_CORE_SOURCE vm/cells/CellString.h vm/cells/CellTraits.h vm/cells/CellUsageTree.h - vm/cells/CellWithStorage.h vm/cells/DataCell.h vm/cells/ExtCell.h vm/cells/LevelMask.h diff --git a/crypto/vm/cells/CellBuilder.cpp b/crypto/vm/cells/CellBuilder.cpp index 772b5f6b7..a9ad449e1 100644 --- a/crypto/vm/cells/CellBuilder.cpp +++ b/crypto/vm/cells/CellBuilder.cpp @@ -50,7 +50,7 @@ Ref CellBuilder::finalize_copy(bool special) const { if (vm_state_interface) { vm_state_interface->register_cell_create(); } - auto res = DataCell::create(data, size(), td::span(refs.data(), size_refs()), special); + auto res = DataCell::create(td::Slice{data, Cell::max_bytes}, size(), td::span(refs.data(), size_refs()), special); if (res.is_error()) { LOG(DEBUG) << res.error(); throw CellWriteError{}; @@ -68,7 +68,8 @@ Ref CellBuilder::finalize_copy(bool special) const { } td::Result> CellBuilder::finalize_novm_nothrow(bool special) { - auto res = DataCell::create(data, size(), td::mutable_span(refs.data(), size_refs()), special); + auto res = + DataCell::create(td::Slice{data, Cell::max_bytes}, size(), td::mutable_span(refs.data(), size_refs()), special); bits = refs_cnt = 0; return res; } diff --git a/crypto/vm/cells/CellWithStorage.h b/crypto/vm/cells/CellWithStorage.h deleted file mode 100644 index f3fedfc31..000000000 --- a/crypto/vm/cells/CellWithStorage.h +++ /dev/null @@ -1,105 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once - -namespace vm { -namespace detail { - -template -struct DefaultAllocator { - template - std::unique_ptr make_unique(ArgsT&&... args) { - return std::make_unique(std::forward(args)...); - } -}; - -template -class CellWithArrayStorage : public CellT { - public: - template - CellWithArrayStorage(ArgsT&&... args) : CellT(std::forward(args)...) { - } - ~CellWithArrayStorage() { - CellT::destroy_storage(get_storage()); - } - template - static auto create(Allocator allocator, size_t storage_size, ArgsT&&... args) { - static_assert(CellT::max_storage_size <= 40 * 8, ""); - //size = 128 + 32 + 8; - auto size = (storage_size + 7) / 8; -#define CASE(size) \ - case (size): \ - return allocator. template make_unique>(std::forward(args)...); -#define CASE2(offset) CASE(offset) CASE(offset + 1) -#define CASE8(offset) CASE2(offset) CASE2(offset + 2) CASE2(offset + 4) CASE2(offset + 6) -#define CASE32(offset) CASE8(offset) CASE8(offset + 8) CASE8(offset + 16) CASE8(offset + 24) - switch (size) { CASE32(0) CASE8(32) } -#undef CASE -#undef CASE2 -#undef CASE8 -#undef CASE32 - LOG(FATAL) << "TOO BIG " << storage_size; - UNREACHABLE(); - } - template - static std::unique_ptr create(size_t storage_size, ArgsT&&... args) { - return create(DefaultAllocator{}, storage_size, std::forward(args)...); - } - - private: - alignas(alignof(void*)) char storage_[Size]; - - const char* get_storage() const final { - return storage_; - } - char* get_storage() final { - return storage_; - } -}; - -template -class CellWithUniquePtrStorage : public CellT { - public: - template - CellWithUniquePtrStorage(size_t storage_size, ArgsT&&... args) - : CellT(std::forward(args)...), storage_(std::make_unique(storage_size)) { - } - ~CellWithUniquePtrStorage() { - CellT::destroy_storage(get_storage()); - } - - template - static std::unique_ptr create(size_t storage_size, ArgsT&&... args) { - return std::make_unique(storage_size, std::forward(args)...); - } - - private: - std::unique_ptr storage_; - - const char* get_storage() const final { - CHECK(storage_); - return storage_.get(); - } - char* get_storage() final { - CHECK(storage_); - return storage_.get(); - } -}; -} // namespace detail -} // namespace vm diff --git a/crypto/vm/cells/DataCell.cpp b/crypto/vm/cells/DataCell.cpp index 5134123d8..14da8e85a 100644 --- a/crypto/vm/cells/DataCell.cpp +++ b/crypto/vm/cells/DataCell.cpp @@ -16,335 +16,368 @@ Copyright 2017-2020 Telegram Systems LLP */ -#include "vm/cells/DataCell.h" #include "openssl/digest.hpp" - -#include "td/utils/ScopeGuard.h" - -#include "vm/cells/CellWithStorage.h" +#include "vm/cells/DataCell.h" namespace vm { -thread_local bool DataCell::use_arena = false; namespace { -template -struct ArenaAllocator { - template - std::unique_ptr make_unique(ArgsT&&... args) { - auto* ptr = fast_alloc(sizeof(T)); - T* obj = new (ptr) T(std::forward(args)...); - return std::unique_ptr(obj); - } - private: - td::MutableSlice alloc_batch() { - size_t batch_size = 1 << 20; - auto batch = std::make_unique(batch_size); - return td::MutableSlice(batch.release(), batch_size); +class CellChecker { + public: + CellChecker(bool is_special, td::Slice data, int bit_length, td::Span> refs) + : is_special_(is_special) + , refs_(refs) + , refs_cnt_(static_cast(refs.size())) + , data_(data) + , bit_length_(bit_length) { } - char* fast_alloc(size_t size) { - thread_local td::MutableSlice batch; - auto aligned_size = (size + 7) / 8 * 8; - if (batch.size() < size) { - batch = alloc_batch(); + + td::Status check_and_compute_level_info() { + // First, we figure out what is the type of the cell. + type_ = Cell::SpecialType::Ordinary; + + if (is_special_) { + if (bit_length_ < 8) { + return td::Status::Error("Not enough data for a special cell"); + } + + type_ = static_cast(read_byte(0)); + if (type_ == Cell::SpecialType::Ordinary) { + return td::Status::Error("Invalid special cell type"); + } } - auto res = batch.begin(); - batch.remove_prefix(aligned_size); - return res; - } -}; -} // namespace -std::unique_ptr DataCell::create_empty_data_cell(Info info) { - if (use_arena) { - ArenaAllocator allocator; - auto res = detail::CellWithArrayStorage::create(allocator, info.get_storage_size(), info); - // this is dangerous - Ref(res.get()).release(); - return res; + + // Next, we populate everything except for virtualization and hashes. `check_*` functions also + // perform type-specific checks. + switch (type_) { + case Cell::SpecialType::Ordinary: + TRY_STATUS(check_ordinary_cell()); + break; + case Cell::SpecialType::PrunnedBranch: + TRY_STATUS(check_pruned_branch()); + break; + case Cell::SpecialType::Library: + TRY_STATUS(check_library()); + break; + case Cell::SpecialType::MerkleProof: + TRY_STATUS(check_merkle_proof()); + break; + case Cell::SpecialType::MerkleUpdate: + TRY_STATUS(check_merkle_update()); + break; + default: + return td::Status::Error("Invalid special cell type"); + } + + // Afterwards, we do some common checks and compute virtualization level. + if (*std::max_element(depth_.begin(), depth_.end()) > CellTraits::max_depth) { + return td::Status::Error("Depth is too big"); + } + + for (int i = 0; i < refs_cnt_; ++i) { + virtualization_ = std::max(virtualization_, refs_[i]->get_virtualization()); + } + if (virtualization_ > std::numeric_limits::max()) { + return td::Status::Error("Virtualization is too big to be stored in vm::DataCell"); + } + + // And finally, we compute cell hashes. + // NOTE: Hash computation algorithm is not described correctly (or at all) in the documentation. + int last_computed_hash = -1; + + for (int i = 0; i <= max_level; ++i) { + if (!level_mask_.is_significant(i + 1) && i != max_level) { + continue; + } + + compute_hash(i, last_computed_hash); + for (int j = last_computed_hash + 1; j < i; ++j) { + hash_[j] = hash_[i]; + } + last_computed_hash = i; + } + + return {}; } - return detail::CellWithUniquePtrStorage::create(info.get_storage_size(), info); -} + // Getters for computed values + Cell::SpecialType type() const { + return type_; + } -DataCell::DataCell(Info info) : info_(std::move(info)) { - get_thread_safe_counter().add(1); -} -DataCell::~DataCell() { - get_thread_safe_counter().add(-1); -} + Cell::LevelMask level_mask() const { + return level_mask_; + } -void DataCell::destroy_storage(char* storage) { - auto* refs = info_.get_refs(storage); - for (size_t i = 0; i < get_refs_cnt(); i++) { - Ref(refs[i], Ref::acquire_t{}); // call destructor + td::uint8 virtualization() const { + return static_cast(virtualization_); } -} -td::Result> DataCell::create(td::ConstBitPtr data, unsigned bits, td::Span> refs, - bool special) { - std::array, max_refs> copied_refs; - CHECK(refs.size() <= copied_refs.size()); - for (size_t i = 0; i < refs.size(); i++) { - copied_refs[i] = refs[i]; + std::array const& depths() const { + return depth_; } - return create(std::move(data), bits, td::MutableSpan>(copied_refs.data(), refs.size()), special); -} -DataCell::SpecialType DataCell::special_type() const { - if (is_special()) { - return static_cast(td::bitstring::bits_load_ulong(get_data(), 8)); + std::array const& hashes() const { + return hash_; } - return SpecialType::Ordinary; -} -td::Result> DataCell::create(td::ConstBitPtr data, unsigned bits, td::MutableSpan> refs, - bool special) { - for (auto& ref : refs) { - if (ref.is_null()) { - return td::Status::Error("Has null cell reference"); - } + private: + static constexpr int max_level = CellTraits::max_level; + + static constexpr int hash_bytes = CellTraits::hash_bytes; + static_assert(hash_bytes == sizeof(CellHash)); + + static constexpr int depth_bytes = CellTraits::depth_bytes; + static_assert(depth_bytes == 2); + + td::uint8 read_byte(size_t i) { + return data_[i]; } - SpecialType type = SpecialType::Ordinary; - if (special) { - if (bits < 8) { - return td::Status::Error("Not enough data for a special cell"); + td::Status check_ordinary_cell() { + for (int i = 0; i < refs_cnt_; ++i) { + level_mask_ = level_mask_.apply_or(refs_[i]->get_level_mask()); + + for (int j = 0; j <= max_level; ++j) { + depth_[j] = std::max(depth_[j], refs_[i]->get_depth(j)); + } } - type = static_cast(td::bitstring::bits_load_ulong(data, 8)); - if (type == SpecialType::Ordinary) { - return td::Status::Error("Special cell has Ordinary type"); + + if (refs_cnt_ != 0) { + for (auto& depth : depth_) { + ++depth; + } } + + return {}; } - LevelMask level_mask; - td::uint32 virtualization = 0; - switch (type) { - case SpecialType::Ordinary: { - for (auto& ref : refs) { - level_mask = level_mask.apply_or(ref->get_level_mask()); - virtualization = td::max(virtualization, ref->get_virtualization()); - } - break; + td::Status check_pruned_branch() { + if (refs_cnt_ != 0) { + return td::Status::Error("Pruned branch cannot have references"); + } + if (bit_length_ < 16) { + return td::Status::Error("Length mismatch in a pruned branch"); } - case SpecialType::PrunnedBranch: { - if (refs.size() != 0) { - return td::Status::Error("PrunnedBranch special cell has a cell reference"); - } - if (bits < 16) { - return td::Status::Error("Not enough data for a PrunnedBranch special cell"); - } - level_mask = LevelMask((td::bitstring::bits_load_ulong(data + 8, 8)) & 0xff); - auto level = level_mask.get_level(); - if (level > max_level || level == 0) { - return td::Status::Error("Prunned Branch has an invalid level"); - } - if (bits != (2 + level_mask.apply(level - 1).get_hashes_count() * (hash_bytes + depth_bytes)) * 8) { - return td::Status::Error("Not enouch data for a PrunnedBranch special cell"); - } - // depth will be checked later! - break; + level_mask_ = Cell::LevelMask{read_byte(1)}; + if (level_mask_.get_level() == 0 || level_mask_.get_level() > max_level) { + return td::Status::Error("Invalid level mask in a pruned branch"); } - case SpecialType::Library: { - if (bits != 8 + hash_bytes * 8) { - return td::Status::Error("Not enouch data for a Library special cell"); - } - if (!refs.empty()) { - return td::Status::Error("Library special cell has a cell reference"); - } - break; + int hashes_count = level_mask_.get_hash_i(); + auto expected_byte_size = 2 + hashes_count * (hash_bytes + depth_bytes); + + if (bit_length_ != static_cast(expected_byte_size * 8)) { + return td::Status::Error("Length mismatch in a pruned branch"); } - case SpecialType::MerkleProof: { - if (bits != 8 + (hash_bytes + depth_bytes) * 8) { - return td::Status::Error("Not enouch data for a MerkleProof special cell"); - } - if (refs.size() != 1) { - return td::Status::Error("Wrong references count for a MerkleProof special cell"); - } - if (td::bitstring::bits_memcmp(data + 8, refs[0]->get_hash(0).as_bitslice().get_ptr(), hash_bits) != 0) { - return td::Status::Error("Hash mismatch in a MerkleProof special cell"); - } - if (td::bitstring::bits_load_ulong(data + 8 + hash_bits, depth_bytes * 8) != refs[0]->get_depth(0)) { - return td::Status::Error("Depth mismatch in a MerkleProof special cell"); + // depth[max_level] = 0; + + for (int i = max_level; i--;) { + if (level_mask_.is_significant(i + 1)) { + int hashes_before = level_mask_.apply(i).get_hash_i(); + auto offset = 2 + hashes_count * hash_bytes + hashes_before * depth_bytes; + depth_[i] = DataCell::load_depth(data_.ubegin() + offset); + } else { + depth_[i] = depth_[i + 1]; } - level_mask = refs[0]->get_level_mask().shift_right(); - virtualization = refs[0]->get_virtualization(); - break; } - case SpecialType::MerkleUpdate: { - if (bits != 8 + (hash_bytes + depth_bytes) * 8 * 2) { - return td::Status::Error("Not enouch data for a MerkleUpdate special cell"); - } - if (refs.size() != 2) { - return td::Status::Error("Wrong references count for a MerkleUpdate special cell"); - } - if (td::bitstring::bits_memcmp(data + 8, refs[0]->get_hash(0).as_bitslice().get_ptr(), hash_bits) != 0) { - return td::Status::Error("First hash mismatch in a MerkleProof special cell"); - } - if (td::bitstring::bits_memcmp(data + 8 + hash_bits, refs[1]->get_hash(0).as_bitslice().get_ptr(), hash_bits) != - 0) { - return td::Status::Error("Second hash mismatch in a MerkleProof special cell"); - } - if (td::bitstring::bits_load_ulong(data + 8 + 2 * hash_bits, depth_bytes * 8) != refs[0]->get_depth(0)) { - return td::Status::Error("First depth mismatch in a MerkleProof special cell"); - } - if (td::bitstring::bits_load_ulong(data + 8 + 2 * hash_bits + depth_bytes * 8, depth_bytes * 8) != - refs[1]->get_depth(0)) { - return td::Status::Error("Second depth mismatch in a MerkleProof special cell"); - } + return {}; + } - level_mask = refs[0]->get_level_mask().apply_or(refs[1]->get_level_mask()).shift_right(); - virtualization = td::max(refs[0]->get_virtualization(), refs[1]->get_virtualization()); - break; + td::Status check_library() { + if (refs_cnt_ != 0) { + return td::Status::Error("Library cell cannot have references"); + } + if (bit_length_ != 8 * (1 + hash_bytes)) { + return td::Status::Error("Length mismatch in a library cell"); } - default: - return td::Status::Error("Unknown special cell type"); + return {}; } - Info info; - if (td::unlikely(bits > max_bits)) { - return td::Status::Error("Too many bits"); - } - if (td::unlikely(refs.size() > max_refs)) { - return td::Status::Error("Too many cell references"); - } - if (td::unlikely(virtualization > max_virtualization)) { - return td::Status::Error("Too big virtualization"); - } + td::Status check_merkle_child(int child_idx, int hash_offset, int depth_offset) { + CellHash stored_hash; + std::memcpy(&stored_hash, data_.begin() + hash_offset, hash_bytes); + if (stored_hash != refs_[child_idx]->get_hash(0)) { + return td::Status::Error("Invalid hash in a Merkle proof or update"); + } - CHECK(level_mask.get_level() <= max_level); - - auto hash_count = type == SpecialType::PrunnedBranch ? 1 : level_mask.get_hashes_count(); - DCHECK(hash_count <= max_level + 1); - - info.bits_ = bits; - info.refs_count_ = refs.size() & 7; - info.is_special_ = special; - info.level_mask_ = level_mask.get_mask() & 7; - info.hash_count_ = hash_count & 7; - info.virtualization_ = virtualization & 7; - - auto data_cell = create_empty_data_cell(info); - auto* storage = data_cell->get_storage(); - - // init data - auto* data_ptr = info.get_data(storage); - td::BitPtr{data_ptr}.copy_from(data, bits); - // prepare for serialization - if (bits & 7) { - int m = (0x80 >> (bits & 7)); - unsigned l = bits / 8; - data_ptr[l] = static_cast((data_ptr[l] & -m) | m); - } + td::uint16 stored_depth = DataCell::load_depth(data_.ubegin() + depth_offset); + if (stored_depth != refs_[child_idx]->get_depth(0)) { + return td::Status::Error("Invalid depth in a Merkle proof or update"); + } - // init refs - auto refs_ptr = info.get_refs(storage); - for (size_t i = 0; i < refs.size(); i++) { - refs_ptr[i] = refs[i].release(); - } + for (int i = 0; i <= max_level; ++i) { + depth_[i] = std::max(depth_[i], refs_[child_idx]->get_depth(i + 1) + 1); + } - // init hashes and depth - auto* hashes_ptr = info.get_hashes(storage); - auto* depth_ptr = info.get_depth(storage); + return {}; + } - // NB: be careful with special cells - auto total_hash_count = level_mask.get_hashes_count(); - auto hash_i_offset = total_hash_count - hash_count; - for (td::uint32 level_i = 0, hash_i = 0, level = level_mask.get_level(); level_i <= level; level_i++) { - if (!level_mask.is_significant(level_i)) { - continue; + td::Status check_merkle_proof() { + if (refs_cnt_ != 1) { + return td::Status::Error("Merkle proof must have exactly one reference"); } - SCOPE_EXIT { - hash_i++; - }; - if (hash_i < hash_i_offset) { - continue; + if (bit_length_ != 8 * (1 + hash_bytes + depth_bytes)) { + return td::Status::Error("Length mismatch in a Merkle proof"); } - unsigned char tmp[2]; - tmp[0] = info.d1(level_mask.apply(level_i)); - tmp[1] = info.d2(); - digest::SHA256 hasher; + TRY_STATUS(check_merkle_child(0, 1, 1 + hash_bytes)); - hasher.feed(td::Slice(tmp, 2)); + level_mask_ = refs_[0]->get_level_mask().shift_right(); - if (hash_i == hash_i_offset) { - DCHECK(level_i == 0 || type == SpecialType::PrunnedBranch); - hasher.feed(td::Slice(data_ptr, (bits + 7) >> 3)); - } else { - DCHECK(level_i != 0 && type != SpecialType::PrunnedBranch); - hasher.feed(hashes_ptr[hash_i - hash_i_offset - 1].as_slice()); + return {}; + } + + td::Status check_merkle_update() { + if (refs_cnt_ != 2) { + return td::Status::Error("Merkle update must have exactly two references"); + } + if (bit_length_ != 8 * (1 + (hash_bytes + depth_bytes) * 2)) { + return td::Status::Error("Length mismatch in a Merkle update"); } - auto dest_i = hash_i - hash_i_offset; + TRY_STATUS(check_merkle_child(0, 1, 1 + 2 * hash_bytes)); + TRY_STATUS(check_merkle_child(1, 1 + hash_bytes, 1 + 2 * hash_bytes + 2)); - // calc depth - td::uint16 depth = 0; - for (int i = 0; i < info.refs_count_; i++) { - td::uint16 child_depth = 0; - if (type == SpecialType::MerkleProof || type == SpecialType::MerkleUpdate) { - child_depth = refs_ptr[i]->get_depth(level_i + 1); - } else { - child_depth = refs_ptr[i]->get_depth(level_i); - } + level_mask_ = refs_[0]->get_level_mask().apply_or(refs_[1]->get_level_mask()).shift_right(); - // add depth into hash - td::uint8 child_depth_buf[depth_bytes]; - store_depth(child_depth_buf, child_depth); - hasher.feed(td::Slice(child_depth_buf, depth_bytes)); + return {}; + } - depth = std::max(depth, child_depth); + void compute_hash(int level, int last_computed_hash) { + if (level != max_level && type_ == Cell::SpecialType::PrunnedBranch) { + int hashes_before = level_mask_.apply(level).get_hash_i(); + auto offset = 2 + hashes_before * hash_bytes; + std::memcpy(&hash_[level], data_.begin() + offset, hash_bytes); + return; } - if (info.refs_count_ != 0) { - if (depth >= max_depth) { - return td::Status::Error("Depth is too big"); + + digest::SHA256 hasher; + + auto d1 = refs_cnt_ + (is_special_ << 3) + (level_mask_.apply(level).get_mask() << 5); + auto d2 = (bit_length_ >> 3 << 1) + ((bit_length_ & 7) != 0); + td::uint8 header[] = {static_cast(d1), static_cast(d2)}; + hasher.feed(header, 2); + + if (last_computed_hash != -1 && type_ != Cell::SpecialType::PrunnedBranch) { + hasher.feed(hash_[last_computed_hash].as_slice()); + } else { + hasher.feed(data_.substr(0, bit_length_ / 8)); + // If we are not byte-aligned, some bit gymnastics is required to correctly pad the last byte. + if (bit_length_ % 8 != 0) { + td::uint8 last_byte = data_[bit_length_ / 8]; + last_byte >>= 7 - bit_length_ % 8; + last_byte |= 1; + last_byte <<= 7 - bit_length_ % 8; + hasher.feed({&last_byte, 1}); } - depth++; } - depth_ptr[dest_i] = depth; - // children hash - for (int i = 0; i < info.refs_count_; i++) { - if (type == SpecialType::MerkleProof || type == SpecialType::MerkleUpdate) { - hasher.feed(refs_ptr[i]->get_hash(level_i + 1).as_slice()); - } else { - hasher.feed(refs_ptr[i]->get_hash(level_i).as_slice()); - } + bool is_merkle_node = type_ == Cell::SpecialType::MerkleUpdate || type_ == Cell::SpecialType::MerkleProof; + auto child_level = (is_merkle_node ? std::min(max_level, level + 1) : level); + + for (int i = 0; i < refs_cnt_; ++i) { + td::uint8 stored_depth[depth_bytes]; + DataCell::store_depth(stored_depth, refs_[i]->get_depth(child_level)); + + hasher.feed(stored_depth, depth_bytes); + } + + for (int i = 0; i < refs_cnt_; ++i) { + hasher.feed(refs_[i]->get_hash(child_level).as_slice()); } - auto extracted_size = hasher.extract(hashes_ptr[dest_i].as_slice()); - DCHECK(extracted_size == hash_bytes); + + hasher.extract(hash_[level].as_slice()); } - return Ref(data_cell.release(), Ref::acquire_t{}); + bool is_special_; + Cell::SpecialType type_; + td::Span> refs_; + int refs_cnt_; + td::Slice data_; + int bit_length_; + + Cell::LevelMask level_mask_; + td::uint32 virtualization_{0}; + std::array depth_{}; + std::array hash_{}; +}; + +char* allocate_in_arena(size_t size) { + constexpr size_t batch_size = 1 << 20; + thread_local td::MutableSlice batch; + + auto aligned_size = (size + 7) / 8 * 8; + if (batch.size() < size) { + batch = td::MutableSlice(new char[batch_size], batch_size); + } + auto res = batch.begin(); + batch.remove_prefix(aligned_size); + return res; } -const DataCell::Hash DataCell::do_get_hash(td::uint32 level) const { - auto hash_i = get_level_mask().apply(level).get_hash_i(); - if (special_type() == SpecialType::PrunnedBranch) { - auto this_hash_i = get_level_mask().get_hash_i(); - if (hash_i != this_hash_i) { - return reinterpret_cast(info_.get_data(get_storage()) + 2)[hash_i]; - } - hash_i = 0; +} // namespace + +thread_local bool DataCell::use_arena = false; + +td::Result> DataCell::create(td::Slice data, int bit_length, td::Span> refs, bool is_special) { + CHECK(bit_length >= 0 && data.size() * 8 >= static_cast(bit_length)); + if (refs.size() > CellTraits::max_refs) { + return td::Status::Error("Too many references"); + } + if (bit_length > CellTraits::max_bits) { + return td::Status::Error("Too many data bits"); + } + + CellChecker checker{is_special, data, bit_length, refs}; + TRY_STATUS(checker.check_and_compute_level_info()); + + auto level_info_size = sizeof(detail::LevelInfo) * (checker.level_mask().get_level() + 1); + auto cell_size = sizeof(DataCell) + level_info_size + (bit_length + 7) / 8; + + void* storage = use_arena ? allocate_in_arena(cell_size) : ::operator new(cell_size); + DataCell* allocated_cell = new (storage) + DataCell{bit_length, refs.size(), checker.type(), checker.level_mask(), use_arena, checker.virtualization()}; + auto& cell = *allocated_cell; + + auto mutable_data = cell.trailer_ + level_info_size; + + std::memcpy(mutable_data, data.data(), (bit_length + 7) / 8); + if (bit_length % 8 != 0) { + auto& last_byte = mutable_data[bit_length / 8]; + // This is the same padding as was used in CellChecker::compute_hash above. + last_byte >>= (7 - bit_length % 8); + last_byte |= 1; + last_byte <<= (7 - bit_length % 8); + } + + auto level_info = new (cell.trailer_) detail::LevelInfo[checker.level_mask().get_level() + 1]; + + for (int i = 0; i <= cell.level_; ++i) { + level_info[i] = { + .hash = checker.hashes()[i], + .depth = checker.depths()[i], + }; + } + + for (int i = 0; i < cell.refs_cnt_; ++i) { + cell.refs_[i] = refs[i]; } - return info_.get_hashes(get_storage())[hash_i]; + + return Ref{allocated_cell, Ref::acquire_t{}}; } -td::uint16 DataCell::do_get_depth(td::uint32 level) const { - auto hash_i = get_level_mask().apply(level).get_hash_i(); - if (special_type() == SpecialType::PrunnedBranch) { - auto this_hash_i = get_level_mask().get_hash_i(); - if (hash_i != this_hash_i) { - return load_depth(info_.get_data(get_storage()) + 2 + hash_bytes * this_hash_i + hash_i * depth_bytes); - } - hash_i = 0; +DataCell::~DataCell() { + for (size_t i = 0; i < level_ + 1; ++i) { + level_info()[i].~LevelInfo(); } - return info_.get_depth(get_storage())[hash_i]; + get_thread_safe_counter().add(-1); } int DataCell::serialize(unsigned char* buff, int buff_size, bool with_hashes) const { @@ -352,8 +385,8 @@ int DataCell::serialize(unsigned char* buff, int buff_size, bool with_hashes) co if (len > buff_size) { return 0; } - buff[0] = static_cast(info_.d1() | (with_hashes * 16)); - buff[1] = info_.d2(); + buff[0] = static_cast(construct_d1(max_level) | (with_hashes * 16)); + buff[1] = construct_d2(); int hs = 0; if (with_hashes) { hs = (get_level_mask().get_hashes_count()) * (hash_bytes + depth_bytes); @@ -361,7 +394,7 @@ int DataCell::serialize(unsigned char* buff, int buff_size, bool with_hashes) co std::memset(buff + 2, 0, hs); auto dest = td::MutableSlice(buff + 2, hs); auto level = get_level(); - // TODO: optimize for prunned brandh + // TODO: optimize for pruned branch for (unsigned i = 0; i <= level; i++) { if (!get_level_mask().is_significant(i)) { continue; @@ -376,7 +409,6 @@ int DataCell::serialize(unsigned char* buff, int buff_size, bool with_hashes) co store_depth(dest.ubegin(), get_depth(i)); dest.remove_prefix(depth_bytes); } - // buff[2] = 0; // for testing hash verification in deserialization buff += hs; len -= hs; } @@ -400,8 +432,16 @@ std::string DataCell::to_hex() const { return hex_buff; } -std::ostream& operator<<(std::ostream& os, const DataCell& c) { - return os << c.to_hex(); +DataCell::DataCell(int bit_length, size_t refs_cnt, Cell::SpecialType type, LevelMask level_mask, + bool allocated_in_arena, td::uint8 virtualization) + : bit_length_(bit_length) + , refs_cnt_(static_cast(refs_cnt)) + , type_(static_cast(type)) + , level_(static_cast(level_mask.get_level())) + , level_mask_(level_mask.get_mask()) + , allocated_in_arena_(allocated_in_arena) + , virtualization_(virtualization) { + get_thread_safe_counter().add(1); } } // namespace vm diff --git a/crypto/vm/cells/DataCell.h b/crypto/vm/cells/DataCell.h index b39ee1d4b..172b3390a 100644 --- a/crypto/vm/cells/DataCell.h +++ b/crypto/vm/cells/DataCell.h @@ -17,219 +17,197 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "vm/cells/Cell.h" #include "td/utils/Span.h" - #include "td/utils/ThreadSafeCounter.h" +#include "vm/cells/Cell.h" namespace vm { -class DataCell : public Cell { +namespace detail { + +struct LevelInfo { + CellHash hash; + td::uint16 depth; +}; + +} // namespace detail + +class DataCell final : public Cell { public: // NB: cells created with use_arena=true are never freed static thread_local bool use_arena; - DataCell(const DataCell& other) = delete; - DataCell(DataCell&& other) = delete; - DataCell& operator=(const DataCell& other) = delete; - DataCell& operator=(DataCell&& other) = delete; - ~DataCell() override; + static td::Result> create(td::Slice data, int bit_length, td::Span> refs, bool is_special); static void store_depth(td::uint8* dest, td::uint16 depth) { td::bitstring::bits_store_long(dest, depth, depth_bits); } + static td::uint16 load_depth(const td::uint8* src) { return td::bitstring::bits_load_ulong(src, depth_bits) & 0xffff; } - protected: - struct Info { - unsigned bits_; - - // d1 - unsigned char refs_count_ : 3; - bool is_special_ : 1; - unsigned char level_mask_ : 3; - - unsigned char hash_count_ : 3; - - unsigned char virtualization_ : 3; - - unsigned char d1() const { - return d1(LevelMask{level_mask_}); - } - unsigned char d1(LevelMask level_mask) const { - // d1 = refs_count + 8 * is_special + 32 * level - // + 16 * with_hashes - for seriazlization - // d1 = 7 + 16 + 32 * l - for absent cells - return static_cast(refs_count_ + 8 * is_special_ + 32 * level_mask.get_mask()); - } - unsigned char d2() const { - auto res = static_cast((bits_ / 8) * 2); - if ((bits_ & 7) != 0) { - return static_cast(res + 1); - } - return res; - } - size_t get_hashes_offset() const { - return 0; - } - size_t get_refs_offset() const { - return get_hashes_offset() + hash_bytes * hash_count_; - } - size_t get_depth_offset() const { - return get_refs_offset() + refs_count_ * sizeof(Cell*); - } - size_t get_data_offset() const { - return get_depth_offset() + sizeof(td::uint16) * hash_count_; - } - size_t get_storage_size() const { - return get_data_offset() + (bits_ + 7) / 8; - } - - const Hash* get_hashes(const char* storage) const { - return reinterpret_cast(storage + get_hashes_offset()); - } - - Hash* get_hashes(char* storage) const { - return reinterpret_cast(storage + get_hashes_offset()); - } - - const td::uint16* get_depth(const char* storage) const { - return reinterpret_cast(storage + get_depth_offset()); + void operator delete(DataCell* ptr, std::destroying_delete_t) { + bool allocated_in_arena = ptr->allocated_in_arena_; + ptr->~DataCell(); + if (!allocated_in_arena) { + ::operator delete(ptr); } + } - td::uint16* get_depth(char* storage) const { - return reinterpret_cast(storage + get_depth_offset()); - } + DataCell(DataCell const&) = delete; + DataCell(DataCell&&) = delete; - const unsigned char* get_data(const char* storage) const { - return reinterpret_cast(storage + get_data_offset()); - } - unsigned char* get_data(char* storage) const { - return reinterpret_cast(storage + get_data_offset()); - } + ~DataCell(); - Cell* const* get_refs(const char* storage) const { - return reinterpret_cast(storage + get_refs_offset()); - } - Cell** get_refs(char* storage) const { - return reinterpret_cast(storage + get_refs_offset()); - } - }; + virtual td::Status set_data_cell(Ref&& data_cell) const override { + CHECK(get_hash() == data_cell->get_hash()); + return td::Status::OK(); + } - Info info_; - virtual char* get_storage() = 0; - virtual const char* get_storage() const = 0; - // TODO: we may also save three different pointers + virtual td::Result load_cell() const override { + return LoadedCell{ + .data_cell = Ref{this}, + .virt = {}, + .tree_node = {}, + }; + } - void destroy_storage(char* storage); + virtual td::uint32 get_virtualization() const override { + return virtualization_; + } - explicit DataCell(Info info); + virtual CellUsageTree::NodePtr get_tree_node() const override { + return {}; + } - public: - td::Status set_data_cell(Ref&& data_cell) const override { - CHECK(get_hash() == data_cell->get_hash()); - return td::Status::OK(); + virtual bool is_loaded() const override { + return true; } - td::Result load_cell() const override { - return LoadedCell{Ref{this}, {}, {}}; + + virtual LevelMask get_level_mask() const override { + return LevelMask{level_mask_}; } + unsigned get_refs_cnt() const { - return info_.refs_count_; + return refs_cnt_; } + unsigned get_bits() const { - return info_.bits_; + return bit_length_; } + unsigned size_refs() const { - return info_.refs_count_; + return refs_cnt_; } + unsigned size() const { - return info_.bits_; + return bit_length_; } - const unsigned char* get_data() const { - return info_.get_data(get_storage()); + + unsigned char const* get_data() const { + return reinterpret_cast(trailer_ + sizeof(detail::LevelInfo) * (level_ + 1)); } + Ref get_ref(unsigned idx) const { - if (idx >= get_refs_cnt()) { - return Ref{}; + if (idx >= refs_cnt_) { + return {}; } - return Ref(get_ref_raw_ptr(idx)); + return refs_[idx]; } Cell* get_ref_raw_ptr(unsigned idx) const { - DCHECK(idx < get_refs_cnt()); - return info_.get_refs(get_storage())[idx]; + DCHECK(idx < refs_cnt_); + return const_cast(refs_[idx].get()); } Ref reset_ref_unsafe(unsigned idx, Ref ref, bool check_hash = true) { CHECK(idx < get_refs_cnt()); - auto refs = info_.get_refs(get_storage()); - CHECK(!check_hash || refs[idx]->get_hash() == ref->get_hash()); - auto res = Ref(refs[idx], Ref::acquire_t{}); // call destructor - refs[idx] = ref.release(); - return res; + CHECK(!check_hash || refs_[idx]->get_hash() == ref->get_hash()); + return std::exchange(refs_[idx], std::move(ref)); } - td::uint32 get_virtualization() const override { - return info_.virtualization_; - } - CellUsageTree::NodePtr get_tree_node() const override { - return {}; - } - bool is_loaded() const override { - return true; - } - LevelMask get_level_mask() const override { - return LevelMask{info_.level_mask_}; + bool is_special() const { + return type_ != static_cast(SpecialType::Ordinary); } - bool is_special() const { - return info_.is_special_; + SpecialType special_type() const { + return static_cast(type_); } - SpecialType special_type() const; + int get_serialized_size(bool with_hashes = false) const { return ((get_bits() + 23) >> 3) + (with_hashes ? get_level_mask().get_hashes_count() * (hash_bytes + depth_bytes) : 0); } + size_t get_storage_size() const { - return info_.get_storage_size(); + return sizeof(DataCell) + sizeof(detail::LevelInfo) * (level_ + 1) + (bit_length_ + 7) / 8; } + int serialize(unsigned char* buff, int buff_size, bool with_hashes = false) const; + std::string serialize() const; + std::string to_hex() const; + static td::int64 get_total_data_cells() { return get_thread_safe_counter().sum(); } template void store(StorerT& storer) const { - storer.template store_binary(info_.d1()); - storer.template store_binary(info_.d2()); + storer.template store_binary(construct_d1(max_level)); + storer.template store_binary(construct_d2()); storer.store_slice(td::Slice(get_data(), (get_bits() + 7) / 8)); } - protected: - static constexpr auto max_storage_size = max_refs * sizeof(void*) + (max_level + 1) * hash_bytes + max_bytes; - private: static td::NamedThreadSafeCounter::CounterRef get_thread_safe_counter() { static auto res = td::NamedThreadSafeCounter::get_default().get_counter("DataCell"); return res; } - static std::unique_ptr create_empty_data_cell(Info info); - const Hash do_get_hash(td::uint32 level) const override; - td::uint16 do_get_depth(td::uint32 level) const override; + DataCell(int bit_length, size_t refs_cnt, Cell::SpecialType type, LevelMask level_mask, bool allocated_in_arena, + td::uint8 virtualization); - friend class CellBuilder; - static td::Result> create(td::ConstBitPtr data, unsigned bits, td::Span> refs, bool special); - static td::Result> create(td::ConstBitPtr data, unsigned bits, td::MutableSpan> refs, - bool special); + detail::LevelInfo const* level_info() const { + return reinterpret_cast(trailer_); + } + + virtual td::uint16 do_get_depth(td::uint32 level) const override { + return level_info()[std::min(level_, level)].depth; + } + + virtual const Hash do_get_hash(td::uint32 level) const override { + return level_info()[std::min(level_, level)].hash; + } + + td::uint8 construct_d1(td::uint32 level) const { + return static_cast(refs_cnt_ + (is_special() << 3) + (get_level_mask().apply(level).get_mask() << 5)); + } + + td::uint8 construct_d2() const { + return static_cast(bit_length_ / 8 + (bit_length_ + 7) / 8); + } + + unsigned bit_length_ : 11; + unsigned refs_cnt_ : 3; + unsigned type_ : 3; + unsigned level_ : 3; + unsigned level_mask_ : 3; + unsigned allocated_in_arena_ : 1; + unsigned virtualization_ : 8; + + std::array, max_refs> refs_{}; + + alignas(detail::LevelInfo) char trailer_[]; }; -std::ostream& operator<<(std::ostream& os, const DataCell& c); +inline std::ostream& operator<<(std::ostream& os, const DataCell& c) { + return os << c.to_hex(); +} + inline CellHash as_cell_hash(const Ref& cell) { return cell->get_hash(); } diff --git a/crypto/vm/cells/PrunnedCell.h b/crypto/vm/cells/PrunnedCell.h index 6e8b77093..f4d40c285 100644 --- a/crypto/vm/cells/PrunnedCell.h +++ b/crypto/vm/cells/PrunnedCell.h @@ -17,8 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "vm/cells/CellWithStorage.h" -#include "vm/cells/Cell.h" + +#include "vm/cells/DataCell.h" namespace vm { struct PrunnedCellInfo { @@ -28,7 +28,7 @@ struct PrunnedCellInfo { }; template -class PrunnedCell : public Cell { +class PrunnedCell final : public Cell { public: ExtraT& get_extra() { return extra_; @@ -37,22 +37,33 @@ class PrunnedCell : public Cell { return extra_; } + void operator delete(PrunnedCell* ptr, std::destroying_delete_t) { + bool allocated_in_arena = ptr->info_.allocated_in_arena_; + ptr->~PrunnedCell(); + if (!allocated_in_arena) { + ::operator delete(ptr); + } + } + static td::Result>> create(const PrunnedCellInfo& prunned_cell_info, ExtraT&& extra) { - return create(detail::DefaultAllocator>(), prunned_cell_info, std::forward(extra)); + auto allocator = [](size_t bytes) { return ::operator new(bytes); }; + return create(allocator, true, prunned_cell_info, std::move(extra)); } - template - static td::Result>> create(AllocatorT allocator, const PrunnedCellInfo& prunned_cell_info, - ExtraT&& extra) { + template + static td::Result>> create(AllocatorFunc&& allocator, bool should_free, + const PrunnedCellInfo& prunned_cell_info, ExtraT&& extra) { auto level_mask = prunned_cell_info.level_mask; if (level_mask.get_level() > max_level) { return td::Status::Error("Level is too big"); } Info info(level_mask); - auto prunned_cell = - detail::CellWithArrayStorage>::create(allocator, info.get_storage_size(), info, std::move(extra)); - TRY_STATUS(prunned_cell->init(prunned_cell_info)); - return Ref>(prunned_cell.release(), typename Ref>::acquire_t{}); + + auto storage = allocator(sizeof(PrunnedCell) + info.get_storage_size()); + auto* result = new (storage) PrunnedCell{info, std::move(extra)}; + result->info_.allocated_in_arena_ = !should_free; + TRY_STATUS(result->init(prunned_cell_info)); + return Ref>(result, typename Ref>::acquire_t{}); } LevelMask get_level_mask() const override { @@ -68,6 +79,7 @@ class PrunnedCell : public Cell { } unsigned char level_mask_ : 3; unsigned char hash_count_ : 3; + unsigned char allocated_in_arena_ : 1; size_t get_hashes_offset() const { return 0; } @@ -93,16 +105,10 @@ class PrunnedCell : public Cell { Info info_; ExtraT extra_; - virtual char* get_storage() = 0; - virtual const char* get_storage() const = 0; - void destroy_storage(char* storage) { - // noop - } td::Status init(const PrunnedCellInfo& prunned_cell_info) { - auto storage = get_storage(); auto& new_hash = prunned_cell_info.hash; - auto* hash = info_.get_hashes(storage); + auto* hash = info_.get_hashes(trailer_); size_t n = prunned_cell_info.level_mask.get_hashes_count(); CHECK(new_hash.size() == n * hash_bytes); for (td::uint32 i = 0; i < n; i++) { @@ -111,7 +117,7 @@ class PrunnedCell : public Cell { auto& new_depth = prunned_cell_info.depth; CHECK(new_depth.size() == n * depth_bytes); - auto* depth = info_.get_depth(storage); + auto* depth = info_.get_depth(trailer_); for (td::uint32 i = 0; i < n; i++) { depth[i] = DataCell::load_depth(new_depth.substr(i * Cell::depth_bytes, Cell::depth_bytes).ubegin()); if (depth[i] > max_depth) { @@ -135,11 +141,11 @@ class PrunnedCell : public Cell { private: const Hash do_get_hash(td::uint32 level) const override { - return info_.get_hashes(get_storage())[get_level_mask().apply(level).get_hash_i()]; + return info_.get_hashes(trailer_)[get_level_mask().apply(level).get_hash_i()]; } td::uint16 do_get_depth(td::uint32 level) const override { - return info_.get_depth(get_storage())[get_level_mask().apply(level).get_hash_i()]; + return info_.get_depth(trailer_)[get_level_mask().apply(level).get_hash_i()]; } td::Status set_data_cell(Ref &&data_cell) const override { @@ -149,5 +155,7 @@ class PrunnedCell : public Cell { td::Result load_cell() const override { return td::Status::Error("Can't load prunned branch"); } + + char trailer_[]; }; } // namespace vm diff --git a/crypto/vm/db/InMemoryBagOfCellsDb.cpp b/crypto/vm/db/InMemoryBagOfCellsDb.cpp index e43cfde4e..44be4ca8b 100644 --- a/crypto/vm/db/InMemoryBagOfCellsDb.cpp +++ b/crypto/vm/db/InMemoryBagOfCellsDb.cpp @@ -157,17 +157,9 @@ class ArenaPrunnedCellCreator : public ExtCellCreator { } }; - struct Allocator { - template - std::unique_ptr> make_unique(ArgsT &&...args) { - auto *ptr = arena_.alloc(sizeof(T)); - T *obj = new (ptr) T(std::forward(args)...); - return std::unique_ptr(obj); - } - }; td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { - Allocator allocator; - TRY_RESULT(cell, PrunnedCell::create(allocator, PrunnedCellInfo{level_mask, hash, depth}, Counter())); + TRY_RESULT(cell, PrunnedCell::create([&](size_t bytes) { return arena_.alloc(bytes); }, false, + PrunnedCellInfo{level_mask, hash, depth}, Counter())); return cell; } static td::int64 count() { @@ -601,7 +593,7 @@ class CellStorage { std::vector bucket_stats(buckets_.size()); std::atomic boc_count{0}; for_each_bucket(options.extra_threads, [&](size_t bucket_id, auto &bucket) { - bucket_stats[bucket_id] = validate_bucket_a(bucket, options.use_arena); + bucket_stats[bucket_id] = validate_bucket_a(bucket); boc_count += bucket.boc_count_; }); for_each_bucket(options.extra_threads, [&](size_t bucket_id, auto &bucket) { validate_bucket_b(bucket); }); @@ -730,12 +722,12 @@ class CellStorage { }); } - DynamicBagOfCellsDb::Stats validate_bucket_a(CellBucket &bucket, bool use_arena) { + DynamicBagOfCellsDb::Stats validate_bucket_a(CellBucket &bucket) { DynamicBagOfCellsDb::Stats stats; bucket.infos_.for_each([&](auto &it) { int cell_ref_cnt = it.cell->get_refcnt(); - CHECK(it.db_refcnt + 1 + use_arena >= cell_ref_cnt); - auto extra_refcnt = it.db_refcnt + 1 + use_arena - cell_ref_cnt; + CHECK(it.db_refcnt + 1 >= cell_ref_cnt); + auto extra_refcnt = it.db_refcnt + 1 - cell_ref_cnt; if (extra_refcnt != 0) { bucket.roots_.push_back(it.cell); stats.roots_total_count++; diff --git a/crypto/vm/db/StaticBagOfCellsDb.cpp b/crypto/vm/db/StaticBagOfCellsDb.cpp index c65d2624f..215dcfa3b 100644 --- a/crypto/vm/db/StaticBagOfCellsDb.cpp +++ b/crypto/vm/db/StaticBagOfCellsDb.cpp @@ -18,7 +18,6 @@ */ #include "vm/db/StaticBagOfCellsDb.h" -#include "vm/cells/CellWithStorage.h" #include "vm/boc.h" #include "vm/cells/ExtCell.h" diff --git a/tdutils/td/utils/ThreadSafeCounter.h b/tdutils/td/utils/ThreadSafeCounter.h index 46dc16bf7..8601275a9 100644 --- a/tdutils/td/utils/ThreadSafeCounter.h +++ b/tdutils/td/utils/ThreadSafeCounter.h @@ -20,10 +20,12 @@ #pragma once #include "port/thread.h" -#include "td/utils/common.h" #include "td/utils/Slice.h" #include "td/utils/StringBuilder.h" #include "td/utils/ThreadLocalStorage.h" +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/port/Clocks.h" #include #include From 9c9198253fc600b5ec69c7c298a7d2244803ae6f Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Sat, 29 Mar 2025 15:48:00 -0400 Subject: [PATCH 186/388] Avoid CellBuilder during deserialization in CellSerializationInfo Another free 2.5% of speed-up on block verification. --- crypto/vm/boc.cpp | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index 72afb9988..6377674d7 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -89,14 +89,9 @@ td::Result CellSerializationInfo::get_bits(td::Slice cell) const { // TODO: check usage when result is empty td::Result> CellSerializationInfo::create_data_cell(td::Slice cell_slice, td::Span> refs) const { - CellBuilder cb; - TRY_RESULT(bits, get_bits(cell_slice)); - cb.store_bits(cell_slice.ubegin() + data_offset, bits); DCHECK(refs_cnt == (td::int64)refs.size()); - for (int k = 0; k < refs_cnt; k++) { - cb.store_ref(std::move(refs[k])); - } - TRY_RESULT(res, cb.finalize_novm_nothrow(special)); + TRY_RESULT(bits, get_bits(cell_slice)); + TRY_RESULT(res, DataCell::create(cell_slice.substr(data_offset), bits, refs, special)); CHECK(!res.is_null()); if (res->is_special() != special) { return td::Status::Error("is_special mismatch"); From 22003ae9ff1859b7f64f72df02ca41b4079a47d7 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Mon, 31 Mar 2025 14:35:01 -0400 Subject: [PATCH 187/388] Do not recompute validator set in MasterchainStateQ::mc_reinit This does not improve performance (as unpack_validator_set result is cached regardless) but makes the code clearer. --- crypto/block/mc-config.h | 4 ++-- validator/impl/shard.cpp | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index a44522f11..8af2877db 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -643,8 +643,8 @@ class Config { const WorkchainSet& get_workchain_list() const { return workchains_; } - const ValidatorSet* get_cur_validator_set() const { - return cur_validators_.get(); + std::shared_ptr const& get_cur_validator_set() const& { + return cur_validators_; } std::pair get_validator_set_start_stop(int next = 0) const; ton::ValidatorSessionConfig get_consensus_config() const; diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index d2d402dd6..22091a0f7 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -388,11 +388,8 @@ td::Status MasterchainStateQ::mc_reinit() { CHECK(config_); CHECK(config_->set_block_id_ext(get_block_id())); - auto cv_root = config_->get_config_param(35, 34); - if (cv_root.not_null()) { - TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(cv_root), true)); - cur_validators_ = std::move(validators); - } + cur_validators_ = config_->get_cur_validator_set(); + auto nv_root = config_->get_config_param(37, 36); if (nv_root.not_null()) { TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(nv_root), true)); From 1ac7247fbbb95840f662105ca14a25e5a090a61e Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Mon, 31 Mar 2025 14:30:35 -0400 Subject: [PATCH 188/388] Do a single SHA256::feed call in CellChecker::compute_hash This speeds up DataCell::create by 20% and the verification benchmark by 3.5%. --- crypto/vm/cells/DataCell.cpp | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/crypto/vm/cells/DataCell.cpp b/crypto/vm/cells/DataCell.cpp index 14da8e85a..b6ff8021f 100644 --- a/crypto/vm/cells/DataCell.cpp +++ b/crypto/vm/cells/DataCell.cpp @@ -258,24 +258,33 @@ class CellChecker { return; } - digest::SHA256 hasher; + static_assert(2 + CellTraits::max_bytes + CellTraits::max_refs * (hash_bytes + depth_bytes) <= 512); + char data_to_hash[512]; + int pointer = 0; + + auto add_byte_to_hash = [&](char byte) { data_to_hash[pointer++] = byte; }; + + auto add_slice_to_hash = [&](td::Slice slice) { + std::memcpy(data_to_hash + pointer, slice.data(), slice.size()); + pointer += slice.size(); + }; auto d1 = refs_cnt_ + (is_special_ << 3) + (level_mask_.apply(level).get_mask() << 5); + add_byte_to_hash(static_cast(d1)); auto d2 = (bit_length_ >> 3 << 1) + ((bit_length_ & 7) != 0); - td::uint8 header[] = {static_cast(d1), static_cast(d2)}; - hasher.feed(header, 2); + add_byte_to_hash(static_cast(d2)); if (last_computed_hash != -1 && type_ != Cell::SpecialType::PrunnedBranch) { - hasher.feed(hash_[last_computed_hash].as_slice()); + add_slice_to_hash(hash_[last_computed_hash].as_slice()); } else { - hasher.feed(data_.substr(0, bit_length_ / 8)); + add_slice_to_hash(data_.substr(0, bit_length_ / 8)); // If we are not byte-aligned, some bit gymnastics is required to correctly pad the last byte. if (bit_length_ % 8 != 0) { td::uint8 last_byte = data_[bit_length_ / 8]; last_byte >>= 7 - bit_length_ % 8; last_byte |= 1; last_byte <<= 7 - bit_length_ % 8; - hasher.feed({&last_byte, 1}); + add_byte_to_hash(last_byte); } } @@ -283,16 +292,17 @@ class CellChecker { auto child_level = (is_merkle_node ? std::min(max_level, level + 1) : level); for (int i = 0; i < refs_cnt_; ++i) { - td::uint8 stored_depth[depth_bytes]; - DataCell::store_depth(stored_depth, refs_[i]->get_depth(child_level)); - - hasher.feed(stored_depth, depth_bytes); + auto depth = refs_[i]->get_depth(child_level); + add_byte_to_hash((depth >> 8) & 255); + add_byte_to_hash((depth >> 0) & 255); } for (int i = 0; i < refs_cnt_; ++i) { - hasher.feed(refs_[i]->get_hash(child_level).as_slice()); + add_slice_to_hash(refs_[i]->get_hash(child_level).as_slice()); } + digest::SHA256 hasher; + hasher.feed(data_to_hash, pointer); hasher.extract(hash_[level].as_slice()); } From c69315fee589cf298de4f2066485ca33c7a76f2c Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Tue, 1 Apr 2025 08:32:13 -0400 Subject: [PATCH 189/388] Set CMAKE_POLICY_VERSION_MINIMUM to 3.10 for crc32 This fixes Wasm CI job. --- CMakeLists.txt | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2c65d66a9..ad1aaf506 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -87,16 +87,15 @@ set(CRC32C_BUILD_BENCHMARKS OFF CACHE BOOL "Build CRC32C's benchmarks") set(CRC32C_USE_GLOG OFF CACHE BOOL "Build CRC32C's tests with Google Logging") set(CRC32C_INSTALL OFF CACHE BOOL "Install CRC32C's header and library") message("Add crc32c") -if (NOT MSVC) - set(OLD_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) - # fix aarch64 build @ crc32c/src/crc32c_arm64_linux_check.h - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=address") - add_subdirectory(third-party/crc32c EXCLUDE_FROM_ALL) - set(CMAKE_CXX_FLAGS ${OLD_CMAKE_CXX_FLAGS}) - unset(OLD_CMAKE_CXX_FLAGS) -else() +function(crc32_scope) + set(CMAKE_POLICY_VERSION_MINIMUM "3.10") + if (NOT MSVC) + # fix aarch64 build @ crc32c/src/crc32c_arm64_linux_check.h + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=address") + endif() add_subdirectory(third-party/crc32c EXCLUDE_FROM_ALL) -endif() +endfunction() +crc32_scope() set(CRC32C_FOUND 1) if (TON_USE_ROCKSDB) From 751ce0e044edb981fbe280f12af3edab776a90ba Mon Sep 17 00:00:00 2001 From: neodix42 Date: Tue, 1 Apr 2025 21:46:51 +0400 Subject: [PATCH 190/388] Merge pull request #1598 from neodix42/improve-docker-img-1 Speed up docker image arm64 build --- .../workflows/docker-ubuntu-branch-image.yml | 83 +++++++++++++--- .github/workflows/docker-ubuntu-image.yml | 97 ++++++++++++++++--- 2 files changed, 154 insertions(+), 26 deletions(-) diff --git a/.github/workflows/docker-ubuntu-branch-image.yml b/.github/workflows/docker-ubuntu-branch-image.yml index 00aa5015e..bd97ed233 100644 --- a/.github/workflows/docker-ubuntu-branch-image.yml +++ b/.github/workflows/docker-ubuntu-branch-image.yml @@ -11,21 +11,26 @@ env: IMAGE_NAME: ${{ github.repository }} jobs: - build-and-push: - runs-on: ubuntu-22.04 + build-arm64: + runs-on: ubuntu-22.04-arm steps: - name: Check out repository uses: actions/checkout@v3 with: submodules: 'recursive' + - name: Get tag as branch name + id: tag + run: | + echo "TAG=${GITHUB_REF##*/}" >> $GITHUB_OUTPUT + - name: Set up QEMU uses: docker/setup-qemu-action@v3.5.0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3.10.0 with: - driver-opts: image=moby/buildkit:v0.11.0 + driver-opts: image=moby/buildkit:v0.20.2 - name: Login to GitHub Container Registry uses: docker/login-action@v3 @@ -34,28 +39,80 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and export to Docker + - name: Build and push uses: docker/build-push-action@v6 with: - load: true + platforms: linux/arm64 + push: true context: ./ - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-arm64 - - name: Test - run: | - docker run --rm -e "TEST=1" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test + + build-amd64: + runs-on: ubuntu-22.04 + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' - name: Get tag as branch name id: tag run: | echo "TAG=${GITHUB_REF##*/}" >> $GITHUB_OUTPUT + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.5.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.10.0 + with: + driver-opts: image=moby/buildkit:v0.20.2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push - id: docker_build uses: docker/build-push-action@v6 with: - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 push: true context: ./ - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }} + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-amd64 + + + merge: + runs-on: ubuntu-22.04 + needs: + - build-amd64 + - build-arm64 + steps: + - name: Get tag as branch name + id: tag + run: | + echo "TAG=${GITHUB_REF##*/}" >> $GITHUB_OUTPUT + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.10.0 + + - name: Create manifest list and push + run: | + docker buildx imagetools create \ + --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }} \ + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-arm64 \ + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-amd64 + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }} diff --git a/.github/workflows/docker-ubuntu-image.yml b/.github/workflows/docker-ubuntu-image.yml index aa4eaeef5..0284dcd54 100644 --- a/.github/workflows/docker-ubuntu-image.yml +++ b/.github/workflows/docker-ubuntu-image.yml @@ -11,8 +11,8 @@ env: IMAGE_NAME: ${{ github.repository }} jobs: - build-and-push: - runs-on: ubuntu-22.04 + build-arm64: + runs-on: ubuntu-22.04-arm steps: - name: Check out repository uses: actions/checkout@v3 @@ -32,16 +32,47 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and export to Docker + - name: Get next tag + id: tag + run: | + git fetch --all --tags + git tag -l + NEW_TAG=v$(date +'%Y.%m') + FOUND=$(git tag -l | grep $NEW_TAG | wc -l) + if [ $FOUND -eq 0 ]; then + echo "TAG=$NEW_TAG" >> $GITHUB_OUTPUT + else + echo "TAG=$NEW_TAG-$FOUND" >> $GITHUB_OUTPUT + fi + + - name: Build and push uses: docker/build-push-action@v6 with: - load: true + platforms: linux/arm64 + push: true context: ./ - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-arm64 - - name: Test - run: | - docker run --rm -e "TEST=1" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test + build-amd64: + runs-on: ubuntu-22.04 + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.5.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.10.0 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Get next tag id: tag @@ -57,12 +88,52 @@ jobs: fi - name: Build and push - id: docker_build uses: docker/build-push-action@v6 with: - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 push: true context: ./ - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }} + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-amd64 + + merge: + runs-on: ubuntu-22.04 + needs: + - build-amd64 + - build-arm64 + steps: + - name: Check out repository + uses: actions/checkout@v3 + + - name: Get next tag + id: tag + run: | + git fetch --all --tags + git tag -l + NEW_TAG=v$(date +'%Y.%m') + FOUND=$(git tag -l | grep $NEW_TAG | wc -l) + if [ $FOUND -eq 0 ]; then + echo "TAG=$NEW_TAG" >> $GITHUB_OUTPUT + else + echo "TAG=$NEW_TAG-$FOUND" >> $GITHUB_OUTPUT + fi + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.10.0 + + - name: Create manifest list and push + run: | + docker buildx imagetools create \ + --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest \ + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-arm64 \ + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}-amd64 + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest From 4366b6a66ab49f70a62646823a4178df95cbbabb Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 2 Apr 2025 11:01:46 +0300 Subject: [PATCH 191/388] Don't calculate account storage dict when not needed --- crypto/block/account-storage-stat.cpp | 153 +++++++++++++++----------- crypto/block/account-storage-stat.h | 72 ++++-------- crypto/block/transaction.cpp | 42 ++++--- 3 files changed, 142 insertions(+), 125 deletions(-) diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp index 918d2ac00..9fab8f880 100644 --- a/crypto/block/account-storage-stat.cpp +++ b/crypto/block/account-storage-stat.cpp @@ -23,44 +23,29 @@ AccountStorageStat::AccountStorageStat() : AccountStorageStat({}, {}, 0, 0) { AccountStorageStat::AccountStorageStat(Ref dict_root, std::vector> roots, td::uint64 total_cells, td::uint64 total_bits) - : dict_(std::move(dict_root), 256), roots_(std::move(roots)), total_cells_(total_cells), total_bits_(total_bits) { + : dict_(std::move(dict_root), 256), total_cells_(total_cells), total_bits_(total_bits), roots_(std::move(roots)) { } -AccountStorageStat::AccountStorageStat(const AccountStorageStat& other) - : AccountStorageStat(other.dict_.get_root_cell(), other.roots_, other.total_cells_, other.total_bits_) { +AccountStorageStat::AccountStorageStat(const AccountStorageStat* parent) + : dict_(parent->dict_) + , dict_up_to_date_(parent->dict_up_to_date_) + , total_cells_(parent->total_cells_) + , total_bits_(parent->total_bits_) + , roots_(parent->roots_) + , parent_(parent) { + CHECK(parent_->parent_ == nullptr); } -AccountStorageStat::AccountStorageStat(AccountStorageStat&& other) - : AccountStorageStat(other.dict_.get_root_cell(), std::move(other.roots_), other.total_cells_, other.total_bits_) { - cache_ = std::move(other.cache_); -} - -AccountStorageStat& AccountStorageStat::operator=(const AccountStorageStat& other) { - dict_ = other.dict_; - total_cells_ = other.total_cells_; - total_bits_ = other.total_bits_; - roots_ = other.roots_; - cache_ = {}; - return *this; -} - -AccountStorageStat& AccountStorageStat::operator=(AccountStorageStat&& other) { - dict_ = std::move(other.dict_); - total_cells_ = other.total_cells_; - total_bits_ = other.total_bits_; - roots_ = std::move(other.roots_); - cache_ = std::move(other.cache_); - return *this; -} - -td::Result AccountStorageStat::replace_roots(std::vector> new_roots) { +td::Status AccountStorageStat::replace_roots(std::vector> new_roots, bool check_merkle_depth) { std::erase_if(new_roots, [](const Ref& c) { return c.is_null(); }); if (new_roots.empty()) { roots_.clear(); total_bits_ = total_cells_ = 0; dict_ = vm::Dictionary{256}; cache_ = {}; - return CellInfo{}; + dict_up_to_date_ = true; + parent_ = nullptr; + return td::Status::OK(); } auto cmp = [](const Ref& c1, const Ref& c2) { return c1->get_hash() < c2->get_hash(); }; @@ -71,29 +56,26 @@ td::Result AccountStorageStat::replace_roots(std:: cmp); std::set_difference(roots_.begin(), roots_.end(), new_roots.begin(), new_roots.end(), std::back_inserter(to_del), cmp); + if (to_add.empty() && to_del.empty()) { + return td::Status::OK(); + } - td::uint32 max_merkle_depth = 0; for (const Ref& root : to_add) { TRY_RESULT(info, add_cell(root)); - max_merkle_depth = std::max(max_merkle_depth, info.max_merkle_depth); + if (check_merkle_depth && info.max_merkle_depth > MAX_MERKLE_DEPTH) { + return td::Status::Error("too big Merkle depth"); + } } for (const Ref& root : to_del) { TRY_STATUS(remove_cell(root)); } roots_ = std::move(new_roots); - td::Status S = td::Status::OK(); - std::vector>> values; - cache_.for_each([&](Entry& e) { - if (S.is_ok()) { - S = commit_entry(e, values); - } - }); - TRY_STATUS(std::move(S)); - if (!dict_.multiset(values)) { - return td::Status::Error("failed to update dictionary"); + dict_up_to_date_ = false; + for (auto& [_, e] : cache_) { + TRY_STATUS(finalize_entry(e)); } - return CellInfo{max_merkle_depth}; + return td::Status::OK(); } void AccountStorageStat::add_hint(const td::HashSet& hint) { @@ -105,7 +87,7 @@ void AccountStorageStat::add_hint(const td::HashSet& hint) { Entry& e = get_entry(cell); e.exists = e.exists_known = true; if (is_root) { - fetch_entry(e).ignore(); + fetch_from_dict(e).ignore(); if (e.max_merkle_depth && e.max_merkle_depth.value() != 0) { return; } @@ -126,12 +108,12 @@ void AccountStorageStat::add_hint(const td::HashSet& hint) { td::Result AccountStorageStat::add_cell(const Ref& cell) { Entry& e = get_entry(cell); if (!e.exists_known || e.refcnt_diff < 0) { - TRY_STATUS(fetch_entry(e)); + TRY_STATUS(fetch_from_dict(e)); } ++e.refcnt_diff; if (e.exists || e.refcnt_diff > 1 || (e.refcnt && e.refcnt.value() + e.refcnt_diff != 1)) { if (!e.max_merkle_depth) { - TRY_STATUS(fetch_entry(e)); + TRY_STATUS(fetch_from_dict(e)); if (!e.max_merkle_depth) { return td::Status::Error(PSTRING() << "unexpected unknown Merkle depth of cell " << cell->get_hash()); } @@ -159,7 +141,7 @@ td::Result AccountStorageStat::add_cell(const Ref< td::Status AccountStorageStat::remove_cell(const Ref& cell) { Entry& e = get_entry(cell); if (!e.exists_known) { - TRY_STATUS(fetch_entry(e)); + TRY_STATUS(fetch_from_dict(e)); } if (!e.exists) { return td::Status::Error(PSTRING() << "Failed to remove cell " << cell->get_hash().to_hex() @@ -167,7 +149,7 @@ td::Status AccountStorageStat::remove_cell(const Ref& cell) { } --e.refcnt_diff; if (e.refcnt_diff < 0 && !e.refcnt) { - TRY_STATUS(fetch_entry(e)); + TRY_STATUS(fetch_from_dict(e)); } if (e.refcnt.value() + e.refcnt_diff != 0) { return td::Status::OK(); @@ -180,18 +162,70 @@ td::Status AccountStorageStat::remove_cell(const Ref& cell) { return td::Status::OK(); } +td::Result> AccountStorageStat::get_dict_root() { + if (!dict_up_to_date_) { + std::vector>> values; + for (auto& [_, e] : cache_) { + if (e.dict_refcnt_diff == 0) { + continue; + } + if (!e.exists_known || !e.refcnt || (e.exists && !e.max_merkle_depth)) { + return td::Status::Error("unexpected state of storage stat"); + } + if (e.exists) { + Ref cbr{true}; + auto& cb = cbr.write(); + CHECK(cb.store_long_bool(e.refcnt.value(), 32) && cb.store_long_bool(e.max_merkle_depth.value(), 2)); + values.emplace_back(e.hash.bits(), std::move(cbr)); + } else { + values.emplace_back(e.hash.bits(), Ref{}); + } + e.dict_refcnt_diff = 0; + } + if (!dict_.multiset(values)) { + return td::Status::Error("failed to update dictionary"); + } + dict_up_to_date_ = true; + } + return dict_.get_root_cell(); +} + +void AccountStorageStat::apply_child_stat(AccountStorageStat&& child) { + CHECK(parent_ == nullptr); + if (child.parent_ == nullptr) { + *this = std::move(child); + return; + } + CHECK(child.parent_ == this); + total_bits_ = child.total_bits_; + total_cells_ = child.total_cells_; + dict_ = std::move(child.dict_); + dict_up_to_date_ = child.dict_up_to_date_; + roots_ = std::move(child.roots_); + for (auto& [hash, e] : child.cache_) { + cache_[hash] = std::move(e); + } +} + AccountStorageStat::Entry& AccountStorageStat::get_entry(const Ref& cell) { - return cache_.apply(cell->get_hash().as_slice(), [&](Entry& e) { - if (e.inited) { - return; + Entry& e = cache_[cell->get_hash()]; + if (e.cell.not_null()) { + return e; + } + if (parent_) { + auto it = parent_->cache_.find(cell->get_hash()); + if (it != parent_->cache_.end()) { + CHECK(it->second.cell.not_null()); + e = it->second; + return e; } - e.inited = true; - e.cell = cell; - e.hash = cell->get_hash(); - }); + } + e.cell = cell; + e.hash = cell->get_hash(); + return e; } -td::Status AccountStorageStat::fetch_entry(Entry& e) { +td::Status AccountStorageStat::fetch_from_dict(Entry& e) { if (e.exists_known && e.refcnt && (!e.exists || e.max_merkle_depth)) { return td::Status::OK(); } @@ -214,20 +248,19 @@ td::Status AccountStorageStat::fetch_entry(Entry& e) { return td::Status::OK(); } -td::Status AccountStorageStat::commit_entry(Entry& e, - std::vector>>& values) { +td::Status AccountStorageStat::finalize_entry(Entry& e) { if (e.refcnt_diff == 0) { return td::Status::OK(); } - TRY_STATUS(fetch_entry(e)); + TRY_STATUS(fetch_from_dict(e)); e.refcnt.value() += e.refcnt_diff; + e.dict_refcnt_diff += e.refcnt_diff; e.refcnt_diff = 0; bool spec; if (e.refcnt.value() == 0) { --total_cells_; total_bits_ -= vm::load_cell_slice_special(e.cell, spec).size(); e.exists = false; - values.emplace_back(e.hash.bits(), td::Ref{}); } else { if (!e.exists) { ++total_cells_; @@ -237,10 +270,6 @@ td::Status AccountStorageStat::commit_entry(Entry& e, if (!e.max_merkle_depth) { return td::Status::Error(PSTRING() << "Failed to store entry " << e.hash.to_hex() << " : unknown merkle depth"); } - td::Ref cbr{true}; - auto& cb = cbr.write(); - CHECK(cb.store_long_bool(e.refcnt.value(), 32) && cb.store_long_bool(e.max_merkle_depth.value(), 2)); - values.emplace_back(e.hash.bits(), std::move(cbr)); } return td::Status::OK(); } diff --git a/crypto/block/account-storage-stat.h b/crypto/block/account-storage-stat.h index 83eb3af5a..a7da8288f 100644 --- a/crypto/block/account-storage-stat.h +++ b/crypto/block/account-storage-stat.h @@ -28,19 +28,19 @@ using td::Ref; class AccountStorageStat { public: - struct CellInfo { - td::uint32 max_merkle_depth = 0; - }; - AccountStorageStat(); AccountStorageStat(Ref dict_root, std::vector> roots, td::uint64 total_cells, td::uint64 total_bits); - AccountStorageStat(const AccountStorageStat &other); - AccountStorageStat(AccountStorageStat &&other); + explicit AccountStorageStat(const AccountStorageStat *parent); + AccountStorageStat(const AccountStorageStat &other) = delete; + AccountStorageStat(AccountStorageStat &&other) = default; ~AccountStorageStat() = default; - AccountStorageStat &operator=(const AccountStorageStat &other); - AccountStorageStat &operator=(AccountStorageStat &&other); + AccountStorageStat &operator=(const AccountStorageStat &other) = delete; + AccountStorageStat &operator=(AccountStorageStat &&other) = default; + + td::Status replace_roots(std::vector> new_roots, bool check_merkle_depth = false); + void add_hint(const td::HashSet &visited); td::uint64 get_total_cells() const { return total_cells_; @@ -50,70 +50,46 @@ class AccountStorageStat { return total_bits_; } - Ref get_dict_root() const { - return dict_.get_root_cell(); - } + td::Result> get_dict_root(); - td::Bits256 get_dict_hash() const { - return dict_.is_empty() ? td::Bits256::zero() : td::Bits256{dict_.get_root_cell()->get_hash().bits()}; + td::Result get_dict_hash() { + TRY_RESULT(root, get_dict_root()); + return root.is_null() ? td::Bits256::zero() : td::Bits256{root->get_hash().bits()}; } - td::Result replace_roots(std::vector> hint); - void add_hint(const td::HashSet &visited); + void apply_child_stat(AccountStorageStat &&child); private: vm::Dictionary dict_; + bool dict_up_to_date_ = true; td::uint64 total_cells_, total_bits_; std::vector> roots_; + const AccountStorageStat *parent_ = nullptr; + + struct CellInfo { + td::uint32 max_merkle_depth = 0; + }; td::Result add_cell(const Ref &cell); td::Status remove_cell(const Ref &cell); struct Entry { - bool inited = false; vm::CellHash hash; Ref cell; bool exists_known = false; bool exists = false; td::optional refcnt, max_merkle_depth; td::int32 refcnt_diff = 0; - - vm::Cell::Hash key() const { - return hash; - } - bool operator<(const Entry &other) const { - return key() < other.key(); - } - struct Eq { - using is_transparent = void; // Pred to use - bool operator()(const Entry &info, const Entry &other_info) const { - return info.key() == other_info.key(); - } - bool operator()(const Entry &info, td::Slice hash) const { - return info.key().as_slice() == hash; - } - bool operator()(td::Slice hash, const Entry &info) const { - return info.key().as_slice() == hash; - } - }; - struct Hash { - using is_transparent = void; // Pred to use - using transparent_key_equal = Eq; - size_t operator()(td::Slice hash) const { - return cell_hash_slice_hash(hash); - } - size_t operator()(const Entry &info) const { - return cell_hash_slice_hash(info.key().as_slice()); - } - }; + td::int32 dict_refcnt_diff = 0; }; - vm::CellHashTable cache_; + td::HashMap cache_; Entry &get_entry(const Ref &cell); - td::Status fetch_entry(Entry &e); - td::Status commit_entry(Entry &e, std::vector>> &values); + td::Status fetch_from_dict(Entry &e); + td::Status finalize_entry(Entry &e); static constexpr td::uint32 MERKLE_DEPTH_LIMIT = 3; + static constexpr td::uint32 MAX_MERKLE_DEPTH = 2; }; class StorageStatCalculationContext : public td::Context { diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 7dbc5249f..f24a03676 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -2951,15 +2951,12 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, } AccountStorageStat storage_stat; if (update_storage_stat && account.account_storage_stat) { - storage_stat = account.account_storage_stat.value(); + storage_stat = AccountStorageStat{&account.account_storage_stat.value()}; } { TD_PERF_COUNTER(transaction_storage_stat_a); td::Timer timer; - TRY_RESULT(info, storage_stat.replace_roots({new_code, new_data, new_library})); - if (info.max_merkle_depth > max_allowed_merkle_depth) { - return td::Status::Error("too big Merkle depth"); - } + TRY_STATUS(storage_stat.replace_roots({new_code, new_data, new_library}, /* check_merkle_depth = */ true)); if (timer.elapsed() > 0.1) { LOG(INFO) << "Compute used storage (1) took " << timer.elapsed() << "s"; } @@ -3276,23 +3273,33 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { } } - if (storage_refs_changed || (cfg.store_storage_dict_hash && !account.storage_dict_hash)) { + if (storage_refs_changed || + (cfg.store_storage_dict_hash && !account.storage_dict_hash && account.storage_used.cells > 25)) { TD_PERF_COUNTER(transaction_storage_stat_b); td::Timer timer; if (!new_account_storage_stat && account.account_storage_stat) { - new_account_storage_stat = account.account_storage_stat; + new_account_storage_stat = AccountStorageStat(&account.account_storage_stat.value()); } AccountStorageStat& stats = new_account_storage_stat.value_force(); // Don't check Merkle depth and size here - they were checked in check_state_limits - auto S = stats.replace_roots(new_storage_for_stat->prefetch_all_refs()).move_as_status(); + td::Status S = stats.replace_roots(new_storage_for_stat->prefetch_all_refs()); if (S.is_error()) { LOG(ERROR) << "Cannot recompute storage stats for account " << account.addr.to_hex() << ": " << S.move_as_error(); return false; } - new_storage_dict_hash = stats.get_dict_hash(); // Root of AccountStorage is not counted in AccountStorageStat new_storage_used.cells = stats.get_total_cells() + 1; new_storage_used.bits = stats.get_total_bits() + new_storage_for_stat->size(); + // TODO: think about this limit (25) + if (cfg.store_storage_dict_hash && new_storage_used.cells > 25) { + auto r_hash = stats.get_dict_hash(); + if (r_hash.is_error()) { + LOG(ERROR) << "Cannot compute storage dict hash for account " << account.addr.to_hex() << ": " + << r_hash.move_as_error(); + return false; + } + new_storage_dict_hash = r_hash.move_as_ok(); + } if (timer.elapsed() > 0.1) { LOG(INFO) << "Compute used storage (2) took " << timer.elapsed() << "s"; } @@ -3300,11 +3307,10 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { new_storage_used = account.storage_used; new_storage_used.bits -= old_storage_for_stat->size(); new_storage_used.bits += new_storage_for_stat->size(); - new_storage_dict_hash = account.storage_dict_hash; - new_account_storage_stat = account.account_storage_stat; - } - if (!cfg.store_storage_dict_hash || (new_storage_dict_hash && new_storage_dict_hash.value().is_zero())) { - new_storage_dict_hash = {}; + new_account_storage_stat = {}; + if (cfg.store_storage_dict_hash) { + new_storage_dict_hash = account.storage_dict_hash; + } } CHECK(cb.store_long_bool(1, 1) // account$1 @@ -3683,7 +3689,13 @@ Ref Transaction::commit(Account& acc) { acc.last_trans_hash_ = root->get_hash().bits(); acc.last_paid = last_paid; acc.storage_used = new_storage_used; - acc.account_storage_stat = std::move(new_account_storage_stat); + if (new_account_storage_stat) { + if (acc.account_storage_stat) { + acc.account_storage_stat.value().apply_child_stat(std::move(new_account_storage_stat.value())); + } else { + acc.account_storage_stat = std::move(new_account_storage_stat); + } + } acc.storage_dict_hash = new_storage_dict_hash; acc.storage = new_storage; acc.balance = std::move(balance); From 5c777002bc55cc25b9605ae42f912ba8b85ca53d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 2 Apr 2025 12:17:30 +0300 Subject: [PATCH 192/388] Fix compilation errors --- crypto/block/transaction.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index e0345c537..bc8961164 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -617,7 +617,7 @@ td::Result> Account::compute_account_storage_dict() const { if (storage_for_stat.is_null()) { return td::Status::Error("cannot compute storage dict: invalid storage"); } - TRY_STATUS(stat.replace_roots(storage_for_stat->prefetch_all_refs()).move_as_status()); + TRY_STATUS(stat.replace_roots(storage_for_stat->prefetch_all_refs())); // Root of AccountStorage is not counted in AccountStorageStat td::uint64 expected_cells = stat.get_total_cells() + 1; td::uint64 expected_bits = stat.get_total_bits() + storage->size(); @@ -626,9 +626,10 @@ td::Result> Account::compute_account_storage_dict() const { << " bits=" << expected_bits << ", found cells" << storage_used.cells << " bits=" << storage_used.bits); } - if (storage_dict_hash.value() != stat.get_dict_hash()) { - return td::Status::Error(PSTRING() << "invalid storage dict hash: computed " << stat.get_dict_hash().to_hex() - << ", found " << storage_dict_hash.value().to_hex()); + TRY_RESULT(root_hash, stat.get_dict_hash()); + if (storage_dict_hash.value() != root_hash) { + return td::Status::Error(PSTRING() << "invalid storage dict hash: computed " << root_hash.to_hex() << ", found " + << storage_dict_hash.value().to_hex()); } return stat.get_dict_root(); } @@ -662,9 +663,10 @@ td::Status Account::init_account_storage_stat(Ref dict_root) { } AccountStorageStat new_stat(std::move(dict_root), storage->prefetch_all_refs(), storage_used.cells - 1, storage_used.bits - storage->size()); - if (storage_dict_hash.value() != new_stat.get_dict_hash()) { - return td::Status::Error(PSTRING() << "invalid storage dict hash: computed " << new_stat.get_dict_hash().to_hex() - << ", found " << storage_dict_hash.value().to_hex()); + TRY_RESULT(root_hash, new_stat.get_dict_hash()); + if (storage_dict_hash.value() != root_hash) { + return td::Status::Error(PSTRING() << "invalid storage dict hash: computed " << root_hash.to_hex() << ", found " + << storage_dict_hash.value().to_hex()); } account_storage_stat = std::move(new_stat); return td::Status::OK(); From dd5068163c253f6e925dfecd5137fc4bca8f3843 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 3 Apr 2025 12:59:24 +0300 Subject: [PATCH 193/388] Optimize AccountStorageStat::add_hint call --- crypto/block/transaction.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index bc8961164..f316bca96 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -3363,12 +3363,12 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { td::Timer timer; if (!new_account_storage_stat && account.account_storage_stat) { new_account_storage_stat = AccountStorageStat(&account.account_storage_stat.value()); + if (compute_phase) { + new_account_storage_stat.value().add_hint(compute_phase->vm_loaded_cells); + } } AccountStorageStat& stats = new_account_storage_stat.value_force(); // Don't check Merkle depth and size here - they were checked in check_state_limits - if (compute_phase) { - stats.add_hint(compute_phase->vm_loaded_cells); - } auto roots = new_storage_for_stat->prefetch_all_refs(); storage_stats_updates.insert(storage_stats_updates.end(), roots.begin(), roots.end()); { From 3b4b00c0752d3ecd9926a567d561522a3b674912 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 3 Apr 2025 16:05:57 +0300 Subject: [PATCH 194/388] Optimize cell loading in account storage stat --- crypto/block/account-storage-stat.cpp | 20 ++++++++++++++------ crypto/block/account-storage-stat.h | 6 ++++-- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp index 9fab8f880..56cc8f1e1 100644 --- a/crypto/block/account-storage-stat.cpp +++ b/crypto/block/account-storage-stat.cpp @@ -95,6 +95,7 @@ void AccountStorageStat::add_hint(const td::HashSet& hint) { if (hint.contains(cell->get_hash())) { bool spec; vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); + e.size_bits = cs.size(); for (unsigned i = 0; i < cs.size_refs(); ++i) { dfs(cs.prefetch_ref(i), false); } @@ -124,6 +125,7 @@ td::Result AccountStorageStat::add_cell(const Ref< td::uint32 max_merkle_depth = 0; bool spec; vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); + e.size_bits = cs.size(); for (unsigned i = 0; i < cs.size_refs(); ++i) { TRY_RESULT(info, add_cell(cs.prefetch_ref(i))); max_merkle_depth = std::max(max_merkle_depth, info.max_merkle_depth); @@ -156,6 +158,7 @@ td::Status AccountStorageStat::remove_cell(const Ref& cell) { } bool spec; vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); + e.size_bits = cs.size(); for (unsigned i = 0; i < cs.size_refs(); ++i) { TRY_STATUS(remove_cell(cs.prefetch_ref(i))); } @@ -209,18 +212,18 @@ void AccountStorageStat::apply_child_stat(AccountStorageStat&& child) { AccountStorageStat::Entry& AccountStorageStat::get_entry(const Ref& cell) { Entry& e = cache_[cell->get_hash()]; - if (e.cell.not_null()) { + if (e.inited) { return e; } if (parent_) { auto it = parent_->cache_.find(cell->get_hash()); if (it != parent_->cache_.end()) { - CHECK(it->second.cell.not_null()); + CHECK(it->second.inited); e = it->second; return e; } } - e.cell = cell; + e.inited = true; e.hash = cell->get_hash(); return e; } @@ -256,15 +259,20 @@ td::Status AccountStorageStat::finalize_entry(Entry& e) { e.refcnt.value() += e.refcnt_diff; e.dict_refcnt_diff += e.refcnt_diff; e.refcnt_diff = 0; - bool spec; if (e.refcnt.value() == 0) { + if (!e.size_bits) { + return td::Status::Error(PSTRING() << "Failed to store entry " << e.hash.to_hex() << " : unknown cell bits"); + } --total_cells_; - total_bits_ -= vm::load_cell_slice_special(e.cell, spec).size(); + total_bits_ -= e.size_bits.value(); e.exists = false; } else { if (!e.exists) { + if (!e.size_bits) { + return td::Status::Error(PSTRING() << "Failed to store entry " << e.hash.to_hex() << " : unknown cell bits"); + } ++total_cells_; - total_bits_ += vm::load_cell_slice_special(e.cell, spec).size(); + total_bits_ += e.size_bits.value(); } e.exists = true; if (!e.max_merkle_depth) { diff --git a/crypto/block/account-storage-stat.h b/crypto/block/account-storage-stat.h index a7da8288f..645f0744d 100644 --- a/crypto/block/account-storage-stat.h +++ b/crypto/block/account-storage-stat.h @@ -74,15 +74,17 @@ class AccountStorageStat { td::Status remove_cell(const Ref &cell); struct Entry { + bool inited = false; vm::CellHash hash; - Ref cell; + td::optional size_bits; bool exists_known = false; bool exists = false; td::optional refcnt, max_merkle_depth; td::int32 refcnt_diff = 0; td::int32 dict_refcnt_diff = 0; }; - td::HashMap cache_; + + td::HashMap> cache_; Entry &get_entry(const Ref &cell); td::Status fetch_from_dict(Entry &e); From 0e2c9bf75644f5720b0b31242e8be87557085b19 Mon Sep 17 00:00:00 2001 From: Marat <98183742+dungeon-master-666@users.noreply.github.com> Date: Thu, 3 Apr 2025 16:46:13 +0200 Subject: [PATCH 195/388] Fix calculating normalized hash (#1602) * fix calculating normalized hash * add get_ext_in_msg_hash_norm to .h --- tonlib/tonlib/TonlibClient.cpp | 32 +++++++++++++++++--------------- tonlib/tonlib/TonlibClient.h | 1 + 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index b137ee769..00dcf22db 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -3308,14 +3308,9 @@ td::Status TonlibClient::do_request(const tonlib_api::raw_sendMessage& request, return td::Status::OK(); } -td::Status TonlibClient::do_request(const tonlib_api::raw_sendMessageReturnHash& request, - td::Promise>&& promise) { - TRY_RESULT_PREFIX(body, vm::std_boc_deserialize(request.body_), TonlibError::InvalidBagOfCells("body")); - auto hash = body->get_hash().as_slice().str(); - - // compute hash normalized +td::Result get_ext_in_msg_hash_norm(td::Ref ext_in_msg_cell) { block::gen::Message::Record message; - if (!tlb::type_unpack_cell(body, block::gen::t_Message_Any, message)) { + if (!tlb::type_unpack_cell(ext_in_msg_cell, block::gen::t_Message_Any, message)) { return td::Status::Error("Failed to unpack Message"); } auto tag = block::gen::CommonMsgInfo().get_tag(*message.info); @@ -3327,32 +3322,39 @@ td::Status TonlibClient::do_request(const tonlib_api::raw_sendMessageReturnHash& return td::Status::Error("Failed to unpack CommonMsgInfo::ext_in_msg_info"); } - td::Ref body_norm; + td::Ref body; auto body_cs = message.body.write(); - if (body_cs.fetch_long(1) == 1) { - body_norm = body_cs.fetch_ref(); + if (body_cs.fetch_ulong(1) == 1) { + body = body_cs.fetch_ref(); } else { - body_norm = vm::CellBuilder().append_cellslice(body_cs).finalize(); + body = vm::CellBuilder().append_cellslice(body_cs).finalize(); } auto cb = vm::CellBuilder(); - bool status = + bool status = cb.store_long_bool(2, 2) && // message$_ -> info:CommonMsgInfo -> ext_in_msg_info$10 cb.store_long_bool(0, 2) && // message$_ -> info:CommonMsgInfo -> src:MsgAddressExt -> addr_none$00 cb.append_cellslice_bool(msg_info.dest) && // message$_ -> info:CommonMsgInfo -> dest:MsgAddressInt cb.store_long_bool(0, 4) && // message$_ -> info:CommonMsgInfo -> import_fee:Grams -> 0 cb.store_long_bool(0, 1) && // message$_ -> init:(Maybe (Either StateInit ^StateInit)) -> nothing$0 cb.store_long_bool(1, 1) && // message$_ -> body:(Either X ^X) -> right$1 - cb.store_ref_bool(body_norm); + cb.store_ref_bool(body); if (!status) { return td::Status::Error("Failed to build normalized message"); } - auto hash_norm = cb.finalize()->get_hash().as_slice().str(); + return cb.finalize()->get_hash().bits(); +} + +td::Status TonlibClient::do_request(const tonlib_api::raw_sendMessageReturnHash& request, + td::Promise>&& promise) { + TRY_RESULT_PREFIX(body, vm::std_boc_deserialize(request.body_), TonlibError::InvalidBagOfCells("body")); + auto hash = body->get_hash().as_slice().str(); + TRY_RESULT(hash_norm, get_ext_in_msg_hash_norm(body)); make_request(int_api::SendMessage{std::move(body)}, promise.wrap([hash = std::move(hash), hash_norm = std::move(hash_norm)](auto res) { - return tonlib_api::make_object(std::move(hash), std::move(hash_norm)); + return tonlib_api::make_object(std::move(hash), hash_norm.as_slice().str()); })); return td::Status::OK(); } diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index 28239a8a4..ed19be65e 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -59,6 +59,7 @@ class RunEmulator; td::Result> to_tonlib_api( const ton::ManualDns::EntryData& entry_data); td::Result to_dns_entry_data(tonlib_api::dns_EntryData& entry_data); +td::Result get_ext_in_msg_hash_norm(td::Ref ext_in_msg_cell); class TonlibClient : public td::actor::Actor { public: From 5661301db5c0d7a6facea4906fe160c9802bf655 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Sat, 5 Apr 2025 10:38:50 +0300 Subject: [PATCH 196/388] Fix archive-ttl overflow, remove unused code, clarify validator-engine flags (#1605) --- validator-engine/validator-engine.cpp | 8 +- validator/db/archive-manager.cpp | 6 +- validator/db/archive-manager.hpp | 4 +- validator/db/rootdb.cpp | 6 +- validator/db/rootdb.hpp | 4 +- validator/interfaces/db.h | 2 +- validator/interfaces/validator-manager.h | 9 -- validator/manager-disk.hpp | 25 ----- validator/manager-hardfork.hpp | 25 ----- validator/manager.cpp | 119 +---------------------- validator/manager.hpp | 26 ----- 11 files changed, 15 insertions(+), 219 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index cbcc3ab1f..4b4e47348 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -4359,7 +4359,7 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_max_mempool_num, v); }); return td::Status::OK(); }); - p.add_checked_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=86400", + p.add_checked_option('b', "block-ttl", "deprecated", [&](td::Slice fname) { auto v = td::to_double(fname); if (v <= 0) { @@ -4369,7 +4369,9 @@ int main(int argc, char *argv[]) { return td::Status::OK(); }); p.add_checked_option( - 'A', "archive-ttl", "archived blocks will be deleted after this time (in seconds) default=7*86400", + 'A', "archive-ttl", + "ttl for archived blocks (in seconds) default=7*86400. Note: archived blocks are gc'd after state-ttl + " + "archive-ttl seconds", [&](td::Slice fname) { auto v = td::to_double(fname); if (v <= 0) { @@ -4379,7 +4381,7 @@ int main(int argc, char *argv[]) { return td::Status::OK(); }); p.add_checked_option( - 'K', "key-proof-ttl", "key blocks will be deleted after this time (in seconds) default=365*86400*10", + 'K', "key-proof-ttl", "deprecated", [&](td::Slice fname) { auto v = td::to_double(fname); if (v <= 0) { diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index 9b1ebf6c2..9e6ea797e 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -965,8 +965,8 @@ void ArchiveManager::alarm() { } } -void ArchiveManager::run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) { - auto p = get_temp_package_id_by_unixtime(mc_ts - TEMP_PACKAGES_TTL); +void ArchiveManager::run_gc(UnixTime mc_ts, UnixTime gc_ts, double archive_ttl) { + auto p = get_temp_package_id_by_unixtime((double)mc_ts - TEMP_PACKAGES_TTL); std::vector vec; for (auto &x : temp_files_) { if (x.first < p) { @@ -994,7 +994,7 @@ void ArchiveManager::run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl if (it == desc.first_blocks.end()) { continue; } - if (it->second.ts < gc_ts - archive_ttl) { + if ((double)it->second.ts < (double)gc_ts - archive_ttl) { vec.push_back(f.first); } } diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index cd79ccc6e..33dd1f3d6 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -61,7 +61,7 @@ class ArchiveManager : public td::actor::Actor { void truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise); //void truncate_continue(BlockSeqno masterchain_seqno, td::Promise promise); - void run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl); + void run_gc(UnixTime mc_ts, UnixTime gc_ts, double archive_ttl); /* from LTDB */ void get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, td::Promise promise); @@ -240,7 +240,7 @@ class ArchiveManager : public td::actor::Actor { void update_permanent_slices(); - static const td::uint32 TEMP_PACKAGES_TTL = 3600; + static constexpr double TEMP_PACKAGES_TTL = 3600; }; } // namespace validator diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 2b370eb06..90546fbef 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -431,10 +431,6 @@ void RootDb::allow_state_gc(BlockIdExt block_id, td::Promise promise) { td::actor::send_closure(validator_manager_, &ValidatorManager::allow_block_state_gc, block_id, std::move(promise)); } -void RootDb::allow_block_gc(BlockIdExt block_id, td::Promise promise) { - td::actor::send_closure(validator_manager_, &ValidatorManager::allow_block_info_gc, block_id, std::move(promise)); -} - void RootDb::prepare_stats(td::Promise>> promise) { auto merger = StatsMerger::create(std::move(promise)); td::actor::send_closure(cell_db_, &CellDb::prepare_stats, merger.make_promise("celldb.")); @@ -519,7 +515,7 @@ void RootDb::set_async_mode(bool mode, td::Promise promise) { td::actor::send_closure(archive_db_, &ArchiveManager::set_async_mode, mode, std::move(promise)); } -void RootDb::run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) { +void RootDb::run_gc(UnixTime mc_ts, UnixTime gc_ts, double archive_ttl) { td::actor::send_closure(archive_db_, &ArchiveManager::run_gc, mc_ts, gc_ts, archive_ttl); } diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 9e5b0b302..2ccd7f942 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -118,8 +118,6 @@ class RootDb : public Db { void archive(BlockHandle handle, td::Promise promise) override; void allow_state_gc(BlockIdExt block_id, td::Promise promise); - void allow_block_gc(BlockIdExt block_id, td::Promise promise); - //void allow_gc(FileDb::RefId ref_id, bool is_archive, td::Promise promise); void prepare_stats(td::Promise>> promise) override; @@ -137,7 +135,7 @@ class RootDb : public Db { td::Promise promise) override; void set_async_mode(bool mode, td::Promise promise) override; - void run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) override; + void run_gc(UnixTime mc_ts, UnixTime gc_ts, double archive_ttl) override; void add_persistent_state_description(td::Ref desc, td::Promise promise) override; void get_persistent_state_descriptions(td::Promise>> promise) override; diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index 9a2eea56e..f21c6f902 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -123,7 +123,7 @@ class Db : public td::actor::Actor { td::Promise promise) = 0; virtual void set_async_mode(bool mode, td::Promise promise) = 0; - virtual void run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) = 0; + virtual void run_gc(UnixTime mc_ts, UnixTime gc_ts, double archive_ttl) = 0; virtual void add_persistent_state_description(td::Ref desc, td::Promise promise) = 0; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 00fb77e1e..1f08c4ef5 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -160,16 +160,7 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void try_get_static_file(FileHash file_hash, td::Promise promise) = 0; - virtual void allow_block_data_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) = 0; virtual void allow_block_state_gc(BlockIdExt block_id, td::Promise promise) = 0; - virtual void allow_zero_state_file_gc(BlockIdExt block_id, td::Promise promise) = 0; - virtual void allow_persistent_state_file_gc(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) = 0; - virtual void allow_block_signatures_gc(BlockIdExt block_id, td::Promise promise) = 0; - virtual void allow_block_proof_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) = 0; - virtual void allow_block_proof_link_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) = 0; - virtual void allow_block_candidate_gc(BlockIdExt block_id, td::Promise promise) = 0; - virtual void allow_block_info_gc(BlockIdExt block_id, td::Promise promise) = 0; virtual void archive(BlockHandle handle, td::Promise promise) = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 62f54b2e9..8526ffa4d 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -341,34 +341,9 @@ class ValidatorManagerImpl : public ValidatorManager { void update_gc_block_handle(BlockHandle handle, td::Promise promise) override { promise.set_value(td::Unit()); } - void allow_block_data_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - promise.set_result(false); - } void allow_block_state_gc(BlockIdExt block_id, td::Promise promise) override { promise.set_result(false); } - void allow_zero_state_file_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } - void allow_persistent_state_file_gc(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override { - promise.set_result(false); - } - void allow_block_signatures_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } - void allow_block_proof_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - promise.set_result(false); - } - void allow_block_proof_link_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - promise.set_result(false); - } - void allow_block_candidate_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } - void allow_block_info_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } void archive(BlockHandle handle, td::Promise promise) override { td::actor::send_closure(db_, &Db::archive, std::move(handle), std::move(promise)); } diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 486c185f7..b2bd9250b 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -406,34 +406,9 @@ class ValidatorManagerImpl : public ValidatorManager { void update_gc_block_handle(BlockHandle handle, td::Promise promise) override { promise.set_value(td::Unit()); } - void allow_block_data_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - promise.set_result(false); - } void allow_block_state_gc(BlockIdExt block_id, td::Promise promise) override { promise.set_result(false); } - void allow_zero_state_file_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } - void allow_persistent_state_file_gc(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override { - promise.set_result(false); - } - void allow_block_signatures_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } - void allow_block_proof_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - promise.set_result(false); - } - void allow_block_proof_link_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - promise.set_result(false); - } - void allow_block_candidate_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } - void allow_block_info_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } void archive(BlockHandle handle, td::Promise promise) override { UNREACHABLE(); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 12702ddd8..85f3a30ab 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2374,7 +2374,7 @@ void ValidatorManagerImpl::try_advance_gc_masterchain_block() { gc_masterchain_handle_->id().id.seqno < last_masterchain_state_->last_key_block_id().seqno() && gc_masterchain_handle_->id().id.seqno < min_confirmed_masterchain_seqno_ && gc_masterchain_handle_->id().id.seqno < state_serializer_masterchain_seqno_ && - gc_masterchain_state_->get_unix_time() < td::Clocks::system() - state_ttl()) { + (double)gc_masterchain_state_->get_unix_time() < td::Clocks::system() - state_ttl()) { gc_advancing_ = true; auto block_id = gc_masterchain_handle_->one_next(true); @@ -2386,100 +2386,6 @@ void ValidatorManagerImpl::try_advance_gc_masterchain_block() { } } -void ValidatorManagerImpl::allow_persistent_state_file_gc(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { - if (!gc_masterchain_handle_) { - promise.set_result(false); - return; - } - if (masterchain_block_id.seqno() == 0) { - promise.set_result(false); - return; - } - if (masterchain_block_id.seqno() >= gc_masterchain_handle_->id().seqno()) { - promise.set_result(false); - return; - } - auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { - R.ensure(); - auto handle = R.move_as_ok(); - CHECK(handle->is_key_block()); - promise.set_result(ValidatorManager::persistent_state_ttl(handle->unix_time()) < td::Clocks::system()); - }); - get_block_handle(masterchain_block_id, false, std::move(P)); -} - -void ValidatorManagerImpl::allow_archive(BlockIdExt block_id, td::Promise promise) { - /*if (!gc_masterchain_handle_) { - promise.set_result(false); - return; - } - if (!block_id.is_masterchain()) { - if (!gc_masterchain_state_->workchain_is_active(block_id.id.workchain)) { - promise.set_result(false); - return; - } - bool found = false; - auto S = gc_masterchain_state_->get_shard_from_config(block_id.shard_full()); - if (S.not_null()) { - if (block_id.id.seqno >= S->top_block_id().id.seqno) { - promise.set_result(false); - return; - } - found = true; - } else { - auto shards = gc_masterchain_state_->get_shards(); - for (auto shard : shards) { - if (shard_intersects(shard->shard(), block_id.shard_full())) { - if (block_id.id.seqno >= shard->top_block_id().id.seqno) { - promise.set_result(false); - return; - } - found = true; - } - } - } - CHECK(found); - } else { - if (block_id.id.seqno >= gc_masterchain_handle_->id().id.seqno) { - promise.set_result(false); - return; - } - } - auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - } else { - promise.set_result(true); - } - }); - td::actor::send_closure(db_, &Db::archive, block_id, std::move(P));*/ - promise.set_result(false); -} - -void ValidatorManagerImpl::allow_delete(BlockIdExt block_id, td::Promise promise) { - auto key_ttl = td::Clocks::system() - opts_->key_proof_ttl(); - auto ttl = td::Clocks::system() - opts_->archive_ttl(); - auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), promise = std::move(promise), ttl, key_ttl](td::Result R) mutable { - if (R.is_error()) { - promise.set_result(true); - return; - } - auto handle = R.move_as_ok(); - if (!handle->inited_unix_time()) { - promise.set_result(true); - return; - } - if (!handle->inited_is_key_block() || !handle->is_key_block()) { - promise.set_result(handle->unix_time() <= ttl); - } else { - promise.set_result(handle->unix_time() <= key_ttl); - } - }); - get_block_handle(block_id, false, std::move(P)); -} - void ValidatorManagerImpl::allow_block_state_gc(BlockIdExt block_id, td::Promise promise) { if (!gc_masterchain_handle_) { promise.set_result(false); @@ -2508,27 +2414,6 @@ void ValidatorManagerImpl::allow_block_state_gc(BlockIdExt block_id, td::Promise UNREACHABLE(); } -void ValidatorManagerImpl::allow_block_info_gc(BlockIdExt block_id, td::Promise promise) { - auto P = - td::PromiseCreator::lambda([db = db_.get(), promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_result(false); - } else { - auto handle = R.move_as_ok(); - if (!handle->moved_to_archive() || !handle->is_applied()) { - promise.set_result(false); - } else { - auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { - R.ensure(); - promise.set_result(true); - }); - td::actor::send_closure(db, &Db::store_block_handle, handle, std::move(P)); - } - } - }); - get_block_handle(block_id, false, std::move(P)); -} - void ValidatorManagerImpl::got_next_gc_masterchain_handle(BlockHandle handle) { CHECK(gc_advancing_); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { @@ -2610,7 +2495,7 @@ void ValidatorManagerImpl::alarm() { alarm_timestamp() = td::Timestamp::in(1.0); if (shard_client_handle_ && gc_masterchain_handle_) { td::actor::send_closure(db_, &Db::run_gc, shard_client_handle_->unix_time(), gc_masterchain_handle_->unix_time(), - static_cast(opts_->archive_ttl())); + opts_->archive_ttl()); } if (log_status_at_.is_in_past()) { if (last_masterchain_block_handle_) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 347b8f7ef..c406d36b0 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -568,30 +568,7 @@ class ValidatorManagerImpl : public ValidatorManager { } public: - void allow_delete(BlockIdExt block_id, td::Promise promise); - void allow_archive(BlockIdExt block_id, td::Promise promise); - void allow_block_data_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - allow_archive(block_id, std::move(promise)); - } void allow_block_state_gc(BlockIdExt block_id, td::Promise promise) override; - void allow_zero_state_file_gc(BlockIdExt block_id, td::Promise promise) override { - promise.set_result(false); - } - void allow_persistent_state_file_gc(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) override; - void allow_block_signatures_gc(BlockIdExt block_id, td::Promise promise) override { - allow_archive(block_id, std::move(promise)); - } - void allow_block_proof_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - allow_archive(block_id, std::move(promise)); - } - void allow_block_proof_link_gc(BlockIdExt block_id, bool is_archive, td::Promise promise) override { - allow_archive(block_id, std::move(promise)); - } - void allow_block_candidate_gc(BlockIdExt block_id, td::Promise promise) override { - allow_block_state_gc(block_id, std::move(promise)); - } - void allow_block_info_gc(BlockIdExt block_id, td::Promise promise) override; void archive(BlockHandle handle, td::Promise promise) override { td::actor::send_closure(db_, &Db::archive, std::move(handle), std::move(promise)); } @@ -716,9 +693,6 @@ class ValidatorManagerImpl : public ValidatorManager { double state_ttl() const { return opts_->state_ttl(); } - double block_ttl() const { - return opts_->block_ttl(); - } double max_mempool_num() const { return opts_->max_mempool_num(); } From c3160e37f4d445a871bcb9a52e4cac3557023e35 Mon Sep 17 00:00:00 2001 From: Marat <98183742+dungeon-master-666@users.noreply.github.com> Date: Sat, 5 Apr 2025 09:47:38 +0200 Subject: [PATCH 197/388] Enable bloom filters for CellDB (#1523) * enable bloom filters for celldb * 2 level index and filter for state ttl >= 30 days * set block cache size to 16gb in case of 2 level index * fix rebase artifacts * refactor setting block cache size --- tddb/td/db/RocksDb.cpp | 12 ++++++++++-- tddb/td/db/RocksDb.h | 2 ++ validator-engine/validator-engine.cpp | 7 +++++++ validator-engine/validator-engine.hpp | 4 ++++ validator/db/celldb.cpp | 7 +++++++ validator/validator-options.hpp | 7 +++++++ validator/validator.h | 2 ++ 7 files changed, 39 insertions(+), 2 deletions(-) diff --git a/tddb/td/db/RocksDb.cpp b/tddb/td/db/RocksDb.cpp index b56f3b145..d9040b695 100644 --- a/tddb/td/db/RocksDb.cpp +++ b/tddb/td/db/RocksDb.cpp @@ -24,10 +24,9 @@ #include "rocksdb/write_batch.h" #include "rocksdb/utilities/optimistic_transaction_db.h" #include "rocksdb/utilities/transaction.h" +#include "rocksdb/filter_policy.h" #include "td/utils/misc.h" -#include - namespace td { namespace { static Status from_rocksdb(const rocksdb::Status &status) { @@ -81,6 +80,15 @@ Result RocksDb::open(std::string path, RocksDbOptions options) { } else { table_options.block_cache = options.block_cache; } + if (options.enable_bloom_filter) { + table_options.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false)); + if (options.two_level_index_and_filter) { + table_options.index_type = rocksdb::BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch; + table_options.partition_filters = true; + table_options.cache_index_and_filter_blocks = true; + table_options.pin_l0_filter_and_index_blocks_in_cache = true; + } + } db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); // table_options.block_align = true; diff --git a/tddb/td/db/RocksDb.h b/tddb/td/db/RocksDb.h index d24a20dd7..71ece1e2a 100644 --- a/tddb/td/db/RocksDb.h +++ b/tddb/td/db/RocksDb.h @@ -75,6 +75,8 @@ struct RocksDbOptions { bool use_direct_reads = false; bool no_block_cache = false; + bool enable_bloom_filter = false; + bool two_level_index_and_filter = false; }; class RocksDb : public KeyValue { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 4b4e47348..945d5456f 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1470,6 +1470,7 @@ td::Status ValidatorEngine::load_global_config() { validator_options_.write().set_celldb_compress_depth(celldb_compress_depth_); validator_options_.write().set_celldb_in_memory(celldb_in_memory_); validator_options_.write().set_celldb_v2(celldb_v2_); + validator_options_.write().set_celldb_disable_bloom_filter(celldb_disable_bloom_filter_); validator_options_.write().set_max_open_archive_files(max_open_archive_files_); validator_options_.write().set_archive_preload_period(archive_preload_period_); validator_options_.write().set_disable_rocksdb_stats(disable_rocksdb_stats_); @@ -4538,6 +4539,12 @@ int main(int argc, char *argv[]) { [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_v2, true); }); }); + p.add_option( + '\0', "celldb-disable-bloom-filter", + "disable using bloom filter in CellDb. Enabled bloom filter reduces read latency, but increases memory usage", + [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_disable_bloom_filter, true); }); + }); p.add_checked_option( '\0', "catchain-max-block-delay", "delay before creating a new catchain block, in seconds (default: 0.4)", [&](td::Slice s) -> td::Status { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 5a1db7f3f..77030d68b 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -219,6 +219,7 @@ class ValidatorEngine : public td::actor::Actor { bool celldb_preload_all_ = false; bool celldb_in_memory_ = false; bool celldb_v2_ = false; + bool celldb_disable_bloom_filter_ = false; td::optional catchain_max_block_delay_, catchain_max_block_delay_slow_; bool read_config_ = false; bool started_keyring_ = false; @@ -315,6 +316,9 @@ class ValidatorEngine : public td::actor::Actor { void set_celldb_v2(bool value) { celldb_v2_ = value; } + void set_celldb_disable_bloom_filter(bool value) { + celldb_disable_bloom_filter_ = value; + } void set_catchain_max_block_delay(double value) { catchain_max_block_delay_ = value; } diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 90c659cc4..c67203f08 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -227,6 +227,13 @@ void CellDbIn::start_up() { LOG(WARNING) << "Using V1 DynamicBagOfCells with options " << *boc_v1_options; } + db_options.enable_bloom_filter = !opts_->get_celldb_disable_bloom_filter(); + db_options.two_level_index_and_filter = db_options.enable_bloom_filter + && opts_->state_ttl() >= 60 * 60 * 24 * 30; // 30 days + if (db_options.two_level_index_and_filter && !opts_->get_celldb_in_memory()) { + o_celldb_cache_size = std::max(o_celldb_cache_size ? o_celldb_cache_size.value() : 0UL, 16UL << 30); + } + if (o_celldb_cache_size) { db_options.block_cache = td::RocksDb::create_cache(o_celldb_cache_size.value()); LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(o_celldb_cache_size.value()); diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 45b8d7ec2..55552635c 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -142,6 +142,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool get_celldb_v2() const override { return celldb_v2_; } + bool get_celldb_disable_bloom_filter() const override { + return celldb_disable_bloom_filter_; + } td::optional get_catchain_max_block_delay() const override { return catchain_max_block_delay_; } @@ -243,6 +246,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_celldb_v2(bool value) override { celldb_v2_ = value; } + void set_celldb_disable_bloom_filter(bool value) override { + celldb_disable_bloom_filter_ = value; + } void set_catchain_max_block_delay(double value) override { catchain_max_block_delay_ = value; } @@ -311,6 +317,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool celldb_preload_all_ = false; bool celldb_in_memory_ = false; bool celldb_v2_ = false; + bool celldb_disable_bloom_filter_ = false; td::optional catchain_max_block_delay_, catchain_max_block_delay_slow_; bool state_serializer_enabled_ = true; td::Ref collator_options_{true}; diff --git a/validator/validator.h b/validator/validator.h index 9e1dba7db..7868bc16f 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -112,6 +112,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual td::optional get_celldb_cache_size() const = 0; virtual bool get_celldb_direct_io() const = 0; virtual bool get_celldb_preload_all() const = 0; + virtual bool get_celldb_disable_bloom_filter() const = 0; virtual td::optional get_catchain_max_block_delay() const = 0; virtual td::optional get_catchain_max_block_delay_slow() const = 0; virtual bool get_state_serializer_enabled() const = 0; @@ -146,6 +147,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_celldb_preload_all(bool value) = 0; virtual void set_celldb_in_memory(bool value) = 0; virtual void set_celldb_v2(bool value) = 0; + virtual void set_celldb_disable_bloom_filter(bool value) = 0; virtual void set_catchain_max_block_delay(double value) = 0; virtual void set_catchain_max_block_delay_slow(double value) = 0; virtual void set_state_serializer_enabled(bool value) = 0; From 724ec937208a1a3049773b451541bd0dbad2abbd Mon Sep 17 00:00:00 2001 From: Marat <98183742+dungeon-master-666@users.noreply.github.com> Date: Sat, 5 Apr 2025 13:50:25 +0200 Subject: [PATCH 198/388] Speed up LargeBocSerializer with bulk cells reading (#1533) * serialize boc with bfs (with multiget) * revert in-memory implementation * std_boc_serialize_to_file_large -> boc_serialize_to_file_large * provide load_bulk as default and for celldbv2 * remove excessive check --- crypto/test/test-db.cpp | 7 +- crypto/vm/boc.cpp | 6 +- crypto/vm/boc.h | 19 ++- crypto/vm/db/CellStorage.cpp | 22 +++ crypto/vm/db/CellStorage.h | 1 + crypto/vm/db/DynamicBagOfCellsDb.cpp | 46 ++++-- crypto/vm/db/DynamicBagOfCellsDb.h | 2 + crypto/vm/db/DynamicBagOfCellsDbV2.cpp | 18 +++ crypto/vm/db/InMemoryBagOfCellsDb.cpp | 13 ++ crypto/vm/large-boc-serializer.cpp | 214 ++++++++++++++++++------- tddb/td/db/KeyValue.h | 20 ++- tddb/td/db/MemoryKeyValue.cpp | 12 ++ tddb/td/db/MemoryKeyValue.h | 1 + tddb/td/db/RocksDb.cpp | 34 ++++ tddb/td/db/RocksDb.h | 1 + validator/state-serializer.cpp | 37 ++++- 16 files changed, 367 insertions(+), 86 deletions(-) diff --git a/crypto/test/test-db.cpp b/crypto/test/test-db.cpp index 2b77e7118..47d833d56 100644 --- a/crypto/test/test-db.cpp +++ b/crypto/test/test-db.cpp @@ -2991,10 +2991,13 @@ TEST(TonDb, LargeBocSerializer) { td::unlink(path).ignore(); fd = td::FileFd::open(path, td::FileFd::Flags::Create | td::FileFd::Flags::Truncate | td::FileFd::Flags::Write) .move_as_ok(); - std_boc_serialize_to_file_large(dboc->get_cell_db_reader(), root->get_hash(), fd, 31); + boc_serialize_to_file_large(dboc->get_cell_db_reader(), root->get_hash(), fd, 31); fd.close(); auto b = td::read_file_str(path).move_as_ok(); - CHECK(a == b); + + auto a_cell = vm::deserialize_boc(td::BufferSlice(a)); + auto b_cell = vm::deserialize_boc(td::BufferSlice(b)); + ASSERT_EQ(a_cell->get_hash(), b_cell->get_hash()); } TEST(TonDb, DoNotMakeListsPrunned) { diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index 6377674d7..3aef7dd10 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -209,7 +209,7 @@ td::Result BagOfCells::import_cell(td::Ref cell, int depth) { return td::Status::Error("error while importing a cell into a bag of cells: cell is null"); } if (logger_ptr_) { - TRY_STATUS(logger_ptr_->on_cell_processed()); + TRY_STATUS(logger_ptr_->on_cells_processed(1)); } auto it = cells.find(cell->get_hash()); if (it != cells.end()) { @@ -555,7 +555,7 @@ td::Result BagOfCells::serialize_to_impl(WriterT& writer, int mode) } store_offset(fixed_offset); if (logger_ptr_) { - TRY_STATUS(logger_ptr_->on_cell_processed()); + TRY_STATUS(logger_ptr_->on_cells_processed(1)); } } if (logger_ptr_) { @@ -588,7 +588,7 @@ td::Result BagOfCells::serialize_to_impl(WriterT& writer, int mode) } // std::cerr << std::endl; if (logger_ptr_) { - TRY_STATUS(logger_ptr_->on_cell_processed()); + TRY_STATUS(logger_ptr_->on_cells_processed(1)); } } writer.chk(); diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index 17e7eb69d..4f3fe5fcf 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -210,6 +210,7 @@ class BagOfCellsLogger { void start_stage(std::string stage) { log_speed_at_ = td::Timestamp::in(LOG_SPEED_PERIOD); + last_speed_log_ = td::Timestamp::now(); processed_cells_ = 0; timer_ = {}; stage_ = std::move(stage); @@ -217,15 +218,19 @@ class BagOfCellsLogger { void finish_stage(td::Slice desc) { LOG(ERROR) << "serializer: " << stage_ << " took " << timer_.elapsed() << "s, " << desc; } - td::Status on_cell_processed() { - ++processed_cells_; - if (processed_cells_ % 1000 == 0) { + td::Status on_cells_processed(size_t count) { + processed_cells_ += count; + if (processed_cells_ / 1000 > last_token_check_) { TRY_STATUS(cancellation_token_.check()); + last_token_check_ = processed_cells_ / 1000; } if (log_speed_at_.is_in_past()) { - log_speed_at_ += LOG_SPEED_PERIOD; - LOG(WARNING) << "serializer: " << stage_ << " " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s"; + double period = td::Timestamp::now().at() - last_speed_log_.at(); + + LOG(WARNING) << "serializer: " << stage_ << " " << (double)processed_cells_ / period << " cells/s"; processed_cells_ = 0; + last_speed_log_ = td::Timestamp::now(); + log_speed_at_ = td::Timestamp::in(LOG_SPEED_PERIOD); } return td::Status::OK(); } @@ -236,6 +241,8 @@ class BagOfCellsLogger { td::CancellationToken cancellation_token_; td::Timestamp log_speed_at_; size_t processed_cells_ = 0; + size_t last_token_check_ = 0; + td::Timestamp last_speed_log_; static constexpr double LOG_SPEED_PERIOD = 120.0; }; class BagOfCells { @@ -390,7 +397,7 @@ td::Result std_boc_serialize_multi(std::vector> root, td::Status std_boc_serialize_to_file(Ref root, td::FileFd& fd, int mode = 0, td::CancellationToken cancellation_token = {}); -td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, +td::Status boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, int mode = 0, td::CancellationToken cancellation_token = {}); } // namespace vm diff --git a/crypto/vm/db/CellStorage.cpp b/crypto/vm/db/CellStorage.cpp index a07d85e87..f93e3fa59 100644 --- a/crypto/vm/db/CellStorage.cpp +++ b/crypto/vm/db/CellStorage.cpp @@ -177,6 +177,28 @@ td::Result CellLoader::load(td::Slice hash, bool need_da return res; } +td::Result> CellLoader::load_bulk(td::Span hashes, bool need_data, + ExtCellCreator &ext_cell_creator) { + std::vector values; + TRY_RESULT(get_statuses, reader_->get_multi(hashes, &values)); + std::vector res; + res.reserve(hashes.size()); + for (size_t i = 0; i < hashes.size(); i++) { + auto get_status = get_statuses[i]; + if (get_status != KeyValue::GetStatus::Ok) { + DCHECK(get_status == KeyValue::GetStatus::NotFound); + res.push_back(LoadResult{}); + continue; + } + TRY_RESULT(load_res, load(hashes[i], values[i], need_data, ext_cell_creator)); + if (on_load_callback_) { + on_load_callback_(load_res); + } + res.push_back(std::move(load_res)); + } + return res; +} + td::Result CellLoader::load(td::Slice hash, td::Slice value, bool need_data, ExtCellCreator &ext_cell_creator) { LoadResult res; diff --git a/crypto/vm/db/CellStorage.h b/crypto/vm/db/CellStorage.h index ca32a8007..7ae586793 100644 --- a/crypto/vm/db/CellStorage.h +++ b/crypto/vm/db/CellStorage.h @@ -49,6 +49,7 @@ class CellLoader { }; CellLoader(std::shared_ptr reader, std::function on_load_callback = {}); td::Result load(td::Slice hash, bool need_data, ExtCellCreator &ext_cell_creator); + td::Result> load_bulk(td::Span hashes, bool need_data, ExtCellCreator &ext_cell_creator); static td::Result load(td::Slice hash, td::Slice value, bool need_data, ExtCellCreator &ext_cell_creator); td::Result load_refcnt(td::Slice hash); // This only loads refcnt_, cell_ == null KeyValueReader &key_value_reader() const { diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp index d6731b039..1a30f76d4 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDb.cpp @@ -138,22 +138,24 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat return td::Status::OK(); } td::Result> load_cell(td::Slice hash) override { - auto info = hash_table_.get_if_exists(hash); - if (info && info->sync_with_db) { - TRY_RESULT(loaded_cell, info->cell->load_cell()); - return std::move(loaded_cell.data_cell); - } - TRY_RESULT(res, loader_->load(hash, true, *this)); - if (res.status != CellLoader::LoadResult::Ok) { - return td::Status::Error("cell not found"); - } - Ref cell = res.cell(); - hash_table_.apply(hash, [&](CellInfo &info) { update_cell_info_loaded(info, hash, std::move(res)); }); - return cell; + TRY_RESULT(loaded_cell, get_cell_info_force(hash).cell->load_cell()); + return std::move(loaded_cell.data_cell); } td::Result> load_root(td::Slice hash) override { return load_cell(hash); } + td::Result>> load_bulk(td::Span hashes) override { + std::vector> result; + result.reserve(hashes.size()); + for (auto &hash : hashes) { + auto cell = load_cell(hash); + if (cell.is_error()) { + return cell.move_as_error(); + } + result.push_back(cell.move_as_ok()); + } + return result; + } td::Result> load_root_thread_safe(td::Slice hash) const override { return td::Status::Error("Not implemented"); } @@ -193,6 +195,9 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat promise->set_result(std::move(cell)); }); } + CellInfo &get_cell_info_force(td::Slice hash) { + return hash_table_.apply(hash, [&](CellInfo &info) { update_cell_info_force(info, hash); }); + } CellInfo &get_cell_info_lazy(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) { return hash_table_.apply(hash.substr(hash.size() - Cell::hash_bytes), [&](CellInfo &info) { update_cell_info_lazy(info, level_mask, hash, depth); }); @@ -383,6 +388,23 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat return std::move(load_result.cell()); } + td::Result>> load_bulk(td::Span hashes) override { + if (db_) { + return db_->load_bulk(hashes); + } + TRY_RESULT(load_result, cell_loader_->load_bulk(hashes, true, *this)); + + std::vector> res; + res.reserve(load_result.size()); + for (auto &load_res : load_result) { + if (load_res.status != CellLoader::LoadResult::Ok) { + return td::Status::Error("cell not found"); + } + res.push_back(std::move(load_res.cell())); + } + return res; + } + private: static td::NamedThreadSafeCounter::CounterRef get_thread_safe_counter() { static auto res = td::NamedThreadSafeCounter::get_default().get_counter("DynamicBagOfCellsDbLoader"); diff --git a/crypto/vm/db/DynamicBagOfCellsDb.h b/crypto/vm/db/DynamicBagOfCellsDb.h index 82028f3fe..ec0aee9c1 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.h +++ b/crypto/vm/db/DynamicBagOfCellsDb.h @@ -45,6 +45,7 @@ class CellDbReader { public: virtual ~CellDbReader() = default; virtual td::Result> load_cell(td::Slice hash) = 0; + virtual td::Result>> load_bulk(td::Span hashes) = 0; }; class DynamicBagOfCellsDb { @@ -57,6 +58,7 @@ class DynamicBagOfCellsDb { virtual td::Status meta_erase(td::Slice key) = 0; virtual td::Result> load_cell(td::Slice hash) = 0; + virtual td::Result>> load_bulk(td::Span hashes) = 0; virtual td::Result> load_root(td::Slice hash) = 0; virtual td::Result> load_root_thread_safe(td::Slice hash) const = 0; virtual td::Result>> load_known_roots() const { diff --git a/crypto/vm/db/DynamicBagOfCellsDbV2.cpp b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp index eff74e214..ecf3b76cb 100644 --- a/crypto/vm/db/DynamicBagOfCellsDbV2.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp @@ -811,6 +811,10 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { td::Result> load_root(td::Slice hash) override { return load_cell(hash); } + td::Result>> load_bulk(td::Span hashes) override { + CHECK(cell_db_reader_); + return cell_db_reader_->load_bulk(hashes); + } td::Result> load_root_thread_safe(td::Slice hash) const override { // TODO: it is better to use AtomicRef, or atomic shared pointer // But to use AtomicRef we need a little refactoring @@ -1102,6 +1106,20 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { return load_cell_slow_path(hash); } + td::Result>> load_bulk(td::Span hashes) override { + // thread safe function + std::vector> result; + result.reserve(hashes.size()); + for (auto &hash : hashes) { + auto maybe_cell = load_cell(hash); + if (maybe_cell.is_error()) { + return maybe_cell.move_as_error(); + } + result.push_back(maybe_cell.move_as_ok()); + } + return result; + } + td::Result> load_ext_cell(Ref ext_cell) override { // thread safe function. // Called by external cell diff --git a/crypto/vm/db/InMemoryBagOfCellsDb.cpp b/crypto/vm/db/InMemoryBagOfCellsDb.cpp index 44be4ca8b..829ed38d8 100644 --- a/crypto/vm/db/InMemoryBagOfCellsDb.cpp +++ b/crypto/vm/db/InMemoryBagOfCellsDb.cpp @@ -454,6 +454,16 @@ class CellStorage { return td::Status::Error("not found"); } + td::Result>> load_bulk(td::Span hashes) const { + std::vector> res; + res.reserve(hashes.size()); + for (auto &hash : hashes) { + TRY_RESULT(cell, load_cell(hash)); + res.push_back(std::move(cell)); + } + return res; + } + td::Result> load_root_local(const CellHash &hash) const { auto lock = local_access_.lock(); if (auto it = local_roots_.find(hash); it != local_roots_.end()) { @@ -839,6 +849,9 @@ class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { td::Result> load_root(td::Slice hash) override { return storage_->load_root_local(CellHash::from_slice(hash)); } + td::Result>> load_bulk(td::Span hashes) override { + return storage_->load_bulk(td::transform(hashes, [](auto &hash) { return CellHash::from_slice(hash); })); + } td::Result> load_root_thread_safe(td::Slice hash) const override { return storage_->load_root_shared(CellHash::from_slice(hash)); } diff --git a/crypto/vm/large-boc-serializer.cpp b/crypto/vm/large-boc-serializer.cpp index d209c88ed..839e6235a 100644 --- a/crypto/vm/large-boc-serializer.cpp +++ b/crypto/vm/large-boc-serializer.cpp @@ -32,6 +32,7 @@ namespace { class LargeBocSerializer { public: using Hash = Cell::Hash; + constexpr static int load_batch_size = 4'000'000; explicit LargeBocSerializer(std::shared_ptr reader) : reader(std::move(reader)) { } @@ -46,7 +47,6 @@ class LargeBocSerializer { private: std::shared_ptr reader; struct CellInfo { - Cell::Hash hash; std::array ref_idx; int idx; unsigned short serialized_size; @@ -95,6 +95,8 @@ void LargeBocSerializer::add_root(Hash root) { roots.emplace_back(root, -1); } +// Unlike crypto/vm/boc.cpp this implementation does not load all cells into memory +// and traverses them in BFS order to utilize bulk load of cells on the same level. td::Status LargeBocSerializer::import_cells() { if (logger_ptr_) { logger_ptr_->start_stage("import_cells"); @@ -111,46 +113,124 @@ td::Status LargeBocSerializer::import_cells() { return td::Status::OK(); } -td::Result LargeBocSerializer::import_cell(Hash hash, int depth) { - if (depth > Cell::max_depth) { - return td::Status::Error("error while importing a cell into a bag of cells: cell depth too large"); +td::Result LargeBocSerializer::import_cell(Hash root_hash, int root_depth) { + const int start_ind = cell_count; + td::HashMap> current_depth_hashes; + + auto existing_it = cells.find(root_hash); + if (existing_it != cells.end()) { + existing_it->second.should_cache = true; + } else { + current_depth_hashes.emplace(root_hash, std::make_pair(cell_count, false)); + } + int current_depth = root_depth; + int next_child_idx = cell_count + 1; + while (!current_depth_hashes.empty()) { + if (current_depth > Cell::max_depth) { + return td::Status::Error("error while importing a cell into a bag of cells: cell depth too large"); + } + + cell_list.resize(cell_list.size() + current_depth_hashes.size()); + td::HashMap> next_depth_hashes; + auto batch_start = current_depth_hashes.begin(); + while (batch_start != current_depth_hashes.end()) { + std::vector batch_hashes; + batch_hashes.reserve(load_batch_size); + std::vector*> batch_idxs_should_cache; + batch_idxs_should_cache.reserve(load_batch_size); + + while (batch_hashes.size() < load_batch_size && batch_start != current_depth_hashes.end()) { + batch_hashes.push_back(batch_start->first.as_slice()); + batch_idxs_should_cache.push_back(&batch_start->second); + ++batch_start; + } + + TRY_RESULT_PREFIX(loaded_results, reader->load_bulk(batch_hashes), + "error while importing a cell into a bag of cells: "); + DCHECK(loaded_results.size() == batch_hashes.size()); + + for (size_t i = 0; i < loaded_results.size(); ++i) { + auto& cell = loaded_results[i]; + + if (cell->get_virtualization() != 0) { + return td::Status::Error( + "error while importing a cell into a bag of cells: cell has non-zero virtualization level"); + } + + const auto hash = cell->get_hash(); + CellSlice cs(std::move(cell)); + + DCHECK(cs.size_refs() <= 4); + std::array refs{-1, -1, -1, -1}; + for (unsigned j = 0; j < cs.size_refs(); j++) { + auto child = cs.prefetch_ref(j); + const auto child_hash = child->get_hash(); + + auto existing_global_it = cells.find(child_hash); + if (existing_global_it != cells.end()) { + existing_global_it->second.should_cache = true; + refs[j] = existing_global_it->second.idx; + continue; + } + auto current_depth_it = current_depth_hashes.find(child_hash); + if (current_depth_it != current_depth_hashes.end()) { + current_depth_it->second.second = true; + refs[j] = current_depth_it->second.first; + continue; + } + auto next_depth_it = next_depth_hashes.find(child_hash); + if (next_depth_it != next_depth_hashes.end()) { + next_depth_it->second.second = true; + refs[j] = next_depth_it->second.first; + continue; + } + auto res = next_depth_hashes.emplace(child_hash, std::make_pair(next_child_idx, false)); + refs[j] = next_child_idx++; + } + + auto dc = cs.move_as_loaded_cell().data_cell; + auto idx_should_cache = batch_idxs_should_cache[i]; + auto res = cells.emplace(hash, CellInfo(idx_should_cache->first, std::move(refs))); + DCHECK(res.second); + cell_list[idx_should_cache->first] = &*res.first; + CellInfo& dc_info = res.first->second; + dc_info.should_cache = idx_should_cache->second; + dc_info.hcnt = static_cast(dc->get_level_mask().get_hashes_count()); + DCHECK(dc_info.hcnt <= 4); + dc_info.wt = 0; // will be calculated after traversing + TRY_RESULT(serialized_size, td::narrow_cast_safe(dc->get_serialized_size())); + data_bytes += dc_info.serialized_size = serialized_size; + cell_count++; + } + if (logger_ptr_) { + TRY_STATUS(logger_ptr_->on_cells_processed(batch_hashes.size())); + } + } + + current_depth_hashes = std::move(next_depth_hashes); + next_depth_hashes.clear(); + current_depth++; + } + DCHECK(next_child_idx == cell_count); + + for (int idx = cell_count - 1; idx >= start_ind; --idx) { + CellInfo& cell_info = cell_list[idx]->second; + + unsigned sum_child_wt = 1; + for (size_t j = 0; j < cell_info.ref_idx.size(); ++j) { + int child_idx = cell_info.ref_idx[j]; + if (child_idx == -1) { + continue; + } + sum_child_wt += cell_list[child_idx]->second.wt; + ++int_refs; + } + cell_info.wt = static_cast(std::min(0xffU, sum_child_wt)); } - if (logger_ptr_) { - TRY_STATUS(logger_ptr_->on_cell_processed()); - } - auto it = cells.find(hash); - if (it != cells.end()) { - it->second.should_cache = true; - return it->second.idx; - } - TRY_RESULT(cell, reader->load_cell(hash.as_slice())); - if (cell->get_virtualization() != 0) { - return td::Status::Error( - "error while importing a cell into a bag of cells: cell has non-zero virtualization level"); - } - CellSlice cs(std::move(cell)); - std::array refs; - std::fill(refs.begin(), refs.end(), -1); - DCHECK(cs.size_refs() <= 4); - unsigned sum_child_wt = 1; - for (unsigned i = 0; i < cs.size_refs(); i++) { - TRY_RESULT(ref, import_cell(cs.prefetch_ref(i)->get_hash(), depth + 1)); - refs[i] = ref; - sum_child_wt += cell_list[ref]->second.wt; - ++int_refs; - } - auto dc = cs.move_as_loaded_cell().data_cell; - auto res = cells.emplace(hash, CellInfo(cell_count, refs)); - DCHECK(res.second); - cell_list.push_back(&*res.first); - CellInfo& dc_info = res.first->second; - dc_info.wt = (unsigned char)std::min(0xffU, sum_child_wt); - unsigned hcnt = dc->get_level_mask().get_hashes_count(); - DCHECK(hcnt <= 4); - dc_info.hcnt = (unsigned char)hcnt; - TRY_RESULT(serialized_size, td::narrow_cast_safe(dc->get_serialized_size())); - data_bytes += dc_info.serialized_size = serialized_size; - return cell_count++; + + auto root_it = cells.find(root_hash); + DCHECK(root_it != cells.end()); + return root_it->second.idx; } void LargeBocSerializer::reorder_cells() { @@ -386,7 +466,7 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { } store_offset(fixed_offset); if (logger_ptr_) { - TRY_STATUS(logger_ptr_->on_cell_processed()); + TRY_STATUS(logger_ptr_->on_cells_processed(1)); } } DCHECK(offs == info.data_size); @@ -399,26 +479,42 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { if (logger_ptr_) { logger_ptr_->start_stage("serialize"); } - for (int i = 0; i < cell_count; ++i) { - auto hash = cell_list[cell_count - 1 - i]->first; - const auto& dc_info = cell_list[cell_count - 1 - i]->second; - TRY_RESULT(dc, reader->load_cell(hash.as_slice())); - bool with_hash = (mode & Mode::WithIntHashes) && !dc_info.wt; - if (dc_info.is_root_cell && (mode & Mode::WithTopHash)) { - with_hash = true; + for (int batch_start = 0; batch_start < cell_count; batch_start += load_batch_size) { + int batch_end = std::min(batch_start + static_cast(load_batch_size), cell_count); + + std::vector batch_hashes; + batch_hashes.reserve(batch_end - batch_start); + for (int i = batch_start; i < batch_end; ++i) { + int cell_index = cell_count - 1 - i; + batch_hashes.push_back(cell_list[cell_index]->first.as_slice()); } - unsigned char buf[256]; - int s = dc->serialize(buf, 256, with_hash); - writer.store_bytes(buf, s); - DCHECK(dc->size_refs() == dc_info.get_ref_num()); - unsigned ref_num = dc_info.get_ref_num(); - for (unsigned j = 0; j < ref_num; ++j) { - int k = cell_count - 1 - dc_info.ref_idx[j]; - DCHECK(k > i && k < cell_count); - store_ref(k); + + TRY_RESULT(batch_cells, reader->load_bulk(std::move(batch_hashes))); + + for (int i = batch_start; i < batch_end; ++i) { + int idx_in_batch = i - batch_start; + int cell_index = cell_count - 1 - i; + + const auto& dc_info = cell_list[cell_index]->second; + auto& dc = batch_cells[idx_in_batch]; + + bool with_hash = (mode & Mode::WithIntHashes) && !dc_info.wt; + if (dc_info.is_root_cell && (mode & Mode::WithTopHash)) { + with_hash = true; + } + unsigned char buf[256]; + int s = dc->serialize(buf, 256, with_hash); + writer.store_bytes(buf, s); + DCHECK(dc->size_refs() == dc_info.get_ref_num()); + unsigned ref_num = dc_info.get_ref_num(); + for (unsigned j = 0; j < ref_num; ++j) { + int k = cell_count - 1 - dc_info.ref_idx[j]; + DCHECK(k > i && k < cell_count); + store_ref(k); + } } if (logger_ptr_) { - TRY_STATUS(logger_ptr_->on_cell_processed()); + TRY_STATUS(logger_ptr_->on_cells_processed(batch_hashes.size())); } } DCHECK(writer.position() - keep_position == info.data_size); @@ -435,7 +531,7 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { } } // namespace -td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, +td::Status boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, int mode, td::CancellationToken cancellation_token) { td::Timer timer; CHECK(reader != nullptr) diff --git a/tddb/td/db/KeyValue.h b/tddb/td/db/KeyValue.h index c3f83919b..6bbe7df3c 100644 --- a/tddb/td/db/KeyValue.h +++ b/tddb/td/db/KeyValue.h @@ -20,6 +20,7 @@ #include "td/utils/Status.h" #include "td/utils/Time.h" #include "td/utils/logging.h" +#include "td/utils/Span.h" #include "td/utils/ThreadSafeCounter.h" #include namespace td { @@ -59,7 +60,8 @@ class KeyValueReader { virtual ~KeyValueReader() = default; enum class GetStatus : int32 { Ok, NotFound }; - virtual Result get(Slice key, std::string& value) = 0; + virtual Result get(Slice key, std::string &value) = 0; + virtual Result> get_multi(td::Span keys, std::vector *values) = 0; virtual Result count(Slice prefix) = 0; virtual Status for_each(std::function f) { return Status::Error("for_each is not supported"); @@ -77,6 +79,14 @@ class PrefixedKeyValueReader : public KeyValueReader { Result get(Slice key, std::string& value) override { return reader_->get(PSLICE() << prefix_ << key, value); } + Result> get_multi(td::Span keys, std::vector *values) override { + std::vector prefixed_keys; + prefixed_keys.reserve(keys.size()); + for (auto &key : keys) { + prefixed_keys.push_back(PSLICE() << prefix_ << key); + } + return reader_->get_multi(prefixed_keys, values); + } Result count(Slice prefix) override { return reader_->count(PSLICE() << prefix_ << prefix); } @@ -125,6 +135,14 @@ class PrefixedKeyValue : public KeyValue { Result get(Slice key, std::string& value) override { return kv_->get(PSLICE() << prefix_ << key, value); } + Result> get_multi(td::Span keys, std::vector *values) override { + std::vector prefixed_keys; + prefixed_keys.reserve(keys.size()); + for (auto &key : keys) { + prefixed_keys.push_back(PSLICE() << prefix_ << key); + } + return kv_->get_multi(prefixed_keys, values); + } Result count(Slice prefix) override { return kv_->count(PSLICE() << prefix_ << prefix); } diff --git a/tddb/td/db/MemoryKeyValue.cpp b/tddb/td/db/MemoryKeyValue.cpp index 7105f72b9..c79daccea 100644 --- a/tddb/td/db/MemoryKeyValue.cpp +++ b/tddb/td/db/MemoryKeyValue.cpp @@ -19,6 +19,7 @@ #include "td/db/MemoryKeyValue.h" #include "td/utils/format.h" +#include "td/utils/Span.h" namespace td { Result MemoryKeyValue::get(Slice key, std::string &value) { @@ -41,6 +42,17 @@ std::unique_ptr MemoryKeyValue:: return lock(buckets_[bucket_id]); } +Result> MemoryKeyValue::get_multi(td::Span keys, + std::vector *values) { + values->resize(keys.size()); + std::vector res; + res.reserve(keys.size()); + for (size_t i = 0; i < keys.size(); i++) { + res.push_back(get(keys[i], (*values)[i]).move_as_ok()); + } + return res; +} + Status MemoryKeyValue::for_each(std::function f) { for (auto &unlocked_bucket : buckets_) { auto bucket = lock(unlocked_bucket); diff --git a/tddb/td/db/MemoryKeyValue.h b/tddb/td/db/MemoryKeyValue.h index cf896095d..6a5be7866 100644 --- a/tddb/td/db/MemoryKeyValue.h +++ b/tddb/td/db/MemoryKeyValue.h @@ -34,6 +34,7 @@ class MemoryKeyValue : public KeyValue { MemoryKeyValue(std::shared_ptr merger) : merger_(std::move(merger)) { } Result get(Slice key, std::string& value) override; + Result> get_multi(td::Span keys, std::vector *values) override; Status for_each(std::function f) override; Status for_each_in_range(Slice begin, Slice end, std::function f) override; Status set(Slice key, Slice value) override; diff --git a/tddb/td/db/RocksDb.cpp b/tddb/td/db/RocksDb.cpp index d9040b695..660381a31 100644 --- a/tddb/td/db/RocksDb.cpp +++ b/tddb/td/db/RocksDb.cpp @@ -187,6 +187,40 @@ Result RocksDb::get(Slice key, std::string &value) { return from_rocksdb(status); } +Result> RocksDb::get_multi(td::Span keys, std::vector *values) { + std::vector statuses(keys.size()); + std::vector keys_rocksdb; + keys_rocksdb.reserve(keys.size()); + for (auto &key : keys) { + keys_rocksdb.push_back(to_rocksdb(key)); + } + std::vector values_rocksdb(keys.size()); + rocksdb::ReadOptions options; + if (snapshot_) { + options.snapshot = snapshot_.get(); + db_->MultiGet(options, db_->DefaultColumnFamily(), keys_rocksdb.size(), keys_rocksdb.data(), values_rocksdb.data(), statuses.data()); + } else if (transaction_) { + transaction_->MultiGet(options, db_->DefaultColumnFamily(), keys_rocksdb.size(), keys_rocksdb.data(), values_rocksdb.data(), statuses.data()); + } else { + db_->MultiGet(options, db_->DefaultColumnFamily(), keys_rocksdb.size(), keys_rocksdb.data(), values_rocksdb.data(), statuses.data()); + } + std::vector res(statuses.size()); + values->resize(statuses.size()); + for (size_t i = 0; i < statuses.size(); i++) { + auto &status = statuses[i]; + if (status.ok()) { + res[i] = GetStatus::Ok; + values->at(i) = values_rocksdb[i].ToString(); + } else if (status.code() == rocksdb::Status::kNotFound) { + res[i] = GetStatus::NotFound; + values->at(i) = ""; + } else { + return from_rocksdb(status); + } + } + return res; +} + Status RocksDb::set(Slice key, Slice value) { if (write_batch_) { return from_rocksdb(write_batch_->Put(to_rocksdb(key), to_rocksdb(value))); diff --git a/tddb/td/db/RocksDb.h b/tddb/td/db/RocksDb.h index 71ece1e2a..c9fa93e10 100644 --- a/tddb/td/db/RocksDb.h +++ b/tddb/td/db/RocksDb.h @@ -86,6 +86,7 @@ class RocksDb : public KeyValue { static Result open(std::string path, RocksDbOptions options = {}); Result get(Slice key, std::string &value) override; + Result> get_multi(td::Span keys, std::vector *values) override; Status set(Slice key, Slice value) override; Status merge(Slice key, Slice value) override; Status erase(Slice key) override; diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index bc3d7b5e0..b8fd84d17 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -282,8 +282,38 @@ class CachedCellDbReader : public vm::CellDbReader { } return parent_->load_cell(hash); } + td::Result>> load_bulk(td::Span hashes) override { + total_reqs_ += hashes.size(); + if (!cache_) { + ++bulk_reqs_; + return parent_->load_bulk(hashes); + } + std::vector missing_hashes; + std::vector missing_indices; + std::vector> res(hashes.size()); + for (size_t i = 0; i < hashes.size(); i++) { + auto it = cache_->find(hashes[i]); + if (it != cache_->end()) { + ++cached_reqs_; + TRY_RESULT(loaded_cell, (*it)->load_cell()); + res[i] = loaded_cell.data_cell; + continue; + } + missing_hashes.push_back(hashes[i]); + missing_indices.push_back(i); + } + if (missing_hashes.empty()) { + return std::move(res); + } + TRY_RESULT(missing_cells, parent_->load_bulk(missing_hashes)); + for (size_t i = 0; i < missing_indices.size(); i++) { + res[missing_indices[i]] = missing_cells[i]; + } + return res; + }; void print_stats() const { - LOG(WARNING) << "CachedCellDbReader stats : " << total_reqs_ << " reads, " << cached_reqs_ << " cached"; + LOG(WARNING) << "CachedCellDbReader stats : " << total_reqs_ << " reads, " << cached_reqs_ << " cached, " + << bulk_reqs_ << " bulk reqs"; } private: std::shared_ptr parent_; @@ -291,6 +321,7 @@ class CachedCellDbReader : public vm::CellDbReader { td::uint64 total_reqs_ = 0; td::uint64 cached_reqs_ = 0; + td::uint64 bulk_reqs_ = 0; }; void AsyncStateSerializer::PreviousStateCache::prepare_cache(ShardIdFull shard) { @@ -373,7 +404,7 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state previous_state_cache->prepare_cache(shard); } auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache->cache); - auto res = vm::std_boc_serialize_to_file_large(new_cell_db_reader, root->get_hash(), fd, 31, std::move(cancellation_token)); + auto res = vm::boc_serialize_to_file_large(new_cell_db_reader, root->get_hash(), fd, 31, std::move(cancellation_token)); new_cell_db_reader->print_stats(); return res; }; @@ -443,7 +474,7 @@ void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Refprepare_cache(shard); } auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache->cache); - auto res = vm::std_boc_serialize_to_file_large(new_cell_db_reader, root->get_hash(), fd, 31, std::move(cancellation_token)); + auto res = vm::boc_serialize_to_file_large(new_cell_db_reader, root->get_hash(), fd, 31, std::move(cancellation_token)); new_cell_db_reader->print_stats(); return res; }; From 80ecebd5a1157176c0c18f9f29f39d9b497f5c44 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Sat, 5 Apr 2025 15:51:32 +0400 Subject: [PATCH 199/388] Fix failing windows build (#1603) * test builds - testnet * set CMAKE_POLICY_VERSION_MINIMUM for WIN32 too * adjust as per review comment --- .github/workflows/docker-ubuntu-image.yml | 1 - CMakeLists.txt | 7 +++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-ubuntu-image.yml b/.github/workflows/docker-ubuntu-image.yml index 0284dcd54..7a5eee10a 100644 --- a/.github/workflows/docker-ubuntu-image.yml +++ b/.github/workflows/docker-ubuntu-image.yml @@ -5,7 +5,6 @@ on: push: branches: - 'master' - env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} diff --git a/CMakeLists.txt b/CMakeLists.txt index ad1aaf506..b002b2844 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,9 +77,12 @@ endif() if (WIN32) message("Add wingetopt") - add_subdirectory(third-party/wingetopt EXCLUDE_FROM_ALL) + function(wingetopt_scope) + set(CMAKE_POLICY_VERSION_MINIMUM "3.10") + add_subdirectory(third-party/wingetopt EXCLUDE_FROM_ALL) + endfunction() + wingetopt_scope() set(WINGETOPT_FOUND 1) - message(STATUS "Use wingetopt") endif() set(CRC32C_BUILD_TESTS OFF CACHE BOOL "Build CRC32C's unit tests") From e745696757a298594e54d3fedbca604c3eb59a3f Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sat, 5 Apr 2025 16:58:38 +0300 Subject: [PATCH 200/388] [Tolk] Type aliases `type NewName = ` An alias creates a new name for an existing type but remains interchangeable with it: > type UserId = int32; > type MaybeOwnerHash = bytes32?; > type MyStorage = (int, int, slice, cell); Aliases never occur at runtime, they are compile-time only (they exist as types until IR generation). That's why lots of code comparing types use `type->unwrap_alias()` or `try_as`. In stdlib, a special alias was introduced: > type dict = cell?; Functions working with dictionaries use `dict`, improving code clarity. --- crypto/smartcont/tolk-stdlib/common.tolk | 18 +- crypto/smartcont/tolk-stdlib/tvm-dicts.tolk | 140 ++++++------- tolk-tester/tests/constants-tests.tolk | 13 +- tolk-tester/tests/generics-1.tolk | 52 ++++- tolk-tester/tests/indexed-access.tolk | 15 +- tolk-tester/tests/inference-tests.tolk | 2 +- .../tests/invalid-declaration/err-1483.tolk | 8 + .../tests/invalid-semantics/err-4304.tolk | 10 + .../tests/invalid-syntax/err-3482.tolk | 6 + .../tests/invalid-typing/err-6188.tolk | 7 +- .../tests/invalid-typing/err-6337.tolk | 2 +- .../tests/invalid-typing/err-6450.tolk | 6 +- .../tests/invalid-typing/err-6553.tolk | 6 +- .../tests/invalid-typing/err-6580.tolk | 4 +- .../tests/invalid-typing/err-6734.tolk | 6 +- .../tests/invalid-typing/err-6737.tolk | 4 +- tolk-tester/tests/nullable-tensors.tolk | 30 ++- tolk-tester/tests/nullable-types.tolk | 15 +- tolk-tester/tests/smart-cast-tests.tolk | 4 +- tolk-tester/tests/type-aliases-tests.tolk | 189 ++++++++++++++++++ tolk-tester/tests/var-apply-tests.tolk | 34 +++- tolk/abscode.cpp | 2 + tolk/ast-from-tokens.cpp | 19 ++ tolk/ast-stringifier.h | 4 + tolk/ast.cpp | 8 + tolk/ast.h | 20 ++ tolk/fwd-declarations.h | 2 + tolk/generics-helpers.cpp | 8 +- tolk/pipe-ast-to-legacy.cpp | 37 +++- tolk/pipe-check-inferred-types.cpp | 53 ++++- tolk/pipe-infer-types-and-calls.cpp | 32 +-- tolk/pipe-optimize-boolean-expr.cpp | 29 ++- tolk/pipe-register-symbols.cpp | 10 + tolk/pipe-resolve-identifiers.cpp | 41 +++- tolk/smart-casts-cfg.cpp | 15 +- tolk/symtable.cpp | 12 ++ tolk/symtable.h | 13 ++ tolk/type-system.cpp | 163 ++++++++++++++- tolk/type-system.h | 35 ++++ 39 files changed, 909 insertions(+), 165 deletions(-) create mode 100644 tolk-tester/tests/invalid-declaration/err-1483.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4304.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3482.tolk create mode 100644 tolk-tester/tests/type-aliases-tests.tolk diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 0075764b6..dc0ec4a61 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -3,6 +3,12 @@ // More specific functions are required to be imported explicitly, like "@stdlib/tvm-dicts". tolk 0.10 +/// In Tolk v1.x there would be a type `map`. +/// Currently, working with dictionaries is still low-level, with raw cells. +/// But just for clarity, we use "dict" instead of a "cell?" where a cell-dictionary is assumed. +/// Every dictionary object can be null. TVM NULL is essentially "empty dictionary". +type dict = cell?; + /** Tuple manipulation primitives. Elements of a tuple can be of arbitrary type. @@ -144,9 +150,9 @@ fun getMyOriginalBalance(): int /// Same as [getMyOriginalBalance], but returns a tuple: /// `int` — balance in nanotoncoins; -/// `cell` — a dictionary with 32-bit keys representing the balance of "extra currencies". +/// `dict` — a dictionary with 32-bit keys representing the balance of "extra currencies". @pure -fun getMyOriginalBalanceWithExtraCurrencies(): [int, cell?] +fun getMyOriginalBalanceWithExtraCurrencies(): [int, dict] asm "BALANCE"; /// Returns the logical time of the current transaction. @@ -476,12 +482,12 @@ fun getLastBits(self: slice, len: int): slice /// Loads a dictionary (TL HashMapE structure, represented as TVM cell) from a slice. /// Returns `null` if `nothing` constructor is used. @pure -fun loadDict(mutate self: slice): cell? +fun loadDict(mutate self: slice): dict asm( -> 1 0) "LDDICT"; /// Preloads a dictionary (cell) from a slice. @pure -fun preloadDict(self: slice): cell? +fun preloadDict(self: slice): dict asm "PLDDICT"; /// Loads a dictionary as [loadDict], but returns only the remainder of the slice. @@ -556,7 +562,7 @@ fun storeBool(mutate self: builder, x: bool): self /// Stores dictionary (represented by TVM `cell` or `null`) into a builder. /// In other words, stores a `1`-bit and a reference to [c] if [c] is not `null` and `0`-bit otherwise. @pure -fun storeDict(mutate self: builder, c: cell?): self +fun storeDict(mutate self: builder, c: dict): self asm(c self) "STDICT"; /// Stores (Maybe ^Cell) into a builder. @@ -720,7 +726,7 @@ fun reserveToncoinsOnBalance(nanoTonCoins: int, reserveMode: int): void /// Similar to [reserveToncoinsOnBalance], but also accepts a dictionary extraAmount (represented by a cell or null) /// with extra currencies. In this way currencies other than Toncoin can be reserved. -fun reserveExtraCurrenciesOnBalance(nanoTonCoins: int, extraAmount: cell?, reserveMode: int): void +fun reserveExtraCurrenciesOnBalance(nanoTonCoins: int, extraAmount: dict, reserveMode: int): void asm "RAWRESERVEX"; diff --git a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk index d4babd1ef..83fe705c4 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk @@ -13,285 +13,289 @@ tolk 0.10 Every dictionary object (`self` parameter) can be null. TVM NULL is essentially "empty dictionary". */ +/// In @stdlib/common.tolk, there is a type alias: +/// `type dict = cell?;` +/// For clarity, we use "dict" instead of a "cell?" where a cell-dictionary is assumed. + /// Creates an empty dictionary, which is actually a null value. Equivalent to PUSHNULL @pure -fun createEmptyDict(): cell? +fun createEmptyDict(): dict asm "NEWDICT"; /// Checks whether a dictionary is empty. @pure -fun dictIsEmpty(self: cell?): bool +fun dictIsEmpty(self: dict): bool asm "DICTEMPTY"; @pure -fun iDictGet(self: cell?, keyLen: int, key: int): (slice?, bool) +fun iDictGet(self: dict, keyLen: int, key: int): (slice?, bool) asm(key self keyLen) "DICTIGET" "NULLSWAPIFNOT"; @pure -fun uDictGet(self: cell?, keyLen: int, key: int): (slice?, bool) +fun uDictGet(self: dict, keyLen: int, key: int): (slice?, bool) asm(key self keyLen) "DICTUGET" "NULLSWAPIFNOT"; @pure -fun sDictGet(self: cell?, keyLen: int, key: slice): (slice?, bool) +fun sDictGet(self: dict, keyLen: int, key: slice): (slice?, bool) asm(key self keyLen) "DICTGET" "NULLSWAPIFNOT"; @pure -fun iDictSet(mutate self: cell?, keyLen: int, key: int, value: slice): void +fun iDictSet(mutate self: dict, keyLen: int, key: int, value: slice): void asm(value key self keyLen) "DICTISET"; @pure -fun uDictSet(mutate self: cell?, keyLen: int, key: int, value: slice): void +fun uDictSet(mutate self: dict, keyLen: int, key: int, value: slice): void asm(value key self keyLen) "DICTUSET"; @pure -fun sDictSet(mutate self: cell?, keyLen: int, key: slice, value: slice): void +fun sDictSet(mutate self: dict, keyLen: int, key: slice, value: slice): void asm(value key self keyLen) "DICTSET"; @pure -fun iDictSetRef(mutate self: cell?, keyLen: int, key: int, value: cell): void +fun iDictSetRef(mutate self: dict, keyLen: int, key: int, value: cell): void asm(value key self keyLen) "DICTISETREF"; @pure -fun uDictSetRef(mutate self: cell?, keyLen: int, key: int, value: cell): void +fun uDictSetRef(mutate self: dict, keyLen: int, key: int, value: cell): void asm(value key self keyLen) "DICTUSETREF"; @pure -fun sDictSetRef(mutate self: cell?, keyLen: int, key: slice, value: cell): void +fun sDictSetRef(mutate self: dict, keyLen: int, key: slice, value: cell): void asm(value key self keyLen) "DICTSETREF"; @pure -fun iDictSetIfNotExists(mutate self: cell?, keyLen: int, key: int, value: slice): bool +fun iDictSetIfNotExists(mutate self: dict, keyLen: int, key: int, value: slice): bool asm(value key self keyLen) "DICTIADD"; @pure -fun uDictSetIfNotExists(mutate self: cell?, keyLen: int, key: int, value: slice): bool +fun uDictSetIfNotExists(mutate self: dict, keyLen: int, key: int, value: slice): bool asm(value key self keyLen) "DICTUADD"; @pure -fun iDictSetIfExists(mutate self: cell?, keyLen: int, key: int, value: slice): bool +fun iDictSetIfExists(mutate self: dict, keyLen: int, key: int, value: slice): bool asm(value key self keyLen) "DICTIREPLACE"; @pure -fun uDictSetIfExists(mutate self: cell?, keyLen: int, key: int, value: slice): bool +fun uDictSetIfExists(mutate self: dict, keyLen: int, key: int, value: slice): bool asm(value key self keyLen) "DICTUREPLACE"; @pure -fun iDictGetRef(self: cell?, keyLen: int, key: int): (cell?, bool) +fun iDictGetRef(self: dict, keyLen: int, key: int): (dict, bool) asm(key self keyLen) "DICTIGETREF" "NULLSWAPIFNOT"; @pure -fun uDictGetRef(self: cell?, keyLen: int, key: int): (cell?, bool) +fun uDictGetRef(self: dict, keyLen: int, key: int): (dict, bool) asm(key self keyLen) "DICTUGETREF" "NULLSWAPIFNOT"; @pure -fun sDictGetRef(self: cell?, keyLen: int, key: slice): (cell?, bool) +fun sDictGetRef(self: dict, keyLen: int, key: slice): (dict, bool) asm(key self keyLen) "DICTGETREF" "NULLSWAPIFNOT"; @pure -fun iDictGetRefOrNull(self: cell?, keyLen: int, key: int): cell? +fun iDictGetRefOrNull(self: dict, keyLen: int, key: int): dict asm(key self keyLen) "DICTIGETOPTREF"; @pure -fun uDictGetRefOrNull(self: cell?, keyLen: int, key: int): cell? +fun uDictGetRefOrNull(self: dict, keyLen: int, key: int): dict asm(key self keyLen) "DICTUGETOPTREF"; @pure -fun sDictGetRefOrNull(self: cell?, keyLen: int, key: slice): cell? +fun sDictGetRefOrNull(self: dict, keyLen: int, key: slice): dict asm(key self keyLen) "DICTGETOPTREF"; @pure -fun iDictDelete(mutate self: cell?, keyLen: int, key: int): bool +fun iDictDelete(mutate self: dict, keyLen: int, key: int): bool asm(key self keyLen) "DICTIDEL"; @pure -fun uDictDelete(mutate self: cell?, keyLen: int, key: int): bool +fun uDictDelete(mutate self: dict, keyLen: int, key: int): bool asm(key self keyLen) "DICTUDEL"; @pure -fun sDictDelete(mutate self: cell?, keyLen: int, key: slice): bool +fun sDictDelete(mutate self: dict, keyLen: int, key: slice): bool asm(key self keyLen) "DICTDEL"; @pure -fun iDictSetAndGet(mutate self: cell?, keyLen: int, key: int, value: slice): (slice?, bool) +fun iDictSetAndGet(mutate self: dict, keyLen: int, key: int, value: slice): (slice?, bool) asm(value key self keyLen) "DICTISETGET" "NULLSWAPIFNOT"; @pure -fun uDictSetAndGet(mutate self: cell?, keyLen: int, key: int, value: slice): (slice?, bool) +fun uDictSetAndGet(mutate self: dict, keyLen: int, key: int, value: slice): (slice?, bool) asm(value key self keyLen) "DICTUSETGET" "NULLSWAPIFNOT"; @pure -fun sDictSetAndGet(mutate self: cell?, keyLen: int, key: slice, value: slice): (slice?, bool) +fun sDictSetAndGet(mutate self: dict, keyLen: int, key: slice, value: slice): (slice?, bool) asm(value key self keyLen) "DICTSETGET" "NULLSWAPIFNOT"; @pure -fun iDictSetAndGetRefOrNull(mutate self: cell?, keyLen: int, key: int, value: cell): cell? +fun iDictSetAndGetRefOrNull(mutate self: dict, keyLen: int, key: int, value: cell): dict asm(value key self keyLen) "DICTISETGETOPTREF"; @pure -fun uDictSetAndGetRefOrNull(mutate self: cell?, keyLen: int, key: int, value: cell): cell? +fun uDictSetAndGetRefOrNull(mutate self: dict, keyLen: int, key: int, value: cell): dict asm(value key self keyLen) "DICTUSETGETOPTREF"; @pure -fun iDictDeleteAndGet(mutate self: cell?, keyLen: int, key: int): (slice?, bool) +fun iDictDeleteAndGet(mutate self: dict, keyLen: int, key: int): (slice?, bool) asm(key self keyLen) "DICTIDELGET" "NULLSWAPIFNOT"; @pure -fun uDictDeleteAndGet(mutate self: cell?, keyLen: int, key: int): (slice?, bool) +fun uDictDeleteAndGet(mutate self: dict, keyLen: int, key: int): (slice?, bool) asm(key self keyLen) "DICTUDELGET" "NULLSWAPIFNOT"; @pure -fun sDictDeleteAndGet(mutate self: cell?, keyLen: int, key: slice): (slice?, bool) +fun sDictDeleteAndGet(mutate self: dict, keyLen: int, key: slice): (slice?, bool) asm(key self keyLen) "DICTDELGET" "NULLSWAPIFNOT"; @pure -fun iDictSetBuilder(mutate self: cell?, keyLen: int, key: int, value: builder): void +fun iDictSetBuilder(mutate self: dict, keyLen: int, key: int, value: builder): void asm(value key self keyLen) "DICTISETB"; @pure -fun uDictSetBuilder(mutate self: cell?, keyLen: int, key: int, value: builder): void +fun uDictSetBuilder(mutate self: dict, keyLen: int, key: int, value: builder): void asm(value key self keyLen) "DICTUSETB"; @pure -fun sDictSetBuilder(mutate self: cell?, keyLen: int, key: slice, value: builder): void +fun sDictSetBuilder(mutate self: dict, keyLen: int, key: slice, value: builder): void asm(value key self keyLen) "DICTSETB"; @pure -fun iDictSetBuilderIfNotExists(mutate self: cell?, keyLen: int, key: int, value: builder): bool +fun iDictSetBuilderIfNotExists(mutate self: dict, keyLen: int, key: int, value: builder): bool asm(value key self keyLen) "DICTIADDB"; @pure -fun uDictSetBuilderIfNotExists(mutate self: cell?, keyLen: int, key: int, value: builder): bool +fun uDictSetBuilderIfNotExists(mutate self: dict, keyLen: int, key: int, value: builder): bool asm(value key self keyLen) "DICTUADDB"; @pure -fun iDictSetBuilderIfExists(mutate self: cell?, keyLen: int, key: int, value: builder): bool +fun iDictSetBuilderIfExists(mutate self: dict, keyLen: int, key: int, value: builder): bool asm(value key self keyLen) "DICTIREPLACEB"; @pure -fun uDictSetBuilderIfExists(mutate self: cell?, keyLen: int, key: int, value: builder): bool +fun uDictSetBuilderIfExists(mutate self: dict, keyLen: int, key: int, value: builder): bool asm(value key self keyLen) "DICTUREPLACEB"; @pure -fun iDictDeleteFirstAndGet(mutate self: cell?, keyLen: int): (int?, slice?, bool) +fun iDictDeleteFirstAndGet(mutate self: dict, keyLen: int): (int?, slice?, bool) asm(-> 0 2 1 3) "DICTIREMMIN" "NULLSWAPIFNOT2"; @pure -fun uDictDeleteFirstAndGet(mutate self: cell?, keyLen: int): (int?, slice?, bool) +fun uDictDeleteFirstAndGet(mutate self: dict, keyLen: int): (int?, slice?, bool) asm(-> 0 2 1 3) "DICTUREMMIN" "NULLSWAPIFNOT2"; @pure -fun sDictDeleteFirstAndGet(mutate self: cell?, keyLen: int): (slice?, slice?, bool) +fun sDictDeleteFirstAndGet(mutate self: dict, keyLen: int): (slice?, slice?, bool) asm(-> 0 2 1 3) "DICTREMMIN" "NULLSWAPIFNOT2"; @pure -fun iDictDeleteLastAndGet(mutate self: cell?, keyLen: int): (int?, slice?, bool) +fun iDictDeleteLastAndGet(mutate self: dict, keyLen: int): (int?, slice?, bool) asm(-> 0 2 1 3) "DICTIREMMAX" "NULLSWAPIFNOT2"; @pure -fun uDictDeleteLastAndGet(mutate self: cell?, keyLen: int): (int?, slice?, bool) +fun uDictDeleteLastAndGet(mutate self: dict, keyLen: int): (int?, slice?, bool) asm(-> 0 2 1 3) "DICTUREMMAX" "NULLSWAPIFNOT2"; @pure -fun sDictDeleteLastAndGet(mutate self: cell?, keyLen: int): (slice?, slice?, bool) +fun sDictDeleteLastAndGet(mutate self: dict, keyLen: int): (slice?, slice?, bool) asm(-> 0 2 1 3) "DICTREMMAX" "NULLSWAPIFNOT2"; @pure -fun iDictGetFirst(self: cell?, keyLen: int): (int?, slice?, bool) +fun iDictGetFirst(self: dict, keyLen: int): (int?, slice?, bool) asm (-> 1 0 2) "DICTIMIN" "NULLSWAPIFNOT2"; @pure -fun uDictGetFirst(self: cell?, keyLen: int): (int?, slice?, bool) +fun uDictGetFirst(self: dict, keyLen: int): (int?, slice?, bool) asm (-> 1 0 2) "DICTUMIN" "NULLSWAPIFNOT2"; @pure -fun sDictGetFirst(self: cell?, keyLen: int): (slice?, slice?, bool) +fun sDictGetFirst(self: dict, keyLen: int): (slice?, slice?, bool) asm (-> 1 0 2) "DICTMIN" "NULLSWAPIFNOT2"; @pure -fun iDictGetFirstAsRef(self: cell?, keyLen: int): (int?, cell?, bool) +fun iDictGetFirstAsRef(self: dict, keyLen: int): (int?, dict, bool) asm (-> 1 0 2) "DICTIMINREF" "NULLSWAPIFNOT2"; @pure -fun uDictGetFirstAsRef(self: cell?, keyLen: int): (int?, cell?, bool) +fun uDictGetFirstAsRef(self: dict, keyLen: int): (int?, dict, bool) asm (-> 1 0 2) "DICTUMINREF" "NULLSWAPIFNOT2"; @pure -fun sDictGetFirstAsRef(self: cell?, keyLen: int): (slice?, cell?, bool) +fun sDictGetFirstAsRef(self: dict, keyLen: int): (slice?, dict, bool) asm (-> 1 0 2) "DICTMINREF" "NULLSWAPIFNOT2"; @pure -fun iDictGetLast(self: cell?, keyLen: int): (int?, slice?, bool) +fun iDictGetLast(self: dict, keyLen: int): (int?, slice?, bool) asm (-> 1 0 2) "DICTIMAX" "NULLSWAPIFNOT2"; @pure -fun uDictGetLast(self: cell?, keyLen: int): (int?, slice?, bool) +fun uDictGetLast(self: dict, keyLen: int): (int?, slice?, bool) asm (-> 1 0 2) "DICTUMAX" "NULLSWAPIFNOT2"; @pure -fun sDictGetLast(self: cell?, keyLen: int): (slice?, slice?, bool) +fun sDictGetLast(self: dict, keyLen: int): (slice?, slice?, bool) asm (-> 1 0 2) "DICTMAX" "NULLSWAPIFNOT2"; @pure -fun iDictGetLastAsRef(self: cell?, keyLen: int): (int?, cell?, bool) +fun iDictGetLastAsRef(self: dict, keyLen: int): (int?, dict, bool) asm (-> 1 0 2) "DICTIMAXREF" "NULLSWAPIFNOT2"; @pure -fun uDictGetLastAsRef(self: cell?, keyLen: int): (int?, cell?, bool) +fun uDictGetLastAsRef(self: dict, keyLen: int): (int?, dict, bool) asm (-> 1 0 2) "DICTUMAXREF" "NULLSWAPIFNOT2"; @pure -fun sDictGetLastAsRef(self: cell?, keyLen: int): (slice?, cell?, bool) +fun sDictGetLastAsRef(self: dict, keyLen: int): (slice?, dict, bool) asm (-> 1 0 2) "DICTMAXREF" "NULLSWAPIFNOT2"; @pure -fun iDictGetNext(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) +fun iDictGetNext(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXT" "NULLSWAPIFNOT2"; @pure -fun uDictGetNext(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) +fun uDictGetNext(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXT" "NULLSWAPIFNOT2"; @pure -fun iDictGetNextOrEqual(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) +fun iDictGetNextOrEqual(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXTEQ" "NULLSWAPIFNOT2"; @pure -fun uDictGetNextOrEqual(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) +fun uDictGetNextOrEqual(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXTEQ" "NULLSWAPIFNOT2"; @pure -fun iDictGetPrev(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) +fun iDictGetPrev(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTIGETPREV" "NULLSWAPIFNOT2"; @pure -fun uDictGetPrev(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) +fun uDictGetPrev(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTUGETPREV" "NULLSWAPIFNOT2"; @pure -fun iDictGetPrevOrEqual(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) +fun iDictGetPrevOrEqual(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTIGETPREVEQ" "NULLSWAPIFNOT2"; @pure -fun uDictGetPrevOrEqual(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) +fun uDictGetPrevOrEqual(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTUGETPREVEQ" "NULLSWAPIFNOT2"; @@ -300,13 +304,13 @@ fun uDictGetPrevOrEqual(self: cell?, keyLen: int, pivot: int): (int?, slice?, bo */ @pure -fun prefixDictGet(self: cell?, keyLen: int, key: slice): (slice, slice?, slice?, bool) +fun prefixDictGet(self: dict, keyLen: int, key: slice): (slice, slice?, slice?, bool) asm(key self keyLen) "PFXDICTGETQ" "NULLSWAPIFNOT2"; @pure -fun prefixDictSet(mutate self: cell?, keyLen: int, key: slice, value: slice): bool +fun prefixDictSet(mutate self: dict, keyLen: int, key: slice, value: slice): bool asm(value key self keyLen) "PFXDICTSET"; @pure -fun prefixDictDelete(mutate self: cell?, keyLen: int, key: slice): bool +fun prefixDictDelete(mutate self: dict, keyLen: int, key: slice): bool asm(key self keyLen) "PFXDICTDEL"; diff --git a/tolk-tester/tests/constants-tests.tolk b/tolk-tester/tests/constants-tests.tolk index e145306ea..d5519b0ef 100644 --- a/tolk-tester/tests/constants-tests.tolk +++ b/tolk-tester/tests/constants-tests.tolk @@ -1,13 +1,16 @@ +type MInt = int; +type MSlice = slice; + const int1 = 1; const int2 = 2; const int101: int = 101; -const int111: int = 111; +const int111: MInt = 111; const int1r = int1; const str1 = "const1"; -const str2 = stringHexToSlice("aabbcc"); +const str2: MSlice = stringHexToSlice("aabbcc"); const str2r: slice = str2; @@ -17,7 +20,7 @@ const str2int = 0xAABBCC; const nibbles: int = 4; const strange_zero = (!10 as int); -const strange_minus_1: int = (!0 as int); +const strange_minus_1: MInt = (!0 as int); const true1 = true; const true2 = !!true; @@ -38,7 +41,7 @@ fun sget2r(): slice { return str2r; } const int240: int = ((int1+int2)*10)<<3; -fun iget240(): int { return int240; } +fun iget240(): MInt { return int240; } @pure fun newc(): builder @@ -47,7 +50,7 @@ asm "NEWC"; fun endcs(b: builder): slice asm "ENDC" "CTOS"; @pure -fun sdeq(s1: slice, s2: slice): int +fun sdeq(s1: slice, s2: slice): MInt asm "SDEQ"; @pure fun stslicer(b: builder, s: slice): builder diff --git a/tolk-tester/tests/generics-1.tolk b/tolk-tester/tests/generics-1.tolk index ca3109275..70cd2371c 100644 --- a/tolk-tester/tests/generics-1.tolk +++ b/tolk-tester/tests/generics-1.tolk @@ -1,3 +1,7 @@ +type MInt = int; +type Tensor2Int = (int, int); +type Tensor2IntN = (int, int)?; + fun eq1(value: X): X { return value; } fun eq2(value: X) { return value; } fun eq3(value: X): X { var cp: [X] = [eq1(value)]; var ((([v: X]))) = cp; return v; } @@ -26,6 +30,7 @@ fun test102(): (int, int, int, [int, int]) { var ab_tup = [0, [1, 2]]; ab_tup.0 = getTwo(); ab_tup.1.1 = getTwo(); + var ab_tens_al: Tensor2Int = (getTwo(), getTwo()); return (eq1(a), eq2(b), takeInt(getTwo()), [getTwo(), ab_tens.1.1]); } @@ -88,11 +93,12 @@ fun test106() { fun callTupleFirst(t: X): Y { return t.tupleFirst(); } fun callTuplePush(mutate self: T, v1: V, v2: V): self { self.tuplePush(v1); tuplePush(mutate self, v2); return self; } fun getTupleLastInt(t: tuple) { return t.tupleLast(); } -fun getTupleSize(t: tuple) { return t.tupleSize(); } +fun getTupleSize(t: MTuple) { return t.tupleSize(); } fun callAnyFn(f: (TObj) -> TResult, arg: TObj) { return f(arg); } fun callAnyFn2(f: TCallback, arg: tuple) { return f(arg); } -global t107: tuple; +global t107: MTuple; +type MTuple = tuple; @method_id(107) fun test107() { @@ -105,7 +111,7 @@ fun test107() { callAnyFn2(getTupleSize, t107), first, callTupleFirst(t107) as int, - callAnyFn(getTupleLastInt, t107), + callAnyFn(getTupleLastInt, t107 as tuple), callAnyFn2(getTupleLastInt, t107) ); } @@ -131,7 +137,41 @@ fun test108() { return g108; } -fun main(x: int): (int, [[int, int]]) { +@method_id(109) +fun test109(initialX: int32) { + var x: MInt = initialX; + __expect_type(eq1(x), "MInt"); + __expect_type(eq1(x), "int"); + __expect_type(eq1(x), "int8?"); + return (eq1(x), eq1(null), eq1(null)); +} + +@method_id(110) +fun test110() { + var ab_tens_al: Tensor2Int = (getTwo(), getTwo()); + ab_tens_al.0 = getTwo(); + var ab_tens_al_n: Tensor2Int? = (5, 6); + ab_tens_al_n.1 = getTwo(); + return (ab_tens_al, ab_tens_al_n); +} + +type Tup2Int = [int, int]; + +fun isSomeNullableNull(v: T?) { return v == null; } + +type MIntN = int?; +type Tup2IntN = Tup2Int?; + +@method_id(111) +fun test111() { + var i1: MIntN = 3; + var i2: Tup2IntN = [1, 2]; + return (isSomeNullableNull(i1), isSomeNullableNull(i2), isSomeNullableNull(i1), isSomeNullableNull(i1)); +} + +fun main(x: int): (int, [Tup2Int]) { + __expect_type(test110, "() -> (Tensor2Int, Tensor2Int)"); + try { if(x) { throw (1, x); } } catch (excNo, arg) { return (arg as int, [[eq2(arg as int), getTwo()]]); } return (0, [[x, 1]]); @@ -147,11 +187,15 @@ fun main(x: int): (int, [[int, int]]) { @testcase | 106 | | [ 106 2 6 6 ] @testcase | 107 | | 6 6 1 1 6 6 @testcase | 108 | | 45 +@testcase | 109 | 5 | 5 (null) (null) +@testcase | 110 | | 2 2 5 2 +@testcase | 111 | | 0 0 0 0 @fif_codegen DECLPROC eq1 @fif_codegen DECLPROC eq1 @fif_codegen DECLPROC eq1<(int,int)> @fif_codegen DECLPROC eq1<[int,int]> +@fif_codegen DECLPROC eq1 @fif_codegen DECLPROC getTwo @fif_codegen_avoid DECLPROC eq1 diff --git a/tolk-tester/tests/indexed-access.tolk b/tolk-tester/tests/indexed-access.tolk index e2bd3dd93..4b3a0a4b2 100644 --- a/tolk-tester/tests/indexed-access.tolk +++ b/tolk-tester/tests/indexed-access.tolk @@ -1,5 +1,8 @@ +type Tup2Int = [int, MInt]; +type Tensor2Int = (int, MInt); +type MInt = int; -fun increment(mutate self: int) { +fun increment(mutate self: MInt) { self += 1; } @@ -8,7 +11,7 @@ fun increment2(mutate a: int, mutate b: int) { b += 1; } -fun assign1020(mutate a: int, mutate b: int) { +fun assign1020(mutate a: int, mutate b: MInt) { a = 10; b = 20; } @@ -51,7 +54,7 @@ fun test101() { return t; } -global t102: (int, (int, int), [int, int, [int, int]], int); +global t102: (int, (int, MInt), [int, int, Tup2Int], MInt); @method_id(102) fun test102() { @@ -137,7 +140,7 @@ fun test110(f: int, s: int) { return (x, x.1, x.1.plus(x.1 / 20), x.1, x.1 = x.1 * 2, x.1, x.1 += 1, x.1, x); } -global xx: (int, int); +global xx: Tensor2Int; @method_id(111) fun test111(x: (int, int)) { @@ -154,7 +157,7 @@ fun test112(f: int, s: int) { } @pure -fun getConstTuple() { +fun getConstTuple(): Tup2Int { return [1,2]; } @@ -197,7 +200,7 @@ fun test117() { var t = createEmptyTuple(); t.tuplePush(1); try { - return (t.0 as tuple).0 as int; + return (t.0 as tuple).0 as MInt; } catch(excNo) { return excNo; } diff --git a/tolk-tester/tests/inference-tests.tolk b/tolk-tester/tests/inference-tests.tolk index 436848317..affe42ee1 100644 --- a/tolk-tester/tests/inference-tests.tolk +++ b/tolk-tester/tests/inference-tests.tolk @@ -75,7 +75,7 @@ fun test5(x: int) { __expect_type([x, x >= 1], "[int, bool]"); __expect_type([x, x >= 1, null as slice?], "[int, bool, slice?]"); __expect_type((x, [x], [[x], x]), "(int, [int], [[int], int])"); - __expect_type(getMyOriginalBalanceWithExtraCurrencies(), "[int, cell?]"); + __expect_type(getMyOriginalBalanceWithExtraCurrencies(), "[int, dict]"); } fun test6() { diff --git a/tolk-tester/tests/invalid-declaration/err-1483.tolk b/tolk-tester/tests/invalid-declaration/err-1483.tolk new file mode 100644 index 000000000..01812fedc --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1483.tolk @@ -0,0 +1,8 @@ +type A = B; +type B = (int, A); + +/** +@compilation_should_fail +@stderr type `A` circularly references itself +@stderr type A = B + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4304.tolk b/tolk-tester/tests/invalid-semantics/err-4304.tolk new file mode 100644 index 000000000..963693414 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4304.tolk @@ -0,0 +1,10 @@ +type MInt = int; + +fun main() { + var value = MInt; +} + +/** +@compilation_should_fail +@stderr type `MInt` can not be used as a value +*/ diff --git a/tolk-tester/tests/invalid-syntax/err-3482.tolk b/tolk-tester/tests/invalid-syntax/err-3482.tolk new file mode 100644 index 000000000..cfe2d3884 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3482.tolk @@ -0,0 +1,6 @@ +type MInt int; + +/** +@compilation_should_fail +@stderr expected `=`, got `int` +*/ diff --git a/tolk-tester/tests/invalid-typing/err-6188.tolk b/tolk-tester/tests/invalid-typing/err-6188.tolk index 8c1df4a26..7a7fb15e3 100644 --- a/tolk-tester/tests/invalid-typing/err-6188.tolk +++ b/tolk-tester/tests/invalid-typing/err-6188.tolk @@ -1,8 +1,11 @@ +type MInt = int; + fun failMathOnBoolean(c: cell) { - return (null == c) * 10; + var ten: MInt = 10; + return (null == c) * ten; } /** @compilation_should_fail -@stderr can not apply operator `*` to `bool` and `int` +@stderr can not apply operator `*` to `bool` and `MInt` */ diff --git a/tolk-tester/tests/invalid-typing/err-6337.tolk b/tolk-tester/tests/invalid-typing/err-6337.tolk index 1dca7822c..5c9cbdc4c 100644 --- a/tolk-tester/tests/invalid-typing/err-6337.tolk +++ b/tolk-tester/tests/invalid-typing/err-6337.tolk @@ -9,5 +9,5 @@ fun testCantCallDictMethodsOnNullable(c: cell) { /** @compilation_should_fail -@stderr can not call method for `cell` with object of type `cell?` +@stderr can not call method for `cell` with object of type `dict` */ diff --git a/tolk-tester/tests/invalid-typing/err-6450.tolk b/tolk-tester/tests/invalid-typing/err-6450.tolk index 052596e4c..b14f3b6d5 100644 --- a/tolk-tester/tests/invalid-typing/err-6450.tolk +++ b/tolk-tester/tests/invalid-typing/err-6450.tolk @@ -1,9 +1,11 @@ +type MInt = int; + fun main() { - var tri: (int, int) = (10, false); + var tri: (int, MInt) = (10, false); return; } /** @compilation_should_fail -@stderr can not assign `(int, bool)` to variable of type `(int, int)` +@stderr can not assign `(int, bool)` to variable of type `(int, MInt)` */ diff --git a/tolk-tester/tests/invalid-typing/err-6553.tolk b/tolk-tester/tests/invalid-typing/err-6553.tolk index bf5a1165c..ba6807713 100644 --- a/tolk-tester/tests/invalid-typing/err-6553.tolk +++ b/tolk-tester/tests/invalid-typing/err-6553.tolk @@ -1,4 +1,6 @@ -fun getNullableInt(): int? { return 5; } +type MInt = int; + +fun getNullableInt(): MInt? { return 5; } fun testAssertThrowIsConditional() { var (x, y) = (getNullableInt(), getNullableInt()); @@ -8,5 +10,5 @@ fun testAssertThrowIsConditional() { /** @compilation_should_fail -@stderr can not apply operator `+` to `int` and `int?` +@stderr can not apply operator `+` to `MInt` and `MInt?` */ diff --git a/tolk-tester/tests/invalid-typing/err-6580.tolk b/tolk-tester/tests/invalid-typing/err-6580.tolk index d2a815eee..e2c088b04 100644 --- a/tolk-tester/tests/invalid-typing/err-6580.tolk +++ b/tolk-tester/tests/invalid-typing/err-6580.tolk @@ -1,4 +1,6 @@ -fun getNullableInt(): int? { return 5; } +type MIntN = int?; + +fun getNullableInt(): MIntN { return 5; } fun testNeverTypeOccurs() { var x: int? = getNullableInt(); diff --git a/tolk-tester/tests/invalid-typing/err-6734.tolk b/tolk-tester/tests/invalid-typing/err-6734.tolk index 3a5b1fe28..0f805cbe5 100644 --- a/tolk-tester/tests/invalid-typing/err-6734.tolk +++ b/tolk-tester/tests/invalid-typing/err-6734.tolk @@ -1,10 +1,12 @@ +type MTensor = (int, int); + fun failAssignNullToTensor() { - var ab = (1, 2); + var ab: MTensor = (1, 2); ab = null; return ab; } /** @compilation_should_fail -@stderr can not assign `null` to variable of type `(int, int)` +@stderr can not assign `null` to variable of type `MTensor` */ diff --git a/tolk-tester/tests/invalid-typing/err-6737.tolk b/tolk-tester/tests/invalid-typing/err-6737.tolk index d112b3606..cc4bc2f59 100644 --- a/tolk-tester/tests/invalid-typing/err-6737.tolk +++ b/tolk-tester/tests/invalid-typing/err-6737.tolk @@ -1,5 +1,7 @@ +type MInt32 = int32; + fun testAssignBetweenDifferentIntN(op: int32, qid: uint64) { - op = qid as int32; // ok + op = qid as MInt32; // ok op = qid; } diff --git a/tolk-tester/tests/nullable-tensors.tolk b/tolk-tester/tests/nullable-tensors.tolk index 6b53306b6..9e3a269b5 100644 --- a/tolk-tester/tests/nullable-tensors.tolk +++ b/tolk-tester/tests/nullable-tensors.tolk @@ -1,3 +1,7 @@ +type TensorOf2IntNull = (int?, int?); +type MInt = int; +type MIntN = int?; + fun getNullableInt(): int? { return 5; } fun sumOfNullableTensorComponents(t: (int, int)?): int { @@ -385,7 +389,9 @@ fun getNormalNullableTensorWidth1(vLess100: int?): ([int?], ())? { return ([vLess100], ()); // such a nullable tensor can store NULL in the same slot } -fun getTrickyNullableTensorWidth1(vLess100: int?): (int?, ())? { +type TrickyActually2SlotTensor = (MIntN, ())?; + +fun getTrickyNullableTensorWidth1(vLess100: int?): TrickyActually2SlotTensor { if (vLess100 != null && vLess100 >= 100) { return null; } @@ -429,6 +435,26 @@ fun test140(x: int8, y: int16) { return (t1, t2); } +@method_id(141) +fun test141() { + var t: TensorOf2IntNull = (1, 2); + if (t.0 != null) { + t.0 = null; + } + t.1 != null ? t.1 = 10 : t.1 = null; + return t; +} + +type T142 = (int, MInt)?; + +@method_id(142) +fun test142() { + var t = (1, 2) as T142; + __expect_type(t!, "(int, MInt)"); + var sum1 = sumOfNullableTensorComponents(t); + var sum2 = sumOfTensor(t!); + return (sum1, sum2, t!); +} fun main(){} @@ -482,6 +508,8 @@ fun main(){} @testcase | 136 | 9 | 9 0 @testcase | 136 | null | (null) -1 @testcase | 140 | 8 9 | 8 9 -1 (null) (null) 0 +@testcase | 141 | | (null) 10 +@testcase | 142 | | 3 3 1 2 @fif_codegen """ diff --git a/tolk-tester/tests/nullable-types.tolk b/tolk-tester/tests/nullable-types.tolk index 24aa7f8ae..f5ee8b550 100644 --- a/tolk-tester/tests/nullable-types.tolk +++ b/tolk-tester/tests/nullable-types.tolk @@ -1,5 +1,6 @@ +type MIntN = int?; -fun getNullable4(): int? { return 4; } +fun getNullable4(): MIntN { return 4; } fun getNullableIntNull(): int? asm "PUSHNULL"; fun eqInt(x: int) { return x; } @@ -36,7 +37,7 @@ fun test103(x: int?): (bool, bool, int) { } @method_id(104) -fun test104(x: int?) { +fun test104(x: MIntN) { var x2 = eq(x = 10); var ab = (x2, getNullableIntNull()); return (unwrap(ab.0) + (ab.1 == null ? -100 : ab.1!), ab.1); @@ -44,7 +45,7 @@ fun test104(x: int?) { @method_id(105) fun test105() { - var xy: (int?, int?) = (5, null); + var xy: (int?, MIntN) = (5, null); var ab = [1 ? [xy.0, xy.1] : null]; ab.0!.0 = intOr0(ab.0!.0); ab.0!.1 = intOr0(ab.0!.1); @@ -52,7 +53,7 @@ fun test105() { } global gTup106: tuple?; -global gInt106: int?; +global gInt106: MIntN; @method_id(106) fun test106() { @@ -91,7 +92,11 @@ fun test109() { return ([a, b] = [3, 4], a, b); } -fun main(x: int?, y: int?) { +fun main(x: int?, y: MIntN) { + __expect_type(x! == y!, "bool"); + __expect_type(5 != y!, "bool"); + __expect_type(y == null, "bool"); + __expect_type(true == (true as bool?)!, "bool"); } /** diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index 4d71bb63a..813218026 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -9,10 +9,12 @@ fun increment(mutate self: int) { self += 1; } fun assignToInt(mutate self: int, value: int) { self = value; } fun assignToNullableInt(mutate self: int?, value: int) { self = value; } fun sameTensor(t: (int, int)) { return t; } -fun sameTensor2(t: (int?, (slice, slice, slice, builder)?)) { return t; } +fun sameTensor2(t: (int?, Pair4N)) { return t; } fun eq(v: T) { return v; } fun getTwo(): X { return 2 as X; } +type Pair4N = (slice, slice, slice, builder)?; + fun test1(): int { var x = getNullableInt(); var y = getNullableInt(); diff --git a/tolk-tester/tests/type-aliases-tests.tolk b/tolk-tester/tests/type-aliases-tests.tolk new file mode 100644 index 000000000..54899a0fc --- /dev/null +++ b/tolk-tester/tests/type-aliases-tests.tolk @@ -0,0 +1,189 @@ +type MIntN = MInt?; +type MInt = int; +type MInt_v2 = int; +type MVoid = void; +type Pair2_v1 = (int, int); +type Pair2_v2 = (MInt, MInt); +type MBool = bool; +type Tuple2Int = [int, int]; + + +fun test1(x: MInt): MVoid { + var y = x; + var z: MInt = 2; + __expect_type(x, "MInt"); + __expect_type(y, "MInt"); + __expect_type(z, "MInt"); + + __expect_type(x + y, "int"); + __expect_type((x + y) as MInt, "MInt"); + __expect_type(x!, "MInt"); + __expect_type(~x, "int"); + __expect_type(x as int, "int"); + __expect_type(x as int8, "int8"); + __expect_type(random() ? x : y, "MInt"); + __expect_type((x, 1, y), "(MInt, int, MInt)"); + + __expect_type(random() ? (1, 2) : (1, 2) as Pair2_v1, "(int, int)"); + __expect_type(random() ? (1, 2) : (1, 2) as Pair2_v2, "(int, int)"); + __expect_type(random() ? (1, 2) as Pair2_v1 : (1, 2), "Pair2_v1"); + __expect_type(random() ? (1, 2) as Pair2_v1 : (1, 2) as Pair2_v2, "Pair2_v1"); + __expect_type(random() ? (1, 2) as Pair2_v2 : (1, 2) as Pair2_v2, "Pair2_v2"); + + __expect_type(!x, "bool"); + + __expect_type(x as int?, "int?"); + __expect_type(x as MInt?, "MInt?"); + __expect_type(x as MIntN, "MIntN"); + + if (x) { return; } +} + +fun test2() { + __expect_type(test1, "(MInt) -> MVoid"); + __expect_type(test1(1), "MVoid"); +} + +fun test3(x: MIntN, y: MInt?) { + __expect_type(x, "MIntN"); + __expect_type(x!, "MInt"); + __expect_type(y!, "MInt"); + if (x != null) { + __expect_type(x, "MInt"); + } + __expect_type(x, "MInt?"); // merge null + MInt = MInt?, not MIntN as original + var (z1, z2) = (x, x!, ); + __expect_type(z1, "MInt?"); + __expect_type(z2, "MInt"); +} + +@method_id(104) +fun test4(x: MIntN, y: MIntN) { + if (x != null && y != null) { + __expect_type(x, "MInt"); + __expect_type(x + y, "int"); + return x + y; + } + __expect_type(x!, "MInt"); + __expect_type(random() ? x : y, "MInt?"); + __expect_type(random() ? x : 0, "MInt?"); + __expect_type(random() ? 0 : x, "MInt?"); + __expect_type(random() ? 0 : x!, "int"); + return y!; +} + +fun takeTensor_v1(v: Pair2_v1) { return v.0 + v.1; } +fun takeTensor_v2(v: Pair2_v2) { return v.0 + v.1; } +fun takeTensor_v3(v: (int, int)) { return v.0 + v.1; } + +fun test5() { + var x: Pair2_v1 = (1, 2); + var y = (3, 4) as Pair2_v2; + var z = (5, 6); + + __expect_type(x, "Pair2_v1"); + __expect_type(y, "Pair2_v2"); + + takeTensor_v1(x); takeTensor_v2(x); takeTensor_v3(x); + takeTensor_v1(y); takeTensor_v2(y); takeTensor_v3(y); + takeTensor_v1(z); takeTensor_v2(z); takeTensor_v3(z); + + var t = (y, x); + return t.0.1 + t.1.0; +} + +fun test6() { + var (x1: MInt?, x2: MIntN) = (5, 5); // smart cast + __expect_type(x1, "MInt"); + __expect_type(x2, "MInt"); + var (y1: MInt?, y2: MIntN) = (null, null); + __expect_type(y1, "null"); + __expect_type(y2, "null"); + var z: Pair2_v2? = (5, 5); + __expect_type(z, "Pair2_v2"); + __expect_type(z.0, "MInt"); +} + +fun someFn1(v: MInt): MIntN { return v; } + +fun test7() { + var f1: (int) -> int? = someFn1; + var f2: (int) -> MInt? = someFn1; + var f3: (MInt_v2) -> MInt_v2? = someFn1; + + f1 = f2; f1 = f3; + f2 = f1; f2 = f3; + f3 = f1; f3 = f2; +} + +fun test8() { + 0 as MInt; + 0 as MInt?; + 0 as MIntN; + + (1, 2) as Pair2_v2; + (1, 2) as Pair2_v2?; + (((1, 2) as Pair2_v2?) as (int, int)?) as Pair2_v1?; + + someFn1 as (int) -> int?; + someFn1 as (int) -> MInt?; + someFn1 as (MInt_v2) -> MInt_v2?; +} + +fun test9(b: MBool): MBool { + if (!b) { + __expect_type(b, "MBool"); + return !b; + } + return !!(5 as MInt); +} + +@method_id(110) +fun test10() { + var x1: Pair2_v1 = (5, 6); + var (a1, b1) = x1; + __expect_type(a1, "int"); + var x2: Pair2_v2? = x1; + var (a2, b2) = x2; + __expect_type(a2, "MInt"); + var x3: Tuple2Int = [9, 10]; + var [a3, b3] = x3; + return a1 + a2 + a3 + b1 + b2 + b3; +} + +fun analyzeTensor1(a: (T1, T2)): (T1, T2) { return a; } +fun analyzeTensor2(a: (T1, T2)?): (T1, T2)? { return a; } + +fun test11() { + var (x1: (int8, int16), x2: Pair2_v1, x3: Pair2_v2) = ((1,2), (3,4), (5,6)); + __expect_type(analyzeTensor1(x1), "(int8, int16)"); + __expect_type(analyzeTensor1(x2), "(int, int)"); + __expect_type(analyzeTensor1(x3), "(MInt, MInt)"); + __expect_type(analyzeTensor2(x1), "(int8, int16)?"); + __expect_type(analyzeTensor2(x2), "(int, int)?"); + __expect_type(analyzeTensor2(x3), "(MInt, MInt)?"); +} + + +fun main(x: MInt, y: MInt?) { + return y == null ? x : x + y; +} + +/** +@testcase | 0 | 3 4 | 7 +@testcase | 104 | 1 2 | 3 +@testcase | 110 | | 41 + +@fif_codegen +""" + test9 PROC:<{ + // b + DUP // b b + IFNOTJMP:<{ // b + NOT // '2 + }> // b + DROP // + TRUE // '5 + }> +""" + */ diff --git a/tolk-tester/tests/var-apply-tests.tolk b/tolk-tester/tests/var-apply-tests.tolk index 96b4d134d..76fdab21f 100644 --- a/tolk-tester/tests/var-apply-tests.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -1,3 +1,8 @@ +type MInt = int; +type MIntN = int?; +type MBuilder = builder; +type CallbackIntToInt = int -> int; + fun getBeginCell() { return beginCell; } @@ -9,7 +14,7 @@ fun getBeginParse() { @method_id(101) fun testVarApply1() { var (_, f_end_cell) = (0, endCell); - var b: builder = (getBeginCell())().storeInt(1, 32); + var b: MBuilder = (getBeginCell())().storeInt(1, 32); b.storeInt(2, 32); var s = (getBeginParse())(f_end_cell(b)); return (s.loadInt(32), s.loadInt(32)); @@ -55,7 +60,7 @@ fun demo_handler(op: int, query_id: int, a: int, b: int): int { return 0; // result not used, we test that func is nevertheless called } if (op == 0xF4) { - val func = query_id % 2 == 0 ? sum : mul; + val func: (MInt, int) -> MInt = query_id % 2 == 0 ? sum : mul; val result = func(a, b); return result; } @@ -92,7 +97,7 @@ fun always_throw2(x: int) { throw 239 + x; } -global global_f: int -> void; +global global_f: MInt -> void; @method_id(104) fun testGlobalVarApply() { @@ -124,7 +129,7 @@ fun testVarApply3() { t.tuplePush(1); t.tuplePush([2]); var getIntAt = t.tupleAt; - var getTupleFirstInt = createEmptyTuple().tupleFirst; + var getTupleFirstInt = createEmptyTuple().tupleFirst; var getTupleLastTuple = getTupleLastGetter(); return (getIntAt(t, 0), getTupleFirstInt(t), getTupleLastTuple(t), getTupleLastGetter()(t)); } @@ -143,9 +148,9 @@ fun myBeginCell(): builder? asm "NEWC"; @method_id(108) fun testCallingNotNull() { - var n4: () -> int? = getNullable4; + var n4: () -> MIntN = getNullable4; var creator: (() -> builder?)? = myBeginCell; - var end2: [int, (builder -> cell)?] = [0, endCell]; + var end2: [int, (MBuilder -> cell)?] = [0, endCell]; var c: cell = end2.1!((creator!()!)!.storeInt(getNullable4()!, 32)); return c.beginParse().loadInt(32); } @@ -186,8 +191,8 @@ fun testApplyNativePlus(x: int, y: int, z: int): bool { global op: (int, int) -> int; -fun check_assoc_2(a: int, b: int, c: int): bool { - return op(op(a, b), c) == op(a, op(b, c)); +fun check_assoc_2(a: int, b: MInt, c: MIntN): bool { + return op(op(a, b), c!) == op(a, op(b, c!)); } @method_id(112) @@ -197,6 +202,15 @@ fun testApplyGlobalVar(x: int, y: int, z: int): bool? { return check_assoc_2(x, y, z); } +fun justAdd2(x: MInt): int64 { return x + 2; } + +@method_id(113) +fun testCallbackAlias(secondNull: bool) { + var adder1: CallbackIntToInt = justAdd2; + var adder2: CallbackIntToInt? = secondNull ? null : justAdd2; + return (adder1(10), adder2 == null ? -1 : adder2(10)); +} + fun main() {} /** @@ -217,4 +231,6 @@ fun main() {} @testcase | 112 | 2 3 9 | -1 @testcase | 112 | 11 22 44 | -1 @testcase | 112 | -1 -10 -20 | -1 -*/ +@testcase | 113 | 0 | 12 12 +@testcase | 113 | -1 | 12 -1 + */ diff --git a/tolk/abscode.cpp b/tolk/abscode.cpp index fc1609846..eb2e9a0a6 100644 --- a/tolk/abscode.cpp +++ b/tolk/abscode.cpp @@ -414,6 +414,8 @@ std::vector CodeBlob::create_var(TypePtr var_type, SrcLocation loc, s std::string null_flag_name = name.empty() ? name : name + ".NNFlag"; ir_idx = create_var(t_nullable->inner, loc, std::move(name)); ir_idx.emplace_back(create_var(TypeDataBool::create(), loc, std::move(null_flag_name))[0]); + } else if (const TypeDataAlias* t_alias = var_type->try_as()) { + ir_idx = create_var(t_alias->underlying_type, loc, std::move(name)); } else if (var_type != TypeDataVoid::create() && var_type != TypeDataNever::create()) { #ifdef TOLK_DEBUG tolk_assert(stack_w == 1); diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 94ef848c1..729204579 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -227,6 +227,21 @@ static AnyV parse_constant_declaration(Lexer& lex, const std::vector(loc, v_ident, declared_type, init_value); } +static AnyV parse_type_alias_declaration(Lexer& lex, const std::vector>& annotations) { + if (!annotations.empty()) { + lex.error("@annotations are not applicable to type alias declaration"); + } + SrcLocation loc = lex.cur_location(); + lex.expect(tok_type, "`type`"); + lex.check(tok_identifier, "type name"); + auto v_ident = createV(lex.cur_location(), lex.cur_str()); + lex.next(); + lex.expect(tok_assign, "`=`"); + TypePtr underlying_type = parse_type_from_tokens(lex); + lex.expect(tok_semicolon, "`;`"); + return createV(loc, v_ident, underlying_type); +} + // "parameters" are at function declaration: `fun f(param1: int, mutate param2: slice)` static V parse_parameter_list(Lexer& lex) { SrcLocation loc = lex.cur_location(); @@ -1180,6 +1195,10 @@ AnyV parse_src_file_to_ast(const SrcFile* file) { toplevel_declarations.push_back(parse_constant_declaration(lex, annotations)); annotations.clear(); break; + case tok_type: + toplevel_declarations.push_back(parse_type_alias_declaration(lex, annotations)); + annotations.clear(); + break; case tok_fun: case tok_get: toplevel_declarations.push_back(parse_function_declaration(lex, annotations)); diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index 07f03745c..77390c1ca 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -81,6 +81,7 @@ class ASTStringifier final : public ASTVisitor { {ast_function_declaration, "ast_function_declaration"}, {ast_global_var_declaration, "ast_global_var_declaration"}, {ast_constant_declaration, "ast_constant_declaration"}, + {ast_type_alias_declaration, "ast_type_alias_declaration"}, {ast_tolk_required_version, "ast_tolk_required_version"}, {ast_import_directive, "ast_import_directive"}, {ast_tolk_file, "ast_tolk_file"}, @@ -157,6 +158,8 @@ class ASTStringifier final : public ASTVisitor { return static_cast(v->as()->get_identifier()->name); case ast_constant_declaration: return static_cast(v->as()->get_identifier()->name); + case ast_type_alias_declaration: + return "type " + static_cast(v->as()->get_identifier()->name); case ast_assign: return "="; case ast_set_assign: @@ -291,6 +294,7 @@ class ASTStringifier final : public ASTVisitor { case ast_function_declaration: return handle_vertex(v->as()); case ast_global_var_declaration: return handle_vertex(v->as()); case ast_constant_declaration: return handle_vertex(v->as()); + case ast_type_alias_declaration: return handle_vertex(v->as()); case ast_tolk_required_version: return handle_vertex(v->as()); case ast_import_directive: return handle_vertex(v->as()); case ast_tolk_file: return handle_vertex(v->as()); diff --git a/tolk/ast.cpp b/tolk/ast.cpp index ac93d21c3..f383ebcc7 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -154,6 +154,14 @@ void Vertex::assign_resolved_type(TypePtr declared_typ this->declared_type = declared_type; } +void Vertex::assign_alias_ref(AliasDefPtr alias_ref) { + this->alias_ref = alias_ref; +} + +void Vertex::assign_resolved_type(TypePtr underlying_type) { + this->underlying_type = underlying_type; +} + void Vertex::assign_resolved_type(TypePtr substituted_type) { this->substituted_type = substituted_type; } diff --git a/tolk/ast.h b/tolk/ast.h index 385e1ff89..b62aa4c53 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -113,6 +113,7 @@ enum ASTNodeType { ast_function_declaration, ast_global_var_declaration, ast_constant_declaration, + ast_type_alias_declaration, ast_tolk_required_version, ast_import_directive, ast_tolk_file, @@ -1036,6 +1037,25 @@ struct Vertex final : ASTOtherVararg { , declared_type(declared_type) {} }; +template<> +// ast_type_alias_declaration is declaring a structural type alias (fully interchangeable with original type) +// example: `type UserId = int;` +// see TypeDataAlias in type-system.h +struct Vertex final : ASTOtherVararg { + AliasDefPtr alias_ref = nullptr; // filled after register, contains TypeDataAlias + TypePtr underlying_type; // at the right of `=` + + auto get_identifier() const { return children.at(0)->as(); } + + Vertex* mutate() const { return const_cast(this); } + void assign_alias_ref(AliasDefPtr alias_ref); + void assign_resolved_type(TypePtr underlying_type); + + Vertex(SrcLocation loc, V name_identifier, TypePtr underlying_type) + : ASTOtherVararg(ast_type_alias_declaration, loc, {name_identifier}) + , underlying_type(underlying_type) {} +}; + template<> // ast_tolk_required_version is a preamble fixating compiler's version at the top of the file // example: `tolk 0.6` diff --git a/tolk/fwd-declarations.h b/tolk/fwd-declarations.h index 8d3b24a8f..7255a8588 100644 --- a/tolk/fwd-declarations.h +++ b/tolk/fwd-declarations.h @@ -31,11 +31,13 @@ struct LocalVarData; struct FunctionData; struct GlobalVarData; struct GlobalConstData; +struct AliasDefData; using LocalVarPtr = const LocalVarData*; using FunctionPtr = const FunctionData*; using GlobalVarPtr = const GlobalVarData*; using GlobalConstPtr = const GlobalConstData*; +using AliasDefPtr = const AliasDefData*; class TypeData; using TypePtr = const TypeData*; diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index 9dae3f009..b335683f3 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -83,7 +83,7 @@ void GenericSubstitutionsDeduceForCall::consider_next_condition(TypePtr param_ty provide_deducedT(asT->nameT, arg_type); } else if (const auto* p_nullable = param_type->try_as()) { // `arg: T?` called as `f(nullableInt)` => T is int - if (const auto* a_nullable = arg_type->try_as()) { + if (const auto* a_nullable = arg_type->unwrap_alias()->try_as()) { consider_next_condition(p_nullable->inner, a_nullable->inner); } // `arg: T?` called as `f(int)` => T is int @@ -92,21 +92,21 @@ void GenericSubstitutionsDeduceForCall::consider_next_condition(TypePtr param_ty } } else if (const auto* p_tensor = param_type->try_as()) { // `arg: (int, T)` called as `f((5, cs))` => T is slice - if (const auto* a_tensor = arg_type->try_as(); a_tensor && a_tensor->size() == p_tensor->size()) { + if (const auto* a_tensor = arg_type->unwrap_alias()->try_as(); a_tensor && a_tensor->size() == p_tensor->size()) { for (int i = 0; i < a_tensor->size(); ++i) { consider_next_condition(p_tensor->items[i], a_tensor->items[i]); } } } else if (const auto* p_tuple = param_type->try_as()) { // `arg: [int, T]` called as `f([5, cs])` => T is slice - if (const auto* a_tuple = arg_type->try_as(); a_tuple && a_tuple->size() == p_tuple->size()) { + if (const auto* a_tuple = arg_type->unwrap_alias()->try_as(); a_tuple && a_tuple->size() == p_tuple->size()) { for (int i = 0; i < a_tuple->size(); ++i) { consider_next_condition(p_tuple->items[i], a_tuple->items[i]); } } } else if (const auto* p_callable = param_type->try_as()) { // `arg: fun(TArg) -> TResult` called as `f(calcTupleLen)` => TArg is tuple, TResult is int - if (const auto* a_callable = arg_type->try_as(); a_callable && a_callable->params_size() == p_callable->params_size()) { + if (const auto* a_callable = arg_type->unwrap_alias()->try_as(); a_callable && a_callable->params_size() == p_callable->params_size()) { for (int i = 0; i < a_callable->params_size(); ++i) { consider_next_condition(p_callable->params_types[i], a_callable->params_types[i]); } diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index b26c9d2c4..10647de94 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -150,7 +150,7 @@ class LValContext { LValContext local_lval; local_lval.enter_rval_inside_lval(); std::vector obj_ir_idx = pre_compile_expr(tensor_obj, code, nullptr, &local_lval); - const TypeDataTensor* t_tensor = tensor_obj->inferred_type->try_as(); + const TypeDataTensor* t_tensor = tensor_obj->inferred_type->unwrap_alias()->try_as(); tolk_assert(t_tensor); int stack_width = t_tensor->items[index_at]->get_width_on_stack(); int stack_offset = 0; @@ -384,7 +384,7 @@ static std::vector pre_compile_let(CodeBlob& code, AnyExprV lhs, AnyE std::vector left = pre_compile_tensor(code, lhs->as()->get_items(), &local_lval); vars_modification_watcher.trigger_callbacks(left, loc); std::vector right = pre_compile_expr(rhs, code, nullptr); - const TypeDataTypedTuple* inferred_tuple = rhs->inferred_type->try_as(); + const TypeDataTypedTuple* inferred_tuple = rhs->inferred_type->unwrap_alias()->try_as(); std::vector types_list = inferred_tuple->items; std::vector rvect = code.create_tmp_var(TypeDataTensor::create(std::move(types_list)), rhs->loc, "(unpack-tuple)"); code.emplace_back(lhs->loc, Op::_UnTuple, rvect, std::move(right)); @@ -432,6 +432,13 @@ static std::vector gen_op_call(CodeBlob& code, TypePtr ret_type, SrcL // Here rvect is a list of IR vars for inferred_type, probably patched due to target_type. GNU_ATTRIBUTE_NOINLINE static std::vector transition_expr_to_runtime_type_impl(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc) { +#ifdef TOLK_DEBUG + tolk_assert(static_cast(rvect.size()) == original_type->get_width_on_stack()); +#endif + + original_type = original_type->unwrap_alias(); + target_type = target_type->unwrap_alias(); + // pass `T` to `T` // could occur for passing tensor `(..., T, ...)` to `(..., T, ...)` while traversing tensor's components if (target_type == original_type) { @@ -635,6 +642,22 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as()) { + return transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type, target_alias->underlying_type, loc); + } + // pass `UserId` (or some other alias) to `T` + if (const TypeDataAlias* orig_alias = original_type->try_as()) { + return transition_expr_to_runtime_type_impl(std::move(rvect), code, orig_alias->underlying_type, target_type, loc); + } + + // pass callable to callable + // their types aren't exactly equal, but they match (containing aliases, for example) + if (original_type->try_as() && target_type->try_as()) { + tolk_assert(rvect.size() == 1); + return rvect; + } + throw Fatal("unhandled transition_expr_to_runtime_type_impl() combination"); } @@ -819,7 +842,7 @@ static std::vector process_cast_as_operator(V v } static std::vector process_not_null_operator(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { - TypePtr child_target_type = v->get_expr()->inferred_type; + TypePtr child_target_type = v->get_expr()->inferred_type->unwrap_alias(); if (const auto* as_nullable = child_target_type->try_as()) { child_target_type = as_nullable->inner; } @@ -830,7 +853,7 @@ static std::vector process_not_null_operator(V static std::vector process_is_null_check(V v, CodeBlob& code, TypePtr target_type) { std::vector expr_ir_idx = pre_compile_expr(v->get_expr(), code, nullptr); std::vector isnull_ir_idx = code.create_tmp_var(TypeDataBool::create(), v->loc, "(is-null)"); - TypePtr expr_type = v->get_expr()->inferred_type; + TypePtr expr_type = v->get_expr()->inferred_type->unwrap_alias(); if (const TypeDataNullable* t_nullable = expr_type->try_as()) { if (!t_nullable->is_primitive_nullable()) { @@ -858,7 +881,7 @@ static std::vector process_dot_access(V v, CodeBlob& // it's NOT a method call `t.tupleSize()` (since such cases are handled by process_function_call) // it's `t.0`, `getUser().id`, and `t.tupleSize` (as a reference, not as a call) if (!v->is_target_fun_ref()) { - TypePtr obj_type = v->get_obj()->inferred_type; + TypePtr obj_type = v->get_obj()->inferred_type->unwrap_alias(); int index_at = std::get(v->target); // `tensorVar.0` if (const auto* t_tensor = obj_type->try_as()) { @@ -930,7 +953,7 @@ static std::vector process_function_call(V v, Code for (int i = 0; i < v->get_num_args(); ++i) { args.push_back(v->get_arg(i)->get_expr()); } - std::vector params_types = v->get_callee()->inferred_type->try_as()->params_types; + std::vector params_types = v->get_callee()->inferred_type->unwrap_alias()->try_as()->params_types; const TypeDataTensor* tensor_tt = TypeDataTensor::create(std::move(params_types))->try_as(); std::vector> vars_per_arg = pre_compile_tensor_inner(code, args, tensor_tt, nullptr); std::vector args_vars; @@ -1249,7 +1272,7 @@ static void process_do_while_statement(V v, CodeBlob& co until_cond = createV(cond->loc, "<", tok_lt, v_geq->get_lhs(), v_geq->get_rhs()); } else if (auto v_gt = cond->try_as(); v_gt && v_gt->tok == tok_gt) { until_cond = createV(cond->loc, "<=", tok_geq, v_gt->get_lhs(), v_gt->get_rhs()); - } else if (cond->inferred_type == TypeDataBool::create()) { + } else if (cond->inferred_type->unwrap_alias() == TypeDataBool::create()) { until_cond = createV(cond->loc, "!b", tok_logical_not, cond); } else { until_cond = createV(cond->loc, "!", tok_logical_not, cond); diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 4b5abbc8b..927cfc4e1 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -118,6 +118,19 @@ static void handle_possible_compiler_internal_call(FunctionPtr cur_f, Vname == "__expect_type") { tolk_assert(v->get_num_args() == 2); TypePtr expected_type = parse_type_from_string(v->get_arg(1)->get_expr()->as()->str_val); + if (expected_type->has_unresolved_inside()) { // only aliases can be inside + expected_type = expected_type->replace_children_custom([](TypePtr child) -> TypePtr { + if (const auto* as_unresolved = child->try_as()) { + const Symbol* sym = lookup_global_symbol(as_unresolved->text); + tolk_assert(sym); + if (AliasDefPtr alias_ref = sym->try_as()) { + return TypeDataAlias::create(alias_ref->name, alias_ref->underlying_type); + } + tolk_assert(false); + } + return child; + }); + } TypePtr expr_type = v->get_arg(0)->inferred_type; if (expected_type != expr_type) { fire(cur_f, v->loc, "__expect_type failed: expected " + to_string(expected_type) + ", got " + to_string(expr_type)); @@ -125,14 +138,36 @@ static void handle_possible_compiler_internal_call(FunctionPtr cur_f, Vtry_as() || inferred_type == TypeDataCoins::create()) { + return true; + } + if (const TypeDataAlias* as_alias = inferred_type->try_as()) { + return expect_integer(as_alias->underlying_type); + } + return false; +} + static bool expect_integer(AnyExprV v_inferred) { - return v_inferred->inferred_type == TypeDataInt::create() || v_inferred->inferred_type->try_as() || v_inferred->inferred_type == TypeDataCoins::create(); + return expect_integer(v_inferred->inferred_type); } -static bool expect_boolean(AnyExprV v_inferred) { - return v_inferred->inferred_type == TypeDataBool::create(); +static bool expect_boolean(TypePtr inferred_type) { + if (inferred_type == TypeDataBool::create()) { + return true; + } + if (const TypeDataAlias* as_alias = inferred_type->try_as()) { + return expect_boolean(as_alias->underlying_type); + } + return false; } +static bool expect_boolean(AnyExprV v_inferred) { + return expect_boolean(v_inferred->inferred_type); +} class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { FunctionPtr cur_f = nullptr; // may be nullptr if checking `const a = ...` init_value @@ -282,9 +317,9 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { parent::visit(v); - TypePtr obj_type = v->get_obj()->inferred_type; if (v->is_target_indexed_access()) { - if (obj_type->try_as() && v->inferred_type->get_width_on_stack() != 1) { + TypePtr obj_type = v->get_obj()->inferred_type; + if (obj_type->unwrap_alias()->try_as() && v->inferred_type->get_width_on_stack() != 1) { fire_error_cannot_put_non1_stack_width_arg_to_tuple(cur_f, v->loc, v->inferred_type); } } @@ -296,7 +331,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { FunctionPtr fun_ref = v->fun_maybe; if (!fun_ref) { // `local_var(args)` and similar - const TypeDataFunCallable* f_callable = v->get_callee()->inferred_type->try_as(); + const TypeDataFunCallable* f_callable = v->get_callee()->inferred_type->unwrap_alias()->try_as(); tolk_assert(f_callable && f_callable->params_size() == v->get_num_args()); for (int i = 0; i < v->get_num_args(); ++i) { auto arg_i = v->get_arg(i)->get_expr(); @@ -381,7 +416,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // `(v1, v2) = rhs` / `var (v1, v2) = rhs` (rhs may be `(1,2)` or `tensorVar` or `someF()`, doesn't matter) // dig recursively into v1 and v2 with corresponding rhs i-th item of a tensor if (auto lhs_tensor = lhs->try_as()) { - const TypeDataTensor* rhs_type_tensor = rhs_type->try_as(); + const TypeDataTensor* rhs_type_tensor = rhs_type->unwrap_alias()->try_as(); if (!rhs_type_tensor) { fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to a tensor"); } @@ -398,7 +433,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // `[v1, v2] = rhs` / `var [v1, v2] = rhs` (rhs may be `[1,2]` or `tupleVar` or `someF()`, doesn't matter) // dig recursively into v1 and v2 with corresponding rhs i-th item of a tuple if (auto lhs_tuple = lhs->try_as()) { - const TypeDataTypedTuple* rhs_type_tuple = rhs_type->try_as(); + const TypeDataTypedTuple* rhs_type_tuple = rhs_type->unwrap_alias()->try_as(); if (!rhs_type_tuple) { fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to a tuple"); } @@ -414,7 +449,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // check `untypedTuple.0 = rhs_tensor` and other non-1 width elements if (auto lhs_dot = lhs->try_as()) { - if (lhs_dot->is_target_indexed_access() && lhs_dot->get_obj()->inferred_type == TypeDataTuple::create()) { + if (lhs_dot->is_target_indexed_access() && lhs_dot->get_obj()->inferred_type->unwrap_alias() == TypeDataTuple::create()) { if (rhs_type->get_width_on_stack() != 1) { fire_error_cannot_put_non1_stack_width_arg_to_tuple(cur_f, err_loc->loc, rhs_type); } diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 9db82f576..236037c3e 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -401,7 +401,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // `(v1, v2) = rhs` / `var (v1, v2) = rhs` (rhs may be `(1,2)` or `tensorVar` or `someF()`, doesn't matter) // dig recursively into v1 and v2 with corresponding rhs i-th item of a tensor if (auto lhs_tensor = lhs->try_as()) { - const TypeDataTensor* rhs_type_tensor = rhs_type->try_as(); + const TypeDataTensor* rhs_type_tensor = rhs_type->unwrap_alias()->try_as(); std::vector types_list; types_list.reserve(lhs_tensor->size()); for (int i = 0; i < lhs_tensor->size(); ++i) { @@ -416,7 +416,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // `[v1, v2] = rhs` / `var [v1, v2] = rhs` (rhs may be `[1,2]` or `tupleVar` or `someF()`, doesn't matter) // dig recursively into v1 and v2 with corresponding rhs i-th item of a tuple if (auto lhs_tuple = lhs->try_as()) { - const TypeDataTypedTuple* rhs_type_tuple = rhs_type->try_as(); + const TypeDataTypedTuple* rhs_type_tuple = rhs_type->unwrap_alias()->try_as(); std::vector types_list; types_list.reserve(lhs_tuple->size()); for (int i = 0; i < lhs_tuple->size(); ++i) { @@ -478,7 +478,7 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, TypeDataInt::create()); break; case tok_logical_not: - if (rhs->inferred_type == TypeDataBool::create()) { + if (rhs->inferred_type->unwrap_alias() == TypeDataBool::create()) { builtin_func = "!b"; // "overloaded" for bool } assign_inferred_type(v, TypeDataBool::create()); @@ -520,7 +520,7 @@ class InferTypesAndCallsAndFieldsVisitor final { case tok_bitwise_xor: flow = infer_any_expr(lhs, std::move(flow), false).out_flow; flow = infer_any_expr(rhs, std::move(flow), false).out_flow; - if (lhs->inferred_type == TypeDataBool::create() && rhs->inferred_type == TypeDataBool::create()) { + if (lhs->inferred_type->unwrap_alias() == TypeDataBool::create() && rhs->inferred_type->unwrap_alias() == TypeDataBool::create()) { assign_inferred_type(v, TypeDataBool::create()); } else { assign_inferred_type(v, TypeDataInt::create()); @@ -615,7 +615,7 @@ class InferTypesAndCallsAndFieldsVisitor final { ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false); assign_inferred_type(v, TypeDataBool::create()); - TypePtr expr_type = v->get_expr()->inferred_type; + TypePtr expr_type = v->get_expr()->inferred_type->unwrap_alias(); TypePtr non_null_type = calculate_type_subtract_null(expr_type); if (expr_type == TypeDataNullLiteral::create()) { // `expr == null` is always true v->mutate()->assign_always_true_or_false(v->is_negated ? 2 : 1); @@ -652,7 +652,7 @@ class InferTypesAndCallsAndFieldsVisitor final { ExprFlow infer_not_null_operator(V v, FlowContext&& flow, bool used_as_condition) { ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false); - if (const auto* as_nullable = v->get_expr()->inferred_type->try_as()) { + if (const auto* as_nullable = v->get_expr()->inferred_type->unwrap_alias()->try_as()) { assign_inferred_type(v, as_nullable->inner); } else { assign_inferred_type(v, v->get_expr()); @@ -710,6 +710,9 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, fun_ref->inferred_full_type); return ExprFlow(std::move(flow), used_as_condition); + } else if (AliasDefPtr alias_ref = v->sym->try_as()) { + fire(cur_f, v->loc, "type `" + alias_ref->name + "` can not be used as a value"); + } else { tolk_assert(false); } @@ -766,6 +769,7 @@ class InferTypesAndCallsAndFieldsVisitor final { flow = infer_any_expr(v->get_obj(), std::move(flow), false).out_flow; TypePtr obj_type = v->get_obj()->inferred_type; + TypePtr unwrapped_obj_type = obj_type->unwrap_alias(); // our goal is to fill v->target knowing type of obj V v_ident = v->get_identifier(); // field/method name vertex V v_instantiationTs = v->get_instantiationTs(); @@ -775,7 +779,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // at first, check for indexed access if (field_name[0] >= '0' && field_name[0] <= '9') { int index_at = std::stoi(std::string(field_name)); - if (const auto* t_tensor = obj_type->try_as()) { + if (const auto* t_tensor = unwrapped_obj_type->try_as()) { if (index_at >= t_tensor->size()) { fire(cur_f, v_ident->loc, "invalid tensor index, expected 0.." + std::to_string(t_tensor->items.size() - 1)); } @@ -789,7 +793,7 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, inferred_type); return ExprFlow(std::move(flow), used_as_condition); } - if (const auto* t_tuple = obj_type->try_as()) { + if (const auto* t_tuple = unwrapped_obj_type->try_as()) { if (index_at >= t_tuple->size()) { fire(cur_f, v_ident->loc, "invalid tuple index, expected 0.." + std::to_string(t_tuple->items.size() - 1)); } @@ -803,7 +807,7 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, inferred_type); return ExprFlow(std::move(flow), used_as_condition); } - if (obj_type->try_as()) { + if (unwrapped_obj_type->try_as()) { TypePtr item_type = nullptr; if (v->is_lvalue && !hint) { // left side of assignment item_type = TypeDataUnknown::create(); @@ -899,7 +903,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // treat callee like a usual expression flow = infer_any_expr(callee, std::move(flow), false).out_flow; // it must have "callable" inferred type - const TypeDataFunCallable* f_callable = callee->inferred_type->try_as(); + const TypeDataFunCallable* f_callable = callee->inferred_type->unwrap_alias()->try_as(); if (!f_callable) { // `5()` / `SOME_CONST()` / `null()` fire(cur_f, v->loc, "calling a non-function " + to_string(callee->inferred_type)); } @@ -951,7 +955,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (param_type->has_genericT_inside()) { param_type = deducingTs->auto_deduce_from_argument(cur_f, dot_obj->loc, param_type, dot_obj->inferred_type); } - if (param_0.is_mutate_parameter() && dot_obj->inferred_type != param_type) { + if (param_0.is_mutate_parameter() && dot_obj->inferred_type->unwrap_alias() != param_type->unwrap_alias()) { if (SinkExpression s_expr = extract_sink_expression_from_vertex(dot_obj)) { assign_inferred_type(dot_obj, calc_declared_type_before_smart_cast(dot_obj)); flow.register_known_type(s_expr, param_type); @@ -974,7 +978,7 @@ class InferTypesAndCallsAndFieldsVisitor final { flow = infer_any_expr(arg_i, std::move(flow), false, param_type).out_flow; } assign_inferred_type(v->get_arg(i), arg_i); // arg itself is an expression - if (param_i.is_mutate_parameter() && arg_i->inferred_type != param_type) { + if (param_i.is_mutate_parameter() && arg_i->inferred_type->unwrap_alias() != param_type->unwrap_alias()) { if (SinkExpression s_expr = extract_sink_expression_from_vertex(arg_i)) { assign_inferred_type(arg_i, calc_declared_type_before_smart_cast(arg_i)); flow.register_known_type(s_expr, param_type); @@ -1023,7 +1027,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } ExprFlow infer_tensor(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { - const TypeDataTensor* tensor_hint = hint ? hint->try_as() : nullptr; + const TypeDataTensor* tensor_hint = hint ? hint->unwrap_alias()->try_as() : nullptr; std::vector types_list; types_list.reserve(v->get_items().size()); for (int i = 0; i < v->size(); ++i) { @@ -1036,7 +1040,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } ExprFlow infer_typed_tuple(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { - const TypeDataTypedTuple* tuple_hint = hint ? hint->try_as() : nullptr; + const TypeDataTypedTuple* tuple_hint = hint ? hint->unwrap_alias()->try_as() : nullptr; std::vector types_list; types_list.reserve(v->get_items().size()); for (int i = 0; i < v->size(); ++i) { diff --git a/tolk/pipe-optimize-boolean-expr.cpp b/tolk/pipe-optimize-boolean-expr.cpp index c4c5d1dcf..642defa03 100644 --- a/tolk/pipe-optimize-boolean-expr.cpp +++ b/tolk/pipe-optimize-boolean-expr.cpp @@ -57,6 +57,29 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { return v_not; } + static bool expect_integer(TypePtr inferred_type) { + if (inferred_type == TypeDataInt::create()) { + return true; + } + if (inferred_type->try_as() || inferred_type == TypeDataCoins::create()) { + return true; + } + if (const TypeDataAlias* as_alias = inferred_type->try_as()) { + return expect_integer(as_alias->underlying_type); + } + return false; + } + + static bool expect_boolean(TypePtr inferred_type) { + if (inferred_type == TypeDataBool::create()) { + return true; + } + if (const TypeDataAlias* as_alias = inferred_type->try_as()) { + return expect_boolean(as_alias->underlying_type); + } + return false; + } + protected: AnyExprV replace(V v) override { @@ -66,11 +89,11 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { if (auto inner_not = v->get_rhs()->try_as(); inner_not && inner_not->tok == tok_logical_not) { AnyExprV cond_not_not = inner_not->get_rhs(); // `!!boolVar` => `boolVar` - if (cond_not_not->inferred_type == TypeDataBool::create()) { + if (expect_boolean(cond_not_not->inferred_type)) { return cond_not_not; } // `!!intVar` => `intVar != 0` - if (cond_not_not->inferred_type == TypeDataInt::create()) { + if (expect_integer(cond_not_not->inferred_type)) { auto v_zero = create_int_const(v->loc, td::make_refint(0)); auto v_neq = createV(v->loc, "!=", tok_neq, cond_not_not, v_zero); v_neq->mutate()->assign_rvalue_true(); @@ -94,7 +117,7 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { if (v->tok == tok_eq || v->tok == tok_neq) { AnyExprV lhs = v->get_lhs(); AnyExprV rhs = v->get_rhs(); - if (lhs->inferred_type == TypeDataBool::create() && rhs->type == ast_bool_const) { + if (expect_boolean(lhs->inferred_type) && rhs->type == ast_bool_const) { // `boolVar == true` / `boolVar != false` if (rhs->as()->bool_val ^ (v->tok == tok_neq)) { return lhs; diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index ea4cc8110..e76f5b479 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -133,6 +133,13 @@ static void register_global_var(V v) { v->mutate()->assign_var_ref(g_sym); } +static void register_type_alias(V v) { + AliasDefData* a_sym = new AliasDefData(static_cast(v->get_identifier()->name), v->loc, v->underlying_type); + + G.symtable.add_type_alias(a_sym); + v->mutate()->assign_alias_ref(a_sym); +} + static LocalVarData register_parameter(V v, int idx) { if (v->is_underscore()) { return {"", v->loc, v->declared_type, 0, idx}; @@ -238,6 +245,9 @@ static void iterate_through_file_symbols(const SrcFile* file) { case ast_global_var_declaration: register_global_var(v->as()); break; + case ast_type_alias_declaration: + register_type_alias(v->as()); + break; case ast_function_declaration: register_function(v->as()); break; diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 2ace2e719..3f8f622c0 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -136,6 +136,15 @@ struct NameAndScopeResolver { }; struct TypeDataResolver { + static TypePtr finalize_type_data(FunctionPtr cur_f, TypePtr type_data, const GenericsDeclaration* genericTs) { + if (type_data) { + if (type_data->has_unresolved_inside()) { + type_data = resolve_identifiers_in_type_data(cur_f, type_data, genericTs); + } + } + return type_data; + } + GNU_ATTRIBUTE_NOINLINE static TypePtr resolve_identifiers_in_type_data(FunctionPtr cur_f, TypePtr type_data, const GenericsDeclaration* genericTs) { return type_data->replace_children_custom([cur_f, genericTs](TypePtr child) { @@ -144,6 +153,14 @@ struct TypeDataResolver { std::string nameT = un->text; return TypeDataGenericT::create(std::move(nameT)); } + if (const Symbol* sym = lookup_global_symbol(un->text)) { + if (AliasDefPtr alias_ref = sym->try_as()) { + if (alias_ref->underlying_type->has_unresolved_inside()) { + resolve_and_mutate_type_alias(alias_ref); + } + return TypeDataAlias::create(alias_ref->name, alias_ref->underlying_type); + } + } if (un->text == "auto") { throw ParseError(cur_f, un->loc, "`auto` type does not exist; just omit a type for local variable (will be inferred from assignment); parameters should always be typed"); } @@ -155,13 +172,25 @@ struct TypeDataResolver { return child; }); } + + static void resolve_and_mutate_type_alias(AliasDefPtr alias_ref) { + static std::vector called_stack; + + // prevent recursion like `type A = B; type B = A` + bool contains = std::find(called_stack.begin(), called_stack.end(), alias_ref) != called_stack.end(); + if (contains) { + throw ParseError(alias_ref->loc, "type `" + alias_ref->name + "` circularly references itself"); + } + + called_stack.push_back(alias_ref); + TypePtr underlying_type = finalize_type_data(nullptr, alias_ref->underlying_type, nullptr); + alias_ref->mutate()->assign_resolved_type(underlying_type); + called_stack.pop_back(); + } }; static TypePtr finalize_type_data(FunctionPtr cur_f, TypePtr type_data, const GenericsDeclaration* genericTs) { - if (!type_data || !type_data->has_unresolved_inside()) { - return type_data; - } - return TypeDataResolver::resolve_identifiers_in_type_data(cur_f, type_data, genericTs); + return TypeDataResolver::finalize_type_data(cur_f, type_data, genericTs); } @@ -344,6 +373,10 @@ void pipeline_resolve_identifiers_and_assign_symbols() { v_const->mutate()->assign_resolved_type(declared_type); v_const->const_ref->mutate()->assign_resolved_type(declared_type); } + + } else if (auto v_alias = v->try_as()) { + TypeDataResolver::resolve_and_mutate_type_alias(v_alias->alias_ref); + v_alias->mutate()->assign_resolved_type(v_alias->alias_ref->underlying_type); } } } diff --git a/tolk/smart-casts-cfg.cpp b/tolk/smart-casts-cfg.cpp index 9a2f7ef4d..d3ddb5af7 100644 --- a/tolk/smart-casts-cfg.cpp +++ b/tolk/smart-casts-cfg.cpp @@ -196,6 +196,13 @@ static TypePtr calculate_type_lca(TypePtr a, TypePtr b) { return TypeDataInt::create(); } + if (const auto* a_alias = a->try_as()) { + return calculate_type_lca(a_alias->underlying_type, b); + } + if (const auto* b_alias = b->try_as()) { + return calculate_type_lca(a, b_alias->underlying_type); + } + return nullptr; } @@ -352,7 +359,7 @@ FlowContext FlowContext::merge_flow(FlowContext&& c1, FlowContext&& c2) { // return `T`, so that `T?` = type // what for: `if (x != null)`, to smart cast x inside if TypePtr calculate_type_subtract_null(TypePtr type) { - if (const auto* as_nullable = type->try_as()) { + if (const auto* as_nullable = type->unwrap_alias()->try_as()) { return as_nullable->inner; } // union types will be handled here @@ -415,7 +422,7 @@ TypePtr calc_declared_type_before_smart_cast(AnyExprV v) { } if (auto as_dot = v->try_as(); as_dot && as_dot->is_target_indexed_access()) { - TypePtr obj_type = as_dot->get_obj()->inferred_type; // v already inferred; hence, index_at is correct + TypePtr obj_type = as_dot->get_obj()->inferred_type->unwrap_alias(); // v already inferred; hence, index_at is correct int index_at = std::get(as_dot->target); if (const auto* t_tensor = obj_type->try_as()) { return t_tensor->items[index_at]; @@ -436,7 +443,7 @@ TypePtr calc_declared_type_before_smart_cast(AnyExprV v) { TypePtr calc_smart_cast_type_on_assignment(TypePtr lhs_declared_type, TypePtr rhs_inferred_type) { // assign `T` to `T?` (or at least "assignable-to-T" to "T?") // smart cast to `T` - if (const auto* lhs_nullable = lhs_declared_type->try_as()) { + if (const auto* lhs_nullable = lhs_declared_type->unwrap_alias()->try_as()) { if (lhs_nullable->inner->can_rhs_be_assigned(rhs_inferred_type)) { return lhs_nullable->inner; } @@ -444,7 +451,7 @@ TypePtr calc_smart_cast_type_on_assignment(TypePtr lhs_declared_type, TypePtr rh // assign `null` to `T?` // smart cast to `null` - if (lhs_declared_type->try_as() && rhs_inferred_type == TypeDataNullLiteral::create()) { + if (lhs_declared_type->unwrap_alias()->try_as() && rhs_inferred_type == TypeDataNullLiteral::create()) { return TypeDataNullLiteral::create(); } diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index ea009957a..4942fe870 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -113,6 +113,10 @@ void LocalVarData::assign_inferred_type(TypePtr inferred_type) { this->declared_type = inferred_type; } +void AliasDefData::assign_resolved_type(TypePtr underlying_type) { + this->underlying_type = underlying_type; +} + GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_redefinition_of_symbol(SrcLocation loc, const Symbol* previous) { SrcLocation prev_loc = previous->loc; @@ -149,6 +153,14 @@ void GlobalSymbolTable::add_global_const(GlobalConstPtr c_sym) { } } +void GlobalSymbolTable::add_type_alias(AliasDefPtr a_sym) { + auto key = key_hash(a_sym->name); + auto [it, inserted] = entries.emplace(key, a_sym); + if (!inserted) { + fire_error_redefinition_of_symbol(a_sym->loc, it->second); + } +} + const Symbol* lookup_global_symbol(std::string_view name) { return G.symtable.lookup(name); } diff --git a/tolk/symtable.h b/tolk/symtable.h index e78eef6e8..a27aecd44 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -216,6 +216,18 @@ struct GlobalConstData final : Symbol { void assign_const_value(ConstantValue&& value); }; +struct AliasDefData final : Symbol { + TypePtr underlying_type; + + AliasDefData(std::string name, SrcLocation loc, TypePtr underlying_type) + : Symbol(std::move(name), loc) + , underlying_type(underlying_type) { + } + + AliasDefData* mutate() const { return const_cast(this); } + void assign_resolved_type(TypePtr underlying_type); +}; + class GlobalSymbolTable { std::unordered_map entries; @@ -227,6 +239,7 @@ class GlobalSymbolTable { void add_function(FunctionPtr f_sym); void add_global_var(GlobalVarPtr g_sym); void add_global_const(GlobalConstPtr c_sym); + void add_type_alias(AliasDefPtr a_sym); const Symbol* lookup(std::string_view name) const { const auto it = entries.find(key_hash(name)); diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index ebcfc6fe6..459ad8136 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -113,6 +113,18 @@ void type_system_init() { // and creates an object only if it isn't found in a global hashtable // +TypePtr TypeDataAlias::create(const std::string& alias_name, TypePtr underlying_type) { + TypeDataTypeIdCalculation hash(5694590762732189561ULL); + hash.feed_string(alias_name); + hash.feed_child(underlying_type); + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + std::string name_copy = alias_name; + return hash.register_unique(new TypeDataAlias(hash.type_id(), hash.children_flags(), std::move(name_copy), underlying_type)); +} + TypePtr TypeDataNullable::create(TypePtr inner) { TypeDataTypeIdCalculation hash(1774084920039440885ULL); hash.feed_child(inner); @@ -286,6 +298,11 @@ std::string TypeDataBytesN::as_human_readable() const { // only non-trivial implementations are here; by default (no children), `callback(this)` is executed // +void TypeDataAlias::traverse(const TraverserCallbackT& callback) const { + callback(this); + underlying_type->traverse(callback); +} + void TypeDataNullable::traverse(const TraverserCallbackT& callback) const { callback(this); inner->traverse(callback); @@ -322,6 +339,10 @@ void TypeDataTypedTuple::traverse(const TraverserCallbackT& callback) const { // only non-trivial implementations are here; by default (no children), `return callback(this)` is executed // +TypePtr TypeDataAlias::replace_children_custom(const ReplacerCallbackT& callback) const { + return callback(create(alias_name, underlying_type->replace_children_custom(callback))); +} + TypePtr TypeDataNullable::replace_children_custom(const ReplacerCallbackT& callback) const { return callback(create(inner->replace_children_custom(callback))); } @@ -361,6 +382,13 @@ TypePtr TypeDataTypedTuple::replace_children_custom(const ReplacerCallbackT& cal // the same goes for passing arguments, returning values, etc. — where the "receiver" (lhs) checks "applier" (rhs) // +bool TypeDataAlias::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + return underlying_type->can_rhs_be_assigned(rhs); +} + bool TypeDataInt::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; @@ -371,6 +399,9 @@ bool TypeDataInt::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == TypeDataCoins::create()) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -378,6 +409,9 @@ bool TypeDataBool::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -385,6 +419,9 @@ bool TypeDataCell::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -392,6 +429,9 @@ bool TypeDataSlice::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); // note, that bytesN is NOT automatically cast to slice without `as` operator } @@ -399,6 +439,9 @@ bool TypeDataBuilder::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -406,6 +449,9 @@ bool TypeDataTuple::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -413,6 +459,9 @@ bool TypeDataContinuation::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -420,6 +469,9 @@ bool TypeDataNullLiteral::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -436,6 +488,9 @@ bool TypeDataNullable::can_rhs_be_assigned(TypePtr rhs) const { if (inner->can_rhs_be_assigned(rhs)) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -443,6 +498,23 @@ bool TypeDataFunCallable::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataFunCallable* rhs_callable = rhs->try_as()) { + if (rhs_callable->params_size() != params_size()) { + return false; + } + for (int i = 0; i < params_size(); ++i) { + if (!rhs_callable->params_types[i]->can_rhs_be_assigned(params_types[i])) { + return false; + } + if (!params_types[i]->can_rhs_be_assigned(rhs_callable->params_types[i])) { + return false; + } + } + return return_type->can_rhs_be_assigned(rhs_callable->return_type) && rhs_callable->return_type->can_rhs_be_assigned(return_type); + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -460,6 +532,9 @@ bool TypeDataTensor::can_rhs_be_assigned(TypePtr rhs) const { } return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -472,6 +547,9 @@ bool TypeDataTypedTuple::can_rhs_be_assigned(TypePtr rhs) const { } return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -482,6 +560,9 @@ bool TypeDataIntN::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == TypeDataInt::create()) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); // `int8` is NOT assignable to `int32` without `as` } @@ -491,7 +572,10 @@ bool TypeDataBytesN::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } - return rhs == TypeDataNever::create(); + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } + return false; } bool TypeDataCoins::can_rhs_be_assigned(TypePtr rhs) const { @@ -501,6 +585,9 @@ bool TypeDataCoins::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == TypeDataInt::create()) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -521,6 +608,9 @@ bool TypeDataVoid::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } return rhs == TypeDataNever::create(); } @@ -532,6 +622,10 @@ bool TypeDataVoid::can_rhs_be_assigned(TypePtr rhs) const { // note, that it's not auto-casts `var lhs: = rhs`, it's an expression `rhs as ` // +bool TypeDataAlias::can_be_casted_with_as_operator(TypePtr cast_to) const { + return underlying_type->can_be_casted_with_as_operator(cast_to); +} + bool TypeDataInt::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { // `int` as `int?` return can_be_casted_with_as_operator(to_nullable->inner); @@ -542,6 +636,9 @@ bool TypeDataInt::can_be_casted_with_as_operator(TypePtr cast_to) const { if (cast_to == TypeDataCoins::create()) { // `int` as `coins` return true; } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } return cast_to == this; } @@ -555,6 +652,9 @@ bool TypeDataBool::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_intN = cast_to->try_as()) { return !to_intN->is_unsigned; // `bool` as `int8` ok, `bool` as `uintN` not (true is -1) } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } return cast_to == this; } @@ -562,6 +662,9 @@ bool TypeDataCell::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { return can_be_casted_with_as_operator(to_nullable->inner); } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } return cast_to == this; } @@ -572,6 +675,9 @@ bool TypeDataSlice::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { return can_be_casted_with_as_operator(to_nullable->inner); } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } return cast_to == this; } @@ -579,6 +685,9 @@ bool TypeDataBuilder::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { return can_be_casted_with_as_operator(to_nullable->inner); } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } return cast_to == this; } @@ -586,6 +695,9 @@ bool TypeDataTuple::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { return can_be_casted_with_as_operator(to_nullable->inner); } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } return cast_to == this; } @@ -593,17 +705,29 @@ bool TypeDataContinuation::can_be_casted_with_as_operator(TypePtr cast_to) const if (const auto* to_nullable = cast_to->try_as()) { return can_be_casted_with_as_operator(to_nullable->inner); } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } return cast_to == this; } bool TypeDataNullLiteral::can_be_casted_with_as_operator(TypePtr cast_to) const { - return cast_to == this || cast_to->try_as(); + if (cast_to->try_as()) { + return true; + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } + return cast_to == this; } bool TypeDataNullable::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { return inner->can_be_casted_with_as_operator(to_nullable->inner); } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } return false; } @@ -611,7 +735,21 @@ bool TypeDataFunCallable::can_be_casted_with_as_operator(TypePtr cast_to) const if (const auto* to_nullable = cast_to->try_as()) { return can_be_casted_with_as_operator(to_nullable->inner); } - return this == cast_to; + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } + if (const TypeDataFunCallable* to_callable = cast_to->try_as()) { + if (to_callable->params_size() != params_size()) { + return false; + } + for (int i = 0; i < params_size(); ++i) { + if (!params_types[i]->can_be_casted_with_as_operator(to_callable->params_types[i])) { + return false; + } + } + return return_type->can_be_casted_with_as_operator(to_callable->return_type); + } + return false; } bool TypeDataGenericT::can_be_casted_with_as_operator(TypePtr cast_to) const { @@ -630,6 +768,9 @@ bool TypeDataTensor::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { return can_be_casted_with_as_operator(to_nullable->inner); } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } return false; } @@ -645,6 +786,9 @@ bool TypeDataTypedTuple::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { return can_be_casted_with_as_operator(to_nullable->inner); } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } return false; } @@ -655,6 +799,9 @@ bool TypeDataIntN::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { // `int8` as `int32?` return can_be_casted_with_as_operator(to_nullable->inner); } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } return cast_to == TypeDataInt::create() || cast_to == TypeDataCoins::create(); } @@ -665,6 +812,9 @@ bool TypeDataBytesN::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { // `bytes8` as `slice?` return can_be_casted_with_as_operator(to_nullable->inner); } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } return cast_to == TypeDataSlice::create(); } @@ -675,6 +825,9 @@ bool TypeDataCoins::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_nullable = cast_to->try_as()) { // `coins` as `coins?` / `coins` as `int?` return can_be_casted_with_as_operator(to_nullable->inner); } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } if (cast_to == TypeDataInt::create()) { return true; } @@ -708,6 +861,10 @@ bool TypeDataVoid::can_be_casted_with_as_operator(TypePtr cast_to) const { // though still, tricky situations like `(int, ())?` can still "embed" TVM NULL in parallel with original value // +bool TypeDataAlias::can_hold_tvm_null_instead() const { + return underlying_type->can_hold_tvm_null_instead(); +} + bool TypeDataNullable::can_hold_tvm_null_instead() const { if (get_width_on_stack() != 1) { // `(int, int)?` / `()?` can not hold null instead return false; // only `int?` / `cell?` / `StructWith1IntField?` can diff --git a/tolk/type-system.h b/tolk/type-system.h index 07a2496ca..ebb25312e 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -83,6 +83,8 @@ class TypeData { bool has_genericT_inside() const { return flags & flag_contains_genericT_inside; } bool has_unresolved_inside() const { return flags & flag_contains_unresolved_inside; } + inline TypePtr unwrap_alias() const; + using TraverserCallbackT = std::function; using ReplacerCallbackT = std::function; @@ -103,6 +105,32 @@ class TypeData { } }; +/* + * `type AliasName = underlying_type` is an alias, which is fully interchangeable with its original type. + * It never occurs at runtime: at IR generation it's erased, replaced by an underlying type. + * But until IR generation, aliases exists, and `var t: MyTensor2 = (1,2)` is alias "MyTensor", not tensor (int,int). + * That's why lots of code comparing types use `type->unwrap_alias()` or `try_as`. + */ +class TypeDataAlias final : public TypeData { + explicit TypeDataAlias(uint64_t type_id, int children_flags, std::string&& alias_name, TypePtr underlying_type) + : TypeData(type_id, children_flags, underlying_type->get_width_on_stack()) + , alias_name(std::move(alias_name)) + , underlying_type(underlying_type) {} + +public: + const std::string alias_name; + const TypePtr underlying_type; + + static TypePtr create(const std::string& alias_name, TypePtr underlying_type); + + std::string as_human_readable() const override { return alias_name; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + void traverse(const TraverserCallbackT& callback) const override; + TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; + bool can_hold_tvm_null_instead() const override; +}; + /* * `int` is TypeDataInt, representation of TVM int. */ @@ -519,6 +547,13 @@ class TypeDataVoid final : public TypeData { // -------------------------------------------- +inline TypePtr TypeData::unwrap_alias() const { + TypePtr unwrapped = this; + while (const TypeDataAlias* as_alias = unwrapped->try_as()) { + unwrapped = as_alias->underlying_type; + } + return unwrapped; +} class Lexer; TypePtr parse_type_from_tokens(Lexer& lex); From 249badf4b39851db69fd1cc0351d7088f768a7bd Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sat, 5 Apr 2025 17:11:48 +0300 Subject: [PATCH 201/388] [Tolk] Union types `T1 | T2 | ...` and pattern matching Union types allow a variable to hold multiple possible types. They are key to message parsing and routing in future struct-based designs. At the TVM level, they are tagged unions, like enums in Rust. Pattern matching is the only way of handling union types: > match (result) { > slice => { /* result is smart-casted to slice */ } > UserId => { /* result is smart-casted to UserId */ } > } `match` must cover all union cases (should be exhaustive). It can also be used as an expression. Variable declaration inside `match` is allowed. Besides `match`, you can test a union type by `is`: > if (v is cell) { > // v is smart-casted to cell --- tolk-tester/tests/generics-1.tolk | 28 + tolk-tester/tests/intN-tests.tolk | 16 +- .../tests/invalid-declaration/err-1140.tolk | 26 + .../tests/invalid-declaration/err-1473.tolk | 16 + .../tests/invalid-semantics/err-4355.tolk | 21 + .../tests/invalid-symbol/err-2208.tolk | 13 + .../tests/invalid-symbol/err-2355.tolk | 12 + .../tests/invalid-symbol/err-2898.tolk | 16 + .../tests/invalid-syntax/err-3360.tolk | 10 + .../tests/invalid-syntax/err-3390.tolk | 4 +- .../tests/invalid-syntax/err-3457.tolk | 18 + .../tests/invalid-typing/err-6089.tolk | 8 + .../tests/invalid-typing/err-6110.tolk | 15 + .../tests/invalid-typing/err-6144.tolk | 13 + .../tests/invalid-typing/err-6204.tolk | 7 +- .../tests/invalid-typing/err-6274.tolk | 19 + .../tests/invalid-typing/err-6353.tolk | 30 + .../tests/invalid-typing/err-6366.tolk | 13 + .../tests/invalid-typing/err-6427.tolk | 13 + .../tests/invalid-typing/err-6460.tolk | 19 + .../tests/invalid-typing/err-6498.tolk | 14 + .../tests/invalid-typing/err-6502.tolk | 11 + .../tests/invalid-typing/err-6518.tolk | 16 + .../tests/invalid-typing/err-6583.tolk | 12 + .../tests/invalid-typing/err-6659.tolk | 18 + .../tests/invalid-typing/err-6673.tolk | 14 + .../tests/invalid-typing/err-6690.tolk | 12 + .../tests/invalid-typing/err-6712.tolk | 12 + .../tests/invalid-typing/err-6767.tolk | 10 + .../tests/invalid-typing/err-6801.tolk | 9 + .../tests/invalid-typing/err-6808.tolk | 11 + .../tests/invalid-typing/err-6829.tolk | 16 + .../tests/invalid-typing/err-6853.tolk | 8 + .../tests/invalid-typing/err-6920.tolk | 12 + .../tests/invalid-typing/err-6930.tolk | 15 + tolk-tester/tests/nullable-tensors.tolk | 65 +- tolk-tester/tests/smart-cast-tests.tolk | 30 +- tolk-tester/tests/type-aliases-tests.tolk | 11 +- tolk-tester/tests/union-types-tests.tolk | 873 ++++++++++++++++++ tolk-tester/tests/unreachable-5.tolk | 24 + tolk-tester/tests/var-apply-tests.tolk | 17 +- .../warnings-not-errors/unreachable-3.tolk | 2 +- .../tests/warnings-not-errors/warnings-1.tolk | 8 +- .../tests/warnings-not-errors/warnings-2.tolk | 8 +- .../tests/warnings-not-errors/warnings-3.tolk | 26 + tolk-tester/tolk-tester.py | 2 +- tolk/abscode.cpp | 15 +- tolk/ast-from-tokens.cpp | 317 ++++--- tolk/ast-replacer.h | 16 +- tolk/ast-replicator.h | 18 +- tolk/ast-stringifier.h | 22 +- tolk/ast-visitor.h | 14 +- tolk/ast.cpp | 18 +- tolk/ast.h | 95 +- tolk/generics-helpers.cpp | 15 +- tolk/lexer.cpp | 6 +- tolk/lexer.h | 4 + tolk/pipe-ast-to-legacy.cpp | 464 +++++++--- tolk/pipe-calc-rvalue-lvalue.cpp | 141 ++- tolk/pipe-check-inferred-types.cpp | 116 ++- tolk/pipe-check-rvalue-lvalue.cpp | 90 +- tolk/pipe-constant-folding.cpp | 4 +- tolk/pipe-infer-types-and-calls.cpp | 130 ++- tolk/pipe-optimize-boolean-expr.cpp | 6 +- tolk/pipe-resolve-identifiers.cpp | 50 +- tolk/smart-casts-cfg.cpp | 159 ++-- tolk/smart-casts-cfg.h | 7 +- tolk/type-system.cpp | 586 +++++++++--- tolk/type-system.h | 201 ++-- 69 files changed, 3361 insertions(+), 706 deletions(-) create mode 100644 tolk-tester/tests/invalid-declaration/err-1140.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1473.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4355.tolk create mode 100644 tolk-tester/tests/invalid-symbol/err-2208.tolk create mode 100644 tolk-tester/tests/invalid-symbol/err-2355.tolk create mode 100644 tolk-tester/tests/invalid-symbol/err-2898.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3360.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3457.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6089.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6110.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6144.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6274.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6353.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6366.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6427.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6460.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6498.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6502.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6518.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6583.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6659.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6673.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6690.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6712.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6767.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6801.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6808.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6829.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6853.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6920.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6930.tolk create mode 100644 tolk-tester/tests/union-types-tests.tolk create mode 100644 tolk-tester/tests/unreachable-5.tolk create mode 100644 tolk-tester/tests/warnings-not-errors/warnings-3.tolk diff --git a/tolk-tester/tests/generics-1.tolk b/tolk-tester/tests/generics-1.tolk index 70cd2371c..feb7a8393 100644 --- a/tolk-tester/tests/generics-1.tolk +++ b/tolk-tester/tests/generics-1.tolk @@ -24,6 +24,7 @@ fun test102(): (int, int, int, [int, int]) { var b = getTwo() as int; var c: int = 1 ? getTwo() : getTwo(); var c redef = getTwo(); + var d: int = match (1 as int|slice|builder) { int => getTwo(), slice => getTwo(), builder => getTwo() }; var ab_tens = (0, (1, 2)); ab_tens.0 = getTwo(); ab_tens.1.1 = getTwo(); @@ -169,6 +170,29 @@ fun test111() { return (isSomeNullableNull(i1), isSomeNullableNull(i2), isSomeNullableNull(i1), isSomeNullableNull(i1)); } +@method_id(112) +fun test112(v: int | slice?) { + return (eq1(v), eq4(v == null)); +} + +fun makeNullable(a: T1 | T2): T1 | T2 | null { + return a; +} + +@method_id(113) +fun test113(a: int | slice) { + var b: int | (int, int) = match (a) { + int => a, + slice => (a.loadInt(32), -1) + }; + __expect_type(makeNullable(b), "int | (int, int) | null"); + return ( + makeNullable(a), -100, makeNullable(b), -100, makeNullable(9), -100, makeNullable(null), -100, + makeNullable(b), -100, makeNullable(null) + ); +} + + fun main(x: int): (int, [Tup2Int]) { __expect_type(test110, "() -> (Tensor2Int, Tensor2Int)"); @@ -190,12 +214,16 @@ fun main(x: int): (int, [Tup2Int]) { @testcase | 109 | 5 | 5 (null) (null) @testcase | 110 | | 2 2 5 2 @testcase | 111 | | 0 0 0 0 +@testcase | 112 | 5 1 | 5 1 0 +@testcase | 112 | 0 0 | 0 0 -1 +@testcase | 113 | 5 1 | 5 1 -100 (null) 5 1 -100 9 -100 (null) -100 (null) 5 1 -100 (null) (null) (null) 0 @fif_codegen DECLPROC eq1 @fif_codegen DECLPROC eq1 @fif_codegen DECLPROC eq1<(int,int)> @fif_codegen DECLPROC eq1<[int,int]> @fif_codegen DECLPROC eq1 +@fif_codegen DECLPROC eq1 @fif_codegen DECLPROC getTwo @fif_codegen_avoid DECLPROC eq1 diff --git a/tolk-tester/tests/intN-tests.tolk b/tolk-tester/tests/intN-tests.tolk index abafdf1d7..d12436f5e 100644 --- a/tolk-tester/tests/intN-tests.tolk +++ b/tolk-tester/tests/intN-tests.tolk @@ -23,13 +23,13 @@ fun autoInferUInt16Opt(x: uint16) { return null; } -fun autoInferInt_v1() { +fun inferInt_v1(): int { if (random()) { return 0; } else if (random()) { return 0 as uint16; } else { return 0 as int8; } } -fun autoInferInt_v2() { +fun inferInt_v2(): int { if (random()) { return 0 as uint1; } else if (random()) { return 0 as int123; } else { return 0 as int8; } @@ -87,8 +87,8 @@ fun test3(op: int32, qid: uint64) { __expect_type(autoInferInt8, "(int8) -> int8"); __expect_type(autoInferUInt16Opt(0), "uint16?"); - __expect_type(autoInferInt_v1(), "int"); - __expect_type(autoInferInt_v2(), "int"); + __expect_type(inferInt_v1(), "int"); + __expect_type(inferInt_v2(), "int"); var amount: uint100 = 1000; var percent: uint8 = 50; @@ -124,7 +124,7 @@ fun test5() { fun test6() { var x: int11 = ~(0 as int22); while (x) {} - return x ? x : 0; + return x ? x as int : 0; } fun test7() { @@ -211,7 +211,7 @@ fun test13(x1: int?, x2: int8?, x3: int, x4: int8): (int8?, int8?, int8?, int8?, @method_id(114) fun test14(firstComponent: int8?): (int, int16)? { - return firstComponent! < 10 ? null : (firstComponent!, 2); + return firstComponent! < 10 ? null : (firstComponent! as int, 2 as int16); } fun assign0(mutate v: T) { v = 0; } @@ -232,7 +232,7 @@ fun main() { __expect_type(0 as coins, "coins"); __expect_type(someN_coins as coins?, "coins?"); __expect_type(someN_coins as int8?, "int8?"); - __expect_type(10>3 ? 0 as uint8 : 0 as uint16, "int"); + __expect_type(10>3 ? (0 as uint8) as uint8 | uint16 : 0 as uint16, "uint8 | uint16"); return t; } @@ -245,7 +245,7 @@ fun main() { @testcase | 110 | | 50000000 50000100 1234000000 51000000 @testcase | 111 | | [ 1000000000 1000000000 1000000000 -321123456789 321123456789 1100000000 ] @testcase | 114 | 5 | (null) (null) 0 -@testcase | 114 | 15 | 15 2 -1 +@testcase | 114 | 15 | 15 2 130 @fif_codegen DECLPROC assign0 @fif_codegen DECLPROC assign0 diff --git a/tolk-tester/tests/invalid-declaration/err-1140.tolk b/tolk-tester/tests/invalid-declaration/err-1140.tolk new file mode 100644 index 000000000..eb1ca04e5 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1140.tolk @@ -0,0 +1,26 @@ +fun okWithJustInt(a: int8, b: int16): int { + if (a > 10) { + return a; + } + return b; +} + +fun okWithManualUnion(a: int8, b: int16): int16 | int8 { + if (a > 10) { + return a; + } + return b; +} + +fun autoInferUnionType(a: int8, b: int16) { + if (a > 10) { + return a; + } + return b; +} + +/** +@compilation_should_fail +@stderr function `autoInferUnionType` calculated return type is `int8 | int16`; probably, it's not what you expected; declare `fun (...): ` manually +@stderr fun autoInferUnionType + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1473.tolk b/tolk-tester/tests/invalid-declaration/err-1473.tolk new file mode 100644 index 000000000..ef79b5033 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1473.tolk @@ -0,0 +1,16 @@ +fun getIntOrSlice() { return 5 as int | slice; } + +fun autoInferUnionType(a: int | slice | null) { + var cc = match (a) { + int => a = getIntOrSlice(), + slice => return beginCell(), + null => return -20 + }; + return cc; +} + +/** +@compilation_should_fail +@stderr function `autoInferUnionType` calculated return type is `builder | int | slice`; probably, it's not what you expected +@stderr fun autoInferUnionType + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4355.tolk b/tolk-tester/tests/invalid-semantics/err-4355.tolk new file mode 100644 index 000000000..bb648dfa4 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4355.tolk @@ -0,0 +1,21 @@ +fun increment(mutate x: int) { + x += 1; +} + +fun cantModifyValDeclaredInMatch() { + match (var a = (0 + 10 + 90) as int | slice) { + int => increment(mutate a), + slice => {}, + }; + match (val b = (0 + 10 + 90) as int | slice) { + int => increment(mutate b), + slice => {}, + }; +} + +/** +@compilation_should_fail +@stderr modifying immutable variable `b` +@stderr in function `cantModifyValDeclaredInMatch` +@stderr increment(mutate b) + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2208.tolk b/tolk-tester/tests/invalid-symbol/err-2208.tolk new file mode 100644 index 000000000..b918e0abf --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2208.tolk @@ -0,0 +1,13 @@ +fun main(x: int | slice) { + match (var a = x) { // a variable declared in `match` subject exists only within a match + int => a + 0, + slice => a.loadInt(32), + } + return a; +} + +/** +@compilation_should_fail +@stderr undefined symbol `a` +@stderr return a; + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2355.tolk b/tolk-tester/tests/invalid-symbol/err-2355.tolk new file mode 100644 index 000000000..f85dc2591 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2355.tolk @@ -0,0 +1,12 @@ + +fun main() { + match (10) { + asdf => 1, + }; +} + +/** +@compilation_should_fail +@stderr unknown type name `asdf` +@stderr asdf => 1 + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2898.tolk b/tolk-tester/tests/invalid-symbol/err-2898.tolk new file mode 100644 index 000000000..ba1de7751 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2898.tolk @@ -0,0 +1,16 @@ +fun main(a: int | slice) { + match (a) { + int => { + var cc: int = a; + }, + slice => { + return cc; + }, + }; +} + +/** +@compilation_should_fail +@stderr undefined symbol `cc` +@stderr return cc; + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3360.tolk b/tolk-tester/tests/invalid-syntax/err-3360.tolk new file mode 100644 index 000000000..7f1b46602 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3360.tolk @@ -0,0 +1,10 @@ + +fun main(r: int | slice) { + // after `is` we expect a type, parse `int?`, and expression becomes incorrect + return r is int ? 10 : 20; +} + +/** +@compilation_should_fail +@stderr expected `;`, got `10` + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3390.tolk b/tolk-tester/tests/invalid-syntax/err-3390.tolk index 58927d3a0..404aa959c 100644 --- a/tolk-tester/tests/invalid-syntax/err-3390.tolk +++ b/tolk-tester/tests/invalid-syntax/err-3390.tolk @@ -1,8 +1,8 @@ fun main(): int { - ;; here is not a comment + ;; here not a comment } /** @compilation_should_fail -@stderr error: expected `;`, got `is` +@stderr error: expected `;`, got `not` */ diff --git a/tolk-tester/tests/invalid-syntax/err-3457.tolk b/tolk-tester/tests/invalid-syntax/err-3457.tolk new file mode 100644 index 000000000..a2ef6e96b --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3457.tolk @@ -0,0 +1,18 @@ +type asdf = int; + +fun main() { + match (10) { + asdf => 1, // it's match by type + }; + + var asdf = 5; + match (10) { + asdf => 2, // it's match by expression now + }; +} + +/** +@compilation_should_fail +@stderr `match` by expression is not supported yet +@stderr asdf => 2 + */ diff --git a/tolk-tester/tests/invalid-typing/err-6089.tolk b/tolk-tester/tests/invalid-typing/err-6089.tolk new file mode 100644 index 000000000..8548d1781 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6089.tolk @@ -0,0 +1,8 @@ +fun unionTypesNotAllowedInIs(a: int | slice | builder) { + if (a is int | slice) {} +} + +/** +@compilation_should_fail +@stderr union types are not allowed, use concrete types in `is` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6110.tolk b/tolk-tester/tests/invalid-typing/err-6110.tolk new file mode 100644 index 000000000..fbef5abe6 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6110.tolk @@ -0,0 +1,15 @@ +fun makeNullable(a: T1 | T2): T1 | T2 | null { + return a; +} + +fun cantDeduceDefiniteUnion(a: int | slice | builder) { + makeNullable(a); // ok + makeNullable(a); // ok + makeNullable(a); +} + +/** +@compilation_should_fail +@stderr can not deduce T1 for generic function `makeNullable` +@stderr makeNullable(a); + */ diff --git a/tolk-tester/tests/invalid-typing/err-6144.tolk b/tolk-tester/tests/invalid-typing/err-6144.tolk new file mode 100644 index 000000000..6fe610eff --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6144.tolk @@ -0,0 +1,13 @@ +type IntOrSlice = int | slice; + +fun unionTypesNotAllowedInMatch(a: int | slice) { + match (a) { + IntOrSlice => 1, + }; +} + +/** +@compilation_should_fail +@stderr wrong pattern matching: union types are not allowed, use concrete types in `match` +@stderr IntOrSlice + */ diff --git a/tolk-tester/tests/invalid-typing/err-6204.tolk b/tolk-tester/tests/invalid-typing/err-6204.tolk index 58208caab..8f2b540da 100644 --- a/tolk-tester/tests/invalid-typing/err-6204.tolk +++ b/tolk-tester/tests/invalid-typing/err-6204.tolk @@ -1,10 +1,13 @@ fun cantUnifyCoinsAndUInt8(n: int8, c: coins) { __expect_type(random() ? n : c as int8, "int8"); - __expect_type(random() ? n : c as int16, "int"); - random() ? n : c; + __expect_type(random() ? n as int16 : c as int16, "int16"); + + var withHint: int = random() ? n : c; // ok + var withoutHint = random() ? n : c; // error } /** @compilation_should_fail @stderr types of ternary branches are incompatible: `int8` and `coins` +@stderr var withoutHint */ diff --git a/tolk-tester/tests/invalid-typing/err-6274.tolk b/tolk-tester/tests/invalid-typing/err-6274.tolk new file mode 100644 index 000000000..d586efe41 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6274.tolk @@ -0,0 +1,19 @@ +type MInt = int; +type MInt2 = MInt | int; +type MSlice = slice; +type MCell = cell; +type MBuilder = builder; + +fun cantMatchAgainstNotVariant(a: int | slice | MBuilder) { + match (a) { + MInt2 => 10, + MSlice => 20, + MCell => 30, + }; +} + +/** +@compilation_should_fail +@stderr wrong pattern matching: `cell` is not a variant of `int | slice | MBuilder` +@stderr MCell + */ diff --git a/tolk-tester/tests/invalid-typing/err-6353.tolk b/tolk-tester/tests/invalid-typing/err-6353.tolk new file mode 100644 index 000000000..fb2537592 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6353.tolk @@ -0,0 +1,30 @@ +fun cantInferUnionTypeWithoutHint(a: int | slice | builder) { + var ok1: builder | (slice | int | int) = match (a) { + int => a, + slice => a, + builder => a, + }; + var ok2 = match (a) { + int => a, + slice => a, + builder => a, + } as builder | (slice | int | int); + var ok3 = match (a) { + int => a as builder | slice | int, + slice => a as builder | slice | int, + builder => a as builder | slice, + }; + + var err = match (a) { + int => a!, + slice => a!, + builder => a!, + }; +} + +/** +@compilation_should_fail +@stderr type of `match` was inferred as `int | slice | builder`; probably, it's not what you expected +@stderr assign it to a variable `var v: = match (...) { ... }` manually +@stderr err = match (a) + */ diff --git a/tolk-tester/tests/invalid-typing/err-6366.tolk b/tolk-tester/tests/invalid-typing/err-6366.tolk new file mode 100644 index 000000000..cc6052d99 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6366.tolk @@ -0,0 +1,13 @@ + +fun main() { + match (0 as int | slice) { + int => 1, + else => 10, + }; +} + +/** +@compilation_should_fail +@stderr `else` is not allowed in `match` by type; you should cover all possible types +@stderr else => 10 + */ diff --git a/tolk-tester/tests/invalid-typing/err-6427.tolk b/tolk-tester/tests/invalid-typing/err-6427.tolk new file mode 100644 index 000000000..37df89a1e --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6427.tolk @@ -0,0 +1,13 @@ +fun cantReturnVoidAndNonVoid(a: int | slice | builder) { + match (a) { + int => { return; } + slice => {} + builder => "" + }; + return 123; +} + +/** +@compilation_should_fail +@stderr mixing void and non-void returns in function + */ diff --git a/tolk-tester/tests/invalid-typing/err-6460.tolk b/tolk-tester/tests/invalid-typing/err-6460.tolk new file mode 100644 index 000000000..554f8b4d3 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6460.tolk @@ -0,0 +1,19 @@ +type MNull = null; +type MNull2 = MNull; + +fun main(t: MNull2) { + t as null; + t as MNull; + t as MNull2; + t as int?; + t as int | slice | null; + t as (int, int) | slice | MNull; + + t as int; // error +} + +/** +@compilation_should_fail +@stderr type `null` can not be cast to `int` +@stderr t as int + */ diff --git a/tolk-tester/tests/invalid-typing/err-6498.tolk b/tolk-tester/tests/invalid-typing/err-6498.tolk new file mode 100644 index 000000000..79da03b65 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6498.tolk @@ -0,0 +1,14 @@ + +fun main() { + var a = 0 as int | slice; + var bb = match (a) { + }; + __expect_type(a, "int | slice"); + return a; +} + +/** +@compilation_should_fail +@stderr empty `match` can't be used as expression +@stderr var bb = match + */ diff --git a/tolk-tester/tests/invalid-typing/err-6502.tolk b/tolk-tester/tests/invalid-typing/err-6502.tolk new file mode 100644 index 000000000..2f9e664d8 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6502.tolk @@ -0,0 +1,11 @@ +fun cantAutoInferUnionVariant() { + var a: (int?, int?) | (int, int?) | int = 5; + a = (1, 2) as (int?, int?); // ok + a = (1 as int?, 2 as int?); // ok + a = (1, 2); +} + +/** +@compilation_should_fail +@stderr can not assign `(int, int)` to variable of type `(int?, int?) | (int, int?) | int` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6518.tolk b/tolk-tester/tests/invalid-typing/err-6518.tolk new file mode 100644 index 000000000..382767744 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6518.tolk @@ -0,0 +1,16 @@ +type MNull = null; +type IntOr16 = int | int16; + +fun main(t: int?) { + t as int? | null; + t as int? | slice; + t as (int, int) | MNull | int8 | IntOr16; + + t as slice?; +} + +/** +@compilation_should_fail +@stderr type `int?` can not be cast to `slice?` +@stderr t as slice? + */ diff --git a/tolk-tester/tests/invalid-typing/err-6583.tolk b/tolk-tester/tests/invalid-typing/err-6583.tolk new file mode 100644 index 000000000..530ff95cc --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6583.tolk @@ -0,0 +1,12 @@ +fun cantAssignIncompatibleUnionTypes(a: int | slice | builder) { + var b: int | slice = match (a) { + int => a, + slice => a, + builder => null, + }; +} + +/** +@compilation_should_fail +@stderr can not assign `int | slice | null` to variable of type `int | slice` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6659.tolk b/tolk-tester/tests/invalid-typing/err-6659.tolk new file mode 100644 index 000000000..82fa462b6 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6659.tolk @@ -0,0 +1,18 @@ +fun cantAutoInferUnionInTernary() { + var dd: int | slice = random() ? 1 : ""; // ok + var cc = (random() ? 1 : "") as int|slice; // ok + random() ? 1 as int|slice : "" as int|slice; // ok + random() ? 1 as int|slice : ""; // ok + random() ? 1 : "" as int|slice|builder; // ok + random() ? 1 as int|slice|null : "" as slice?; // ok + + var sub = random() ? (1, 2 as int|slice) : (1, ""); + __expect_type(sub, "(int, int | slice)"); + + random() ? 1 : ""; +} + +/** +@compilation_should_fail +@stderr types of ternary branches are incompatible: `int` and `slice` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6673.tolk b/tolk-tester/tests/invalid-typing/err-6673.tolk new file mode 100644 index 000000000..59a6ba283 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6673.tolk @@ -0,0 +1,14 @@ +fun cantAutoInferUnionVariant() { + var a: int8 | int16 = 0 as int8; // ok + a = 0 as int16; // ok + if (a is int8) { + a = 1 as int8; // ok + } + a = 2; +} + +/** +@compilation_should_fail +@stderr can not assign `int` to variable of type `int8 | int16` +@stderr a = 2; + */ diff --git a/tolk-tester/tests/invalid-typing/err-6690.tolk b/tolk-tester/tests/invalid-typing/err-6690.tolk new file mode 100644 index 000000000..cfbcd8cf4 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6690.tolk @@ -0,0 +1,12 @@ +fun main(t: int?) { + t!; + t! as int; + + t as int; +} + +/** +@compilation_should_fail +@stderr type `int?` can not be cast to `int` +@stderr t as int; + */ diff --git a/tolk-tester/tests/invalid-typing/err-6712.tolk b/tolk-tester/tests/invalid-typing/err-6712.tolk new file mode 100644 index 000000000..c0210a7bc --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6712.tolk @@ -0,0 +1,12 @@ +fun matchDoesntCoverAllCases(a: int | slice?) { + match (a) { + int => {} + slice => {} + }; +} + +/** +@compilation_should_fail +@stderr `match` does not cover all possible types; missing types are: `null` +@stderr match (a) + */ diff --git a/tolk-tester/tests/invalid-typing/err-6767.tolk b/tolk-tester/tests/invalid-typing/err-6767.tolk new file mode 100644 index 000000000..f9ddf7273 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6767.tolk @@ -0,0 +1,10 @@ +type MInt = int; + +fun main() { + return (5 as int8?) as MInt | slice | null; +} + +/** +@compilation_should_fail +@stderr type `int8?` can not be cast to `MInt | slice | null` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6801.tolk b/tolk-tester/tests/invalid-typing/err-6801.tolk new file mode 100644 index 000000000..4646bd7be --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6801.tolk @@ -0,0 +1,9 @@ + +fun main(a: slice | null | int8) { + a as slice | null; +} + +/** +@compilation_should_fail +@stderr type `slice | null | int8` can not be cast to `slice?` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6808.tolk b/tolk-tester/tests/invalid-typing/err-6808.tolk new file mode 100644 index 000000000..46db0d6de --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6808.tolk @@ -0,0 +1,11 @@ +fun main() { + (1 as int16) as (int | int16); // ok, exact match + (1 as int8) as (int | int32); // ok, int accepts int8, but int32 not + + 1 as (int16 | int32); // error, ambiguous +} + +/** +@compilation_should_fail +@stderr type `int` can not be cast to `int16 | int32` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6829.tolk b/tolk-tester/tests/invalid-typing/err-6829.tolk new file mode 100644 index 000000000..fbf462367 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6829.tolk @@ -0,0 +1,16 @@ +type Pair2 = (int, int); +type Pair2Or3 = Pair2 | (int, int, int); + +fun matchDoesntCoverAllCases(a: Pair2Or3 | slice | [[int]] | builder) { + if (a !is slice) { + match (a) { + (int, int) => {} + [[int]] => {} + } + } +} + +/** +@compilation_should_fail +@stderr `match` does not cover all possible types; missing types are: `(int, int, int)`, `builder` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6853.tolk b/tolk-tester/tests/invalid-typing/err-6853.tolk new file mode 100644 index 000000000..f901ad48f --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6853.tolk @@ -0,0 +1,8 @@ +fun cantAssignIncompatiblePrimitiveNullable(m: int?) { + var a: slice? = m; +} + +/** +@compilation_should_fail +@stderr can not assign `int?` to variable of type `slice?` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6920.tolk b/tolk-tester/tests/invalid-typing/err-6920.tolk new file mode 100644 index 000000000..dc2a5ffce --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6920.tolk @@ -0,0 +1,12 @@ +fun matchOverNotSubtypeIsNever(m: slice | cell) { + match (m) { + int => return m + 0 + }; + return 3; +} + +/** +@compilation_should_fail +@stderr `int` is not a variant of `slice | cell` +@stderr int => + */ diff --git a/tolk-tester/tests/invalid-typing/err-6930.tolk b/tolk-tester/tests/invalid-typing/err-6930.tolk new file mode 100644 index 000000000..80269c3dc --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6930.tolk @@ -0,0 +1,15 @@ +type MInt = int; + +fun f1(v: int): int { return v; } + +fun main(t: int?) { + f1 as (MInt) -> MInt; + f1 as (int8) -> uint16; + + f1 as ((int | slice) -> int); +} + +/** +@compilation_should_fail +@stderr type `(int) -> int` can not be cast to `(int | slice) -> int` + */ diff --git a/tolk-tester/tests/nullable-tensors.tolk b/tolk-tester/tests/nullable-tensors.tolk index 9e3a269b5..c1de8c0b7 100644 --- a/tolk-tester/tests/nullable-tensors.tolk +++ b/tolk-tester/tests/nullable-tensors.tolk @@ -430,8 +430,8 @@ fun test136(x: int?, ) { @method_id(140) fun test140(x: int8, y: int16) { - var t1: (int32, int)? = (x, y) as (int32, int64)?; - var t2: (int32, int)? = null as (int32, int64)?; + var t1: (int32, int64)? = ((x, y) as (int32, int64)) as (int32, int64)?; + var t2: (int32, int64)? = null as (int32, int64)?; return (t1, t2); } @@ -456,66 +456,83 @@ fun test142() { return (sum1, sum2, t!); } +@method_id(143) +fun test143(setBNull: bool, setANullMid: bool) { + var a: (int, int | slice | null, int)? = (1, 2, 3); + var b = setBNull ? null : a; + if (setANullMid) { + a.1 = null; + } + if (a.1 != null && b != null) { + b.0 += match (a.1) { int => 100, slice => 200 }; + } + __expect_type(a, "(int, int | slice | null, int)"); + __expect_type(b, "(int, int | slice | null, int)?"); + return (a, 777, b); +} + fun main(){} /** -@testcase | 101 | | 1 2 -1 -@testcase | 102 | | 1 2 -1 (null) (null) 0 +@testcase | 101 | | 1 2 129 +@testcase | 102 | | 1 2 129 (null) (null) 0 @testcase | 103 | 1 2 | 3 3 0 1 2 @testcase | 104 | | 1 2 (null) (null) 0 -@testcase | 105 | | (null) (null) (null) 0 1 2 3 -1 +@testcase | 105 | | (null) (null) (null) 0 1 2 3 131 @testcase | 106 | | 1 2 @testcase | 107 | | 0 0 -1 0 0 -1 -@testcase | 108 | 5 6 | 7 8 10 11 -1 (null) (null) 0 +@testcase | 108 | 5 6 | 7 8 10 11 129 (null) (null) 0 @testcase | 109 | | 0 0 -1 0 -1 0 0 -1 -1 -@testcase | 110 | | 3 4 (null) (null) 0 6 7 -1 +@testcase | 110 | | 3 4 (null) (null) 0 6 7 129 @testcase | 111 | | 50 30 70 90 100 @testcase | 112 | | 12 22 @testcase | 113 | | -1 @testcase | 114 | | (null) (null) (null) 0 (null) (null) (null) 0 -@testcase | 115 | | 2 3 7 (null) (null) 0 5 0 -1 0 +@testcase | 115 | | 2 3 7 (null) (null) 0 5 0 129 0 @testcase | 116 | -1 | (null) (null) 0 (null) (null) 0 -@testcase | 116 | 0 | 1 2 -1 1 2 -1 +@testcase | 116 | 0 | 1 2 129 1 2 129 @testcase | 117 | | (null) 1 3 -@testcase | 118 | 5 | 5 10 -1 +@testcase | 118 | 5 | 5 10 129 @testcase | 118 | null | (null) (null) 0 -@testcase | 119 | | (null) (null) 1 2 -1 100 +@testcase | 119 | | (null) (null) 1 2 129 100 @testcase | 120 | -1 | (null) (null) 0 -@testcase | 120 | 0 | 1 2 -1 +@testcase | 120 | 0 | 1 2 129 @testcase | 121 | | [ 1 [ 3 4 ] ] @testcase | 122 | 0 | [ 1 [ 3 4 ] 4 (null) ] @testcase | 122 | -1 | [ 1 (null) 4 (null) ] -@testcase | 123 | | 1 3 4 -1 -@testcase | 124 | 0 | 1 3 4 -1 4 (null) (null) 0 +@testcase | 123 | | 1 3 4 132 +@testcase | 124 | 0 | 1 3 4 132 4 (null) (null) 0 @testcase | 124 | -1 | 1 (null) (null) 0 4 (null) (null) 0 @testcase | 125 | | 3 @testcase | 126 | | 1 (null) 2 @testcase | 127 | 1 | 1 (null) (null) 0 2 -@testcase | 127 | 2 | 1 2 3 -1 4 +@testcase | 127 | 2 | 1 2 3 129 4 @testcase | 127 | 3 | 1 (null) (null) 0 5 -@testcase | 128 | 1 | 1 (null) (null) 0 2 -1 +@testcase | 128 | 1 | 1 (null) (null) 0 2 139 @testcase | 128 | 2 | (null) (null) (null) (null) (null) 0 -@testcase | 128 | 3 | 1 2 3 -1 4 -1 +@testcase | 128 | 3 | 1 2 3 129 4 139 @testcase | 129 | 0 | 5 5 0 -1 1 2 0 -1 @testcase | 129 | -1 | 5 5 0 -1 (null) (null) 0 -1 -@testcase | 130 | 0 | 1 2 3 -1 +@testcase | 130 | 0 | 1 2 3 129 @testcase | 130 | -1 | 1 (null) (null) 0 -@testcase | 131 | | -1 777 0 777 777 777 0 0 -1 -1 777 -1 -1 -1 777 +@testcase | 131 | | 140 777 0 777 777 777 0 0 140 140 777 140 140 141 777 @testcase | 132 | | -1 0 -1 0 777 (null) (null) -1 0 0 @testcase | 133 | | 60 -@testcase | 134 | | 11 21 -1 -@testcase | 135 | | [ 10 ] [ (null) ] (null) 777 10 -1 (null) -1 (null) 0 777 10 -1 (null) -1 (null) 0 777 0 0 -1 0 0 -1 0 0 -1 777 0 -1 0 0 -1 0 +@testcase | 134 | | 11 21 129 +@testcase | 135 | | [ 10 ] [ (null) ] (null) 777 10 146 (null) 146 (null) 0 777 10 143 (null) 143 (null) 0 777 0 0 -1 0 0 -1 0 0 -1 777 0 -1 0 0 -1 0 @testcase | 136 | 9 | 9 0 @testcase | 136 | null | (null) -1 -@testcase | 140 | 8 9 | 8 9 -1 (null) (null) 0 +@testcase | 140 | 8 9 | 8 9 144 (null) (null) 0 @testcase | 141 | | (null) 10 @testcase | 142 | | 3 3 1 2 +@testcase | 143 | -1 0 | 1 2 1 3 777 (null) (null) (null) (null) 0 +@testcase | 143 | 0 -1 | 1 (null) 0 3 777 1 2 1 3 145 @fif_codegen """ isTensorNull PROC:<{ - // t.0 t.1 t.NNFlag - 2 1 BLKDROP2 // t.NNFlag + // t.0 t.1 t.UTag + 2 1 BLKDROP2 // t.UTag 0 EQINT // '3 }> """ diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index 813218026..ffedcb441 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -645,6 +645,18 @@ fun test59() { __expect_type(x1, "int"); } +@method_id(160) +fun test60() { + var cc1 = match (val a: int | slice = 0 + 10 + 90) { + int => a + 1, + }; + var cc2 = match (var a = (0 + 10 + 90) as int | slice) { + int => (a = 108) + 1, + slice => 120, + }; + return (cc1, cc2); +} + fun main(x: int?): int { @@ -663,18 +675,28 @@ fun main(x: int?): int { @testcase | 139 | | 16 @testcase | 140 | 5 | 25 @testcase | 141 | | 1 2 -@testcase | 142 | | 5 3 (null) (null) 0 -1 3 (null) (null) 0 +@testcase | 142 | | 5 3 (null) (null) 0 132 3 (null) (null) 0 @testcase | 143 | | 10 11 (null) 10 11 (null) (null) 0 @testcase | 144 | | 10 11 (null) 10 11 (null) (null) 0 -@testcase | 145 | | 5 3 (null) (null) 0 -1 3 (null) (null) (null) (null) 0 3 (null) (null) 0 -@testcase | 146 | | 3 4 5 3 4 5 -1 +@testcase | 145 | | 5 3 (null) (null) 0 132 3 (null) (null) (null) (null) 0 3 (null) (null) 0 +@testcase | 146 | | 3 4 5 3 4 5 131 @testcase | 147 | | (null) (null) 100 (null) 100 (null) (null) 0 @testcase | 158 | | 123 10 123 5 +@testcase | 160 | | 101 109 -@stderr warning: expression of type `int` is always not null, this condition is always true +@stderr warning: expression of type `int` can never be `null`, this condition is always true @stderr warning: unreachable code @stderr var t2 redef = getNullableInt()!; @fif_codegen eq55 PROC:<{ @fif_codegen eq55 PROC:<{ + +@fif_codegen +""" + test60 PROC:<{ + // + 101 PUSHINT // cc1 + 109 PUSHINT // cc1 cc2 + }> +""" */ diff --git a/tolk-tester/tests/type-aliases-tests.tolk b/tolk-tester/tests/type-aliases-tests.tolk index 54899a0fc..aa67bcad7 100644 --- a/tolk-tester/tests/type-aliases-tests.tolk +++ b/tolk-tester/tests/type-aliases-tests.tolk @@ -27,7 +27,7 @@ fun test1(x: MInt): MVoid { __expect_type(random() ? (1, 2) : (1, 2) as Pair2_v1, "(int, int)"); __expect_type(random() ? (1, 2) : (1, 2) as Pair2_v2, "(int, int)"); __expect_type(random() ? (1, 2) as Pair2_v1 : (1, 2), "Pair2_v1"); - __expect_type(random() ? (1, 2) as Pair2_v1 : (1, 2) as Pair2_v2, "Pair2_v1"); + __expect_type(random() ? (1, 2) as Pair2_v1 : (1, 2) as Pair2_v2, "(int, int)"); __expect_type(random() ? (1, 2) as Pair2_v2 : (1, 2) as Pair2_v2, "Pair2_v2"); __expect_type(!x, "bool"); @@ -40,8 +40,8 @@ fun test1(x: MInt): MVoid { } fun test2() { - __expect_type(test1, "(MInt) -> MVoid"); - __expect_type(test1(1), "MVoid"); + __expect_type(test1, "(MInt) -> void"); + __expect_type(test1(1), "void"); } fun test3(x: MIntN, y: MInt?) { @@ -51,7 +51,7 @@ fun test3(x: MIntN, y: MInt?) { if (x != null) { __expect_type(x, "MInt"); } - __expect_type(x, "MInt?"); // merge null + MInt = MInt?, not MIntN as original + __expect_type(x, "MInt?"); var (z1, z2) = (x, x!, ); __expect_type(z1, "MInt?"); __expect_type(z2, "MInt"); @@ -64,10 +64,11 @@ fun test4(x: MIntN, y: MIntN) { __expect_type(x + y, "int"); return x + y; } + __expect_type(x, "MInt?"); __expect_type(x!, "MInt"); __expect_type(random() ? x : y, "MInt?"); __expect_type(random() ? x : 0, "MInt?"); - __expect_type(random() ? 0 : x, "MInt?"); + __expect_type(random() ? 0 : x, "int?"); __expect_type(random() ? 0 : x!, "int"); return y!; } diff --git a/tolk-tester/tests/union-types-tests.tolk b/tolk-tester/tests/union-types-tests.tolk new file mode 100644 index 000000000..04446f8e7 --- /dev/null +++ b/tolk-tester/tests/union-types-tests.tolk @@ -0,0 +1,873 @@ +type MInt = int; +type MNull = null; +type MSlice = slice; +type MIntN = int | MNull; +type IntOrSlice = MInt | slice; +type IntOrSliceN = null | slice | int; +type IntOrSliceOrBuilder = int | slice | builder; + +type Pair2 = (int, int); +type Pair2Or3 = Pair2 | (int, int, int); +type SomeIntBits = int8 | int9 | int10 | int32 | uint64; + +fun getSlice32(): slice { return beginCell().storeInt(32, 32).endCell().beginParse(); } +fun getIntOrSlice(): int | slice { return 5; } + +fun testFlatten() { + __expect_type(0 as (int | int), "int"); + __expect_type(0 as (int | int | slice | builder | slice), "int | slice | builder"); + __expect_type(0 as (int? | int), "int?"); + __expect_type(0 as (slice | int? | slice? | cell), "slice | int | null | cell"); + __expect_type(0 as (int | (int | int)), "int"); + __expect_type(0 as (slice | (slice | (slice | int) | (cell | slice?))), "slice | int | cell | null"); + + __expect_type(0 as (MInt | MNull), "MInt?"); + __expect_type(0 as (MInt | slice | int), "MInt | slice"); + __expect_type(0 as (MIntN), "MIntN"); + __expect_type(0 as (MIntN | null), "int?"); + __expect_type(0 as (MIntN | MNull | MSlice), "int | null | MSlice"); + __expect_type(0 as (MIntN | MNull? | MSlice), "int | null | MSlice"); + __expect_type(0 as (MInt | IntOrSliceOrBuilder | MSlice), "MInt | slice | builder"); + __expect_type((1, 2) as (Pair2 | Pair2Or3 | (slice, int?)), "Pair2 | (int, int, int) | (slice, int?)"); + + __expect_type(0 as (int | int8), "int | int8"); + __expect_type(0 as (MInt | int8), "MInt | int8"); + __expect_type((0 as MInt) as (int | int8), "int | int8"); + __expect_type((0 as MInt) as (MInt | int8), "MInt | int8"); + + __expect_type(0 as IntOrSlice, "IntOrSlice"); + __expect_type(0 as IntOrSlice?, "MInt | slice | null"); + __expect_type(0 as IntOrSliceN, "IntOrSliceN"); + __expect_type(0 as IntOrSliceN?, "null | slice | int"); + + __expect_type(0 as (int?)?, "int?"); + __expect_type(0 as (int?)? | (int)?, "int?"); + + __expect_type(0 is builder, "bool"); + __expect_type(0 !is builder, "bool"); + __expect_type(0!!is builder, "bool"); + + __expect_type(match (0) { int => [] }, "[]"); + __expect_type(match (0 as int|slice) { int => 0, slice => 0 }, "int"); + __expect_type(match (0 as int|slice) { int => 0, slice => 0 as int8 } as int|int8, "int | int8"); + __expect_type(match (0 as int?) { int => "", null => null }, "slice?"); + __expect_type(match (0 as int?) { + int => 10>3 ? match(0) { int => 0 } : match (10>3 ? 6 : null) { null => 0, int => { return; } }, + null => match(null) { null => 0 } + }, "int"); +} + +@method_id(101) +fun test1() { + var a = getIntOrSlice(); + var b: MInt | MSlice = getIntOrSlice(); + var c: IntOrSlice = getIntOrSlice(); + __expect_type(a, "int | slice"); + __expect_type(b, "MInt | MSlice"); + __expect_type(c, "IntOrSlice"); + return (a, b, c); +} + +@method_id(102) +fun test2() { + var a: int | slice = 4; + __expect_type(a, "int"); + a = getIntOrSlice(); + __expect_type(a, "int | slice"); + a = getSlice32(); + __expect_type(a, "slice"); + return a.loadInt(32); +} + +@method_id(103) +fun test3(case: int): int | slice | null { + var a: int | slice | null = 4; + if (case == 0) { + a = getSlice32(); + } else if (case == 1) { + a = 10; + } else { + a = null; + } + return a; +} + +@method_id(104) +fun test4() { + return ( + 5 as int | int, + (5 as int8) as int8 | int8, + true as bool | bool, + 5 as int | int | (int | slice), + (5 as int | int | (int | slice)) as int | slice | null + ); +} + +@method_id(105) +fun test5() { + var a: int | int | IntOrSlice = getIntOrSlice(); + __expect_type(a, "int | slice"); + a = 5; + return (a, a as IntOrSlice, a as IntOrSliceOrBuilder, a as MIntN); +} + +@method_id(106) +fun test6(case: int): int | slice | builder | null { + var a: int? | slice? | builder = null; + __expect_type(a, "null"); + if (case == 1) { + a = null as slice? | builder; + __expect_type(a, "null | slice | builder"); + __expect_type(a!, "slice | builder"); + return a; + } + if (case == 2) { + __expect_type(a = 2 as MInt, "MInt"); + __expect_type(a, "int"); + a = 2; + __expect_type(a, "int"); + return a + 0; + } + if (case == 3) { + a = null as slice?; + __expect_type(a, "slice?"); + return a; + } + a = null as MNull; + __expect_type(a, "null"); + return a; +} + +@method_id(107) +fun test7(case: int) { + var a = (2, 3); + var b: Pair2 = (4, 5); + var c: Pair2Or3 = case == 1 ? a : case == 2 ? b : (6, 7, 8); + __expect_type(c, "Pair2Or3"); + c as Pair2Or3; + c as Pair2Or3 | Pair2Or3; + c as (int, int, int) | Pair2; + c as (int, int) | (int, int, int); + return (c, c as (int, int)? | (int, int, int) | null); +} + +@method_id(108) +fun test8() { + return ( + (2, 3) as (int, int) | (int, int, int) | null, + null as (int, int) | null | (int, int, int), + (2, 3) as Pair2Or3 | null, + (2, 3) as null | Pair2Or3, + null as Pair2Or3? + ); +} + +@method_id(109) +fun test9() { + var c: null | Pair2Or3 = (6, 7, 8) as Pair2Or3?; + __expect_type(c, "null | Pair2 | (int, int, int)"); + __expect_type(c!, "Pair2 | (int, int, int)"); + c! as Pair2Or3; + c! as (int, int, int) | Pair2; + (c! as (int, int) | (int, int, int)) as Pair2Or3; + return (c, c == null, c = null, c == null, c = null as Pair2Or3?, c == null, c as Pair2? | (int, int, int)? | null); +} + +@method_id(110) +fun test10() { + var t = getSlice32() as (int, (int, int)?, (int?, int)?) | slice; + if (10 > 3) { + t = (1, null, null) as (int, (int, int)?, (int?, int)?); + } + return t; +} + +@method_id(120) +fun test20() { + var a = getIntOrSlice(); + if (a is int) { + __expect_type(a, "int"); + return a + 0; + } + __expect_type(a, "slice"); + return a.loadInt(32); +} + +@method_id(121) +fun test21() { + var a = getIntOrSlice(); + if (a is int) { + if (a !is int8) { + __expect_type(a, "int"); + } + if (a is slice) { + __expect_type(a, "never"); + } + if (a !is slice) { + __expect_type(a, "int"); + } + if (a !is builder) { + __expect_type(a, "int"); + } + } + if (a !is int && a !is builder && a !is int && a !is null) { + __expect_type(a, "slice"); + } + if (a !is int || (a !is int && a !is builder)) { + __expect_type(a, "slice"); + } + if (a is int || a is slice) { + __expect_type(a, "int | slice"); + } else { + __expect_type(a, "never"); + } + if (a is slice || a is int) { + __expect_type(a, "slice | int"); + } else { + __expect_type(a, "never"); + } + if (a is int || a is slice || a is builder) { + __expect_type(a, "int | slice"); + } + if (a !is int && a !is slice && a !is builder) { + __expect_type(a, "never"); + } + return (a is int, a !is int, a is slice, a !is slice, a is int || a is slice || a is builder, a is builder); +} + +@method_id(122) +fun test22(a: int, b: int | slice) { + var result = 0; + if (a is int) { result += 1; } + if (a is int || a is slice) { result += 2; } + if (a is builder || a is slice) { result += 3; } + if (b is int) { result += 10; } + if ((b is int) | (b is slice)) { result += 11; } + if ((b is builder) | (b is int)) { result += 12; } + return result; +} + +@method_id(123) +fun test23(p2: bool) { + var p: Pair2Or3 = p2 ? (1, 2) : (3, 4, 5); + __expect_type(p, "Pair2Or3"); + if (p is (int, int) && 10 < 0) { + __expect_type(p, "(int, int)"); + } + __expect_type(p, "(int, int) | (int, int, int)"); + if (p is int) { + __expect_type(p, "never"); + return (0, 0); + } + __expect_type(p, "(int, int) | (int, int, int)"); + if (p !is (int, int)) { + __expect_type(p, "(int, int, int)"); + p = (10, 20); + } + __expect_type(p, "Pair2"); + return p; +} + +fun eqUnion2(v: T1 | T2): T2 | T1 { return v; } + +fun detectUnionSide(v: T1 | T2) { + if (v is T1) { + assert(v is T1 && v !is T2 && v !is (T1, T2), 101); + return 0; + } + assert(v is T2 && v !is T1 && !(v is null || v is never), 102); + return 1; +} + +@method_id(124) +fun test24() { + var bc = beginCell() as builder | cell; + var mi1 = 5 as MInt | int8; + var mi2 = 5 as int8 | MInt; + var mi3 = (5 as int8) as int8 | MInt; + __expect_type(eqUnion2(bc), "cell | builder"); + __expect_type(eqUnion2(mi1), "int8 | MInt"); + return ( + detectUnionSide(5), + detectUnionSide(5), + detectUnionSide(getIntOrSlice()), + detectUnionSide((getIntOrSlice() is int) ? getSlice32() : beginCell()), + detectUnionSide(bc), + detectUnionSide(mi1), + detectUnionSide(mi2), + detectUnionSide(mi3) + ); +} + +@method_id(125) +fun test25(a: int? | slice) { + var result = 0; + if (a is null) { result |= 0x0001; } + if (a is int || a == null) { result |= 0x0002; } + if ((a is null) | a is slice) { result |= 0x0004; } + if (a is int) { result |= 0x0008; } + if ((a is builder) | a is null) { result |= 0x0010; } + if (a is int || a is builder) { result |= 0x0020; } + if (a is int8 || a is null || a is int16) { result |= 0x0040; } + return result; +} + +@method_id(126) +fun test26() { + var a = 1 as int? | slice; + var result = 0; + if (a is null) { result |= 0x0001; } + if (a is int || a == null) { result |= 0x0002; } + if ((a is null) | a is slice) { result |= 0x0004; } + if (a is int) { result |= 0x0008; } + if ((a is builder) | a == null) { result |= 0x0010; } + if ((a is int) | a is builder) { result |= 0x0020; } + if ((a is int8) | (a is int16) | (a is null)) { result |= 0x0040; } + return result; +} + +@method_id(127) +fun test27() { + var a: (MInt, int) | slice = (1, 2); + __expect_type(a, "(MInt, int)"); + a = (1 as MInt, 2 as MInt); + __expect_type(a, "(MInt, int)"); + var b: IntOrSliceOrBuilder = 5 as (MInt | slice); + return (a, b); +} + +@method_id(128) +fun test28() { + var t = null as int | (int, int)?; + return (t, 777, 2 as int | (int, int)?, 777, (3, 4) as int | (int, int)?); +} + +@method_id(129) +fun test29() { + var t1 = (null as int?) as (int, int) | int | null; + var t2 = (5 as int?) as (int, int) | int | null; + return (t1, t2); +} + +fun get5OrNull(getNull: bool) { return getNull ? null : 5; } + +@method_id(130) +fun test30(getNull: bool) { + var t1 = get5OrNull(getNull) as (int, int) | int | null; + var t2 = (null as (int, int)?) as (int, int) | int | null; + var t3 = ((1, 2) as (int, int)?) as (int, int) | int | null; + var t4 = ([6] as [int]?) as (int, int) | [int] | null; + var t5 = (null as [int]?) as (int, int) | [int] | null; + return (t1, 777, t2, 777, t3, 777, t4, 777, t5); +} + +@method_id(131) +fun test31() { + // all of them are smart casted (narrowed from a wide union type to a subtype) + var t1: (int, int) | int | null = null; + var t2: (int, int) | int | null = 5; + var t3: (int, int) | int | null = (1, 2); + var t4: (int, int) | int | null = null as int?; + var t5: (int, int) | int | null = 5 as int?; + var t6: (int, int) | int | null = null as (int, int)?; + var t7: (int, int) | int | null = (1, 2) as (int, int)?; + __expect_type(t1, "null"); + __expect_type(t2, "int"); + __expect_type(t3, "(int, int)"); + __expect_type(t4, "int?"); + __expect_type(t6, "(int, int)?"); + return (t1, 777, t2, 777, t3, 777, t4, 777, t5, 777, t6, 777, t7); +} + +@method_id(132) +fun test32() { + var t1 = 5 as (int, slice?, int) | (int, int) | int | int8 | null; + var t2 = (5 as int8) as (int, slice?, int) | (int, int) | int | int8 | null; + var t3 = (4, 5) as (int, slice?, int) | (int, int) | int | int8 | null; + var t4 = null as (int, slice?, int) | (int, int) | int | int8 | null; + return (t1, 777, t2, 777, t3, 777, t4); +} + +@method_id(133) +fun test33() { + var t1 = (5 as (int, MInt) | int) as (int, slice?, int) | (int, int) | MInt | int8 | null; + var t2 = ((5 as int8) as null | int8 | int) as (int, slice?, int) | (MInt, int) | int | int8 | null; + var t3 = ((4, 5) as (int, MInt) | MInt) as (int, slice?, int) | (MInt, int) | int | int8 | null; + var t4 = (null as MInt | int8 | null) as (int, slice?, int) | (int, MInt) | int | int8 | null; + return (t1, 777, t2, 777, t3, 777, t4); +} + +@method_id(134) +fun test34() { + // all of them are smart casted (narrowed from a wide union type to a subtype) + var t1: (int, cell, slice?, int) | (int, int) | int | int8 | null = 5 as (int, int) | int; + var t2: (int, cell, slice?, int) | (int, MInt) | int | int8 | null = 5 as (int, int) | MInt | int8; + var t3: (int, cell, slice?, int) | (int, int) | MInt | int8 | null = (1, 2) as (int, int) | MInt; + var t4: (int, cell, slice?, int) | (MInt, MInt) | int | int8 | null = (1, 2) as (int, int) | int | null; + var t5: (int, cell, slice?, int) | (int, int) | int | int8 | null = (5 as int8) as (int, MInt) | int | int8; + var t6: (int, cell, slice?, int) | (MInt, int) | MInt | int8 | null = (5 as int8) as int8 | int; + var t7: (int, cell, slice?, int) | (MInt, MInt) | int | int8 | null = (5 as int8) as int8 | MInt | null; + __expect_type(t1, "(int, int) | int"); + __expect_type(t7, "int | int8 | null"); + return (t1, 777, t2, 777, t3, 777, t4, 777, t5, 777, t6, 777, t7); +} + +@method_id(135) +fun test35() { + var t1 = 5 as int | (int, MInt) | null; + var t2 = (1, 2) as int | (int, int) | (int, MInt, slice, slice); + var t3 = (1, 2) as int | null | (MInt, int) | (MInt, int, slice, slice); + var t4 = null as int? | (MInt, MInt)?; + __expect_type(t1!, "int | (int, MInt)"); + __expect_type(t2!, "int | (int, int) | (int, MInt, slice, slice)"); + __expect_type(t3!, "int | (MInt, int) | (MInt, int, slice, slice)"); + __expect_type(t4!, "int | (MInt, MInt)"); + return (t1!, 777, t2!, 777, t3!, 777, t4!); +} + +@method_id(136) +fun test36() { + var t1 = (1 as int8, 2 as int16) as (int, int) | slice; + var t2 = (1 as int8, 2 as int16) as (int, MInt) | (int8, int16) | slice; + var t3 = (1 as int8, 2 as int16) as (int?, MInt?) | builder | null; + var t4 = (1 as int8, 2 as int16) as (int8?, int16) | (int, int, int) | (cell, cell, cell); + return (t1, 777, t2, 777, t3, 777, t4); +} + +@method_id(137) +fun test37(one: int8, two: int16) { + var t1: (int, MInt) | slice = (one, two); + var t2: (int, int) | (int8, int16) | slice = (one, two); + var t3: (int?, int?) | builder | null = (one, two); + var t4: [int8?, int16] | (int, int, int) | (cell, cell, cell) = [one, two]; + __expect_type(t1, "(int, MInt)"); + __expect_type(t2, "(int8, int16)"); + __expect_type(t3, "(int?, int?)"); + __expect_type(t4, "[int8?, int16]"); + return (t1, 777, t2, 777, t3, 777, t4); +} + +@method_id(138) +fun test38() { + var t1 = (1, null, 2) as (int, (int, MInt, int, int)?, int) | slice; + var t2 = (1, null, 2) as (int, MInt | slice | null, int) | slice; + var t3 = (1, 5, 2) as (int, int | slice | null, int) | slice; + var t4 = (1, 5, 2) as (int, int | slice | (int, int), int) | slice; + return (t1, 777, t2, 777, t3, 777, t4); +} + +@method_id(139) +fun test39() { + var t1: (int, (MInt, int, int, MInt)?, int) | slice = (1, null, 2); + var t2: (int, int | slice | null, int) | slice = (1, null, 2); + var t3: (int, MInt | slice | null, int) | slice = (1, 5, 2); + var t4: (int, int | slice | (int, int), int) | slice = (1, 5, 2); + __expect_type(t1, "(int, (MInt, int, int, MInt)?, int)"); + __expect_type(t3, "(int, MInt | slice | null, int)"); + return (t1, 777, t2, 777, t3, 777, t4); +} + +@method_id(140) +fun test40(x: int): SomeIntBits { + if (x < 10) { return x as int8; } + if (x < 20) { return x as int9; } + if (x < 30) { return x as int10; } + return x as uint64; +} + +@method_id(141) +fun test41(x: int?, y: MInt | MNull) { + var t1 = x as int | MNull; + var t2 = x as int | slice | MNull; + var t3 = y as int?; + var t4 = y as int? | slice; + return (t1, t2, t3, t4); +} + +@method_id(142) +fun test42(x: int?, y: MInt | MNull) { + var t1: int | MNull = x; + var t2: int | slice | MNull = x; + var t3: int? = y; + var t4: int? | slice = y; + __expect_type(t1, "int?"); + __expect_type(t2, "int?"); + __expect_type(t3, "int?"); + __expect_type(t4, "int?"); + return (t1, t2, t3, t4); +} + +@method_id(143) +fun test43() { + var a: MNull = null; + __expect_type(a, "null"); + return (a as null, a as int?, a as MIntN, a as (int, int)?, a as (int, int) | slice | MNull); +} + +@method_id(144) +fun test44(a: int) { + return (a is slice, a is int, a is null, a != null, a is builder, a !is (int, int), a is int8); +} + +@method_id(145) +fun test45(a: slice?) { + return (a is slice, a is null, a != null, a is builder, a !is (int, int), a is int); +} + +@method_id(146) +fun test46(a: (int, int) | int) { + return (a is slice, a is null, a != null, a is builder, a !is (int, int), a is int && a == 0); +} + +fun checkSimple(a: int | slice | builder) { + return match (a) { + int => 1, + slice => 2, + builder => 3, + }; +} + +@method_id(147) +fun test47() { + __expect_type(checkSimple, "(int | slice | builder) -> int"); + return (checkSimple(1), checkSimple(beginCell().endCell().beginParse()), checkSimple(100), checkSimple(beginCell())); +} + +fun checkGeneric3(a: T1 | T2 | T3): (int, T1 | T2?) { + var vv: T1? = match (a) { T1 => a, T2 => null, T3 => null }; + return match (a) { + T1 => (1, a), + T2 => (2, match(a) { T2 => a }), + T3 => (3, null), + }; +} + +type IntOrInt8OrInt16 = int | int8 | int16; + +@method_id(148) +fun test48(a: IntOrInt8OrInt16) { + var b = a; + __expect_type(b, "IntOrInt8OrInt16"); + match (b) { MInt => {} int8 => {} int16 => {} } + __expect_type(b, "int | int8 | int16"); + match (b) { int => {}, int8 => { return ((0, null), (0, null)); }, int16 => "" }; + __expect_type(b, "int | int16"); + return (checkGeneric3(a), checkGeneric3(5 as int | slice | builder)); +} + +@method_id(149) +fun test49(a: slice? | int) { + if (a !is int) { + return match (a) { + slice => 1, + null => 2, + }; + } + return a + 0; +} + +type VeryComplexType = int | (MInt, int)? | builder? | int; + +@method_id(150) +fun test50() { + var a = 5 as int | slice | null; + var b = 5 as VeryComplexType; + test49(a); + test49(match (a) { int => a, slice => a, null => a, }); + __expect_type(b, "VeryComplexType"); + match (b) { int => {} Pair2 => {} builder => {} null => {} } + __expect_type(b, "int | (int, int) | builder | null"); + __expect_type(b!, "int | (int, int) | builder"); + match (b) { int => 100, Pair2 => {} builder => return -2, null => "null" } + __expect_type(b, "int | (int, int) | null"); + if (b !is null) { + match (b) { int => {}, Pair2 => {} } + } else { + match (b) { null => {} } + } + match (b is null) { + bool => {} + } + match (b = 5 as VeryComplexType) { int => b, (int, int) => { return -1; } null => {} builder => return -2 } + __expect_type(b, "int?"); + b is int && b!is int && b!!is int; + if (match(b) { int => 1, null => null } !is int) { + return 100; + } + return (b! + (match (a) { int => a, slice => a, null => a } as int|slice?).test49()); +} + +@method_id(151) +fun test51(init: int) { + var a = init as int | slice; + match(a){} + __expect_type(a, "int | slice"); + match (a) { + int => { + a = init; + match (a) {}; + __expect_type(a, "int"); + a = a > 10 ? beginCell().storeInt(init, 32).endCell().beginParse() : 10; + __expect_type(a, "int | slice"); + if (a is slice) { + a = a.loadInt(32); + } + } + slice => { + if (a !is int) { + a = a.loadInt(32) as int8; + } + } + } + __expect_type(a, "int"); + return a; +} + +@method_id(152) +fun test52(init: int): int | builder { + var m = get5OrNull(false); + if (init > 10 && init < 15) { + return init > 20 ? init : beginCell(); + } + return match (init as int?) { + int => init, + null => beginCell() + }; +} + +fun inc53(mutate x: int) { + x += 1; + return x; +} + +@method_id(153) +fun test53(a: int | builder) { + match (var a = 5 as int | slice) { + int => {} + slice => { __expect_type(a, "slice"); var b: slice? = a; return b.loadInt(32); } + } + match (val a = getIntOrSlice()) { + int => a + 0, + slice => {} + } + __expect_type(a, "int | builder"); + var cc = match (var a = getIntOrSlice()) { + int => inc53(mutate a), + slice => { return 777; } + }; + __expect_type(cc, "int"); + match (var (a, (b, c)) = (10, (20, 30))) { + (int, Pair2) => {} + } + return cc; +} + +@method_id(154) +fun test54(): int|slice { + match (var a = getIntOrSlice()) { + int => { + __expect_type(a, "int"); + a = beginCell().storeInt(100, 32).endCell().beginParse(); + __expect_type(a, "slice"); + if (a.getRemainingBitsCount() < 100) { a = a.loadInt(32); } + else { a = 0; } + __expect_type(a, "int"); + return a; + } + slice => throw 123 + } + return 777; +} + +@method_id(155) +fun test55() { + var a = getIntOrSlice() as int | slice | null; + var cc = match (a) { + int => a = getIntOrSlice(), + slice => return (-10, 0), + null => throw 123 + }; + __expect_type(a, "int | slice"); + return (a, cc); +} + +@method_id(156) +fun test56(a: int8, b: int16, useSecond: bool) { + var r1: int8 | int16 = useSecond ? b: a; + var r2: int = useSecond ? b: a; + var r3: int8 | (int, int) | int16? = useSecond ? b : a; + var r4 = (useSecond ? b : a) as int; + var r5 = (useSecond ? b : a) as int8? | int16; + var r6: int8 | int16 = match (val c: int16 | int8 | int8 = useSecond ? b : a) { int8 => c, int16 => c }; + var r7 = match (val c: int16 | int8 = useSecond ? b : a) { int8 => c, int16 => c } as int; + var r8 = match (val c: int16 | int8 = useSecond ? b : a) { int8 => c as int, int16 => c as int }; + var r9: int = match (val c = (useSecond ? b : a) as IntOrInt8OrInt16) { int8 => c, int16 => c, int => c }; + return (r1, r2, r3, r4, r5, r6, r7, r8, r9); +} + +@method_id(157) +fun test57(r3v: int) { + var r1 = null as int | () | null; + var r2 = () as int | () | null; + var r3 = () as (int, int) | () | null; + var r4 = null as (int, int) | () | null; + if (r3v == 1) { + r3 = (1, 2); + } + return (r1, r2, 777, r3, 777, r4 = r3v == 1 ? () : null, r4, 777, r1 is int, r1 is (), r1 is null, r3 is int, r3 is (int,int), r3 is (), r3 == null, 777, r4 is (), r4 is null); +} + +@method_id(158) +fun test58() { + var tn = () as ()?; + var aln: null = (tn = null); + __expect_type(tn, "null"); + return ((()) as () | (int, int) | null, (() as () | null) as () | (int, int) | null, tn, tn as ()?, aln); +} + +fun alwaysThrows123(): never { throw 123; } + +@method_id(159) +fun test59(a: int | slice | builder | null) { + try { + var b = 0; + var c = match (a) { + int => b + a, + slice => { + b = 10; + throw 456; + } + builder => alwaysThrows123(), + null => { + alwaysThrows123(); + } + }; + __expect_type(c, "int"); + return b + c; + } catch (excCode) { + return excCode; + } +} + +fun takeIntReturnVoid(a: int) { } + +fun returnVoidOrNull(willBeVoid: bool) { + var (i: int?, _: builder) = (willBeVoid ? 10 : null, beginCell()); + + var res = i != null ? takeIntReturnVoid(i) : null; + __expect_type(res, "void?"); + return res; +} + +@method_id(160) +fun test60() { + return (returnVoidOrNull(true), returnVoidOrNull(false)); +} + + + +fun main() { + return 0; +} + +/** +@testcase | 0 | | 0 +@testcase | 101 | | 5 1 5 1 5 1 +@testcase | 102 | | 32 +@testcase | 103 | 1 | 10 1 +@testcase | 103 | 2 | (null) 0 +@testcase | 104 | | 5 5 -1 5 1 5 1 +@testcase | 105 | | 5 5 1 5 1 5 +@testcase | 106 | 1 | (null) 0 +@testcase | 106 | 2 | 2 1 +@testcase | 107 | 1 | (null) 2 3 131 (null) 2 3 131 +@testcase | 107 | 3 | 6 7 8 132 6 7 8 132 +@testcase | 108 | | (null) 2 3 131 (null) (null) (null) 0 (null) 2 3 131 (null) 2 3 131 (null) (null) (null) 0 +@testcase | 109 | | 6 7 8 132 0 (null) -1 (null) (null) (null) 0 -1 (null) (null) (null) 0 +@testcase | 110 | | 1 (null) (null) 0 (null) (null) 0 134 +@testcase | 120 | | 5 +@testcase | 121 | | -1 0 0 -1 -1 0 +@testcase | 122 | 0 0 1 | 36 +@testcase | 122 | 0 0 4 | 14 +@testcase | 123 | 1 | 1 2 +@testcase | 123 | 0 | 10 20 +@testcase | 124 | | 0 1 0 1 0 0 1 0 +@testcase | 125 | null 0 | 87 +@testcase | 125 | 1 1 | 42 +@testcase | 126 | | 42 +@testcase | 127 | | 1 2 5 1 +@testcase | 128 | | (null) (null) 0 777 (null) 2 1 777 3 4 131 +@testcase | 129 | | (null) (null) 0 (null) 5 1 +@testcase | 130 | 0 | (null) 5 1 777 (null) (null) 0 777 1 2 131 777 (null) [ 6 ] 135 777 (null) (null) 0 +@testcase | 130 | -1 | (null) (null) 0 777 (null) (null) 0 777 1 2 131 777 (null) [ 6 ] 135 777 (null) (null) 0 +@testcase | 131 | | (null) 777 5 777 1 2 777 (null) 777 5 777 (null) (null) 0 777 1 2 131 +@testcase | 132 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 131 777 (null) (null) (null) 0 +@testcase | 133 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 131 777 (null) (null) (null) 0 +@testcase | 134 | | (null) 5 1 777 (null) 5 1 777 1 2 131 777 1 2 131 777 (null) 5 42 777 5 42 777 5 42 +@testcase | 135 | | (null) 5 1 777 (null) (null) 1 2 131 777 (null) (null) 1 2 131 777 (null) (null) 0 +@testcase | 136 | | 1 2 131 777 1 2 140 777 1 2 141 777 (null) 1 2 138 +@testcase | 137 | 1 2 | 1 2 777 1 2 777 1 2 777 [ 1 2 ] +@testcase | 138 | | 1 (null) (null) (null) (null) 0 2 149 777 1 (null) 0 2 143 777 1 5 1 2 143 777 1 (null) 5 1 2 144 +@testcase | 139 | | 1 (null) (null) (null) (null) 0 2 777 1 (null) 0 2 777 1 5 1 2 777 1 (null) 5 1 2 +@testcase | 140 | 5 | 5 42 +@testcase | 140 | 15 | 15 129 +@testcase | 140 | 90 | 90 49 +@testcase | 141 | 7 8 | 7 7 1 8 8 1 +@testcase | 141 | null null | (null) (null) 0 (null) (null) 0 +@testcase | 142 | 7 8 | 7 7 8 8 +@testcase | 142 | null null | (null) (null) (null) (null) +@testcase | 143 | | (null) (null) (null) (null) (null) 0 (null) (null) 0 +@testcase | 144 | 0 | 0 -1 0 -1 0 -1 0 +@testcase | 145 | null | 0 -1 0 0 -1 0 +@testcase | 146 | null 0 1 | 0 0 -1 0 -1 -1 +@testcase | 147 | | 1 2 1 3 +@testcase | 148 | 9 1 | 1 9 1 1 5 1 +@testcase | 148 | 9 44 | 3 (null) 0 1 5 1 +@testcase | 149 | null 0 | 2 +@testcase | 150 | | 10 +@testcase | 151 | 88 | 88 +@testcase | 152 | 52 | 52 1 +@testcase | 153 | 1 1 | 6 +@testcase | 154 | | 100 1 +@testcase | 155 | | 5 1 5 1 +@testcase | 156 | 1 2 -1 | 2 44 2 2 44 2 2 44 2 44 2 2 2 +@testcase | 157 | 1 | (null) 0 (null) 145 777 1 2 131 777 145 145 777 0 0 -1 0 -1 0 0 777 -1 0 +@testcase | 157 | 0 | (null) 0 (null) 145 777 (null) (null) 145 777 0 0 777 0 0 -1 0 0 -1 0 777 0 -1 +@testcase | 158 | | (null) (null) 145 (null) (null) 145 (null) 0 (null) +@testcase | 159 | 0 4 | 456 +@testcase | 159 | null 0 | 123 +@testcase | 160 | | 10 0 + + +@fif_codegen +""" + test26 PROC:<{ + // + 42 PUSHINT // result + }> +""" + +@fif_codegen +""" + checkSimple PROC:<{ + // a.USlot1 a.UTag + NIP // a.UTag + DUP // a.UTag a.UTag + 1 EQINT // a.UTag '3 + IF:<{ // a.UTag + DROP // + 1 PUSHINT // '2=1 + }>ELSE<{ // a.UTag + 4 EQINT // '6 + IF:<{ // + 2 PUSHINT // '2=2 + }>ELSE<{ // + 3 PUSHINT // '2=3 + }> + }> + }> +""" + +@fif_codegen DECLPROC checkGeneric3 + + */ diff --git a/tolk-tester/tests/unreachable-5.tolk b/tolk-tester/tests/unreachable-5.tolk new file mode 100644 index 000000000..9371ee127 --- /dev/null +++ b/tolk-tester/tests/unreachable-5.tolk @@ -0,0 +1,24 @@ +fun main(x: int | slice | null) { + if (x is builder) { + __expect_type(x, "never"); + return 10; + } + if (x !is cell) { + __expect_type(x, "int | slice | null"); + } + if (x is int) { + return x; + } + return (x is null) ? -1 : 20; +} + +/** +@testcase | 0 | 0 1 | 0 +@testcase | 0 | null 0 | -1 +@testcase | 0 | -1 2 | 20 + +@stderr warning: variable `x` of type `int | slice | null` can never be `builder` +@stderr if (x is builder) +@stderr warning: variable `x` of type `int | slice | null` can never be `cell` +@stderr if (x !is cell) + */ diff --git a/tolk-tester/tests/var-apply-tests.tolk b/tolk-tester/tests/var-apply-tests.tolk index 76fdab21f..11241bf13 100644 --- a/tolk-tester/tests/var-apply-tests.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -202,7 +202,7 @@ fun testApplyGlobalVar(x: int, y: int, z: int): bool? { return check_assoc_2(x, y, z); } -fun justAdd2(x: MInt): int64 { return x + 2; } +fun justAdd2(x: MInt): int { return x + 2; } @method_id(113) fun testCallbackAlias(secondNull: bool) { @@ -211,6 +211,17 @@ fun testCallbackAlias(secondNull: bool) { return (adder1(10), adder2 == null ? -1 : adder2(10)); } +fun justGetTensor2() { return (2, 3); } + +@method_id(114) +fun testCallbackUnion(call1st: bool): (int, int) | int { + var cb: (int -> int) | (() -> (MInt, int)) = call1st ? justGetTensor2 : justAdd2; + if (cb is () -> (int, int)) { + return cb(); + } + return cb(10) as int; +} + fun main() {} /** @@ -223,7 +234,7 @@ fun main() {} @testcase | 107 | | 65537 @testcase | 108 | | 4 @testcase | 109 | | 0 3 0 7 -@testcase | 110 | 5 | 5 10 100 100 100 -1 +@testcase | 110 | 5 | 5 10 100 100 100 134 @testcase | 110 | 0 | (null) (null) (null) (null) (null) 0 @testcase | 111 | 2 3 9 | -1 @testcase | 111 | 11 22 44 | -1 @@ -233,4 +244,6 @@ fun main() {} @testcase | 112 | -1 -10 -20 | -1 @testcase | 113 | 0 | 12 12 @testcase | 113 | -1 | 12 -1 +@testcase | 114 | -1 | 2 3 130 +@testcase | 114 | 0 | (null) 12 1 */ diff --git a/tolk-tester/tests/warnings-not-errors/unreachable-3.tolk b/tolk-tester/tests/warnings-not-errors/unreachable-3.tolk index fab21fd24..14bc43264 100644 --- a/tolk-tester/tests/warnings-not-errors/unreachable-3.tolk +++ b/tolk-tester/tests/warnings-not-errors/unreachable-3.tolk @@ -15,7 +15,7 @@ fun main(x: int?) { @testcase | 0 | 5 | -2 @testcase | 0 | null | -1 -@stderr warning: variable `x` of type `int` is always not null +@stderr warning: variable `x` of type `int` can never be `null`, this condition is always true @stderr if (x != null) @stderr warning: unreachable code @stderr return 3 + 4 diff --git a/tolk-tester/tests/warnings-not-errors/warnings-1.tolk b/tolk-tester/tests/warnings-not-errors/warnings-1.tolk index 040057d17..5dc3acdc3 100644 --- a/tolk-tester/tests/warnings-not-errors/warnings-1.tolk +++ b/tolk-tester/tests/warnings-not-errors/warnings-1.tolk @@ -21,8 +21,8 @@ fun main() { /** @testcase | 0 | | 0 -@stderr warning: variable `c` of type `int` is always not null, this condition is always false -@stderr warning: variable `d` of type `int` is always not null, this condition is always true -@stderr warning: variable `e` is always null, this condition is always true -@stderr warning: expression is always null, this condition is always false +@stderr warning: variable `c` of type `int` can never be `null`, this condition is always false +@stderr warning: variable `d` of type `int` can never be `null`, this condition is always true +@stderr warning: variable `e` is always `null`, this condition is always true +@stderr warning: expression is always `null`, this condition is always false */ diff --git a/tolk-tester/tests/warnings-not-errors/warnings-2.tolk b/tolk-tester/tests/warnings-not-errors/warnings-2.tolk index 57ecb21ac..2c5d97e84 100644 --- a/tolk-tester/tests/warnings-not-errors/warnings-2.tolk +++ b/tolk-tester/tests/warnings-not-errors/warnings-2.tolk @@ -15,12 +15,12 @@ fun main() { /** @testcase | 0 | | -1 -@stderr warning: variable `a` of type `int` is always not null, this condition is always true +@stderr warning: variable `a` of type `int` can never be `null`, this condition is always true @stderr warning: condition of ternary operator is always true -@stderr warning: variable `c` of type `slice` is always not null, this condition is always false +@stderr warning: variable `c` of type `slice` can never be `null`, this condition is always false @stderr warning: condition of `if` is always true -@stderr warning: variable `b` of type `builder` is always not null, this condition is always false +@stderr warning: variable `b` of type `builder` can never be `null`, this condition is always false @stderr warning: condition of `assert` is always false @stderr warning: condition of `while` is always false -@stderr warning: expression of type `bool` is always not null, this condition is always true +@stderr warning: expression of type `bool` can never be `null`, this condition is always true */ diff --git a/tolk-tester/tests/warnings-not-errors/warnings-3.tolk b/tolk-tester/tests/warnings-not-errors/warnings-3.tolk new file mode 100644 index 000000000..1198bc104 --- /dev/null +++ b/tolk-tester/tests/warnings-not-errors/warnings-3.tolk @@ -0,0 +1,26 @@ +fun main() { + var x = 0 as int | slice; + if (x !is builder) {} + while ((x = x) is builder) {} + + match (val tt = x) { + int => { + assert (tt is int, 505); + } + slice => {} + } + assert(x is int) throw 400; + __expect_type(x, "int"); + return x == null; +} + +/** +@testcase | 0 | | 0 + +@stderr warning: variable `x` of type `int | slice` can never be `builder`, this condition is always true +@stderr warning: condition of `if` is always true +@stderr warning: expression of type `int | slice` can never be `builder`, this condition is always false +@stderr warning: condition of `while` is always false +@stderr warning: variable `tt` is always `int`, this condition is always true +@stderr warning: condition of `assert` is always true + */ diff --git a/tolk-tester/tolk-tester.py b/tolk-tester/tolk-tester.py index 3b4fb5b7f..f8a7a5512 100644 --- a/tolk-tester/tolk-tester.py +++ b/tolk-tester/tolk-tester.py @@ -402,7 +402,7 @@ def run_all_tests(tests: List[str]): if testcase.compilation_should_fail: print(" OK, stderr match", file=sys.stderr) else: - print(" OK, %d cases, gas %d" % (len(testcase.input_output), gas_used), file=sys.stderr) + print(" OK, %d cases" % (len(testcase.input_output)), file=sys.stderr) except ParseInputError as e: print(" Error parsing input (cur line #%d):" % (testcase.line_idx + 1), e, file=sys.stderr) exit(2) diff --git a/tolk/abscode.cpp b/tolk/abscode.cpp index eb2e9a0a6..c33fcd89d 100644 --- a/tolk/abscode.cpp +++ b/tolk/abscode.cpp @@ -410,12 +410,19 @@ std::vector CodeBlob::create_var(TypePtr var_type, SrcLocation loc, s std::vector nested = create_var(t_tensor->items[i], loc, std::move(sub_name)); ir_idx.insert(ir_idx.end(), nested.begin(), nested.end()); } - } else if (const TypeDataNullable* t_nullable = var_type->try_as(); t_nullable && stack_w != 1) { - std::string null_flag_name = name.empty() ? name : name + ".NNFlag"; - ir_idx = create_var(t_nullable->inner, loc, std::move(name)); - ir_idx.emplace_back(create_var(TypeDataBool::create(), loc, std::move(null_flag_name))[0]); } else if (const TypeDataAlias* t_alias = var_type->try_as()) { ir_idx = create_var(t_alias->underlying_type, loc, std::move(name)); + } else if (const TypeDataUnion* t_union = var_type->try_as(); t_union && stack_w != 1) { + std::string utag_name = name.empty() ? "'UTag" : name + ".UTag"; + if (t_union->or_null) { // in stack comments, `a:(int,int)?` will be "a.0 a.1 a.UTag" + ir_idx = create_var(t_union->or_null, loc, std::move(name)); + } else { // in stack comments, `a:int|slice` will be "a.USlot1 a.UTag" + for (int i = 0; i < stack_w - 1; ++i) { + std::string slot_name = name.empty() ? "'USlot" + std::to_string(i + 1) : name + ".USlot" + std::to_string(i + 1); + ir_idx.emplace_back(create_var(TypeDataUnknown::create(), loc, std::move(slot_name))[0]); + } + } + ir_idx.emplace_back(create_var(TypeDataInt::create(), loc, std::move(utag_name))[0]); } else if (var_type != TypeDataVoid::create() && var_type != TypeDataNever::create()) { #ifdef TOLK_DEBUG tolk_assert(stack_w == 1); diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 729204579..36e6ce5a2 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -111,7 +111,7 @@ static void diagnose_addition_in_bitshift(SrcLocation loc, std::string_view bits } } -// replace (a == null) and similar to ast_is_null_check(a) (special AST vertex) +// replace (a == null) and similar to ast_is_type_operator(a, null) (as if `a is null` was written) static AnyExprV maybe_replace_eq_null_with_isNull_check(V v) { bool has_null = v->get_lhs()->type == ast_null_keyword || v->get_rhs()->type == ast_null_keyword; bool replace = has_null && (v->tok == tok_eq || v->tok == tok_neq); @@ -120,7 +120,7 @@ static AnyExprV maybe_replace_eq_null_with_isNull_check(V v } AnyExprV v_nullable = v->get_lhs()->type == ast_null_keyword ? v->get_rhs() : v->get_lhs(); - return createV(v->loc, v_nullable, v->tok == tok_neq); + return createV(v->loc, v_nullable, TypeDataNullLiteral::create(), v->tok == tok_neq); } // parse `123` / `0xFF` / `0b10001` to td::RefInt256 @@ -151,6 +151,7 @@ static td::RefInt256 parse_tok_int_const(std::string_view text) { AnyExprV parse_expr(Lexer& lex); +AnyV parse_statement(Lexer& lex); static AnyV parse_parameter(Lexer& lex, bool is_first) { SrcLocation loc = lex.cur_location(); @@ -242,6 +243,80 @@ static AnyV parse_type_alias_declaration(Lexer& lex, const std::vector(loc, v_ident, underlying_type); } +static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable) { + SrcLocation loc = lex.cur_location(); + if (lex.tok() == tok_oppar) { + lex.next(); + AnyExprV first = parse_var_declaration_lhs(lex, is_immutable); + if (lex.tok() == tok_clpar) { + lex.next(); + return first; + } + std::vector args(1, first); + while (lex.tok() == tok_comma) { + lex.next(); + args.push_back(parse_var_declaration_lhs(lex, is_immutable)); + } + lex.expect(tok_clpar, "`)`"); + return createV(loc, std::move(args)); + } + if (lex.tok() == tok_opbracket) { + lex.next(); + std::vector args(1, parse_var_declaration_lhs(lex, is_immutable)); + while (lex.tok() == tok_comma) { + lex.next(); + args.push_back(parse_var_declaration_lhs(lex, is_immutable)); + } + lex.expect(tok_clbracket, "`]`"); + return createV(loc, std::move(args)); + } + if (lex.tok() == tok_identifier) { + auto v_ident = createV(loc, lex.cur_str()); + TypePtr declared_type = nullptr; + bool marked_as_redef = false; + lex.next(); + if (lex.tok() == tok_colon) { + lex.next(); + declared_type = parse_type_from_tokens(lex); + } else if (lex.tok() == tok_redef) { + lex.next(); + marked_as_redef = true; + } + return createV(loc, v_ident, declared_type, is_immutable, marked_as_redef); + } + if (lex.tok() == tok_underscore) { + TypePtr declared_type = nullptr; + lex.next(); + if (lex.tok() == tok_colon) { + lex.next(); + declared_type = parse_type_from_tokens(lex); + } + return createV(loc, createV(loc, ""), declared_type, true, false); + } + lex.unexpected("variable name"); +} + +static AnyExprV parse_local_vars_declaration_assignment(Lexer& lex, bool is_statement) { + SrcLocation loc = lex.cur_location(); + bool is_immutable = lex.tok() == tok_val; + lex.next(); + + AnyExprV lhs = createV(loc, parse_var_declaration_lhs(lex, is_immutable)); + if (lex.tok() != tok_assign) { + lex.error("variables declaration must be followed by assignment: `var xxx = ...`"); + } + lex.next(); + AnyExprV rhs = parse_expr(lex); + + if (lex.tok() == tok_comma) { + lex.error("multiple declarations are not allowed, split variables on separate lines"); + } + if (is_statement) { + lex.expect(tok_semicolon, "`;`"); + } + return createV(loc, lhs, rhs); +} + // "parameters" are at function declaration: `fun f(param1: int, mutate param2: slice)` static V parse_parameter_list(Lexer& lex) { SrcLocation loc = lex.cur_location(); @@ -314,6 +389,123 @@ static V parse_maybe_instantiationTs_after_identifier(L } } +// `throw code` / `throw (code)` / `throw (code, arg)` +// it's technically a statement (can't occur "in any place of expression"), +// but inside `match` arm it can appear without semicolon: `pattern => throw 123` +static AnyV parse_throw_expression(Lexer& lex, bool is_statement) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_throw, "`throw`"); + + AnyExprV thrown_code, thrown_arg; + if (lex.tok() == tok_oppar) { // throw (code) or throw (code, arg) + lex.next(); + thrown_code = parse_expr(lex); + if (lex.tok() == tok_comma) { + lex.next(); + thrown_arg = parse_expr(lex); + } else { + thrown_arg = createV(loc); + } + lex.expect(tok_clpar, "`)`"); + } else { // throw code + thrown_code = parse_expr(lex); + thrown_arg = createV(loc); + } + + if (is_statement) { + lex.expect(tok_semicolon, "`;`"); + } + return createV(loc, thrown_code, thrown_arg); +} + +// `pattern => body` inside `match` +static V parse_match_arm(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + MatchArmKind pattern_kind = static_cast(-1); + TypePtr exact_type = nullptr; + AnyExprV pattern_expr = nullptr; + + Lexer::SavedPositionForLookahead backup = lex.save_parsing_position(); + try { + exact_type = parse_type_from_tokens(lex); + pattern_kind = MatchArmKind::exact_type; + } catch (const ParseError&) { + } + if (!exact_type || lex.tok() != tok_double_arrow) { + exact_type = nullptr; + lex.restore_position(backup); + try { + pattern_expr = parse_expr(lex); + pattern_kind = MatchArmKind::const_expression; // any expr at parsing, should result in const int/bool + } catch (const ParseError&) { + } + } + if (!exact_type && !pattern_expr && lex.tok() == tok_else) { + lex.next(); + pattern_kind = MatchArmKind::else_branch; + } + + if (pattern_kind == static_cast(-1)) { + lex.restore_position(backup); + throw ParseError(loc, "expected or in `match` before `=>`"); + } + lex.expect(tok_double_arrow, "`=>`"); + + AnyExprV body; + if (lex.tok() == tok_opbrace) { // pattern => { ... } + AnyV v_seq = parse_statement(lex); + body = createV(v_seq->loc, v_seq); + } else if (lex.tok() == tok_throw) { // pattern => throw 123 (allow without braces) + AnyV v_throw = parse_throw_expression(lex, false); + AnyV v_seq = createV(v_throw->loc, v_throw->loc, {v_throw}); + body = createV(v_seq->loc, v_seq); + } else if (lex.tok() == tok_return) { // pattern => return 123 (allow without braces, like throw) + lex.next(); + AnyV v_return = createV(lex.cur_location(), parse_expr(lex)); + AnyV v_seq = createV(v_return->loc, v_return->loc, {v_return}); + body = createV(v_seq->loc, v_seq); + } else { + body = parse_expr(lex); + } + + if (pattern_expr == nullptr) { // for match by type / default case, empty vertex, not nullptr + pattern_expr = createV(loc); + } + return createV(loc, pattern_kind, exact_type, pattern_expr, body); +} + +static V parse_match_expression(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_match, "`match`"); + + lex.expect(tok_oppar, "`(`"); + AnyExprV subject = lex.tok() == tok_var || lex.tok() == tok_val // `match (var x = rhs)` + ? parse_local_vars_declaration_assignment(lex, false) + : parse_expr(lex); + lex.expect(tok_clpar, "`)`"); + + std::vector subject_and_arms = {subject}; + lex.expect(tok_opbrace, "`{`"); + while (lex.tok() != tok_clbrace) { + auto v_arm = parse_match_arm(lex); + subject_and_arms.push_back(v_arm); + + // after `pattern => { ... }` comma is optional, after `pattern => expr` mandatory + bool was_comma = lex.tok() == tok_comma; // trailing comma is allowed always + if (was_comma) { + lex.next(); + } + if (lex.tok() == tok_clbrace) { + break; + } + if (!was_comma && v_arm->get_body()->type != ast_braced_expression) { + lex.unexpected("`,`"); + } + } + lex.expect(tok_clbrace, "`}`"); + return createV(loc, std::move(subject_and_arms)); +} + // parse (expr) / [expr] / identifier / number static AnyExprV parse_expr100(Lexer& lex) { SrcLocation loc = lex.cur_location(); @@ -404,6 +596,8 @@ static AnyExprV parse_expr100(Lexer& lex) { } return createV(loc, v_ident, v_instantiationTs); } + case tok_match: + return parse_match_expression(lex); default: lex.unexpected(""); } @@ -474,7 +668,7 @@ static AnyExprV parse_expr75(Lexer& lex) { return parse_expr80(lex); } -// parse E as +// parse E as / is / !is static AnyExprV parse_expr40(Lexer& lex) { AnyExprV lhs = parse_expr75(lex); if (lex.tok() == tok_as) { @@ -482,6 +676,15 @@ static AnyExprV parse_expr40(Lexer& lex) { lex.next(); TypePtr cast_to_type = parse_type_from_tokens(lex); lhs = createV(loc, lhs, cast_to_type); + } else if (lex.tok() == tok_is) { + SrcLocation loc = lex.cur_location(); + lex.next(); + TypePtr rhs_type = parse_type_from_tokens(lex); + bool is_negated = lhs->type == ast_not_null_operator; // `a !is ...`, now lhs = `a!` + if (is_negated) { + lhs = lhs->as()->get_expr(); + } + lhs = createV(loc, lhs, rhs_type, is_negated); } return lhs; } @@ -616,80 +819,6 @@ AnyExprV parse_expr(Lexer& lex) { return parse_expr10(lex); } -AnyV parse_statement(Lexer& lex); - -static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable) { - SrcLocation loc = lex.cur_location(); - if (lex.tok() == tok_oppar) { - lex.next(); - AnyExprV first = parse_var_declaration_lhs(lex, is_immutable); - if (lex.tok() == tok_clpar) { - lex.next(); - return first; - } - std::vector args(1, first); - while (lex.tok() == tok_comma) { - lex.next(); - args.push_back(parse_var_declaration_lhs(lex, is_immutable)); - } - lex.expect(tok_clpar, "`)`"); - return createV(loc, std::move(args)); - } - if (lex.tok() == tok_opbracket) { - lex.next(); - std::vector args(1, parse_var_declaration_lhs(lex, is_immutable)); - while (lex.tok() == tok_comma) { - lex.next(); - args.push_back(parse_var_declaration_lhs(lex, is_immutable)); - } - lex.expect(tok_clbracket, "`]`"); - return createV(loc, std::move(args)); - } - if (lex.tok() == tok_identifier) { - auto v_ident = createV(loc, lex.cur_str()); - TypePtr declared_type = nullptr; - bool marked_as_redef = false; - lex.next(); - if (lex.tok() == tok_colon) { - lex.next(); - declared_type = parse_type_from_tokens(lex); - } else if (lex.tok() == tok_redef) { - lex.next(); - marked_as_redef = true; - } - return createV(loc, v_ident, declared_type, is_immutable, marked_as_redef); - } - if (lex.tok() == tok_underscore) { - TypePtr declared_type = nullptr; - lex.next(); - if (lex.tok() == tok_colon) { - lex.next(); - declared_type = parse_type_from_tokens(lex); - } - return createV(loc, createV(loc, ""), declared_type, true, false); - } - lex.unexpected("variable name"); -} - -static AnyV parse_local_vars_declaration_assignment(Lexer& lex) { - SrcLocation loc = lex.cur_location(); - bool is_immutable = lex.tok() == tok_val; - lex.next(); - - AnyExprV lhs = createV(loc, parse_var_declaration_lhs(lex, is_immutable)); - if (lex.tok() != tok_assign) { - lex.error("variables declaration must be followed by assignment: `var xxx = ...`"); - } - lex.next(); - AnyExprV rhs = parse_expr(lex); - - if (lex.tok() == tok_comma) { - lex.error("multiple declarations are not allowed, split variables on separate lines"); - } - lex.expect(tok_semicolon, "`;`"); - return createV(loc, lhs, rhs); -} - static V parse_sequence(Lexer& lex) { SrcLocation loc = lex.cur_location(); lex.expect(tok_opbrace, "`{`"); @@ -699,7 +828,7 @@ static V parse_sequence(Lexer& lex) { } SrcLocation loc_end = lex.cur_location(); lex.expect(tok_clbrace, "`}`"); - return createV(loc, loc_end, items); + return createV(loc, loc_end, std::move(items)); } static AnyV parse_return_statement(Lexer& lex) { @@ -789,30 +918,6 @@ static AnyExprV create_catch_underscore_variable(const Lexer& lex) { return createV(lex.cur_location(), v_ident, nullptr); } -static AnyV parse_throw_statement(Lexer& lex) { - SrcLocation loc = lex.cur_location(); - lex.expect(tok_throw, "`throw`"); - - AnyExprV thrown_code, thrown_arg; - if (lex.tok() == tok_oppar) { // throw (code) or throw (code, arg) - lex.next(); - thrown_code = parse_expr(lex); - if (lex.tok() == tok_comma) { - lex.next(); - thrown_arg = parse_expr(lex); - } else { - thrown_arg = createV(loc); - } - lex.expect(tok_clpar, "`)`"); - } else { // throw code - thrown_code = parse_expr(lex); - thrown_arg = createV(loc); - } - - lex.expect(tok_semicolon, "`;`"); - return createV(loc, thrown_code, thrown_arg); -} - static AnyV parse_assert_statement(Lexer& lex) { SrcLocation loc = lex.cur_location(); lex.expect(tok_assert, "`assert`"); @@ -866,7 +971,7 @@ AnyV parse_statement(Lexer& lex) { switch (lex.tok()) { case tok_var: // `var x = 0` is technically an expression, but can not appear in "any place", case tok_val: // only as a separate declaration - return parse_local_vars_declaration_assignment(lex); + return parse_local_vars_declaration_assignment(lex, true); case tok_opbrace: return parse_sequence(lex); case tok_return: @@ -880,7 +985,7 @@ AnyV parse_statement(Lexer& lex) { case tok_while: return parse_while_statement(lex); case tok_throw: - return parse_throw_statement(lex); + return parse_throw_expression(lex, true); case tok_assert: return parse_assert_statement(lex); case tok_try: @@ -895,7 +1000,9 @@ AnyV parse_statement(Lexer& lex) { lex.error("break/continue from loops are not supported yet"); default: { AnyExprV expr = parse_expr(lex); - lex.expect(tok_semicolon, "`;`"); + if (expr->type != ast_match_expression) { // match statement, as if statement, doesn't require semicolon after + lex.expect(tok_semicolon, "`;`"); + } return expr; } } diff --git a/tolk/ast-replacer.h b/tolk/ast-replacer.h index d1ab4c450..5efae1e2f 100644 --- a/tolk/ast-replacer.h +++ b/tolk/ast-replacer.h @@ -60,6 +60,12 @@ class ASTReplacer { return v_mutable; } + GNU_ATTRIBUTE_ALWAYS_INLINE AnyExprV replace_children(const ASTExprBlockOfStatements* v) { + auto* v_mutable = const_cast(v); + v_mutable->child_sequence = replace(v_mutable->child_sequence->as()); + return v_mutable; + } + GNU_ATTRIBUTE_ALWAYS_INLINE AnyV replace_children(const ASTStatementUnary* v) { auto* v_mutable = const_cast(v); v_mutable->child = replace(v_mutable->child); @@ -88,6 +94,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { // expressions virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } @@ -108,8 +115,10 @@ class ASTReplacerInFunctionBody : public ASTReplacer { virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } - virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } // statements virtual AnyV replace(V v) { return replace_children(v); } virtual AnyV replace(V v) { return replace_children(v); } @@ -126,6 +135,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { switch (v->type) { case ast_empty_expression: return replace(v->as()); case ast_parenthesized_expression: return replace(v->as()); + case ast_braced_expression: return replace(v->as()); case ast_tensor: return replace(v->as()); case ast_typed_tuple: return replace(v->as()); case ast_reference: return replace(v->as()); @@ -146,8 +156,10 @@ class ASTReplacerInFunctionBody : public ASTReplacer { case ast_binary_operator: return replace(v->as()); case ast_ternary_operator: return replace(v->as()); case ast_cast_as_operator: return replace(v->as()); + case ast_is_type_operator: return replace(v->as()); case ast_not_null_operator: return replace(v->as()); - case ast_is_null_check: return replace(v->as()); + case ast_match_expression: return replace(v->as()); + case ast_match_arm: return replace(v->as()); default: throw UnexpectedASTNodeType(v, "ASTReplacerInFunctionBody::replace"); } diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index bf48c2389..9f0738d3f 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -61,6 +61,9 @@ class ASTReplicatorFunction : public ASTReplicator { virtual V clone(V v) { return createV(v->loc, clone(v->get_expr())); } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_sequence())); + } virtual V clone(V v) { return createV(v->loc, clone(v->get_items())); } @@ -121,11 +124,17 @@ class ASTReplicatorFunction : public ASTReplicator { virtual V clone(V v) { return createV(v->loc, clone(v->get_expr()), clone(v->cast_to_type)); } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_expr()), clone(v->rhs_type), v->is_negated); + } virtual V clone(V v) { return createV(v->loc, clone(v->get_expr())); } - virtual V clone(V v) { - return createV(v->loc, clone(v->get_expr()), v->is_negated); + virtual V clone(V v) { + return createV(v->loc, clone(v->get_all_children())); + } + virtual V clone(V v) { + return createV(v->loc, v->pattern_kind, clone(v->exact_type), clone(v->get_pattern_expr()), clone(v->get_body())); } // statements @@ -186,6 +195,7 @@ class ASTReplicatorFunction : public ASTReplicator { switch (v->type) { case ast_empty_expression: return clone(v->as()); case ast_parenthesized_expression: return clone(v->as()); + case ast_braced_expression: return clone(v->as()); case ast_tensor: return clone(v->as()); case ast_typed_tuple: return clone(v->as()); case ast_reference: return clone(v->as()); @@ -206,8 +216,10 @@ class ASTReplicatorFunction : public ASTReplicator { case ast_binary_operator: return clone(v->as()); case ast_ternary_operator: return clone(v->as()); case ast_cast_as_operator: return clone(v->as()); + case ast_is_type_operator: return clone(v->as()); case ast_not_null_operator: return clone(v->as()); - case ast_is_null_check: return clone(v->as()); + case ast_match_expression: return clone(v->as()); + case ast_match_arm: return clone(v->as()); default: throw UnexpectedASTNodeType(v, "ASTReplicatorFunction::clone"); } diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index 77390c1ca..33082a6e9 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -36,6 +36,7 @@ class ASTStringifier final : public ASTVisitor { // expressions {ast_empty_expression, "ast_empty_expression"}, {ast_parenthesized_expression, "ast_parenthesized_expression"}, + {ast_braced_expression, "ast_braced_expression"}, {ast_tensor, "ast_tensor"}, {ast_typed_tuple, "ast_typed_tuple"}, {ast_reference, "ast_reference"}, @@ -56,8 +57,10 @@ class ASTStringifier final : public ASTVisitor { {ast_binary_operator, "ast_binary_operator"}, {ast_ternary_operator, "ast_ternary_operator"}, {ast_cast_as_operator, "ast_cast_as_operator"}, + {ast_is_type_operator, "ast_is_type_operator"}, {ast_not_null_operator, "ast_not_null_operator"}, - {ast_is_null_check, "ast_is_null_check"}, + {ast_match_expression, "ast_match_expression"}, + {ast_match_arm, "ast_match_arm"}, // statements {ast_empty_statement, "ast_empty_statement"}, {ast_sequence, "ast_sequence"}, @@ -170,6 +173,10 @@ class ASTStringifier final : public ASTVisitor { return static_cast(v->as()->operator_name); case ast_cast_as_operator: return v->as()->cast_to_type->as_human_readable(); + case ast_is_type_operator: { + std::string prefix = v->as()->is_negated ? "!is " : "is "; + return prefix + v->as()->rhs_type->as_human_readable(); + } case ast_sequence: return "↓" + std::to_string(v->as()->get_items().size()); case ast_instantiationT_item: @@ -209,6 +216,14 @@ class ASTStringifier final : public ASTVisitor { } return result + ">"; } + case ast_match_arm: + if (v->as()->pattern_kind == MatchArmKind::exact_type) { + return v->as()->exact_type->as_human_readable(); + } + if (v->as()->pattern_kind == MatchArmKind::const_expression) { + return "(expression)"; + } + return "(else)"; case ast_tolk_required_version: return static_cast(v->as()->semver); case ast_import_directive: @@ -249,6 +264,7 @@ class ASTStringifier final : public ASTVisitor { // expressions case ast_empty_expression: return handle_vertex(v->as()); case ast_parenthesized_expression: return handle_vertex(v->as()); + case ast_braced_expression: return handle_vertex(v->as()); case ast_tensor: return handle_vertex(v->as()); case ast_typed_tuple: return handle_vertex(v->as()); case ast_reference: return handle_vertex(v->as()); @@ -269,8 +285,10 @@ class ASTStringifier final : public ASTVisitor { case ast_binary_operator: return handle_vertex(v->as()); case ast_ternary_operator: return handle_vertex(v->as()); case ast_cast_as_operator: return handle_vertex(v->as()); + case ast_is_type_operator: return handle_vertex(v->as()); case ast_not_null_operator: return handle_vertex(v->as()); - case ast_is_null_check: return handle_vertex(v->as()); + case ast_match_expression: return handle_vertex(v->as()); + case ast_match_arm: return handle_vertex(v->as()); // statements case ast_empty_statement: return handle_vertex(v->as()); case ast_sequence: return handle_vertex(v->as()); diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h index 737646bb5..2f5f269f7 100644 --- a/tolk/ast-visitor.h +++ b/tolk/ast-visitor.h @@ -56,6 +56,10 @@ class ASTVisitor { } } + GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTExprBlockOfStatements* v) { + visit_children(v->child_sequence->as()); + } + GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTStatementUnary* v) { visit(v->child); } @@ -89,6 +93,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { // expressions virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } @@ -109,8 +114,10 @@ class ASTVisitorFunctionBody : public ASTVisitor { virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } - virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } // statements virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } @@ -128,6 +135,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { // expressions case ast_empty_expression: return visit(v->as()); case ast_parenthesized_expression: return visit(v->as()); + case ast_braced_expression: return visit(v->as()); case ast_tensor: return visit(v->as()); case ast_typed_tuple: return visit(v->as()); case ast_reference: return visit(v->as()); @@ -148,8 +156,10 @@ class ASTVisitorFunctionBody : public ASTVisitor { case ast_binary_operator: return visit(v->as()); case ast_ternary_operator: return visit(v->as()); case ast_cast_as_operator: return visit(v->as()); + case ast_is_type_operator: return visit(v->as()); case ast_not_null_operator: return visit(v->as()); - case ast_is_null_check: return visit(v->as()); + case ast_match_expression: return visit(v->as()); + case ast_match_arm: return visit(v->as()); // statements case ast_empty_statement: return visit(v->as()); case ast_sequence: return visit(v->as()); diff --git a/tolk/ast.cpp b/tolk/ast.cpp index f383ebcc7..d740970da 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -138,6 +138,20 @@ void Vertex::assign_resolved_type(TypePtr cast_to_type) { this->cast_to_type = cast_to_type; } +void Vertex::assign_resolved_type(TypePtr rhs_type) { + this->rhs_type = rhs_type; +} + +void Vertex::assign_is_negated(bool is_negated) { + this->is_negated = is_negated; +} + +void Vertex::assign_resolved_pattern(MatchArmKind pattern_kind, TypePtr exact_type, AnyExprV pattern_expr) { + this->pattern_kind = pattern_kind; + this->exact_type = exact_type; + this->lhs = pattern_expr; +} + void Vertex::assign_var_ref(GlobalVarPtr var_ref) { this->var_ref = var_ref; } @@ -186,10 +200,6 @@ void Vertex::assign_fun_ref(FunctionPtr fun_ref) { this->fun_ref = fun_ref; } -void Vertex::assign_is_negated(bool is_negated) { - this->is_negated = is_negated; -} - void Vertex::assign_first_unreachable(AnyV first_unreachable) { this->first_unreachable = first_unreachable; } diff --git a/tolk/ast.h b/tolk/ast.h index b62aa4c53..fea11ebcb 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -68,6 +68,7 @@ enum ASTNodeType { // expressions ast_empty_expression, ast_parenthesized_expression, + ast_braced_expression, ast_tensor, ast_typed_tuple, ast_reference, @@ -88,8 +89,10 @@ enum ASTNodeType { ast_binary_operator, ast_ternary_operator, ast_cast_as_operator, + ast_is_type_operator, ast_not_null_operator, - ast_is_null_check, + ast_match_expression, + ast_match_arm, // statements ast_empty_statement, ast_sequence, @@ -128,6 +131,12 @@ enum class AnnotationKind { unknown, }; +enum class MatchArmKind { // for `match` expression, each of arms `pattern => body` can be: + const_expression, // `-1 => body` / `SOME_CONST + ton("0.05") => body` (any expr at parsing, resulting in const) + exact_type, // `int => body` / `User | slice => body` + else_branch, // `else => body` +}; + template struct Vertex; @@ -244,7 +253,7 @@ struct ASTExprVararg : ASTNodeExpressionBase { AnyExprV child(int i) const { return children.at(i); } - ASTExprVararg(ASTNodeType type, SrcLocation loc, std::vector children) + ASTExprVararg(ASTNodeType type, SrcLocation loc, std::vector&& children) : ASTNodeExpressionBase(type, loc), children(std::move(children)) {} public: @@ -252,6 +261,17 @@ struct ASTExprVararg : ASTNodeExpressionBase { bool empty() const { return children.empty(); } }; +struct ASTExprBlockOfStatements : ASTNodeExpressionBase { + friend class ASTVisitor; + friend class ASTReplacer; + +protected: + AnyV child_sequence; + + ASTExprBlockOfStatements(ASTNodeType type, SrcLocation loc, AnyV child_sequence) + : ASTNodeExpressionBase(type, loc), child_sequence(child_sequence) {} +}; + struct ASTStatementUnary : ASTNodeStatementBase { friend class ASTVisitor; friend class ASTReplacer; @@ -351,6 +371,18 @@ struct Vertex final : ASTExprUnary { : ASTExprUnary(ast_parenthesized_expression, loc, expr) {} }; +template<> +// ast_braced_expression is a sequence, but in a context of expression (it has a type) +// it can contain arbitrary statements inside +// it can occur only in special places within the input code, not anywhere +// example: `match (intV) { 0 => { ... } }` rhs of 0 is braced expression +struct Vertex final : ASTExprBlockOfStatements { + auto get_sequence() const { return child_sequence->as(); } + + Vertex(SrcLocation loc, AnyV child_sequence) + : ASTExprBlockOfStatements(ast_braced_expression, loc, child_sequence) {} +}; + template<> // ast_tensor is a set of expressions embraced by (parenthesis) // in most languages, it's called "tuple", but in TVM, "tuple" is a TVM primitive, that's why "tensor" @@ -683,6 +715,24 @@ struct Vertex final : ASTExprUnary { , cast_to_type(cast_to_type) {} }; +template<> +// ast_is_type_operator is type matching with "is" or "!is" keywords and "== null" / "!= null" (same as "is null") +// examples: `v is SomeStruct` / `getF() !is slice` / `v == null` / `v !is null` +struct Vertex final : ASTExprUnary { + AnyExprV get_expr() const { return child; } + + TypePtr rhs_type; + bool is_negated; // `!is type`, `!= null` + + Vertex* mutate() const { return const_cast(this); } + void assign_resolved_type(TypePtr rhs_type); + void assign_is_negated(bool is_negated); + + Vertex(SrcLocation loc, AnyExprV expr, TypePtr rhs_type, bool is_negated) + : ASTExprUnary(ast_is_type_operator, loc, expr) + , rhs_type(rhs_type), is_negated(is_negated) {} +}; + template<> // ast_not_null_operator is non-null assertion: like TypeScript ! or Kotlin !! // examples: `nullableInt!` / `getNullableBuilder()!` @@ -694,19 +744,40 @@ struct Vertex final : ASTExprUnary { }; template<> -// ast_is_null_check is an artificial vertex for "expr == null" / "expr != null" / same but null on the left -// it's created instead of a general binary expression to emphasize its purpose -struct Vertex final : ASTExprUnary { - bool is_negated; +// ast_match_expression is `match (subject) { ... arms ... }`, used either as a statement or as an expression +// example: `match (intOrSliceVar) { int => 1, slice => 2 }` +// example: `match (var c = getIntOrSlice()) { int => return 0, slice => throw 123 }` +struct Vertex final : ASTExprVararg { + AnyExprV get_subject() const { return child(0); } + int get_arms_count() const { return size() - 1; } + auto get_arm(int i) const { return child(i + 1)->as(); } + const std::vector& get_all_children() const { return children; } - AnyExprV get_expr() const { return child; } + bool is_statement() const { return !is_rvalue && !is_lvalue; } + + Vertex(SrcLocation loc, std::vector&& subject_and_arms) + : ASTExprVararg(ast_match_expression, loc, std::move(subject_and_arms)) {} +}; + +template<> +// ast_match_arm is one `pattern => body` inside `match` expression/statement +// pattern can be a custom expression / a type / `else` (see comments in MatchArmKind) +// body can be any expression; particularly, braced expression `{ ... }` +// example: `int => variable` (match by type, inferred_type of body variable's type) +// example: `a+b => { ...; return 0; }` (match by expression, inferred_type of body is "never" (unreachable end)) +struct Vertex final : ASTExprBinary { + MatchArmKind pattern_kind; + TypePtr exact_type; // for MatchArmKind::exact_type; otherwise, nullptr + + AnyExprV get_pattern_expr() const { return lhs; } + AnyExprV get_body() const { return rhs; } // remember, it may be V Vertex* mutate() const { return const_cast(this); } - void assign_is_negated(bool is_negated); + void assign_resolved_pattern(MatchArmKind pattern_kind, TypePtr exact_type, AnyExprV pattern_expr); - Vertex(SrcLocation loc, AnyExprV expr, bool is_negated) - : ASTExprUnary(ast_is_null_check, loc, expr) - , is_negated(is_negated) {} + Vertex(SrcLocation loc, MatchArmKind pattern_kind, TypePtr exact_type, AnyExprV pattern_expr, AnyExprV body) + : ASTExprBinary(ast_match_arm, loc, pattern_expr, body) + , pattern_kind(pattern_kind), exact_type(exact_type) {} }; @@ -739,7 +810,7 @@ struct Vertex final : ASTStatementVararg { Vertex* mutate() const { return const_cast(this); } void assign_first_unreachable(AnyV first_unreachable); - Vertex(SrcLocation loc, SrcLocation loc_end, std::vector items) + Vertex(SrcLocation loc, SrcLocation loc_end, std::vector&& items) : ASTStatementVararg(ast_sequence, loc, std::move(items)) , loc_end(loc_end) {} }; diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index b335683f3..94a43e9fc 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -81,14 +81,14 @@ void GenericSubstitutionsDeduceForCall::consider_next_condition(TypePtr param_ty if (const auto* asT = param_type->try_as()) { // `(arg: T)` called as `f([1, 2])` => T is [int, int] provide_deducedT(asT->nameT, arg_type); - } else if (const auto* p_nullable = param_type->try_as()) { + } else if (const auto* p_nullable = param_type->try_as(); p_nullable && p_nullable->or_null) { // `arg: T?` called as `f(nullableInt)` => T is int - if (const auto* a_nullable = arg_type->unwrap_alias()->try_as()) { - consider_next_condition(p_nullable->inner, a_nullable->inner); + if (const auto* a_nullable = arg_type->unwrap_alias()->try_as(); a_nullable && a_nullable->or_null) { + consider_next_condition(p_nullable->or_null, a_nullable->or_null); } // `arg: T?` called as `f(int)` => T is int else { - consider_next_condition(p_nullable->inner, arg_type); + consider_next_condition(p_nullable->or_null, arg_type); } } else if (const auto* p_tensor = param_type->try_as()) { // `arg: (int, T)` called as `f((5, cs))` => T is slice @@ -112,6 +112,13 @@ void GenericSubstitutionsDeduceForCall::consider_next_condition(TypePtr param_ty } consider_next_condition(p_callable->return_type, a_callable->return_type); } + } else if (const auto* p_union = param_type->try_as()) { + // `arg: T1 | T2` called as `f(intOrBuilder)` => T1 is int, T2 is builder + if (const auto* a_union = arg_type->unwrap_alias()->try_as(); a_union && a_union->variants.size() == p_union->variants.size()) { + for (int i = 0; i < static_cast(p_union->variants.size()); ++i) { + consider_next_condition(p_union->variants[i], a_union->variants[i]); + } + } } } diff --git a/tolk/lexer.cpp b/tolk/lexer.cpp index 961be1674..822abe931 100644 --- a/tolk/lexer.cpp +++ b/tolk/lexer.cpp @@ -337,6 +337,7 @@ struct ChunkIdentifierOrKeyword final : ChunkLexerBase { case 2: if (str == "do") return tok_do; if (str == "if") return tok_if; + if (str == "is") return tok_is; if (str == "as") return tok_as; break; case 3: @@ -359,6 +360,7 @@ struct ChunkIdentifierOrKeyword final : ChunkLexerBase { case 5: if (str == "const") return tok_const; if (str == "false") return tok_false; + if (str == "match") return tok_match; if (str == "redef") return tok_redef; if (str == "while") return tok_while; if (str == "break") return tok_break; @@ -521,6 +523,7 @@ struct TolkLanguageGrammar { register_token("|=", 2, tok_set_bitwise_or); register_token("^=", 2, tok_set_bitwise_xor); register_token("->", 2, tok_arrow); + register_token("=>", 2, tok_double_arrow); register_token("<=>", 3, tok_spaceship); register_token("~>>", 3, tok_rshiftR); register_token("^>>", 3, tok_rshiftC); @@ -587,13 +590,14 @@ void Lexer::next_special(TokenType parse_next_as, const char* str_expected) { } Lexer::SavedPositionForLookahead Lexer::save_parsing_position() const { - return {p_next, cur_token_idx, cur_token}; + return {p_next, cur_token_idx, cur_token, location}; } void Lexer::restore_position(SavedPositionForLookahead saved) { p_next = saved.p_next; cur_token_idx = last_token_idx = saved.cur_token_idx; cur_token = saved.cur_token; + location = saved.loc; } void Lexer::error(const std::string& err_msg) const { diff --git a/tolk/lexer.h b/tolk/lexer.h index 34c792ebc..6e9ab6a41 100644 --- a/tolk/lexer.h +++ b/tolk/lexer.h @@ -116,9 +116,12 @@ enum TokenType { tok_assert, tok_if, tok_else, + tok_match, tok_arrow, + tok_double_arrow, tok_as, + tok_is, tok_tolk, tok_semver, @@ -161,6 +164,7 @@ class Lexer { const char* p_next = nullptr; int cur_token_idx = 0; Token cur_token; + SrcLocation loc; }; explicit Lexer(const SrcFile* file); diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 10647de94..167c56147 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -21,6 +21,7 @@ #include "type-system.h" #include "common/refint.h" #include "constant-evaluator.h" +#include "smart-casts-cfg.h" #include /* @@ -41,7 +42,7 @@ * Example: `nullableInt!`; for `nullableInt` inferred_type is `int?`, and target_type is `int` * (this doesn't lead to stack reorganization, but in case `nullableTensor!` does) * (inferred_type of `nullableInt!` is `int`, and its target_type depends on its usage). - * The same mechanism will work for union types in the future. + * Example: `var a: int|slice = 5`. This `5` should be extended as "5 1" (5 for value, 1 for type_id of `int`). */ namespace tolk { @@ -410,6 +411,39 @@ static std::vector pre_compile_let(CodeBlob& code, AnyExprV lhs, AnyE return right; } +static std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, TypePtr cmp_type, const std::vector& expr_ir_idx, SrcLocation loc, const char* debug_desc) { + FunctionPtr eq_sym = lookup_global_symbol("_==_")->try_as(); + FunctionPtr isnull_sym = lookup_global_symbol("__isNull")->try_as(); + FunctionPtr not_sym = lookup_global_symbol("!b_")->try_as(); + std::vector result_ir_idx = code.create_tmp_var(TypeDataBool::create(), loc, debug_desc); + + const TypeDataUnion* lhs_union = expr_type->try_as(); + if (!lhs_union) { + // `int` is `int` / `int` is `builder`, it's compile-time, either 0, or -1 + bool types_eq = expr_type->get_type_id() == cmp_type->get_type_id(); + code.emplace_back(loc, Op::_IntConst, result_ir_idx, td::make_refint(types_eq ? -1 : 0)); + } else if (lhs_union->is_primitive_nullable() && cmp_type == TypeDataNullLiteral::create()) { + // `int?` is `null` for primitive 1-slot nullables, they hold either value of TVM NULL, no extra union tag slot + code.emplace_back(loc, Op::_Call, result_ir_idx, expr_ir_idx, isnull_sym); + } else if (lhs_union->is_primitive_nullable()) { + // `int?` is `int` (check for null actually) / `int?` is `builder` (compile-time false actually) + bool cant_happen = lhs_union->or_null->get_type_id() != cmp_type->get_type_id(); + if (cant_happen) { + code.emplace_back(loc, Op::_IntConst, result_ir_idx, td::make_refint(0)); + } else { + code.emplace_back(loc, Op::_Call, result_ir_idx, expr_ir_idx, isnull_sym); + code.emplace_back(loc, Op::_Call, result_ir_idx, result_ir_idx, not_sym); + } + } else { + // `int | slice` is `int`, check type id + std::vector typeid_ir_idx = code.create_tmp_var(TypeDataInt::create(), loc, "(type-id)"); + code.emplace_back(loc, Op::_IntConst, typeid_ir_idx, td::make_refint(cmp_type->get_type_id())); + code.emplace_back(loc, Op::_Call, result_ir_idx, std::vector{typeid_ir_idx[0], expr_ir_idx.back()}, eq_sym); + } + + return result_ir_idx; +} + static std::vector gen_op_call(CodeBlob& code, TypePtr ret_type, SrcLocation loc, std::vector&& args_vars, FunctionPtr fun_ref, const char* debug_desc) { std::vector rvect = code.create_tmp_var(ret_type, loc, debug_desc); @@ -428,8 +462,8 @@ static std::vector gen_op_call(CodeBlob& code, TypePtr ret_type, SrcL // `null` (inferred_type) is 1 stack slots, but target_type is 3, we should add 2 nulls. // Another example: `var t1 = (1, null); var t2: (int, (int,int)?) = t1;`. // Then t1's rvect is 2 vars (1 and null), but t1's `null` should be converted to 3 stack slots (resulting in 4 total). -// The same mechanism will work for union types in the future. -// Here rvect is a list of IR vars for inferred_type, probably patched due to target_type. +// The same mechanism works for union types, but there is a union tag (UTag) slot instead of null flag. +// Another example: `var i: int|slice = 5;`. This "5" is represented as "5 1" (5 for value, 1 is type_id of `int`). GNU_ATTRIBUTE_NOINLINE static std::vector transition_expr_to_runtime_type_impl(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc) { #ifdef TOLK_DEBUG @@ -446,176 +480,299 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectorget_width_on_stack(); - const TypeDataNullable* t_nullable = target_type->try_as(); - const TypeDataNullable* o_nullable = original_type->try_as(); + int orig_w = original_type->get_width_on_stack(); // = rvect.size() + const TypeDataUnion* t_union = target_type->try_as(); + const TypeDataUnion* o_union = original_type->try_as(); + + // most common case, simple nullability: + // - `int` to `int?` + // - `null` to `int?` + // - `int8?` to `int16?` + // - `null` to `StructWith1Int?` + // in general, pass `T1` to `T2?` when `T2` is a primitive and `T2?` still occupies 1 stack slot (value or TVM NULL) + if (t_union && target_w == 1 && orig_w == 1 && t_union->or_null->get_width_on_stack() == 1) { + // a generalized union type "T | ..." takes at least 2 slots, but "T | null" can be 1 + tolk_assert(t_union->or_null); + // rvect has 1 slot, either value or TVM NULL + return rvect; + } - // handle `never` - // it may occur due to smart cast and in unreachable branches - // we can't do anything reasonable here, but (hopefully) execution will never reach this point, and stack won't be polluted - if (original_type == TypeDataNever::create()) { - std::vector dummy_rvect; - dummy_rvect.reserve(target_w); - for (int i = 0; i < target_w; ++i) { - dummy_rvect.push_back(code.create_tmp_var(TypeDataUnknown::create(), loc, "(never)")[0]); - } - return dummy_rvect; + // smart cast of a primitive 1-slot nullable: + // - `int?` to `int` + // - `int?` to `null` + // - `StructWith1Int?` to `null` + // this value (one slot) is either a TVM primitive or TVM NULL at runtime + if (o_union && orig_w == 1 && target_w == 1 && o_union->or_null->get_width_on_stack() == 1) { + tolk_assert(o_union->or_null); + return rvect; } - if (target_type == TypeDataNever::create()) { - return {}; + + // pass `T` to `never` + // it occurs due to smart cast, in unreachable branches, for example `if (intVal == null) { return intVal; }` + // we can't do anything reasonable here, but (hopefully) execution will never reach this point, and stack won't be polluted + if (target_type == TypeDataNever::create() || original_type == TypeDataNever::create() || target_type == TypeDataUnknown::create()) { + return rvect; } - // pass `null` to `T?` - // for primitives like `int?`, no changes in rvect, null occupies the same TVM slot - // for tensors like `(int,int)?`, `null` is represented as N nulls + 1 null flag, insert N nulls - if (t_nullable && original_type == TypeDataNullLiteral::create()) { - tolk_assert(rvect.size() == 1); - if (target_w == 1 && !t_nullable->is_primitive_nullable()) { // `null` to `()?` - rvect = code.create_tmp_var(TypeDataInt::create(), loc, "(NNFlag)"); - code.emplace_back(loc, Op::_IntConst, rvect, td::make_refint(0)); - } - if (target_w > 1) { - FunctionPtr builtin_sym = lookup_global_symbol("__null")->try_as(); - rvect.reserve(target_w + 1); - for (int i = 1; i < target_w - 1; ++i) { - std::vector ith_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); - code.emplace_back(loc, Op::_Call, ith_null, std::vector{}, builtin_sym); - rvect.push_back(ith_null[0]); - } - std::vector null_flag_ir = code.create_tmp_var(TypeDataInt::create(), loc, "(NNFlag)"); - var_idx_t null_flag_ir_idx = null_flag_ir[0]; - code.emplace_back(loc, Op::_IntConst, std::move(null_flag_ir), td::make_refint(0)); - rvect.push_back(null_flag_ir_idx); + // smart cast to a primitive 1-slot nullable: + // - `int | slice | null` to `slice?` + // - `A | int | null` to `int?` + // so, originally a type occupies N slots, but needs to be converted to 1 slot + if (t_union && target_w == 1 && orig_w > 0 && t_union->or_null->get_width_on_stack() == 1) { + // nothing except "T1 | T2 | ... null" can be cast to 1-slot nullable `T1?` + tolk_assert(o_union && o_union->has_null() && o_union->has_variant_with_type_id(t_union->or_null)); + // here we exploit rvect shape, how union types and multi-slot nullables are stored on a stack + // `T1 | T2 | ... | null` occupies N+1 slots, where the last is for UTag + // when it holds null value, N slots are null, and UTag slot is 0 (it's type_id of TypeDataNullLiteral) + return {rvect[rvect.size() - 2]}; + } + + // pass `null` to `T?` when T is not a primitive (1-slot T was handled above) + // - `null` to `(int, int)?` + // - `null` to `int | slice | null` + // to represent a non-primitive null value, we need N nulls + 1 null flag (UTag=0, type_id of TypeDataNullLiteral) + if (t_union && target_w > 1 && original_type == TypeDataNullLiteral::create()) { + tolk_assert(t_union->has_null()); + FunctionPtr null_sym = lookup_global_symbol("__null")->try_as(); + rvect.reserve(target_w); // keep rvect[0], it's already null + for (int i = 1; i < target_w - 1; ++i) { + std::vector ith_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); + code.emplace_back(loc, Op::_Call, ith_null, std::vector{}, null_sym); + rvect.push_back(ith_null[0]); } + std::vector last_null = code.create_tmp_var(TypeDataInt::create(), loc, "(UTag)"); + code.emplace_back(loc, Op::_IntConst, last_null, td::make_refint(0)); + rvect.push_back(last_null[0]); return rvect; } - // pass `T` to `T?` - // for primitives like `int?`, no changes in rvect: `int` and `int?` occupy the same TVM slot (null is represented as NULL TVM value) - // for passing `(int, int)` to `(int, int)?` / `(int, null)` to `(int, (int,int)?)?`, add a null flag equals to 0 - if (t_nullable && !o_nullable) { - if (!t_nullable->is_primitive_nullable()) { - rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type, t_nullable->inner, loc); - tolk_assert(target_w == static_cast(rvect.size() + 1)); - std::vector null_flag_ir = code.create_tmp_var(TypeDataInt::create(), loc, "(NNFlag)"); - var_idx_t null_flag_ir_idx = null_flag_ir[0]; - code.emplace_back(loc, Op::_IntConst, std::move(null_flag_ir), td::make_refint(-1)); - rvect.push_back(null_flag_ir_idx); + + // pass `null` to nullable empty tensor + // - `null` to `()?` + // - `null` to `EmptyStruct?` + // so, rvect contains TVM NULL, but instead, we should push UTag=0 + if (t_union && original_type == TypeDataNullLiteral::create()) { + tolk_assert(t_union->or_null && t_union->or_null->get_width_on_stack() == 0); + std::vector new_rvect = code.create_tmp_var(TypeDataInt::create(), loc, "(UTag)"); + code.emplace_back(loc, Op::_IntConst, new_rvect, td::make_refint(0)); + return new_rvect; + } + + // smart cast of a wide nullable union to plain `null` + // - `(int, int)?` to `null` + // - `int | slice | null` to `null` + if (o_union && target_type == TypeDataNullLiteral::create() && orig_w > 1) { + tolk_assert(o_union->has_null()); + // if we are here, it's guaranteed that original value holds null + // it means, that its shape is N nulls + 1 UTag (equals 0) + return {rvect[rvect.size() - 2]}; + } + + // smart cast of nullable empty tensor to plain `null` + // - `()?` to `null` + // `EmptyStruct?` to `null` + // so, rvect contains UTag, we need TVM NULL + if (o_union && target_type == TypeDataNullLiteral::create()) { + tolk_assert(orig_w == 1 && o_union->or_null); + FunctionPtr null_sym = lookup_global_symbol("__null")->try_as(); + std::vector new_rvect = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); + code.emplace_back(loc, Op::_Call, new_rvect, std::vector{}, null_sym); + return new_rvect; + } + + // pass primitive 1-slot `T?` to a wider nullable union + // - `int?` to `int | slice | null` + // - `slice?` to `(int, int) | slice | builder | null` + // so, originally `T?` is 1-slot, but needs to be converted to N+1 slots, keeping its value + if (o_union && orig_w == 1 && t_union && o_union->or_null->get_width_on_stack() == 1) { + tolk_assert(t_union->has_null() && t_union->has_variant_with_type_id(o_union->or_null) && target_w > 1); + // the transformation is tricky: + // when value is null, we need to achieve "... (null) 0" (value is already null, so "... value 0") + // when value is not null, we need to get "... value {type_id}" + // this can be done only via IFs at runtime; luckily, this case is very uncommon in practice + // for "...", we might need N-1 nulls: `int?` to `(int,int,int) | int | null` is `(null) (null) value/(null) 0/1` + FunctionPtr null_sym = lookup_global_symbol("__null")->try_as(); + std::vector new_rvect; + new_rvect.resize(target_w); + for (int i = 0; i < target_w - 2; ++i) { // N-1 nulls + std::vector ith_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); + code.emplace_back(loc, Op::_Call, ith_null, std::vector{}, null_sym); + new_rvect[i] = ith_null[0]; } + new_rvect[target_w - 2] = rvect[0]; // value + new_rvect[target_w - 1] = code.create_tmp_var(TypeDataInt::create(), loc, "(UTag)")[0]; + + std::vector eq_null_cond = code.create_tmp_var(TypeDataBool::create(), loc, "(value-is-null)"); + FunctionPtr isnull_sym = lookup_global_symbol("__isNull")->try_as(); + code.emplace_back(loc, Op::_Call, eq_null_cond, rvect, isnull_sym); + Op& if_op = code.emplace_back(loc, Op::_If, eq_null_cond); + code.push_set_cur(if_op.block0); + code.emplace_back(loc, Op::_IntConst, std::vector{new_rvect[target_w - 1]}, td::make_refint(0)); + code.close_pop_cur(loc); + code.push_set_cur(if_op.block1); + code.emplace_back(loc, Op::_IntConst, std::vector{new_rvect[target_w - 1]}, td::make_refint(o_union->or_null->get_type_id())); + code.close_pop_cur(loc); + return new_rvect; + } + + // extend a single type into a union type + // - `int` to `int | slice` + // - `int` to `int | (int, int) | null` + // - `(int, int)` to `(int, int, cell) | builder | (int, int)` + // - `(int, null)` to `(int, (int, int)?) | ...`: mind transition + // - `(int, null)` to `(int, int | slice | null) | ...`: mind transition + // so, probably need to prepend some nulls, and need to append UTag + if (t_union && !o_union) { + TypePtr t_subtype = t_union->calculate_exact_variant_to_fit_rhs(original_type); + tolk_assert(t_subtype && target_w > t_subtype->get_width_on_stack()); + rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type, t_subtype, loc); + std::vector prepend_nulls; + prepend_nulls.reserve(target_w - t_subtype->get_width_on_stack() - 1); + for (int i = 0; i < target_w - t_subtype->get_width_on_stack() - 1; ++i) { + FunctionPtr null_sym = lookup_global_symbol("__null")->try_as(); + std::vector ith_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(UVar.null)"); + prepend_nulls.push_back(ith_null[0]); + code.emplace_back(loc, Op::_Call, std::move(ith_null), std::vector{}, null_sym); + } + rvect.insert(rvect.begin(), prepend_nulls.begin(), prepend_nulls.end()); + + std::vector last_utag = code.create_tmp_var(TypeDataInt::create(), loc, "(UTag)"); + code.emplace_back(loc, Op::_IntConst, last_utag, td::make_refint(t_subtype->get_type_id())); + rvect.push_back(last_utag[0]); return rvect; } - // pass `T1?` to `T2?` - // for example, `int8?` to `int16?` - // transition inner types, leaving nullable flag unchanged for tensors - if (t_nullable && o_nullable) { - if (target_w > 1) { - var_idx_t null_flag_ir_idx = rvect.back(); - rvect.pop_back(); - rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, o_nullable->inner, t_nullable->inner, loc); - rvect.push_back(null_flag_ir_idx); - } + + // smart cast a union type to a single type + // - `int | slice` to `int` + // - `int | (int, int) | null` to `int` + // - `(int, (int, int)?) | ...` to `(int, null)`: mind transition + // so, cut off UTag and probably some unused tags from the start + if (o_union && !t_union) { + TypePtr o_subtype = o_union->calculate_exact_variant_to_fit_rhs(target_type); + tolk_assert(o_subtype && orig_w > o_subtype->get_width_on_stack()); + rvect = std::vector(rvect.begin() + orig_w - o_subtype->get_width_on_stack() - 1, rvect.end() - 1); + rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, o_subtype, target_type, loc); return rvect; } - // pass `T?` to `null` - // it may occur due to smart cast, when a `T?` variable is guaranteed to be always null - // (for instance, always-null `(int,int)?` will be represented as 1 TVM NULL value, not 3) - if (target_type == TypeDataNullLiteral::create() && original_type->can_rhs_be_assigned(target_type)) { - tolk_assert(o_nullable || original_type == TypeDataUnknown::create()); - if (o_nullable && !o_nullable->is_primitive_nullable()) { - FunctionPtr builtin_sym = lookup_global_symbol("__null")->try_as(); - rvect = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); - code.emplace_back(loc, Op::_Call, rvect, std::vector{}, builtin_sym); + + // extend a union type to a wider one + // - `int | slice` to `int | slice | builder` + // - `int | slice` to `int | (int, int) | slice | null` + // so, both original and target have UTag slot, but rvect probably needs to be prepended by nulls + if (t_union && o_union && t_union->variants.size() >= o_union->variants.size()) { + tolk_assert(target_w >= orig_w && t_union->has_all_variants_of(o_union)); + std::vector prepend_nulls; + prepend_nulls.reserve(target_w - orig_w); + for (int i = 0; i < target_w - orig_w; ++i) { + FunctionPtr null_sym = lookup_global_symbol("__null")->try_as(); + std::vector ith_null_ir_idx = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(UVar.null)"); + prepend_nulls.push_back(ith_null_ir_idx[0]); + code.emplace_back(loc, Op::_Call, std::move(ith_null_ir_idx), std::vector{}, null_sym); } + rvect.insert(rvect.begin(), prepend_nulls.begin(), prepend_nulls.end()); return rvect; } - // pass `T?` to `T` (or, more generally, `T1?` to `T2`) - // it may occur due to operator `!` or smart cast - // for primitives like `int?`, no changes in rvect - // for passing `(int, int)?` to `(int, int)`, drop the null flag from the tail - // for complex scenarios like passing `(int, (int,int)?)?` to `(int, null)`, recurse the call - // (it may occur on `someF(t = (3,null))` when `(3,null)` at first targeted to lhs, but actually its result is rhs) - if (!t_nullable && o_nullable) { - if (!o_nullable->is_primitive_nullable()) { - rvect.pop_back(); - rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type->try_as()->inner, target_type, loc); - } + + // smart cast a wider union type to a narrow one + // - `int | slice | builder` to `int | slice` + // - `int | (int, int) | slice | null` to `int | slice` + // so, both original and target have UTag slot, but rvect needs to be cut off from the left + if (t_union && o_union) { + tolk_assert(target_w <= orig_w && o_union->has_all_variants_of(t_union)); + rvect = std::vector(rvect.begin() + (orig_w - target_w), rvect.end()); return rvect; } + // pass `bool` to `int` // in code, it's done via `as` operator, like `boolVar as int` // no changes in rvect, boolVar is guaranteed to be -1 or 0 at TVM level if (original_type == TypeDataBool::create() && target_type == TypeDataInt::create()) { return rvect; } + // pass `bool` to `int8` // same as above if (original_type == TypeDataBool::create() && target_type->try_as()) { return rvect; } + // pass `int8` to `int` // it comes from auto cast when an integer (even a literal) is assigned to intN // to changes in rvect, intN is int at TVM level if (target_type == TypeDataInt::create() && original_type->try_as()) { return rvect; } + // pass `coins` to `int` // same as above if (target_type == TypeDataInt::create() && original_type == TypeDataCoins::create()) { return rvect; } + // pass `int` to `int8` // in code, it's probably done with `as` operator // no changes in rvect if (original_type == TypeDataInt::create() && target_type->try_as()) { return rvect; } + // pass `int` to `coins` // same as above if (original_type == TypeDataInt::create() && target_type == TypeDataCoins::create()) { return rvect; } + // pass `int8` to `int16` / `int8` to `uint8` // in code, it's probably done with `as` operator // no changes in rvect if (original_type->try_as() && target_type->try_as()) { return rvect; } + // pass `int8` to `coins` // same as above if (target_type == TypeDataCoins::create() && original_type->try_as()) { return rvect; } + // pass `coins` to `int8` // same as above if (original_type == TypeDataCoins::create() && target_type->try_as()) { return rvect; } + // pass `bytes32` to `slice` // in code, it's probably done with `as` operator // no changes in rvect, since bytesN is slice at TVM level if (target_type == TypeDataSlice::create() && original_type->try_as()) { return rvect; } + // pass `slice` to `bytes32` // same as above if (original_type == TypeDataSlice::create() && target_type->try_as()) { return rvect; } + // pass `bytes32` to `bytes64` / `bits128` to `bytes16` // no changes in rvect if (original_type->try_as() && target_type->try_as()) { return rvect; } + // pass something to `unknown` // probably, it comes from `_ = rhs`, type of `_` is unknown, it's target_type of rhs // no changes in rvect if (target_type == TypeDataUnknown::create()) { return rvect; } + // pass `unknown` to something // probably, it comes from `arg` in exception, it's inferred as `unknown` and could be cast to any value if (original_type == TypeDataUnknown::create()) { tolk_assert(rvect.size() == 1); return rvect; } + // pass tensor to tensor, e.g. `(1, null)` to `(int, slice?)` / `(1, null)` to `(int, (int,int)?)` // every element of rhs tensor should be transitioned if (target_type->try_as() && original_type->try_as()) { @@ -635,22 +792,14 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as() && original_type->try_as()) { - tolk_assert(target_type->get_width_on_stack() == original_type->get_width_on_stack()); + tolk_assert(target_w == 1 && orig_w == 1); return rvect; } - // pass `T` to `UserId` (or some other alias) - if (const TypeDataAlias* target_alias = target_type->try_as()) { - return transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type, target_alias->underlying_type, loc); - } - // pass `UserId` (or some other alias) to `T` - if (const TypeDataAlias* orig_alias = original_type->try_as()) { - return transition_expr_to_runtime_type_impl(std::move(rvect), code, orig_alias->underlying_type, target_type, loc); - } - // pass callable to callable // their types aren't exactly equal, but they match (containing aliases, for example) if (original_type->try_as() && target_type->try_as()) { @@ -817,7 +966,7 @@ static std::vector process_ternary_operator(V v std::vector cond = pre_compile_expr(v->get_cond(), code, nullptr); tolk_assert(cond.size() == 1); std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(cond)"); - + if (v->get_cond()->is_always_true) { code.emplace_back(v->get_when_true()->loc, Op::_Let, rvect, pre_compile_expr(v->get_when_true(), code, v->inferred_type)); } else if (v->get_cond()->is_always_false) { @@ -841,40 +990,91 @@ static std::vector process_cast_as_operator(V v return transition_to_target_type(std::move(rvect), code, target_type, v); } -static std::vector process_not_null_operator(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { - TypePtr child_target_type = v->get_expr()->inferred_type->unwrap_alias(); - if (const auto* as_nullable = child_target_type->try_as()) { - child_target_type = as_nullable->inner; +static std::vector process_is_type_operator(V v, CodeBlob& code, TypePtr target_type) { + TypePtr lhs_type = v->get_expr()->inferred_type->unwrap_alias(); + TypePtr cmp_type = v->rhs_type->unwrap_alias(); + bool is_null_check = cmp_type == TypeDataNullLiteral::create(); // `v == null`, not `v is T` + tolk_assert(!cmp_type->try_as()); // `v is int|slice` is a type checker error + + std::vector expr_ir_idx = pre_compile_expr(v->get_expr(), code, nullptr); + std::vector result_ir_idx = pre_compile_is_type(code, lhs_type, cmp_type, expr_ir_idx, v->loc, is_null_check ? "(is-null)" : "(is-type)"); + + if (v->is_negated) { + FunctionPtr not_sym = lookup_global_symbol("!b_")->try_as(); + code.emplace_back(v->loc, Op::_Call, result_ir_idx, result_ir_idx, not_sym); } + return transition_to_target_type(std::move(result_ir_idx), code, target_type, v); +} + +static std::vector process_not_null_operator(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + TypePtr expr_type = v->get_expr()->inferred_type->unwrap_alias(); + TypePtr without_null_type = calculate_type_subtract_rhs_type(expr_type, TypeDataNullLiteral::create()); + TypePtr child_target_type = without_null_type != TypeDataNever::create() ? without_null_type : expr_type; + std::vector rvect = pre_compile_expr(v->get_expr(), code, child_target_type, lval_ctx); return transition_to_target_type(std::move(rvect), code, target_type, v); } -static std::vector process_is_null_check(V v, CodeBlob& code, TypePtr target_type) { - std::vector expr_ir_idx = pre_compile_expr(v->get_expr(), code, nullptr); - std::vector isnull_ir_idx = code.create_tmp_var(TypeDataBool::create(), v->loc, "(is-null)"); - TypePtr expr_type = v->get_expr()->inferred_type->unwrap_alias(); +static std::vector process_match_expression(V v, CodeBlob& code, TypePtr target_type) { + TypePtr lhs_type = v->get_subject()->inferred_type->unwrap_alias(); + + std::vector subj_ir_idx = pre_compile_expr(v->get_subject(), code, nullptr); + std::vector result_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(match-expression)"); + + // detect whether it's `match` by type covering all cases; type checker has already checked that + // match by type can not be mixed with match by expression and can't contain `else`, and that types are distinct and cover all variants + bool is_exhaustive_match_by_type = v->get_arms_count() > 0 && v->get_arm(0)->pattern_kind == MatchArmKind::exact_type; + if (is_exhaustive_match_by_type) { + // example: `match (v) { int => ... slice => ... builder => ... }` + // construct nested IFs: IF is int { ... } ELSE { IF is slice { ... } ELSE { ... } } + for (int i = 0; i < v->get_arms_count() - 1; ++i) { + auto v_ith_arm = v->get_arm(i); + TypePtr cmp_type = v_ith_arm->exact_type->unwrap_alias(); + tolk_assert(!cmp_type->try_as()); // `match` over `int|slice` is a type checker error + std::vector eq_ith_ir_idx = pre_compile_is_type(code, lhs_type, cmp_type, subj_ir_idx, v_ith_arm->loc, "(arm-cond-eq)"); + Op& if_op = code.emplace_back(v_ith_arm->loc, Op::_If, std::move(eq_ith_ir_idx)); + code.push_set_cur(if_op.block0); + if (v->is_statement()) { + pre_compile_expr(v_ith_arm->get_body(), code); + } else { + std::vector arm_ir_idx = pre_compile_expr(v_ith_arm->get_body(), code, v->inferred_type); + code.emplace_back(v->loc, Op::_Let, result_ir_idx, std::move(arm_ir_idx)); + } + code.close_pop_cur(v->loc); + code.push_set_cur(if_op.block1); // open ELSE + } - if (const TypeDataNullable* t_nullable = expr_type->try_as()) { - if (!t_nullable->is_primitive_nullable()) { - std::vector zero_ir_idx = code.create_tmp_var(TypeDataInt::create(), v->loc, "(zero)"); - code.emplace_back(v->loc, Op::_IntConst, zero_ir_idx, td::make_refint(0)); - FunctionPtr eq_sym = lookup_global_symbol("_==_")->try_as(); - code.emplace_back(v->loc, Op::_Call, isnull_ir_idx, std::vector{expr_ir_idx.back(), zero_ir_idx[0]}, eq_sym); + // we're inside the last ELSE, close all outer IFs + auto v_last_arm = v->get_arm(v->get_arms_count() - 1); + if (v->is_statement()) { + pre_compile_expr(v_last_arm->get_body(), code); } else { - FunctionPtr builtin_sym = lookup_global_symbol("__isNull")->try_as(); - code.emplace_back(v->loc, Op::_Call, isnull_ir_idx, expr_ir_idx, builtin_sym); + std::vector arm_ir_idx = pre_compile_expr(v_last_arm->get_body(), code, v->inferred_type); + code.emplace_back(v->loc, Op::_Let, result_ir_idx, std::move(arm_ir_idx)); + } + for (int i = 0; i < v->get_arms_count() - 1; ++i) { + code.close_pop_cur(v->loc); } + } else { - bool always_null = expr_type == TypeDataNullLiteral::create(); - code.emplace_back(v->loc, Op::_IntConst, isnull_ir_idx, td::make_refint(always_null ? -1 : 0)); + // not exhaustive `match` by type + for (int i = 0; i < v->get_arms_count(); ++i) { + auto v_arm = v->get_arm(i); + switch (v_arm->pattern_kind) { + case MatchArmKind::const_expression: + tolk_assert(false); // match by expressions is not supported yet, checked earlier + break; + case MatchArmKind::exact_type: + tolk_assert(false); // match by type is exhaustive, checked earlier + break; + case MatchArmKind::else_branch: + tolk_assert(false); // `else` in match can be only inside match by expression, unsupported yet + break; + } + } } - if (v->is_negated) { - FunctionPtr not_sym = lookup_global_symbol("!b_")->try_as(); - code.emplace_back(v->loc, Op::_Call, isnull_ir_idx, std::vector{isnull_ir_idx}, not_sym); - } - return transition_to_target_type(std::move(isnull_ir_idx), code, target_type, v); + return transition_to_target_type(std::move(result_ir_idx), code, target_type, v); } static std::vector process_dot_access(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { @@ -1048,6 +1248,15 @@ static std::vector process_function_call(V v, Code return transition_to_target_type(std::move(rvect_apply), code, target_type, v); } +static std::vector process_braced_expression(V v, CodeBlob& code, TypePtr target_type) { + // `{ ... }` used as an expression can not return a value currently (there is no syntax in a language) + // that's why it can appear only in special places, and its usage correctness has been checked + tolk_assert(v->inferred_type == TypeDataVoid::create() || v->inferred_type == TypeDataNever::create()); + process_any_statement(v->get_sequence(), code); + static_cast(target_type); + return {}; +} + static std::vector process_tensor(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { // tensor is compiled "as is", for example `(1, null)` occupies 2 slots // and if assigned/passed to something other, like `(int, (int,int)?)`, a whole tensor is transitioned, it works @@ -1068,6 +1277,8 @@ static std::vector process_typed_tuple(V v, CodeBlob static std::vector process_int_const(V v, CodeBlob& code, TypePtr target_type) { std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(int-const)"); code.emplace_back(v->loc, Op::_IntConst, rvect, v->intval); + // here, like everywhere, even for just `int`, there might be a potential transition due to union types + // example: passing `1` to `int | slice` puts actually "1 5" on a stack (1 for value, 5 for UTag = type_id of `int`) return transition_to_target_type(std::move(rvect), code, target_type, v); } @@ -1113,6 +1324,11 @@ static std::vector process_underscore(V v, CodeBlob& return code.create_tmp_var(v->inferred_type, v->loc, "(underscore)"); } +static std::vector process_empty_expression(V v, CodeBlob& code, TypePtr target_type) { + std::vector empty_rvect; + return transition_to_target_type(std::move(empty_rvect), code, target_type, v); +} + std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { switch (v->type) { case ast_reference: @@ -1129,16 +1345,20 @@ std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr targ return process_ternary_operator(v->as(), code, target_type); case ast_cast_as_operator: return process_cast_as_operator(v->as(), code, target_type, lval_ctx); + case ast_is_type_operator: + return process_is_type_operator(v->as(), code, target_type); case ast_not_null_operator: return process_not_null_operator(v->as(), code, target_type, lval_ctx); - case ast_is_null_check: - return process_is_null_check(v->as(), code, target_type); + case ast_match_expression: + return process_match_expression(v->as(), code, target_type); case ast_dot_access: return process_dot_access(v->as(), code, target_type, lval_ctx); case ast_function_call: return process_function_call(v->as(), code, target_type); case ast_parenthesized_expression: return pre_compile_expr(v->as()->get_expr(), code, target_type, lval_ctx); + case ast_braced_expression: + return process_braced_expression(v->as(), code, target_type); case ast_tensor: return process_tensor(v->as(), code, target_type, lval_ctx); case ast_typed_tuple: @@ -1157,6 +1377,8 @@ std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr targ return process_local_vars_declaration(v->as(), code); case ast_underscore: return process_underscore(v->as(), code); + case ast_empty_expression: + return process_empty_expression(v->as(), code, target_type); default: throw UnexpectedASTNodeType(v, "pre_compile_expr"); } @@ -1313,14 +1535,12 @@ static void process_throw_statement(V v, CodeBlob& code) { } static void process_return_statement(V v, CodeBlob& code) { - std::vector return_vars; - if (v->has_return_value()) { - TypePtr child_target_type = code.fun_ref->inferred_return_type; - if (code.fun_ref->does_return_self()) { - child_target_type = code.fun_ref->parameters[0].declared_type; - } - return_vars = pre_compile_expr(v->get_return_value(), code, child_target_type); + TypePtr child_target_type = code.fun_ref->inferred_return_type; + if (code.fun_ref->does_return_self()) { + child_target_type = code.fun_ref->parameters[0].declared_type; } + std::vector return_vars = pre_compile_expr(v->get_return_value(), code, child_target_type); + if (code.fun_ref->does_return_self()) { return_vars = {}; } diff --git a/tolk/pipe-calc-rvalue-lvalue.cpp b/tolk/pipe-calc-rvalue-lvalue.cpp index 1f374bc81..769d31c01 100644 --- a/tolk/pipe-calc-rvalue-lvalue.cpp +++ b/tolk/pipe-calc-rvalue-lvalue.cpp @@ -24,6 +24,8 @@ * * Example: `a = b`, `a` is lvalue, `b` is rvalue. * Example: `a + b`, both are rvalue. + * Example: `a;`, it's none (not rvalue, not lvalue). + * For instance, difference between none and rvalue helps detect whether `match` is expression or statement. * * Note, that this pass only assigns, not checks. So, for `f() = 4`, expr `f()` is lvalue. * Checking (firing this as incorrect later) is performed after type inferring, see pipe-check-rvalue-lvalue. @@ -47,79 +49,101 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { return saved; } + MarkingState enter_rvalue_if_none() { + if (cur_state != MarkingState::None) { + return cur_state; + } + cur_state = MarkingState::RValue; + return MarkingState::None; + } + void restore_state(MarkingState saved) { cur_state = saved; } - void mark_vertex_cur_or_rvalue(AnyExprV v) const { + void mark_vertex(AnyExprV v) const { if (cur_state == MarkingState::LValue || cur_state == MarkingState::LValueAndRValue) { v->mutate()->assign_lvalue_true(); } - if (cur_state == MarkingState::RValue || cur_state == MarkingState::LValueAndRValue || cur_state == MarkingState::None) { + if (cur_state == MarkingState::RValue || cur_state == MarkingState::LValueAndRValue) { v->mutate()->assign_rvalue_true(); } } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); + MarkingState saved = enter_rvalue_if_none(); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + mark_vertex(v); + MarkingState saved = enter_state(MarkingState::None); parent::visit(v); + restore_state(saved); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); + MarkingState saved = enter_rvalue_if_none(); parent::visit(v); + restore_state(saved); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); + MarkingState saved = enter_rvalue_if_none(); parent::visit(v); + restore_state(saved); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); MarkingState saved = enter_state(v->passed_as_mutate ? MarkingState::LValueAndRValue : MarkingState::RValue); parent::visit(v); restore_state(saved); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + tolk_assert(cur_state == MarkingState::RValue); + mark_vertex(v); parent::visit(v); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); MarkingState saved = enter_state(MarkingState::RValue); parent::visit(v->get_obj()); restore_state(saved); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); MarkingState saved = enter_state(MarkingState::RValue); parent::visit(v); restore_state(saved); @@ -130,11 +154,11 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { // so, if current state is "lvalue", `_` will be marked as lvalue, and ok // but if used incorrectly, like `f(_)` or just `_;`, it will be marked rvalue // and will fire an error later, in pipe lvalue/rvalue check - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); MarkingState saved = enter_state(MarkingState::LValue); parent::visit(v->get_lhs()); enter_state(MarkingState::RValue); @@ -143,7 +167,7 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); MarkingState saved = enter_state(MarkingState::LValueAndRValue); parent::visit(v->get_lhs()); enter_state(MarkingState::RValue); @@ -152,53 +176,114 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); MarkingState saved = enter_state(MarkingState::RValue); parent::visit(v); restore_state(saved); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); MarkingState saved = enter_state(MarkingState::RValue); parent::visit(v); restore_state(saved); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); MarkingState saved = enter_state(MarkingState::RValue); parent::visit(v); // both cond, when_true and when_false are rvalue, `(cond ? a : b) = 5` prohibited restore_state(saved); } void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); parent::visit(v->get_expr()); // leave lvalue state unchanged, for `mutate (t.0 as int)` both `t.0 as int` and `t.0` are lvalue } + void visit(V v) override { + mark_vertex(v); + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v->get_expr()); + restore_state(saved); + } + void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); parent::visit(v->get_expr()); // leave lvalue state unchanged, for `mutate x!` both `x!` and `x` are lvalue } - void visit(V v) override { - mark_vertex_cur_or_rvalue(v); + void visit(V v) override { + mark_vertex(v); MarkingState saved = enter_state(MarkingState::RValue); - parent::visit(v->get_expr()); + parent::visit(v); restore_state(saved); } + void visit(V v) override { + tolk_assert(cur_state == MarkingState::RValue); + mark_vertex(v); + parent::visit(v); + } + void visit(V v) override { tolk_assert(cur_state == MarkingState::LValue); - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); parent::visit(v); } void visit(V v) override { tolk_assert(cur_state == MarkingState::LValue); - mark_vertex_cur_or_rvalue(v); + mark_vertex(v); + parent::visit(v); + } + + void visit(V v) override { + MarkingState saved = enter_state(MarkingState::None); parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); } void visit(V v) override { diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 927cfc4e1..2174560fd 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -119,12 +119,14 @@ static void handle_possible_compiler_internal_call(FunctionPtr cur_f, Vget_num_args() == 2); TypePtr expected_type = parse_type_from_string(v->get_arg(1)->get_expr()->as()->str_val); if (expected_type->has_unresolved_inside()) { // only aliases can be inside - expected_type = expected_type->replace_children_custom([](TypePtr child) -> TypePtr { + expected_type = expected_type->replace_children_custom([cur_f, v](TypePtr child) -> TypePtr { if (const auto* as_unresolved = child->try_as()) { const Symbol* sym = lookup_global_symbol(as_unresolved->text); - tolk_assert(sym); + if (!sym) { + throw ParseError(cur_f, v->loc, "invalid __expect_type"); + } if (AliasDefPtr alias_ref = sym->try_as()) { - return TypeDataAlias::create(alias_ref->name, alias_ref->underlying_type); + return TypeDataAlias::create(alias_ref); } tolk_assert(false); } @@ -132,7 +134,7 @@ static void handle_possible_compiler_internal_call(FunctionPtr cur_f, Vget_arg(0)->inferred_type; - if (expected_type != expr_type) { + if (expected_type->as_human_readable() != expr_type->as_human_readable()) { fire(cur_f, v->loc, "__expect_type failed: expected " + to_string(expected_type) + ", got " + to_string(expr_type)); } } @@ -282,25 +284,29 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } } - void visit(V v) override { + void visit(V v) override { parent::visit(v->get_expr()); - if (v->get_expr()->inferred_type == TypeDataNullLiteral::create()) { - // operator `!` used for always-null (proven by smart casts, for example), it's an error - fire(cur_f, v->loc, "operator `!` used for always null expression"); + if (v->rhs_type->unwrap_alias()->try_as()) { // `v is T1 | T2` / `v is T?` is disallowed + fire(cur_f, v->loc, "union types are not allowed, use concrete types in `is`"); } - // if operator `!` used for non-nullable, probably a warning should be printed - } - - void visit(V v) override { - parent::visit(v->get_expr()); if ((v->is_always_true && !v->is_negated) || (v->is_always_false && v->is_negated)) { - v->loc.show_warning(expression_as_string(v->get_expr()) + " is always null, this condition is always " + (v->is_always_true ? "true" : "false")); + v->loc.show_warning(expression_as_string(v->get_expr()) + " is always " + to_string(v->rhs_type) + ", this condition is always " + (v->is_always_true ? "true" : "false")); } if ((v->is_always_false && !v->is_negated) || (v->is_always_true && v->is_negated)) { - v->loc.show_warning(expression_as_string(v->get_expr()) + " of type " + to_string(v->get_expr()) + " is always not null, this condition is always " + (v->is_always_true ? "true" : "false")); + v->loc.show_warning(expression_as_string(v->get_expr()) + " of type " + to_string(v->get_expr()) + " can never be " + to_string(v->rhs_type) + ", this condition is always " + (v->is_always_true ? "true" : "false")); + } + } + + void visit(V v) override { + parent::visit(v->get_expr()); + + if (v->get_expr()->inferred_type == TypeDataNullLiteral::create()) { + // operator `!` used for always-null (proven by smart casts, for example), it's an error + fire(cur_f, v->loc, "operator `!` used for always null expression"); } + // if operator `!` used for non-nullable, probably a warning should be printed } void visit(V v) override { @@ -318,8 +324,8 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { parent::visit(v); if (v->is_target_indexed_access()) { - TypePtr obj_type = v->get_obj()->inferred_type; - if (obj_type->unwrap_alias()->try_as() && v->inferred_type->get_width_on_stack() != 1) { + TypePtr obj_type = v->get_obj()->inferred_type->unwrap_alias(); + if (v->inferred_type->get_width_on_stack() != 1 && (obj_type->try_as() || obj_type->try_as())) { fire_error_cannot_put_non1_stack_width_arg_to_tuple(cur_f, v->loc, v->inferred_type); } } @@ -513,6 +519,82 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } } + void visit(V v) override { + parent::visit(v); + + bool has_type_arm = false; + bool has_expr_arm = false; + bool has_else_arm = false; + TypePtr subject_type = v->get_subject()->inferred_type->unwrap_alias(); + const TypeDataUnion* subject_union = subject_type->try_as(); + + std::vector covered_type_ids; // for type-based `match`, what types are on the left of `=>` + + for (int i = 0; i < v->get_arms_count(); ++i) { + auto v_arm = v->get_arm(i); + switch (v_arm->pattern_kind) { + case MatchArmKind::exact_type: { + if (has_expr_arm) { + fire(cur_f, v_arm->loc, "can not mix type and expression patterns in `match`"); + } + if (has_else_arm) { + fire(cur_f, v_arm->loc, "`else` branch should be the last"); + } + has_type_arm = true; + + TypePtr lhs_type = v_arm->exact_type->unwrap_alias(); // `lhs_type => ...` + if (lhs_type->try_as()) { + fire(cur_f, v_arm->loc, "wrong pattern matching: union types are not allowed, use concrete types in `match`"); + } + bool can_happen = (subject_union && subject_union->has_variant_with_type_id(lhs_type)) || + (!subject_union && subject_type->equal_to(lhs_type)); + if (!can_happen) { + fire(cur_f, v_arm->loc, "wrong pattern matching: " + to_string(lhs_type) + " is not a variant of " + to_string(subject_type)); + } + if (std::find(covered_type_ids.begin(), covered_type_ids.end(), lhs_type->get_type_id()) != covered_type_ids.end()) { + fire(cur_f, v_arm->loc, "wrong pattern matching: duplicated " + to_string(lhs_type)); + } + covered_type_ids.push_back(lhs_type->get_type_id()); + break; + } + case MatchArmKind::const_expression: { + if (has_type_arm) { + fire(cur_f, v_arm->loc, "can not mix type and expression patterns in `match`"); + } + if (has_else_arm) { + fire(cur_f, v_arm->loc, "`else` branch should be the last"); + } + has_expr_arm = true; + fire(cur_f, v_arm->loc, "`match` by expression is not supported yet"); + break; + } + default: + if (has_else_arm) { + fire(cur_f, v_arm->loc, "duplicated `else` branch"); + } + if (has_type_arm) { + fire(cur_f, v_arm->loc, "`else` is not allowed in `match` by type; you should cover all possible types"); + } + has_else_arm = true; + fire(cur_f, v_arm->loc, "`match` by expression is not supported yet"); + } + } + + // fire if `match` by type is not exhaustive + if (has_type_arm && subject_union && subject_union->variants.size() != covered_type_ids.size()) { + std::string missing; + for (TypePtr variant : subject_union->variants) { + if (std::find(covered_type_ids.begin(), covered_type_ids.end(), variant->get_type_id()) == covered_type_ids.end()) { + if (!missing.empty()) { + missing += ", "; + } + missing += to_string(variant); + } + } + throw ParseError(cur_f, v->loc, "`match` does not cover all possible types; missing types are: " + missing); + } + } + void visit(V v) override { parent::visit(v); diff --git a/tolk/pipe-check-rvalue-lvalue.cpp b/tolk/pipe-check-rvalue-lvalue.cpp index 3ec47a16b..886091127 100644 --- a/tolk/pipe-check-rvalue-lvalue.cpp +++ b/tolk/pipe-check-rvalue-lvalue.cpp @@ -29,65 +29,80 @@ namespace tolk { +// fire a general error, just a wrapper over `throw` GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_cannot_be_used_as_lvalue(AnyV v, const std::string& details) { +static void fire(FunctionPtr cur_f, SrcLocation loc, const std::string& message) { + throw ParseError(cur_f, loc, message); +} + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_cannot_be_used_as_lvalue(FunctionPtr cur_f, AnyV v, const std::string& details) { // example: `f() = 32` // example: `loadUint(c.beginParse(), 32)` (since `loadUint()` mutates the first argument) - v->error(details + " can not be used as lvalue"); + throw ParseError(cur_f, v->loc, details + " can not be used as lvalue"); } GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_modifying_immutable_variable(AnyExprV v, LocalVarPtr var_ref) { +static void fire_error_modifying_immutable_variable(FunctionPtr cur_f, AnyExprV v, LocalVarPtr var_ref) { if (var_ref->param_idx == 0 && var_ref->name == "self") { - v->error("modifying `self`, which is immutable by default; probably, you want to declare `mutate self`"); + throw ParseError(cur_f, v->loc, "modifying `self`, which is immutable by default; probably, you want to declare `mutate self`"); } else { - v->error("modifying immutable variable `" + var_ref->name + "`"); + throw ParseError(cur_f, v->loc, "modifying immutable variable `" + var_ref->name + "`"); } } // validate a function used as rvalue, like `var cb = f` // it's not a generic function (ensured earlier at type inferring) and has some more restrictions -static void validate_function_used_as_noncall(AnyExprV v, FunctionPtr fun_ref) { +static void validate_function_used_as_noncall(FunctionPtr cur_f, AnyExprV v, FunctionPtr fun_ref) { if (!fun_ref->arg_order.empty() || !fun_ref->ret_order.empty()) { - v->error("saving `" + fun_ref->name + "` into a variable will most likely lead to invalid usage, since it changes the order of variables on the stack"); + fire(cur_f, v->loc, "saving `" + fun_ref->name + "` into a variable will most likely lead to invalid usage, since it changes the order of variables on the stack"); } if (fun_ref->has_mutate_params()) { - v->error("saving `" + fun_ref->name + "` into a variable is impossible, since it has `mutate` parameters and thus can only be called directly"); + fire(cur_f, v->loc, "saving `" + fun_ref->name + "` into a variable is impossible, since it has `mutate` parameters and thus can only be called directly"); } } class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { + FunctionPtr cur_f = nullptr; + + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(cur_f, v, "braced expression"); + } + parent::visit(v); + } + void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "assignment"); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "assignment"); } parent::visit(v); } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "assignment"); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "assignment"); } parent::visit(v); } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "operator " + static_cast(v->operator_name)); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "operator " + static_cast(v->operator_name)); } parent::visit(v); } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "operator " + static_cast(v->operator_name)); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "operator " + static_cast(v->operator_name)); } parent::visit(v); } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "operator ?:"); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "operator ?:"); } parent::visit(v); } @@ -97,52 +112,52 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { parent::visit(v->get_expr()); } - void visit(V v) override { - // if `x!` is lvalue, then `x` is also lvalue, so check that `x` is ok + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(cur_f, v, v->is_negated ? "operator !is" : "operator is"); + } parent::visit(v->get_expr()); } - void visit(V v) override { - if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, v->is_negated ? "operator !=" : "operator =="); - } + void visit(V v) override { + // if `x!` is lvalue, then `x` is also lvalue, so check that `x` is ok parent::visit(v->get_expr()); } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "literal"); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "literal"); } } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "literal"); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "literal"); } } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "literal"); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "literal"); } } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "literal"); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "literal"); } } void visit(V v) override { // a reference to a method used as rvalue, like `var v = t.tupleAt` if (v->is_rvalue && v->is_target_fun_ref()) { - validate_function_used_as_noncall(v, std::get(v->target)); + validate_function_used_as_noncall(cur_f, v, std::get(v->target)); } } void visit(V v) override { if (v->is_lvalue) { - fire_error_cannot_be_used_as_lvalue(v, "function call"); + fire_error_cannot_be_used_as_lvalue(cur_f, v, "function call"); } if (!v->fun_maybe) { parent::visit(v->get_callee()); @@ -158,11 +173,18 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { } } + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(cur_f, v, "`match` expression"); + } + parent::visit(v); + } + void visit(V v) override { if (v->marked_as_redef) { tolk_assert(v->var_ref); if (v->var_ref->is_immutable()) { - v->error("`redef` for immutable variable"); + fire(cur_f, v->loc, "`redef` for immutable variable"); } } } @@ -171,23 +193,23 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { if (v->is_lvalue) { tolk_assert(v->sym); if (LocalVarPtr var_ref = v->sym->try_as(); var_ref && var_ref->is_immutable()) { - fire_error_modifying_immutable_variable(v, var_ref); + fire_error_modifying_immutable_variable(cur_f, v, var_ref); } else if (v->sym->try_as()) { - v->error("modifying immutable constant"); + fire(cur_f, v->loc, "modifying immutable constant"); } else if (v->sym->try_as()) { - v->error("function can't be used as lvalue"); + fire(cur_f, v->loc, "function can't be used as lvalue"); } } // a reference to a function used as rvalue, like `var v = someFunction` if (FunctionPtr fun_ref = v->sym->try_as(); fun_ref && v->is_rvalue) { - validate_function_used_as_noncall(v, fun_ref); + validate_function_used_as_noncall(cur_f, v, fun_ref); } } void visit(V v) override { if (v->is_rvalue) { - v->error("`_` can't be used as a value; it's a placeholder for a left side of assignment"); + fire(cur_f, v->loc, "`_` can't be used as a value; it's a placeholder for a left side of assignment"); } } @@ -201,6 +223,12 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function(); } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + parent::visit(v_function->get_body()); + cur_f = nullptr; + } }; void pipeline_check_rvalue_lvalue() { diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp index 13ecece51..1a071f492 100644 --- a/tolk/pipe-constant-folding.cpp +++ b/tolk/pipe-constant-folding.cpp @@ -96,11 +96,11 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { return v; } - AnyExprV replace(V v) override { + AnyExprV replace(V v) override { parent::replace(v); // `null == null` / `null != null` - if (v->get_expr()->type == ast_null_keyword) { + if (v->get_expr()->type == ast_null_keyword && v->rhs_type == TypeDataNullLiteral::create()) { return create_bool_const(v->loc, !v->is_negated); } diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 236037c3e..c0c7c1d39 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -238,12 +238,14 @@ class InferTypesAndCallsAndFieldsVisitor final { return infer_ternary_operator(v->as(), std::move(flow), used_as_condition, hint); case ast_cast_as_operator: return infer_cast_as_operator(v->as(), std::move(flow), used_as_condition); + case ast_is_type_operator: + return infer_is_type_operator(v->as(), std::move(flow), used_as_condition); case ast_not_null_operator: return infer_not_null_operator(v->as(), std::move(flow), used_as_condition); - case ast_is_null_check: - return infer_is_null_check(v->as(), std::move(flow), used_as_condition); case ast_parenthesized_expression: return infer_parenthesized(v->as(), std::move(flow), used_as_condition, hint); + case ast_braced_expression: + return infer_braced_expression(v->as(), std::move(flow), used_as_condition); case ast_reference: return infer_reference(v->as(), std::move(flow), used_as_condition); case ast_dot_access: @@ -256,6 +258,8 @@ class InferTypesAndCallsAndFieldsVisitor final { return infer_typed_tuple(v->as(), std::move(flow), used_as_condition, hint); case ast_null_keyword: return infer_null_keyword(v->as(), std::move(flow), used_as_condition); + case ast_match_expression: + return infer_match_expression(v->as(), std::move(flow), used_as_condition, hint); case ast_underscore: return infer_underscore(v->as(), std::move(flow), used_as_condition, hint); case ast_empty_expression: @@ -589,12 +593,18 @@ class InferTypesAndCallsAndFieldsVisitor final { return after_false; } - TypeInferringUnifyStrategy tern_type; - tern_type.unify_with(v->get_when_true()->inferred_type); - if (!tern_type.unify_with(v->get_when_false()->inferred_type)) { - fire(cur_f, v->loc, "types of ternary branches are incompatible: " + to_string(v->get_when_true()) + " and " + to_string(v->get_when_false())); + TypeInferringUnifyStrategy branches_unifier; + branches_unifier.unify_with(v->get_when_true()->inferred_type, hint); + branches_unifier.unify_with(v->get_when_false()->inferred_type, hint); + if (branches_unifier.is_union_of_different_types()) { + // `... ? intVar : sliceVar` results in `int | slice`, probably it's not what the user expected + // example: `var v = ternary`, show an inference error + // do NOT show an error for `var v: T = ternary` (T is hint); it will be checked by type checker later + if (hint == nullptr || hint == TypeDataUnknown::create()) { + fire(cur_f, v->loc, "types of ternary branches are incompatible: " + to_string(v->get_when_true()) + " and " + to_string(v->get_when_false())); + } } - assign_inferred_type(v, tern_type.get_result()); + assign_inferred_type(v, branches_unifier.get_result()); FlowContext out_flow = FlowContext::merge_flow(std::move(after_true.out_flow), std::move(after_false.out_flow)); return ExprFlow(std::move(out_flow), std::move(after_true.true_flow), std::move(after_false.false_flow)); @@ -611,15 +621,16 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(after_expr.out_flow), true); } - ExprFlow infer_is_null_check(V v, FlowContext&& flow, bool used_as_condition) { + ExprFlow infer_is_type_operator(V v, FlowContext&& flow, bool used_as_condition) { ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false); assign_inferred_type(v, TypeDataBool::create()); + TypePtr rhs_type = v->rhs_type->unwrap_alias(); TypePtr expr_type = v->get_expr()->inferred_type->unwrap_alias(); - TypePtr non_null_type = calculate_type_subtract_null(expr_type); - if (expr_type == TypeDataNullLiteral::create()) { // `expr == null` is always true + TypePtr non_rhs_type = calculate_type_subtract_rhs_type(expr_type, rhs_type); + if (expr_type->equal_to(v->rhs_type)) { // `expr is ` is always true v->mutate()->assign_always_true_or_false(v->is_negated ? 2 : 1); - } else if (non_null_type == TypeDataNever::create()) { // `expr == null` is always false + } else if (non_rhs_type == TypeDataNever::create()) { // `expr is ` is always false v->mutate()->assign_always_true_or_false(v->is_negated ? 1 : 2); } else { v->mutate()->assign_always_true_or_false(0); @@ -639,11 +650,11 @@ class InferTypesAndCallsAndFieldsVisitor final { true_flow.mark_unreachable(UnreachableKind::CantHappen); true_flow.register_known_type(s_expr, TypeDataNever::create()); } else if (!v->is_negated) { - true_flow.register_known_type(s_expr, TypeDataNullLiteral::create()); - false_flow.register_known_type(s_expr, non_null_type); + true_flow.register_known_type(s_expr, rhs_type); + false_flow.register_known_type(s_expr, non_rhs_type); } else { - true_flow.register_known_type(s_expr, non_null_type); - false_flow.register_known_type(s_expr, TypeDataNullLiteral::create()); + true_flow.register_known_type(s_expr, non_rhs_type); + false_flow.register_known_type(s_expr, rhs_type); } } return ExprFlow(std::move(after_expr.out_flow), std::move(true_flow), std::move(false_flow)); @@ -651,12 +662,9 @@ class InferTypesAndCallsAndFieldsVisitor final { ExprFlow infer_not_null_operator(V v, FlowContext&& flow, bool used_as_condition) { ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false); - - if (const auto* as_nullable = v->get_expr()->inferred_type->unwrap_alias()->try_as()) { - assign_inferred_type(v, as_nullable->inner); - } else { - assign_inferred_type(v, v->get_expr()); - } + TypePtr expr_type = v->get_expr()->inferred_type; + TypePtr without_null_type = calculate_type_subtract_rhs_type(expr_type->unwrap_alias(), TypeDataNullLiteral::create()); + assign_inferred_type(v, without_null_type != TypeDataNever::create() ? without_null_type : expr_type); if (!used_as_condition) { return after_expr; @@ -670,6 +678,13 @@ class InferTypesAndCallsAndFieldsVisitor final { return after_expr; } + ExprFlow infer_braced_expression(V v, FlowContext&& flow, bool used_as_condition) { + // `{ ... }` used as an expression can not return a value currently (there is no syntax in a language) + flow = process_any_statement(v->get_sequence(), std::move(flow)); + assign_inferred_type(v, flow.is_unreachable() ? TypeDataNever::create() : TypeDataVoid::create()); + return ExprFlow(std::move(flow), used_as_condition); + } + ExprFlow infer_reference(V v, FlowContext&& flow, bool used_as_condition) { if (LocalVarPtr var_ref = v->sym->try_as()) { TypePtr declared_or_smart_casted = flow.smart_cast_if_exists(SinkExpression(var_ref)); @@ -1058,6 +1073,45 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(flow), used_as_condition); } + ExprFlow infer_match_expression(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + flow = infer_any_expr(v->get_subject(), std::move(flow), false).out_flow; + SinkExpression s_expr = extract_sink_expression_from_vertex(v->get_subject()); + TypeInferringUnifyStrategy branches_unifier; + FlowContext arms_entry_facts = flow.clone(); + FlowContext match_out_flow; + + for (int i = 0; i < v->get_arms_count(); ++i) { + auto v_arm = v->get_arm(i); + FlowContext arm_flow = infer_any_expr(v_arm->get_pattern_expr(), arms_entry_facts.clone(), false, nullptr).out_flow; + if (s_expr && v_arm->pattern_kind == MatchArmKind::exact_type) { + arm_flow.register_known_type(s_expr, v_arm->exact_type); + } + arm_flow = infer_any_expr(v_arm->get_body(), std::move(arm_flow), false, hint).out_flow; + match_out_flow = i == 0 ? std::move(arm_flow) : FlowContext::merge_flow(std::move(match_out_flow), std::move(arm_flow)); + branches_unifier.unify_with(v_arm->get_body()->inferred_type, hint); + } + if (v->get_arms_count() == 0) { + match_out_flow = std::move(arms_entry_facts); + } + + if (v->is_statement()) { + assign_inferred_type(v, TypeDataVoid::create()); + } else { + if (v->get_arms_count() == 0) { // still allow empty `match` statements, for probable codegen + fire(cur_f, v->loc, "empty `match` can't be used as expression"); + } + if (branches_unifier.is_union_of_different_types()) { + // same as in ternary: `match (...) { t1 => someSlice, t2 => someInt }` is `int|slice`, probably unexpected + if (hint == nullptr || hint == TypeDataUnknown::create()) { + fire(cur_f, v->loc, "type of `match` was inferred as " + to_string(branches_unifier.get_result()) + "; probably, it's not what you expected; assign it to a variable `var v: = match (...) { ... }` manually"); + } + } + assign_inferred_type(v, branches_unifier.get_result()); + } + + return ExprFlow(std::move(match_out_flow), used_as_condition); + } + static ExprFlow infer_underscore(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { // if execution is here, underscore is either used as lhs of assignment, or incorrectly, like `f(_)` // more precise is to always set unknown here, but for incorrect usages, instead of an error @@ -1229,21 +1283,35 @@ class InferTypesAndCallsAndFieldsVisitor final { if (fun_ref->does_return_self()) { return_unifier.unify_with(fun_ref->parameters[0].declared_type); } - for (AnyExprV return_value : return_statements) { - if (!return_unifier.unify_with(return_value->inferred_type)) { - fire(cur_f, return_value->loc, "can not unify type " + to_string(return_value) + " with previous return type " + to_string(return_unifier.get_result())); - } - } - if (!body_end.is_unreachable()) { - if (!return_unifier.unify_with_implicit_return_void()) { - fire(cur_f, v_function->get_body()->as()->loc_end, "missing return"); - } + bool has_void_returns = false; + bool has_non_void_returns = false; + for (AnyExprV return_expr : return_statements) { + TypePtr cur_type = return_expr->inferred_type; // `return expr` - type of expr; `return` - void + return_unifier.unify_with(cur_type); + has_void_returns |= cur_type == TypeDataVoid::create(); + has_non_void_returns |= cur_type != TypeDataVoid::create(); } inferred_return_type = return_unifier.get_result(); - if (inferred_return_type == nullptr && body_end.is_unreachable()) { + if (inferred_return_type == nullptr) { // if no return statements at all inferred_return_type = TypeDataVoid::create(); } + + if (!body_end.is_unreachable() && inferred_return_type != TypeDataVoid::create()) { + fire(fun_ref, v_function->get_body()->as()->loc_end, "missing return"); + } + if (has_void_returns && has_non_void_returns) { + for (AnyExprV return_expr : return_statements) { + if (return_expr->inferred_type == TypeDataVoid::create()) { + fire(fun_ref, return_expr->loc, "mixing void and non-void returns in function `" + fun_ref->as_human_readable() + "`"); + } + } + } + if (return_unifier.is_union_of_different_types()) { + // `return intVar` + `return sliceVar` results in `int | slice`, probably unexpected + fire(fun_ref, v_function->get_body()->loc, "function `" + fun_ref->as_human_readable() + "` calculated return type is " + to_string(inferred_return_type) + "; probably, it's not what you expected; declare `fun (...): ` manually"); + } } + } else { // asm functions should be strictly typed, this was checked earlier tolk_assert(fun_ref->declared_return_type); diff --git a/tolk/pipe-optimize-boolean-expr.cpp b/tolk/pipe-optimize-boolean-expr.cpp index 642defa03..09282cfd9 100644 --- a/tolk/pipe-optimize-boolean-expr.cpp +++ b/tolk/pipe-optimize-boolean-expr.cpp @@ -141,9 +141,9 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { v = createV(v->loc, !v->is_ifnot, v_cond_unary->get_rhs(), v->get_if_body(), v->get_else_body()); } // `if (x != null)` -> ifnot(x == null) - if (auto v_cond_isnull = v->get_cond()->try_as(); v_cond_isnull && v_cond_isnull->is_negated) { - v_cond_isnull->mutate()->assign_is_negated(!v_cond_isnull->is_negated); - v = createV(v->loc, !v->is_ifnot, v_cond_isnull, v->get_if_body(), v->get_else_body()); + if (auto v_cond_istype = v->get_cond()->try_as(); v_cond_istype && v_cond_istype->is_negated) { + v_cond_istype->mutate()->assign_is_negated(!v_cond_istype->is_negated); + v = createV(v->loc, !v->is_ifnot, v_cond_istype, v->get_if_body(), v->get_else_body()); } return v; diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 3f8f622c0..0b116bbe8 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -158,7 +158,7 @@ struct TypeDataResolver { if (alias_ref->underlying_type->has_unresolved_inside()) { resolve_and_mutate_type_alias(alias_ref); } - return TypeDataAlias::create(alias_ref->name, alias_ref->underlying_type); + return TypeDataAlias::create(alias_ref); } } if (un->text == "auto") { @@ -260,6 +260,48 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { } } + void visit(V v) override { + current_scope.open_scope(v->loc); + parent::visit(v->get_sequence()); + current_scope.close_scope(v->loc); + } + + void visit(V v) override { + current_scope.open_scope(v->loc); // `match (var a = init_val) { ... }` + parent::visit(v); // then `a` exists only inside `match` arms + current_scope.close_scope(v->loc); + } + + void visit(V v) override { + // resolve identifiers after => at first + parent::visit(v->get_body()); + // because handling lhs of => is comprehensive + + switch (v->pattern_kind) { + case MatchArmKind::exact_type: { + if (const TypeDataUnresolved* maybe_ident = v->exact_type->try_as()) { + if (const Symbol* sym = current_scope.lookup_symbol(maybe_ident->text); sym && !sym->try_as()) { + auto v_ident = createV(v->loc, sym->name); + AnyExprV pattern_expr = createV(v->loc, v_ident, nullptr); + parent::visit(pattern_expr); + v->mutate()->assign_resolved_pattern(MatchArmKind::const_expression, nullptr, pattern_expr); + return; + } + } + TypePtr resolved_exact_type = finalize_type_data(cur_f, v->exact_type, current_genericTs); + v->mutate()->assign_resolved_pattern(MatchArmKind::exact_type, resolved_exact_type, v->get_pattern_expr()); + break; + } + case MatchArmKind::const_expression: { + parent::visit(v->get_pattern_expr()); + break; + } + default: + // for `else` match branch, do nothing: its body was already traversed above + break; + } + } + void visit(V v) override { // for `t.tupleAt` / `obj.method`, resolve "MyAlias" and "T" // (for function call `t.tupleAt()`, this v (ast_dot_access `t.tupleAt`) is callee) @@ -278,6 +320,12 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { parent::visit(v->get_expr()); } + void visit(V v) override { + TypePtr rhs_type = finalize_type_data(cur_f, v->rhs_type, current_genericTs); + v->mutate()->assign_resolved_type(rhs_type); + parent::visit(v->get_expr()); + } + void visit(V v) override { if (v->empty()) { return; diff --git a/tolk/smart-casts-cfg.cpp b/tolk/smart-casts-cfg.cpp index d3ddb5af7..65ab7fb80 100644 --- a/tolk/smart-casts-cfg.cpp +++ b/tolk/smart-casts-cfg.cpp @@ -133,10 +133,15 @@ static AnyExprV unwrap_not_null_operator(AnyExprV expr) { // 3) when two data flows rejoin // example: `if (tensorVar != null) ... else ...` rejoin `(int,int)` and `null` into `(int,int)?` // when lca can't be calculated (example: `(int,int)` and `(int,int,int)`), nullptr is returned -static TypePtr calculate_type_lca(TypePtr a, TypePtr b) { - if (a == b) { +static TypePtr calculate_type_lca(TypePtr a, TypePtr b, bool* became_union = nullptr) { + if (a->equal_to(b)) { return a; } + + if (a == TypeDataUnknown::create() || b == TypeDataUnknown::create()) { + return TypeDataUnknown::create(); + } + if (a == TypeDataNever::create()) { return b; } @@ -144,22 +149,11 @@ static TypePtr calculate_type_lca(TypePtr a, TypePtr b) { return a; } - if (a->can_rhs_be_assigned(b)) { - return a; - } - if (b->can_rhs_be_assigned(a)) { - return b; - } - - if (a == TypeDataUnknown::create() || b == TypeDataUnknown::create()) { - return TypeDataUnknown::create(); - } - if (a == TypeDataNullLiteral::create()) { - return TypeDataNullable::create(b); + return TypeDataUnion::create_nullable(b); } if (b == TypeDataNullLiteral::create()) { - return TypeDataNullable::create(a); + return TypeDataUnion::create_nullable(a); } const auto* tensor1 = a->try_as(); @@ -168,7 +162,7 @@ static TypePtr calculate_type_lca(TypePtr a, TypePtr b) { std::vector types_lca; types_lca.reserve(tensor1->size()); for (int i = 0; i < tensor1->size(); ++i) { - TypePtr next = calculate_type_lca(tensor1->items[i], tensor2->items[i]); + TypePtr next = calculate_type_lca(tensor1->items[i], tensor2->items[i], became_union); if (next == nullptr) { return nullptr; } @@ -183,7 +177,7 @@ static TypePtr calculate_type_lca(TypePtr a, TypePtr b) { std::vector types_lca; types_lca.reserve(tuple1->size()); for (int i = 0; i < tuple1->size(); ++i) { - TypePtr next = calculate_type_lca(tuple1->items[i], tuple2->items[i]); + TypePtr next = calculate_type_lca(tuple1->items[i], tuple2->items[i], became_union); if (next == nullptr) { return nullptr; } @@ -192,18 +186,18 @@ static TypePtr calculate_type_lca(TypePtr a, TypePtr b) { return TypeDataTypedTuple::create(std::move(types_lca)); } - if (a->try_as() && b->try_as()) { // cond ? int32 : int64 - return TypeDataInt::create(); - } - if (const auto* a_alias = a->try_as()) { - return calculate_type_lca(a_alias->underlying_type, b); + return calculate_type_lca(a_alias->underlying_type, b, became_union); } if (const auto* b_alias = b->try_as()) { - return calculate_type_lca(a, b_alias->underlying_type); + return calculate_type_lca(a, b_alias->underlying_type, became_union); } - return nullptr; + TypePtr resulting_union = TypeDataUnion::create(std::vector{a, b}); + if (became_union != nullptr && !a->equal_to(resulting_union) && !b->equal_to(resulting_union)) { + *became_union = true; + } + return resulting_union; } // merge (unify) of two sign states: what sign do we definitely have @@ -248,31 +242,28 @@ BoolState calculate_bool_lca(BoolState a, BoolState b) { // see comments above TypeInferringUnifyStrategy // this function calculates lca or currently stored result and next -bool TypeInferringUnifyStrategy::unify_with(TypePtr next) { +void TypeInferringUnifyStrategy::unify_with(TypePtr next, TypePtr dest_hint) { + // example: `var r = ... ? int8 : int16`, will be inferred as `int8 | int16` (via unification) + // but `var r: int = ... ? int8 : int16`, will be inferred as `int` (it's dest_hint) + if (dest_hint && dest_hint != TypeDataUnknown::create() && !dest_hint->unwrap_alias()->try_as()) { + if (dest_hint->can_rhs_be_assigned(next)) { + next = dest_hint; + } + } + if (unified_result == nullptr) { unified_result = next; - return true; + return; } if (unified_result == next) { - return true; + return; } - TypePtr combined = calculate_type_lca(unified_result, next); - if (!combined) { - return false; - } + bool became_union = false; + TypePtr combined = calculate_type_lca(unified_result, next, &became_union); + different_types_became_union |= became_union; unified_result = combined; - return true; -} - -bool TypeInferringUnifyStrategy::unify_with_implicit_return_void() { - if (unified_result == nullptr) { - unified_result = TypeDataVoid::create(); - return true; - } - - return unified_result == TypeDataVoid::create(); } // invalidate knowledge about sub-fields of a variable or its field @@ -356,14 +347,43 @@ FlowContext FlowContext::merge_flow(FlowContext&& c1, FlowContext&& c2) { return FlowContext(std::move(unified), c1.unreachable && c2.unreachable); } -// return `T`, so that `T?` = type -// what for: `if (x != null)`, to smart cast x inside if -TypePtr calculate_type_subtract_null(TypePtr type) { - if (const auto* as_nullable = type->unwrap_alias()->try_as()) { - return as_nullable->inner; +// return `T`, so that `T + subtract_type` = type +// example: `int?` - `null` = `int` +// example: `int | slice | builder | bool` - `bool | slice` = `int | builder` +// what for: `if (x != null)` / `if (x is T)`, to smart cast x inside if +TypePtr calculate_type_subtract_rhs_type(TypePtr type, TypePtr subtract_type) { + const TypeDataUnion* lhs_union = type->try_as(); + if (!lhs_union) { + return TypeDataNever::create(); + } + + std::vector rest_variants; + + if (const TypeDataUnion* sub_union = subtract_type->try_as()) { + if (lhs_union->has_all_variants_of(sub_union)) { + rest_variants.reserve(lhs_union->variants.size() - sub_union->variants.size()); + for (TypePtr lhs_variant : lhs_union->variants) { + if (!sub_union->has_variant_with_type_id(lhs_variant)) { + rest_variants.push_back(lhs_variant); + } + } + } + } else if (lhs_union->has_variant_with_type_id(subtract_type)) { + rest_variants.reserve(lhs_union->variants.size() - 1); + for (TypePtr lhs_variant : lhs_union->variants) { + if (lhs_variant->get_type_id() != subtract_type->get_type_id()) { + rest_variants.push_back(lhs_variant); + } + } + } + + if (rest_variants.empty()) { + return TypeDataNever::create(); + } + if (rest_variants.size() == 1) { + return rest_variants[0]; } - // union types will be handled here - return TypeDataNever::create(); + return TypeDataUnion::create(std::move(rest_variants)); } // given any expression vertex, extract SinkExpression is possible @@ -407,6 +427,13 @@ SinkExpression extract_sink_expression_from_vertex(AnyExprV v) { return extract_sink_expression_from_vertex(as_assign->get_lhs()); } + if (auto as_decl = v->try_as()) { + if (auto decl_var = as_decl->get_expr()->try_as()) { + tolk_assert(decl_var->var_ref); + return SinkExpression(decl_var->var_ref); + } + } + return {}; } @@ -441,18 +468,32 @@ TypePtr calc_declared_type_before_smart_cast(AnyExprV v) { // obvious example: `var x: (int,int)? = null`, it's `null` (`x == null` is always true, `x` can be passed to any `T?`) // not obvious example: `var x: (int?, int?)? = (3,null)`, result is `(int?,int?)`, whereas type of rhs is `(int,null)` TypePtr calc_smart_cast_type_on_assignment(TypePtr lhs_declared_type, TypePtr rhs_inferred_type) { - // assign `T` to `T?` (or at least "assignable-to-T" to "T?") - // smart cast to `T` - if (const auto* lhs_nullable = lhs_declared_type->unwrap_alias()->try_as()) { - if (lhs_nullable->inner->can_rhs_be_assigned(rhs_inferred_type)) { - return lhs_nullable->inner; + if (const TypeDataUnion* lhs_union = lhs_declared_type->unwrap_alias()->try_as()) { + // example: `var x: T? = null`, result is null + // example: `var x: int | (int, User?) = (5, null)`, result is `(int, User?)` + if (TypePtr lhs_subtype = lhs_union->calculate_exact_variant_to_fit_rhs(rhs_inferred_type)) { + return lhs_subtype; + } + // example: `var x: int | slice | cell = 4`, result is int + // example: `var x: T1 | T2 | T3 = y as T3 | T1`, result is `T1 | T3` + if (const TypeDataUnion* rhs_union = rhs_inferred_type->try_as()) { + bool lhs_has_all_variants_of_rhs = true; + for (TypePtr rhs_variant : rhs_union->variants) { + lhs_has_all_variants_of_rhs &= lhs_union->has_variant_with_type_id(rhs_variant); + } + if (lhs_has_all_variants_of_rhs && rhs_union->variants.size() < lhs_union->variants.size()) { + std::vector subtypes_of_lhs; + for (TypePtr lhs_variant : lhs_union->variants) { + if (rhs_union->has_variant_with_type_id(lhs_variant)) { + subtypes_of_lhs.push_back(lhs_variant); + } + } + if (subtypes_of_lhs.size() == 1) { + return subtypes_of_lhs[0]; + } + return TypeDataUnion::create(std::move(subtypes_of_lhs)); + } } - } - - // assign `null` to `T?` - // smart cast to `null` - if (lhs_declared_type->unwrap_alias()->try_as() && rhs_inferred_type == TypeDataNullLiteral::create()) { - return TypeDataNullLiteral::create(); } // no smart cast, type is the same as declared diff --git a/tolk/smart-casts-cfg.h b/tolk/smart-casts-cfg.h index b97c8864c..31d4ebe0d 100644 --- a/tolk/smart-casts-cfg.h +++ b/tolk/smart-casts-cfg.h @@ -34,12 +34,13 @@ namespace tolk { */ class TypeInferringUnifyStrategy { TypePtr unified_result = nullptr; + bool different_types_became_union = false; public: - bool unify_with(TypePtr next); - bool unify_with_implicit_return_void(); + void unify_with(TypePtr next, TypePtr dest_hint = nullptr); TypePtr get_result() const { return unified_result; } + bool is_union_of_different_types() const { return different_types_became_union; } }; /* @@ -200,7 +201,7 @@ struct ExprFlow { std::ostream& operator<<(std::ostream& os, const FactsAboutExpr& facts); std::ostream& operator<<(std::ostream& os, const FlowContext& flow); -TypePtr calculate_type_subtract_null(TypePtr type); +TypePtr calculate_type_subtract_rhs_type(TypePtr type, TypePtr subtract_type); SinkExpression extract_sink_expression_from_vertex(AnyExprV v); TypePtr calc_declared_type_before_smart_cast(AnyExprV v); TypePtr calc_smart_cast_type_on_assignment(TypePtr lhs_declared_type, TypePtr rhs_inferred_type); diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index 459ad8136..a63d555c1 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -24,18 +24,19 @@ namespace tolk { /* - * This class stores a big hashtable [hash => TypeData] + * This class stores a big hashtable [hash => TypePtr] * Every non-trivial TypeData*::create() method at first looks here, and allocates an object only if not found. - * That's why all allocated TypeData objects are unique, storing unique type_id. + * That's why all allocated TypeData objects are unique, and can be compared as pointers. + * But compare pointers carefully due to type aliases. */ -class TypeDataTypeIdCalculation { +class TypeDataHasherForUnique { uint64_t cur_hash; int children_flags_mask = 0; static std::unordered_map all_unique_occurred_types; public: - explicit TypeDataTypeIdCalculation(uint64_t initial_arbitrary_unique_number) + explicit TypeDataHasherForUnique(uint64_t initial_arbitrary_unique_number) : cur_hash(initial_arbitrary_unique_number) {} void feed_hash(uint64_t val) { @@ -47,14 +48,10 @@ class TypeDataTypeIdCalculation { } void feed_child(TypePtr inner) { - feed_hash(inner->type_id); + feed_hash(reinterpret_cast(inner)); children_flags_mask |= inner->flags; } - uint64_t type_id() const { - return cur_hash; - } - int children_flags() const { return children_flags_mask; } @@ -68,14 +65,52 @@ class TypeDataTypeIdCalculation { GNU_ATTRIBUTE_NOINLINE TypePtr register_unique(TypePtr newly_created) const { #ifdef TOLK_DEBUG - assert(newly_created->type_id == cur_hash); + assert(all_unique_occurred_types.find(cur_hash) == all_unique_occurred_types.end()); #endif all_unique_occurred_types[cur_hash] = newly_created; return newly_created; } }; -std::unordered_map TypeDataTypeIdCalculation::all_unique_occurred_types; +/* + * This class stores a hashtable [TypePtr => type_id] + * We need type_id to support union types, that are stored as tagged unions on a stack. + * Every type that can be contained inside a union, has type_id. + * Some type_id are predefined (1 = int, etc.), but all user-defined types are assigned type_id. + */ +class TypeIdCalculation { + static int last_type_id; + static std::unordered_map map_ptr_to_type_id; + +public: + static int assign_type_id(TypePtr self) { + if (self->has_type_alias_inside()) { // type_id is calculated without aliases + self = unwrap_type_alias_deeply(self); // `(int,int)` equals `(IntAlias,IntAlias)`. + } + if (auto it = map_ptr_to_type_id.find(self); it != map_ptr_to_type_id.end()) { + return it->second; + } + + int type_id = ++last_type_id; + map_ptr_to_type_id[self] = type_id; + return type_id; + } + + static TypePtr unwrap_type_alias_deeply(TypePtr type) { + return type->replace_children_custom([](TypePtr child) { + if (const TypeDataAlias* as_alias = child->try_as()) { + return as_alias->underlying_type->unwrap_alias(); + } + return child; + }); + } +}; + + +int TypeIdCalculation::last_type_id = 128; // below 128 reserved for built-in types +std::unordered_map TypeDataHasherForUnique::all_unique_occurred_types; +std::unordered_map TypeIdCalculation::map_ptr_to_type_id; + TypePtr TypeDataInt::singleton; TypePtr TypeDataBool::singleton; TypePtr TypeDataCell::singleton; @@ -105,41 +140,61 @@ void type_system_init() { } +bool TypeData::equal_to_slow_path(TypePtr lhs, TypePtr rhs) { + if (lhs->has_type_alias_inside()) { + lhs = TypeIdCalculation::unwrap_type_alias_deeply(lhs); + } + if (rhs->has_type_alias_inside()) { + rhs = TypeIdCalculation::unwrap_type_alias_deeply(rhs); + } + if (lhs == rhs) { + return true; + } + + if (const TypeDataUnion* lhs_union = lhs->try_as()) { + if (const TypeDataUnion* rhs_union = rhs->try_as()) { + return lhs_union->variants.size() == rhs_union->variants.size() && lhs_union->has_all_variants_of(rhs_union); + } + } + return false; +} + +TypePtr TypeData::unwrap_alias_slow_path(TypePtr lhs) { + TypePtr unwrapped = lhs; + while (const TypeDataAlias* as_alias = unwrapped->try_as()) { + unwrapped = as_alias->underlying_type; + } + return unwrapped; +} + + // -------------------------------------------- // create() // // all constructors of TypeData classes are private, only TypeData*::create() is allowed -// each non-trivial create() method calculates hash (type_id) +// each non-trivial create() method calculates hash // and creates an object only if it isn't found in a global hashtable // -TypePtr TypeDataAlias::create(const std::string& alias_name, TypePtr underlying_type) { - TypeDataTypeIdCalculation hash(5694590762732189561ULL); - hash.feed_string(alias_name); - hash.feed_child(underlying_type); +TypePtr TypeDataAlias::create(AliasDefPtr alias_ref) { + TypeDataHasherForUnique hash(5694590762732189561ULL); + hash.feed_string(alias_ref->name); + hash.feed_child(alias_ref->underlying_type); if (TypePtr existing = hash.get_existing()) { return existing; } - std::string name_copy = alias_name; - return hash.register_unique(new TypeDataAlias(hash.type_id(), hash.children_flags(), std::move(name_copy), underlying_type)); -} - -TypePtr TypeDataNullable::create(TypePtr inner) { - TypeDataTypeIdCalculation hash(1774084920039440885ULL); - hash.feed_child(inner); - if (TypePtr existing = hash.get_existing()) { - return existing; + TypePtr underlying_type = alias_ref->underlying_type; + if (underlying_type == TypeDataNullLiteral::create() || underlying_type == TypeDataNever::create() || underlying_type == TypeDataVoid::create()) { + return underlying_type; // aliasing these types is strange, don't store an alias } - // most types (int?, slice?, etc.), when nullable, still occupy 1 stack slot (holding TVM NULL at runtime) - // but for example for `(int, int)` we need an extra stack slot "null flag" - int width_on_stack = inner->can_hold_tvm_null_instead() ? 1 : inner->get_width_on_stack() + 1; - return hash.register_unique(new TypeDataNullable(hash.type_id(), hash.children_flags(), width_on_stack, inner)); + + return hash.register_unique(new TypeDataAlias(hash.children_flags(), alias_ref, underlying_type)); } TypePtr TypeDataFunCallable::create(std::vector&& params_types, TypePtr return_type) { - TypeDataTypeIdCalculation hash(3184039965511020991ULL); + TypeDataHasherForUnique hash(3184039965511020991ULL); for (TypePtr param : params_types) { hash.feed_child(param); hash.feed_hash(767721); @@ -150,21 +205,21 @@ TypePtr TypeDataFunCallable::create(std::vector&& params_types, TypePtr if (TypePtr existing = hash.get_existing()) { return existing; } - return hash.register_unique(new TypeDataFunCallable(hash.type_id(), hash.children_flags(), std::move(params_types), return_type)); + return hash.register_unique(new TypeDataFunCallable(hash.children_flags(), std::move(params_types), return_type)); } TypePtr TypeDataGenericT::create(std::string&& nameT) { - TypeDataTypeIdCalculation hash(9145033724911680012ULL); + TypeDataHasherForUnique hash(9145033724911680012ULL); hash.feed_string(nameT); if (TypePtr existing = hash.get_existing()) { return existing; } - return hash.register_unique(new TypeDataGenericT(hash.type_id(), std::move(nameT))); + return hash.register_unique(new TypeDataGenericT(std::move(nameT))); } TypePtr TypeDataTensor::create(std::vector&& items) { - TypeDataTypeIdCalculation hash(3159238551239480381ULL); + TypeDataHasherForUnique hash(3159238551239480381ULL); for (TypePtr item : items) { hash.feed_child(item); hash.feed_hash(819613); @@ -177,11 +232,11 @@ TypePtr TypeDataTensor::create(std::vector&& items) { for (TypePtr item : items) { width_on_stack += item->get_width_on_stack(); } - return hash.register_unique(new TypeDataTensor(hash.type_id(), hash.children_flags(), width_on_stack, std::move(items))); + return hash.register_unique(new TypeDataTensor(hash.children_flags(), width_on_stack, std::move(items))); } TypePtr TypeDataTypedTuple::create(std::vector&& items) { - TypeDataTypeIdCalculation hash(9189266157349499320ULL); + TypeDataHasherForUnique hash(9189266157349499320ULL); for (TypePtr item : items) { hash.feed_child(item); hash.feed_hash(735911); @@ -190,11 +245,11 @@ TypePtr TypeDataTypedTuple::create(std::vector&& items) { if (TypePtr existing = hash.get_existing()) { return existing; } - return hash.register_unique(new TypeDataTypedTuple(hash.type_id(), hash.children_flags(), std::move(items))); + return hash.register_unique(new TypeDataTypedTuple(hash.children_flags(), std::move(items))); } TypePtr TypeDataIntN::create(bool is_unsigned, bool is_variadic, int n_bits) { - TypeDataTypeIdCalculation hash(1678330938771108027ULL); + TypeDataHasherForUnique hash(1678330938771108027ULL); hash.feed_hash(is_unsigned); hash.feed_hash(is_variadic); hash.feed_hash(n_bits); @@ -202,29 +257,173 @@ TypePtr TypeDataIntN::create(bool is_unsigned, bool is_variadic, int n_bits) { if (TypePtr existing = hash.get_existing()) { return existing; } - return hash.register_unique(new TypeDataIntN(hash.type_id(), is_unsigned, is_variadic, n_bits)); + return hash.register_unique(new TypeDataIntN(is_unsigned, is_variadic, n_bits)); } TypePtr TypeDataBytesN::create(bool is_bits, int n_width) { - TypeDataTypeIdCalculation hash(7810988137199333041ULL); + TypeDataHasherForUnique hash(7810988137199333041ULL); hash.feed_hash(is_bits); hash.feed_hash(n_width); if (TypePtr existing = hash.get_existing()) { return existing; } - return hash.register_unique(new TypeDataBytesN(hash.type_id(), is_bits, n_width)); + return hash.register_unique(new TypeDataBytesN(is_bits, n_width)); +} + +TypePtr TypeDataUnion::create(std::vector&& variants) { + TypeDataHasherForUnique hash(8719233194368471403ULL); + for (TypePtr variant : variants) { + hash.feed_child(variant); + hash.feed_hash(817663); + } + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + + // at the moment of parsing, union type can contain unresolved symbols + // in this case, don't try to flatten: we have no info + // after symbols resolving, a new union type (with resolved variants) will be created + bool not_ready_yet = false; + for (TypePtr variant : variants) { + not_ready_yet |= variant->has_unresolved_inside() || variant->has_genericT_inside(); + } + if (not_ready_yet) { + TypePtr or_null = nullptr; + if (variants.size() == 2) { + if (variants[0] == TypeDataNullLiteral::create() || variants[1] == TypeDataNullLiteral::create()) { + or_null = variants[variants[0] == TypeDataNullLiteral::create()]; + } + } + return hash.register_unique(new TypeDataUnion(hash.children_flags(), -999999, or_null, std::move(variants))); + } + + // flatten variants and remove duplicates + // note, that `int | slice` and `int | int | slice` are different TypePtr, but actually the same variants + std::vector flat_variants; + flat_variants.reserve(variants.size()); + for (TypePtr variant : variants) { + if (const TypeDataUnion* nested_union = variant->unwrap_alias()->try_as()) { + for (TypePtr nested_variant : nested_union->variants) { + append_union_type_variant(nested_variant, flat_variants); + } + } else { + append_union_type_variant(variant, flat_variants); + } + } + // detect, whether it's `T?` or `T1 | T2 | ...` + TypePtr or_null = nullptr; + if (flat_variants.size() == 2) { + if (flat_variants[0] == TypeDataNullLiteral::create() || flat_variants[1] == TypeDataNullLiteral::create()) { + or_null = flat_variants[flat_variants[0] == TypeDataNullLiteral::create()]; + } + } + + int width_on_stack; + if (or_null && or_null->can_hold_tvm_null_instead()) { + width_on_stack = 1; + } else { + // `T1 | T2 | ...` occupy max(W[i]) + 1 slot for UTag (stores type_id or 0 for null) + int max_child_width = 0; + for (TypePtr i : flat_variants) { + if (i != TypeDataNullLiteral::create()) { // `Empty | () | null` totally should be 1 (0 + 1 for UTag) + max_child_width = std::max(max_child_width, i->get_width_on_stack()); + } + } + width_on_stack = max_child_width + 1; + } + + if (flat_variants.size() == 1) { // `int | int` + return flat_variants[0]; + } + return hash.register_unique(new TypeDataUnion(hash.children_flags(), width_on_stack, or_null, std::move(flat_variants))); +} + +TypePtr TypeDataUnion::create_nullable(TypePtr nullable) { + // calculate exactly the same hash as for `T | null` to create std::vector only if type seen the first time + TypeDataHasherForUnique hash(8719233194368471403ULL); + hash.feed_child(nullable); + hash.feed_hash(817663); + hash.feed_child(TypeDataNullLiteral::create()); + hash.feed_hash(817663); + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + return create({nullable, TypeDataNullLiteral::create()}); } TypePtr TypeDataUnresolved::create(std::string&& text, SrcLocation loc) { - TypeDataTypeIdCalculation hash(3680147223540048162ULL); + TypeDataHasherForUnique hash(3680147223540048162ULL); hash.feed_string(text); // hash.feed_hash(*reinterpret_cast(&loc)); if (TypePtr existing = hash.get_existing()) { return existing; } - return hash.register_unique(new TypeDataUnresolved(hash.type_id(), std::move(text), loc)); + return hash.register_unique(new TypeDataUnresolved(std::move(text), loc)); +} + + +// -------------------------------------------- +// get_type_id() +// +// in order to support union types, every type that can be stored inside a union has a unique type_id +// some are predefined (1 = int, etc. in .h file), the others are here +// + +int TypeDataAlias::get_type_id() const { + return underlying_type->get_type_id(); +} + +int TypeDataFunCallable::get_type_id() const { + return TypeIdCalculation::assign_type_id(this); +} + +int TypeDataGenericT::get_type_id() const { + return TypeIdCalculation::assign_type_id(this); +} + +int TypeDataTensor::get_type_id() const { + assert(!has_genericT_inside()); + return TypeIdCalculation::assign_type_id(this); +} + +int TypeDataTypedTuple::get_type_id() const { + assert(!has_genericT_inside()); + return TypeIdCalculation::assign_type_id(this); +} + +int TypeDataIntN::get_type_id() const { + switch (n_bits) { + case 8: return 42 + is_unsigned; // for common intN, use predefined small numbers + case 16: return 44 + is_unsigned; + case 32: return 46 + is_unsigned; + case 64: return 48 + is_unsigned; + case 128: return 50 + is_unsigned; + case 256: return 52 + is_unsigned; + default: return TypeIdCalculation::assign_type_id(this); + } +} + +int TypeDataBytesN::get_type_id() const { + return TypeIdCalculation::assign_type_id(this); +} + +int TypeDataUnion::get_type_id() const { + assert(false); // a union can not be inside a union + throw Fatal("unexpected get_type_id() call"); +} + +int TypeDataUnknown::get_type_id() const { + assert(false); // unknown can not be inside a union + throw Fatal("unexpected get_type_id() call"); +} + +int TypeDataUnresolved::get_type_id() const { + assert(false); // unresolved can be inside a union at parsing, but is resolved is advance + throw Fatal("unexpected get_type_id() call"); } @@ -235,10 +434,8 @@ TypePtr TypeDataUnresolved::create(std::string&& text, SrcLocation loc) { // only non-trivial implementations are here; trivial are defined in .h file // -std::string TypeDataNullable::as_human_readable() const { - std::string nested = inner->as_human_readable(); - bool embrace = inner->try_as(); - return embrace ? "(" + nested + ")?" : nested + "?"; +std::string TypeDataAlias::as_human_readable() const { + return alias_ref->name; } std::string TypeDataFunCallable::as_human_readable() const { @@ -290,6 +487,30 @@ std::string TypeDataBytesN::as_human_readable() const { return s_bytes + std::to_string(n_width); } +std::string TypeDataUnion::as_human_readable() const { + // stringify `T?`, not `T | null` + if (or_null) { + bool embrace = or_null->try_as(); + return embrace ? "(" + or_null->as_human_readable() + ")?" : or_null->as_human_readable() + "?"; + } + + std::string result; + for (TypePtr variant : variants) { + if (!result.empty()) { + result += " | "; + } + bool embrace = variant->try_as(); + if (embrace) { + result += "("; + } + result += variant->as_human_readable(); + if (embrace) { + result += ")"; + } + } + return result; +} + // -------------------------------------------- // traverse() @@ -303,11 +524,6 @@ void TypeDataAlias::traverse(const TraverserCallbackT& callback) const { underlying_type->traverse(callback); } -void TypeDataNullable::traverse(const TraverserCallbackT& callback) const { - callback(this); - inner->traverse(callback); -} - void TypeDataFunCallable::traverse(const TraverserCallbackT& callback) const { callback(this); for (TypePtr param : params_types) { @@ -330,6 +546,13 @@ void TypeDataTypedTuple::traverse(const TraverserCallbackT& callback) const { } } +void TypeDataUnion::traverse(const TraverserCallbackT& callback) const { + callback(this); + for (TypePtr variant : variants) { + variant->traverse(callback); + } +} + // -------------------------------------------- // replace_children_custom() @@ -339,14 +562,6 @@ void TypeDataTypedTuple::traverse(const TraverserCallbackT& callback) const { // only non-trivial implementations are here; by default (no children), `return callback(this)` is executed // -TypePtr TypeDataAlias::replace_children_custom(const ReplacerCallbackT& callback) const { - return callback(create(alias_name, underlying_type->replace_children_custom(callback))); -} - -TypePtr TypeDataNullable::replace_children_custom(const ReplacerCallbackT& callback) const { - return callback(create(inner->replace_children_custom(callback))); -} - TypePtr TypeDataFunCallable::replace_children_custom(const ReplacerCallbackT& callback) const { std::vector mapped; mapped.reserve(params_types.size()); @@ -374,12 +589,23 @@ TypePtr TypeDataTypedTuple::replace_children_custom(const ReplacerCallbackT& cal return callback(create(std::move(mapped))); } +TypePtr TypeDataUnion::replace_children_custom(const ReplacerCallbackT& callback) const { + std::vector mapped; + mapped.reserve(variants.size()); + for (TypePtr variant : variants) { + mapped.push_back(variant->replace_children_custom(callback)); + } + return callback(create(std::move(mapped))); +} + // -------------------------------------------- // can_rhs_be_assigned() // // on `var lhs: = rhs`, having inferred rhs_type, check that it can be assigned without any casts // the same goes for passing arguments, returning values, etc. — where the "receiver" (lhs) checks "applier" (rhs) +// note, that `int8 | int16` is not assignable to `int` (even though both are assignable), +// because the only way to work with union types is to use `match`/`is` operators // bool TypeDataAlias::can_rhs_be_assigned(TypePtr rhs) const { @@ -475,25 +701,6 @@ bool TypeDataNullLiteral::can_rhs_be_assigned(TypePtr rhs) const { return rhs == TypeDataNever::create(); } -bool TypeDataNullable::can_rhs_be_assigned(TypePtr rhs) const { - if (rhs == this) { - return true; - } - if (rhs == TypeDataNullLiteral::create()) { - return true; - } - if (const TypeDataNullable* rhs_nullable = rhs->try_as()) { - return inner->can_rhs_be_assigned(rhs_nullable->inner); - } - if (inner->can_rhs_be_assigned(rhs)) { - return true; - } - if (const TypeDataAlias* rhs_alias = rhs->try_as()) { - return can_rhs_be_assigned(rhs_alias->underlying_type); - } - return rhs == TypeDataNever::create(); -} - bool TypeDataFunCallable::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; @@ -510,7 +717,8 @@ bool TypeDataFunCallable::can_rhs_be_assigned(TypePtr rhs) const { return false; } } - return return_type->can_rhs_be_assigned(rhs_callable->return_type) && rhs_callable->return_type->can_rhs_be_assigned(return_type); + return return_type->can_rhs_be_assigned(rhs_callable->return_type) && + rhs_callable->return_type->can_rhs_be_assigned(return_type); } if (const TypeDataAlias* rhs_alias = rhs->try_as()) { return can_rhs_be_assigned(rhs_alias->underlying_type); @@ -591,6 +799,23 @@ bool TypeDataCoins::can_rhs_be_assigned(TypePtr rhs) const { return rhs == TypeDataNever::create(); } +bool TypeDataUnion::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + if (calculate_exact_variant_to_fit_rhs(rhs)) { // `int` to `int | slice`, `int?` to `int8?`, `(int, null)` to `(int, T?) | slice` + return true; + } + if (const TypeDataUnion* rhs_union = rhs->try_as()) { + return has_all_variants_of(rhs_union); + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } + + return rhs == TypeDataNever::create(); +} + bool TypeDataUnknown::can_rhs_be_assigned(TypePtr rhs) const { return true; } @@ -608,9 +833,6 @@ bool TypeDataVoid::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } - if (const TypeDataAlias* rhs_alias = rhs->try_as()) { - return can_rhs_be_assigned(rhs_alias->underlying_type); - } return rhs == TypeDataNever::create(); } @@ -622,13 +844,27 @@ bool TypeDataVoid::can_rhs_be_assigned(TypePtr rhs) const { // note, that it's not auto-casts `var lhs: = rhs`, it's an expression `rhs as ` // +// common helper for union types: +// - `int as int?` is ok +// - `int8 as int16?` is ok (primitive 1-slot nullable don't store UTag, rules are less strict) +// - `int as int | int16` is ok (exact match one of types) +// - `int as slice | null` is NOT ok (no rhs subtype fits) +// - `int as int8 | int16` is NOT ok (ambiguity) +static bool can_be_casted_to_union(TypePtr self, const TypeDataUnion* rhs_union) { + if (rhs_union->is_primitive_nullable()) { // casting to primitive 1-slot nullable + return self == TypeDataNullLiteral::create() || self->can_be_casted_with_as_operator(rhs_union->or_null); + } + + return rhs_union->calculate_exact_variant_to_fit_rhs(self) != nullptr; +} + bool TypeDataAlias::can_be_casted_with_as_operator(TypePtr cast_to) const { return underlying_type->can_be_casted_with_as_operator(cast_to); } bool TypeDataInt::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_nullable = cast_to->try_as()) { // `int` as `int?` - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { // `int` as `int?` / `int` as `int | slice` + return can_be_casted_to_union(this, to_union); } if (cast_to->try_as()) { // `int` as `int8` / `int` as `uint2` return true; @@ -646,8 +882,8 @@ bool TypeDataBool::can_be_casted_with_as_operator(TypePtr cast_to) const { if (cast_to == TypeDataInt::create()) { return true; } - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); } if (const auto* to_intN = cast_to->try_as()) { return !to_intN->is_unsigned; // `bool` as `int8` ok, `bool` as `uintN` not (true is -1) @@ -659,8 +895,8 @@ bool TypeDataBool::can_be_casted_with_as_operator(TypePtr cast_to) const { } bool TypeDataCell::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); @@ -672,8 +908,8 @@ bool TypeDataSlice::can_be_casted_with_as_operator(TypePtr cast_to) const { if (cast_to->try_as()) { // `slice` to `bytes32` / `slice` to `bits8` return true; } - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); @@ -682,8 +918,8 @@ bool TypeDataSlice::can_be_casted_with_as_operator(TypePtr cast_to) const { } bool TypeDataBuilder::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); @@ -692,8 +928,8 @@ bool TypeDataBuilder::can_be_casted_with_as_operator(TypePtr cast_to) const { } bool TypeDataTuple::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); @@ -702,8 +938,8 @@ bool TypeDataTuple::can_be_casted_with_as_operator(TypePtr cast_to) const { } bool TypeDataContinuation::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); @@ -712,8 +948,8 @@ bool TypeDataContinuation::can_be_casted_with_as_operator(TypePtr cast_to) const } bool TypeDataNullLiteral::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (cast_to->try_as()) { - return true; + if (const TypeDataUnion* to_union = cast_to->try_as()) { // `null` to `T?` / `null` to `... | null` + return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); @@ -721,19 +957,9 @@ bool TypeDataNullLiteral::can_be_casted_with_as_operator(TypePtr cast_to) const return cast_to == this; } -bool TypeDataNullable::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_nullable = cast_to->try_as()) { - return inner->can_be_casted_with_as_operator(to_nullable->inner); - } - if (const TypeDataAlias* to_alias = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_alias->underlying_type); - } - return false; -} - bool TypeDataFunCallable::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); @@ -746,8 +972,12 @@ bool TypeDataFunCallable::can_be_casted_with_as_operator(TypePtr cast_to) const if (!params_types[i]->can_be_casted_with_as_operator(to_callable->params_types[i])) { return false; } + if (!to_callable->params_types[i]->can_be_casted_with_as_operator(params_types[i])) { + return false; + } } - return return_type->can_be_casted_with_as_operator(to_callable->return_type); + return return_type->can_be_casted_with_as_operator(to_callable->return_type) && + to_callable->return_type->can_be_casted_with_as_operator(return_type); } return false; } @@ -765,8 +995,8 @@ bool TypeDataTensor::can_be_casted_with_as_operator(TypePtr cast_to) const { } return true; } - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); @@ -783,8 +1013,8 @@ bool TypeDataTypedTuple::can_be_casted_with_as_operator(TypePtr cast_to) const { } return true; } - if (const auto* to_nullable = cast_to->try_as()) { - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); @@ -796,8 +1026,8 @@ bool TypeDataIntN::can_be_casted_with_as_operator(TypePtr cast_to) const { if (cast_to->try_as()) { // `int8` as `int32`, `int256` as `uint5`, anything return true; } - if (const auto* to_nullable = cast_to->try_as()) { // `int8` as `int32?` - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { // `int8` as `int32?` + return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); @@ -809,8 +1039,8 @@ bool TypeDataBytesN::can_be_casted_with_as_operator(TypePtr cast_to) const { if (cast_to->try_as()) { // `bytes256` as `bytes512`, `bits1` as `bytes8` return true; } - if (const auto* to_nullable = cast_to->try_as()) { // `bytes8` as `slice?` - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { // `bytes8` as `slice?` + return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); @@ -822,8 +1052,8 @@ bool TypeDataCoins::can_be_casted_with_as_operator(TypePtr cast_to) const { if (cast_to->try_as()) { // `coins` as `int8` return true; } - if (const auto* to_nullable = cast_to->try_as()) { // `coins` as `coins?` / `coins` as `int?` - return can_be_casted_with_as_operator(to_nullable->inner); + if (const TypeDataUnion* to_union = cast_to->try_as()) { // `coins` as `coins?` / `coins` as `int?` + return can_be_casted_to_union(this, to_union); } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); @@ -834,6 +1064,19 @@ bool TypeDataCoins::can_be_casted_with_as_operator(TypePtr cast_to) const { return cast_to == this; } +bool TypeDataUnion::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const TypeDataUnion* to_union = cast_to->try_as()) { // `int8 | int16` as `int16 | int8 | slice` + if (to_union->is_primitive_nullable()) { + return or_null && or_null->can_be_casted_with_as_operator(to_union->or_null); + } + return to_union->has_all_variants_of(this); + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } + return false; +} + bool TypeDataUnknown::can_be_casted_with_as_operator(TypePtr cast_to) const { // 'unknown' can be cast to any TVM value return cast_to->get_width_on_stack() == 1; @@ -865,13 +1108,6 @@ bool TypeDataAlias::can_hold_tvm_null_instead() const { return underlying_type->can_hold_tvm_null_instead(); } -bool TypeDataNullable::can_hold_tvm_null_instead() const { - if (get_width_on_stack() != 1) { // `(int, int)?` / `()?` can not hold null instead - return false; // only `int?` / `cell?` / `StructWith1IntField?` can - } // and some tricky situations like `(int, ())?`, but not `(int?, ())?` - return !inner->can_hold_tvm_null_instead(); -} - bool TypeDataTensor::can_hold_tvm_null_instead() const { if (get_width_on_stack() != 1) { // `(int, int)` / `()` can not hold null instead, since null is 1 slot return false; // only `((), int)` and similar can: @@ -884,6 +1120,13 @@ bool TypeDataTensor::can_hold_tvm_null_instead() const { return true; } +bool TypeDataUnion::can_hold_tvm_null_instead() const { + if (get_width_on_stack() != 1) { // `(int, int)?` / `()?` can not hold null instead + return false; // only `int?` / `cell?` / `StructWith1IntField?` can + } // and some tricky situations like `(int, ())?`, but not `(int?, ())?` + return or_null && !or_null->can_hold_tvm_null_instead(); +} + bool TypeDataNever::can_hold_tvm_null_instead() const { return false; } @@ -893,12 +1136,73 @@ bool TypeDataVoid::can_hold_tvm_null_instead() const { } +// union types creation is a bit tricky: nested unions are flattened, duplicates are removed +// so, a resolved union type has variants, each with unique type_id +// (type_id is calculated with aliases erasure) +void TypeDataUnion::append_union_type_variant(TypePtr variant, std::vector& out_unique_variants) { + for (TypePtr existing : out_unique_variants) { + if (existing->get_type_id() == variant->get_type_id()) { + return; + } + } + + out_unique_variants.push_back(variant); +} + +bool TypeDataUnion::has_variant_with_type_id(int type_id) const { + for (TypePtr self_variant : variants) { + if (self_variant->get_type_id() == type_id) { + return true; + } + } + return false; +} + +bool TypeDataUnion::has_all_variants_of(const TypeDataUnion* rhs_type) const { + for (TypePtr rhs_variant : rhs_type->variants) { + if (!has_variant_with_type_id(rhs_variant->get_type_id())) { + return false; + } + } + return true; +} + +// given this = `T1 | T2 | ...` and rhs_type, find the only (not ambiguous) T_i that can accept it +TypePtr TypeDataUnion::calculate_exact_variant_to_fit_rhs(TypePtr rhs_type) const { + // primitive 1-slot nullable don't store type_id, they can be assigned less strict, like `int?` to `int16?` + if (const TypeDataUnion* rhs_union = rhs_type->unwrap_alias()->try_as()) { + if (is_primitive_nullable() && rhs_union->is_primitive_nullable() && or_null->can_rhs_be_assigned(rhs_union->or_null)) { + return this; + } + return nullptr; + } + // `int` to `int | int8` is okay: exact type matching + for (TypePtr variant : variants) { + if (variant->get_type_id() == rhs_type->get_type_id()) { + return variant; + } + } + + // find the only T_i; it would also be used for transition at IR generation, like `(int,null)` to `(int, User?) | int` + TypePtr first_covering = nullptr; + for (TypePtr variant : variants) { + if (variant->can_rhs_be_assigned(rhs_type)) { + if (first_covering) { + return nullptr; + } + first_covering = variant; + } + } + return first_covering; +} + + // -------------------------------------------- // parsing type from tokens // // here we implement parsing types (mostly after colon) to TypeData // example: `var v: int` is TypeDataInt -// example: `var v: (builder?, [cell])` is TypeDataTensor(TypeDataNullable(TypeDataBuilder), TypeDataTypedTuple(TypeDataCell)) +// example: `var v: (builder?, [cell])` is TypeDataTensor(TypeDataUnion(TypeDataBuilder,TypeDataNullLiteral), TypeDataTypedTuple(TypeDataCell)) // example: `fun f(): ()` is TypeDataTensor() (an empty one) // // note, that unrecognized type names (MyEnum, MyStruct, T) are parsed as TypeDataUnresolved, @@ -1036,7 +1340,7 @@ static TypePtr parse_type_nullable(Lexer& lex) { if (lex.tok() == tok_question) { lex.next(); - result = TypeDataNullable::create(result); + result = TypeDataUnion::create_nullable(result); } return result; @@ -1045,21 +1349,27 @@ static TypePtr parse_type_nullable(Lexer& lex) { static TypePtr parse_type_expression(Lexer& lex) { TypePtr result = parse_type_nullable(lex); - if (lex.tok() == tok_arrow) { // `int -> int`, `(cell, slice) -> void` + if (lex.tok() == tok_bitwise_or) { // `int | slice`, `Pair2 | (Pair3 | null)` + std::vector items; + items.emplace_back(result); + while (lex.tok() == tok_bitwise_or) { + lex.next(); + items.emplace_back(parse_type_nullable(lex)); + } + result = TypeDataUnion::create(std::move(items)); + } + + if (lex.tok() == tok_arrow) { // `int -> int`, `(cell, slice) -> void`, `int -> int -> int`, `int | cell -> void` lex.next(); TypePtr return_type = parse_type_expression(lex); std::vector params_types = {result}; if (const auto* as_tensor = result->try_as()) { params_types = as_tensor->items; } - return TypeDataFunCallable::create(std::move(params_types), return_type); + result = TypeDataFunCallable::create(std::move(params_types), return_type); } - if (lex.tok() != tok_bitwise_or) { - return result; - } - - lex.error("union types are not supported yet"); + return result; } TypePtr parse_type_from_tokens(Lexer& lex) { diff --git a/tolk/type-system.h b/tolk/type-system.h index ebb25312e..56fd02e5c 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -27,7 +27,7 @@ namespace tolk { * TypeData is both a user-given and an inferred type representation. * `int`, `cell`, `T`, `(int, [tuple])` are instances of TypeData. * Every unique TypeData is created only once, so for example TypeDataTensor::create(int, int) - * returns one and the same pointer always. This "uniqueness" is called type_id, calculated before creation. + * returns one and the same pointer always. * * In Tolk code, types after colon `var v: (int, T)` are parsed to TypeData. * See parse_type_from_tokens(). @@ -46,28 +46,29 @@ namespace tolk { * For instance, `fun f(v: T)`, at first "T" of `v` is unresolved, and then converted to TypeDataGenericT. */ class TypeData { - // all unique types have unique type_id; it's used both for allocating memory once and for tagged unions - const uint64_t type_id; // bits of flag_mask, to store often-used properties and return them without tree traversing const int flags; // how many slots on a stack this type occupies (calculated on creation), e.g. `int`=1, `(int,int)`=2, `(int,int)?`=3 const int width_on_stack; - friend class TypeDataTypeIdCalculation; + friend class TypeDataHasherForUnique; protected: enum flag_mask { flag_contains_unknown_inside = 1 << 1, flag_contains_genericT_inside = 1 << 2, flag_contains_unresolved_inside = 1 << 3, + flag_contains_type_alias_inside = 1 << 4, }; - explicit TypeData(uint64_t type_id, int flags_with_children, int width_on_stack) - : type_id(type_id) - , flags(flags_with_children) + explicit TypeData(int flags_with_children, int width_on_stack) + : flags(flags_with_children) , width_on_stack(width_on_stack) { } + static bool equal_to_slow_path(TypePtr lhs, TypePtr rhs); + static TypePtr unwrap_alias_slow_path(TypePtr lhs); + public: virtual ~TypeData() = default; @@ -76,18 +77,24 @@ class TypeData { return dynamic_cast(this); } - uint64_t get_type_id() const { return type_id; } int get_width_on_stack() const { return width_on_stack; } + bool equal_to(TypePtr rhs) const { + return this == rhs || equal_to_slow_path(this, rhs); + } + TypePtr unwrap_alias() const { + return has_type_alias_inside() ? unwrap_alias_slow_path(this) : this; + } + bool has_unknown_inside() const { return flags & flag_contains_unknown_inside; } bool has_genericT_inside() const { return flags & flag_contains_genericT_inside; } bool has_unresolved_inside() const { return flags & flag_contains_unresolved_inside; } - - inline TypePtr unwrap_alias() const; + bool has_type_alias_inside() const { return flags & flag_contains_type_alias_inside; } using TraverserCallbackT = std::function; using ReplacerCallbackT = std::function; + virtual int get_type_id() const = 0; virtual std::string as_human_readable() const = 0; virtual bool can_rhs_be_assigned(TypePtr rhs) const = 0; virtual bool can_be_casted_with_as_operator(TypePtr cast_to) const = 0; @@ -112,22 +119,22 @@ class TypeData { * That's why lots of code comparing types use `type->unwrap_alias()` or `try_as`. */ class TypeDataAlias final : public TypeData { - explicit TypeDataAlias(uint64_t type_id, int children_flags, std::string&& alias_name, TypePtr underlying_type) - : TypeData(type_id, children_flags, underlying_type->get_width_on_stack()) - , alias_name(std::move(alias_name)) + explicit TypeDataAlias(int children_flags, AliasDefPtr alias_ref, TypePtr underlying_type) + : TypeData(children_flags | flag_contains_type_alias_inside, underlying_type->get_width_on_stack()) + , alias_ref(alias_ref) , underlying_type(underlying_type) {} public: - const std::string alias_name; + const AliasDefPtr alias_ref; const TypePtr underlying_type; - static TypePtr create(const std::string& alias_name, TypePtr underlying_type); + static TypePtr create(AliasDefPtr alias_ref); - std::string as_human_readable() const override { return alias_name; } + int get_type_id() const override; + std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; void traverse(const TraverserCallbackT& callback) const override; - TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; bool can_hold_tvm_null_instead() const override; }; @@ -135,7 +142,7 @@ class TypeDataAlias final : public TypeData { * `int` is TypeDataInt, representation of TVM int. */ class TypeDataInt final : public TypeData { - TypeDataInt() : TypeData(1ULL, 0, 1) {} + TypeDataInt() : TypeData(0, 1) {} static TypePtr singleton; friend void type_system_init(); @@ -143,6 +150,7 @@ class TypeDataInt final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 1; } std::string as_human_readable() const override { return "int"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -153,7 +161,7 @@ class TypeDataInt final : public TypeData { * From the type system point of view, int and bool are different, not-autocastable types. */ class TypeDataBool final : public TypeData { - TypeDataBool() : TypeData(2ULL, 0, 1) {} + TypeDataBool() : TypeData(0, 1) {} static TypePtr singleton; friend void type_system_init(); @@ -161,6 +169,7 @@ class TypeDataBool final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 2; } std::string as_human_readable() const override { return "bool"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -170,7 +179,7 @@ class TypeDataBool final : public TypeData { * `cell` is TypeDataCell, representation of TVM cell. */ class TypeDataCell final : public TypeData { - TypeDataCell() : TypeData(3ULL, 0, 1) {} + TypeDataCell() : TypeData(0, 1) {} static TypePtr singleton; friend void type_system_init(); @@ -178,6 +187,7 @@ class TypeDataCell final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 3; } std::string as_human_readable() const override { return "cell"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -187,7 +197,7 @@ class TypeDataCell final : public TypeData { * `slice` is TypeDataSlice, representation of TVM slice. */ class TypeDataSlice final : public TypeData { - TypeDataSlice() : TypeData(4ULL, 0, 1) {} + TypeDataSlice() : TypeData(0, 1) {} static TypePtr singleton; friend void type_system_init(); @@ -195,6 +205,7 @@ class TypeDataSlice final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 4; } std::string as_human_readable() const override { return "slice"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -204,7 +215,7 @@ class TypeDataSlice final : public TypeData { * `builder` is TypeDataBuilder, representation of TVM builder. */ class TypeDataBuilder final : public TypeData { - TypeDataBuilder() : TypeData(5ULL, 0, 1) {} + TypeDataBuilder() : TypeData(0, 1) {} static TypePtr singleton; friend void type_system_init(); @@ -212,6 +223,7 @@ class TypeDataBuilder final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 5; } std::string as_human_readable() const override { return "builder"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -223,7 +235,7 @@ class TypeDataBuilder final : public TypeData { * so getting its element results in TypeDataUnknown (which must be assigned/cast explicitly). */ class TypeDataTuple final : public TypeData { - TypeDataTuple() : TypeData(6ULL, 0, 1) {} + TypeDataTuple() : TypeData(0, 1) {} static TypePtr singleton; friend void type_system_init(); @@ -231,6 +243,7 @@ class TypeDataTuple final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 6; } std::string as_human_readable() const override { return "tuple"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -241,7 +254,7 @@ class TypeDataTuple final : public TypeData { * It's like "untyped callable", not compatible with other types. */ class TypeDataContinuation final : public TypeData { - TypeDataContinuation() : TypeData(7ULL, 0, 1) {} + TypeDataContinuation() : TypeData(0, 1) {} static TypePtr singleton; friend void type_system_init(); @@ -249,6 +262,7 @@ class TypeDataContinuation final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 7; } std::string as_human_readable() const override { return "continuation"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -261,7 +275,7 @@ class TypeDataContinuation final : public TypeData { * (it's much better for user to see an error here than when he passes this variable somewhere). */ class TypeDataNullLiteral final : public TypeData { - TypeDataNullLiteral() : TypeData(8ULL, 0, 1) {} + TypeDataNullLiteral() : TypeData(0, 1) {} static TypePtr singleton; friend void type_system_init(); @@ -269,43 +283,20 @@ class TypeDataNullLiteral final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 0; } std::string as_human_readable() const override { return "null"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; }; -/* - * `T?` is "nullable T". - * It can be converted to T either with ! (non-null assertion operator) or with smart casts. - */ -class TypeDataNullable final : public TypeData { - TypeDataNullable(uint64_t type_id, int children_flags, int width_on_stack, TypePtr inner) - : TypeData(type_id, children_flags, width_on_stack) - , inner(inner) {} - -public: - const TypePtr inner; - - static TypePtr create(TypePtr inner); - - bool is_primitive_nullable() const { return get_width_on_stack() == 1 && inner->get_width_on_stack() == 1; } - - std::string as_human_readable() const override; - bool can_rhs_be_assigned(TypePtr rhs) const override; - bool can_be_casted_with_as_operator(TypePtr cast_to) const override; - void traverse(const TraverserCallbackT& callback) const override; - TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; - bool can_hold_tvm_null_instead() const override; -}; - /* * `fun(int, int) -> void` is TypeDataFunCallable, think of is as a typed continuation. * A type of function `fun f(x: int) { return x; }` is actually `fun(int) -> int`. * So, when assigning it to a variable `var cb = f`, this variable also has this type. */ class TypeDataFunCallable final : public TypeData { - TypeDataFunCallable(uint64_t type_id, int children_flags, std::vector&& params_types, TypePtr return_type) - : TypeData(type_id, children_flags, 1) + TypeDataFunCallable(int children_flags, std::vector&& params_types, TypePtr return_type) + : TypeData(children_flags, 1) , params_types(std::move(params_types)) , return_type(return_type) {} @@ -317,6 +308,7 @@ class TypeDataFunCallable final : public TypeData { int params_size() const { return static_cast(params_types.size()); } + int get_type_id() const override; std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -330,8 +322,8 @@ class TypeDataFunCallable final : public TypeData { * On instantiation like `f(1,"")`, a new function `f` is created with type `fun(int,slice)->[int,slice]`. */ class TypeDataGenericT final : public TypeData { - TypeDataGenericT(uint64_t type_id, std::string&& nameT) - : TypeData(type_id, flag_contains_genericT_inside, -999999) // width undefined until instantiated + explicit TypeDataGenericT(std::string&& nameT) + : TypeData(flag_contains_genericT_inside, -999999) // width undefined until instantiated , nameT(std::move(nameT)) {} public: @@ -339,6 +331,7 @@ class TypeDataGenericT final : public TypeData { static TypePtr create(std::string&& nameT); + int get_type_id() const override; std::string as_human_readable() const override { return nameT; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -351,8 +344,8 @@ class TypeDataGenericT final : public TypeData { * A tensor can be empty. */ class TypeDataTensor final : public TypeData { - TypeDataTensor(uint64_t type_id, int children_flags, int width_on_stack, std::vector&& items) - : TypeData(type_id, children_flags, width_on_stack) + TypeDataTensor(int children_flags, int width_on_stack, std::vector&& items) + : TypeData(children_flags, width_on_stack) , items(std::move(items)) {} public: @@ -362,6 +355,7 @@ class TypeDataTensor final : public TypeData { int size() const { return static_cast(items.size()); } + int get_type_id() const override; std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -376,8 +370,8 @@ class TypeDataTensor final : public TypeData { * `var [i, cs] = [0, ""]` (where a and b become two separate variables on a stack, int and slice). */ class TypeDataTypedTuple final : public TypeData { - TypeDataTypedTuple(uint64_t type_id, int children_flags, std::vector&& items) - : TypeData(type_id, children_flags, 1) + TypeDataTypedTuple(int children_flags, std::vector&& items) + : TypeData(children_flags, 1) , items(std::move(items)) {} public: @@ -387,6 +381,7 @@ class TypeDataTypedTuple final : public TypeData { int size() const { return static_cast(items.size()); } + int get_type_id() const override; std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -401,8 +396,8 @@ class TypeDataTypedTuple final : public TypeData { * intN is smoothly cast from/to plain int, mathematical operators on intN also "fall back" to general int. */ class TypeDataIntN final : public TypeData { - TypeDataIntN(uint64_t type_id, bool is_unsigned, bool is_variadic, int n_bits) - : TypeData(type_id, 0, 1) + TypeDataIntN(bool is_unsigned, bool is_variadic, int n_bits) + : TypeData(0, 1) , is_unsigned(is_unsigned) , is_variadic(is_variadic) , n_bits(n_bits) {} @@ -414,6 +409,7 @@ class TypeDataIntN final : public TypeData { static TypePtr create(bool is_unsigned, bool is_variadic, int n_bits); + int get_type_id() const override; std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -424,7 +420,7 @@ class TypeDataIntN final : public TypeData { * Example: `var cost = ton("0.05")` has type `coins`. */ class TypeDataCoins final : public TypeData { - TypeDataCoins() : TypeData(17ULL, 0, 1) {} + TypeDataCoins() : TypeData(0, 1) {} static TypePtr singleton; friend void type_system_init(); @@ -432,6 +428,7 @@ class TypeDataCoins final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 17; } std::string as_human_readable() const override { return "coins"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -444,8 +441,8 @@ class TypeDataCoins final : public TypeData { * Note, that unlike intN automatically cast to/from int, bytesN does NOT auto cast to slice (without `as`). */ class TypeDataBytesN final : public TypeData { - TypeDataBytesN(uint64_t type_id, bool is_bits, int n_width) - : TypeData(type_id, 0, 1) + TypeDataBytesN(bool is_bits, int n_width) + : TypeData(0, 1) , is_bits(is_bits) , n_width(n_width) {} @@ -455,11 +452,69 @@ class TypeDataBytesN final : public TypeData { static TypePtr create(bool is_bits, int n_width); + int get_type_id() const override; std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; }; +/* + * `T1 | T2 | ...` is a union type. Unions are supported in Tolk, similar to TypeScript, stored like enums in Rust. + * `T | null` (denoted as `T?`) is also a union from a type system point of view. + * There is no TypeDataNullable, because mixing unions and nullables would result in a mess. + * At TVM level: + * - `T | null`, if T is 1 slot (like `int | null`), then it's still 1 slot + * - `T | null`, if T is N slots (like `(int, int)?`), it's stored as N+1 slots (the last for type_id if T or 0 if null) + * - `T1 | T2 | ...` is a tagged union: occupy max(T_i)+1 slots (1 for type_id) + */ +class TypeDataUnion final : public TypeData { + TypeDataUnion(int children_flags, int width_on_stack, TypePtr or_null, std::vector&& variants) + : TypeData(children_flags, width_on_stack) + , or_null(or_null) + , variants(std::move(variants)) {} + + bool has_variant_with_type_id(int type_id) const; + static void append_union_type_variant(TypePtr variant, std::vector& out_unique_variants); + +public: + const TypePtr or_null; // if `T | null`, then T is here (variants = [T, null] then); otherwise, nullptr + const std::vector variants; // T_i, flattened, no duplicates; may include aliases, but not other unions + + static TypePtr create(std::vector&& variants); + static TypePtr create_nullable(TypePtr nullable); + + // "primitive nullable" is `T?` which holds TVM NULL in the same slot (it other words, has no UTag slot) + // true : `int?`, `slice?`, `StructWith1IntField?` + // false: `(int, int)?`, `ComplexStruct?`, `()?` + bool is_primitive_nullable() const { + return get_width_on_stack() == 1 && or_null != nullptr && or_null->get_width_on_stack() == 1; + } + bool has_null() const { + if (or_null) { + return true; + } + return has_variant_with_type_id(0); + } + bool has_variant_with_type_id(TypePtr rhs_type) const { + int type_id = rhs_type->get_type_id(); + if (or_null) { + return type_id == 0 || type_id == or_null->get_type_id(); + } + return has_variant_with_type_id(type_id); + } + + TypePtr calculate_exact_variant_to_fit_rhs(TypePtr rhs_type) const; + bool has_all_variants_of(const TypeDataUnion* rhs_type) const; + + int get_type_id() const override; + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + void traverse(const TraverserCallbackT& callback) const override; + TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; + bool can_hold_tvm_null_instead() const override; +}; + /* * `unknown` is a special type, which can appear in corner cases. * The type of exception argument (which can hold any TVM value at runtime) is unknown. @@ -467,7 +522,7 @@ class TypeDataBytesN final : public TypeData { * The only thing available to do with unknown is to cast it: `catch (excNo, arg) { var i = arg as int; }` */ class TypeDataUnknown final : public TypeData { - TypeDataUnknown() : TypeData(20ULL, flag_contains_unknown_inside, 1) {} + TypeDataUnknown() : TypeData(flag_contains_unknown_inside, 1) {} static TypePtr singleton; friend void type_system_init(); @@ -475,6 +530,7 @@ class TypeDataUnknown final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override; std::string as_human_readable() const override { return "unknown"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -487,8 +543,8 @@ class TypeDataUnknown final : public TypeData { * Example: `fun f(v: T)` at first v is TypeDataUnresolved("T"), later becomes TypeDataGenericT. */ class TypeDataUnresolved final : public TypeData { - TypeDataUnresolved(uint64_t type_id, std::string&& text, SrcLocation loc) - : TypeData(type_id, flag_contains_unresolved_inside, -999999) + TypeDataUnresolved(std::string&& text, SrcLocation loc) + : TypeData(flag_contains_unresolved_inside, -999999) , text(std::move(text)) , loc(loc) {} @@ -498,6 +554,7 @@ class TypeDataUnresolved final : public TypeData { static TypePtr create(std::string&& text, SrcLocation loc); + int get_type_id() const override; std::string as_human_readable() const override { return text + "*"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -510,7 +567,7 @@ class TypeDataUnresolved final : public TypeData { * Such variables can not be cast to any other types, all their usage will trigger type mismatch errors. */ class TypeDataNever final : public TypeData { - TypeDataNever() : TypeData(19ULL, 0, 0) {} + TypeDataNever() : TypeData(0, 0) {} static TypePtr singleton; friend void type_system_init(); @@ -518,6 +575,7 @@ class TypeDataNever final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 19; } std::string as_human_readable() const override { return "never"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -530,7 +588,7 @@ class TypeDataNever final : public TypeData { * Empty tensor is not compatible with void, although at IR level they are similar, 0 stack slots. */ class TypeDataVoid final : public TypeData { - TypeDataVoid() : TypeData(10ULL, 0, 0) {} + TypeDataVoid() : TypeData(0, 0) {} static TypePtr singleton; friend void type_system_init(); @@ -538,6 +596,7 @@ class TypeDataVoid final : public TypeData { public: static TypePtr create() { return singleton; } + int get_type_id() const override { return 10; } std::string as_human_readable() const override { return "void"; } bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; @@ -547,14 +606,6 @@ class TypeDataVoid final : public TypeData { // -------------------------------------------- -inline TypePtr TypeData::unwrap_alias() const { - TypePtr unwrapped = this; - while (const TypeDataAlias* as_alias = unwrapped->try_as()) { - unwrapped = as_alias->underlying_type; - } - return unwrapped; -} - class Lexer; TypePtr parse_type_from_tokens(Lexer& lex); TypePtr parse_type_from_string(std::string_view text); From ac57626d342d656af3f120764949294bdb2b801d Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sat, 5 Apr 2025 17:15:07 +0300 Subject: [PATCH 202/388] [Tolk] Pattern matching for expressions (switch-like behavior) `match` can also be used for constant expressions, similar to `switch`: > val nextValue = match (curValue) { > 1 => 0, > 0 => 1, > else => -1 > }; Only constant expressions are allowed on the left-hand side. `else` is required for expression form but optional for statement form. --- tolk-tester/tests/if-else-tests.tolk | 107 +++++++++ .../tests/invalid-declaration/err-1808.tolk | 6 + .../tests/invalid-semantics/err-4258.tolk | 13 ++ .../tests/invalid-semantics/err-4517.tolk | 14 ++ .../tests/invalid-syntax/err-3080.tolk | 11 + .../tests/invalid-syntax/err-3419.tolk | 12 + .../tests/invalid-syntax/err-3457.tolk | 2 +- .../tests/invalid-syntax/err-3837.tolk | 14 ++ .../tests/invalid-syntax/err-3913.tolk | 8 + tolk-tester/tests/match-by-expr-tests.tolk | 205 ++++++++++++++++++ tolk-tester/tests/union-types-tests.tolk | 29 +++ tolk/analyzer.cpp | 27 ++- tolk/constant-evaluator.cpp | 22 +- tolk/constant-evaluator.h | 1 + tolk/pipe-ast-to-legacy.cpp | 89 ++++---- tolk/pipe-check-inferred-types.cpp | 29 ++- tolk/pipe-constant-folding.cpp | 17 +- 17 files changed, 551 insertions(+), 55 deletions(-) create mode 100644 tolk-tester/tests/invalid-declaration/err-1808.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4258.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4517.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3080.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3419.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3837.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3913.tolk create mode 100644 tolk-tester/tests/match-by-expr-tests.tolk diff --git a/tolk-tester/tests/if-else-tests.tolk b/tolk-tester/tests/if-else-tests.tolk index 0f1d31f94..b16aef15e 100644 --- a/tolk-tester/tests/if-else-tests.tolk +++ b/tolk-tester/tests/if-else-tests.tolk @@ -72,6 +72,33 @@ fun test7() { // here we test that no "missing return" error } +fun withNoElse(op: int): int { + if (op == 123) { return 100; } + if (op == 234) { return 200; } + if (op == 345) { return 300; } + throw 0xFF; +} + +fun withElse(op: int): int { + if (op == 123) { return 100; } + else if (op == 234) { return 200; } + else if (op == 345) { return 300; } + return 0xFF; +} + +fun withMatch(op: int): int { + match (op) { + 123 => return 100, + 234 => return 200, + 345 => return 300, + } + throw 0xFF; +} + +@method_id(108) +fun test8(op: int) { + return (withNoElse(op), withElse(op), withMatch(op)); +} fun main() { @@ -96,6 +123,8 @@ fun main() { @testcase | 105 | 3 | 7 222 @testcase | 106 | -5 | -1 @testcase | 106 | 5 | 5000 +@testcase | 108 | 123 | 100 100 100 +@testcase | 108 | 345 | 300 300 300 @fif_codegen """ @@ -111,4 +140,82 @@ fun main() { 50 EQINT // x '5 IFNOTJMP:<{ // x """ + +@fif_codegen +""" + withNoElse PROC:<{ + // op + DUP + 123 EQINT + IFJMP:<{ + DROP + 100 PUSHINT + }> + DUP + 234 PUSHINT + EQUAL + IFJMP:<{ + DROP + 200 PUSHINT + }> + 345 PUSHINT + EQUAL + IFJMP:<{ + 300 PUSHINT + }> + 255 THROW + }> +""" + +@fif_codegen +""" + withElse PROC:<{ + // op + DUP + 123 EQINT + IFJMP:<{ + DROP + 100 PUSHINT + }> + DUP + 234 PUSHINT + EQUAL + IFJMP:<{ + DROP + 200 PUSHINT + }> + 345 PUSHINT + EQUAL + IFJMP:<{ + 300 PUSHINT + }> + 8 PUSHPOW2DEC + }> +""" + +@fif_codegen +""" + withMatch PROC:<{ + // op + DUP + 123 EQINT + IFJMP:<{ + DROP + 100 PUSHINT + }> + DUP + 234 PUSHINT + EQUAL + IFJMP:<{ + DROP + 200 PUSHINT + }> + 345 PUSHINT + EQUAL + IFJMP:<{ + 300 PUSHINT + }> + 255 THROW + }> +""" */ diff --git a/tolk-tester/tests/invalid-declaration/err-1808.tolk b/tolk-tester/tests/invalid-declaration/err-1808.tolk new file mode 100644 index 000000000..a7faa6ff3 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1808.tolk @@ -0,0 +1,6 @@ +const C = match (10) { 10 => 0, else => -1 }; + +/** +@compilation_should_fail +@stderr not a constant expression + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4258.tolk b/tolk-tester/tests/invalid-semantics/err-4258.tolk new file mode 100644 index 000000000..d60d1adf3 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4258.tolk @@ -0,0 +1,13 @@ +fun f(x: int): int { + return match(x) { + -1 => -1, + f(0) => 0, + else => 1, + }; +} + +/** +@compilation_should_fail +@stderr not a constant expression +@stderr f(0) => 0 + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4517.tolk b/tolk-tester/tests/invalid-semantics/err-4517.tolk new file mode 100644 index 000000000..f84e3ac1b --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4517.tolk @@ -0,0 +1,14 @@ +global g: int; + +fun f(x: int): int { + match(x) { + -1 => -1, + g => 0, + }; + return 10; +} + +/** +@compilation_should_fail +@stderr symbol `g` is not a constant + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3080.tolk b/tolk-tester/tests/invalid-syntax/err-3080.tolk new file mode 100644 index 000000000..e79e8ee95 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3080.tolk @@ -0,0 +1,11 @@ +fun f(x: int) { + return match(x) { + -1 => -1, + 0 => 0 + }; +} + +/** +@compilation_should_fail +@stderr `match` expression should have `else` branch + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3419.tolk b/tolk-tester/tests/invalid-syntax/err-3419.tolk new file mode 100644 index 000000000..a59ca0e68 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3419.tolk @@ -0,0 +1,12 @@ +fun f(x: int) { + return match(x) { + -1 => -1, + else => 0, + 0 => 0 + }; +} + +/** +@compilation_should_fail +@stderr `else` branch should be the last + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3457.tolk b/tolk-tester/tests/invalid-syntax/err-3457.tolk index a2ef6e96b..8cfcf8e6c 100644 --- a/tolk-tester/tests/invalid-syntax/err-3457.tolk +++ b/tolk-tester/tests/invalid-syntax/err-3457.tolk @@ -13,6 +13,6 @@ fun main() { /** @compilation_should_fail -@stderr `match` by expression is not supported yet +@stderr symbol `asdf` is not a constant @stderr asdf => 2 */ diff --git a/tolk-tester/tests/invalid-syntax/err-3837.tolk b/tolk-tester/tests/invalid-syntax/err-3837.tolk new file mode 100644 index 000000000..c0ee50b30 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3837.tolk @@ -0,0 +1,14 @@ +fun f(x: int) { + match(x) { + -1 => -1, + 0 => 0, + else => 0, + else => -1 + }; +} + +/** +@compilation_should_fail +@stderr duplicated `else` branch +@stderr else => -1 + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3913.tolk b/tolk-tester/tests/invalid-syntax/err-3913.tolk new file mode 100644 index 000000000..033743259 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3913.tolk @@ -0,0 +1,8 @@ +fun f(x: bool) { + return match (x) { true => 0, true => -1 }; +} + +/** +@compilation_should_fail +@stderr `match` expression should have `else` branch + */ diff --git a/tolk-tester/tests/match-by-expr-tests.tolk b/tolk-tester/tests/match-by-expr-tests.tolk new file mode 100644 index 000000000..4bb71c664 --- /dev/null +++ b/tolk-tester/tests/match-by-expr-tests.tolk @@ -0,0 +1,205 @@ +const C1 = 1; +const C2 = C1 + 1; + +@method_id(101) +fun test1(x: int) { + return match (x) { + 1 => 100, + C2 => 200, + 2 + 1 => 300, + else => 400 + }; +} + +@method_id(102) +fun test2() { + var x = 3; + return match (x) { + 1 => 100, + C2 => 200, + 2 + 1 => 300, + else => 400 + }; +} + +fun isGt10(x: int) { return x > 10; } + +@method_id(103) +fun test3(init: int) { + var r1 = match (isGt10(init)) { true => 10, false => 20 }; + var r2 = match (isGt10(init)) { !!true => 10, !!false => 20, else => 8 }; + var r3 = match (isGt10(init)) { 10 > 3 => 10, 10 < 3 => 20, 100 > 1 => 30, else => 8 }; + return (r1, r2, r3); +} + +@method_id(104) +fun test4(x: int?) { + return match(match(x) { null => 0, int => x * 2 }) { 0 => 0, 30 => 30, else => throw 123 }; +} + +@method_id(105) +fun test5(x: int) { + try { + return match (val r = x * 2) { + 10/C1 => 10, + C2*10 => throw 500, + else => 50 + }; + } catch (excCode) { + return excCode; + } +} + +@method_id(106) +fun test6(x: coins) { + var result1 = 0; + match (x) { + ton("0.05") => { result1 = 5; } + ton("0.1") => { result1 = 10; } + else => { result1 = 20; } + } + var result2 = 0; + match (x) { + ton("0.05") => { result2 = 5; } + ton("0.1") => { result2 = 10; } + } + return (result1, result2); +} + +@method_id(107) +fun test7(x: int8) { + match (x) { 10 => {} } + return match (x) { + 10 => 10, + 20 as int8 => 20, + 30 as int16 => 30, + 40 as uint255 => 40, + else => 50, + }; +} + +@method_id(108) +fun test8(x: int) { + var r1 = match (x) { else => 10 }; + match (x) { } + match (x) { else => { r1 += 20; } } + return r1; +} + +@method_id(109) +fun test9(x: int | slice) { + if (x is slice) { + match (x.loadInt(32)) { -1 => {}, 0 => {} } + } + return match (x) { + int => match (x) { 10 => 5, else => -1 }, + slice => match (x.loadInt(32)) { -1 => 0, 0 => 0, else => throw 20 } + }; +} + +@method_id(110) +fun test10(x: int) { + var result1 = match (x) { + 10 => throw 10, + 20 => return 20, + 30 => { x += 10; return x; } + else => x + }; + return result1 * 2; +} + +@method_id(111) +fun test11(x: bool) { + return match (x) { + true => false, + else => !x, + }; +} + + +fun main() {} + +/** +@testcase | 101 | 1 | 100 +@testcase | 101 | 2 | 200 +@testcase | 101 | 3 | 300 +@testcase | 101 | 4 | 400 +@testcase | 102 | | 300 +@testcase | 103 | 5 | 20 20 20 +@testcase | 103 | 15 | 10 10 10 +@testcase | 104 | 15 | 30 +@testcase | 104 | null | 0 +@testcase | 105 | 10 | 500 +@testcase | 105 | 11 | 50 +@testcase | 106 | 50000000 | 5 5 +@testcase | 106 | 300 | 20 0 +@testcase | 107 | 30 | 30 +@testcase | 107 | 40 | 40 +@testcase | 107 | 41 | 50 +@testcase | 108 | 5 | 30 +@testcase | 109 | 1 1 | -1 +@testcase | 110 | 20 | 20 +@testcase | 110 | 30 | 40 +@testcase | 110 | 50 | 100 +@testcase | 111 | 0 | -1 +@testcase | 111 | -1 | 0 + +@fif_codegen +""" + test1 PROC:<{ + // x + DUP // x x + 1 EQINT // x '3 + IF:<{ // x + DROP // + 100 PUSHINT // '1=100 + }>ELSE<{ // x + DUP // x x + 2 EQINT // x '6 + IF:<{ // x + DROP // + 200 PUSHINT // '1=200 + }>ELSE<{ // x + 3 EQINT // '9 + IF:<{ // + 300 PUSHINT // '1=300 + }>ELSE<{ // + 400 PUSHINT // '1=400 + }> + }> + }> + }> +""" + +@fif_codegen +""" + test2 PROC:<{ + // + 300 PUSHINT + }> +""" + +@fif_codegen +""" + test3 PROC:<{ + // init + DUP // init init + isGt10 CALLDICT // init '2 + -1 EQINT // init '5 + IF:<{ // init + 10 PUSHINT // init '3=10 + }>ELSE<{ // init + 20 PUSHINT // init '3=20 + }> // init r1 +""" + +@fif_codegen +""" + test8 PROC:<{ + // x + DROP // + 30 PUSHINT // r1 + }> +""" + + */ diff --git a/tolk-tester/tests/union-types-tests.tolk b/tolk-tester/tests/union-types-tests.tolk index 04446f8e7..e451f13dd 100644 --- a/tolk-tester/tests/union-types-tests.tolk +++ b/tolk-tester/tests/union-types-tests.tolk @@ -764,6 +764,15 @@ fun test60() { return (returnVoidOrNull(true), returnVoidOrNull(false)); } +@method_id(161) +fun test61(i: int32 | int64 | int) { + match (i) { + int32 => { return 10; } + int64 => { return 20; } + int => { return 30; } + } +} + fun main() { @@ -837,6 +846,7 @@ fun main() { @testcase | 159 | 0 4 | 456 @testcase | 159 | null 0 | 123 @testcase | 160 | | 10 0 +@testcase | 161 | 100 48 | 20 @fif_codegen @@ -868,6 +878,25 @@ fun main() { }> """ +@fif_codegen +""" + test61 PROC:<{ + // i.USlot1 i.UTag + NIP + DUP + 46 EQINT + IFJMP:<{ + DROP + 10 PUSHINT + }> + 48 EQINT + IFJMP:<{ + 20 PUSHINT + }> + 30 PUSHINT + }> +""" + @fif_codegen DECLPROC checkGeneric3 */ diff --git a/tolk/analyzer.cpp b/tolk/analyzer.cpp index c38b0bfa4..2cd65c419 100644 --- a/tolk/analyzer.cpp +++ b/tolk/analyzer.cpp @@ -874,7 +874,32 @@ bool Op::mark_noreturn() { return set_noreturn(); case _Call: return set_noreturn(next->mark_noreturn() || does_function_always_throw(f_sym)); - case _If: + case _If: { + // this very-not-beautiful code does the following: + // replace `if (cond) { return; } else { block1; } next;` with `if (cond) { return; } block1; next` + // purpose: to make code like `if (...) { ... return; } else if (...) { ... return; } ...` act like without else + // similarly, `match (...) { v1 => { ... return; } ...}` is internally transformed to IF-ELSE + // (that's why such transformation is done at IR level, not in AST) + // without these code (without else removed), extra RETALT instructions are inserted, not necessary actually + // implementation is UGLY, because currently there is no way to perform IR replacements + // in the future, anyway, IR implementation should be rewritten, for easier traversing and replacement + // btw, now it doesn't work with `if (!...)` (v->is_ifnot), else "keyword" is not removed + if (block0->mark_noreturn() && !block1->is_empty()) { + VarDescrList block1_var_info = block1->var_info; // important to keep it + Op* last_in_block1 = block1.get(); + while (last_in_block1->next->cl != Op::_Nop) { // find the tail of a forward list of Ops + last_in_block1 = last_in_block1->next.get(); + } + last_in_block1->next = std::move(next); + next = std::move(block1); + block1 = std::make_unique(where, Op::_Nop); + block1->var_info = std::move(block1_var_info); + } else { + block1->mark_noreturn(); + } + bool next_noreturn = next->mark_noreturn(); + return set_noreturn((block0->noreturn() && block1->noreturn()) || next_noreturn); + } case _TryCatch: // note, that & | (not && ||) here and below is mandatory to invoke both left and right calls return set_noreturn((static_cast(block0->mark_noreturn()) & static_cast(block1 && block1->mark_noreturn())) | static_cast(next->mark_noreturn())); diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp index 7daebc1da..ff8dbbce1 100644 --- a/tolk/constant-evaluator.cpp +++ b/tolk/constant-evaluator.cpp @@ -384,12 +384,11 @@ struct ConstantEvaluator { v->error("not a constant expression"); } - // evaluate `const a = 2 + 3` into 5 - // type inferring has already passed, to types are correct, `const a = 1 + ""` can not occur - // recursive initializers `const a = b; const b = a` also 100% don't exist, checked on type inferring - // if init_value is not a constant expression like `const a = foo()`, an exception is thrown - static ConstantValue eval_const_init_value(GlobalConstPtr const_ref) { - return visit(const_ref->init_value); + // evaluate `2 + 3` into 5 + // type inferring has already passed, to types are correct, `1 + ""` can not occur + // if v is not a constant expression like `foo()`, an exception is thrown + static ConstantValue eval_expression_expected_to_be_constant(AnyExprV v) { + return visit(v); } }; @@ -405,8 +404,17 @@ ConstantValue eval_call_to_compile_time_function(AnyExprV v_call) { return parse_vertex_call_to_compile_time_function(v, v->fun_maybe->name); } +// evaluate `2 + 3` into `5` +// if v is not a constant expression like `gVar`, an exception is thrown +ConstantValue eval_expression_expected_to_be_constant(AnyExprV v) { + return ConstantEvaluator::eval_expression_expected_to_be_constant(v); +} + +// evaluate `const a = 2 + 3` into `const a = 5` +// recursive initializers `const a = b; const b = a` don't exist, checked on type inferring +// if init_value is not a constant expression like `const a = foo()`, an exception is thrown void eval_and_assign_const_init_value(GlobalConstPtr const_ref) { - ConstantValue init_value = ConstantEvaluator::eval_const_init_value(const_ref); + ConstantValue init_value = ConstantEvaluator::eval_expression_expected_to_be_constant(const_ref->init_value); const_ref->mutate()->assign_const_value(std::move(init_value)); } diff --git a/tolk/constant-evaluator.h b/tolk/constant-evaluator.h index b065e6f49..1a7f364fb 100644 --- a/tolk/constant-evaluator.h +++ b/tolk/constant-evaluator.h @@ -49,6 +49,7 @@ class ConstantValue { ConstantValue eval_string_const_standalone(AnyExprV v_string); ConstantValue eval_call_to_compile_time_function(AnyExprV v_call); +ConstantValue eval_expression_expected_to_be_constant(AnyExprV v); void eval_and_assign_const_init_value(GlobalConstPtr const_ref); } // namespace tolk diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 167c56147..bba79ea92 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -1018,60 +1018,67 @@ static std::vector process_not_null_operator(V static std::vector process_match_expression(V v, CodeBlob& code, TypePtr target_type) { TypePtr lhs_type = v->get_subject()->inferred_type->unwrap_alias(); + int n_arms = v->get_arms_count(); std::vector subj_ir_idx = pre_compile_expr(v->get_subject(), code, nullptr); std::vector result_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(match-expression)"); - // detect whether it's `match` by type covering all cases; type checker has already checked that - // match by type can not be mixed with match by expression and can't contain `else`, and that types are distinct and cover all variants - bool is_exhaustive_match_by_type = v->get_arms_count() > 0 && v->get_arm(0)->pattern_kind == MatchArmKind::exact_type; - if (is_exhaustive_match_by_type) { - // example: `match (v) { int => ... slice => ... builder => ... }` - // construct nested IFs: IF is int { ... } ELSE { IF is slice { ... } ELSE { ... } } - for (int i = 0; i < v->get_arms_count() - 1; ++i) { - auto v_ith_arm = v->get_arm(i); + if (!n_arms) { // `match (subject) {}` + tolk_assert(v->is_statement()); + return {}; + } + + // it's either `match` by type (all arms patterns are types) or `match` by expression + bool is_match_by_type = v->get_arm(0)->pattern_kind == MatchArmKind::exact_type; + // detect whether `match` is exhaustive + bool is_exhaustive = is_match_by_type // match by type covers all cases, checked earlier + || !v->is_statement() // match by expression is guaranteely exhaustive, checked earlier + || v->get_arm(n_arms - 1)->pattern_kind == MatchArmKind::else_branch; + + // example 1 (exhaustive): `match (v) { int => ... slice => ... builder => ... }` + // construct nested IFs: IF is int { ... } ELSE { IF is slice { ... } ELSE { ... } } + // example 2 (exhaustive): `match (v) { -1 => ... 0 => ... else => ... }` + // construct nested IFs: IF is int { ... } ELSE { IF is slice { ... } ELSE { ... } } + // example 3 (not exhaustive): `match (v) { -1 => ... 0 => ... 1 => ... }` + // construct nested IFs: IF == -1 { ... } ELSE { IF == 0 { ... } ELSE { IF == 1 { ... } } } + FunctionPtr eq_sym = lookup_global_symbol("_==_")->try_as(); + for (int i = 0; i < n_arms - is_exhaustive; ++i) { + auto v_ith_arm = v->get_arm(i); + std::vector eq_ith_ir_idx; + if (is_match_by_type) { TypePtr cmp_type = v_ith_arm->exact_type->unwrap_alias(); tolk_assert(!cmp_type->try_as()); // `match` over `int|slice` is a type checker error - std::vector eq_ith_ir_idx = pre_compile_is_type(code, lhs_type, cmp_type, subj_ir_idx, v_ith_arm->loc, "(arm-cond-eq)"); - Op& if_op = code.emplace_back(v_ith_arm->loc, Op::_If, std::move(eq_ith_ir_idx)); - code.push_set_cur(if_op.block0); - if (v->is_statement()) { - pre_compile_expr(v_ith_arm->get_body(), code); - } else { - std::vector arm_ir_idx = pre_compile_expr(v_ith_arm->get_body(), code, v->inferred_type); - code.emplace_back(v->loc, Op::_Let, result_ir_idx, std::move(arm_ir_idx)); - } - code.close_pop_cur(v->loc); - code.push_set_cur(if_op.block1); // open ELSE + eq_ith_ir_idx = pre_compile_is_type(code, lhs_type, cmp_type, subj_ir_idx, v_ith_arm->loc, "(arm-cond-eq)"); + } else { + std::vector ith_ir_idx = pre_compile_expr(v_ith_arm->get_pattern_expr(), code); + tolk_assert(subj_ir_idx.size() == 1 && ith_ir_idx.size() == 1); + eq_ith_ir_idx = code.create_tmp_var(TypeDataBool::create(), v_ith_arm->loc, "(arm-cond-eq)"); + code.emplace_back(v_ith_arm->loc, Op::_Call, eq_ith_ir_idx, std::vector{subj_ir_idx[0], ith_ir_idx[0]}, eq_sym); + } + Op& if_op = code.emplace_back(v_ith_arm->loc, Op::_If, std::move(eq_ith_ir_idx)); + code.push_set_cur(if_op.block0); + if (v->is_statement()) { + pre_compile_expr(v_ith_arm->get_body(), code); + } else { + std::vector arm_ir_idx = pre_compile_expr(v_ith_arm->get_body(), code, v->inferred_type); + code.emplace_back(v->loc, Op::_Let, result_ir_idx, std::move(arm_ir_idx)); } + code.close_pop_cur(v->loc); + code.push_set_cur(if_op.block1); // open ELSE + } - // we're inside the last ELSE, close all outer IFs - auto v_last_arm = v->get_arm(v->get_arms_count() - 1); + if (is_exhaustive) { + // we're inside the last ELSE + auto v_last_arm = v->get_arm(n_arms - 1); if (v->is_statement()) { pre_compile_expr(v_last_arm->get_body(), code); } else { std::vector arm_ir_idx = pre_compile_expr(v_last_arm->get_body(), code, v->inferred_type); code.emplace_back(v->loc, Op::_Let, result_ir_idx, std::move(arm_ir_idx)); } - for (int i = 0; i < v->get_arms_count() - 1; ++i) { - code.close_pop_cur(v->loc); - } - - } else { - // not exhaustive `match` by type - for (int i = 0; i < v->get_arms_count(); ++i) { - auto v_arm = v->get_arm(i); - switch (v_arm->pattern_kind) { - case MatchArmKind::const_expression: - tolk_assert(false); // match by expressions is not supported yet, checked earlier - break; - case MatchArmKind::exact_type: - tolk_assert(false); // match by type is exhaustive, checked earlier - break; - case MatchArmKind::else_branch: - tolk_assert(false); // `else` in match can be only inside match by expression, unsupported yet - break; - } - } + } + // close all outer IFs + for (int i = 0; i < n_arms - is_exhaustive; ++i) { + code.close_pop_cur(v->loc); } return transition_to_target_type(std::move(result_ir_idx), code, target_type, v); diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 2174560fd..b9bcdaced 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -227,7 +227,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { bool both_int = expect_integer(lhs) && expect_integer(rhs); bool both_bool = expect_boolean(lhs) && expect_boolean(rhs); if (!both_int && !both_bool) { - if (lhs->inferred_type == rhs->inferred_type) { // compare slice with slice, int? with int? + if (lhs->inferred_type->equal_to(rhs->inferred_type)) { // compare slice with slice, int? with int? fire(cur_f, v->loc, "type " + to_string(lhs) + " can not be compared with `== !=`"); } else { fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); @@ -525,7 +525,8 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { bool has_type_arm = false; bool has_expr_arm = false; bool has_else_arm = false; - TypePtr subject_type = v->get_subject()->inferred_type->unwrap_alias(); + AnyExprV v_subject = v->get_subject(); + TypePtr subject_type = v_subject->inferred_type->unwrap_alias(); const TypeDataUnion* subject_union = subject_type->try_as(); std::vector covered_type_ids; // for type-based `match`, what types are on the left of `=>` @@ -565,7 +566,16 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { fire(cur_f, v_arm->loc, "`else` branch should be the last"); } has_expr_arm = true; - fire(cur_f, v_arm->loc, "`match` by expression is not supported yet"); + TypePtr pattern_type = v_arm->get_pattern_expr()->inferred_type->unwrap_alias(); + bool both_int = expect_integer(pattern_type) && expect_integer(subject_type); + bool both_bool = expect_boolean(pattern_type) && expect_boolean(subject_type); + if (!both_int && !both_bool) { + if (pattern_type->equal_to(subject_type)) { // `match` over `slice` etc., where operator `==` can't be applied + fire(cur_f, v_arm->loc, "wrong pattern matching: can not compare type " + to_string(subject_type) + " in `match`"); + } else { + fire(cur_f, v_arm->loc, "wrong pattern matching: can not compare type " + to_string(v_arm->get_pattern_expr()) + " with match subject of type " + to_string(v_subject)); + } + } break; } default: @@ -576,7 +586,6 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { fire(cur_f, v_arm->loc, "`else` is not allowed in `match` by type; you should cover all possible types"); } has_else_arm = true; - fire(cur_f, v_arm->loc, "`match` by expression is not supported yet"); } } @@ -593,6 +602,18 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } throw ParseError(cur_f, v->loc, "`match` does not cover all possible types; missing types are: " + missing); } + // `match` by expression, if it's not statement, should have `else` (unless it's match over bool with const true/false) + if (has_expr_arm && !has_else_arm && !v->is_statement()) { + bool needs_else_branch = true; + if (expect_boolean(subject_type) && v->get_arms_count() == 2) { + auto arm0 = v->get_arm(0)->get_pattern_expr()->try_as(); + auto arm1 = v->get_arm(1)->get_pattern_expr()->try_as(); + needs_else_branch = !(arm0 && arm1 && arm0->bool_val != arm1->bool_val); + } + if (needs_else_branch) { + fire(cur_f, v->loc, "`match` expression should have `else` branch"); + } + } } void visit(V v) override { diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp index 1a071f492..c64f7bff5 100644 --- a/tolk/pipe-constant-folding.cpp +++ b/tolk/pipe-constant-folding.cpp @@ -134,7 +134,22 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { return v; } - public: + AnyExprV replace(V v) override { + parent::replace(v); + + // replace `2 + 3 => ...` with `5 => ...` + // non-constant expressions like `foo() => ...` fire an error here + if (v->pattern_kind == MatchArmKind::const_expression && v->get_pattern_expr()->type != ast_int_const) { + ConstantValue value = eval_expression_expected_to_be_constant(v->get_pattern_expr()); + tolk_assert(value.is_int()); + AnyExprV v_new_pattern = create_int_const(v->get_pattern_expr()->loc, value.as_int()); + v->mutate()->assign_resolved_pattern(MatchArmKind::const_expression, nullptr, v_new_pattern); + } + + return v; + } + +public: bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function(); } From 9c0f893f6a53661e71dbb2c2502167c763bdb5ea Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Fri, 28 Mar 2025 13:26:40 +0300 Subject: [PATCH 203/388] [Tolk] Optional semicolon for the last statement in a block While semicolons are required between statements to disambiguate sequencing, the trailing semicolon on the last statement is now optional. This change is backward-compatible and does not affect existing code. --- tolk-tester/tests/a-tests.tolk | 14 ++-- tolk-tester/tests/if-else-tests.tolk | 20 ++--- tolk-tester/tests/logical-operators.tolk | 6 +- tolk-tester/tests/match-by-expr-tests.tolk | 18 ++--- tolk-tester/tests/no-spaces.tolk | 16 ++-- tolk-tester/tests/nullable-types.tolk | 4 +- tolk-tester/tests/smart-cast-tests.tolk | 8 +- tolk-tester/tests/unbalanced-ret.tolk | 6 +- tolk-tester/tests/union-types-tests.tolk | 74 +++++++++--------- tolk-tester/tests/var-apply-tests.tolk | 26 +++---- tolk/ast-from-tokens.cpp | 88 ++++++++++------------ tolk/ast-from-tokens.h | 25 ------ tolk/ast-replacer.h | 6 +- tolk/ast-replicator.h | 10 +-- tolk/ast-stringifier.h | 8 +- tolk/ast-visitor.h | 6 +- tolk/ast.cpp | 2 +- tolk/ast.h | 52 ++++++------- tolk/pipe-ast-to-legacy.cpp | 10 +-- tolk/pipe-calc-rvalue-lvalue.cpp | 2 +- tolk/pipe-check-inferred-types.cpp | 4 +- tolk/pipe-discover-parse-sources.cpp | 5 +- tolk/pipe-infer-types-and-calls.cpp | 12 +-- tolk/pipe-register-symbols.cpp | 2 +- tolk/pipe-resolve-identifiers.cpp | 10 +-- 25 files changed, 201 insertions(+), 233 deletions(-) delete mode 100644 tolk/ast-from-tokens.h diff --git a/tolk-tester/tests/a-tests.tolk b/tolk-tester/tests/a-tests.tolk index 65bff5ad1..077303d64 100644 --- a/tolk-tester/tests/a-tests.tolk +++ b/tolk-tester/tests/a-tests.tolk @@ -4,19 +4,19 @@ fun f(a: int, b: int, c: int, d: int, e: int, f: int): (int, int) { __expect_type(D, "int"); __expect_type(D*D, "int"); __expect_type(calc_phi, "() -> int"); - return (Dx/D,Dy/D,); + return (Dx/D,Dy/D,) };;;; fun calc_phi(): int { var n = 1; - repeat (70) { n*=10; }; + repeat (70) { n*=10 }; var p= 1; var `q`=1; _=`q`; do { - (p,q)=(q,p+q); + (p,q)=(q,p+q) } while (q <= n); //;; - return mulDivRound(p, n, q,); + return mulDivRound(p, n, q,) } fun calc_sqrt2(): int { @@ -43,7 +43,7 @@ fun calc_root(m: int,) { k+=1; (a1, b1, c1) = (a, b, c); c+=b; - c += b += a; + c += b += a } while (c <= 0); (a, b, c) = (-c1, -b1, -a1); (p1, q1) = (k * p1+q1, p1); @@ -67,12 +67,12 @@ fun ataninv(base: int, q: int): int { // computes base*atan(1/q) fun calc_pi(): int { var base: int = 64; - repeat (70) { base *= 10; } + repeat (70) { base *= 10 } return (ataninv(base << 2, 5) - ataninv(base, 239))~>>4; } fun main(): int { - return calc_pi(); + return calc_pi() } /** diff --git a/tolk-tester/tests/if-else-tests.tolk b/tolk-tester/tests/if-else-tests.tolk index b16aef15e..f40336a9e 100644 --- a/tolk-tester/tests/if-else-tests.tolk +++ b/tolk-tester/tests/if-else-tests.tolk @@ -68,22 +68,22 @@ fun doNothing(): void {} @method_id(107) fun test7() { - if (random()) { return doNothing(); } + if (random()) { return doNothing() } // here we test that no "missing return" error } fun withNoElse(op: int): int { - if (op == 123) { return 100; } - if (op == 234) { return 200; } - if (op == 345) { return 300; } - throw 0xFF; + if (op == 123) { return 100 } + if (op == 234) { return 200 } + if (op == 345) { return 300 } + throw 0xFF } fun withElse(op: int): int { - if (op == 123) { return 100; } - else if (op == 234) { return 200; } - else if (op == 345) { return 300; } - return 0xFF; + if (op == 123) { return 100 } + else if (op == 234) { return 200 } + else if (op == 345) { return 300 } + return 0xFF } fun withMatch(op: int): int { @@ -92,7 +92,7 @@ fun withMatch(op: int): int { 234 => return 200, 345 => return 300, } - throw 0xFF; + throw 0xFF } @method_id(108) diff --git a/tolk-tester/tests/logical-operators.tolk b/tolk-tester/tests/logical-operators.tolk index 700f2a3c5..cbc82f15e 100644 --- a/tolk-tester/tests/logical-operators.tolk +++ b/tolk-tester/tests/logical-operators.tolk @@ -13,9 +13,9 @@ fun compileTimeEval1(x: int) { @method_id(101) fun withIfNot(x: int, y: int) { - if (!x) { return 10; } - else if (!y) { return 20; } - return x+y; + if (!x) { return 10 } + else if (!y) { { return 20 } } + return x+y } @method_id(102) diff --git a/tolk-tester/tests/match-by-expr-tests.tolk b/tolk-tester/tests/match-by-expr-tests.tolk index 4bb71c664..d9ecb3cfe 100644 --- a/tolk-tester/tests/match-by-expr-tests.tolk +++ b/tolk-tester/tests/match-by-expr-tests.tolk @@ -8,7 +8,7 @@ fun test1(x: int) { C2 => 200, 2 + 1 => 300, else => 400 - }; + } } @method_id(102) @@ -19,7 +19,7 @@ fun test2() { C2 => 200, 2 + 1 => 300, else => 400 - }; + } } fun isGt10(x: int) { return x > 10; } @@ -34,7 +34,7 @@ fun test3(init: int) { @method_id(104) fun test4(x: int?) { - return match(match(x) { null => 0, int => x * 2 }) { 0 => 0, 30 => 30, else => throw 123 }; + return match(match(x) { null => 0, int => x * 2 }) { 0 => 0, 30 => 30, else => throw 123 } } @method_id(105) @@ -44,9 +44,9 @@ fun test5(x: int) { 10/C1 => 10, C2*10 => throw 500, else => 50 - }; + } } catch (excCode) { - return excCode; + return excCode } } @@ -60,8 +60,8 @@ fun test6(x: coins) { } var result2 = 0; match (x) { - ton("0.05") => { result2 = 5; } - ton("0.1") => { result2 = 10; } + ton("0.05") => { result2 = 5 } + ton("0.1") => { result2 = 10 } } return (result1, result2); } @@ -94,7 +94,7 @@ fun test9(x: int | slice) { return match (x) { int => match (x) { 10 => 5, else => -1 }, slice => match (x.loadInt(32)) { -1 => 0, 0 => 0, else => throw 20 } - }; + } } @method_id(110) @@ -113,7 +113,7 @@ fun test11(x: bool) { return match (x) { true => false, else => !x, - }; + } } diff --git a/tolk-tester/tests/no-spaces.tolk b/tolk-tester/tests/no-spaces.tolk index d63ca1411..3fe2573a9 100644 --- a/tolk-tester/tests/no-spaces.tolk +++ b/tolk-tester/tests/no-spaces.tolk @@ -1,13 +1,13 @@ const int10:int=10; fun just10(): int { return int10; } -fun eq(v: int): int { return`v`; } +fun eq(v: int): int { return`v` } -@method_id(101,) fun `get_-1` (): int {return-1;} -@method_id(102,) fun `get_--1` (): int {return--1;} -@method_id(103,) fun `get_---1`(): int {return---1;} -@method_id(104,) fun `get_+++1`(): int {return+++1;} -@method_id(105,) fun `get_+-+1`(): int {return+-+1;} +@method_id(101,) fun `get_-1` (): int {return-1} +@method_id(102,) fun `get_--1` (): int {return--1} +@method_id(103,) fun `get_---1`(): int {return---1} +@method_id(104,) fun `get_+++1`(): int {return+++1} +@method_id(105,) fun `get_+-+1`(): int {return+-+1} global `some()var`:int; @@ -26,7 +26,7 @@ global `some()var`:int; return[ (just10()-3==just10()-(4)--1)|((2==2)&(eq(eq(10)) -3==just10()--13)), ((flags&0xFF)!=0) - ]; + ] } @method_id(113)fun`unary+bitwise-constant`():[int,int,int]{ @@ -48,7 +48,7 @@ fun add3(a: int, b: int, c: int) { return a+b+c; } } fun `load:u32`(mutate self: slice): int { - return self.loadUint(32); + return self.loadUint(32) } @method_id(116) fun `call_~_via_backticks`():[int,int,int,int] { diff --git a/tolk-tester/tests/nullable-types.tolk b/tolk-tester/tests/nullable-types.tolk index f5ee8b550..74e8f1b1b 100644 --- a/tolk-tester/tests/nullable-types.tolk +++ b/tolk-tester/tests/nullable-types.tolk @@ -3,8 +3,8 @@ type MIntN = int?; fun getNullable4(): MIntN { return 4; } fun getNullableIntNull(): int? asm "PUSHNULL"; -fun eqInt(x: int) { return x; } -fun eq(x: T) { return x; } +fun eqInt(x: int) { return x } +fun eq(x: T) { return x } fun unwrap(x: T?): T { return x!; } fun intOr0(x: int?): int { return null == x ? 0 : x!; } diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index ffedcb441..62477c135 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -47,8 +47,8 @@ fun test3(): int { return x; } if (random() > -1) { - if (y == null) { return -1; } - else { return y; } + if (y == null) { return -1 } + else { return y } } return 0; } @@ -344,8 +344,8 @@ fun test33(): int { fun test34() { var (x, y) = (getNullableInt(), getNullableInt()); - if (random()) { throw (x = 1, y = 2); } - else { throw (x = 3, y = (1, getNullableInt()!).1); } + if (random()) { throw (x = 1, y = 2) } + else { throw (x = 3, y = (1, getNullableInt()!).1) } return x + y; } diff --git a/tolk-tester/tests/unbalanced-ret.tolk b/tolk-tester/tests/unbalanced-ret.tolk index eca5dd2a5..05c7f46d9 100644 --- a/tolk-tester/tests/unbalanced-ret.tolk +++ b/tolk-tester/tests/unbalanced-ret.tolk @@ -4,10 +4,10 @@ fun main(x: int): (int, int) { x *= 2; y += 1; if (x == -10) { - return (111, 0); + return (111, 0) } } - return (x + 1, y); + return (x + 1, y) } @inline @@ -41,7 +41,7 @@ fun bar2(x: int, y: int): (int, int) { y = foo2(y); x *= 2; if (x == -10) { - return (111, y); + return (111, y) } } return (x + 1, y); diff --git a/tolk-tester/tests/union-types-tests.tolk b/tolk-tester/tests/union-types-tests.tolk index e451f13dd..8cd97f941 100644 --- a/tolk-tester/tests/union-types-tests.tolk +++ b/tolk-tester/tests/union-types-tests.tolk @@ -83,11 +83,11 @@ fun test2() { fun test3(case: int): int | slice | null { var a: int | slice | null = 4; if (case == 0) { - a = getSlice32(); + a = getSlice32() } else if (case == 1) { - a = 10; + a = 10 } else { - a = null; + a = null } return a; } @@ -99,8 +99,8 @@ fun test4() { (5 as int8) as int8 | int8, true as bool | bool, 5 as int | int | (int | slice), - (5 as int | int | (int | slice)) as int | slice | null - ); + (5 as int | int | (int | slice)) as int | slice | null, + ) } @method_id(105) @@ -119,19 +119,19 @@ fun test6(case: int): int | slice | builder | null { a = null as slice? | builder; __expect_type(a, "null | slice | builder"); __expect_type(a!, "slice | builder"); - return a; + return a } if (case == 2) { __expect_type(a = 2 as MInt, "MInt"); __expect_type(a, "int"); a = 2; __expect_type(a, "int"); - return a + 0; + return a + 0 } if (case == 3) { a = null as slice?; __expect_type(a, "slice?"); - return a; + return a } a = null as MNull; __expect_type(a, "null"); @@ -159,7 +159,7 @@ fun test8() { (2, 3) as Pair2Or3 | null, (2, 3) as null | Pair2Or3, null as Pair2Or3? - ); + ) } @method_id(109) @@ -177,7 +177,7 @@ fun test9() { fun test10() { var t = getSlice32() as (int, (int, int)?, (int?, int)?) | slice; if (10 > 3) { - t = (1, null, null) as (int, (int, int)?, (int?, int)?); + t = (1, null, null) as (int, (int, int)?, (int?, int)?) } return t; } @@ -222,9 +222,9 @@ fun test21() { __expect_type(a, "never"); } if (a is slice || a is int) { - __expect_type(a, "slice | int"); + __expect_type(a, "slice | int") } else { - __expect_type(a, "never"); + __expect_type(a, "never") } if (a is int || a is slice || a is builder) { __expect_type(a, "int | slice"); @@ -238,13 +238,13 @@ fun test21() { @method_id(122) fun test22(a: int, b: int | slice) { var result = 0; - if (a is int) { result += 1; } - if (a is int || a is slice) { result += 2; } - if (a is builder || a is slice) { result += 3; } - if (b is int) { result += 10; } - if ((b is int) | (b is slice)) { result += 11; } - if ((b is builder) | (b is int)) { result += 12; } - return result; + if (a is int) { result += 1 } + if (a is int || a is slice) { result += 2 } + if (a is builder || a is slice) { result += 3 } + if (b is int) { result += 10 } + if ((b is int) | (b is slice)) { result += 11 } + if ((b is builder) | (b is int)) { result += 12 } + return result } @method_id(123) @@ -268,7 +268,7 @@ fun test23(p2: bool) { return p; } -fun eqUnion2(v: T1 | T2): T2 | T1 { return v; } +fun eqUnion2(v: T1 | T2): T2 | T1 { return v } fun detectUnionSide(v: T1 | T2) { if (v is T1) { @@ -371,11 +371,13 @@ fun test31() { var t5: (int, int) | int | null = 5 as int?; var t6: (int, int) | int | null = null as (int, int)?; var t7: (int, int) | int | null = (1, 2) as (int, int)?; - __expect_type(t1, "null"); - __expect_type(t2, "int"); - __expect_type(t3, "(int, int)"); - __expect_type(t4, "int?"); - __expect_type(t6, "(int, int)?"); + { + __expect_type(t1, "null"); + __expect_type(t2, "int"); + __expect_type(t3, "(int, int)"); + __expect_type(t4, "int?"); + __expect_type(t6, "(int, int)?") + } return (t1, 777, t2, 777, t3, 777, t4, 777, t5, 777, t6, 777, t7); } @@ -469,9 +471,9 @@ fun test39() { @method_id(140) fun test40(x: int): SomeIntBits { - if (x < 10) { return x as int8; } - if (x < 20) { return x as int9; } - if (x < 30) { return x as int10; } + if (x < 10) { return x as int8 } + if (x < 20) { return x as int9 } + if (x < 30) { return x as int10 } return x as uint64; } @@ -524,7 +526,7 @@ fun checkSimple(a: int | slice | builder) { int => 1, slice => 2, builder => 3, - }; + } } @method_id(147) @@ -539,7 +541,7 @@ fun checkGeneric3(a: T1 | T2 | T3): (int, T1 | T2?) { T1 => (1, a), T2 => (2, match(a) { T2 => a }), T3 => (3, null), - }; + } } type IntOrInt8OrInt16 = int | int8 | int16; @@ -561,9 +563,9 @@ fun test49(a: slice? | int) { return match (a) { slice => 1, null => 2, - }; + } } - return a + 0; + return a + 0 } type VeryComplexType = int | (MInt, int)? | builder? | int; @@ -592,7 +594,7 @@ fun test50() { __expect_type(b, "int?"); b is int && b!is int && b!!is int; if (match(b) { int => 1, null => null } !is int) { - return 100; + return 100 } return (b! + (match (a) { int => a, slice => a, null => a } as int|slice?).test49()); } @@ -610,7 +612,7 @@ fun test51(init: int) { a = a > 10 ? beginCell().storeInt(init, 32).endCell().beginParse() : 10; __expect_type(a, "int | slice"); if (a is slice) { - a = a.loadInt(32); + a = a.loadInt(32) } } slice => { @@ -632,7 +634,7 @@ fun test52(init: int): int | builder { return match (init as int?) { int => init, null => beginCell() - }; + } } fun inc53(mutate x: int) { @@ -659,7 +661,7 @@ fun test53(a: int | builder) { match (var (a, (b, c)) = (10, (20, 30))) { (int, Pair2) => {} } - return cc; + return cc } @method_id(154) diff --git a/tolk-tester/tests/var-apply-tests.tolk b/tolk-tester/tests/var-apply-tests.tolk index 11241bf13..3411f4a06 100644 --- a/tolk-tester/tests/var-apply-tests.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -4,11 +4,11 @@ type MBuilder = builder; type CallbackIntToInt = int -> int; fun getBeginCell() { - return beginCell; + return beginCell } fun getBeginParse() { - return beginParse; + return beginParse } @method_id(101) @@ -76,25 +76,25 @@ fun testVarApplyInTernary() { t.tuplePush(code); } try { - t.tuplePush(demo_handler(0xF4, 122, 100, 200)); + t.tuplePush(demo_handler(0xF4, 122, 100, 200)) } catch(code) { - t.tuplePush(code); + t.tuplePush(code) } try { - t.tuplePush(demo_handler(0xF2, 122, 10, 10)); + t.tuplePush(demo_handler(0xF2, 122, 10, 10)) } catch(code) { - t.tuplePush(code); + t.tuplePush(code) } try { - t.tuplePush(demo_handler(0xF2, 123, 10, 10)); + t.tuplePush(demo_handler(0xF2, 123, 10, 10)) } catch(code) { - t.tuplePush(code); + t.tuplePush(code) } return t; } fun always_throw2(x: int) { - throw 239 + x; + throw 239 + x } global global_f: MInt -> void; @@ -143,7 +143,7 @@ fun testIndexedAccessApply() { return functions2.0(functions1.1(b)).loadInt(32); } -fun getNullable4(): int? { return 4; } +fun getNullable4(): int? { return 4 } fun myBeginCell(): builder? asm "NEWC"; @method_id(108) @@ -211,15 +211,15 @@ fun testCallbackAlias(secondNull: bool) { return (adder1(10), adder2 == null ? -1 : adder2(10)); } -fun justGetTensor2() { return (2, 3); } +fun justGetTensor2() { return (2, 3) } @method_id(114) fun testCallbackUnion(call1st: bool): (int, int) | int { var cb: (int -> int) | (() -> (MInt, int)) = call1st ? justGetTensor2 : justAdd2; if (cb is () -> (int, int)) { - return cb(); + return cb() } - return cb(10) as int; + return cb(10) as int } fun main() {} diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 36e6ce5a2..ec9d689d4 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -14,7 +14,6 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#include "ast-from-tokens.h" #include "ast.h" #include "type-system.h" #include "platform-utils.h" @@ -296,7 +295,7 @@ static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable) { lex.unexpected("variable name"); } -static AnyExprV parse_local_vars_declaration_assignment(Lexer& lex, bool is_statement) { +static AnyExprV parse_local_vars_declaration_assignment(Lexer& lex) { SrcLocation loc = lex.cur_location(); bool is_immutable = lex.tok() == tok_val; lex.next(); @@ -311,9 +310,6 @@ static AnyExprV parse_local_vars_declaration_assignment(Lexer& lex, bool is_stat if (lex.tok() == tok_comma) { lex.error("multiple declarations are not allowed, split variables on separate lines"); } - if (is_statement) { - lex.expect(tok_semicolon, "`;`"); - } return createV(loc, lhs, rhs); } @@ -392,7 +388,7 @@ static V parse_maybe_instantiationTs_after_identifier(L // `throw code` / `throw (code)` / `throw (code, arg)` // it's technically a statement (can't occur "in any place of expression"), // but inside `match` arm it can appear without semicolon: `pattern => throw 123` -static AnyV parse_throw_expression(Lexer& lex, bool is_statement) { +static AnyV parse_throw_expression(Lexer& lex) { SrcLocation loc = lex.cur_location(); lex.expect(tok_throw, "`throw`"); @@ -412,9 +408,6 @@ static AnyV parse_throw_expression(Lexer& lex, bool is_statement) { thrown_arg = createV(loc); } - if (is_statement) { - lex.expect(tok_semicolon, "`;`"); - } return createV(loc, thrown_code, thrown_arg); } @@ -453,17 +446,17 @@ static V parse_match_arm(Lexer& lex) { AnyExprV body; if (lex.tok() == tok_opbrace) { // pattern => { ... } - AnyV v_seq = parse_statement(lex); - body = createV(v_seq->loc, v_seq); + AnyV v_block = parse_statement(lex); + body = createV(v_block->loc, v_block); } else if (lex.tok() == tok_throw) { // pattern => throw 123 (allow without braces) - AnyV v_throw = parse_throw_expression(lex, false); - AnyV v_seq = createV(v_throw->loc, v_throw->loc, {v_throw}); - body = createV(v_seq->loc, v_seq); + AnyV v_throw = parse_throw_expression(lex); + AnyV v_block = createV(v_throw->loc, v_throw->loc, {v_throw}); + body = createV(v_block->loc, v_block); } else if (lex.tok() == tok_return) { // pattern => return 123 (allow without braces, like throw) lex.next(); AnyV v_return = createV(lex.cur_location(), parse_expr(lex)); - AnyV v_seq = createV(v_return->loc, v_return->loc, {v_return}); - body = createV(v_seq->loc, v_seq); + AnyV v_block = createV(v_return->loc, v_return->loc, {v_return}); + body = createV(v_block->loc, v_block); } else { body = parse_expr(lex); } @@ -480,7 +473,7 @@ static V parse_match_expression(Lexer& lex) { lex.expect(tok_oppar, "`(`"); AnyExprV subject = lex.tok() == tok_var || lex.tok() == tok_val // `match (var x = rhs)` - ? parse_local_vars_declaration_assignment(lex, false) + ? parse_local_vars_declaration_assignment(lex) : parse_expr(lex); lex.expect(tok_clpar, "`)`"); @@ -819,16 +812,24 @@ AnyExprV parse_expr(Lexer& lex) { return parse_expr10(lex); } -static V parse_sequence(Lexer& lex) { +static V parse_block_statement(Lexer& lex) { SrcLocation loc = lex.cur_location(); lex.expect(tok_opbrace, "`{`"); std::vector items; while (lex.tok() != tok_clbrace) { - items.push_back(parse_statement(lex)); + AnyV v = parse_statement(lex); + items.push_back(v); + if (lex.tok() == tok_clbrace) { + break; + } + bool does_end_with_brace = v->type == ast_if_statement || v->type == ast_while_statement || v->type == ast_match_expression || v->type == ast_try_catch_statement || v->type == ast_repeat_statement || v->type == ast_block_statement; + if (!does_end_with_brace) { + lex.expect(tok_semicolon, "`;`"); + } } SrcLocation loc_end = lex.cur_location(); lex.expect(tok_clbrace, "`}`"); - return createV(loc, loc_end, std::move(items)); + return createV(loc, loc_end, std::move(items)); } static AnyV parse_return_statement(Lexer& lex) { @@ -837,7 +838,6 @@ static AnyV parse_return_statement(Lexer& lex) { AnyExprV child = lex.tok() == tok_semicolon // `return;` actually means "nothing" (inferred as void) ? createV(lex.cur_location()) : parse_expr(lex); - lex.expect(tok_semicolon, "`;`"); return createV(loc, child); } @@ -849,18 +849,18 @@ static AnyV parse_if_statement(Lexer& lex) { AnyExprV cond = parse_expr(lex); lex.expect(tok_clpar, "`)`"); - V if_body = parse_sequence(lex); - V else_body = nullptr; + V if_body = parse_block_statement(lex); + V else_body = nullptr; if (lex.tok() == tok_else) { // else if(e) { } or else { } lex.next(); if (lex.tok() == tok_if) { AnyV v_inner_if = parse_if_statement(lex); - else_body = createV(v_inner_if->loc, lex.cur_location(), {v_inner_if}); + else_body = createV(v_inner_if->loc, lex.cur_location(), {v_inner_if}); } else { - else_body = parse_sequence(lex); + else_body = parse_block_statement(lex); } } else { // no 'else', create empty block - else_body = createV(lex.cur_location(), lex.cur_location(), {}); + else_body = createV(lex.cur_location(), lex.cur_location(), {}); } return createV(loc, false, cond, if_body, else_body); } @@ -871,7 +871,7 @@ static AnyV parse_repeat_statement(Lexer& lex) { lex.expect(tok_oppar, "`(`"); AnyExprV cond = parse_expr(lex); lex.expect(tok_clpar, "`)`"); - V body = parse_sequence(lex); + V body = parse_block_statement(lex); return createV(loc, cond, body); } @@ -881,19 +881,18 @@ static AnyV parse_while_statement(Lexer& lex) { lex.expect(tok_oppar, "`(`"); AnyExprV cond = parse_expr(lex); lex.expect(tok_clpar, "`)`"); - V body = parse_sequence(lex); + V body = parse_block_statement(lex); return createV(loc, cond, body); } static AnyV parse_do_while_statement(Lexer& lex) { SrcLocation loc = lex.cur_location(); lex.expect(tok_do, "`do`"); - V body = parse_sequence(lex); + V body = parse_block_statement(lex); lex.expect(tok_while, "`while`"); lex.expect(tok_oppar, "`(`"); AnyExprV cond = parse_expr(lex); lex.expect(tok_clpar, "`)`"); - lex.expect(tok_semicolon, "`;`"); return createV(loc, body, cond); } @@ -935,14 +934,13 @@ static AnyV parse_assert_statement(Lexer& lex) { thrown_code = parse_expr(lex); } - lex.expect(tok_semicolon, "`;`"); return createV(loc, cond, thrown_code); } static AnyV parse_try_catch_statement(Lexer& lex) { SrcLocation loc = lex.cur_location(); lex.expect(tok_try, "`try`"); - V try_body = parse_sequence(lex); + V try_body = parse_block_statement(lex); std::vector catch_args; lex.expect(tok_catch, "`catch`"); @@ -963,7 +961,7 @@ static AnyV parse_try_catch_statement(Lexer& lex) { } V catch_expr = createV(catch_loc, std::move(catch_args)); - V catch_body = parse_sequence(lex); + V catch_body = parse_block_statement(lex); return createV(loc, try_body, catch_expr, catch_body); } @@ -971,9 +969,9 @@ AnyV parse_statement(Lexer& lex) { switch (lex.tok()) { case tok_var: // `var x = 0` is technically an expression, but can not appear in "any place", case tok_val: // only as a separate declaration - return parse_local_vars_declaration_assignment(lex, true); + return parse_local_vars_declaration_assignment(lex); case tok_opbrace: - return parse_sequence(lex); + return parse_block_statement(lex); case tok_return: return parse_return_statement(lex); case tok_if: @@ -985,31 +983,23 @@ AnyV parse_statement(Lexer& lex) { case tok_while: return parse_while_statement(lex); case tok_throw: - return parse_throw_expression(lex, true); + return parse_throw_expression(lex); case tok_assert: return parse_assert_statement(lex); case tok_try: return parse_try_catch_statement(lex); - case tok_semicolon: { - SrcLocation loc = lex.cur_location(); - lex.next(); - return createV(loc); - } + case tok_semicolon: + return createV(lex.cur_location()); case tok_break: case tok_continue: lex.error("break/continue from loops are not supported yet"); - default: { - AnyExprV expr = parse_expr(lex); - if (expr->type != ast_match_expression) { // match statement, as if statement, doesn't require semicolon after - lex.expect(tok_semicolon, "`;`"); - } - return expr; - } + default: + return parse_expr(lex); } } static AnyV parse_func_body(Lexer& lex) { - return parse_sequence(lex); + return parse_block_statement(lex); } static AnyV parse_asm_func_body(Lexer& lex, V param_list) { diff --git a/tolk/ast-from-tokens.h b/tolk/ast-from-tokens.h deleted file mode 100644 index 39574f9c2..000000000 --- a/tolk/ast-from-tokens.h +++ /dev/null @@ -1,25 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . -*/ -#pragma once - -#include "fwd-declarations.h" - -namespace tolk { - -AnyV parse_src_file_to_ast(const SrcFile* file); - -} // namespace tolk diff --git a/tolk/ast-replacer.h b/tolk/ast-replacer.h index 5efae1e2f..417ff411c 100644 --- a/tolk/ast-replacer.h +++ b/tolk/ast-replacer.h @@ -62,7 +62,7 @@ class ASTReplacer { GNU_ATTRIBUTE_ALWAYS_INLINE AnyExprV replace_children(const ASTExprBlockOfStatements* v) { auto* v_mutable = const_cast(v); - v_mutable->child_sequence = replace(v_mutable->child_sequence->as()); + v_mutable->child_block_statement = replace(v_mutable->child_block_statement->as()); return v_mutable; } @@ -121,7 +121,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { virtual AnyExprV replace(V v) { return replace_children(v); } // statements virtual AnyV replace(V v) { return replace_children(v); } - virtual AnyV replace(V v) { return replace_children(v); } + virtual AnyV replace(V v) { return replace_children(v); } virtual AnyV replace(V v) { return replace_children(v); } virtual AnyV replace(V v) { return replace_children(v); } virtual AnyV replace(V v) { return replace_children(v); } @@ -168,7 +168,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { AnyV replace(AnyV v) final { switch (v->type) { case ast_empty_statement: return replace(v->as()); - case ast_sequence: return replace(v->as()); + case ast_block_statement: return replace(v->as()); case ast_return_statement: return replace(v->as()); case ast_if_statement: return replace(v->as()); case ast_repeat_statement: return replace(v->as()); diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index 9f0738d3f..66781f1f0 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -62,7 +62,7 @@ class ASTReplicatorFunction : public ASTReplicator { return createV(v->loc, clone(v->get_expr())); } virtual V clone(V v) { - return createV(v->loc, clone(v->get_sequence())); + return createV(v->loc, clone(v->get_block_statement())); } virtual V clone(V v) { return createV(v->loc, clone(v->get_items())); @@ -142,8 +142,8 @@ class ASTReplicatorFunction : public ASTReplicator { virtual V clone(V v) { return createV(v->loc); } - virtual V clone(V v) { - return createV(v->loc, v->loc_end, clone(v->get_items())); + virtual V clone(V v) { + return createV(v->loc, v->loc_end, clone(v->get_items())); } virtual V clone(V v) { return createV(v->loc, clone(v->get_return_value())); @@ -228,7 +228,7 @@ class ASTReplicatorFunction : public ASTReplicator { AnyV clone(AnyV v) final { switch (v->type) { case ast_empty_statement: return clone(v->as()); - case ast_sequence: return clone(v->as()); + case ast_block_statement: return clone(v->as()); case ast_return_statement: return clone(v->as()); case ast_if_statement: return clone(v->as()); case ast_repeat_statement: return clone(v->as()); @@ -263,7 +263,7 @@ class ASTReplicatorFunction : public ASTReplicator { v_function->loc, clone(v_function->get_identifier()), clone(v_function->get_param_list()), - clone(v_function->get_body()->as()), + clone(v_function->get_body()->as()), clone(v_function->declared_return_type), v_function->genericsT_list, v_function->method_id, diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index 33082a6e9..a232888f2 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -63,7 +63,7 @@ class ASTStringifier final : public ASTVisitor { {ast_match_arm, "ast_match_arm"}, // statements {ast_empty_statement, "ast_empty_statement"}, - {ast_sequence, "ast_sequence"}, + {ast_block_statement, "ast_block_statement"}, {ast_return_statement, "ast_return_statement"}, {ast_if_statement, "ast_if_statement"}, {ast_repeat_statement, "ast_repeat_statement"}, @@ -177,8 +177,8 @@ class ASTStringifier final : public ASTVisitor { std::string prefix = v->as()->is_negated ? "!is " : "is "; return prefix + v->as()->rhs_type->as_human_readable(); } - case ast_sequence: - return "↓" + std::to_string(v->as()->get_items().size()); + case ast_block_statement: + return "↓" + std::to_string(v->as()->get_items().size()); case ast_instantiationT_item: return v->as()->substituted_type->as_human_readable(); case ast_if_statement: @@ -291,7 +291,7 @@ class ASTStringifier final : public ASTVisitor { case ast_match_arm: return handle_vertex(v->as()); // statements case ast_empty_statement: return handle_vertex(v->as()); - case ast_sequence: return handle_vertex(v->as()); + case ast_block_statement: return handle_vertex(v->as()); case ast_return_statement: return handle_vertex(v->as()); case ast_if_statement: return handle_vertex(v->as()); case ast_repeat_statement: return handle_vertex(v->as()); diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h index 2f5f269f7..eac7e98ef 100644 --- a/tolk/ast-visitor.h +++ b/tolk/ast-visitor.h @@ -57,7 +57,7 @@ class ASTVisitor { } GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTExprBlockOfStatements* v) { - visit_children(v->child_sequence->as()); + visit_children(v->child_block_statement->as()); } GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTStatementUnary* v) { @@ -120,7 +120,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { virtual void visit(V v) { return visit_children(v); } // statements virtual void visit(V v) { return visit_children(v); } - virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } @@ -162,7 +162,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { case ast_match_arm: return visit(v->as()); // statements case ast_empty_statement: return visit(v->as()); - case ast_sequence: return visit(v->as()); + case ast_block_statement: return visit(v->as()); case ast_return_statement: return visit(v->as()); case ast_if_statement: return visit(v->as()); case ast_repeat_statement: return visit(v->as()); diff --git a/tolk/ast.cpp b/tolk/ast.cpp index d740970da..9fadd5f6c 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -200,7 +200,7 @@ void Vertex::assign_fun_ref(FunctionPtr fun_ref) { this->fun_ref = fun_ref; } -void Vertex::assign_first_unreachable(AnyV first_unreachable) { +void Vertex::assign_first_unreachable(AnyV first_unreachable) { this->first_unreachable = first_unreachable; } diff --git a/tolk/ast.h b/tolk/ast.h index fea11ebcb..9dce458ec 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -95,7 +95,7 @@ enum ASTNodeType { ast_match_arm, // statements ast_empty_statement, - ast_sequence, + ast_block_statement, ast_return_statement, ast_if_statement, ast_repeat_statement, @@ -266,10 +266,10 @@ struct ASTExprBlockOfStatements : ASTNodeExpressionBase { friend class ASTReplacer; protected: - AnyV child_sequence; + AnyV child_block_statement; - ASTExprBlockOfStatements(ASTNodeType type, SrcLocation loc, AnyV child_sequence) - : ASTNodeExpressionBase(type, loc), child_sequence(child_sequence) {} + ASTExprBlockOfStatements(ASTNodeType type, SrcLocation loc, AnyV child_block_statement) + : ASTNodeExpressionBase(type, loc), child_block_statement(child_block_statement) {} }; struct ASTStatementUnary : ASTNodeStatementBase { @@ -377,10 +377,10 @@ template<> // it can occur only in special places within the input code, not anywhere // example: `match (intV) { 0 => { ... } }` rhs of 0 is braced expression struct Vertex final : ASTExprBlockOfStatements { - auto get_sequence() const { return child_sequence->as(); } + auto get_block_statement() const { return child_block_statement->as(); } - Vertex(SrcLocation loc, AnyV child_sequence) - : ASTExprBlockOfStatements(ast_braced_expression, loc, child_sequence) {} + Vertex(SrcLocation loc, AnyV child_block_statement) + : ASTExprBlockOfStatements(ast_braced_expression, loc, child_block_statement) {} }; template<> @@ -797,10 +797,10 @@ struct Vertex final : ASTStatementVararg { }; template<> -// ast_sequence is "some sequence of statements" -// example: function body is a sequence -// example: do while body is a sequence -struct Vertex final : ASTStatementVararg { +// ast_block_statement is "{ statement; statement }" (trailing semicolon is optional) +// example: function body is a block +// example: do while body is a block +struct Vertex final : ASTStatementVararg { SrcLocation loc_end; AnyV first_unreachable = nullptr; @@ -811,7 +811,7 @@ struct Vertex final : ASTStatementVararg { void assign_first_unreachable(AnyV first_unreachable); Vertex(SrcLocation loc, SrcLocation loc_end, std::vector&& items) - : ASTStatementVararg(ast_sequence, loc, std::move(items)) + : ASTStatementVararg(ast_block_statement, loc, std::move(items)) , loc_end(loc_end) {} }; @@ -836,10 +836,10 @@ struct Vertex final : ASTStatementVararg { bool is_ifnot; // if(!cond), to generate more optimal fift code AnyExprV get_cond() const { return child_as_expr(0); } - auto get_if_body() const { return children.at(1)->as(); } - auto get_else_body() const { return children.at(2)->as(); } // always exists (when else omitted, it's empty) + auto get_if_body() const { return children.at(1)->as(); } + auto get_else_body() const { return children.at(2)->as(); } // always exists (when else omitted, it's empty) - Vertex(SrcLocation loc, bool is_ifnot, AnyExprV cond, V if_body, V else_body) + Vertex(SrcLocation loc, bool is_ifnot, AnyExprV cond, V if_body, V else_body) : ASTStatementVararg(ast_if_statement, loc, {cond, if_body, else_body}) , is_ifnot(is_ifnot) {} }; @@ -849,9 +849,9 @@ template<> // example: `repeat (10) { ... }` struct Vertex final : ASTStatementVararg { AnyExprV get_cond() const { return child_as_expr(0); } - auto get_body() const { return children.at(1)->as(); } + auto get_body() const { return children.at(1)->as(); } - Vertex(SrcLocation loc, AnyExprV cond, V body) + Vertex(SrcLocation loc, AnyExprV cond, V body) : ASTStatementVararg(ast_repeat_statement, loc, {cond, body}) {} }; @@ -860,9 +860,9 @@ template<> // example: `while (x > 0) { ... }` struct Vertex final : ASTStatementVararg { AnyExprV get_cond() const { return child_as_expr(0); } - auto get_body() const { return children.at(1)->as(); } + auto get_body() const { return children.at(1)->as(); } - Vertex(SrcLocation loc, AnyExprV cond, V body) + Vertex(SrcLocation loc, AnyExprV cond, V body) : ASTStatementVararg(ast_while_statement, loc, {cond, body}) {} }; @@ -870,10 +870,10 @@ template<> // ast_do_while_statement is a standard "do while" loop // example: `do { ... } while (x > 0);` struct Vertex final : ASTStatementVararg { - auto get_body() const { return children.at(0)->as(); } + auto get_body() const { return children.at(0)->as(); } AnyExprV get_cond() const { return child_as_expr(1); } - Vertex(SrcLocation loc, V body, AnyExprV cond) + Vertex(SrcLocation loc, V body, AnyExprV cond) : ASTStatementVararg(ast_do_while_statement, loc, {body, cond}) {} }; @@ -907,11 +907,11 @@ template<> // there are two formal "arguments" of catch: excNo and arg, but both can be omitted // when omitted, they are stored as underscores, so len of a catch tensor is always 2 struct Vertex final : ASTStatementVararg { - auto get_try_body() const { return children.at(0)->as(); } + auto get_try_body() const { return children.at(0)->as(); } auto get_catch_expr() const { return children.at(1)->as(); } // (excNo, arg), always len 2 - auto get_catch_body() const { return children.at(2)->as(); } + auto get_catch_body() const { return children.at(2)->as(); } - Vertex(SrcLocation loc, V try_body, V catch_expr, V catch_body) + Vertex(SrcLocation loc, V try_body, V catch_expr, V catch_body) : ASTStatementVararg(ast_try_catch_statement, loc, {try_body, catch_expr, catch_body}) {} }; @@ -1049,7 +1049,7 @@ struct Vertex final : ASTOtherVararg { int get_num_params() const { return children.at(1)->as()->size(); } auto get_param_list() const { return children.at(1)->as(); } auto get_param(int i) const { return children.at(1)->as()->get_param(i); } - AnyV get_body() const { return children.at(2); } // ast_sequence / ast_asm_body + AnyV get_body() const { return children.at(2); } // ast_block_statement / ast_asm_body FunctionPtr fun_ref = nullptr; // filled after register TypePtr declared_return_type; // filled at ast parsing; if unspecified (nullptr), means "auto infer" @@ -1058,7 +1058,7 @@ struct Vertex final : ASTOtherVararg { int flags; // from enum in FunctionData bool is_asm_function() const { return children.at(2)->type == ast_asm_body; } - bool is_code_function() const { return children.at(2)->type == ast_sequence; } + bool is_code_function() const { return children.at(2)->type == ast_block_statement; } bool is_builtin_function() const { return children.at(2)->type == ast_empty_statement; } Vertex* mutate() const { return const_cast(this); } diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index bba79ea92..3778b8cab 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -1259,7 +1259,7 @@ static std::vector process_braced_expression(V // `{ ... }` used as an expression can not return a value currently (there is no syntax in a language) // that's why it can appear only in special places, and its usage correctness has been checked tolk_assert(v->inferred_type == TypeDataVoid::create() || v->inferred_type == TypeDataNever::create()); - process_any_statement(v->get_sequence(), code); + process_any_statement(v->get_block_statement(), code); static_cast(target_type); return {}; } @@ -1392,7 +1392,7 @@ std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr targ } -static void process_sequence(V v, CodeBlob& code) { +static void process_block_statement(V v, CodeBlob& code) { for (AnyV item : v->get_items()) { process_any_statement(item, code); } @@ -1582,8 +1582,8 @@ static void append_implicit_return_statement(SrcLocation loc_end, CodeBlob& code void process_any_statement(AnyV v, CodeBlob& code) { switch (v->type) { - case ast_sequence: - return process_sequence(v->as(), code); + case ast_block_statement: + return process_block_statement(v->as(), code); case ast_return_statement: return process_return_statement(v->as(), code); case ast_repeat_statement: @@ -1608,7 +1608,7 @@ void process_any_statement(AnyV v, CodeBlob& code) { } static void convert_function_body_to_CodeBlob(FunctionPtr fun_ref, FunctionBodyCode* code_body) { - auto v_body = fun_ref->ast_root->as()->get_body()->as(); + auto v_body = fun_ref->ast_root->as()->get_body()->as(); CodeBlob* blob = new CodeBlob{fun_ref->name, fun_ref->loc, fun_ref}; std::vector rvect_import; diff --git a/tolk/pipe-calc-rvalue-lvalue.cpp b/tolk/pipe-calc-rvalue-lvalue.cpp index 769d31c01..b910eface 100644 --- a/tolk/pipe-calc-rvalue-lvalue.cpp +++ b/tolk/pipe-calc-rvalue-lvalue.cpp @@ -238,7 +238,7 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { parent::visit(v); } - void visit(V v) override { + void visit(V v) override { MarkingState saved = enter_state(MarkingState::None); parent::visit(v); restore_state(saved); diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index b9bcdaced..4b0f8d53f 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -691,7 +691,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } } - void visit(V v) override { + void visit(V v) override { parent::visit(v); if (v->first_unreachable) { @@ -715,7 +715,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (fun_ref->is_implicit_return() && fun_ref->declared_return_type) { if (!fun_ref->declared_return_type->can_rhs_be_assigned(TypeDataVoid::create()) || fun_ref->does_return_self()) { - fire(fun_ref, v_function->get_body()->as()->loc_end, "missing return"); + fire(fun_ref, v_function->get_body()->as()->loc_end, "missing return"); } } } diff --git a/tolk/pipe-discover-parse-sources.cpp b/tolk/pipe-discover-parse-sources.cpp index d31348ba9..2f7c02d6d 100644 --- a/tolk/pipe-discover-parse-sources.cpp +++ b/tolk/pipe-discover-parse-sources.cpp @@ -25,7 +25,6 @@ */ #include "tolk.h" #include "ast.h" -#include "ast-from-tokens.h" #include "compiler-state.h" /* @@ -38,6 +37,8 @@ namespace tolk { +AnyV parse_src_file_to_ast(const SrcFile* file); + void pipeline_discover_and_parse_sources(const std::string& stdlib_filename, const std::string& entrypoint_filename) { G.all_src_files.locate_and_register_source_file(stdlib_filename, {}); G.all_src_files.locate_and_register_source_file(entrypoint_filename, {}); @@ -64,7 +65,7 @@ void pipeline_discover_and_parse_sources(const std::string& stdlib_filename, con } // todo #ifdef TOLK_PROFILING - lexer_measure_performance(G.all_src_files); + // lexer_measure_performance(G.all_src_files); } } // namespace tolk diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index c0c7c1d39..0e1c5af98 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -186,8 +186,8 @@ class InferTypesAndCallsAndFieldsVisitor final { // traverse children in any statement FlowContext process_any_statement(AnyV v, FlowContext&& flow) { switch (v->type) { - case ast_sequence: - return process_sequence(v->as(), std::move(flow)); + case ast_block_statement: + return process_block_statement(v->as(), std::move(flow)); case ast_return_statement: return process_return_statement(v->as(), std::move(flow)); case ast_if_statement: @@ -680,7 +680,7 @@ class InferTypesAndCallsAndFieldsVisitor final { ExprFlow infer_braced_expression(V v, FlowContext&& flow, bool used_as_condition) { // `{ ... }` used as an expression can not return a value currently (there is no syntax in a language) - flow = process_any_statement(v->get_sequence(), std::move(flow)); + flow = process_any_statement(v->get_block_statement(), std::move(flow)); assign_inferred_type(v, flow.is_unreachable() ? TypeDataNever::create() : TypeDataVoid::create()); return ExprFlow(std::move(flow), used_as_condition); } @@ -1125,7 +1125,7 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(flow), used_as_condition); } - FlowContext process_sequence(V v, FlowContext&& flow) { + FlowContext process_block_statement(V v, FlowContext&& flow) { // we'll print a warning if after some statement, control flow became unreachable // (but don't print a warning if it's already unreachable, for example we're inside always-false if) bool initially_unreachable = flow.is_unreachable(); @@ -1274,7 +1274,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (!body_end.is_unreachable()) { fun_ref->mutate()->assign_is_implicit_return(); if (fun_ref->declared_return_type == TypeDataNever::create()) { // `never` can only be declared, it can't be inferred - fire(fun_ref, v_function->get_body()->as()->loc_end, "a function returning `never` can not have a reachable endpoint"); + fire(fun_ref, v_function->get_body()->as()->loc_end, "a function returning `never` can not have a reachable endpoint"); } } @@ -1297,7 +1297,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } if (!body_end.is_unreachable() && inferred_return_type != TypeDataVoid::create()) { - fire(fun_ref, v_function->get_body()->as()->loc_end, "missing return"); + fire(fun_ref, v_function->get_body()->as()->loc_end, "missing return"); } if (has_void_returns && has_non_void_returns) { for (AnyExprV return_expr : return_statements) { diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index e76f5b479..363952cbd 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -190,7 +190,7 @@ static void register_function(V v) { std::cerr << "fun " << func_name << " : " << v->declared_return_type << std::endl; } - FunctionBody f_body = v->get_body()->type == ast_sequence ? static_cast(new FunctionBodyCode) : static_cast(new FunctionBodyAsm); + FunctionBody f_body = v->get_body()->type == ast_block_statement ? static_cast(new FunctionBodyCode) : static_cast(new FunctionBodyAsm); FunctionData* f_sym = new FunctionData(static_cast(func_name), v->loc, v->declared_return_type, std::move(parameters), 0, genericTs, nullptr, f_body, v); if (const auto* v_asm = v->get_body()->try_as()) { diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 0b116bbe8..797795afd 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -262,7 +262,7 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { current_scope.open_scope(v->loc); - parent::visit(v->get_sequence()); + parent::visit(v->get_block_statement()); current_scope.close_scope(v->loc); } @@ -326,7 +326,7 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { parent::visit(v->get_expr()); } - void visit(V v) override { + void visit(V v) override { if (v->empty()) { return; } @@ -377,13 +377,13 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { fun_ref->mutate()->assign_resolved_type(return_type); if (fun_ref->is_code_function()) { - auto v_seq = v->get_body()->as(); + auto v_block = v->get_body()->as(); current_scope.open_scope(v->loc); for (int i = 0; i < v->get_num_params(); ++i) { current_scope.add_local_var(&fun_ref->parameters[i]); } - parent::visit(v_seq); - current_scope.close_scope(v_seq->loc_end); + parent::visit(v_block); + current_scope.close_scope(v_block->loc_end); tolk_assert(current_scope.scopes.empty()); } From cadb66e9a3d403b56d9a5db36e87cbbc4f9ae49c Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sat, 5 Apr 2025 17:20:17 +0300 Subject: [PATCH 204/388] [Tolk] Bump version to v0.11 --- crypto/smartcont/tolk-stdlib/common.tolk | 2 +- crypto/smartcont/tolk-stdlib/gas-payments.tolk | 2 +- crypto/smartcont/tolk-stdlib/lisp-lists.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-dicts.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk | 2 +- tolk/tolk-version.h | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index dc0ec4a61..6b737ef3e 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -1,7 +1,7 @@ // Standard library for Tolk (LGPL licence). // It contains common functions that are available out of the box, the user doesn't have to import anything. // More specific functions are required to be imported explicitly, like "@stdlib/tvm-dicts". -tolk 0.10 +tolk 0.11 /// In Tolk v1.x there would be a type `map`. /// Currently, working with dictionaries is still low-level, with raw cells. diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk index 11e3db7ff..0e8305e1f 100644 --- a/crypto/smartcont/tolk-stdlib/gas-payments.tolk +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.10 +tolk 0.11 /** Gas and payment related primitives. diff --git a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk index fd6d2c894..a08b2c11c 100644 --- a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk +++ b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.10 +tolk 0.11 /** Lisp-style lists are nested 2-elements tuples: `(1, (2, (3, null)))` represents list `[1, 2, 3]`. diff --git a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk index 83fe705c4..00ae594c6 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.10 +tolk 0.11 /** Dictionaries are represented as `cell` data type (cells can store anything, dicts in particular). diff --git a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk index 48bacec6c..a7f158248 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.10 +tolk 0.11 /// Usually `c3` has a continuation initialized by the whole code of the contract. It is used for function calls. /// The primitive returns the current value of `c3`. diff --git a/tolk/tolk-version.h b/tolk/tolk-version.h index be161a393..b8f2893eb 100644 --- a/tolk/tolk-version.h +++ b/tolk/tolk-version.h @@ -18,6 +18,6 @@ namespace tolk { -constexpr const char* TOLK_VERSION = "0.10.0"; +constexpr const char* TOLK_VERSION = "0.11.0"; } // namespace tolk From 7b0881292472e9c8957f23dfd3636b0589c9f6bc Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 7 Apr 2025 11:22:19 +0300 Subject: [PATCH 205/388] Don't store storage dict hash in masterchain --- crypto/block/transaction.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index f24a03676..417ce0cf4 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -3273,8 +3273,9 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { } } + bool store_storage_dict_hash = cfg.store_storage_dict_hash && !account.is_masterchain(); if (storage_refs_changed || - (cfg.store_storage_dict_hash && !account.storage_dict_hash && account.storage_used.cells > 25)) { + (store_storage_dict_hash && !account.storage_dict_hash && account.storage_used.cells > 25)) { TD_PERF_COUNTER(transaction_storage_stat_b); td::Timer timer; if (!new_account_storage_stat && account.account_storage_stat) { @@ -3291,7 +3292,7 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { new_storage_used.cells = stats.get_total_cells() + 1; new_storage_used.bits = stats.get_total_bits() + new_storage_for_stat->size(); // TODO: think about this limit (25) - if (cfg.store_storage_dict_hash && new_storage_used.cells > 25) { + if (store_storage_dict_hash && new_storage_used.cells > 25) { auto r_hash = stats.get_dict_hash(); if (r_hash.is_error()) { LOG(ERROR) << "Cannot compute storage dict hash for account " << account.addr.to_hex() << ": " @@ -3308,7 +3309,7 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { new_storage_used.bits -= old_storage_for_stat->size(); new_storage_used.bits += new_storage_for_stat->size(); new_account_storage_stat = {}; - if (cfg.store_storage_dict_hash) { + if (store_storage_dict_hash) { new_storage_dict_hash = account.storage_dict_hash; } } From 73751cdda6e69453536cf2690794e069c1384c29 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 8 Apr 2025 08:55:45 +0300 Subject: [PATCH 206/388] Fix compilation error and test-db (#1609) * Fix compilation error * Fix error processing in load_cell * Decrease verbosity of certain ADNL logs --- adnl/adnl-ext-server.cpp | 2 +- adnl/adnl-peer.cpp | 4 ++-- crypto/vm/db/DynamicBagOfCellsDb.cpp | 17 ++++++++++++----- tdnet/td/net/TcpListener.cpp | 2 +- validator/db/celldb.cpp | 2 +- 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/adnl/adnl-ext-server.cpp b/adnl/adnl-ext-server.cpp index 162a53afb..627a25cb5 100644 --- a/adnl/adnl-ext-server.cpp +++ b/adnl/adnl-ext-server.cpp @@ -31,7 +31,7 @@ td::Status AdnlInboundConnection::process_packet(td::BufferSlice data) { td::PromiseCreator::lambda([SelfId = actor_id(this), query_id = f->query_id_](td::Result R) { if (R.is_error()) { auto S = R.move_as_error(); - LOG(WARNING) << "failed ext query: " << S; + LOG(INFO) << "failed ext query: " << S; } else { auto B = create_tl_object(query_id, R.move_as_ok()); td::actor::send_closure(SelfId, &AdnlInboundConnection::send, serialize_tl_object(B, true)); diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp index ab4600581..88a38f7b0 100644 --- a/adnl/adnl-peer.cpp +++ b/adnl/adnl-peer.cpp @@ -598,11 +598,11 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessagePart &messa respond_with_nop(); auto size = message.total_size(); if (size > huge_packet_max_size()) { - VLOG(ADNL_WARNING) << this << ": dropping too big huge message: size=" << size; + VLOG(ADNL_INFO) << this << ": dropping too big huge message: size=" << size; return; } if (message.hash().is_zero()) { - VLOG(ADNL_WARNING) << this << ": dropping huge message with zero hash"; + VLOG(ADNL_INFO) << this << ": dropping huge message with zero hash"; return; } if (message.hash() != huge_message_hash_) { diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp index 1a30f76d4..697e4d67f 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDb.cpp @@ -138,8 +138,18 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat return td::Status::OK(); } td::Result> load_cell(td::Slice hash) override { - TRY_RESULT(loaded_cell, get_cell_info_force(hash).cell->load_cell()); - return std::move(loaded_cell.data_cell); + auto info = hash_table_.get_if_exists(hash); + if (info && info->sync_with_db) { + TRY_RESULT(loaded_cell, info->cell->load_cell()); + return std::move(loaded_cell.data_cell); + } + TRY_RESULT(res, loader_->load(hash, true, *this)); + if (res.status != CellLoader::LoadResult::Ok) { + return td::Status::Error("cell not found"); + } + Ref cell = res.cell(); + hash_table_.apply(hash, [&](CellInfo &info) { update_cell_info_loaded(info, hash, std::move(res)); }); + return cell; } td::Result> load_root(td::Slice hash) override { return load_cell(hash); @@ -195,9 +205,6 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat promise->set_result(std::move(cell)); }); } - CellInfo &get_cell_info_force(td::Slice hash) { - return hash_table_.apply(hash, [&](CellInfo &info) { update_cell_info_force(info, hash); }); - } CellInfo &get_cell_info_lazy(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) { return hash_table_.apply(hash.substr(hash.size() - Cell::hash_bytes), [&](CellInfo &info) { update_cell_info_lazy(info, level_mask, hash, depth); }); diff --git a/tdnet/td/net/TcpListener.cpp b/tdnet/td/net/TcpListener.cpp index e711cbbda..c9bebe2d4 100644 --- a/tdnet/td/net/TcpListener.cpp +++ b/tdnet/td/net/TcpListener.cpp @@ -61,7 +61,7 @@ void TcpListener::loop() { break; } TRY_RESULT(client_socket, std::move(r_socket)); - LOG(ERROR) << "Accept"; + LOG(INFO) << "Accept"; callback_->accept(std::move(client_socket)); } if (td::can_close(server_socket_fd_)) { diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index c67203f08..fe7af04e7 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -231,7 +231,7 @@ void CellDbIn::start_up() { db_options.two_level_index_and_filter = db_options.enable_bloom_filter && opts_->state_ttl() >= 60 * 60 * 24 * 30; // 30 days if (db_options.two_level_index_and_filter && !opts_->get_celldb_in_memory()) { - o_celldb_cache_size = std::max(o_celldb_cache_size ? o_celldb_cache_size.value() : 0UL, 16UL << 30); + o_celldb_cache_size = std::max(o_celldb_cache_size ? o_celldb_cache_size.value() : 0UL, 16UL << 30); } if (o_celldb_cache_size) { From 5c2461b98c56a7fe930b15fc0c6f22c6f7a0d1b8 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Tue, 8 Apr 2025 12:12:11 +0400 Subject: [PATCH 207/388] not always gh mac runners have nproc utility (#1612) --- assembly/native/build-macos-portable.sh | 10 +++++----- assembly/native/build-macos-shared.sh | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index 81ac507b7..277973088 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -55,7 +55,7 @@ git clone https://github.com/lz4/lz4.git ../3pp/lz4 cd ../3pp/lz4 lz4Path=`pwd` git checkout v1.9.4 -make -j$(nproc) +make -j4 test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } cd ../../build # ./lib/liblz4.a @@ -73,7 +73,7 @@ if [ ! -d "../3pp/libsodium" ]; then git checkout 1.0.18 ./autogen.sh ./configure --with-pic --enable-static - make -j$(nproc) + make -j4 test $? -eq 0 || { echo "Can't compile libsodium"; exit 1; } cd ../../build else @@ -87,7 +87,7 @@ if [ ! -d "../3pp/openssl_3" ]; then opensslPath=`pwd` git checkout openssl-3.1.4 ./config - make build_libs -j$(nproc) + make build_libs -j4 test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } cd ../../build else @@ -100,7 +100,7 @@ if [ ! -d "../3pp/zlib" ]; then cd ../3pp/zlib zlibPath=`pwd` ./configure --static - make -j$(nproc) + make -j4 test $? -eq 0 || { echo "Can't compile zlib"; exit 1; } cd ../../build else @@ -116,7 +116,7 @@ if [ ! -d "../3pp/libmicrohttpd" ]; then cd libmicrohttpd-1.0.1 libmicrohttpdPath=`pwd` ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic - make -j$(nproc) + make -j4 test $? -eq 0 || { echo "Can't compile libmicrohttpd"; exit 1; } cd ../../../build else diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh index dbb4e7d29..595c44e3e 100644 --- a/assembly/native/build-macos-shared.sh +++ b/assembly/native/build-macos-shared.sh @@ -53,7 +53,7 @@ if [ ! -d "lz4" ]; then cd lz4 lz4Path=`pwd` git checkout v1.9.4 - make -j$(nproc) + make -j4 test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } cd .. else From 7528a883696f3a74afa3e8731575b19e52e97f33 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Sat, 22 Mar 2025 13:51:15 +0300 Subject: [PATCH 208/388] Cache validator sets for create_shard_state --- crypto/block/mc-config.cpp | 71 ++++++++++++++++++++++++++++++-- crypto/block/mc-config.h | 4 +- lite-client/lite-client.h | 2 +- validator/impl/shard.cpp | 4 +- validator/impl/validator-set.hpp | 2 +- 5 files changed, 73 insertions(+), 10 deletions(-) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 0f019b068..7b4f326f8 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -38,6 +38,7 @@ #include "openssl/digest.hpp" #include #include +#include namespace block { using namespace std::literals::string_literals; @@ -253,7 +254,7 @@ td::Status Config::unpack() { } config_dict = std::make_unique(config_root, 32); if (mode & needValidatorSet) { - auto vset_res = unpack_validator_set(get_config_param(35, 34)); + auto vset_res = unpack_validator_set(get_config_param(35, 34), true); if (vset_res.is_error()) { return vset_res.move_as_error(); } @@ -486,15 +487,74 @@ td::Result Config::unpack_workchain_list(Ref root) { return std::move(pair.first); } -td::Result> Config::unpack_validator_set(Ref vset_root) { +class ValidatorSetCache { +public: + ValidatorSetCache() { + cache_.reserve(MAX_CACHE_SIZE + 1); + } + + std::shared_ptr get(const vm::CellHash& hash) { + std::lock_guard lock{mutex_}; + auto it = cache_.find(hash); + if (it == cache_.end()) { + return {}; + } + auto entry = it->second.get(); + entry->remove(); + lru_.put(entry); + return entry->value; + } + + void set(const vm::CellHash& hash, std::shared_ptr vset) { + std::lock_guard lock{mutex_}; + std::unique_ptr& entry = cache_[hash]; + if (entry == nullptr) { + entry = std::make_unique(hash, std::move(vset)); + } else { + entry->value = std::move(vset); + entry->remove(); + } + lru_.put(entry.get()); + if (cache_.size() > MAX_CACHE_SIZE) { + auto to_remove = (CacheEntry*)lru_.get(); + CHECK(to_remove); + to_remove->remove(); + cache_.erase(to_remove->key); + } + } + +private: + std::mutex mutex_; + + struct CacheEntry : td::ListNode { + explicit CacheEntry(vm::CellHash key, std::shared_ptr value) : key(key), value(std::move(value)) { + } + vm::CellHash key; + std::shared_ptr value; + }; + td::HashMap> cache_; + td::ListNode lru_; + + static constexpr size_t MAX_CACHE_SIZE = 100; +}; + +td::Result> Config::unpack_validator_set(Ref vset_root, bool use_cache) { if (vset_root.is_null()) { return td::Status::Error("validator set is absent"); } + static ValidatorSetCache cache; + if (use_cache) { + auto result = cache.get(vset_root->get_hash()); + if (result) { + return result; + } + } + gen::ValidatorSet::Record_validators_ext rec; Ref dict_root; if (!tlb::unpack_cell(vset_root, rec)) { gen::ValidatorSet::Record_validators rec0; - if (!tlb::unpack_cell(std::move(vset_root), rec0)) { + if (!tlb::unpack_cell(vset_root, rec0)) { return td::Status::Error("validator set is invalid"); } rec.utime_since = rec0.utime_since; @@ -515,7 +575,7 @@ td::Result> Config::unpack_validator_set(Ref(rec.utime_since, rec.utime_until, rec.total, rec.main); + auto ptr = std::make_shared(rec.utime_since, rec.utime_until, rec.total, rec.main); for (int i = 0; i < rec.total; i++) { key_buffer.store_ulong(i); auto descr_cs = dict.lookup(key_buffer.bits(), 16); @@ -548,6 +608,9 @@ td::Result> Config::unpack_validator_set(Reftotal_weight) { return td::Status::Error("validator set declares incorrect total weight"); } + if (use_cache) { + cache.set(vset_root->get_hash(), ptr); + } return std::move(ptr); } diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 98e6a26df..a44522f11 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -559,7 +559,7 @@ class Config { td::BitArray<256> config_addr; Ref config_root; std::unique_ptr config_dict; - std::unique_ptr cur_validators_; + std::shared_ptr cur_validators_; std::unique_ptr workchains_dict_; WorkchainSet workchains_; int version_{-1}; @@ -623,7 +623,7 @@ class Config { bool set_block_id_ext(const ton::BlockIdExt& block_id_ext); td::Result> get_special_smartcontracts(bool without_config = false) const; bool is_special_smartcontract(const ton::StdSmcAddress& addr) const; - static td::Result> unpack_validator_set(Ref valset_root); + static td::Result> unpack_validator_set(Ref valset_root, bool use_cache = false); td::Result> get_storage_prices() const; static td::Result do_get_one_storage_prices(vm::CellSlice cs); td::Result get_gas_limits_prices(bool is_masterchain = false) const; diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index 721d2b20d..3284f255d 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -143,7 +143,7 @@ class TestNode : public td::actor::Actor { ton::LogicalTime end_lt{0}; ton::Bits256 vset_hash; Ref vset_root; - std::unique_ptr vset; + std::shared_ptr vset; std::map vset_map; int special_idx{-1}; std::pair created_total, created_special; diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index 0ab216f71..d2d402dd6 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -390,12 +390,12 @@ td::Status MasterchainStateQ::mc_reinit() { auto cv_root = config_->get_config_param(35, 34); if (cv_root.not_null()) { - TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(cv_root))); + TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(cv_root), true)); cur_validators_ = std::move(validators); } auto nv_root = config_->get_config_param(37, 36); if (nv_root.not_null()) { - TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(nv_root))); + TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(nv_root), true)); next_validators_ = std::move(validators); } diff --git a/validator/impl/validator-set.hpp b/validator/impl/validator-set.hpp index 951ca4b71..81c0bd518 100644 --- a/validator/impl/validator-set.hpp +++ b/validator/impl/validator-set.hpp @@ -74,7 +74,7 @@ class ValidatorSetCompute { private: const block::Config* config_{nullptr}; - std::unique_ptr cur_validators_, next_validators_; + std::shared_ptr cur_validators_, next_validators_; td::Ref compute_validator_set(ShardIdFull shard, const block::ValidatorSet& vset, UnixTime time, CatchainSeqno seqno) const; }; From 21719999796a7d6454a8d903e859c8a3b7e288a6 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 8 Apr 2025 17:12:16 +0300 Subject: [PATCH 209/388] Avoid copying Account object --- emulator/emulator-extern.cpp | 5 +++-- emulator/transaction-emulator.cpp | 2 +- emulator/transaction-emulator.h | 12 ++++++++---- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/emulator/emulator-extern.cpp b/emulator/emulator-extern.cpp index eb5ff9f9e..c5f3c04e0 100644 --- a/emulator/emulator-extern.cpp +++ b/emulator/emulator-extern.cpp @@ -187,7 +187,7 @@ const char *transaction_emulator_emulate_transaction(void *transaction_emulator, external_not_accepted->elapsed_time); } - auto emulation_success = dynamic_cast(*emulation_result); + auto emulation_success = std::move(dynamic_cast(*emulation_result)); auto trans_boc_b64 = cell_to_boc_b64(std::move(emulation_success.transaction)); if (trans_boc_b64.is_error()) { ERROR_RESPONSE(PSTRING() << "Can't serialize Transaction to boc " << trans_boc_b64.move_as_error()); @@ -260,7 +260,8 @@ const char *transaction_emulator_emulate_tick_tock_transaction(void *transaction } auto emulation_result = result.move_as_ok(); - auto emulation_success = dynamic_cast(*emulation_result); + auto emulation_success = + std::move(dynamic_cast(*emulation_result)); auto trans_boc_b64 = cell_to_boc_b64(std::move(emulation_success.transaction)); if (trans_boc_b64.is_error()) { ERROR_RESPONSE(PSTRING() << "Can't serialize Transaction to boc " << trans_boc_b64.move_as_error()); diff --git a/emulator/transaction-emulator.cpp b/emulator/transaction-emulator.cpp index 6267f9bd0..abdf0d2cf 100644 --- a/emulator/transaction-emulator.cpp +++ b/emulator/transaction-emulator.cpp @@ -141,7 +141,7 @@ td::Result TransactionEmulator::emulate_t return td::Status::Error("account hash mismatch"); } - return emulation_result; + return std::move(emulation_result); } else if (auto emulation_not_accepted_ptr = dynamic_cast(emulation.get())) { return td::Status::Error( PSTRING() diff --git a/emulator/transaction-emulator.h b/emulator/transaction-emulator.h index eae109f40..848548c32 100644 --- a/emulator/transaction-emulator.h +++ b/emulator/transaction-emulator.h @@ -33,14 +33,18 @@ class TransactionEmulator { virtual ~EmulationResult() = default; }; - struct EmulationSuccess: EmulationResult { + struct EmulationSuccess : EmulationResult { td::Ref transaction; block::Account account; td::Ref actions; - EmulationSuccess(td::Ref transaction_, block::Account account_, std::string vm_log_, td::Ref actions_, double elapsed_time_) : - EmulationResult(vm_log_, elapsed_time_), transaction(transaction_), account(account_) , actions(actions_) - {} + EmulationSuccess(td::Ref transaction_, block::Account account_, std::string vm_log_, + td::Ref actions_, double elapsed_time_) + : EmulationResult(vm_log_, elapsed_time_) + , transaction(transaction_) + , account(std::move(account_)) + , actions(actions_) { + } }; struct EmulationExternalNotAccepted: EmulationResult { From e873d272ac900c7e854bdd6421aae4f3c9efdffb Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Thu, 10 Apr 2025 10:22:04 +0400 Subject: [PATCH 210/388] Native logger for go tunnel --- tdnet/td/net/UdpServer.cpp | 22 +++++++++++++++++++++- tdnet/td/net/tunnel/libtunnel.h | 8 +++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/tdnet/td/net/UdpServer.cpp b/tdnet/td/net/UdpServer.cpp index fdb94da77..8d05a2bb8 100644 --- a/tdnet/td/net/UdpServer.cpp +++ b/tdnet/td/net/UdpServer.cpp @@ -67,6 +67,26 @@ class UdpServerTunnelImpl : public UdpServer { static void on_recv_batch(void *next, uint8_t *data, size_t num); static void on_reinit(void *next, sockaddr *addr); + static void log(const char *text, const size_t len, const int level) { + const string str(text, len); + switch (level) { + case 0: + LOG(FATAL) << "[TUNNEL] " << str; + break; + case 1: + LOG(ERROR) << "[TUNNEL] " << str; + break; + case 2: + LOG(WARNING) << "[TUNNEL] " << str; + break; + case 3: + LOG(INFO) << "[TUNNEL] " << str; + break; + default: + LOG(DEBUG) << "[TUNNEL] " << str; + break; + } + } }; void UdpServerTunnelImpl::send(td::UdpMessage &&message) { @@ -129,7 +149,7 @@ void UdpServerTunnelImpl::start_up() { auto global_cfg = global_conf_data_R.move_as_ok(); LOG(INFO) << "Initializing ADNL Tunnel..."; - const auto res = PrepareTunnel(&on_recv_batch, &on_reinit, callback_.get(), callback_.get(), tunnel_config_.data(), tunnel_config_.size(), global_cfg.data(), global_cfg.size()); + const auto res = PrepareTunnel(&log, &on_recv_batch, &on_reinit, callback_.get(), callback_.get(), tunnel_config_.data(), tunnel_config_.size(), global_cfg.data(), global_cfg.size()); if (!res.index) { // the reason will be displayed in logs from lib part exit(1); diff --git a/tdnet/td/net/tunnel/libtunnel.h b/tdnet/td/net/tunnel/libtunnel.h index 30e377f0b..3c6aa31ec 100644 --- a/tdnet/td/net/tunnel/libtunnel.h +++ b/tdnet/td/net/tunnel/libtunnel.h @@ -35,6 +35,8 @@ typedef void (*RecvCallback)(void* next, uint8_t* data, size_t num); typedef void (*ReinitCallback)(void* next, struct sockaddr* data); +typedef void (*Logger)(const char *text, const size_t len, const int level); + // we need it because we cannot call C func by pointer directly from go static inline void on_recv_batch_ready(RecvCallback cb, void* next, void* data, size_t num) { @@ -45,6 +47,10 @@ static inline void on_reinit(ReinitCallback cb, void* next, void* data) { cb(next, (struct sockaddr*)data); } +static inline void write_log(Logger log, const char *text, const size_t len, const int level) { + log(text, len, level); +} + #line 1 "cgo-generated-wrapper" @@ -103,7 +109,7 @@ extern "C" { //goland:noinspection ALL -extern Tunnel PrepareTunnel(RecvCallback onRecv, ReinitCallback onReinit, void* nextOnRecv, void* nextOnReinit, char* configJson, int configJsonLen, char* networkConfigJson, int networkConfigJsonLen); +extern Tunnel PrepareTunnel(Logger logger, RecvCallback onRecv, ReinitCallback onReinit, void* nextOnRecv, void* nextOnReinit, char* configPath, int configPathLen, char* networkConfigJson, int networkConfigJsonLen); extern int WriteTunnel(size_t tunIdx, uint8_t* data, size_t num); #ifdef __cplusplus From 10e7b8faac31e8f5b59e59fb8d4c0f1ce463d97e Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Thu, 10 Apr 2025 11:18:49 +0400 Subject: [PATCH 211/388] Not run DHT server when tunnel enabled --- validator-engine/validator-engine.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index ff542ecd1..3da6b516c 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1976,7 +1976,12 @@ void ValidatorEngine::started_adnl() { } void ValidatorEngine::add_dht(ton::PublicKeyHash id) { - auto D = ton::dht::Dht::create(ton::adnl::AdnlNodeIdShort{id}, db_root_, dht_config_, keyring_.get(), adnl_.get()); + auto creator = ton::dht::Dht::create; + if (!tunnel_config_.empty()) { + creator = ton::dht::Dht::create_client; + } + + auto D = creator(ton::adnl::AdnlNodeIdShort{id}, db_root_, dht_config_, keyring_.get(), adnl_.get()); D.ensure(); dht_nodes_[id] = D.move_as_ok(); From 756a628f98dbb69133c8895ba59edafdcff817c2 Mon Sep 17 00:00:00 2001 From: birydrad Date: Mon, 14 Apr 2025 10:48:04 +0400 Subject: [PATCH 212/388] Support celldb_compression_depth in celldbv2 (#1619) Co-authored-by: birydrad <> --- crypto/test/test-db.cpp | 150 ++++++++++++++----------- crypto/vm/db/DynamicBagOfCellsDbV2.cpp | 10 +- validator-engine/validator-engine.cpp | 2 +- 3 files changed, 92 insertions(+), 70 deletions(-) diff --git a/crypto/test/test-db.cpp b/crypto/test/test-db.cpp index 47d833d56..f4dde8df7 100644 --- a/crypto/test/test-db.cpp +++ b/crypto/test/test-db.cpp @@ -71,7 +71,6 @@ #include "td/actor/actor.h" #include "td/utils/overloaded.h" - class ActorExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor { public: ActorExecutor(size_t tn) : tn_(tn) { @@ -1085,7 +1084,9 @@ struct BocOptions { }; KvOptions kv_options; std::variant options; + std::pair compress_depth_range{0, 0}; td::uint64 seed{123}; + td::Random::Xorshift128plus rnd{123}; std::shared_ptr create_kv(std::shared_ptr old_key_value, bool no_reads = false) { if (kv_options.kv_type == KvOptions::InMemory) { @@ -1151,33 +1152,37 @@ struct BocOptions { auto old_kv = std::move(db.kv); old_boc.reset(); using ResT = DB; - return std::visit(td::overloaded( - [&](CreateV1Options &) -> ResT { - auto new_kv = create_kv(std::move(old_kv)); - auto res = DynamicBagOfCellsDb::create(); - res->set_loader(std::make_unique(new_kv->snapshot())); - return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; - }, - [&](CreateV2Options &options) -> ResT { - auto new_kv = create_kv(std::move(old_kv)); - auto res = DynamicBagOfCellsDb::create_v2(options); - res->set_loader(std::make_unique(new_kv->snapshot())); - return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; - }, - [&](CreateInMemoryOptions &options) -> ResT { - auto read_kv = create_kv(std::move(old_kv), false); - auto res = DynamicBagOfCellsDb::create_in_memory(read_kv.get(), options); - auto new_kv = create_kv(std::move(read_kv), true); - res->set_loader(std::make_unique(new_kv->snapshot())); - auto stats = res->get_stats().move_as_ok(); - if (o_root_n) { - ASSERT_EQ(*o_root_n, stats.roots_total_count); - } - VLOG(boc) << "reset roots_n=" << stats.roots_total_count - << " cells_n=" << stats.cells_total_count; - return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; - }), - options); + auto res = std::visit(td::overloaded( + [&](CreateV1Options &) -> ResT { + auto new_kv = create_kv(std::move(old_kv)); + auto res = DynamicBagOfCellsDb::create(); + res->set_loader(std::make_unique(new_kv->snapshot())); + return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; + }, + [&](CreateV2Options &options) -> ResT { + auto new_kv = create_kv(std::move(old_kv)); + auto res = DynamicBagOfCellsDb::create_v2(options); + res->set_loader(std::make_unique(new_kv->snapshot())); + return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; + }, + [&](CreateInMemoryOptions &options) -> ResT { + auto read_kv = create_kv(std::move(old_kv), false); + auto res = DynamicBagOfCellsDb::create_in_memory(read_kv.get(), options); + auto new_kv = create_kv(std::move(read_kv), true); + res->set_loader(std::make_unique(new_kv->snapshot())); + auto stats = res->get_stats().move_as_ok(); + if (o_root_n) { + ASSERT_EQ(*o_root_n, stats.roots_total_count); + } + VLOG(boc) << "reset roots_n=" << stats.roots_total_count + << " cells_n=" << stats.cells_total_count; + return DB{.dboc = std::move(res), .kv = std::move(new_kv)}; + }), + options); + if (compress_depth_range.second != 0) { + res.dboc->set_celldb_compress_depth(rnd.fast(compress_depth_range.first, compress_depth_range.second)); + } + return res; }; void prepare_commit(DynamicBagOfCellsDb &dboc) { td::PerfWarningTimer warning_timer("test_db_prepare_commit"); @@ -1249,6 +1254,9 @@ struct BocOptions { if (async_executor) { sb << ", executor=" << async_executor->describe(); } + if (compress_depth_range.second != 0) { + sb << ", compress_depth=[" << compress_depth_range.first << ";" << compress_depth_range.second << "]"; + } sb << ")"; return sb.as_cslice().str(); @@ -1270,6 +1278,7 @@ void with_all_boc_options(F &&f, size_t tests_n, bool single_thread = false) { auto before = counter(); options.seed = i == 0 ? 123 : i; + options.rnd = td::Random::Xorshift128plus{options.seed}; auto stats_diff = f(options); stats.apply_diff(stats_diff); @@ -1293,39 +1302,51 @@ void with_all_boc_options(F &&f, size_t tests_n, bool single_thread = false) { BocOptions::KvOptions{ .kv_type = BocOptions::KvOptions::RocksDb, .experimental = false, .cache_size = size_t{128 << 20}}, }; + std::vector> compress_depth_ranges = {{0, 5}, {5, 5}, {0, 0}}; std::vector has_executor_options = {false, true}; - for (auto kv_options : kv_options_list) { - for (bool has_executor : has_executor_options) { - std::shared_ptr executor; - if (has_executor) { - executor = std::make_shared( - 4); // 4 - to compare V1 and V2, because V1 has parallel_load = 4 by default - } - // V2 - 4 threads - run({.async_executor = executor, - .kv_options = kv_options, - .options = DynamicBagOfCellsDb::CreateV2Options{ - .extra_threads = 3, .executor = executor, .cache_ttl_max = 5}}); - - // V1 - run({.async_executor = executor, .kv_options = kv_options, .options = DynamicBagOfCellsDb::CreateV1Options{}}); - - // V2 - one thread - run({.async_executor = executor, - .kv_options = kv_options, - .options = - DynamicBagOfCellsDb::CreateV2Options{.extra_threads = 0, .executor = executor, .cache_ttl_max = 5}}); - - // InMemory - for (auto use_arena : {false, true}) { - for (auto less_memory : {false, true}) { - run({.async_executor = executor, - .kv_options = kv_options, - .options = - DynamicBagOfCellsDb::CreateInMemoryOptions{.extra_threads = std::thread::hardware_concurrency(), - .verbose = false, - .use_arena = use_arena, - .use_less_memory_during_creation = less_memory}}); + for (auto compress_depth_range : compress_depth_ranges) { + for (auto kv_options : kv_options_list) { + for (bool has_executor : has_executor_options) { + std::shared_ptr executor; + if (has_executor) { + executor = std::make_shared( + 4); // 4 - to compare V1 and V2, because V1 has parallel_load = 4 by default + } + // V2 - 4 threads + run({ + .async_executor = executor, + .kv_options = kv_options, + .options = + DynamicBagOfCellsDb::CreateV2Options{.extra_threads = 3, .executor = executor, .cache_ttl_max = 5}, + .compress_depth_range = compress_depth_range, + }); + + // V1 + run({.async_executor = executor, + .kv_options = kv_options, + .options = DynamicBagOfCellsDb::CreateV1Options{}, + .compress_depth_range = compress_depth_range}); + + // V2 - one thread + run({.async_executor = executor, + .kv_options = kv_options, + .options = + DynamicBagOfCellsDb::CreateV2Options{.extra_threads = 0, .executor = executor, .cache_ttl_max = 5}, + .compress_depth_range = compress_depth_range}); + + // InMemory + if (compress_depth_range.second == 0) { + for (auto use_arena : {false, true}) { + for (auto less_memory : {false, true}) { + run({.async_executor = executor, + .kv_options = kv_options, + .options = + DynamicBagOfCellsDb::CreateInMemoryOptions{.extra_threads = std::thread::hardware_concurrency(), + .verbose = false, + .use_arena = use_arena, + .use_less_memory_during_creation = less_memory}}); + } + } } } } @@ -1342,7 +1363,7 @@ void with_all_boc_options(F &&f, size_t tests_n, bool single_thread = false) { DynamicBagOfCellsDb::Stats test_dynamic_boc(BocOptions options) { DynamicBagOfCellsDb::Stats stats; - td::Random::Xorshift128plus rnd{options.seed}; + auto &rnd = options.rnd; std::string old_root_hash; std::string old_root_serialization; DB db; @@ -1392,7 +1413,7 @@ TEST(TonDb, DynamicBoc) { }; DynamicBagOfCellsDb::Stats test_dynamic_boc2(BocOptions options) { - td::Random::Xorshift128plus rnd{options.seed}; + auto &rnd = options.rnd; DynamicBagOfCellsDb::Stats stats; int total_roots = rnd.fast(1, !rnd.fast(0, 30) * 100 + 10); @@ -1879,8 +1900,7 @@ DynamicBagOfCellsDb::Stats bench_dboc_get_and_set(BocOptions options) { db.reset_loader(); } - for (auto p : - std::vector>{{10000, 0}, {10000, 5}, {5000, 5000}, {5, 10000}, {0, 10000}}) { + for (auto p : std::vector>{{10000, 0}, {10000, 5}, {5000, 5000}, {5, 10000}, {0, 10000}}) { auto get_n = p.first; auto set_n = p.second; auto hash = arr.root()->get_hash(); @@ -2912,7 +2932,7 @@ TEST(UsageTree, ThreadSafe) { auto cell = vm::gen_random_cell(rnd.fast(2, 100), rnd, false); auto usage_tree = std::make_shared(); auto usage_cell = vm::UsageCell::create(cell, usage_tree->root_ptr()); - std::ptrdiff_t threads_n = 1; // TODO: when CellUsageTree is thread safe, change it to 4 + std::ptrdiff_t threads_n = 1; // TODO: when CellUsageTree is thread safe, change it to 4 auto barrier = std::barrier{threads_n}; std::vector threads; std::vector explorations(threads_n); diff --git a/crypto/vm/db/DynamicBagOfCellsDbV2.cpp b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp index ecf3b76cb..3b7d7c233 100644 --- a/crypto/vm/db/DynamicBagOfCellsDbV2.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDbV2.cpp @@ -1008,7 +1008,7 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { } void set_celldb_compress_depth(td::uint32 value) override { - CHECK(value == 0); + celldb_compress_depth_ = value; } vm::ExtCellCreator &as_ext_cell_creator() override { @@ -1250,6 +1250,7 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { }; CreateV2Options options_; + td::int32 celldb_compress_depth_{0}; std::vector> to_inc_; std::vector> to_dec_; std::vector> diff_chunks_; @@ -1416,6 +1417,7 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { stats_.diff_zero.inc(); return; } + auto should_compress = celldb_compress_depth_ != 0 && info->cell->get_depth() == celldb_compress_depth_; bool merge_supported = true; if (merge_supported) { @@ -1443,7 +1445,7 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { stats_.diff_full.inc(); worker.add_result({.type = CellStorer::Diff::Set, .key = info->cell->get_hash(), - .value = CellStorer::serialize_value(ref_cnt_diff + state.db_ref_cnt, data_cell, false)}); + .value = CellStorer::serialize_value(ref_cnt_diff + state.db_ref_cnt, data_cell, should_compress)}); } else { stats_.diff_ref_cnt.inc(); worker.add_result({.type = CellStorer::Diff::Merge, @@ -1479,7 +1481,7 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { worker.add_result( {.type = CellStorer::Diff::Set, .key = info->cell->get_hash(), - .value = CellStorer::serialize_value(new_ref_cnt, info->cell->load_cell().move_as_ok().data_cell, false)}); + .value = CellStorer::serialize_value(new_ref_cnt, info->cell->load_cell().move_as_ok().data_cell, should_compress)}); stats_.dec_save_full.inc(); } } else { @@ -1496,7 +1498,7 @@ class DynamicBagOfCellsDbImplV2 : public DynamicBagOfCellsDb { worker.add_result( {.type = CellStorer::Diff::Set, .key = info->cell->get_hash(), - .value = CellStorer::serialize_value(new_ref_cnt, info->cell->load_cell().move_as_ok().data_cell, false)}); + .value = CellStorer::serialize_value(new_ref_cnt, info->cell->load_cell().move_as_ok().data_cell, should_compress)}); stats_.inc_save_full.inc(); } } diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 945d5456f..6f47f7741 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1464,7 +1464,7 @@ td::Status ValidatorEngine::load_global_config() { if (!session_logs_file_.empty()) { validator_options_.write().set_session_logs_file(session_logs_file_); } - if (celldb_in_memory_ || celldb_v2_) { + if (celldb_in_memory_) { celldb_compress_depth_ = 0; } validator_options_.write().set_celldb_compress_depth(celldb_compress_depth_); From 688c695ed131fc08cef020a2bd32d8f0caa81708 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 14 Apr 2025 09:56:21 +0300 Subject: [PATCH 213/388] New account storage stat (#1568) * New account storage stat * Fix computing storage stat * Don't store zero dict hash * Dictionary::multiset; optimize account storage stat * Don't calculate account storage dict when not needed * Optimize cell loading in account storage stat * Don't store storage dict hash in masterchain * Avoid copying Account object --- common/global-version.h | 2 +- crypto/CMakeLists.txt | 2 + crypto/block/account-storage-stat.cpp | 285 ++++++++++++++++++++++++++ crypto/block/account-storage-stat.h | 109 ++++++++++ crypto/block/block-parse.cpp | 45 ++-- crypto/block/block-parse.h | 7 - crypto/block/block.tlb | 17 +- crypto/block/create-state.cpp | 7 +- crypto/block/mc-config.h | 6 +- crypto/block/transaction.cpp | 238 ++++++++++----------- crypto/block/transaction.h | 12 +- crypto/common/bitstring.cpp | 3 + crypto/vm/boc.h | 8 +- crypto/vm/cells/CellSlice.cpp | 8 + crypto/vm/cells/CellSlice.h | 1 + crypto/vm/dict.cpp | 254 +++++++++++++++++++++++ crypto/vm/dict.h | 4 +- doc/GlobalVersions.md | 7 + emulator/emulator-extern.cpp | 5 +- emulator/transaction-emulator.cpp | 2 +- emulator/transaction-emulator.h | 12 +- tonlib/tonlib/TonlibClient.cpp | 15 +- validator/impl/validate-query.cpp | 1 + 23 files changed, 866 insertions(+), 184 deletions(-) create mode 100644 crypto/block/account-storage-stat.cpp create mode 100644 crypto/block/account-storage-stat.h diff --git a/common/global-version.h b/common/global-version.h index 2308ce3e9..b54a3bdc5 100644 --- a/common/global-version.h +++ b/common/global-version.h @@ -19,6 +19,6 @@ namespace ton { // See doc/GlobalVersions.md -constexpr int SUPPORTED_VERSION = 10; +constexpr int SUPPORTED_VERSION = 11; } diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index defe2b58e..79d1117ea 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -212,6 +212,8 @@ set(BLOCK_SOURCE block/mc-config.cpp block/output-queue-merger.cpp block/transaction.cpp + block/account-storage-stat.h + block/account-storage-stat.cpp block/precompiled-smc/PrecompiledSmartContract.cpp ${TLB_BLOCK_AUTO} diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp new file mode 100644 index 000000000..56cc8f1e1 --- /dev/null +++ b/crypto/block/account-storage-stat.cpp @@ -0,0 +1,285 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "account-storage-stat.h" + +namespace block { + +AccountStorageStat::AccountStorageStat() : AccountStorageStat({}, {}, 0, 0) { +} + +AccountStorageStat::AccountStorageStat(Ref dict_root, std::vector> roots, + td::uint64 total_cells, td::uint64 total_bits) + : dict_(std::move(dict_root), 256), total_cells_(total_cells), total_bits_(total_bits), roots_(std::move(roots)) { +} + +AccountStorageStat::AccountStorageStat(const AccountStorageStat* parent) + : dict_(parent->dict_) + , dict_up_to_date_(parent->dict_up_to_date_) + , total_cells_(parent->total_cells_) + , total_bits_(parent->total_bits_) + , roots_(parent->roots_) + , parent_(parent) { + CHECK(parent_->parent_ == nullptr); +} + +td::Status AccountStorageStat::replace_roots(std::vector> new_roots, bool check_merkle_depth) { + std::erase_if(new_roots, [](const Ref& c) { return c.is_null(); }); + if (new_roots.empty()) { + roots_.clear(); + total_bits_ = total_cells_ = 0; + dict_ = vm::Dictionary{256}; + cache_ = {}; + dict_up_to_date_ = true; + parent_ = nullptr; + return td::Status::OK(); + } + + auto cmp = [](const Ref& c1, const Ref& c2) { return c1->get_hash() < c2->get_hash(); }; + std::sort(new_roots.begin(), new_roots.end(), cmp); + std::sort(roots_.begin(), roots_.end(), cmp); + std::vector> to_add, to_del; + std::set_difference(new_roots.begin(), new_roots.end(), roots_.begin(), roots_.end(), std::back_inserter(to_add), + cmp); + std::set_difference(roots_.begin(), roots_.end(), new_roots.begin(), new_roots.end(), std::back_inserter(to_del), + cmp); + if (to_add.empty() && to_del.empty()) { + return td::Status::OK(); + } + + for (const Ref& root : to_add) { + TRY_RESULT(info, add_cell(root)); + if (check_merkle_depth && info.max_merkle_depth > MAX_MERKLE_DEPTH) { + return td::Status::Error("too big Merkle depth"); + } + } + for (const Ref& root : to_del) { + TRY_STATUS(remove_cell(root)); + } + + roots_ = std::move(new_roots); + dict_up_to_date_ = false; + for (auto& [_, e] : cache_) { + TRY_STATUS(finalize_entry(e)); + } + return td::Status::OK(); +} + +void AccountStorageStat::add_hint(const td::HashSet& hint) { + td::HashSet visited; + std::function&, bool)> dfs = [&](const Ref& cell, bool is_root) { + if (!visited.insert(cell->get_hash()).second) { + return; + } + Entry& e = get_entry(cell); + e.exists = e.exists_known = true; + if (is_root) { + fetch_from_dict(e).ignore(); + if (e.max_merkle_depth && e.max_merkle_depth.value() != 0) { + return; + } + } + if (hint.contains(cell->get_hash())) { + bool spec; + vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); + e.size_bits = cs.size(); + for (unsigned i = 0; i < cs.size_refs(); ++i) { + dfs(cs.prefetch_ref(i), false); + } + } + }; + for (const Ref& root : roots_) { + dfs(root, true); + } +} + +td::Result AccountStorageStat::add_cell(const Ref& cell) { + Entry& e = get_entry(cell); + if (!e.exists_known || e.refcnt_diff < 0) { + TRY_STATUS(fetch_from_dict(e)); + } + ++e.refcnt_diff; + if (e.exists || e.refcnt_diff > 1 || (e.refcnt && e.refcnt.value() + e.refcnt_diff != 1)) { + if (!e.max_merkle_depth) { + TRY_STATUS(fetch_from_dict(e)); + if (!e.max_merkle_depth) { + return td::Status::Error(PSTRING() << "unexpected unknown Merkle depth of cell " << cell->get_hash()); + } + } + return CellInfo{e.max_merkle_depth.value()}; + } + + td::uint32 max_merkle_depth = 0; + bool spec; + vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); + e.size_bits = cs.size(); + for (unsigned i = 0; i < cs.size_refs(); ++i) { + TRY_RESULT(info, add_cell(cs.prefetch_ref(i))); + max_merkle_depth = std::max(max_merkle_depth, info.max_merkle_depth); + } + if (cs.special_type() == vm::CellTraits::SpecialType::MerkleProof || + cs.special_type() == vm::CellTraits::SpecialType::MerkleUpdate) { + ++max_merkle_depth; + } + max_merkle_depth = std::min(max_merkle_depth, MERKLE_DEPTH_LIMIT); + Entry& e2 = get_entry(cell); + e2.max_merkle_depth = max_merkle_depth; + return CellInfo{max_merkle_depth}; +} + +td::Status AccountStorageStat::remove_cell(const Ref& cell) { + Entry& e = get_entry(cell); + if (!e.exists_known) { + TRY_STATUS(fetch_from_dict(e)); + } + if (!e.exists) { + return td::Status::Error(PSTRING() << "Failed to remove cell " << cell->get_hash().to_hex() + << " : does not exist in the dict"); + } + --e.refcnt_diff; + if (e.refcnt_diff < 0 && !e.refcnt) { + TRY_STATUS(fetch_from_dict(e)); + } + if (e.refcnt.value() + e.refcnt_diff != 0) { + return td::Status::OK(); + } + bool spec; + vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); + e.size_bits = cs.size(); + for (unsigned i = 0; i < cs.size_refs(); ++i) { + TRY_STATUS(remove_cell(cs.prefetch_ref(i))); + } + return td::Status::OK(); +} + +td::Result> AccountStorageStat::get_dict_root() { + if (!dict_up_to_date_) { + std::vector>> values; + for (auto& [_, e] : cache_) { + if (e.dict_refcnt_diff == 0) { + continue; + } + if (!e.exists_known || !e.refcnt || (e.exists && !e.max_merkle_depth)) { + return td::Status::Error("unexpected state of storage stat"); + } + if (e.exists) { + Ref cbr{true}; + auto& cb = cbr.write(); + CHECK(cb.store_long_bool(e.refcnt.value(), 32) && cb.store_long_bool(e.max_merkle_depth.value(), 2)); + values.emplace_back(e.hash.bits(), std::move(cbr)); + } else { + values.emplace_back(e.hash.bits(), Ref{}); + } + e.dict_refcnt_diff = 0; + } + if (!dict_.multiset(values)) { + return td::Status::Error("failed to update dictionary"); + } + dict_up_to_date_ = true; + } + return dict_.get_root_cell(); +} + +void AccountStorageStat::apply_child_stat(AccountStorageStat&& child) { + CHECK(parent_ == nullptr); + if (child.parent_ == nullptr) { + *this = std::move(child); + return; + } + CHECK(child.parent_ == this); + total_bits_ = child.total_bits_; + total_cells_ = child.total_cells_; + dict_ = std::move(child.dict_); + dict_up_to_date_ = child.dict_up_to_date_; + roots_ = std::move(child.roots_); + for (auto& [hash, e] : child.cache_) { + cache_[hash] = std::move(e); + } +} + +AccountStorageStat::Entry& AccountStorageStat::get_entry(const Ref& cell) { + Entry& e = cache_[cell->get_hash()]; + if (e.inited) { + return e; + } + if (parent_) { + auto it = parent_->cache_.find(cell->get_hash()); + if (it != parent_->cache_.end()) { + CHECK(it->second.inited); + e = it->second; + return e; + } + } + e.inited = true; + e.hash = cell->get_hash(); + return e; +} + +td::Status AccountStorageStat::fetch_from_dict(Entry& e) { + if (e.exists_known && e.refcnt && (!e.exists || e.max_merkle_depth)) { + return td::Status::OK(); + } + auto cs = dict_.lookup(e.hash.as_bitslice()); + if (cs.is_null()) { + e.exists = false; + e.refcnt = 0; + } else { + if (cs->size_ext() != 32 + 2) { + return td::Status::Error(PSTRING() << "invalid record for cell " << e.hash.to_hex()); + } + e.exists = true; + e.refcnt = (td::uint32)cs.write().fetch_ulong(32); + e.max_merkle_depth = (td::uint32)cs.write().fetch_ulong(2); + if (e.refcnt.value() == 0) { + return td::Status::Error(PSTRING() << "invalid refcnt=0 for cell " << e.hash.to_hex()); + } + } + e.exists_known = true; + return td::Status::OK(); +} + +td::Status AccountStorageStat::finalize_entry(Entry& e) { + if (e.refcnt_diff == 0) { + return td::Status::OK(); + } + TRY_STATUS(fetch_from_dict(e)); + e.refcnt.value() += e.refcnt_diff; + e.dict_refcnt_diff += e.refcnt_diff; + e.refcnt_diff = 0; + if (e.refcnt.value() == 0) { + if (!e.size_bits) { + return td::Status::Error(PSTRING() << "Failed to store entry " << e.hash.to_hex() << " : unknown cell bits"); + } + --total_cells_; + total_bits_ -= e.size_bits.value(); + e.exists = false; + } else { + if (!e.exists) { + if (!e.size_bits) { + return td::Status::Error(PSTRING() << "Failed to store entry " << e.hash.to_hex() << " : unknown cell bits"); + } + ++total_cells_; + total_bits_ += e.size_bits.value(); + } + e.exists = true; + if (!e.max_merkle_depth) { + return td::Status::Error(PSTRING() << "Failed to store entry " << e.hash.to_hex() << " : unknown merkle depth"); + } + } + return td::Status::OK(); +} + +} // namespace block \ No newline at end of file diff --git a/crypto/block/account-storage-stat.h b/crypto/block/account-storage-stat.h new file mode 100644 index 000000000..645f0744d --- /dev/null +++ b/crypto/block/account-storage-stat.h @@ -0,0 +1,109 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#pragma once +#include "common/refcnt.hpp" +#include "vm/dict.h" +#include "ton/ton-types.h" +#include "ton/ton-shard.h" +#include "common/bitstring.h" +#include "block.h" +#include "vm/db/CellHashTable.h" + +namespace block { +using td::Ref; + +class AccountStorageStat { + public: + AccountStorageStat(); + AccountStorageStat(Ref dict_root, std::vector> roots, td::uint64 total_cells, + td::uint64 total_bits); + explicit AccountStorageStat(const AccountStorageStat *parent); + AccountStorageStat(const AccountStorageStat &other) = delete; + AccountStorageStat(AccountStorageStat &&other) = default; + ~AccountStorageStat() = default; + + AccountStorageStat &operator=(const AccountStorageStat &other) = delete; + AccountStorageStat &operator=(AccountStorageStat &&other) = default; + + td::Status replace_roots(std::vector> new_roots, bool check_merkle_depth = false); + void add_hint(const td::HashSet &visited); + + td::uint64 get_total_cells() const { + return total_cells_; + } + + td::uint64 get_total_bits() const { + return total_bits_; + } + + td::Result> get_dict_root(); + + td::Result get_dict_hash() { + TRY_RESULT(root, get_dict_root()); + return root.is_null() ? td::Bits256::zero() : td::Bits256{root->get_hash().bits()}; + } + + void apply_child_stat(AccountStorageStat &&child); + + private: + vm::Dictionary dict_; + bool dict_up_to_date_ = true; + td::uint64 total_cells_, total_bits_; + std::vector> roots_; + const AccountStorageStat *parent_ = nullptr; + + struct CellInfo { + td::uint32 max_merkle_depth = 0; + }; + + td::Result add_cell(const Ref &cell); + td::Status remove_cell(const Ref &cell); + + struct Entry { + bool inited = false; + vm::CellHash hash; + td::optional size_bits; + bool exists_known = false; + bool exists = false; + td::optional refcnt, max_merkle_depth; + td::int32 refcnt_diff = 0; + td::int32 dict_refcnt_diff = 0; + }; + + td::HashMap> cache_; + + Entry &get_entry(const Ref &cell); + td::Status fetch_from_dict(Entry &e); + td::Status finalize_entry(Entry &e); + + static constexpr td::uint32 MERKLE_DEPTH_LIMIT = 3; + static constexpr td::uint32 MAX_MERKLE_DEPTH = 2; +}; + +class StorageStatCalculationContext : public td::Context { + public: + explicit StorageStatCalculationContext(bool active) : active_(active) { + } + bool calculating_storage_stat() const { + return active_; + } + + private: + bool active_ = false; +}; + +} // namespace block diff --git a/crypto/block/block-parse.cpp b/crypto/block/block-parse.cpp index 50851c795..6d645ac79 100644 --- a/crypto/block/block-parse.cpp +++ b/crypto/block/block-parse.cpp @@ -960,41 +960,34 @@ const MsgEnvelope t_MsgEnvelope; const RefTo t_Ref_MsgEnvelope; bool StorageUsed::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { - return t_VarUInteger_7.validate_skip(ops, cs, weak) // cells:(VarUInteger 7) - && t_VarUInteger_7.validate_skip(ops, cs, weak) // bits:(VarUInteger 7) - && t_VarUInteger_7.validate_skip(ops, cs, weak); // public_cells:(VarUInteger 7) -} - -bool StorageUsed::skip(vm::CellSlice& cs) const { - return t_VarUInteger_7.skip(cs) // cells:(VarUInteger 7) - && t_VarUInteger_7.skip(cs) // bits:(VarUInteger 7) - && t_VarUInteger_7.skip(cs); // public_cells:(VarUInteger 7) -} - -const StorageUsed t_StorageUsed; - -bool StorageUsedShort::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { return t_VarUInteger_7.validate_skip(ops, cs, weak) // cells:(VarUInteger 7) && t_VarUInteger_7.validate_skip(ops, cs, weak); // bits:(VarUInteger 7) } -bool StorageUsedShort::skip(vm::CellSlice& cs) const { +bool StorageUsed::skip(vm::CellSlice& cs) const { return t_VarUInteger_7.skip(cs) // cells:(VarUInteger 7) && t_VarUInteger_7.skip(cs); // bits:(VarUInteger 7) } -const StorageUsedShort t_StorageUsedShort; +const StorageUsed t_StorageUsed; const Maybe t_Maybe_Grams; bool StorageInfo::skip(vm::CellSlice& cs) const { - return t_StorageUsed.skip(cs) // used:StorageUsed - && cs.advance(32) // last_paid:uint32 - && t_Maybe_Grams.skip(cs); // due_payment:(Maybe Grams) + int extra_tag = 0; + return t_StorageUsed.skip(cs) // used:StorageUsed + && cs.fetch_uint_to(3, extra_tag) // storage_extra:StorageExtraInfo + && (extra_tag == 0 || cs.advance(256)) // storage_extra_info$001 dict_hash:uint256 = StorageExtraInfo; + && cs.advance(32) // last_paid:uint32 + && t_Maybe_Grams.skip(cs); // due_payment:(Maybe Grams) } bool StorageInfo::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { - return t_StorageUsed.validate_skip(ops, cs, weak) // used:StorageUsed + int extra_tag = 0; + return t_StorageUsed.validate_skip(ops, cs, weak) // used:StorageUsed + && cs.fetch_uint_to(3, extra_tag) // storage_extra:StorageExtraInfo + && (extra_tag == 0 || + (extra_tag == 1 && cs.advance(256))) // storage_extra_info$001 dict_hash:uint256 = StorageExtraInfo; && cs.advance(32) // last_paid:uint32 && t_Maybe_Grams.validate_skip(ops, cs, weak); // due_payment:(Maybe Grams) } @@ -1368,7 +1361,7 @@ bool TrActionPhase::skip(vm::CellSlice& cs) const { && cs.advance(16 * 4 + 256) // tot_actions:uint16 spec_actions:uint16 // skipped_actions:uint16 msgs_created:uint16 // action_list_hash:uint256 - && t_StorageUsedShort.skip(cs); // tot_msg_size:StorageUsedShort + && t_StorageUsed.skip(cs); // tot_msg_size:StorageUsed } bool TrActionPhase::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { @@ -1381,7 +1374,7 @@ bool TrActionPhase::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const && cs.advance(16 * 4 + 256) // tot_actions:uint16 spec_actions:uint16 // skipped_actions:uint16 msgs_created:uint16 // action_list_hash:uint256 - && t_StorageUsedShort.validate_skip(ops, cs, weak); // tot_msg_size:StorageUsed + && t_StorageUsed.validate_skip(ops, cs, weak); // tot_msg_size:StorageUsed } const TrActionPhase t_TrActionPhase; @@ -1392,11 +1385,11 @@ bool TrBouncePhase::skip(vm::CellSlice& cs) const { return cs.advance(2); // tr_phase_bounce_negfunds$00 case tr_phase_bounce_nofunds: return cs.advance(2) // tr_phase_bounce_nofunds$01 - && t_StorageUsedShort.skip(cs) // msg_size:StorageUsedShort + && t_StorageUsed.skip(cs) // msg_size:StorageUsed && t_Grams.skip(cs); // req_fwd_fees:Grams case tr_phase_bounce_ok: return cs.advance(1) // tr_phase_bounce_ok$1 - && t_StorageUsedShort.skip(cs) // msg_size:StorageUsedShort + && t_StorageUsed.skip(cs) // msg_size:StorageUsed && t_Grams.skip(cs) // msg_fees:Grams && t_Grams.skip(cs); // fwd_fees:Grams } @@ -1409,11 +1402,11 @@ bool TrBouncePhase::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const return cs.advance(2); // tr_phase_bounce_negfunds$00 case tr_phase_bounce_nofunds: return cs.advance(2) // tr_phase_bounce_nofunds$01 - && t_StorageUsedShort.validate_skip(ops, cs, weak) // msg_size:StorageUsedShort + && t_StorageUsed.validate_skip(ops, cs, weak) // msg_size:StorageUsed && t_Grams.validate_skip(ops, cs, weak); // req_fwd_fees:Grams case tr_phase_bounce_ok: return cs.advance(1) // tr_phase_bounce_ok$1 - && t_StorageUsedShort.validate_skip(ops, cs, weak) // msg_size:StorageUsedShort + && t_StorageUsed.validate_skip(ops, cs, weak) // msg_size:StorageUsed && t_Grams.validate_skip(ops, cs, weak) // msg_fees:Grams && t_Grams.validate_skip(ops, cs, weak); // fwd_fees:Grams } diff --git a/crypto/block/block-parse.h b/crypto/block/block-parse.h index 65f8b91fe..fd17c6579 100644 --- a/crypto/block/block-parse.h +++ b/crypto/block/block-parse.h @@ -493,13 +493,6 @@ struct StorageUsed final : TLB_Complex { extern const StorageUsed t_StorageUsed; -struct StorageUsedShort final : TLB_Complex { - bool skip(vm::CellSlice& cs) const override; - bool validate_skip(int* ops, vm::CellSlice& cs, bool weak = false) const override; -}; - -extern const StorageUsedShort t_StorageUsedShort; - struct StorageInfo final : TLB_Complex { bool skip(vm::CellSlice& cs) const override; bool validate_skip(int* ops, vm::CellSlice& cs, bool weak = false) const override; diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 4a8bbc065..5cba3c69f 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -246,14 +246,13 @@ out_msg_queue_extra#0 dispatch_queue:DispatchQueue out_queue_size:(Maybe uint48) _ out_queue:OutMsgQueue proc_info:ProcessedInfo extra:(Maybe OutMsgQueueExtra) = OutMsgQueueInfo; -// -storage_used$_ cells:(VarUInteger 7) bits:(VarUInteger 7) - public_cells:(VarUInteger 7) = StorageUsed; -storage_used_short$_ cells:(VarUInteger 7) - bits:(VarUInteger 7) = StorageUsedShort; +storage_extra_none$000 = StorageExtraInfo; +storage_extra_info$001 dict_hash:uint256 = StorageExtraInfo; + +storage_used$_ cells:(VarUInteger 7) bits:(VarUInteger 7) = StorageUsed; -storage_info$_ used:StorageUsed last_paid:uint32 +storage_info$_ used:StorageUsed storage_extra:StorageExtraInfo last_paid:uint32 due_payment:(Maybe Grams) = StorageInfo; account_none$0 = Account; @@ -341,13 +340,13 @@ tr_phase_action$_ success:Bool valid:Bool no_funds:Bool total_fwd_fees:(Maybe Grams) total_action_fees:(Maybe Grams) result_code:int32 result_arg:(Maybe int32) tot_actions:uint16 spec_actions:uint16 skipped_actions:uint16 msgs_created:uint16 - action_list_hash:bits256 tot_msg_size:StorageUsedShort + action_list_hash:bits256 tot_msg_size:StorageUsed = TrActionPhase; tr_phase_bounce_negfunds$00 = TrBouncePhase; -tr_phase_bounce_nofunds$01 msg_size:StorageUsedShort +tr_phase_bounce_nofunds$01 msg_size:StorageUsed req_fwd_fees:Grams = TrBouncePhase; -tr_phase_bounce_ok$1 msg_size:StorageUsedShort +tr_phase_bounce_ok$1 msg_size:StorageUsed msg_fees:Grams fwd_fees:Grams = TrBouncePhase; // trans_ord$0000 credit_first:Bool diff --git a/crypto/block/create-state.cpp b/crypto/block/create-state.cpp index c8c8b970d..4a74ac0f6 100644 --- a/crypto/block/create-state.cpp +++ b/crypto/block/create-state.cpp @@ -338,10 +338,11 @@ td::RefInt256 create_smartcontract(td::RefInt256 smc_addr, Ref code, R PDO(cb.store_long_rchk_bool(workchain_id, ctor == 2 ? 8 : 32) && cb.store_bits_bool(addr.cbits(), 256)); THRERR("Cannot serialize addr:MsgAddressInt of the new smart contract"); // storage_stat:StorageInfo -> storage_stat.used:StorageUsed - PDO(block::store_UInt7(cb, stats.cells) // cells:(VarUInteger 7) - && block::store_UInt7(cb, stats.bits) // bits:(VarUInteger 7) - && block::store_UInt7(cb, stats.public_cells)); // public_cells:(VarUInteger 7) + PDO(block::store_UInt7(cb, stats.cells) // cells:(VarUInteger 7) + && block::store_UInt7(cb, stats.bits)) // bits:(VarUInteger 7) THRERR("Cannot serialize used:StorageUsed of the new smart contract"); + PDO(cb.store_zeroes_bool(3)); // extra:StorageExtraInfo + THRERR("Cannot serialize storage_extra:StorageExtraInfo of the new smart contract"); PDO(cb.store_long_bool(0, 33)); // last_paid:uint32 due_payment:(Maybe Grams) PDO(cb.append_data_cell_bool(storage)); // storage:AccountStorage THRERR("Cannot create Account of the new smart contract"); diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index a44522f11..8c1afd965 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -156,6 +156,10 @@ class McShardHashI : public td::CntObject { virtual bool before_merge() const = 0; }; +struct StorageUsed { + td::uint64 cells = 0, bits = 0; +}; + struct McShardHash : public McShardHashI { ton::BlockIdExt blk_; ton::LogicalTime start_lt_, end_lt_; @@ -336,7 +340,7 @@ struct StoragePrices { , mc_cell_price(_mc_cprice) { } static td::RefInt256 compute_storage_fees(ton::UnixTime now, const std::vector& pricing, - const vm::CellStorageStat& storage_stat, ton::UnixTime last_paid, + const StorageUsed& storage_used, ton::UnixTime last_paid, bool is_special, bool is_masterchain); }; diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 34d235114..417ce0cf4 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -257,6 +257,11 @@ bool Account::unpack_storage_info(vm::CellSlice& cs) { return false; } last_paid = info.last_paid; + if (info.storage_extra.write().fetch_long(3) == 1) { + info.storage_extra->prefetch_bits_to(storage_dict_hash.value_force()); + } else { + storage_dict_hash = {}; + } if (info.due_payment->prefetch_ulong(1) == 1) { vm::CellSlice& cs2 = info.due_payment.write(); cs2.advance(1); @@ -268,11 +273,9 @@ bool Account::unpack_storage_info(vm::CellSlice& cs) { due_payment = td::zero_refint(); } unsigned long long u = 0; - u |= storage_stat.cells = block::tlb::t_VarUInteger_7.as_uint(*used.cells); - u |= storage_stat.bits = block::tlb::t_VarUInteger_7.as_uint(*used.bits); - u |= storage_stat.public_cells = block::tlb::t_VarUInteger_7.as_uint(*used.public_cells); - LOG(DEBUG) << "last_paid=" << last_paid << "; cells=" << storage_stat.cells << " bits=" << storage_stat.bits - << " public_cells=" << storage_stat.public_cells; + u |= storage_used.cells = block::tlb::t_VarUInteger_7.as_uint(*used.cells); + u |= storage_used.bits = block::tlb::t_VarUInteger_7.as_uint(*used.bits); + LOG(DEBUG) << "last_paid=" << last_paid << "; cells=" << storage_used.cells << " bits=" << storage_used.bits; return (u != std::numeric_limits::max()); } @@ -527,7 +530,8 @@ bool Account::init_new(ton::UnixTime now) { last_trans_hash_.set_zero(); now_ = now; last_paid = 0; - storage_stat.clear(); + storage_used = {}; + storage_dict_hash = {}; due_payment = td::zero_refint(); balance.set_zero(); if (my_addr_exact.is_null()) { @@ -617,12 +621,12 @@ bool Account::belongs_to_shard(ton::ShardIdFull shard) const { * @param payment The total sum to be updated. * @param delta The time delta for which the payment is calculated. * @param prices The storage prices. - * @param storage Account storage statistics. + * @param storage_used Account storage statistics. * @param is_mc A flag indicating whether the account is in the masterchain. */ void add_partial_storage_payment(td::BigInt256& payment, ton::UnixTime delta, const block::StoragePrices& prices, - const vm::CellStorageStat& storage, bool is_mc) { - td::BigInt256 c{(long long)storage.cells}, b{(long long)storage.bits}; + const StorageUsed& storage_used, bool is_mc) { + td::BigInt256 c{(long long)storage_used.cells}, b{(long long)storage_used.bits}; if (is_mc) { // storage.cells * prices.mc_cell_price + storage.bits * prices.mc_bit_price; c.mul_short(prices.mc_cell_price); @@ -643,7 +647,7 @@ void add_partial_storage_payment(td::BigInt256& payment, ton::UnixTime delta, co * * @param now The current Unix time. * @param pricing The vector of storage prices. - * @param storage_stat Account storage statistics. + * @param storage_used Account storage statistics. * @param last_paid The Unix time when the last payment was made. * @param is_special A flag indicating if the account is special. * @param is_masterchain A flag indicating if the account is in the masterchain. @@ -651,7 +655,7 @@ void add_partial_storage_payment(td::BigInt256& payment, ton::UnixTime delta, co * @returns The computed storage fees as RefInt256. */ td::RefInt256 StoragePrices::compute_storage_fees(ton::UnixTime now, const std::vector& pricing, - const vm::CellStorageStat& storage_stat, ton::UnixTime last_paid, + const StorageUsed& storage_used, ton::UnixTime last_paid, bool is_special, bool is_masterchain) { if (now <= last_paid || !last_paid || is_special || pricing.empty() || now <= pricing[0].valid_since) { return td::zero_refint(); @@ -669,7 +673,7 @@ td::RefInt256 StoragePrices::compute_storage_fees(ton::UnixTime now, const std:: ton::UnixTime valid_until = (i < n - 1 ? std::min(now, pricing[i + 1].valid_since) : now); if (upto < valid_until) { assert(upto >= pricing[i].valid_since); - add_partial_storage_payment(total.unique_write(), valid_until - upto, pricing[i], storage_stat, is_masterchain); + add_partial_storage_payment(total.unique_write(), valid_until - upto, pricing[i], storage_used, is_masterchain); } upto = valid_until; } @@ -685,7 +689,7 @@ td::RefInt256 StoragePrices::compute_storage_fees(ton::UnixTime now, const std:: * @returns The computed storage fees as RefInt256. */ td::RefInt256 Account::compute_storage_fees(ton::UnixTime now, const std::vector& pricing) const { - return StoragePrices::compute_storage_fees(now, pricing, storage_stat, last_paid, is_special, is_masterchain()); + return StoragePrices::compute_storage_fees(now, pricing, storage_used, last_paid, is_special, is_masterchain()); } namespace transaction { @@ -1848,7 +1852,7 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { if (S.is_error()) { // Rollback changes to state, fail action phase LOG(INFO) << "Account state size exceeded limits: " << S.move_as_error(); - new_storage_stat.clear(); + new_account_storage_stat = {}; new_code = old_code; new_data = old_data; new_library = old_library; @@ -2104,7 +2108,7 @@ int Transaction::try_action_change_library(vm::CellSlice& cs, ActionPhase& ap, c LOG(DEBUG) << "added " << ((rec.mode >> 1) ? "public" : "private") << " library with hash " << hash.to_hex(); } new_library = std::move(dict).extract_root_cell(); - } catch (vm::VmError& vme) { + } catch (vm::VmError&) { return 42; } ap.spec_actions++; @@ -2931,7 +2935,7 @@ static td::uint32 get_public_libraries_diff_count(const td::Ref& old_l * This function is not called for special accounts. * * @param size_limits The size limits configuration. - * @param update_storage_stat Store storage stat in the Transaction's CellStorageStat. + * @param update_storage_stat Store storage stat in the Transaction's AccountStorageStat. * * @returns A `td::Status` indicating the result of the check. * - If the state limits are within the allowed range, returns OK. @@ -2939,60 +2943,44 @@ static td::uint32 get_public_libraries_diff_count(const td::Ref& old_l */ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, bool update_storage_stat) { auto cell_equal = [](const td::Ref& a, const td::Ref& b) -> bool { - if (a.is_null()) { - return b.is_null(); - } - if (b.is_null()) { - return false; - } - return a->get_hash() == b->get_hash(); + return a.is_null() || b.is_null() ? a.is_null() == b.is_null() : a->get_hash() == b->get_hash(); }; if (cell_equal(account.code, new_code) && cell_equal(account.data, new_data) && cell_equal(account.library, new_library)) { return td::Status::OK(); } - vm::CellStorageStat storage_stat; - storage_stat.limit_cells = size_limits.max_acc_state_cells; - storage_stat.limit_bits = size_limits.max_acc_state_bits; + AccountStorageStat storage_stat; + if (update_storage_stat && account.account_storage_stat) { + storage_stat = AccountStorageStat{&account.account_storage_stat.value()}; + } { TD_PERF_COUNTER(transaction_storage_stat_a); td::Timer timer; - auto add_used_storage = [&](const td::Ref& cell) -> td::Status { - if (cell.not_null()) { - TRY_RESULT(res, storage_stat.add_used_storage(cell)); - if (res.max_merkle_depth > max_allowed_merkle_depth) { - return td::Status::Error("too big merkle depth"); - } - } - return td::Status::OK(); - }; - TRY_STATUS(add_used_storage(new_code)); - TRY_STATUS(add_used_storage(new_data)); - TRY_STATUS(add_used_storage(new_library)); + TRY_STATUS(storage_stat.replace_roots({new_code, new_data, new_library}, /* check_merkle_depth = */ true)); if (timer.elapsed() > 0.1) { - LOG(INFO) << "Compute used storage took " << timer.elapsed() << "s"; + LOG(INFO) << "Compute used storage (1) took " << timer.elapsed() << "s"; } } - if (acc_status == Account::acc_active) { - storage_stat.clear_limit(); - } else { - storage_stat.clear(); - } - td::Status res; - if (storage_stat.cells > size_limits.max_acc_state_cells || storage_stat.bits > size_limits.max_acc_state_bits) { - res = td::Status::Error(PSTRING() << "account state is too big"); - } else if (account.is_masterchain() && !cell_equal(account.library, new_library) && - get_public_libraries_count(new_library) > size_limits.max_acc_public_libraries) { - res = td::Status::Error("too many public libraries"); - } else { - res = td::Status::OK(); + if (storage_stat.get_total_cells() > size_limits.max_acc_state_cells || + storage_stat.get_total_bits() > size_limits.max_acc_state_bits) { + return td::Status::Error(PSTRING() << "account state is too big: cells=" << storage_stat.get_total_cells() + << ", bits=" << storage_stat.get_total_bits() + << " (max cells=" << size_limits.max_acc_state_cells + << ", max bits=" << size_limits.max_acc_state_bits << ")"); + } + if (account.is_masterchain() && !cell_equal(account.library, new_library)) { + auto libraries_count = get_public_libraries_count(new_library); + if (libraries_count > size_limits.max_acc_public_libraries) { + return td::Status::Error(PSTRING() << "too many public libraries: " << libraries_count << " (max " + << size_limits.max_acc_public_libraries << ")"); + } } if (update_storage_stat) { // storage_stat will be reused in compute_state() - new_storage_stat = std::move(storage_stat); + new_account_storage_stat.value_force() = std::move(storage_stat); } - return res; + return td::Status::OK(); } /** @@ -3140,44 +3128,6 @@ bool Account::store_acc_status(vm::CellBuilder& cb, int acc_status) const { return cb.store_long_bool(v, 2); } -/** - * Tries to update the storage statistics based on the old storage statistics and old account state without fully recomputing it. - * - * It succeeds if only root cell of AccountStorage is changed. - * old_cs and new_cell are AccountStorage without extra currencies (if global_version >= 10). - * - * @param old_stat The old storage statistics. - * @param old_cs The old AccountStorage. - * @param new_cell The new AccountStorage. - * - * @returns An optional value of type vm::CellStorageStat. If the update is successful, it returns the new storage statistics. Otherwise, it returns an empty optional. - */ -static td::optional try_update_storage_stat(const vm::CellStorageStat& old_stat, - td::Ref old_cs, - td::Ref new_cell) { - if (old_stat.cells == 0 || old_cs.is_null()) { - return {}; - } - vm::CellSlice new_cs = vm::CellSlice(vm::NoVm(), new_cell); - if (old_cs->size_refs() != new_cs.size_refs()) { - return {}; - } - for (unsigned i = 0; i < old_cs->size_refs(); ++i) { - if (old_cs->prefetch_ref(i)->get_hash() != new_cs.prefetch_ref(i)->get_hash()) { - return {}; - } - } - if (old_stat.bits < old_cs->size()) { - return {}; - } - - vm::CellStorageStat new_stat; - new_stat.cells = old_stat.cells; - new_stat.bits = old_stat.bits - old_cs->size() + new_cs.size(); - new_stat.public_cells = old_stat.public_cells; - return new_stat; -} - /** * Removes extra currencies dict from AccountStorage. * @@ -3185,9 +3135,9 @@ static td::optional try_update_storage_stat(const vm::CellS * * @param storage_cs AccountStorage as CellSlice. * - * @returns AccountStorage without extra currencies as Cell. + * @returns AccountStorage without extra currencies as CellSlice. */ -static td::Ref storage_without_extra_currencies(td::Ref storage_cs) { +static td::Ref storage_without_extra_currencies(td::Ref storage_cs) { block::gen::AccountStorage::Record rec; if (!block::gen::csr_unpack(storage_cs, rec)) { LOG(ERROR) << "failed to unpack AccountStorage"; @@ -3205,18 +3155,20 @@ static td::Ref storage_without_extra_currencies(td::Ref return {}; } } - td::Ref cell; - if (!block::gen::pack_cell(cell, rec)) { + td::Ref result; + if (!block::gen::csr_pack(result, rec)) { LOG(ERROR) << "failed to pack AccountStorage"; return {}; } - return cell; + return result; } namespace transaction { /** * Computes the new state of the account. * + * @param cfg The configuration for the serialization phase. + * * @returns True if the state computation is successful, false otherwise. */ bool Transaction::compute_state(const SerializeConfig& cfg) { @@ -3290,45 +3242,92 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { } else { new_inner_state.clear(); } - vm::CellStorageStat& stats = new_storage_stat; + td::Ref old_storage_for_stat = account.storage; - td::Ref new_storage_for_stat = storage; + td::Ref new_storage_for_stat = new_storage; if (cfg.extra_currency_v2) { new_storage_for_stat = storage_without_extra_currencies(new_storage); if (new_storage_for_stat.is_null()) { return false; } if (old_storage_for_stat.not_null()) { - old_storage_for_stat = vm::load_cell_slice_ref(storage_without_extra_currencies(old_storage_for_stat)); + old_storage_for_stat = storage_without_extra_currencies(old_storage_for_stat); if (old_storage_for_stat.is_null()) { return false; } } + } else if (cfg.store_storage_dict_hash) { + LOG(ERROR) << "unsupported store_storage_dict_hash=true, extra_currency_v2=false"; + return false; } - auto new_stats = try_update_storage_stat(account.storage_stat, old_storage_for_stat, storage); - if (new_stats) { - stats = new_stats.unwrap(); + + bool storage_refs_changed = false; + if (old_storage_for_stat.is_null() || new_storage_for_stat->size_refs() != old_storage_for_stat->size_refs()) { + storage_refs_changed = true; } else { + for (unsigned i = 0; i < new_storage_for_stat->size_refs(); i++) { + if (new_storage_for_stat->prefetch_ref(i)->get_hash() != old_storage_for_stat->prefetch_ref(i)->get_hash()) { + storage_refs_changed = true; + break; + } + } + } + + bool store_storage_dict_hash = cfg.store_storage_dict_hash && !account.is_masterchain(); + if (storage_refs_changed || + (store_storage_dict_hash && !account.storage_dict_hash && account.storage_used.cells > 25)) { TD_PERF_COUNTER(transaction_storage_stat_b); td::Timer timer; - stats.add_used_storage(new_storage_for_stat).ensure(); + if (!new_account_storage_stat && account.account_storage_stat) { + new_account_storage_stat = AccountStorageStat(&account.account_storage_stat.value()); + } + AccountStorageStat& stats = new_account_storage_stat.value_force(); + // Don't check Merkle depth and size here - they were checked in check_state_limits + td::Status S = stats.replace_roots(new_storage_for_stat->prefetch_all_refs()); + if (S.is_error()) { + LOG(ERROR) << "Cannot recompute storage stats for account " << account.addr.to_hex() << ": " << S.move_as_error(); + return false; + } + // Root of AccountStorage is not counted in AccountStorageStat + new_storage_used.cells = stats.get_total_cells() + 1; + new_storage_used.bits = stats.get_total_bits() + new_storage_for_stat->size(); + // TODO: think about this limit (25) + if (store_storage_dict_hash && new_storage_used.cells > 25) { + auto r_hash = stats.get_dict_hash(); + if (r_hash.is_error()) { + LOG(ERROR) << "Cannot compute storage dict hash for account " << account.addr.to_hex() << ": " + << r_hash.move_as_error(); + return false; + } + new_storage_dict_hash = r_hash.move_as_ok(); + } if (timer.elapsed() > 0.1) { - LOG(INFO) << "Compute used storage took " << timer.elapsed() << "s"; + LOG(INFO) << "Compute used storage (2) took " << timer.elapsed() << "s"; } - } - CHECK(cb.store_long_bool(1, 1) // account$1 - && cb.append_cellslice_bool(account.my_addr) // addr:MsgAddressInt - && block::store_UInt7(cb, stats.cells) // storage_used$_ cells:(VarUInteger 7) - && block::store_UInt7(cb, stats.bits) // bits:(VarUInteger 7) - && block::store_UInt7(cb, stats.public_cells) // public_cells:(VarUInteger 7) - && cb.store_long_bool(last_paid, 32)); // last_paid:uint32 + } else { + new_storage_used = account.storage_used; + new_storage_used.bits -= old_storage_for_stat->size(); + new_storage_used.bits += new_storage_for_stat->size(); + new_account_storage_stat = {}; + if (store_storage_dict_hash) { + new_storage_dict_hash = account.storage_dict_hash; + } + } + + CHECK(cb.store_long_bool(1, 1) // account$1 + && cb.append_cellslice_bool(account.my_addr) // addr:MsgAddressInt + && block::store_UInt7(cb, new_storage_used.cells) // storage_used$_ cells:(VarUInteger 7) + && block::store_UInt7(cb, new_storage_used.bits) // bits:(VarUInteger 7) + && cb.store_long_bool(new_storage_dict_hash ? 1 : 0, 3) // extra:StorageExtraInfo + && (!new_storage_dict_hash || cb.store_bits_bool(new_storage_dict_hash.value())) // dict_hash:uint256 + && cb.store_long_bool(last_paid, 32)); // last_paid:uint32 if (due_payment.not_null() && td::sgn(due_payment) != 0) { CHECK(cb.store_long_bool(1, 1) && block::tlb::t_Grams.store_integer_ref(cb, due_payment)); // due_payment:(Maybe Grams) } else { CHECK(cb.store_long_bool(0, 1)); } - CHECK(cb.append_data_cell_bool(std::move(storage))); + CHECK(cb.append_cellslice_bool(new_storage)); new_total_state = cb.finalize(); if (verbosity > 2) { FLOG(INFO) { @@ -3345,6 +3344,8 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { * * Updates root. * + * @param cfg The configuration for the serialization. + * * @returns True if the serialization is successful, False otherwise. */ bool Transaction::serialize(const SerializeConfig& cfg) { @@ -3688,7 +3689,15 @@ Ref Transaction::commit(Account& acc) { acc.last_trans_end_lt_ = end_lt; acc.last_trans_hash_ = root->get_hash().bits(); acc.last_paid = last_paid; - acc.storage_stat = new_storage_stat; + acc.storage_used = new_storage_used; + if (new_account_storage_stat) { + if (acc.account_storage_stat) { + acc.account_storage_stat.value().apply_child_stat(std::move(new_account_storage_stat.value())); + } else { + acc.account_storage_stat = std::move(new_account_storage_stat); + } + } + acc.storage_dict_hash = new_storage_dict_hash; acc.storage = new_storage; acc.balance = std::move(balance); acc.due_payment = std::move(due_payment); @@ -3936,6 +3945,7 @@ td::Status FetchConfigParams::fetch_config_params( } { serialize_cfg->extra_currency_v2 = config.get_global_version() >= 10; + serialize_cfg->store_storage_dict_hash = config.get_global_version() >= 11; } { // fetch block_grams_created diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index 8e612e6a5..aa08719a8 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -17,6 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "account-storage-stat.h" #include "common/refcnt.hpp" #include "common/refint.h" #include "vm/cells.h" @@ -179,6 +180,7 @@ struct ActionPhaseConfig { struct SerializeConfig { bool extra_currency_v2{false}; + bool store_storage_dict_hash{false}; }; struct CreditPhase { @@ -266,8 +268,12 @@ struct Account { ton::LogicalTime last_trans_lt_; ton::Bits256 last_trans_hash_; ton::LogicalTime block_lt; + ton::UnixTime last_paid; - vm::CellStorageStat storage_stat; + StorageUsed storage_used; + td::optional storage_dict_hash; + td::optional account_storage_stat; + block::CurrencyCollection balance; td::RefInt256 due_payment; Ref orig_total_state; // ^Account @@ -377,7 +383,9 @@ struct Transaction { std::unique_ptr compute_phase; std::unique_ptr action_phase; std::unique_ptr bounce_phase; - vm::CellStorageStat new_storage_stat; + StorageUsed new_storage_used; + td::optional new_account_storage_stat; + td::optional new_storage_dict_hash; bool gas_limit_overridden{false}; Transaction(const Account& _account, int ttype, ton::LogicalTime req_start_lt, ton::UnixTime _now, Ref _inmsg = {}); diff --git a/crypto/common/bitstring.cpp b/crypto/common/bitstring.cpp index 52e57c9a8..3a6f33119 100644 --- a/crypto/common/bitstring.cpp +++ b/crypto/common/bitstring.cpp @@ -347,6 +347,9 @@ std::size_t bits_memscan_rev(ConstBitPtr bs, std::size_t bit_count, bool cmp_to) int bits_memcmp(const unsigned char* bs1, int bs1_offs, const unsigned char* bs2, int bs2_offs, std::size_t bit_count, std::size_t* same_upto) { if (!bit_count) { + if (same_upto) { + *same_upto = 0; + } return 0; } bs1 += (bs1_offs >> 3); diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index 4f3fe5fcf..be1de25ca 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -113,21 +113,19 @@ class NewCellStorageStat { struct CellStorageStat { unsigned long long cells; unsigned long long bits; - unsigned long long public_cells; struct CellInfo { td::uint32 max_merkle_depth = 0; }; td::HashMap seen; - CellStorageStat() : cells(0), bits(0), public_cells(0) { + CellStorageStat() : cells(0), bits(0) { } - explicit CellStorageStat(unsigned long long limit_cells) - : cells(0), bits(0), public_cells(0), limit_cells(limit_cells) { + explicit CellStorageStat(unsigned long long limit_cells) : cells(0), bits(0), limit_cells(limit_cells) { } void clear_seen() { seen.clear(); } void clear() { - cells = bits = public_cells = 0; + cells = bits = 0; clear_limit(); clear_seen(); } diff --git a/crypto/vm/cells/CellSlice.cpp b/crypto/vm/cells/CellSlice.cpp index 9cd3e931a..bea20f95d 100644 --- a/crypto/vm/cells/CellSlice.cpp +++ b/crypto/vm/cells/CellSlice.cpp @@ -773,6 +773,14 @@ bool CellSlice::prefetch_maybe_ref(Ref& res) const { } } +std::vector> CellSlice::prefetch_all_refs() const { + std::vector> res(size_refs()); + for (unsigned i = 0; i < size_refs(); ++i) { + res[i] = prefetch_ref(i); + } + return res; +} + bool CellSlice::fetch_maybe_ref(Ref& res) { auto z = prefetch_ulong(1); if (!z) { diff --git a/crypto/vm/cells/CellSlice.h b/crypto/vm/cells/CellSlice.h index ecce30f5c..7525272b5 100644 --- a/crypto/vm/cells/CellSlice.h +++ b/crypto/vm/cells/CellSlice.h @@ -190,6 +190,7 @@ class CellSlice : public td::CntObject { } bool fetch_maybe_ref(Ref& ref); bool prefetch_maybe_ref(Ref& ref) const; + std::vector> prefetch_all_refs() const; td::BitSlice fetch_bits(unsigned bits); td::BitSlice prefetch_bits(unsigned bits) const; td::Ref fetch_subslice(unsigned bits, unsigned refs = 0); diff --git a/crypto/vm/dict.cpp b/crypto/vm/dict.cpp index 41f9c3396..e1e683fe3 100644 --- a/crypto/vm/dict.cpp +++ b/crypto/vm/dict.cpp @@ -1772,6 +1772,260 @@ void Dictionary::map(const simple_map_func_t& simple_map_func) { map(map_func); } +bool Dictionary::multiset(td::MutableSpan>> new_values) { + force_validate(); + auto cmp = [&](const std::pair>& a, + const std::pair>& b) { + return td::bitstring::bits_memcmp(a.first, b.first, key_bits) < 0; + }; + if (!std::is_sorted(new_values.begin(), new_values.end(), cmp)) { + std::sort(new_values.begin(), new_values.end(), cmp); + } + for (size_t i = 0; i + 1 < new_values.size(); ++i) { + if (td::bitstring::bits_memcmp(new_values[i].first, new_values[i + 1].first, key_bits) == 0) { + return false; + } + } + unsigned char key_buffer[max_key_bytes]; + try { + Ref root = dict_multiset(get_root_cell(), new_values, key_buffer, key_bits, key_bits, 0); + set_root_cell(std::move(root)); + return true; + } catch (CombineError) { + return false; + } +} + +static Ref dict_build(td::Span>> values, int total_key_len, + int prefix_len) { + if (values.empty()) { + return {}; + } + if (values.size() == 1) { + if (values[0].second.is_null()) { + throw CombineError{}; + } + CellBuilder cb; + append_dict_label(cb, values[0].first + prefix_len, total_key_len - prefix_len, total_key_len - prefix_len); + if (!cb.append_builder_bool(values[0].second)) { + throw VmError{Excno::cell_ov, "cannot store new value into a dictionary cell"}; + } + return cb.finalize(); + } + size_t common_prefix_len; + td::bitstring::bits_memcmp(values.front().first + prefix_len, values.back().first + prefix_len, + total_key_len - prefix_len, &common_prefix_len); + CHECK(prefix_len + common_prefix_len < total_key_len); + size_t idx = 0; + while (values[idx].first[prefix_len + common_prefix_len] == 0) { + ++idx; + } + Ref left = dict_build(values.substr(0, idx), total_key_len, prefix_len + common_prefix_len + 1); + Ref right = dict_build(values.substr(idx), total_key_len, prefix_len + common_prefix_len + 1); + CellBuilder cb; + append_dict_label(cb, values[0].first + prefix_len, common_prefix_len, total_key_len - prefix_len); + if (!(cb.store_ref_bool(std::move(left)) && cb.store_ref_bool(std::move(right)))) { + throw VmError{Excno::dict_err, "cannot store branch references into a dictionary fork cell"}; + } + return cb.finalize(); +} + +// Based on DictionaryFixed::dict_combine_with, but dict2 is replaced with a list of values, mode is false +Ref Dictionary::dict_multiset(Ref dict1, td::Span>> values2, + td::BitPtr key_buffer, int n, int total_key_len, int skip1) { + int prefix_len = total_key_len - n; + for (auto& [k, _] : values2) { + CHECK(td::bitstring::bits_memcmp(k, key_buffer - prefix_len, prefix_len) == 0); + } + if (dict1.is_null()) { + return dict_build(values2, total_key_len, prefix_len); + } + if (values2.empty()) { + assert(!skip1); + return dict1; + } + size_t common_prefix_len; + td::bitstring::bits_memcmp(values2.front().first + prefix_len, values2.back().first + prefix_len, + total_key_len - prefix_len, &common_prefix_len); + assert(prefix_len + common_prefix_len < total_key_len || values2.size() == 1); + // both dictionaries non-empty + // skip1: remove that much first bits from all keys in dictionary dict1 (its keys are actually n + skip1 bits long) + // resulting dictionary will have n-bit keys + LabelParser label1{dict1, n + skip1, LabelParser::chk_all}; + int l1 = label1.l_bits - skip1, l2 = (int)common_prefix_len; + assert(l1 >= 0 && l2 >= 0); + assert(!skip1 || label1.common_prefix_len(key_buffer - skip1, skip1) == skip1); + int c = label1.common_prefix_len(values2[0].first + prefix_len - skip1, skip1 + l2) - skip1; + label1.extract_label_to(key_buffer - skip1); + assert(c >= 0 && c <= l1 && c <= l2); + if (c < l1 && c < l2) { + // the two dictionaries have disjoint keys + CellBuilder cb; + append_dict_label(cb, key_buffer + c + 1, l1 - c - 1, n - c - 1); + if (!cell_builder_add_slice_bool(cb, *label1.remainder)) { + throw VmError{Excno::cell_ov, "cannot prune label of an old dictionary cell while merging dictionaries"}; + } + label1.remainder.clear(); + dict1 = cb.finalize(); + // cb.reset(); // included into finalize(); + // now dict1 has been "pruned" -- first skip1+c+1 bits removed from its root egde label + Ref dict2 = dict_build(values2, total_key_len, prefix_len + c + 1); + if (!values2[0].first[prefix_len + c]) { + std::swap(dict1, dict2); + } + // put dict1 into the left tree (with smaller labels), dict2 into the right tree + append_dict_label(cb, key_buffer, c, n); + if (!(cb.store_ref_bool(std::move(dict1)) && cb.store_ref_bool(std::move(dict2)))) { + throw VmError{Excno::dict_err, "cannot store branch references into a dictionary fork cell"}; + } + return cb.finalize(); + } + + auto combine_func = [&](CellBuilder& cb, const Ref& cb2) -> bool { + if (cb2.is_null()) { + return false; + } + cb.append_builder(*cb2); + return true; + }; + size_t idx = 0; + while (prefix_len + common_prefix_len < total_key_len && idx < values2.size() && + values2[idx].first[prefix_len + common_prefix_len] == 0) { + ++idx; + } + auto values2_left = values2.substr(0, idx); + auto values2_right = values2.substr(idx); + + if (c == l1 && c == l2) { + // funny enough, the non-skipped parts of labels of l1 and l2 match + CellBuilder cb; + append_dict_label(cb, key_buffer, c, n); + if (c == n) { + // our two dictionaries are in fact leafs with matching edge labels (keys) + if (!combine_func(cb, values2[0].second)) { + // alas, the two values did not combine, this key will be absent from resulting dictionary + return {}; + } + return cb.finalize(); + } + assert(c < n); + key_buffer += c + 1; + key_buffer[-1] = 0; + // combine left subtrees + auto c1 = dict_multiset(label1.remainder->prefetch_ref(0), values2_left, key_buffer, n - c - 1, total_key_len, 0); + key_buffer[-1] = 1; + // combine right subtrees + auto c2 = dict_multiset(label1.remainder->prefetch_ref(1), values2_right, key_buffer, n - c - 1, total_key_len, 0); + label1.remainder.clear(); + // c1 and c2 are merged left and right children of dict1 and dict2 + if (!c1.is_null() && !c2.is_null()) { + // both children non-empty, simply put them into the new node + if (!(cb.store_ref_bool(std::move(c1)) && cb.store_ref_bool(std::move(c2)))) { + throw VmError{Excno::dict_err, "cannot store branch references into a dictionary fork cell"}; + } + return cb.finalize(); + } + if (c1.is_null() && c2.is_null()) { + return {}; // both children empty, resulting dictionary also empty + } + // exactly one of c1 and c2 is non-empty, have to merge labels + bool sw = c1.is_null(); + key_buffer[-1] = sw; + if (sw) { + c1 = std::move(c2); + } + LabelParser label3{std::move(c1), n - c - 1, LabelParser::chk_all}; + label3.extract_label_to(key_buffer); + key_buffer -= c + 1; + // store combined label for the new edge + cb.reset(); + append_dict_label(cb, key_buffer, c + 1 + label3.l_bits, n); + // store payload + if (!cell_builder_add_slice_bool(cb, *label3.remainder)) { + throw VmError{Excno::cell_ov, "cannot change label of an old dictionary cell while merging edges"}; + } + return cb.finalize(); + } + if (c == l1) { + assert(c < l2); + dict1.clear(); + // children of root node of dict1 + auto c1 = label1.remainder->prefetch_ref(0); + auto c2 = label1.remainder->prefetch_ref(1); + label1.remainder.clear(); + // have to merge dict2 with one of the children of dict1 + td::bitstring::bits_memcpy(key_buffer, values2[0].first + prefix_len, l2); // dict2 has longer label, extract it + bool sw = key_buffer[c]; + if (!sw) { + // merge c1 with dict2 + c1 = dict_multiset(std::move(c1), values2, key_buffer + c + 1, n - c - 1, total_key_len, 0); + } else { + // merge c2 with dict2 + c2 = dict_multiset(std::move(c2), values2, key_buffer + c + 1, n - c - 1, total_key_len, 0); + } + if (!c1.is_null() && !c2.is_null()) { + CellBuilder cb; + append_dict_label(cb, key_buffer, c, n); + if (!(cb.store_ref_bool(std::move(c1)) && cb.store_ref_bool(std::move(c2)))) { + throw VmError{Excno::dict_err, "cannot store branch references into a dictionary fork cell"}; + } + return cb.finalize(); + } + // one of children is empty, have to merge root edges + key_buffer[c] = !sw; + if (!sw) { + std::swap(c1, c2); + } + assert(!c1.is_null() && c2.is_null()); + LabelParser label3{std::move(c1), n - c - 1, LabelParser::chk_all}; + label3.extract_label_to(key_buffer + c + 1); + CellBuilder cb; + append_dict_label(cb, key_buffer, c + 1 + label3.l_bits, n); + // store payload + if (!cell_builder_add_slice_bool(cb, *label3.remainder)) { + throw VmError{Excno::cell_ov, "cannot change label of an old dictionary cell while merging edges"}; + } + return cb.finalize(); + } else { + assert(c == l2 && c < l1); + // have to merge dict1 with one of the children of dict2 + bool sw = key_buffer[c]; + Ref c1, c2; + if (!sw) { + // merge dict1 with c1 + c1 = dict_multiset(std::move(dict1), values2_left, key_buffer + c + 1, n - c - 1, total_key_len, skip1 + c + 1); + c2 = dict_build(values2_right, total_key_len, prefix_len + l2 + 1); + } else { + // merge dict1 with c2 + c2 = dict_multiset(std::move(dict1), values2_right, key_buffer + c + 1, n - c - 1, total_key_len, skip1 + c + 1); + c1 = dict_build(values2_left, total_key_len, prefix_len + l2 + 1); + } + if (!c1.is_null() && !c2.is_null()) { + CellBuilder cb; + append_dict_label(cb, key_buffer, c, n); + if (!(cb.store_ref_bool(std::move(c1)) && cb.store_ref_bool(std::move(c2)))) { + throw VmError{Excno::dict_err, "cannot store branch references into a dictionary fork cell"}; + } + return cb.finalize(); + } + // one of children is empty, have to merge root edges + key_buffer[c] = !sw; + if (!sw) { + std::swap(c1, c2); + } + assert(!c1.is_null() && c2.is_null()); + LabelParser label3{std::move(c1), n - c - 1, LabelParser::chk_all}; + label3.extract_label_to(key_buffer + c + 1); + CellBuilder cb; + append_dict_label(cb, key_buffer, c + 1 + label3.l_bits, n); + // store payload + if (!cell_builder_add_slice_bool(cb, *label3.remainder)) { + throw VmError{Excno::cell_ov, "cannot change label of an old dictionary cell while merging edges"}; + } + return cb.finalize(); + } +} + // mode: +1 = forbid empty dict1 with non-empty dict2 // +2 = forbid empty dict2 with non-empty dict1 Ref DictionaryFixed::dict_combine_with(Ref dict1, Ref dict2, td::BitPtr key_buffer, int n, diff --git a/crypto/vm/dict.h b/crypto/vm/dict.h index c4044963f..e22b90946 100644 --- a/crypto/vm/dict.h +++ b/crypto/vm/dict.h @@ -527,13 +527,15 @@ class Dictionary final : public DictionaryFixed { auto range(bool rev = false, bool sgnd = false) { return dict_range(*this, rev, sgnd); } + bool multiset(td::MutableSpan>> new_values); private: bool check_fork(CellSlice& cs, Ref c1, Ref c2, int n) const override { return cs.empty_ext(); } static Ref extract_value_ref(Ref cs); - std::pair, int> dict_filter(Ref dict, td::BitPtr key, int n, const filter_func_t& check_leaf) const; + static Ref dict_multiset(Ref dict1, td::Span>> values2, + td::BitPtr key_buffer, int n, int total_key_len, int skip1); }; class PrefixDictionary final : public DictionaryBase { diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index 77963e959..c0be0b108 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -156,3 +156,10 @@ Example: if the last masterchain block seqno is `19071` then the list contains b ### TVM changes - `SENDMSG` calculates messages size and fees without extra currencies, uses new +64 and +128 mode behavior. - `SENDMSG` does not check the number of extra currencies. + +## Version 11 +### New account storage stat +Along with the storage stat (cells and bits count), each account now stores the hash of the **storage dict**. + +**Storage dict** is the dictionary that stores refcnt for each cell in the account state. +This is required to help computing storage stats in the future, after collator-validator separation. \ No newline at end of file diff --git a/emulator/emulator-extern.cpp b/emulator/emulator-extern.cpp index eb5ff9f9e..c5f3c04e0 100644 --- a/emulator/emulator-extern.cpp +++ b/emulator/emulator-extern.cpp @@ -187,7 +187,7 @@ const char *transaction_emulator_emulate_transaction(void *transaction_emulator, external_not_accepted->elapsed_time); } - auto emulation_success = dynamic_cast(*emulation_result); + auto emulation_success = std::move(dynamic_cast(*emulation_result)); auto trans_boc_b64 = cell_to_boc_b64(std::move(emulation_success.transaction)); if (trans_boc_b64.is_error()) { ERROR_RESPONSE(PSTRING() << "Can't serialize Transaction to boc " << trans_boc_b64.move_as_error()); @@ -260,7 +260,8 @@ const char *transaction_emulator_emulate_tick_tock_transaction(void *transaction } auto emulation_result = result.move_as_ok(); - auto emulation_success = dynamic_cast(*emulation_result); + auto emulation_success = + std::move(dynamic_cast(*emulation_result)); auto trans_boc_b64 = cell_to_boc_b64(std::move(emulation_success.transaction)); if (trans_boc_b64.is_error()) { ERROR_RESPONSE(PSTRING() << "Can't serialize Transaction to boc " << trans_boc_b64.move_as_error()); diff --git a/emulator/transaction-emulator.cpp b/emulator/transaction-emulator.cpp index 6267f9bd0..abdf0d2cf 100644 --- a/emulator/transaction-emulator.cpp +++ b/emulator/transaction-emulator.cpp @@ -141,7 +141,7 @@ td::Result TransactionEmulator::emulate_t return td::Status::Error("account hash mismatch"); } - return emulation_result; + return std::move(emulation_result); } else if (auto emulation_not_accepted_ptr = dynamic_cast(emulation.get())) { return td::Status::Error( PSTRING() diff --git a/emulator/transaction-emulator.h b/emulator/transaction-emulator.h index eae109f40..848548c32 100644 --- a/emulator/transaction-emulator.h +++ b/emulator/transaction-emulator.h @@ -33,14 +33,18 @@ class TransactionEmulator { virtual ~EmulationResult() = default; }; - struct EmulationSuccess: EmulationResult { + struct EmulationSuccess : EmulationResult { td::Ref transaction; block::Account account; td::Ref actions; - EmulationSuccess(td::Ref transaction_, block::Account account_, std::string vm_log_, td::Ref actions_, double elapsed_time_) : - EmulationResult(vm_log_, elapsed_time_), transaction(transaction_), account(account_) , actions(actions_) - {} + EmulationSuccess(td::Ref transaction_, block::Account account_, std::string vm_log_, + td::Ref actions_, double elapsed_time_) + : EmulationResult(vm_log_, elapsed_time_) + , transaction(transaction_) + , account(std::move(account_)) + , actions(actions_) { + } }; struct EmulationExternalNotAccepted: EmulationResult { diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 00dcf22db..26a98408d 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -180,7 +180,7 @@ struct RawAccountState { td::Ref extra_currencies; ton::UnixTime storage_last_paid{0}; - vm::CellStorageStat storage_stat; + block::StorageUsed storage_used; td::Ref code; td::Ref data; @@ -1036,7 +1036,7 @@ class Query { TRY_RESULT(basechain_msg_prices, cfg->get_msg_prices(false)); block::MsgPrices* msg_prices[2] = {&basechain_msg_prices, &masterchain_msg_prices}; auto storage_fee_256 = block::StoragePrices::compute_storage_fees( - raw_.source->get_sync_time(), storage_prices, raw_.source->raw().storage_stat, + raw_.source->get_sync_time(), storage_prices, raw_.source->raw().storage_used, raw_.source->raw().storage_last_paid, false, is_masterchain); auto storage_fee = storage_fee_256.is_null() ? 0 : storage_fee_256->to_long(); @@ -1085,7 +1085,7 @@ class Query { TRY_RESULT(dest_gas_limits_prices, cfg->get_gas_limits_prices(dest_is_masterchain)); auto dest_storage_fee_256 = destination ? block::StoragePrices::compute_storage_fees( - destination->get_sync_time(), storage_prices, destination->raw().storage_stat, + destination->get_sync_time(), storage_prices, destination->raw().storage_used, destination->raw().storage_last_paid, false, is_masterchain) : td::make_refint(0); Fee dst_fee; @@ -1399,17 +1399,16 @@ class GetRawAccountState : public td::actor::Actor { return td::Status::Error("Failed to unpack StorageInfo"); } unsigned long long u = 0; - vm::CellStorageStat storage_stat; + block::StorageUsed storage_stat; u |= storage_stat.cells = block::tlb::t_VarUInteger_7.as_uint(*storage_used.cells); u |= storage_stat.bits = block::tlb::t_VarUInteger_7.as_uint(*storage_used.bits); - u |= storage_stat.public_cells = block::tlb::t_VarUInteger_7.as_uint(*storage_used.public_cells); //LOG(DEBUG) << "last_paid=" << res.storage_last_paid << "; cells=" << storage_stat.cells - //<< " bits=" << storage_stat.bits << " public_cells=" << storage_stat.public_cells; + //<< " bits=" << storage_stat.bits; if (u == std::numeric_limits::max()) { return td::Status::Error("Failed to unpack StorageStat"); } - res.storage_stat = storage_stat; + res.storage_used = storage_stat; } block::gen::AccountStorage::Record storage; @@ -2089,7 +2088,7 @@ class RunEmulator : public TonlibQueryActor { raw.balance = balance.grams->to_long(); raw.extra_currencies = balance.extra; raw.storage_last_paid = std::move(account.last_paid); - raw.storage_stat = std::move(account.storage_stat); + raw.storage_used = account.storage_used; raw.code = std::move(account.code); raw.data = std::move(account.data); raw.state = std::move(account.total_state); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 90966d820..5f2ba6615 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -1008,6 +1008,7 @@ bool ValidateQuery::fetch_config_params() { } { serialize_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; + serialize_cfg_.store_storage_dict_hash = config_->get_global_version() >= 11; } { // fetch block_grams_created From 56506aeab2bb6ea4d707efa8fd10a54f7a53a27f Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Mon, 14 Apr 2025 10:10:39 +0300 Subject: [PATCH 214/388] Fix finish_getDispatchQueueMessages with messages boc mask (#1623) Co-authored-by: ice-charon --- validator/impl/liteserver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 06f40b8fd..8457d234f 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -3672,7 +3672,7 @@ void LiteQuery::finish_getDispatchQueueMessages(StdSmcAddress addr, LogicalTime fatal_error(r_messages_boc.move_as_error()); return; } - messages_boc = std::move(messages_boc); + messages_boc = r_messages_boc.move_as_ok(); } LOG(INFO) << "getDispatchQueueMessages(" << blk_id_.to_str() << ", " << mode_ << ") query completed"; auto b = ton::create_serialize_tl_object( From 86e6726e4125c2fcec7ca7ea3f3be6143b09524c Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Mon, 14 Apr 2025 10:11:14 +0300 Subject: [PATCH 215/388] Report error to correct promise in tonlib dns request (#1622) Co-authored-by: ice-charon --- tonlib/tonlib/TonlibClient.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 26a98408d..4ddcbbfab 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -5043,7 +5043,7 @@ void TonlibClient::do_dns_request(std::string name, td::Bits256 category, td::in return; } - TRY_RESULT_PROMISE(promise, args, ton::DnsInterface::resolve_args(name, category, address)); + TRY_RESULT_PROMISE(new_promise, args, ton::DnsInterface::resolve_args(name, category, address)); int_api::RemoteRunSmcMethod query; query.address = std::move(address); query.args = std::move(args); From 92f865b94f77ab8d2bc286cfdcbdfaa2b2ad3771 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Fri, 21 Mar 2025 08:22:12 -0400 Subject: [PATCH 216/388] Use check_for_each for iterating over validator set This is negligible but still positive (~0.5%) improvement. --- crypto/block/mc-config.cpp | 40 ++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 7b4f326f8..d5aa65d5e 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -576,34 +576,50 @@ td::Result> Config::unpack_validator_set(Ref(rec.utime_since, rec.utime_until, rec.total, rec.main); - for (int i = 0; i < rec.total; i++) { - key_buffer.store_ulong(i); - auto descr_cs = dict.lookup(key_buffer.bits(), 16); - if (descr_cs.is_null()) { - return td::Status::Error("indices in a validator set dictionary must be integers 0..total-1"); - } + + std::vector seen_keys(rec.total); + td::Status error; + + auto validator_set_check_fn = [&](Ref descr_cs, td::ConstBitPtr key, int n) -> bool { + int i = static_cast(key.get_uint(n)); + CHECK(i >= 0 && i < rec.total && !seen_keys[i]); + seen_keys[i] = true; + gen::ValidatorDescr::Record_validator_addr descr; if (!tlb::csr_unpack(descr_cs, descr)) { descr.adnl_addr.set_zero(); if (!(gen::t_ValidatorDescr.unpack_validator(descr_cs.write(), descr.public_key, descr.weight) && descr_cs->empty_ext())) { - return td::Status::Error(PSLICE() << "validator #" << i - << " has an invalid ValidatorDescr record in the validator set dictionary"); + error = td::Status::Error(PSLICE() << "validator #" << i + << " has an invalid ValidatorDescr record in the validator set dictionary"); + return false; } } gen::SigPubKey::Record sig_pubkey; if (!tlb::csr_unpack(std::move(descr.public_key), sig_pubkey)) { - return td::Status::Error(PSLICE() << "validator #" << i - << " has no public key or its public key is in unsupported format"); + error = td::Status::Error(PSLICE() << "validator #" << i + << " has no public key or its public key is in unsupported format"); + return false; } if (!descr.weight) { - return td::Status::Error(PSLICE() << "validator #" << i << " has zero weight"); + error = td::Status::Error(PSLICE() << "validator #" << i << " has zero weight"); + return false; } if (descr.weight > ~(ptr->total_weight)) { - return td::Status::Error("total weight of all validators in validator set exceeds 2^64"); + error = td::Status::Error("total weight of all validators in validator set exceeds 2^64"); + return false; } ptr->list.emplace_back(sig_pubkey.pubkey, descr.weight, ptr->total_weight, descr.adnl_addr); ptr->total_weight += descr.weight; + return true; + }; + + if (!dict.check_for_each(validator_set_check_fn)) { + CHECK(error.is_error()); + return error; + } + if (std::find(seen_keys.begin(), seen_keys.end(), false) != seen_keys.end()) { + return td::Status::Error("indices in a validator set dictionary must be integers 0..total-1"); } if (rec.total_weight && rec.total_weight != ptr->total_weight) { return td::Status::Error("validator set declares incorrect total weight"); From 2c5e8ea71d3fe78c51a349a329885a354a4fbaec Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 16 Apr 2025 20:58:18 +0300 Subject: [PATCH 217/388] Fix SDBEGINS(Q) in Asm.fif (#1626) --- crypto/fift/lib/Asm.fif | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index 976093f80..4d7e2b6fe 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -2,7 +2,7 @@ library TVM_Asm // simple TVM Assembler namespace Asm Asm definitions -"0.4.5" constant asm-fif-version +"0.4.6" constant asm-fif-version variable @atend variable @was-split @@ -750,26 +750,37 @@ x{D723} @Defop SDSKIPLAST x{D724} @Defop SDSUBSTR x{D726} @Defop SDBEGINSX x{D727} @Defop SDBEGINSXQ + +/* +if (rembits >= opcodebits) or (rembits <= 18) + just write +else + PUSHSLICE SDBEGINSXQ + +--- +18 - size of PUSHREFSLICE +*/ + { tuck sbits tuck 5 + 3 >> swap x{D72A_} s, over 7 u, 3 roll s, -rot 3 << 3 + swap - @scomplete } : SDBEGINS:imm -{ tuck sbitrefs abort"no references allowed in slice" dup 26 <= - { drop > 8 * 3 + 21 + // slice builder opcodebits ((sbits + 5) // 8) * 8 + 3 + 21 + over brembits <= // total op bits <= remaining free bits in slice + { > swap x{D72E_} s, over 7 u, 3 roll s, -rot 3 << 3 + swap - @scomplete } : SDBEGINSQ:imm -{ tuck sbitrefs abort"no references allowed in slice" dup 26 <= - { drop > 8 * 3 + 21 + // slice builder opcodebits + over brembits <= // total op bits <= remaining free bits in slice + { Date: Thu, 17 Apr 2025 12:54:16 +0300 Subject: [PATCH 218/388] Process virtualization in cell load callback --- crypto/vm/boc.cpp | 8 +++++--- crypto/vm/boc.h | 2 +- crypto/vm/cells/Cell.h | 12 +++++++----- crypto/vm/cells/CellUsageTree.cpp | 9 +++++---- crypto/vm/cells/CellUsageTree.h | 9 +++++---- crypto/vm/cells/MerkleProof.h | 2 +- crypto/vm/cells/UsageCell.h | 2 +- validator/impl/collator-impl.h | 2 +- validator/impl/collator.cpp | 14 +++++++------- 9 files changed, 33 insertions(+), 27 deletions(-) diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index cfe37c662..d3ef57b74 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -1254,16 +1254,18 @@ bool VmStorageStat::add_storage(const CellSlice& cs) { return true; } -void ProofStorageStat::add_loaded_cell(const Ref& cell) { - auto& [status, size] = cells_[cell->get_hash()]; +void ProofStorageStat::add_loaded_cell(const Ref& cell, td::uint8 max_level) { + auto& [status, size] = cells_[cell->get_hash(max_level)]; if (status == c_loaded) { return; } proof_size_ -= size; status = c_loaded; proof_size_ += size = estimate_serialized_size(cell); + max_level += (cell->special_type() == CellTraits::SpecialType::MerkleProof || + cell->special_type() == CellTraits::SpecialType::MerkleUpdate); for (unsigned i = 0; i < cell->size_refs(); ++i) { - auto& [child_status, child_size] = cells_[cell->get_ref(i)->get_hash()]; + auto& [child_status, child_size] = cells_[cell->get_ref(i)->get_hash(max_level)]; if (child_status == c_none) { child_status = c_prunned; proof_size_ += child_size = estimate_prunned_size(); diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index 7e718ce26..fbf395538 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -165,7 +165,7 @@ struct VmStorageStat { class ProofStorageStat { public: - void add_loaded_cell(const Ref& cell); + void add_loaded_cell(const Ref& cell, td::uint8 max_level = Cell::max_level); void add_loaded_cells(const ProofStorageStat& other); td::uint64 estimate_proof_size() const; diff --git a/crypto/vm/cells/Cell.h b/crypto/vm/cells/Cell.h index e2b47ffc0..25b71fcb2 100644 --- a/crypto/vm/cells/Cell.h +++ b/crypto/vm/cells/Cell.h @@ -35,15 +35,17 @@ namespace vm { using td::Ref; class DataCell; +struct LoadedCell { + Ref data_cell; + detail::VirtualizationParameters virt; + CellUsageTree::NodePtr tree_node; // TODO: inline_vector? +}; + class Cell : public CellTraits { public: using LevelMask = detail::LevelMask; using VirtualizationParameters = detail::VirtualizationParameters; - struct LoadedCell { - Ref data_cell; - VirtualizationParameters virt; - CellUsageTree::NodePtr tree_node; // TODO: inline_vector? - }; + using LoadedCell = vm::LoadedCell; using Hash = CellHash; static_assert(std::is_standard_layout::value, "Cell::Hash is not a standard layout type"); diff --git a/crypto/vm/cells/CellUsageTree.cpp b/crypto/vm/cells/CellUsageTree.cpp index 3874998c2..301ee7da4 100644 --- a/crypto/vm/cells/CellUsageTree.cpp +++ b/crypto/vm/cells/CellUsageTree.cpp @@ -17,17 +17,18 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "vm/cells/CellUsageTree.h" +#include "DataCell.h" namespace vm { // // CellUsageTree::NodePtr // -bool CellUsageTree::NodePtr::on_load(const td::Ref& cell) const { +bool CellUsageTree::NodePtr::on_load(const Cell::LoadedCell& loaded_cell) const { auto tree = tree_weak_.lock(); if (!tree) { return false; } - tree->on_load(node_id_, cell); + tree->on_load(node_id_, loaded_cell); return true; } @@ -111,13 +112,13 @@ void CellUsageTree::set_use_mark_for_is_loaded(bool use_mark) { use_mark_ = use_mark; } -void CellUsageTree::on_load(NodeId node_id, const td::Ref& cell) { +void CellUsageTree::on_load(NodeId node_id, const Cell::LoadedCell& loaded_cell) { if (ignore_loads_ || nodes_[node_id].is_loaded) { return; } nodes_[node_id].is_loaded = true; if (cell_load_callback_) { - cell_load_callback_(cell); + cell_load_callback_(loaded_cell); } } diff --git a/crypto/vm/cells/CellUsageTree.h b/crypto/vm/cells/CellUsageTree.h index d845be10d..902b98d75 100644 --- a/crypto/vm/cells/CellUsageTree.h +++ b/crypto/vm/cells/CellUsageTree.h @@ -27,6 +27,7 @@ namespace vm { class DataCell; +struct LoadedCell; class CellUsageTree : public std::enable_shared_from_this { public: @@ -42,7 +43,7 @@ class CellUsageTree : public std::enable_shared_from_this { return node_id_ == 0 || tree_weak_.expired(); } - bool on_load(const td::Ref& cell) const; + bool on_load(const LoadedCell& loaded_cell) const; NodePtr create_child(unsigned ref_id) const; bool mark_path(CellUsageTree* master_tree) const; bool is_from_tree(const CellUsageTree* master_tree) const; @@ -63,7 +64,7 @@ class CellUsageTree : public std::enable_shared_from_this { void set_use_mark_for_is_loaded(bool use_mark = true); NodeId create_child(NodeId node_id, unsigned ref_id); - void set_cell_load_callback(std::function&)> f) { + void set_cell_load_callback(std::function f) { cell_load_callback_ = std::move(f); } void set_ignore_loads(bool value) { @@ -83,9 +84,9 @@ class CellUsageTree : public std::enable_shared_from_this { }; bool use_mark_{false}; std::vector nodes_{2}; - std::function&)> cell_load_callback_; + std::function cell_load_callback_; - void on_load(NodeId node_id, const td::Ref& cell); + void on_load(NodeId node_id, const LoadedCell& loaded_cell); NodeId create_node(NodeId parent); int ignore_loads_ = 0; }; diff --git a/crypto/vm/cells/MerkleProof.h b/crypto/vm/cells/MerkleProof.h index 96a31d64e..0a56be050 100644 --- a/crypto/vm/cells/MerkleProof.h +++ b/crypto/vm/cells/MerkleProof.h @@ -70,7 +70,7 @@ class MerkleProofBuilder { bool extract_proof_to(Ref &proof_root) const; td::Result extract_proof_boc() const; - void set_cell_load_callback(std::function&)> f) { + void set_cell_load_callback(std::function f) { usage_tree->set_cell_load_callback(std::move(f)); } }; diff --git a/crypto/vm/cells/UsageCell.h b/crypto/vm/cells/UsageCell.h index 978b91f76..c634bc909 100644 --- a/crypto/vm/cells/UsageCell.h +++ b/crypto/vm/cells/UsageCell.h @@ -42,7 +42,7 @@ class UsageCell : public Cell { // load interface td::Result load_cell() const override { TRY_RESULT(loaded_cell, cell_->load_cell()); - if (tree_node_.on_load(loaded_cell.data_cell)) { + if (tree_node_.on_load(loaded_cell)) { CHECK(loaded_cell.tree_node.empty()); loaded_cell.tree_node = tree_node_; } diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 6e5f876ee..405fad334 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -408,7 +408,7 @@ class Collator final : public td::actor::Actor { AccountStorageDict* current_tx_storage_dict_ = nullptr; - void on_cell_loaded(const Ref& cell); + void on_cell_loaded(const vm::LoadedCell& cell); void set_current_tx_storage_dict(const block::Account& account); }; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index e01031f9c..6e7a5b87a 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -944,7 +944,7 @@ void Collator::got_neighbor_msg_queue(unsigned i, Ref res) { neighbor_proof_builders_.push_back(vm::MerkleProofBuilder{res->state_root_}); state_root = neighbor_proof_builders_.back().root(); if (full_collated_data_ && !block_id.is_masterchain()) { - neighbor_proof_builders_.back().set_cell_load_callback([&](const Ref& cell) { + neighbor_proof_builders_.back().set_cell_load_callback([&](const vm::LoadedCell& cell) { on_cell_loaded(cell); }); } @@ -1054,7 +1054,7 @@ bool Collator::unpack_merge_last_state() { // 1. prepare for creating a MerkleUpdate based on previous state state_usage_tree_ = std::make_shared(); if (full_collated_data_ && !is_masterchain()) { - state_usage_tree_->set_cell_load_callback([&](const Ref& cell) { + state_usage_tree_->set_cell_load_callback([&](const vm::LoadedCell& cell) { on_cell_loaded(cell); }); } @@ -1103,7 +1103,7 @@ bool Collator::unpack_last_state() { // prepare for creating a MerkleUpdate based on previous state state_usage_tree_ = std::make_shared(); if (full_collated_data_ && !is_masterchain()) { - state_usage_tree_->set_cell_load_callback([&](const Ref& cell) { + state_usage_tree_->set_cell_load_callback([&](const vm::LoadedCell& cell) { on_cell_loaded(cell); }); } @@ -2648,7 +2648,7 @@ bool Collator::init_account_storage_dict(block::Account& account) { << ": dict is empty"); } dict.mpb = vm::MerkleProofBuilder(res.move_as_ok()); - dict.mpb.set_cell_load_callback([&](const Ref& cell) { + dict.mpb.set_cell_load_callback([&](const vm::LoadedCell& cell) { on_cell_loaded(cell); }); } @@ -6382,9 +6382,9 @@ void Collator::finalize_stats() { * When storage stat is calculated, StorageStatCalculationContext::calculating_storage_stat() returns true. * In this case, loaded cell is stored in a separate StorageStat for the storage dict of the account (if it exists). * - * @param cell Loaded cell + * @param loaded_cell Loaded cell */ -void Collator::on_cell_loaded(const Ref& cell) { +void Collator::on_cell_loaded(const vm::LoadedCell& loaded_cell) { auto context = block::StorageStatCalculationContext::get(); vm::ProofStorageStat* stat = (context && context->calculating_storage_stat() && current_tx_storage_dict_ ? ¤t_tx_storage_dict_->proof_stat @@ -6392,7 +6392,7 @@ void Collator::on_cell_loaded(const Ref& cell) { if (block_limit_status_) { block_limit_status_->collated_data_size_estimate -= stat->estimate_proof_size(); } - stat->add_loaded_cell(cell); + stat->add_loaded_cell(loaded_cell.data_cell, loaded_cell.virt.get_level()); if (block_limit_status_) { block_limit_status_->collated_data_size_estimate += stat->estimate_proof_size(); } From 5fe152621bf9a255a07391a7f4444a715815de4a Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Sat, 19 Apr 2025 17:41:55 +0300 Subject: [PATCH 219/388] Version 10: extra currencies, anycasts (#1595) * Disable anycast address, allow deploying with "fixed prefix length" * GETEXTRABALANCE instruction, changes in reserve * Change exitcode for "too many extra currencies" * Revert end_lt and collect action fine if state limits are exceeded * Fix consuming gas in RUNVM * Fix checking message balance * Add "supported-version" and capability constants to fift * Add unpacked in msg params to TVM c7 * Remove excessive logs in celldb * Fix moving libraries and log in runvm * Move "in msg params" in c7 to version 11 * Add msg value (with extra) to INMSGPARAMS --- crypto/block/block-parse.cpp | 11 +- crypto/block/block.tlb | 15 +- crypto/block/create-state.cpp | 28 +-- crypto/block/mc-config.cpp | 1 + crypto/block/mc-config.h | 1 + crypto/block/transaction.cpp | 332 +++++++++++++++++-------- crypto/block/transaction.h | 38 +-- crypto/fift/lib/Asm.fif | 17 ++ crypto/fift/words.cpp | 5 + crypto/smartcont/CreateState.fif | 3 + crypto/smc-envelope/SmartContract.cpp | 6 +- crypto/test/fift.cpp | 4 + crypto/test/fift/get_extra_balance.fif | 80 ++++++ crypto/vm/tonops.cpp | 181 +++++++++++--- crypto/vm/tonops.h | 9 +- crypto/vm/vm.cpp | 30 ++- crypto/vm/vm.h | 17 +- doc/GlobalVersions.md | 60 ++++- test/regression-tests.ans | 1 + validator/db/celldb.cpp | 4 - validator/impl/liteserver.cpp | 5 +- validator/impl/validate-query.cpp | 3 + 22 files changed, 650 insertions(+), 201 deletions(-) create mode 100644 crypto/test/fift/get_extra_balance.fif diff --git a/crypto/block/block-parse.cpp b/crypto/block/block-parse.cpp index 6d645ac79..71ec2bcf5 100644 --- a/crypto/block/block-parse.cpp +++ b/crypto/block/block-parse.cpp @@ -738,7 +738,7 @@ const TickTock t_TickTock; const RefAnything t_RefCell; bool StateInit::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { - return Maybe{5}.validate_skip(ops, cs, weak) // split_depth:(Maybe (## 5)) + return Maybe{5}.validate_skip(ops, cs, weak) // fixed_prefix_length:(Maybe (## 5)) && Maybe{}.validate_skip(ops, cs, weak) // special:(Maybe TickTock) && Maybe{}.validate_skip(ops, cs, weak) // code:(Maybe ^Cell) && Maybe{}.validate_skip(ops, cs, weak) // data:(Maybe ^Cell) @@ -1078,7 +1078,7 @@ bool Account::skip_copy_depth_balance(vm::CellBuilder& cb, vm::CellSlice& cs) co case account: return cs.advance(1) // account$1 && t_MsgAddressInt.skip_get_depth(cs, depth) // addr:MsgAddressInt - && cb.store_uint_leq(30, depth) // -> store split_depth:(#<= 30) + && cb.store_uint_leq(30, depth) // -> store fixed_prefix_length:(#<= 30) && t_StorageInfo.skip(cs) // storage_stat:StorageInfo && t_AccountStorage.skip_copy_balance(cb, cs); // storage:AccountStorage } @@ -1223,13 +1223,14 @@ bool HashmapAugE::extract_extra(vm::CellSlice& cs) const { bool DepthBalanceInfo::skip(vm::CellSlice& cs) const { return cs.advance(5) && t_CurrencyCollection.skip( - cs); // depth_balance$_ split_depth:(#<= 30) balance:CurrencyCollection = DepthBalanceInfo; + cs); // depth_balance$_ fixed_prefix_length:(#<= 30) balance:CurrencyCollection = DepthBalanceInfo; } bool DepthBalanceInfo::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { return cs.fetch_ulong(5) <= 30 && - t_CurrencyCollection.validate_skip(ops, cs, - weak); // depth_balance$_ split_depth:(#<= 30) balance:CurrencyCollection + t_CurrencyCollection.validate_skip( + ops, cs, + weak); // depth_balance$_ fixed_prefix_length:(#<= 30) balance:CurrencyCollection } bool DepthBalanceInfo::null_value(vm::CellBuilder& cb) const { diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 5cba3c69f..99dfdb522 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -141,12 +141,12 @@ ext_out_msg_info$11 src:MsgAddress dest:MsgAddressExt tick_tock$_ tick:Bool tock:Bool = TickTock; -_ split_depth:(Maybe (## 5)) special:(Maybe TickTock) +_ fixed_prefix_length:(Maybe (## 5)) special:(Maybe TickTock) code:(Maybe ^Cell) data:(Maybe ^Cell) library:(Maybe ^Cell) = StateInit; // StateInitWithLibs is used to validate sent and received messages -_ split_depth:(Maybe (## 5)) special:(Maybe TickTock) +_ fixed_prefix_length:(Maybe (## 5)) special:(Maybe TickTock) code:(Maybe ^Cell) data:(Maybe ^Cell) library:(HashmapE 256 SimpleLib) = StateInitWithLibs; @@ -272,14 +272,6 @@ acc_state_frozen$01 = AccountStatus; acc_state_active$10 = AccountStatus; acc_state_nonexist$11 = AccountStatus; -/* duplicates -tick_tock$_ tick:Bool tock:Bool = TickTock; - -_ split_depth:(Maybe (## 5)) special:(Maybe TickTock) - code:(Maybe ^Cell) data:(Maybe ^Cell) - library:(Maybe ^Cell) = StateInit; -*/ - account_descr$_ account:^Account last_trans_hash:bits256 last_trans_lt:uint64 = ShardAccount; @@ -800,7 +792,8 @@ size_limits_config#01 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells max_ext_msg_size:uint32 max_ext_msg_depth:uint16 = SizeLimitsConfig; size_limits_config_v2#02 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16 max_ext_msg_size:uint32 max_ext_msg_depth:uint16 max_acc_state_cells:uint32 max_acc_state_bits:uint32 - max_acc_public_libraries:uint32 defer_out_queue_size_limit:uint32 max_msg_extra_currencies:uint32 = SizeLimitsConfig; + max_acc_public_libraries:uint32 defer_out_queue_size_limit:uint32 max_msg_extra_currencies:uint32 + max_acc_fixed_prefix_length:uint8 = SizeLimitsConfig; _ SizeLimitsConfig = ConfigParam 43; // key is [ wc:int32 addr:uint256 ] diff --git a/crypto/block/create-state.cpp b/crypto/block/create-state.cpp index 4a74ac0f6..a32564a21 100644 --- a/crypto/block/create-state.cpp +++ b/crypto/block/create-state.cpp @@ -94,12 +94,12 @@ typedef td::BitArray<256> hash_t; struct SmcDescr { hash_t addr; - int split_depth; + int fixed_prefix_length; bool preinit_only; td::RefInt256 gram_balance; Ref state_init; // StateInit Ref account; // Account - SmcDescr(const hash_t& _addr) : addr(_addr), split_depth(0), preinit_only(false) { + SmcDescr(const hash_t& _addr) : addr(_addr), fixed_prefix_length(0), preinit_only(false) { } }; @@ -123,7 +123,7 @@ vm::Dictionary config_dict{32}; ton::UnixTime now; bool set_config_smc(const SmcDescr& smc) { - if (config_addr_set || smc.preinit_only || workchain_id != wc_master || smc.split_depth) { + if (config_addr_set || smc.preinit_only || workchain_id != wc_master || smc.fixed_prefix_length) { return false; } vm::CellSlice cs = load_cell_slice(smc.state_init); @@ -221,7 +221,7 @@ bool add_public_library(hash_t lib_addr, hash_t smc_addr, Ref lib_root } td::RefInt256 create_smartcontract(td::RefInt256 smc_addr, Ref code, Ref data, - Ref library, td::RefInt256 balance, int special, int split_depth, + Ref library, td::RefInt256 balance, int special, int fixed_prefix_length, int mode) { if (is_empty_cell(code)) { code.clear(); @@ -238,12 +238,12 @@ td::RefInt256 create_smartcontract(td::RefInt256 smc_addr, Ref code, R THRERR("not a valid library collection"); } vm::CellBuilder cb; - if (!split_depth) { + if (!fixed_prefix_length) { PDO(cb.store_long_bool(0, 1)); } else { - PDO(cb.store_long_bool(1, 1) && cb.store_ulong_rchk_bool(split_depth, 5)); + PDO(cb.store_long_bool(1, 1) && cb.store_ulong_rchk_bool(fixed_prefix_length, 5)); } - THRERR("invalid split_depth for a smart contract"); + THRERR("invalid fixed_prefix_length for a smart contract"); if (!special) { PDO(cb.store_long_bool(0, 1)); } else { @@ -287,7 +287,7 @@ td::RefInt256 create_smartcontract(td::RefInt256 smc_addr, Ref code, R auto ins = smart_contracts.emplace(addr, addr); assert(ins.second); SmcDescr& smc = ins.first->second; - smc.split_depth = split_depth; + smc.fixed_prefix_length = fixed_prefix_length; smc.preinit_only = (mode == 1); smc.gram_balance = balance; total_smc_balance += balance; @@ -328,10 +328,10 @@ td::RefInt256 create_smartcontract(td::RefInt256 smc_addr, Ref code, R ctor = 2; // addr_std$10 } PDO(cb.store_long_bool(ctor, 2)); // addr_std$10 or addr_var$11 - if (split_depth) { + if (fixed_prefix_length) { PDO(cb.store_long_bool(1, 1) // just$1 - && cb.store_ulong_rchk_bool(split_depth, 5) // depth:(## 5) - && cb.store_bits_bool(addr.cbits(), split_depth)); // rewrite pfx:(depth * Bit) + && cb.store_ulong_rchk_bool(fixed_prefix_length, 5) // depth:(## 5) + && cb.store_bits_bool(addr.cbits(), fixed_prefix_length)); // rewrite pfx:(depth * Bit) } else { PDO(cb.store_long_bool(0, 1)); // nothing$0 } @@ -515,7 +515,7 @@ Ref create_state() { // data (cell) // library (cell) // balance (int) -// split_depth (int 0..32) +// fixed_prefix_length (int 0..32) // special (int 0..3, +2 = tick, +1 = tock) // [ address (uint256) ] // mode (0 = compute address only, 1 = create uninit, 2 = create complete; +4 = with specified address) @@ -537,7 +537,7 @@ void interpret_register_smartcontract(vm::Stack& stack) { if (special && workchain_id != wc_master) { throw fift::IntError{"cannot create special smartcontracts outside of the masterchain"}; } - int split_depth = stack.pop_smallint_range(32); + int fixed_prefix_length = stack.pop_smallint_range(32); td::RefInt256 balance = stack.pop_int_finite(); if (sgn(balance) < 0) { throw fift::IntError{"initial balance of a smartcontract cannot be negative"}; @@ -549,7 +549,7 @@ void interpret_register_smartcontract(vm::Stack& stack) { Ref data = stack.pop_cell(); Ref code = stack.pop_cell(); td::RefInt256 addr = create_smartcontract(std::move(spec_addr), std::move(code), std::move(data), std::move(library), - std::move(balance), special, split_depth, mode); + std::move(balance), special, fixed_prefix_length, mode); if (addr.is_null()) { throw fift::IntError{"internal error while creating smartcontract"}; } diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index d5aa65d5e..d51f20599 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2040,6 +2040,7 @@ td::Result Config::do_get_size_limits_config(td::Ref 30) { - return false; // invalid value for split_depth +bool Account::set_addr_rewrite_length(int new_length) { + if (new_length < 0 || new_length > 30) { + return false; // invalid value } - if (split_depth_set_) { - return split_depth_ == new_split_depth; + if (addr_rewrite_length_set) { + return addr_rewrite_length == new_length; } else { - split_depth_ = (unsigned char)new_split_depth; - split_depth_set_ = true; + addr_rewrite_length = (unsigned char)new_length; + addr_rewrite_length_set = true; return true; } } /** - * Checks if the given split depth is valid for the Account. + * Checks if the given addr rewrite length is valid for the Account. * - * @param split_depth The split depth to be checked. + * @param length The addr rewrite length to be checked. * - * @returns True if the split depth is valid, False otherwise. + * @returns True if the addr rewrite length is valid, False otherwise. */ -bool Account::check_split_depth(int split_depth) const { - return split_depth_set_ ? (split_depth == split_depth_) : (split_depth >= 0 && split_depth <= 30); +bool Account::check_addr_rewrite_length(int length) const { + return addr_rewrite_length_set ? (length == addr_rewrite_length) : (length >= 0 && length <= 30); } /** * Parses anycast data of the account address. * - * Initializes split_depth and addr_rewrite. + * Initializes addr_rewrite. * - * @param cs The cell slice containing partially-parsed account addressa. + * @param cs The cell slice containing partially-parsed account address. * * @returns True if parsing was successful, false otherwise. */ @@ -159,13 +159,13 @@ bool Account::parse_maybe_anycast(vm::CellSlice& cs) { if (t < 0) { return false; } else if (!t) { - return set_split_depth(0); + return set_addr_rewrite_length(0); } int depth; return cs.fetch_uint_leq(30, depth) // anycast_info$_ depth:(#<= 30) && depth // { depth >= 1 } && cs.fetch_bits_to(addr_rewrite.bits(), depth) // rewrite_pfx:(bits depth) - && set_split_depth(depth); + && set_addr_rewrite_length(depth); } /** @@ -176,12 +176,12 @@ bool Account::parse_maybe_anycast(vm::CellSlice& cs) { * @returns True if the anycast information was successfully stored, false otherwise. */ bool Account::store_maybe_anycast(vm::CellBuilder& cb) const { - if (!split_depth_set_ || !split_depth_) { + if (!addr_rewrite_length_set || !addr_rewrite_length) { return cb.store_bool_bool(false); } return cb.store_bool_bool(true) // just$1 - && cb.store_uint_leq(30, split_depth_) // depth:(#<= 30) - && cb.store_bits_bool(addr_rewrite.cbits(), split_depth_); // rewrite_pfx:(bits depth) + && cb.store_uint_leq(30, addr_rewrite_length) // depth:(#<= 30) + && cb.store_bits_bool(addr_rewrite.cbits(), addr_rewrite_length); // rewrite_pfx:(bits depth) } /** @@ -214,14 +214,14 @@ bool Account::unpack_address(vm::CellSlice& addr_cs) { if (workchain == ton::workchainInvalid) { workchain = new_wc; addr = addr_orig; - addr.bits().copy_from(addr_rewrite.cbits(), split_depth_); - } else if (split_depth_) { + addr.bits().copy_from(addr_rewrite.cbits(), addr_rewrite_length); + } else if (addr_rewrite_length) { ton::StdSmcAddress new_addr = addr_orig; - new_addr.bits().copy_from(addr_rewrite.cbits(), split_depth_); + new_addr.bits().copy_from(addr_rewrite.cbits(), addr_rewrite_length); if (new_addr != addr) { LOG(ERROR) << "error unpacking account " << workchain << ":" << addr.to_hex() << " : account header contains different address " << new_addr.to_hex() << " (with splitting depth " - << (int)split_depth_ << ")"; + << (int)addr_rewrite_length << ")"; return false; } } else if (addr != addr_orig) { @@ -235,7 +235,7 @@ bool Account::unpack_address(vm::CellSlice& addr_cs) { return false; } addr_rewrite = addr.bits(); // initialize all 32 bits of addr_rewrite - if (!split_depth_) { + if (!addr_rewrite_length) { my_addr_exact = my_addr; } return true; @@ -283,7 +283,7 @@ bool Account::unpack_storage_info(vm::CellSlice& cs) { * Unpacks the state of an Account from a CellSlice. * * State is serialized using StateInit TLB-scheme. - * Initializes split_depth (from account state - StateInit) + * Initializes fixed_prefix_length (from account state - StateInit) * * @param cs The CellSlice containing the serialized state. * @@ -294,12 +294,9 @@ bool Account::unpack_state(vm::CellSlice& cs) { if (!tlb::unpack_exact(cs, state)) { return false; } - int sd = 0; - if (state.split_depth->size() == 6) { - sd = (int)state.split_depth->prefetch_ulong(6) - 32; - } - if (!set_split_depth(sd)) { - return false; + fixed_prefix_length = 0; + if (state.fixed_prefix_length->size() == 6) { + fixed_prefix_length = (int)state.fixed_prefix_length->prefetch_ulong(6) - 32; } if (state.special->size() > 1) { int z = (int)state.special->prefetch_ulong(3); @@ -366,23 +363,25 @@ bool Account::compute_my_addr(bool force) { /** * Computes the address of the Account. * + * Legacy (used only if global_version < 10). + * * @param tmp_addr A reference to the CellSlice for the result. - * @param split_depth The split depth for the address. - * @param orig_addr_rewrite Address prefox of length split_depth. + * @param fixed_prefix_length The fixed prefix length for the address. + * @param orig_addr_rewrite Address prefix of length fixed_prefix_length. * * @returns True if the address was successfully computed, false otherwise. */ -bool Account::recompute_tmp_addr(Ref& tmp_addr, int split_depth, +bool Account::recompute_tmp_addr(Ref& tmp_addr, int fixed_prefix_length, td::ConstBitPtr orig_addr_rewrite) const { - if (!split_depth && my_addr_exact.not_null()) { + if (!fixed_prefix_length && my_addr_exact.not_null()) { tmp_addr = my_addr_exact; return true; } - if (split_depth == split_depth_ && my_addr.not_null()) { + if (fixed_prefix_length == addr_rewrite_length && my_addr.not_null()) { tmp_addr = my_addr; return true; } - if (split_depth < 0 || split_depth > 30) { + if (fixed_prefix_length < 0 || fixed_prefix_length > 30) { return false; } vm::CellBuilder cb; @@ -390,13 +389,13 @@ bool Account::recompute_tmp_addr(Ref& tmp_addr, int split_depth, if (!cb.store_long_bool(std ? 2 : 3, 2)) { // addr_std$10 or addr_var$11 return false; } - if (!split_depth) { + if (!fixed_prefix_length) { if (!cb.store_bool_bool(false)) { // anycast:(Maybe Anycast) return false; } } else if (!(cb.store_bool_bool(true) // just$1 - && cb.store_long_bool(split_depth, 5) // depth:(#<= 30) - && cb.store_bits_bool(addr.bits(), split_depth))) { // rewrite_pfx:(bits depth) + && cb.store_long_bool(fixed_prefix_length, 5) // depth:(#<= 30) + && cb.store_bits_bool(addr.bits(), fixed_prefix_length))) { // rewrite_pfx:(bits depth) return false; } if (std) { @@ -408,26 +407,26 @@ bool Account::recompute_tmp_addr(Ref& tmp_addr, int split_depth, return false; } Ref cell; - return cb.store_bits_bool(orig_addr_rewrite, split_depth) // address:(bits addr_len) or bits256 - && cb.store_bits_bool(addr.bits() + split_depth, 256 - split_depth) && cb.finalize_to(cell) && + return cb.store_bits_bool(orig_addr_rewrite, fixed_prefix_length) // address:(bits addr_len) or bits256 + && cb.store_bits_bool(addr.bits() + fixed_prefix_length, 256 - fixed_prefix_length) && cb.finalize_to(cell) && (tmp_addr = vm::load_cell_slice_ref(std::move(cell))).not_null(); } /** * Sets address rewriting info for a newly-activated account. * - * @param split_depth The split depth for the account address. - * @param orig_addr_rewrite Address frepix of length split_depth. + * @param rewrite_length The fixed prefix length for the account address. + * @param orig_addr_rewrite Address prefix of length fixed_prefix_length. * * @returns True if the rewriting info was successfully set, false otherwise. */ -bool Account::init_rewrite_addr(int split_depth, td::ConstBitPtr orig_addr_rewrite) { - if (split_depth_set_ || !set_split_depth(split_depth)) { +bool Account::init_rewrite_addr(int rewrite_length, td::ConstBitPtr orig_addr_rewrite) { + if (addr_rewrite_length_set || !set_addr_rewrite_length(rewrite_length)) { return false; } addr_orig = addr; addr_rewrite = addr.bits(); - addr_orig.bits().copy_from(orig_addr_rewrite, split_depth); + addr_orig.bits().copy_from(orig_addr_rewrite, rewrite_length); return compute_my_addr(true); } @@ -483,7 +482,7 @@ bool Account::unpack(Ref shard_account, ton::UnixTime now, bool s case block::gen::AccountState::account_uninit: status = orig_status = acc_uninit; state_hash = addr; - forget_split_depth(); + forget_addr_rewrite_length(); break; case block::gen::AccountState::account_frozen: status = orig_status = acc_frozen; @@ -558,18 +557,18 @@ bool Account::init_new(ton::UnixTime now) { } state_hash = addr_orig; status = orig_status = acc_nonexist; - split_depth_set_ = false; + addr_rewrite_length_set = false; return true; } /** - * Resets the split depth of the account. + * Resets the fixed prefix length of the account. * - * @returns True if the split depth was successfully reset, false otherwise. + * @returns True if the fixed prefix length was successfully reset, false otherwise. */ -bool Account::forget_split_depth() { - split_depth_set_ = false; - split_depth_ = 0; +bool Account::forget_addr_rewrite_length() { + addr_rewrite_length_set = false; + addr_rewrite_length = 0; addr_orig = addr; my_addr = my_addr_exact; addr_rewrite = addr.bits(); @@ -587,9 +586,10 @@ bool Account::deactivate() { } // forget special (tick/tock) info tick = tock = false; + fixed_prefix_length = 0; if (status == acc_nonexist || status == acc_uninit) { - // forget split depth and address rewriting info - forget_split_depth(); + // forget fixed prefix length and address rewriting info + forget_addr_rewrite_length(); // forget specific state hash for deleted or uninitialized accounts (revert to addr) state_hash = addr; } @@ -710,6 +710,7 @@ Transaction::Transaction(const Account& _account, int ttype, ton::LogicalTime re , is_first(_account.transactions.empty()) , new_tick(_account.tick) , new_tock(_account.tock) + , new_fixed_prefix_length(_account.fixed_prefix_length) , now(_now) , account(_account) , my_addr(_account.my_addr) @@ -750,42 +751,52 @@ bool Transaction::unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* }; } auto cs = vm::load_cell_slice(in_msg); - int tag = block::gen::t_CommonMsgInfo.get_tag(cs); - Ref src_addr, dest_addr; + int tag = gen::t_CommonMsgInfo.get_tag(cs); switch (tag) { - case block::gen::CommonMsgInfo::int_msg_info: { - block::gen::CommonMsgInfo::Record_int_msg_info info; - if (!(tlb::unpack(cs, info) && msg_balance_remaining.unpack(std::move(info.value)))) { + case gen::CommonMsgInfo::int_msg_info: { + if (!(tlb::unpack(cs, in_msg_info) && msg_balance_remaining.unpack(in_msg_info.value))) { return false; } - if (info.ihr_disabled && ihr_delivered) { + if (in_msg_info.ihr_disabled && ihr_delivered) { return false; } - bounce_enabled = info.bounce; - src_addr = std::move(info.src); - dest_addr = std::move(info.dest); + bounce_enabled = in_msg_info.bounce; in_msg_type = 1; - td::RefInt256 ihr_fee = block::tlb::t_Grams.as_integer(std::move(info.ihr_fee)); + td::RefInt256 ihr_fee = block::tlb::t_Grams.as_integer(in_msg_info.ihr_fee); if (ihr_delivered) { in_fwd_fee = std::move(ihr_fee); } else { in_fwd_fee = td::zero_refint(); msg_balance_remaining += std::move(ihr_fee); } - if (info.created_lt >= start_lt) { - start_lt = info.created_lt + 1; + if (in_msg_info.created_lt >= start_lt) { + start_lt = in_msg_info.created_lt + 1; end_lt = start_lt + 1; } // ... break; } - case block::gen::CommonMsgInfo::ext_in_msg_info: { - block::gen::CommonMsgInfo::Record_ext_in_msg_info info; + case gen::CommonMsgInfo::ext_in_msg_info: { + gen::CommonMsgInfo::Record_ext_in_msg_info info; if (!tlb::unpack(cs, info)) { return false; } - src_addr = std::move(info.src); - dest_addr = std::move(info.dest); + in_msg_info.ihr_disabled = in_msg_info.bounce = in_msg_info.bounced = false; + in_msg_info.src = info.src; + in_msg_info.dest = info.dest; + in_msg_info.created_at = in_msg_info.created_lt = 0; + if (cfg->disable_anycast) { + // Check that dest is addr_std without anycast + gen::MsgAddressInt::Record_addr_std rec; + if (!gen::csr_unpack(info.dest, rec)) { + LOG(DEBUG) << "destination address of the external message is not a valid addr_std"; + return false; + } + if (rec.anycast->size() > 1) { + LOG(DEBUG) << "destination address of the external message is an anycast address"; + return false; + } + } in_msg_type = 2; in_msg_extern = true; // compute forwarding fees for this external message @@ -1397,11 +1408,63 @@ Ref Transaction::prepare_vm_c7(const ComputePhaseConfig& cfg) const { ? vm::StackEntry(td::make_refint(compute_phase->precompiled_gas_usage.value())) : vm::StackEntry()); // precompiled_gas_usage:Integer } + if (cfg.global_version >= 11) { + // in_msg_params:[...] + tuple.push_back(prepare_in_msg_params_tuple(trans_type == tr_ord ? &in_msg_info : nullptr, in_msg_state, + msg_balance_remaining)); + } auto tuple_ref = td::make_cnt_ref>(std::move(tuple)); LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple_ref).to_string(); return vm::make_tuple_ref(std::move(tuple_ref)); } +/** + * Prepares tuple with unpacked parameters of the inbound message (for the 17th element of c7). + * `info` is: + * - For internal messages - just int_msg_info of the message + * - For external messages - artificial int_msg_info based on ext_msg_info of the messages. + * - For tick-tock transactions and get methods - nullptr. + * + * @param info Pointer to the message info. + * @param state_init State init of the message (null if absent). + * @param msg_balance_remaining Remaining balance of the message (it's sometimes different from value in info). + * + * @returns Tuple with message parameters. + */ +Ref Transaction::prepare_in_msg_params_tuple(const gen::CommonMsgInfo::Record_int_msg_info* info, + const Ref& state_init, + const CurrencyCollection& msg_balance_remaining) { + std::vector in_msg_params(10); + if (info != nullptr) { + in_msg_params[0] = td::make_refint(info->bounce ? -1 : 0); // bounce + in_msg_params[1] = td::make_refint(info->bounced ? -1 : 0); // bounced + in_msg_params[2] = info->src; // src_addr + in_msg_params[3] = info->fwd_fee.is_null() ? td::zero_refint() : tlb::t_Grams.as_integer(info->fwd_fee); // fwd_fee + in_msg_params[4] = td::make_refint(info->created_lt); // created_lt + in_msg_params[5] = td::make_refint(info->created_at); // created_at + auto value = info->value; + in_msg_params[6] = + info->value.is_null() ? td::zero_refint() : tlb::t_Grams.as_integer_skip(value.write()); // original value + in_msg_params[7] = msg_balance_remaining.is_valid() ? msg_balance_remaining.grams : td::zero_refint(); // value + in_msg_params[8] = msg_balance_remaining.is_valid() ? vm::StackEntry::maybe(msg_balance_remaining.extra) + : vm::StackEntry{}; // value extra + in_msg_params[9] = vm::StackEntry::maybe(state_init); // state_init + } else { + in_msg_params[0] = td::zero_refint(); // bounce + in_msg_params[1] = td::zero_refint(); // bounced + static Ref addr_none = vm::CellBuilder{}.store_zeroes(2).as_cellslice_ref(); + in_msg_params[2] = addr_none; // src_addr + in_msg_params[3] = td::zero_refint(); // fed_fee + in_msg_params[4] = td::zero_refint(); // created_lt + in_msg_params[5] = td::zero_refint(); // created_at + in_msg_params[6] = td::zero_refint(); // original value + in_msg_params[7] = td::zero_refint(); // value + in_msg_params[8] = vm::StackEntry{}; // value extra + in_msg_params[9] = vm::StackEntry{}; // state_init + } + return td::make_cnt_ref>(std::move(in_msg_params)); +} + /** * Computes the number of output actions in a list. * @@ -1442,10 +1505,13 @@ bool Transaction::unpack_msg_state(const ComputePhaseConfig& cfg, bool lib_only, in_msg_library = state.library->prefetch_ref(); return true; } - if (state.split_depth->size() == 6) { - new_split_depth = (signed char)(state.split_depth->prefetch_ulong(6) - 32); + if (state.fixed_prefix_length->size() == 6) { + new_fixed_prefix_length = (signed char)(state.fixed_prefix_length->prefetch_ulong(6) - 32); } else { - new_split_depth = 0; + new_fixed_prefix_length = 0; + } + if (!cfg.disable_anycast) { + new_addr_rewrite_length = new_fixed_prefix_length; } if (state.special->size() > 1) { int z = (int)state.special->prefetch_ulong(3); @@ -1500,19 +1566,26 @@ std::vector> Transaction::compute_vm_libraries(const ComputePhaseC /** * Checks if the input message StateInit hash corresponds to the account address. * + * @param cfg The configuration for the compute phase. + * * @returns True if the input message state hash is valid, False otherwise. */ -bool Transaction::check_in_msg_state_hash() { +bool Transaction::check_in_msg_state_hash(const ComputePhaseConfig& cfg) { CHECK(in_msg_state.not_null()); - CHECK(new_split_depth >= 0 && new_split_depth < 32); + CHECK(new_fixed_prefix_length >= 0 && new_fixed_prefix_length < 32); td::Bits256 in_state_hash = in_msg_state->get_hash().bits(); - int d = new_split_depth; + int d = new_fixed_prefix_length; if ((in_state_hash.bits() + d).compare(account.addr.bits() + d, 256 - d)) { return false; } orig_addr_rewrite = in_state_hash.bits(); orig_addr_rewrite_set = true; - return account.recompute_tmp_addr(my_addr, d, orig_addr_rewrite.bits()); + if (cfg.disable_anycast) { + my_addr = my_addr_exact; + return true; + } else { + return account.recompute_tmp_addr(my_addr, d, orig_addr_rewrite.bits()); + } } /** @@ -1643,16 +1716,24 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { use_msg_state = true; bool forbid_public_libs = acc_status == Account::acc_uninit && account.is_masterchain(); // Forbid for deploying, allow for unfreezing - if (!(unpack_msg_state(cfg, false, forbid_public_libs) && account.check_split_depth(new_split_depth))) { - LOG(DEBUG) << "cannot unpack in_msg_state, or it has bad split_depth; cannot init account state"; + if (!(unpack_msg_state(cfg, false, forbid_public_libs) && + account.check_addr_rewrite_length(new_fixed_prefix_length))) { + LOG(DEBUG) << "cannot unpack in_msg_state, or it has bad fixed_prefix_length; cannot init account state"; cp.skip_reason = ComputePhase::sk_bad_state; return true; } - if (acc_status == Account::acc_uninit && !check_in_msg_state_hash()) { + if (acc_status == Account::acc_uninit && !check_in_msg_state_hash(cfg)) { LOG(DEBUG) << "in_msg_state hash mismatch, cannot init account state"; cp.skip_reason = ComputePhase::sk_bad_state; return true; } + if (cfg.disable_anycast && acc_status == Account::acc_uninit && + new_fixed_prefix_length > cfg.size_limits.max_acc_fixed_prefix_length) { + LOG(DEBUG) << "cannot init account state: too big fixed prefix length (" << new_fixed_prefix_length << ", max " + << cfg.size_limits.max_acc_fixed_prefix_length << ")"; + cp.skip_reason = ComputePhase::sk_bad_state; + return true; + } } else if (acc_status != Account::acc_active) { // no state, cannot perform transactions cp.skip_reason = in_msg_state.not_null() ? ComputePhase::sk_bad_state : ComputePhase::sk_no_state; @@ -1675,6 +1756,11 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { return true; } } + if (cfg.disable_anycast) { + my_addr = my_addr_exact; + new_addr_rewrite_length = 0; + force_remove_anycast_address = true; + } td::optional precompiled; if (new_code.not_null() && trans_type == tr_ord) { @@ -1970,7 +2056,7 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { ap.no_funds = true; } LOG(DEBUG) << "invalid action " << ap.result_arg << " in action list: error code " << ap.result_code; - // This is required here because changes to libraries are applied even if actipn phase fails + // This is required here because changes to libraries are applied even if action phase fails enforce_state_limits(); if (cfg.action_fine_enabled) { ap.action_fine = std::min(ap.action_fine, balance.grams); @@ -1994,6 +2080,15 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { } new_data = compute_phase->new_data; // tentative persistent data update applied if (!enforce_state_limits()) { + if (cfg.extra_currency_v2) { + end_lt = ap.end_lt = start_lt + 1; + if (cfg.action_fine_enabled) { + ap.action_fine = std::min(ap.action_fine, balance.grams); + ap.total_action_fees = ap.action_fine; + balance.grams -= ap.action_fine; + total_fees += ap.action_fine; + } + } return true; } @@ -2241,11 +2336,12 @@ bool Transaction::check_replace_src_addr(Ref& src_addr) const { * @param dest_addr A reference to the destination address of the transaction. * @param cfg The configuration for the action phase. * @param is_mc A pointer to a boolean where it will be stored whether the destination is in the masterchain. + * @param allow_anycast Allow anycast the address. * * @returns True if the destination address is valid, false otherwise. */ bool Transaction::check_rewrite_dest_addr(Ref& dest_addr, const ActionPhaseConfig& cfg, - bool* is_mc) const { + bool* is_mc, bool allow_anycast) const { if (!dest_addr->prefetch_ulong(1)) { // all external addresses allowed if (is_mc) { @@ -2305,6 +2401,9 @@ bool Transaction::check_rewrite_dest_addr(Ref& dest_addr, const A } } if (rec.anycast->size() > 1) { + if (!allow_anycast) { + return false; + } // destination address is an anycast vm::CellSlice cs{*rec.anycast}; int d = (int)cs.fetch_ulong(6) - 32; @@ -2472,11 +2571,11 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, return 35; // invalid source address } bool to_mc = false; - if (!check_rewrite_dest_addr(info.dest, cfg, &to_mc)) { + if (!check_rewrite_dest_addr(info.dest, cfg, &to_mc, !cfg.disable_anycast)) { LOG(DEBUG) << "invalid destination address in a proposed outbound message"; return check_skip_invalid(36); // invalid destination address } - if (cfg.extra_currency_v2) { + if (!ext_msg && cfg.extra_currency_v2) { CurrencyCollection value; if (!value.unpack(info.value)) { LOG(DEBUG) << "invalid value:ExtraCurrencies in a proposed outbound message"; @@ -2486,7 +2585,7 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, LOG(DEBUG) << "invalid value:ExtraCurrencies in a proposed outbound message: too many currencies (max " << cfg.size_limits.max_msg_extra_currencies << ")"; // Dict should be valid, since it was checked in t_OutListNode.validate_ref, so error here means limit exceeded - return check_skip_invalid(41); // invalid value:CurrencyCollection : too many extra currencies + return check_skip_invalid(44); // invalid value:CurrencyCollection : too many extra currencies } info.value = value.pack(); } @@ -2667,7 +2766,7 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, if (cfg.extra_currency_v2 && !req.check_extra_currency_limit(cfg.size_limits.max_msg_extra_currencies)) { LOG(DEBUG) << "too many extra currencies in the message : max " << cfg.size_limits.max_msg_extra_currencies; - return check_skip_invalid(41); // to many extra currencies + return check_skip_invalid(44); // to many extra currencies } Ref new_extra; @@ -2832,13 +2931,25 @@ int Transaction::try_action_reserve_currency(vm::CellSlice& cs, ActionPhase& ap, LOG(DEBUG) << "cannot parse currency field in action_reserve_currency"; return -1; } + if (cfg.extra_currency_v2 && reserve.has_extra()) { + LOG(DEBUG) << "cannot reserve extra currencies"; + return -1; + } LOG(DEBUG) << "action_reserve_currency: mode=" << mode << ", reserve=" << reserve.to_str() << ", balance=" << ap.remaining_balance.to_str() << ", original balance=" << original_balance.to_str(); if (mode & 4) { if (mode & 8) { - reserve = original_balance - reserve; + if (cfg.extra_currency_v2) { + reserve.grams = original_balance.grams - reserve.grams; + } else { + reserve = original_balance - reserve; + } } else { - reserve += original_balance; + if (cfg.extra_currency_v2) { + reserve.grams += original_balance.grams; + } else { + reserve += original_balance; + } } } else if (mode & 8) { LOG(DEBUG) << "invalid reserve mode " << mode; @@ -2851,7 +2962,7 @@ int Transaction::try_action_reserve_currency(vm::CellSlice& cs, ActionPhase& ap, if (mode & 2) { if (cfg.reserve_extra_enabled) { if (!reserve.clamp(ap.remaining_balance)) { - LOG(DEBUG) << "failed to clamp reserve amount" << mode; + LOG(DEBUG) << "failed to clamp reserve amount " << mode; return -1; } } else { @@ -2872,7 +2983,11 @@ int Transaction::try_action_reserve_currency(vm::CellSlice& cs, ActionPhase& ap, newc.grams = ap.remaining_balance.grams - reserve.grams; if (mode & 1) { // leave only res_grams, reserve everything else - std::swap(newc, reserve); + if (cfg.extra_currency_v2) { + std::swap(newc.grams, reserve.grams); + } else { + std::swap(newc, reserve); + } } // set remaining_balance to new_grams and new_extra ap.remaining_balance = std::move(newc); @@ -3192,13 +3307,14 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { && balance.store(cb)); // balance:CurrencyCollection int ticktock = new_tick * 2 + new_tock; unsigned si_pos = 0; + int fixed_prefix_length = cfg.disable_anycast ? new_fixed_prefix_length : account.addr_rewrite_length; if (acc_status == Account::acc_uninit) { CHECK(cb.store_long_bool(0, 2)); // account_uninit$00 = AccountState } else if (acc_status == Account::acc_frozen) { if (was_frozen) { vm::CellBuilder cb2; - CHECK(account.split_depth_ ? cb2.store_long_bool(account.split_depth_ + 32, 6) // _ ... = StateInit - : cb2.store_long_bool(0, 1)); // ... split_depth:(Maybe (## 5)) + CHECK(fixed_prefix_length ? cb2.store_long_bool(fixed_prefix_length + 32, 6) // _ ... = StateInit + : cb2.store_long_bool(0, 1)); // ... fixed_prefix_length:(Maybe (## 5)) CHECK(ticktock ? cb2.store_long_bool(ticktock | 4, 3) : cb2.store_long_bool(0, 1)); // special:(Maybe TickTock) CHECK(cb2.store_maybe_ref(new_code) && cb2.store_maybe_ref(new_data) && cb2.store_maybe_ref(new_library)); // code:(Maybe ^Cell) data:(Maybe ^Cell) library:(HashmapE 256 SimpleLib) @@ -3227,8 +3343,8 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { } else { CHECK(acc_status == Account::acc_active && !was_frozen && !was_deleted); si_pos = cb.size_ext() + 1; - CHECK(account.split_depth_ ? cb.store_long_bool(account.split_depth_ + 96, 7) // account_active$1 _:StateInit - : cb.store_long_bool(2, 2)); // ... split_depth:(Maybe (## 5)) + CHECK(fixed_prefix_length ? cb.store_long_bool(fixed_prefix_length + 96, 7) // account_active$1 _:StateInit + : cb.store_long_bool(2, 2)); // ... fixed_prefix_length:(Maybe (## 5)) CHECK(ticktock ? cb.store_long_bool(ticktock | 4, 3) : cb.store_long_bool(0, 1)); // special:(Maybe TickTock) CHECK(cb.store_maybe_ref(new_code) && cb.store_maybe_ref(new_data) && cb.store_maybe_ref(new_library)); // code:(Maybe ^Cell) data:(Maybe ^Cell) library:(HashmapE 256 SimpleLib) @@ -3314,8 +3430,8 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { } } - CHECK(cb.store_long_bool(1, 1) // account$1 - && cb.append_cellslice_bool(account.my_addr) // addr:MsgAddressInt + CHECK(cb.store_long_bool(1, 1) // account$1 + && cb.append_cellslice_bool(cfg.disable_anycast ? my_addr : account.my_addr) // addr:MsgAddressInt && block::store_UInt7(cb, new_storage_used.cells) // storage_used$_ cells:(VarUInteger 7) && block::store_UInt7(cb, new_storage_used.bits) // bits:(VarUInteger 7) && cb.store_long_bool(new_storage_dict_hash ? 1 : 0, 3) // extra:StorageExtraInfo @@ -3677,12 +3793,14 @@ Ref Transaction::commit(Account& acc) { CHECK((const void*)&acc == (const void*)&account); // export all fields modified by the Transaction into original account // NB: this is the only method that modifies account - if (orig_addr_rewrite_set && new_split_depth >= 0 && acc.status != Account::acc_active && + if (force_remove_anycast_address) { + CHECK(acc.forget_addr_rewrite_length()); + } else if (orig_addr_rewrite_set && new_addr_rewrite_length >= 0 && acc.status != Account::acc_active && acc_status == Account::acc_active) { LOG(DEBUG) << "setting address rewriting info for newly-activated account " << acc.addr.to_hex() - << " with split_depth=" << new_split_depth - << ", orig_addr_rewrite=" << orig_addr_rewrite.bits().to_hex(new_split_depth); - CHECK(acc.init_rewrite_addr(new_split_depth, orig_addr_rewrite.bits())); + << " with addr_rewrite_length=" << new_addr_rewrite_length + << ", orig_addr_rewrite=" << orig_addr_rewrite.bits().to_hex(new_addr_rewrite_length); + CHECK(acc.init_rewrite_addr(new_addr_rewrite_length, orig_addr_rewrite.bits())); } acc.status = (acc_status == Account::acc_deleted ? Account::acc_nonexist : acc_status); acc.last_trans_lt_ = start_lt; @@ -3714,6 +3832,7 @@ Ref Transaction::commit(Account& acc) { if (acc.status == Account::acc_active) { acc.tick = new_tick; acc.tock = new_tock; + acc.fixed_prefix_length = new_fixed_prefix_length; } else { CHECK(acc.deactivate()); } @@ -3914,6 +4033,7 @@ td::Status FetchConfigParams::fetch_config_params( compute_phase_cfg->size_limits = size_limits; compute_phase_cfg->precompiled_contracts = config.get_precompiled_contracts_config(); compute_phase_cfg->allow_external_unfreeze = compute_phase_cfg->global_version >= 8; + compute_phase_cfg->disable_anycast = config.get_global_version() >= 10; } { // compute action_phase_cfg @@ -3942,9 +4062,11 @@ td::Status FetchConfigParams::fetch_config_params( action_phase_cfg->reserve_extra_enabled = config.get_global_version() >= 9; action_phase_cfg->mc_blackhole_addr = config.get_burning_config().blackhole_addr; action_phase_cfg->extra_currency_v2 = config.get_global_version() >= 10; + action_phase_cfg->disable_anycast = config.get_global_version() >= 10; } { serialize_cfg->extra_currency_v2 = config.get_global_version() >= 10; + serialize_cfg->disable_anycast = config.get_global_version() >= 10; serialize_cfg->store_storage_dict_hash = config.get_global_version() >= 11; } { diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index aa08719a8..fe7006006 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -31,6 +31,7 @@ #include "block/block.h" #include "block/mc-config.h" #include "precompiled-smc/PrecompiledSmartContract.h" +#include "block/block-auto.h" namespace block { using td::Ref; @@ -131,6 +132,7 @@ struct ComputePhaseConfig { PrecompiledContractsConfig precompiled_contracts; bool dont_run_precompiled_ = false; bool allow_external_unfreeze{false}; + bool disable_anycast{false}; ComputePhaseConfig() : gas_price(0), gas_limit(0), special_gas_limit(0), gas_credit(0) { compute_threshold(); @@ -173,6 +175,7 @@ struct ActionPhaseConfig { bool reserve_extra_enabled{false}; bool extra_currency_v2{false}; td::optional mc_blackhole_addr; + bool disable_anycast{false}; const MsgPrices& fetch_msg_prices(bool is_masterchain) const { return is_masterchain ? fwd_mc : fwd_std; } @@ -180,6 +183,7 @@ struct ActionPhaseConfig { struct SerializeConfig { bool extra_currency_v2{false}; + bool disable_anycast{false}; bool store_storage_dict_hash{false}; }; @@ -254,12 +258,13 @@ struct Account { bool is_special{false}; bool tick{false}; bool tock{false}; - bool split_depth_set_{false}; - unsigned char split_depth_{0}; + int fixed_prefix_length{0}; int verbosity{3 * 0}; ton::UnixTime now_{0}; ton::WorkchainId workchain{ton::workchainInvalid}; - td::BitArray<32> addr_rewrite; // rewrite (anycast) data, split_depth bits + td::BitArray<32> addr_rewrite; // rewrite (anycast) data, addr_rewrite_length bits + bool addr_rewrite_length_set{false}; + unsigned char addr_rewrite_length{0}; ton::StdSmcAddress addr; // rewritten address (by replacing a prefix of `addr_orig` with `addr_rewrite`); it is the key in ShardAccounts ton::StdSmcAddress addr_orig; // address indicated in smart-contract data (must coincide with hash of StateInit) Ref my_addr; // address as stored in the smart contract (MsgAddressInt); corresponds to `addr_orig` + anycast info @@ -286,9 +291,6 @@ struct Account { Account() = default; Account(ton::WorkchainId wc, td::ConstBitPtr _addr) : workchain(wc), addr(_addr) { } - Account(ton::WorkchainId wc, td::ConstBitPtr _addr, int depth) - : split_depth_set_(true), split_depth_((unsigned char)depth), workchain(wc), addr(_addr) { - } block::CurrencyCollection get_balance() const { return balance; } @@ -296,7 +298,7 @@ struct Account { bool unpack(Ref account, ton::UnixTime now, bool special); bool init_new(ton::UnixTime now); bool deactivate(); - bool recompute_tmp_addr(Ref& tmp_addr, int split_depth, td::ConstBitPtr orig_addr_rewrite) const; + bool recompute_tmp_addr(Ref& tmp_addr, int fixed_prefix_length, td::ConstBitPtr orig_addr_rewrite) const; td::RefInt256 compute_storage_fees(ton::UnixTime now, const std::vector& pricing) const; bool is_masterchain() const { return workchain == ton::masterchainId; @@ -312,10 +314,10 @@ struct Account { protected: friend struct transaction::Transaction; - bool set_split_depth(int split_depth); - bool check_split_depth(int split_depth) const; - bool forget_split_depth(); - bool init_rewrite_addr(int split_depth, td::ConstBitPtr orig_addr_rewrite); + bool set_addr_rewrite_length(int new_length); + bool check_addr_rewrite_length(int length) const; + bool forget_addr_rewrite_length(); + bool init_rewrite_addr(int rewrite_length, td::ConstBitPtr orig_addr_rewrite); private: bool unpack_address(vm::CellSlice& addr_cs); @@ -347,12 +349,15 @@ struct Transaction { bool was_created{false}; bool bounce_enabled{false}; bool in_msg_extern{false}; + gen::CommonMsgInfo::Record_int_msg_info in_msg_info; bool use_msg_state{false}; bool is_first{false}; bool orig_addr_rewrite_set{false}; bool new_tick; bool new_tock; - signed char new_split_depth{-1}; + int new_fixed_prefix_length{-1}; + int new_addr_rewrite_length{-1}; + bool force_remove_anycast_address = false; ton::UnixTime now; int acc_status; int verbosity{3 * 0}; @@ -390,7 +395,7 @@ struct Transaction { Transaction(const Account& _account, int ttype, ton::LogicalTime req_start_lt, ton::UnixTime _now, Ref _inmsg = {}); bool unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* cfg); - bool check_in_msg_state_hash(); + bool check_in_msg_state_hash(const ComputePhaseConfig& cfg); bool prepare_storage_phase(const StoragePhaseConfig& cfg, bool force_collect = true, bool adjust_msg_value = false); bool prepare_credit_phase(); td::uint64 gas_bought_for(const ComputePhaseConfig& cfg, td::RefInt256 nanograms); @@ -426,13 +431,18 @@ struct Transaction { int try_action_reserve_currency(vm::CellSlice& cs, ActionPhase& ap, const ActionPhaseConfig& cfg); bool check_replace_src_addr(Ref& src_addr) const; bool check_rewrite_dest_addr(Ref& dest_addr, const ActionPhaseConfig& cfg, - bool* is_mc = nullptr) const; + bool* is_mc = nullptr, bool allow_anycast = true) const; bool serialize_storage_phase(vm::CellBuilder& cb); bool serialize_credit_phase(vm::CellBuilder& cb); bool serialize_compute_phase(vm::CellBuilder& cb); bool serialize_action_phase(vm::CellBuilder& cb); bool serialize_bounce_phase(vm::CellBuilder& cb); bool unpack_msg_state(const ComputePhaseConfig& cfg, bool lib_only = false, bool forbid_public_libs = false); + + public: + static Ref prepare_in_msg_params_tuple(const gen::CommonMsgInfo::Record_int_msg_info* info, + const Ref& state_init, + const CurrencyCollection& msg_balance_remaining); }; } // namespace transaction diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index 4d7e2b6fe..941351397 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -1305,6 +1305,7 @@ x{F814} @Defop SETRAND x{F815} dup @Defop ADDRAND @Defop RANDOMIZE x{F82} @Defop(4u) GETPARAM +x{F881} @Defop(8u) GETPARAMLONG x{F823} @Defop NOW x{F824} @Defop BLOCKLT x{F825} @Defop LTIME @@ -1318,6 +1319,8 @@ x{F82C} @Defop STORAGEFEES x{F82D} @Defop PREVBLOCKSINFOTUPLE x{F82E} @Defop UNPACKEDCONFIGTUPLE x{F82F} @Defop DUEPAYMENT +x{F88111} @Defop INMSGPARAMS + x{F830} @Defop CONFIGDICT x{F832} @Defop CONFIGPARAM x{F833} @Defop CONFIGOPTPARAM @@ -1333,11 +1336,25 @@ x{F83A} @Defop GETORIGINALFWDFEE x{F83B} @Defop GETGASFEESIMPLE x{F83C} @Defop GETFORWARDFEESIMPLE +x{F89} @Defop(4u) INMSGPARAM +x{F890} @Defop INMSG_BOUNCE +x{F891} @Defop INMSG_BOUNCED +x{F892} @Defop INMSG_SRC +x{F893} @Defop INMSG_FWDFEE +x{F894} @Defop INMSG_LT +x{F895} @Defop INMSG_UTIME +x{F896} @Defop INMSG_ORIGVALUE +x{F897} @Defop INMSG_VALUE +x{F898} @Defop INMSG_VALUEEXTRA +x{F899} @Defop INMSG_STATEINIT + x{F840} @Defop GETGLOBVAR { dup 1 31 @rangechk 16 config! } : config.validator_num! diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp index 4d860fbab..db2ab74ab 100644 --- a/crypto/smc-envelope/SmartContract.cpp +++ b/crypto/smc-envelope/SmartContract.cpp @@ -19,6 +19,7 @@ #include "SmartContract.h" #include "GenericAccount.h" +#include "transaction.h" #include "block/block.h" #include "block/block-auto.h" @@ -175,13 +176,16 @@ td::Ref prepare_vm_c7(SmartContract::Args args, td::Ref cod if (args.config && args.config.value()->get_global_version() >= 6) { tuple.push_back(args.config.value()->get_unpacked_config_tuple(now)); // unpacked_config_tuple tuple.push_back(td::zero_refint()); // due_payment - // precomiled_gas_usage:(Maybe Integer) + // precompiled_gas_usage:(Maybe Integer) td::optional precompiled; if (code.not_null()) { precompiled = args.config.value()->get_precompiled_contracts_config().get_contract(code->get_hash().bits()); } tuple.push_back(precompiled ? td::make_refint(precompiled.value().gas_usage) : vm::StackEntry()); } + if (args.config && args.config.value()->get_global_version() >= 11) { + tuple.push_back(block::transaction::Transaction::prepare_in_msg_params_tuple(nullptr, {}, {})); + } auto tuple_ref = td::make_cnt_ref>(std::move(tuple)); //LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple).to_string(); return vm::make_tuple_ref(std::move(tuple_ref)); diff --git a/crypto/test/fift.cpp b/crypto/test/fift.cpp index 9a92e0b1e..38a9ddd92 100644 --- a/crypto/test/fift.cpp +++ b/crypto/test/fift.cpp @@ -171,3 +171,7 @@ TEST(Fift, test_levels) { TEST(Fift, test_secp256k1) { run_fift("secp256k1.fif"); } + +TEST(Fift, test_get_extra_balance) { + run_fift("get_extra_balance.fif"); +} diff --git a/crypto/test/fift/get_extra_balance.fif b/crypto/test/fift/get_extra_balance.fif new file mode 100644 index 000000000..2b475a7ed --- /dev/null +++ b/crypto/test/fift/get_extra_balance.fif @@ -0,0 +1,80 @@ +"Asm.fif" include +"FiftExt.fif" include +"TonUtil.fif" include + +null constant extras +{ null } 7 times +10 extras 2 tuple +8 tuple 1 tuple constant c7 +<{ + GASCONSUMED 1 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 2 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 3 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 4 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 5 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 6 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 7 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 8 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 9 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 10 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 0 INT <{ }>s SLICE 128 RUNVM GASCONSUMED ROT SUB 26 INT SUB +}>s c7 16 runvmx abort"exitcode != 0" +.s { drop } depth 1- times + +dictnew + s SLICE 128 RUNVM GASCONSUMED ROT SUB 26 INT SUB +}>s c7 16 runvmx abort"exitcode != 0" +.s { drop } depth 1- times + +dictnew 1 +{ + =: idx =: d + s SLICE 128 RUNVM GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 1 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 2 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 3 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 4 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 5 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 6 INT GETEXTRABALANCE GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 0 INT <{ }>s SLICE 128 RUNVM GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 0 INT <{ }>s SLICE 128 RUNVM GASCONSUMED ROT SUB 26 INT SUB + GASCONSUMED 0 INT <{ }>s SLICE 128 RUNVM GASCONSUMED ROT SUB 26 INT SUB +}>s c7 16 runvmx abort"exitcode != 0" +.s { drop } depth 1- times diff --git a/crypto/vm/tonops.cpp b/crypto/vm/tonops.cpp index aab1711f4..b62e5fb99 100644 --- a/crypto/vm/tonops.cpp +++ b/crypto/vm/tonops.cpp @@ -63,6 +63,9 @@ bool debug(int x) { #define DEB_START DBG_START #define DEB DBG +static constexpr int randseed_idx = 6; +static constexpr int inmsgparams_idx = 17; + int exec_set_gas_generic(VmState* st, long long new_gas_limit) { if (new_gas_limit < st->gas_consumed()) { throw VmNoGas{}; @@ -152,6 +155,27 @@ int exec_get_var_param(VmState* st, unsigned idx) { return exec_get_param(st, idx, nullptr); } +int exec_get_var_param_long(VmState* st, unsigned idx) { + idx &= 255; + VM_LOG(st) << "execute GETPARAMLONG " << idx; + return exec_get_param(st, idx, nullptr); +} + +int exec_get_in_msg_param(VmState* st, unsigned idx, const char* name) { + if (name) { + VM_LOG(st) << "execute " << name; + } + Ref t = get_param(st, inmsgparams_idx).as_tuple(); + st->get_stack().push(tuple_index(t, idx)); + return 0; +} + +int exec_get_var_in_msg_param(VmState* st, unsigned idx) { + idx &= 15; + VM_LOG(st) << "execute INMSGPARAM " << idx; + return exec_get_in_msg_param(st, idx, nullptr); +} + int exec_get_config_dict(VmState* st) { exec_get_param(st, 9, "CONFIGDICT"); st->get_stack().push_smallint(32); @@ -358,6 +382,75 @@ int exec_get_forward_fee_simple(VmState* st) { return 0; } +int exec_get_extra_currency_balance(VmState* st) { + VM_LOG(st) << "execute GETEXTRABALANCE"; + Stack& stack = st->get_stack(); + auto id = (td::uint32)stack.pop_long_range((1LL << 32) - 1); + + auto tuple = st->get_c7(); + tuple = tuple_index(tuple, 0).as_tuple_range(255); + if (tuple.is_null()) { + throw VmError{Excno::type_chk, "intermediate value is not a tuple"}; + } + tuple = tuple_index(tuple, 7).as_tuple_range(255); // Balance + if (tuple.is_null()) { + throw VmError{Excno::type_chk, "intermediate value is not a tuple"}; + } + auto dict_root = tuple_index(tuple, 1); + if (!dict_root.is_cell() && !dict_root.is_null()) { + throw VmError{Excno::type_chk, "intermediate value is not cell or null"}; + } + + class LocalVmState : public VmStateInterface { + public: + explicit LocalVmState(VmState* st) : st_(st) { + } + ~LocalVmState() override = default; + + Ref load_library(td::ConstBitPtr hash) override { + return st_->load_library(hash); + } + void register_cell_load(const CellHash& cell_hash) override { + auto new_cell = st_->register_cell_load_free(cell_hash); + consume_gas(new_cell ? VmState::cell_load_gas_price : VmState::cell_reload_gas_price); + } + void register_cell_create() override { + // Not expected in this operation + } + int get_global_version() const override { + return st_->get_global_version(); + } + + private: + VmState* st_; + long long remaining = VmState::get_extra_balance_cheap_max_gas_price; + + void consume_gas(long long gas) { + long long consumed = std::min(gas, remaining); + st_->consume_gas(consumed); + remaining -= consumed; + if (remaining == 0) { + st_->consume_free_gas(gas - consumed); + } + } + }; + bool cheap = st->register_get_extra_balance_call(); + LocalVmState local_vm_state{st}; + VmStateInterface::Guard guard{cheap ? (VmStateInterface*)&local_vm_state : st}; + + Dictionary dict{dict_root.as_cell(), 32}; + Ref cs = dict.lookup(td::BitArray<32>(id)); + if (cs.is_null()) { + stack.push_smallint(0); + } else { + td::RefInt256 x; + util::load_var_integer_q(cs.write(), x, /* len_bits = */ 5, /* sgnd = */ false, /* quiet = */ false); + stack.push_int(std::move(x)); + } + + return 0; +} + void register_ton_config_ops(OpcodeTable& cp0) { using namespace std::placeholders; cp0.insert(OpcodeInstr::mkfixedrange(0xf820, 0xf823, 16, 4, instr::dump_1c("GETPARAM "), exec_get_var_param)) @@ -391,11 +484,24 @@ void register_ton_config_ops(OpcodeTable& cp0) { .insert(OpcodeInstr::mksimple(0xf840, 16, "GETGLOBVAR", exec_get_global_var)) .insert(OpcodeInstr::mkfixedrange(0xf841, 0xf860, 16, 5, instr::dump_1c_and(31, "GETGLOB "), exec_get_global)) .insert(OpcodeInstr::mksimple(0xf860, 16, "SETGLOBVAR", exec_set_global_var)) - .insert(OpcodeInstr::mkfixedrange(0xf861, 0xf880, 16, 5, instr::dump_1c_and(31, "SETGLOB "), exec_set_global)); + .insert(OpcodeInstr::mkfixedrange(0xf861, 0xf880, 16, 5, instr::dump_1c_and(31, "SETGLOB "), exec_set_global)) + .insert(OpcodeInstr::mksimple(0xf880, 16, "GETEXTRABALANCE", exec_get_extra_currency_balance)->require_version(10)) + .insert(OpcodeInstr::mkfixedrange(0xf88100, 0xf88111, 24, 8, instr::dump_1c_l_add(0, "GETPARAMLONG "), exec_get_var_param_long)->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf88111, 24, "INMSGPARAMS", std::bind(exec_get_param, _1, 17, "INMSGPARAMS"))->require_version(11)) + .insert(OpcodeInstr::mkfixedrange(0xf88112, 0xf881ff, 24, 8, instr::dump_1c_l_add(0, "GETPARAMLONG "), exec_get_var_param_long)->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf890, 16, "INMSG_BOUNCE", std::bind(exec_get_in_msg_param, _1, 0, "INMSG_BOUNCE"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf891, 16, "INMSG_BOUNCED", std::bind(exec_get_in_msg_param, _1, 1, "INMSG_BOUNCED"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf892, 16, "INMSG_SRC", std::bind(exec_get_in_msg_param, _1, 2, "INMSG_SRC"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf893, 16, "INMSG_FWDFEE", std::bind(exec_get_in_msg_param, _1, 3, "INMSG_FWDFEE"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf894, 16, "INMSG_LT", std::bind(exec_get_in_msg_param, _1, 4, "INMSG_LT"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf895, 16, "INMSG_UTIME", std::bind(exec_get_in_msg_param, _1, 5, "INMSG_UTIME"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf896, 16, "INMSG_ORIGVALUE", std::bind(exec_get_in_msg_param, _1, 6, "INMSG_ORIGVALUE"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf897, 16, "INMSG_VALUE", std::bind(exec_get_in_msg_param, _1, 7, "INMSG_VALUE"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf898, 16, "INMSG_VALUEEXTRA", std::bind(exec_get_in_msg_param, _1, 8, "INMSG_VALUEEXTRA"))->require_version(11)) + .insert(OpcodeInstr::mksimple(0xf899, 16, "INMSG_STATEINIT", std::bind(exec_get_in_msg_param, _1, 9, "INMSG_STATEINIT"))->require_version(11)) + .insert(OpcodeInstr::mkfixedrange(0xf89a, 0xf8a0, 16, 4, instr::dump_1c("INMSGPARAM "), exec_get_var_in_msg_param)->require_version(11)); } -static constexpr int randseed_idx = 6; - td::RefInt256 generate_randu256(VmState* st) { auto tuple = st->get_c7(); auto t1 = tuple_index(tuple, 0).as_tuple_range(255); @@ -1380,10 +1486,13 @@ int exec_store_var_integer(VmState* st, int len_bits, bool sgnd, bool quiet) { return 0; } -bool skip_maybe_anycast(CellSlice& cs) { +bool skip_maybe_anycast(CellSlice& cs, int global_version) { if (cs.prefetch_ulong(1) != 1) { return cs.advance(1); } + if (global_version >= 10) { + return false; + } unsigned depth; return cs.advance(1) // just$1 && cs.fetch_uint_leq(30, depth) // anycast_info$_ depth:(#<= 30) @@ -1391,7 +1500,7 @@ bool skip_maybe_anycast(CellSlice& cs) { && cs.advance(depth); // rewrite_pfx:(bits depth) = Anycast; } -bool skip_message_addr(CellSlice& cs) { +bool skip_message_addr(CellSlice& cs, int global_version) { switch ((unsigned)cs.fetch_ulong(2)) { case 0: // addr_none$00 = MsgAddressExt; return true; @@ -1400,15 +1509,18 @@ bool skip_message_addr(CellSlice& cs) { return cs.fetch_uint_to(9, len) // len:(## 9) && cs.advance(len); // external_address:(bits len) = MsgAddressExt; } - case 2: { // addr_std$10 - return skip_maybe_anycast(cs) // anycast:(Maybe Anycast) - && cs.advance(8 + 256); // workchain_id:int8 address:bits256 = MsgAddressInt; + case 2: { // addr_std$10 + return skip_maybe_anycast(cs, global_version) // anycast:(Maybe Anycast) + && cs.advance(8 + 256); // workchain_id:int8 address:bits256 = MsgAddressInt; } case 3: { // addr_var$11 + if (global_version >= 10) { + return false; + } unsigned len; - return skip_maybe_anycast(cs) // anycast:(Maybe Anycast) - && cs.fetch_uint_to(9, len) // addr_len:(## 9) - && cs.advance(32 + len); // workchain_id:int32 address:(bits addr_len) = MsgAddressInt; + return skip_maybe_anycast(cs, global_version) // anycast:(Maybe Anycast) + && cs.fetch_uint_to(9, len) // addr_len:(## 9) + && cs.advance(32 + len); // workchain_id:int32 address:(bits addr_len) = MsgAddressInt; } default: return false; @@ -1420,7 +1532,7 @@ int exec_load_message_addr(VmState* st, bool quiet) { Stack& stack = st->get_stack(); auto csr = stack.pop_cellslice(); td::Ref addr{true}; - if (util::load_msg_addr_q(csr.write(), addr.write(), quiet)) { + if (util::load_msg_addr_q(csr.write(), addr.write(), st->get_global_version(), quiet)) { stack.push_cellslice(std::move(addr)); stack.push_cellslice(std::move(csr)); if (quiet) { @@ -1433,11 +1545,14 @@ int exec_load_message_addr(VmState* st, bool quiet) { return 0; } -bool parse_maybe_anycast(CellSlice& cs, StackEntry& res) { +bool parse_maybe_anycast(CellSlice& cs, StackEntry& res, int global_version) { res = StackEntry{}; if (cs.prefetch_ulong(1) != 1) { return cs.advance(1); } + if (global_version >= 10) { + return false; + } unsigned depth; Ref pfx; if (cs.advance(1) // just$1 @@ -1450,7 +1565,7 @@ bool parse_maybe_anycast(CellSlice& cs, StackEntry& res) { return false; } -bool parse_message_addr(CellSlice& cs, std::vector& res) { +bool parse_message_addr(CellSlice& cs, std::vector& res, int global_version) { res.clear(); switch ((unsigned)cs.fetch_ulong(2)) { case 0: // addr_none$00 = MsgAddressExt; @@ -1471,9 +1586,9 @@ bool parse_message_addr(CellSlice& cs, std::vector& res) { StackEntry v; int workchain; Ref addr; - if (parse_maybe_anycast(cs, v) // anycast:(Maybe Anycast) - && cs.fetch_int_to(8, workchain) // workchain_id:int8 - && cs.fetch_subslice_to(256, addr)) { // address:bits256 = MsgAddressInt; + if (parse_maybe_anycast(cs, v, global_version) // anycast:(Maybe Anycast) + && cs.fetch_int_to(8, workchain) // workchain_id:int8 + && cs.fetch_subslice_to(256, addr)) { // address:bits256 = MsgAddressInt; res.emplace_back(td::make_refint(2)); res.emplace_back(std::move(v)); res.emplace_back(td::make_refint(workchain)); @@ -1483,13 +1598,16 @@ bool parse_message_addr(CellSlice& cs, std::vector& res) { break; } case 3: { // addr_var$11 + if (global_version >= 10) { + return false; + } StackEntry v; int len, workchain; Ref addr; - if (parse_maybe_anycast(cs, v) // anycast:(Maybe Anycast) - && cs.fetch_uint_to(9, len) // addr_len:(## 9) - && cs.fetch_int_to(32, workchain) // workchain_id:int32 - && cs.fetch_subslice_to(len, addr)) { // address:(bits addr_len) = MsgAddressInt; + if (parse_maybe_anycast(cs, v, global_version) // anycast:(Maybe Anycast) + && cs.fetch_uint_to(9, len) // addr_len:(## 9) + && cs.fetch_int_to(32, workchain) // workchain_id:int32 + && cs.fetch_subslice_to(len, addr)) { // address:(bits addr_len) = MsgAddressInt; res.emplace_back(td::make_refint(3)); res.emplace_back(std::move(v)); res.emplace_back(td::make_refint(workchain)); @@ -1508,7 +1626,7 @@ int exec_parse_message_addr(VmState* st, bool quiet) { auto csr = stack.pop_cellslice(); auto& cs = csr.write(); std::vector res; - if (!(parse_message_addr(cs, res) && cs.empty_ext())) { + if (!(parse_message_addr(cs, res, st->get_global_version()) && cs.empty_ext())) { if (quiet) { stack.push_bool(false); } else { @@ -1548,7 +1666,7 @@ int exec_rewrite_message_addr(VmState* st, bool allow_var_addr, bool quiet) { auto csr = stack.pop_cellslice(); auto& cs = csr.write(); std::vector tuple; - if (!(parse_message_addr(cs, tuple) && cs.empty_ext())) { + if (!(parse_message_addr(cs, tuple, st->get_global_version()) && cs.empty_ext())) { if (quiet) { stack.push_bool(false); return 0; @@ -2026,9 +2144,9 @@ bool load_var_integer_q(CellSlice& cs, td::RefInt256& res, int len_bits, bool sg bool load_coins_q(CellSlice& cs, td::RefInt256& res, bool quiet) { return load_var_integer_q(cs, res, 4, false, quiet); } -bool load_msg_addr_q(CellSlice& cs, CellSlice& res, bool quiet) { +bool load_msg_addr_q(CellSlice& cs, CellSlice& res, int global_version, bool quiet) { res = cs; - if (!skip_message_addr(cs)) { + if (!skip_message_addr(cs, global_version)) { cs = res; if (quiet) { return false; @@ -2038,10 +2156,11 @@ bool load_msg_addr_q(CellSlice& cs, CellSlice& res, bool quiet) { res.cut_tail(cs); return true; } -bool parse_std_addr_q(CellSlice cs, ton::WorkchainId& res_wc, ton::StdSmcAddress& res_addr, bool quiet) { +bool parse_std_addr_q(CellSlice cs, ton::WorkchainId& res_wc, ton::StdSmcAddress& res_addr, int global_version, + bool quiet) { // Like exec_rewrite_message_addr, but for std address case std::vector tuple; - if (!(parse_message_addr(cs, tuple) && cs.empty_ext())) { + if (!(parse_message_addr(cs, tuple, global_version) && cs.empty_ext())) { if (quiet) { return false; } @@ -2076,14 +2195,14 @@ td::RefInt256 load_var_integer(CellSlice& cs, int len_bits, bool sgnd) { td::RefInt256 load_coins(CellSlice& cs) { return load_var_integer(cs, 4, false); } -CellSlice load_msg_addr(CellSlice& cs) { +CellSlice load_msg_addr(CellSlice& cs, int global_version) { CellSlice addr; - load_msg_addr_q(cs, addr, false); + load_msg_addr_q(cs, addr, global_version, false); return addr; } -std::pair parse_std_addr(CellSlice cs) { +std::pair parse_std_addr(CellSlice cs, int global_version) { std::pair res; - parse_std_addr_q(std::move(cs), res.first, res.second, false); + parse_std_addr_q(std::move(cs), res.first, res.second, global_version, false); return res; } diff --git a/crypto/vm/tonops.h b/crypto/vm/tonops.h index bbac078f2..d33505085 100644 --- a/crypto/vm/tonops.h +++ b/crypto/vm/tonops.h @@ -32,14 +32,15 @@ namespace util { // "_q" functions throw on error if not quiet, return false if quiet (leaving cs unchanged) bool load_var_integer_q(CellSlice& cs, td::RefInt256& res, int len_bits, bool sgnd, bool quiet); bool load_coins_q(CellSlice& cs, td::RefInt256& res, bool quiet); -bool load_msg_addr_q(CellSlice& cs, CellSlice& res, bool quiet); -bool parse_std_addr_q(CellSlice cs, ton::WorkchainId& res_wc, ton::StdSmcAddress& res_addr, bool quiet); +bool load_msg_addr_q(CellSlice& cs, CellSlice& res, int global_version, bool quiet); +bool parse_std_addr_q(CellSlice cs, ton::WorkchainId& res_wc, ton::StdSmcAddress& res_addr, int global_version, + bool quiet); // Non-"_q" functions throw on error td::RefInt256 load_var_integer(CellSlice& cs, int len_bits, bool sgnd); td::RefInt256 load_coins(CellSlice& cs); -CellSlice load_msg_addr(CellSlice& cs); -std::pair parse_std_addr(CellSlice cs); +CellSlice load_msg_addr(CellSlice& cs, int global_version); +std::pair parse_std_addr(CellSlice cs, int global_version); // store_... functions throw on error if not quiet, return false if quiet (leaving cb unchanged) bool store_var_integer(CellBuilder& cb, const td::RefInt256& x, int len_bits, bool sgnd, bool quiet = false); diff --git a/crypto/vm/vm.cpp b/crypto/vm/vm.cpp index 3c1118c60..33d37973b 100644 --- a/crypto/vm/vm.cpp +++ b/crypto/vm/vm.cpp @@ -656,12 +656,12 @@ bool VmState::register_library_collection(Ref lib) { } void VmState::register_cell_load(const CellHash& cell_hash) { - if (cell_load_gas_price == cell_reload_gas_price) { - consume_gas(cell_load_gas_price); - } else { - auto ok = loaded_cells.insert(cell_hash); // check whether this is the first time this cell is loaded - consume_gas(ok.second ? cell_load_gas_price : cell_reload_gas_price); - } + auto new_cell = loaded_cells.insert(cell_hash).second; // check whether this is the first time this cell is loaded + consume_gas(new_cell ? cell_load_gas_price : cell_reload_gas_price); +} + +bool VmState::register_cell_load_free(const CellHash& cell_hash) { + return loaded_cells.insert(cell_hash).second; } void VmState::register_cell_create() { @@ -708,17 +708,27 @@ Ref lookup_library_in(td::ConstBitPtr key, Ref lib_root) { void VmState::run_child_vm(VmState&& new_state, bool return_data, bool return_actions, bool return_gas, bool isolate_gas, int ret_vals) { - new_state.log = std::move(log); - new_state.libraries = std::move(libraries); + if (global_version < 10) { + new_state.log = std::move(log); + new_state.libraries = std::move(libraries); + } new_state.stack_trace = stack_trace; new_state.max_data_depth = max_data_depth; if (!isolate_gas) { new_state.loaded_cells = std::move(loaded_cells); } else { - consume_gas(std::min(chksgn_counter, chksgn_free_count) * chksgn_gas_price); + consume_gas(free_gas_consumed); chksgn_counter = 0; + get_extra_balance_counter = 0; + free_gas_consumed = 0; + } + if (global_version >= 10) { + new_state.log = std::move(log); + new_state.libraries = std::move(libraries); } new_state.chksgn_counter = chksgn_counter; + new_state.free_gas_consumed = free_gas_consumed; + new_state.get_extra_balance_counter = get_extra_balance_counter; auto new_parent = std::make_unique(); new_parent->return_data = return_data; @@ -743,6 +753,8 @@ void VmState::restore_parent_vm(int res) { loaded_cells = std::move(child_state.loaded_cells); } chksgn_counter = child_state.chksgn_counter; + get_extra_balance_counter = child_state.get_extra_balance_counter; + free_gas_consumed = child_state.free_gas_consumed; VM_LOG(this) << "Child VM finished. res: " << res << ", steps: " << child_state.steps << ", gas: " << child_state.gas_consumed(); diff --git a/crypto/vm/vm.h b/crypto/vm/vm.h index a171ef27e..a0ca4a4b3 100644 --- a/crypto/vm/vm.h +++ b/crypto/vm/vm.h @@ -103,6 +103,8 @@ class VmState final : public VmStateInterface { td::uint16 max_data_depth = 512; // Default value int global_version{0}; size_t chksgn_counter = 0; + size_t get_extra_balance_counter = 0; + long long free_gas_consumed = 0; std::unique_ptr parent = nullptr; public: @@ -161,7 +163,10 @@ class VmState final : public VmStateInterface { bls_g2_multiexp_coef2_gas_price = 22840, bls_pairing_base_gas_price = 20000, - bls_pairing_element_gas_price = 11800 + bls_pairing_element_gas_price = 11800, + + get_extra_balance_cheap_count = 5, + get_extra_balance_cheap_max_gas_price = 200 }; VmState(); VmState(Ref _code, int global_version, Ref _stack, const GasLimits& _gas, int flags = 0, Ref _data = {}, @@ -214,6 +219,9 @@ class VmState final : public VmStateInterface { consume_stack_gas((unsigned)stk->depth()); } } + void consume_free_gas(long long amount) { + free_gas_consumed += amount; + } GasLimits get_gas_limits() const { return gas; } @@ -226,6 +234,7 @@ class VmState final : public VmStateInterface { Ref load_library( td::ConstBitPtr hash) override; // may throw a dictionary exception; returns nullptr if library is not found void register_cell_load(const CellHash& cell_hash) override; + bool register_cell_load_free(const CellHash& cell_hash); void register_cell_create() override; bool init_cp(int new_cp); bool set_cp(int new_cp); @@ -420,9 +429,15 @@ class VmState final : public VmStateInterface { ++chksgn_counter; if (chksgn_counter > chksgn_free_count) { consume_gas(chksgn_gas_price); + } else { + consume_free_gas(chksgn_gas_price); } } } + bool register_get_extra_balance_call() { + ++get_extra_balance_counter; + return get_extra_balance_counter <= get_extra_balance_cheap_count; + } private: void init_cregs(bool same_c3 = false, bool push_0 = true); diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index c0be0b108..c3205f3c8 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -3,7 +3,9 @@ Global version is a parameter specified in `ConfigParam 8` ([block.tlb](https:// Various features are enabled depending on the global version. ## Version 4 -New features of version 4 are desctibed in detail in [the documentation](https://docs.ton.org/v3/documentation/tvm/changelog/tvm-upgrade-2023-07). +__Enabled in mainnet on 2023-12-20__ + +New features of version 4 are described in detail in [the documentation](https://docs.ton.org/v3/documentation/tvm/changelog/tvm-upgrade-2023-07). ### New TVM instructions * `PREVMCBLOCKS`, `PREVKEYBLOCK` @@ -40,6 +42,7 @@ intermediate value before division (e.g. `(xy+w)/z`). * Unpaid storage fee is now saved to `due_payment` ## Version 5 +__Enabled in mainnet on 2024-02-03__ ### Gas limits Version 5 enables higher gas limits for special contracts. @@ -57,6 +60,7 @@ See [this post](https://t.me/tonstatus/88) for details. * `XLOAD` now works differently. When it takes a library cell, it returns the cell that it points to. This allows loading "nested libraries", if needed. ## Version 6 +__Enabled in mainnet on 2024-03-16__ ### c7 tuple **c7** tuple extended from 14 to 17 elements: @@ -101,10 +105,12 @@ Operations for working with Merkle proofs, where cells can have non-zero level a ## Version 7 +__Enabled in mainnet on 2024-04-18__ [Explicitly nullify](https://github.com/ton-blockchain/ton/pull/957/files) `due_payment` after due reimbursment. ## Version 8 +__Enabled in mainnet on 2024-08-25__ - Check mode on invalid `action_send_msg`. Ignore action if `IGNORE_ERROR` (+2) bit is set, bounce if `BOUNCE_ON_FAIL` (+16) bit is set. - Slightly change random seed generation to fix mix of `addr_rewrite` and `addr`. @@ -113,6 +119,7 @@ Operations for working with Merkle proofs, where cells can have non-zero level a - Don't use user-provided `fwd_fee` and `ihr_fee` for internal messages. ## Version 9 +__Enabled in mainnet on 2025-02-13__ ### c7 tuple c7 tuple parameter number **13** (previous blocks info tuple) now has the third element. It contains ids of the 16 last masterchain blocks with seqno divisible by 100. @@ -152,12 +159,63 @@ Example: if the last masterchain block seqno is `19071` then the list contains b - `SENDMSG` does not check the number of extra currencies. - Extra currency dictionary is not counted in the account size and does not affect storage fees. - Accounts with already existing extra currencies will get their sizes recomputed without EC only after modifying `AccountState`. +- Reserve action cannot reserve extra currencies. +Reserve modes `+1`, `+4` and `+8` ("reserve all except", "add original balance" and "negate amount") now only affect TONs, but not extra currencies. + +### Anycast addresses and address rewrite +- Anycast addresses are not allowed in `dest` of internal and external messages. +- `addr_var` are not allowed in `dest` of external messages. + - Note: as before, `addr_var` in `dest` of internal messages are automatically replaced with `addr_std`. +- TVM instructions `LDMSGADDR(Q)`, `PARSEMSGADDR(Q)`, `REWRITESTDADDR(Q)`, `REWRITEVARADDR(Q)` no more support anycast addresses and `addr_var`. +- `addr:MsgAddressInt` in `Account` cannot be an anycast address. + - Therefore, `src` of outbound messages cannot be an anycast address. + - Existing accounts with anycast addresses change to non-anycast addresses in the first transaction. +- When deploying an account with `fixed_prefix_length` in `StateInit` of the message (it was called `split_depth` before), the first `fixed_prefix_length` bits of the address are not compared against the state hash. + - This allows deploying an account to an arbitrary shard regardless of the hash of state init. + - `fixed_prefix_length` remains in the account state. + - `fixed_prefix_length` of the account can be at most 8. The limit can be changed in size limits config (`ConfigParam 43`). ### TVM changes - `SENDMSG` calculates messages size and fees without extra currencies, uses new +64 and +128 mode behavior. - `SENDMSG` does not check the number of extra currencies. +- New instruction `GETEXTRABALANCE` (`id - amount`). Takes id of the extra currency (integer in range `0..2^32-1`), returns the amount of this extra currency on the account balance. + - This is equivalent to taking the extra currency dictionary (`BALANCE SECOND`), loading value (`UDICTGET`) and parsing it (`LDVARUINT32`). If `id` is not present in the dictionary, `0` is returned. + - `GETEXTRABALANCE` has special gas cost that allows writing gas-efficient code with predictable gas usage even if there are a lot of different extra currencies. + - The full gas cost of `GETEXTRABALANCE` is `26` (normal instruction cost) plus gas for loading cells (up to `3300` if the dictionary has maximum depth). + - However, the first `5` executions of `GETEXTRABALANCE` cost at most `26+200` gas units. All subsequent executions cost the full price. + - `RUNVM` interacts with this instructions in the following way: + - Without "isolate gas" mode, the child VM shares `GETEXTRABALANCE` counter with the parent vm. + - With "isolate gas" mode, in the beginning of `RUNVM` the parent VM spends full gas for all already executed `GETEXTRABALANCE` and resets the counter. +- `LDMSGADDR(Q)`, `PARSEMSGADDR(Q)`, `REWRITESTDADDR(Q)`, `REWRITEVARADDR(Q)` no more support anycast addresses and `addr_var`. +- Fixed bug in `RUNVM` caused by throwing out-of-gas exception with "isolate gas" enabled. + +### Other changes +- Exceeding state limits in transaction now reverts `end_lt` back to `start_lt + 1` and collects action fines. ## Version 11 + +### c7 tuple +**c7** tuple extended from 17 to 18 elements: +* **17**: tuple with inbound message parameters. Asm opcode: `INMSGPARAMS`. + * The tuple contains: + * `bounce` (boolean) + * `bounced` (boolean) + * `src_addr` (slice) + * `fwd_fee` (int) + * `created_lt` (int) + * `created_at` (int) + * `orig_value` (int) - this is sometimes different from the value in `INCOMINGVALUE` and TVM stack because of storage fees + * `value` (int) - same as in `INCOMINGVALUE` and TVM stack. + * `value_extra` (cell or null) - same as in `INCOMINGVALUE`. + * `state_init` (cell or null) + * For external messages, tick-tock transactions and get methods: `bounce`, `bounced`, `fwd_fee`, `created_lt`, `created_at`, `orig_value`, `value` are 0, `value_extra` is null. + * For tick-tock transactions and get methods: `src_addr` is `addr_none`. + +### New TVM instructions +- `x GETPARAMLONG` - same as `x GETPARAM`, but `x` is in range `[0..254]`. Gas cost: `34`. +- `x INMSGPARAM` - equivalent to `INMSGPARAMS` `x INDEX`. Gas cost: `26`. + - Aliases: `INMSG_BOUNCE`, `INMSG_BOUNCED`, `INMSG_SRC`, `INMSG_FWDFEE`, `INMSG_LT`, `INMSG_UTIME`, `INMSG_ORIGVALUE`, `INMSG_VALUE`, `INMSG_VALUEEXTRA`, `INMSG_STATEINIT`. + ### New account storage stat Along with the storage stat (cells and bits count), each account now stores the hash of the **storage dict**. diff --git a/test/regression-tests.ans b/test/regression-tests.ans index 14d5958b2..22e7826ea 100644 --- a/test/regression-tests.ans +++ b/test/regression-tests.ans @@ -16,6 +16,7 @@ Test_Fift_test_dict_default a9c8cbcfdece5573185022cea07f59f1bc404e5d879e5157a574 Test_Fift_test_disasm_default 412cf37d37c5d9d81f44dbf4e3d3e7cda173c23b890614eb8a3bc5f2b92f13e6 Test_Fift_test_fiftext_default 2b0db5d4d4bfbc705b959cc787540d7b3a21a71469eac54756e76953f0d9afca Test_Fift_test_fixed_default 278a19d56b773102caf5c1fe2997ea6c8d0d9e720eff8503feede6398a197eec +Test_Fift_test_get_extra_balance_default cc2428172b660ae66489e1b4868786654195137cc29524022cd1cfcf6708336d Test_Fift_test_hash_ext_default 686fc5680feca5b3bb207768215b27f6872a95128762dee0d7f2c88bc492d62d Test_Fift_test_hmap_default c269246882039824bb5822e896c3e6e82ef8e1251b6b251f5af8ea9fb8d05067 Test_Fift_test_levels_default 9fba4a7c98aec9000f42846d6e5fd820343ba61d68f9139dd16c88ccda757cf3 diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index fe7af04e7..fc8735d31 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -606,9 +606,6 @@ void CellDbIn::gc_cont2(BlockHandle handle) { if (r_cell.is_ok()) { cell = r_cell.move_as_ok(); boc_->dec(cell); - LOG(ERROR) << "GC of " << handle->id().to_str(); - } else { - LOG(ERROR) << "GC of UNKNOWN root: " << handle->id().to_str(); } db_busy_ = true; @@ -806,7 +803,6 @@ void CellDb::load_cell(RootHash hash, td::Promise> promise if (result.is_ok()) { return async_apply("load_cell_result", std::move(promise), std::move(result)); } else { - LOG(ERROR) << "load_root_thread_safe failed - this is suspicious"; send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise)); return; } diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 8457d234f..e9592f626 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -1432,13 +1432,16 @@ static td::Ref prepare_vm_c7(ton::UnixTime now, ton::LogicalTime lt, if (config && config->get_global_version() >= 6) { tuple.push_back(vm::StackEntry::maybe(config->get_unpacked_config_tuple(now))); // unpacked_config_tuple:[...] tuple.push_back(due_payment); // due_payment:Integer - // precomiled_gas_usage:(Maybe Integer) + // precompiled_gas_usage:(Maybe Integer) td::optional precompiled; if (my_code.not_null()) { precompiled = config->get_precompiled_contracts_config().get_contract(my_code->get_hash().bits()); } tuple.push_back(precompiled ? td::make_refint(precompiled.value().gas_usage) : vm::StackEntry()); } + if (config && config->get_global_version() >= 11) { + tuple.push_back(block::transaction::Transaction::prepare_in_msg_params_tuple(nullptr, {}, {})); + } auto tuple_ref = td::make_cnt_ref>(std::move(tuple)); LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple_ref).to_string(); return vm::make_tuple_ref(std::move(tuple_ref)); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 5f2ba6615..88ae031e2 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -977,6 +977,7 @@ bool ValidateQuery::fetch_config_params() { compute_phase_cfg_.size_limits = size_limits; compute_phase_cfg_.precompiled_contracts = config_->get_precompiled_contracts_config(); compute_phase_cfg_.allow_external_unfreeze = compute_phase_cfg_.global_version >= 8; + compute_phase_cfg_.disable_anycast = config_->get_global_version() >= 10; } { // compute action_phase_cfg @@ -1005,9 +1006,11 @@ bool ValidateQuery::fetch_config_params() { action_phase_cfg_.reserve_extra_enabled = config_->get_global_version() >= 9; action_phase_cfg_.mc_blackhole_addr = config_->get_burning_config().blackhole_addr; action_phase_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; + action_phase_cfg_.disable_anycast = config_->get_global_version() >= 10; } { serialize_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; + serialize_cfg_.disable_anycast = config_->get_global_version() >= 10; serialize_cfg_.store_storage_dict_hash = config_->get_global_version() >= 11; } { From 982c38a1fed43b710bb4efa4449f0b4c449a67c3 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Mon, 21 Apr 2025 12:20:54 +0300 Subject: [PATCH 220/388] Increase default broadcast speed multiplier to 3.33 (#1631) --- validator-engine/validator-engine.cpp | 6 +++--- validator-engine/validator-engine.hpp | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 6f47f7741..582902782 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -4588,7 +4588,7 @@ int main(int argc, char *argv[]) { }); p.add_checked_option( '\0', "broadcast-speed-catchain", - "multiplier for broadcast speed in catchain overlays (experimental, default is 1.0, which is ~300 KB/s)", + "multiplier for broadcast speed in catchain overlays (experimental, default is 3.33, which is ~1 MB/s)", [&](td::Slice s) -> td::Status { auto v = td::to_double(s); if (v <= 0.0) { @@ -4600,7 +4600,7 @@ int main(int argc, char *argv[]) { }); p.add_checked_option( '\0', "broadcast-speed-public", - "multiplier for broadcast speed in public shard overlays (experimental, default is 1.0, which is ~300 KB/s)", + "multiplier for broadcast speed in public shard overlays (experimental, default is 3.33, which is ~1 MB/s)", [&](td::Slice s) -> td::Status { auto v = td::to_double(s); if (v <= 0.0) { @@ -4612,7 +4612,7 @@ int main(int argc, char *argv[]) { }); p.add_checked_option( '\0', "broadcast-speed-private", - "multiplier for broadcast speed in private block overlays (experimental, default is 1.0, which is ~300 KB/s)", + "multiplier for broadcast speed in private block overlays (experimental, default is 3.33, which is ~1 MB/s)", [&](td::Slice s) -> td::Status { auto v = td::to_double(s); if (v <= 0.0) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 77030d68b..de6c3ce19 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -231,9 +231,9 @@ class ValidatorEngine : public td::actor::Actor { bool not_all_shards_ = false; std::vector add_shard_cmds_; bool state_serializer_disabled_flag_ = false; - double broadcast_speed_multiplier_catchain_ = 1.0; - double broadcast_speed_multiplier_public_ = 1.0; - double broadcast_speed_multiplier_private_ = 1.0; + double broadcast_speed_multiplier_catchain_ = 3.33; + double broadcast_speed_multiplier_public_ = 3.33; + double broadcast_speed_multiplier_private_ = 3.33; std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; From d065e51d90e1bd845983acd8e1155a79b757250e Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Mon, 21 Apr 2025 15:05:14 +0300 Subject: [PATCH 221/388] Update Changelog (#1633) --- Changelog.md | 12 ++++++++++++ recent_changelog.md | 23 +++++++++++------------ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/Changelog.md b/Changelog.md index 4dce39fcf..344b26cd3 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,15 @@ +## 2025.04 Update + +1. Introduced substantial improvements of CellDB performance: celldb-v2, bloom filters. +2. Accelerated a number of intrinsic node operations: SHA256, cell operations, large boc serialization, validator set checks. +3. [TVM version v10](./doc/GlobalVersions.md) +4. Overlay broadcast speed up and improved network stats. +5. Fixed some issues in tonlib +6. [Added normalized hash](https://github.com/ton-blockchain/TEPs/pull/467) +7. Fix SDBEGINS(Q) in Asm.fif + +Besides the work of the core team, this update is based on the efforts of @Stanislav-Povolotsky (tonlib fixes), @ice-charon (tonlib fixes), RSquad team (due payments improvements in v10), Arayz @ TonBit (improvements in RUNVM), @Skydev0h and @pyAndr3w (Asm.fif). + ## 2025.03 Update 1. New extracurrency behavior introduced, check [GlobalVersions.md](./doc/GlobalVersions.md#version-10) 2. Optmization of validation process, in particular CellStorageStat. diff --git a/recent_changelog.md b/recent_changelog.md index 820d2aa42..558960318 100644 --- a/recent_changelog.md +++ b/recent_changelog.md @@ -1,13 +1,12 @@ -## 2025.03 Update -1. New extracurrency behavior introduced, check [GlobalVersions.md](./doc/GlobalVersions.md#version-10) -2. Optmization of validation process, in particular CellStorageStat. -3. Flag for speeding up broadcasts in various overlays. -4. Fixes for static builds for emulator and tonlibjson -5. Improving getstats output: add - * Liteserver queries count - * Collated/validated blocks count, number of active sessions - * Persistent state sizes - * Initial sync progress -6. Fixes in logging, TON Storage, external message checking, persistent state downloading, UB in tonlib +## 2025.04 Update + +1. Introduced substantial improvements of CellDB performance: celldb-v2, bloom filters. +2. Accelerated a number of intrinsic node operations: SHA256, cell operations, large boc serialization, validator set checks. +3. [TVM version v10](./doc/GlobalVersions.md) +4. Overlay broadcast speed up and improved network stats. +5. Fixed some issues in tonlib +6. [Added normalized hash](https://github.com/ton-blockchain/TEPs/pull/467) +7. Fix SDBEGINS(Q) in Asm.fif + +Besides the work of the core team, this update is based on the efforts of @Stanislav-Povolotsky (tonlib fixes), @ice-charon (tonlib fixes), RSquad team (due payments improvements in v10), Arayz @ TonBit (improvements in RUNVM), @Skydev0h and @pyAndr3w (Asm.fif). -Besides the work of the core team, this update is based on the efforts of @Sild from StonFi(UB in tonlib). From 1835d84602bbaaa1593270d7ab3bb0b499920416 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 22 Apr 2025 20:14:07 +0300 Subject: [PATCH 222/388] Fixed setting gas limits in RUNVM (#1635) --- crypto/vm/vm.cpp | 4 ++++ doc/GlobalVersions.md | 1 + 2 files changed, 5 insertions(+) diff --git a/crypto/vm/vm.cpp b/crypto/vm/vm.cpp index 33d37973b..272e3930e 100644 --- a/crypto/vm/vm.cpp +++ b/crypto/vm/vm.cpp @@ -729,6 +729,10 @@ void VmState::run_child_vm(VmState&& new_state, bool return_data, bool return_ac new_state.chksgn_counter = chksgn_counter; new_state.free_gas_consumed = free_gas_consumed; new_state.get_extra_balance_counter = get_extra_balance_counter; + if (global_version >= 10) { + new_state.gas = GasLimits{std::min(new_state.gas.gas_limit, gas.gas_remaining), + std::min(new_state.gas.gas_max, gas.gas_remaining)}; + } auto new_parent = std::make_unique(); new_parent->return_data = return_data; diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index c3205f3c8..7bd9b0cc9 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -188,6 +188,7 @@ Reserve modes `+1`, `+4` and `+8` ("reserve all except", "add original balance" - With "isolate gas" mode, in the beginning of `RUNVM` the parent VM spends full gas for all already executed `GETEXTRABALANCE` and resets the counter. - `LDMSGADDR(Q)`, `PARSEMSGADDR(Q)`, `REWRITESTDADDR(Q)`, `REWRITEVARADDR(Q)` no more support anycast addresses and `addr_var`. - Fixed bug in `RUNVM` caused by throwing out-of-gas exception with "isolate gas" enabled. +- Fixed setting gas limits in `RUNVM` after consuming "free gas" (e.g. after `CHKSIGN` instructions). ### Other changes - Exceeding state limits in transaction now reverts `end_lt` back to `start_lt + 1` and collects action fines. From e03b11025e74b642a9aa7c46594f3799dcf3951f Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 22 Apr 2025 20:14:57 +0300 Subject: [PATCH 223/388] Don't delete accounts with remaining balance (#1632) --- crypto/block/transaction.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index c9890203f..b892b5a21 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -3295,6 +3295,9 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { acc_status = Account::acc_nonexist; was_created = false; } + if (acc_status == Account::acc_deleted && !balance.is_zero()) { + acc_status = Account::acc_uninit; + } if (acc_status == Account::acc_nonexist || acc_status == Account::acc_deleted) { CHECK(balance.is_zero()); vm::CellBuilder cb; From 0cc297ba58002a3312f72d787908e0a359e41f60 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 23 Apr 2025 21:47:10 +0300 Subject: [PATCH 224/388] Disable validator set cache during Merkle proof generation (#1636) --- crypto/block/mc-config.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index d51f20599..78e2294f8 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -542,6 +542,11 @@ td::Result> Config::unpack_validator_set(Refload_cell()); + if (!loaded_root.tree_node.empty()) { + // Do not use cache during Merkle proof generation + use_cache = false; + } static ValidatorSetCache cache; if (use_cache) { auto result = cache.get(vset_root->get_hash()); From 413b89837a3ae3a517d2de9372e298f6efa44818 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Mon, 28 Apr 2025 10:15:18 +0400 Subject: [PATCH 225/388] Deprecate Ubuntu 20.04 (#1641) * not always gh mac runners have nproc utility * deprecate Ubuntu 20.04 * fix linking issues of portable tonlibjson.dylib * fix libemulator.dylib linking issues --- .../build-ton-linux-arm64-appimage.yml | 14 ++++++------- .../build-ton-linux-x86-64-appimage.yml | 20 +++++++------------ .../build-ton-linux-x86-64-shared.yml | 9 +-------- assembly/native/build-macos-portable.sh | 1 - assembly/native/build-macos-shared.sh | 1 - emulator/CMakeLists.txt | 11 ++++++++++ tonlib/CMakeLists.txt | 11 ++++++++++ 7 files changed, 37 insertions(+), 30 deletions(-) diff --git a/.github/workflows/build-ton-linux-arm64-appimage.yml b/.github/workflows/build-ton-linux-arm64-appimage.yml index 289cd2a7e..a0ba4a022 100644 --- a/.github/workflows/build-ton-linux-arm64-appimage.yml +++ b/.github/workflows/build-ton-linux-arm64-appimage.yml @@ -36,21 +36,21 @@ jobs: uses: actions/cache@v4 with: path: 3pp - key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-3pp-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-arm-3pp-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} - name: Cache OpenSSL id: cache-openssl uses: actions/cache@v4 with: path: openssl_3 - key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-openssl_3-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-arm-openssl_3-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} - name: Restore cache TON uses: actions/cache/restore@v4 with: path: ~/.ccache - key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} - restore-keys: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-portable-ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-arm-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-arm-portable-ccache - name: Build TON run: | @@ -65,7 +65,7 @@ jobs: uses: actions/cache/save@v4 with: path: ~/.ccache - key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-arm-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} - name: Make AppImages run: | @@ -80,8 +80,8 @@ jobs: uses: actions/cache@v4 with: path: ~/.ccache - key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-portable-libs-ccache-${{ steps.date-stamp.outputs.timestamp }} - restore-keys: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-arm-portable-libs-ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-arm-portable-libs-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-arm-portable-libs-ccache - name: Build TON libs run: | diff --git a/.github/workflows/build-ton-linux-x86-64-appimage.yml b/.github/workflows/build-ton-linux-x86-64-appimage.yml index 0f5645a3a..43d989243 100644 --- a/.github/workflows/build-ton-linux-x86-64-appimage.yml +++ b/.github/workflows/build-ton-linux-x86-64-appimage.yml @@ -4,7 +4,7 @@ on: [push,workflow_dispatch,workflow_call] jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Check out repository @@ -25,12 +25,6 @@ jobs: sudo apt remove libgsl-dev mkdir ~/.ccache 3pp - - name: Install gcc-11 g++-11 - run: | - sudo apt install -y manpages-dev software-properties-common - sudo add-apt-repository ppa:ubuntu-toolchain-r/test - sudo apt update && sudo apt install gcc-11 g++-11 - - name: Install clang-16 run: | wget https://apt.llvm.org/llvm.sh @@ -42,20 +36,20 @@ jobs: uses: actions/cache@v4 with: path: 3pp - key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-3pp-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-3pp-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} - name: Cache OpenSSL id: cache-openssl uses: actions/cache@v4 with: path: openssl_3 - key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-openssl_3-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-openssl_3-${{ hashFiles('**/assembly/native/build-ubuntu-appimages.sh') }} - name: Restore cache TON uses: actions/cache/restore@v4 with: path: ~/.ccache - key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-portable-ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-portable-ccache - name: Build TON run: | @@ -70,7 +64,7 @@ jobs: uses: actions/cache/save@v4 with: path: ~/.ccache - key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-portable-ccache-${{ steps.date-stamp.outputs.timestamp }} - name: Make AppImages run: | @@ -85,8 +79,8 @@ jobs: uses: actions/cache@v4 with: path: ~/.ccache - key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-portable-libs-ccache-${{ steps.date-stamp.outputs.timestamp }} - restore-keys: ${{ runner.os }}-${{ runner.arch }}-ubuntu-20.04-portable-libs-ccache + key: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-portable-libs-ccache-${{ steps.date-stamp.outputs.timestamp }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-ubuntu-22.04-portable-libs-ccache - name: Build TON libs run: | diff --git a/.github/workflows/build-ton-linux-x86-64-shared.yml b/.github/workflows/build-ton-linux-x86-64-shared.yml index 37eddf369..ed05d18aa 100644 --- a/.github/workflows/build-ton-linux-x86-64-shared.yml +++ b/.github/workflows/build-ton-linux-x86-64-shared.yml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04] + os: [ubuntu-22.04, ubuntu-24.04] runs-on: ${{ matrix.os }} steps: @@ -28,13 +28,6 @@ jobs: sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev ccache mkdir ~/.ccache - - if: matrix.os == 'ubuntu-20.04' - name: Install gcc-11 - run: | - sudo apt install -y manpages-dev software-properties-common - sudo add-apt-repository ppa:ubuntu-toolchain-r/test - sudo apt update && sudo apt install gcc-11 g++-11 - - if: matrix.os != 'ubuntu-24.04' run: | wget https://apt.llvm.org/llvm.sh diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index 277973088..c46ea416e 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -127,7 +127,6 @@ fi cmake -GNinja .. \ -DPORTABLE=1 \ -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=$OSX_TARGET \ --DCMAKE_CXX_FLAGS="-stdlib=libc++" \ -DCMAKE_BUILD_TYPE=Release \ -DOPENSSL_FOUND=1 \ -DOPENSSL_INCLUDE_DIR=$opensslPath/include \ diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh index 595c44e3e..a32669490 100644 --- a/assembly/native/build-macos-shared.sh +++ b/assembly/native/build-macos-shared.sh @@ -66,7 +66,6 @@ brew install openssl@3 brew unlink openssl@3 && brew link --overwrite openssl@3 cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \ --DCMAKE_CXX_FLAGS="-stdlib=libc++" \ -DLZ4_FOUND=1 \ -DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a \ -DLZ4_INCLUDE_DIRS=$lz4Path/lib diff --git a/emulator/CMakeLists.txt b/emulator/CMakeLists.txt index e1bc38f52..63cb28d6a 100644 --- a/emulator/CMakeLists.txt +++ b/emulator/CMakeLists.txt @@ -30,6 +30,17 @@ else() target_link_libraries(emulator PUBLIC emulator_static git) endif() +if (APPLE) + set(CMAKE_MACOSX_RPATH ON) + set(CMAKE_CXX_VISIBILITY_PRESET hidden) + set(CMAKE_VISIBILITY_INLINES_HIDDEN ON) +endif() + +if (APPLE AND PORTABLE) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libstdc++") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static-libstdc++") +endif() + generate_export_header(emulator EXPORT_FILE_NAME ${CMAKE_CURRENT_BINARY_DIR}/emulator_export.h) if (EMULATOR_STATIC OR USE_EMSCRIPTEN) target_compile_definitions(emulator PUBLIC EMULATOR_STATIC_DEFINE) diff --git a/tonlib/CMakeLists.txt b/tonlib/CMakeLists.txt index 2e3c583d4..f86364411 100644 --- a/tonlib/CMakeLists.txt +++ b/tonlib/CMakeLists.txt @@ -102,6 +102,17 @@ else() target_link_libraries(tonlibjson PRIVATE tonlibjson_private) endif() +if (APPLE) + set(CMAKE_MACOSX_RPATH ON) + set(CMAKE_CXX_VISIBILITY_PRESET hidden) + set(CMAKE_VISIBILITY_INLINES_HIDDEN ON) +endif() + +if (APPLE AND PORTABLE) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libstdc++") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static-libstdc++") +endif() + generate_export_header(tonlibjson EXPORT_FILE_NAME ${CMAKE_CURRENT_BINARY_DIR}/tonlib/tonlibjson_export.h) if (TONLIBJSON_STATIC OR USE_EMSCRIPTEN) target_compile_definitions(tonlibjson PUBLIC TONLIBJSON_STATIC_DEFINE) From 6a1d8832154be8da8cbbd6aba8d742dab03c0d46 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 28 Apr 2025 18:24:08 +0300 Subject: [PATCH 226/388] Shard block verifier for masterchain validators --- tl/generate/scheme/ton_api.tl | 13 ++ tl/generate/scheme/ton_api.tlo | Bin 112248 -> 113468 bytes .../validator-engine-console-query.cpp | 74 +++++++ .../validator-engine-console-query.h | 60 ++++++ .../validator-engine-console.cpp | 3 + validator-engine/validator-engine.cpp | 100 +++++++++ validator-engine/validator-engine.hpp | 13 ++ validator/CMakeLists.txt | 4 + validator/impl/shard.cpp | 4 +- validator/impl/shard.hpp | 2 +- validator/impl/top-shard-descr.hpp | 3 + validator/impl/validate-query.cpp | 46 +++- validator/impl/validate-query.hpp | 3 +- validator/interfaces/shard-block.h | 1 + validator/interfaces/shard.h | 2 +- validator/interfaces/validator-manager.h | 4 + validator/manager.cpp | 199 ++++++++++++------ validator/manager.hpp | 23 +- validator/shard-block-retainer.cpp | 179 ++++++++++++++++ validator/shard-block-retainer.hpp | 71 +++++++ validator/shard-block-verifier.cpp | 194 +++++++++++++++++ validator/shard-block-verifier.hpp | 90 ++++++++ validator/validator-options.cpp | 28 +++ validator/validator-options.hpp | 7 + validator/validator.h | 17 ++ 25 files changed, 1065 insertions(+), 75 deletions(-) create mode 100644 validator/shard-block-retainer.cpp create mode 100644 validator/shard-block-retainer.hpp create mode 100644 validator/shard-block-verifier.cpp create mode 100644 validator/shard-block-verifier.hpp diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index ac0241a1f..c5bc4afe3 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -696,6 +696,11 @@ engine.validator.collatorsList.shard shard_id:tonNode.shardId collators:(vector self_collate:Bool select_mode:string = engine.validator.collatorsList.Shard; engine.validator.collatorsList shards:(vector engine.validator.collatorsList.shard) = engine.validator.CollatorsList; +engine.validator.shardBlockVerifierConfig.shard shard_id:(tonNode.shardId) trusted_nodes:(vector int256) + required_confirms:int = engine.validator.shardBlockVerifierConfig.Shard; +engine.validator.shardBlockVerifierConfig shards:(vector engine.validator.shardBlockVerifierConfig.shard) + = engine.validator.ShardBlockVerifierConfig; + ---functions--- ---types--- @@ -855,6 +860,9 @@ engine.validator.importFastSyncMemberCertificate adnl_id:int256 certificate:over engine.validator.addFastSyncClient adnl_id:int256 slot:int = engine.validator.Success; engine.validator.delFastSyncClient adnl_id:int256 = engine.validator.Success; +engine.validator.setShardBlockVerifierConfig config:engine.validator.shardBlockVerifierConfig = engine.validator.Success; +engine.validator.showShardBlockVerifierConfig = engine.validator.ShardBlockVerifierConfig; + ---types--- storage.pong = storage.Pong; @@ -962,11 +970,16 @@ collatorNode.compressedCandidate flags:# source:PublicKey id:tonNode.blockIdExt collatorNode.pong flags:# = collatorNode.Pong; collatorNode.error code:int message:string = collatorNode.Error; +shardBlockVerifier.subscribed flags:# = shardBlockVerifier.Subscribed; +shardBlockVerifier.confirmBlocks blocks:(vector tonNode.blockIdExt) = shardBlockVerifier.ConfirmBlocks; + ---functions--- collatorNode.generateBlock shard:tonNode.shardId cc_seqno:int prev_blocks:(vector tonNode.blockIdExt) creator:int256 round:int first_block_round:int priority:int = collatorNode.Candidate; collatorNode.ping flags:# = collatorNode.Pong; +shardBlockVerifier.subscribe shard:tonNode.shardId flags:# = shardBlockVerifier.Subscribed; + ---types--- storage.db.key.torrentList = storage.db.key.TorrentList; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 6708e8f0a610115e169ad65c345c7c9c301102fe..744323b6de215969871ddd09ac1cfb41e3619ed6 100644 GIT binary patch delta 801 zcmezIhHcL`Hr_|G^{p77z;Pq5xSU?IX1}XuYF>I~UaDSMVoqjCVo83HUT{WYQHoPe zesXqLYEfodW@?dheqLH;`owion^olgFqs79oLZ%ip{*E)Hi&r>H%M$&P~5?;dP8|p zz5=FNJ+K``xnO&XCo6JGEArlK;+MsuGPpFUIJqb@DRnZVrS#?-8X4k@=Qle#u(z9F z_YbBK5dSbRfYeVuv)(`g2TP#9Rx{kwmH6o-IhfEg6#w8xp_>;hk3H$ zIc-kflA_Y$lGK#=$%(I&Cp#?XnOqRYA}CmtT3DJ{1QbpN1#nUB>tMky*tye)o%lz;)CTew$OxwCj#Z4xlP{cM Tnf_-oqXyW&+e0QXs;B?}7&0kf delta 87 zcmdn5LL80JNMT8UO$Q diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 711d8968a..eaece0e87 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1946,3 +1946,77 @@ td::Status DelFastSyncOverlayClientQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << "success\n"; return td::Status::OK(); } + +td::Status SetShardBlockVerifierConfigQuery::run() { + TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status SetShardBlockVerifierConfigQuery::send() { + TRY_RESULT(data, td::read_file(file_name_)); + TRY_RESULT(json, td::json_decode(data.as_slice())); + auto list = ton::create_tl_object(); + TRY_STATUS(ton::ton_api::from_json(*list, json.get_object())); + auto b = ton::create_serialize_tl_object(std::move(list)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status SetShardBlockVerifierConfigQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status ClearShardBlockVerifierConfigQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status ClearShardBlockVerifierConfigQuery::send() { + auto list = ton::create_tl_object(); + auto b = ton::create_serialize_tl_object(std::move(list)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ClearShardBlockVerifierConfigQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status ShowShardBlockVerifierConfigQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status ShowShardBlockVerifierConfigQuery::send() { + auto b = ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ShowShardBlockVerifierConfigQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX( + config, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "Shard block verifier config:\n"; + if (config->shards_.empty()) { + td::TerminalIO::out() << "Config is empty\n"; + return td::Status::OK(); + } + for (const auto &shard : config->shards_) { + td::TerminalIO::out() << "Shard " << create_shard_id(shard->shard_id_).to_str() << "\n"; + td::TerminalIO::out() << " Required confirms = " << shard->required_confirms_ << "/" + << shard->trusted_nodes_.size() << "\n"; + td::TerminalIO::out() << " Trusted nodes:\n"; + for (const auto &node : shard->trusted_nodes_) { + td::TerminalIO::out() << " " << node << "\n"; + } + } + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index ad9f7c3b3..c06924d9a 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1747,3 +1747,63 @@ class DelFastSyncOverlayClientQuery : public Query { private: td::Bits256 adnl_id_; }; + +class SetShardBlockVerifierConfigQuery : public Query { + public: + SetShardBlockVerifierConfigQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "set-shard-block-verifier-config"; + } + static std::string get_help() { + return "set-shard-block-verifier-config \tset config for shard block verifier from file "; + } + std::string name() const override { + return get_name(); + } + + private: + std::string file_name_; +}; + +class ClearShardBlockVerifierConfigQuery : public Query { + public: + ClearShardBlockVerifierConfigQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "clear-shard-block-verifier-config"; + } + static std::string get_help() { + return "clear-shard-block-verifier-config \treset config for shard block verifier"; + } + std::string name() const override { + return get_name(); + } +}; + +class ShowShardBlockVerifierConfigQuery : public Query { + public: + ShowShardBlockVerifierConfigQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "show-shard-block-verifier-config"; + } + static std::string get_help() { + return "show-shard-block-verifier-config\tshow config of shard block verifier"; + } + std::string name() const override { + return get_name(); + } +}; diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 9520dd111..a747a648f 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -170,6 +170,9 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index ea137a94b..223ba7864 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1708,6 +1708,36 @@ void ValidatorEngine::load_collators_list() { } } +void ValidatorEngine::load_shard_block_verifier_config() { + shard_block_verifier_config_ = {}; + auto data_R = td::read_file(shard_block_verifier_config_file()); + if (data_R.is_error()) { + return; + } + auto data = data_R.move_as_ok(); + auto json_R = td::json_decode(data.as_slice()); + if (json_R.is_error()) { + LOG(ERROR) << "Failed to parse shard block verifier config: " << json_R.move_as_error(); + return; + } + auto json = json_R.move_as_ok(); + shard_block_verifier_config_ = ton::create_tl_object(); + auto S = ton::ton_api::from_json(*shard_block_verifier_config_, json.get_object()); + if (S.is_error()) { + LOG(ERROR) << "Failed to parse shard block verifier config: " << S; + shard_block_verifier_config_ = {}; + return; + } + td::Ref config{true}; + S = config.write().unpack(*shard_block_verifier_config_); + if (S.is_ok()) { + validator_options_.write().set_shard_block_verifier_config(std::move(config)); + } else { + LOG(ERROR) << "Invalid shard block verifier config: " << S.move_as_error(); + shard_block_verifier_config_ = {}; + } +} + void ValidatorEngine::load_empty_local_config(td::Promise promise) { auto ret_promise = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { @@ -2018,6 +2048,7 @@ void ValidatorEngine::got_key(ton::PublicKey key) { void ValidatorEngine::start() { set_shard_check_function(); load_collators_list(); + load_shard_block_verifier_config(); read_config_ = true; start_adnl(); } @@ -2153,6 +2184,15 @@ void ValidatorEngine::start_validator() { } } + if (!shard_block_retainer_adnl_id_.is_zero()) { + if (config_.adnl_ids.contains(shard_block_retainer_adnl_id_.pubkey_hash())) { + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_shard_block_retainer, + shard_block_retainer_adnl_id_); + } else { + LOG(ERROR) << "Cannot start shard block retainer: no adnl id " << shard_block_retainer_adnl_id_; + } + } + started_validator(); } @@ -4974,6 +5014,55 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delFastSy }); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setShardBlockVerifierConfig &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + td::Ref config{true}; + auto S = config.write().unpack(*query.config_); + if (S.is_error()) { + promise.set_value(create_control_query_error(S.move_as_error_prefix("Invalid config: "))); + return; + } + auto s = td::json_encode(td::ToJson(*query.config_), true); + S = td::write_file(shard_block_verifier_config_file(), s); + if (S.is_error()) { + promise.set_value(create_control_query_error(std::move(S))); + return; + } + shard_block_verifier_config_ = std::move(query.config_); + validator_options_.write().set_shard_block_verifier_config(std::move(config)); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showShardBlockVerifierConfig &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + if (shard_block_verifier_config_) { + promise.set_value(ton::serialize_tl_object(shard_block_verifier_config_, true)); + } else { + promise.set_value(create_control_query_error(td::Status::Error("shard block verifier config is empty"))); + } +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { @@ -5461,6 +5550,17 @@ int main(int argc, char *argv[]) { [&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_broadcast_speed_multiplier_private, v); }); return td::Status::OK(); }); + p.add_checked_option( + '\0', "shard-block-retainer", "adnl id for shard block retainer (hex)", [&](td::Slice arg) -> td::Status { + td::Bits256 v; + if (v.from_hex(arg) != 256) { + return td::Status::Error("invalid adnl-id"); + } + acts.push_back([&x, v]() { + td::actor::send_closure(x, &ValidatorEngine::set_shard_block_retainer_adnl_id, ton::adnl::AdnlNodeIdShort{v}); + }); + return td::Status::OK(); + }); auto S = p.run(argc, argv); if (S.is_error()) { LOG(ERROR) << "failed to parse options: " << S.move_as_error(); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index b5155052c..1893cf517 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -193,6 +193,7 @@ class ValidatorEngine : public td::actor::Actor { Config config_; ton::tl_object_ptr custom_overlays_config_; ton::tl_object_ptr collators_list_; + ton::tl_object_ptr shard_block_verifier_config_; std::set running_gc_; @@ -254,6 +255,7 @@ class ValidatorEngine : public td::actor::Actor { double broadcast_speed_multiplier_catchain_ = 3.33; double broadcast_speed_multiplier_public_ = 3.33; double broadcast_speed_multiplier_private_ = 3.33; + ton::adnl::AdnlNodeIdShort shard_block_retainer_adnl_id_ = ton::adnl::AdnlNodeIdShort::zero(); std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; @@ -369,6 +371,9 @@ class ValidatorEngine : public td::actor::Actor { void set_broadcast_speed_multiplier_private(double value) { broadcast_speed_multiplier_private_ = value; } + void set_shard_block_retainer_adnl_id(ton::adnl::AdnlNodeIdShort id) { + shard_block_retainer_adnl_id_ = id; + } void start_up() override; ValidatorEngine() { @@ -381,6 +386,7 @@ class ValidatorEngine : public td::actor::Actor { void load_config(td::Promise promise); void set_shard_check_function(); void load_collators_list(); + void load_shard_block_verifier_config(); void start(); @@ -479,6 +485,9 @@ class ValidatorEngine : public td::actor::Actor { std::string collators_list_file() const { return db_root_ + "/collators-list.json"; } + std::string shard_block_verifier_config_file() const { + return db_root_ + "/shard-block-verifier-config.json"; + } void load_custom_overlays_config(); td::Status write_custom_overlays_config(); @@ -614,6 +623,10 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_delFastSyncClient &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_setShardBlockVerifierConfig &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_showShardBlockVerifierConfig &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index 003d9b1b0..753ae9d29 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -56,6 +56,8 @@ set(VALIDATOR_HEADERS import-db-slice.hpp queue-size-counter.hpp validator-telemetry.hpp + shard-block-verifier.hpp + shard-block-retainer.hpp collation-manager.hpp collator-node.hpp @@ -87,6 +89,8 @@ set(VALIDATOR_SOURCE validator-options.cpp queue-size-counter.cpp validator-telemetry.cpp + shard-block-verifier.cpp + shard-block-retainer.cpp downloaders/wait-block-data.cpp downloaders/wait-block-state.cpp diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index d2d402dd6..7302b2611 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -501,11 +501,11 @@ std::vector> MasterchainStateQ::get_shards() const { return v; } -td::Ref MasterchainStateQ::get_shard_from_config(ShardIdFull shard) const { +td::Ref MasterchainStateQ::get_shard_from_config(ShardIdFull shard, bool exact) const { if (!config_) { return {}; } - return config_->get_shard_hash(shard); + return config_->get_shard_hash(shard, exact); } bool MasterchainStateQ::rotated_all_shards() const { diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index b13be0219..1dd6804be 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -114,7 +114,7 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { Ref get_validator_set(ShardIdFull shard, UnixTime ts, CatchainSeqno cc_seqno) const; bool rotated_all_shards() const override; std::vector> get_shards() const override; - td::Ref get_shard_from_config(ShardIdFull shard) const override; + td::Ref get_shard_from_config(ShardIdFull shard, bool exact) const override; bool ancestor_is_valid(BlockIdExt id) const override { return check_old_mc_block_id(id); } diff --git a/validator/impl/top-shard-descr.hpp b/validator/impl/top-shard-descr.hpp index 5c866a6e3..e1fd5778f 100644 --- a/validator/impl/top-shard-descr.hpp +++ b/validator/impl/top-shard-descr.hpp @@ -74,6 +74,9 @@ class ShardTopBlockDescrQ final : public ShardTopBlockDescrQBase { std::size_t size() const { return chain_blk_ids_.size(); } + const std::vector& get_chain_blocks() const override { + return chain_blk_ids_; + } UnixTime generated_at() const override { return gen_utime_; } diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index aae718360..dffdf21bc 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -1801,11 +1801,13 @@ void ValidateQuery::after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result< * @param sibling The sibling shard information. * @param wc_info The workchain information. * @param ccvc The Catchain validators configuration. + * @param is_new Set to true if the top shard block is new, false if it existed in the previous shard configuration. * * @returns True if the validation wasa successful, false otherwise. */ bool ValidateQuery::check_one_shard(const block::McShardHash& info, const block::McShardHash* sibling, - const block::WorkchainInfo* wc_info, const block::CatchainValidatorsConfig& ccvc) { + const block::WorkchainInfo* wc_info, const block::CatchainValidatorsConfig& ccvc, + bool& is_new) { auto shard = info.shard(); LOG(DEBUG) << "checking shard " << shard.to_str() << " in new shard configuration"; if (info.next_validator_shard_ != shard.shard) { @@ -1814,6 +1816,7 @@ bool ValidateQuery::check_one_shard(const block::McShardHash& info, const block: ton::ShardIdFull{shard.workchain, info.next_validator_shard_}.to_str()); } auto old = old_shard_conf_->get_shard_hash(shard - 1, false); + is_new = (old.is_null() || old->top_block_id() != info.top_block_id()); Ref prev; CatchainSeqno cc_seqno; bool old_before_merge = false, fsm_inherited = false, workchain_created = false; @@ -2113,8 +2116,10 @@ bool ValidateQuery::check_shard_layout() { WorkchainId wc_id{ton::workchainInvalid}; Ref wc_info; - if (!new_shard_conf_->process_sibling_shard_hashes([self = this, &wc_set, &wc_id, &wc_info, &ccvc]( - block::McShardHash& cur, const block::McShardHash* sibling) { + std::vector new_top_shard_blocks; + if (!new_shard_conf_->process_sibling_shard_hashes([self = this, &new_top_shard_blocks, &wc_set, &wc_id, &wc_info, + &ccvc](block::McShardHash& cur, + const block::McShardHash* sibling) { if (!cur.is_valid()) { return -2; } @@ -2131,7 +2136,15 @@ bool ValidateQuery::check_shard_layout() { wc_info = it->second; } } - return self->check_one_shard(cur, sibling, wc_info.get(), ccvc) ? 0 : -1; + bool is_new; + int res = self->check_one_shard(cur, sibling, wc_info.get(), ccvc, is_new); + if (!res) { + return -1; + } + if (is_new) { + new_top_shard_blocks.push_back(cur.top_block_id()); + } + return 0; })) { return reject_query("new shard configuration is invalid"); } @@ -2148,6 +2161,14 @@ bool ValidateQuery::check_shard_layout() { << " is active, but is absent from new shard configuration"); } } + if (!new_top_shard_blocks.empty()) { + ++pending; + td::actor::send_closure(manager, &ValidatorManager::wait_verify_shard_blocks, std::move(new_top_shard_blocks), + [SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(SelfId, &ValidateQuery::verified_shard_blocks, + R.move_as_status()); + }); + } return check_mc_validator_info(is_key_block_ || (now_ / ccvc.mc_cc_lifetime > prev_now_ / ccvc.mc_cc_lifetime)); } @@ -2351,6 +2372,23 @@ void ValidateQuery::got_out_queue_size(size_t i, td::Result res) { try_validate(); } +/** + * Handles the result of ValidatorManager::wait_verify_shard_blocks. + * + * This is called after new top shard blocks were confirmed by trusted nodes. + * + * @param S The status of the operation (OK on success). + */ +void ValidateQuery::verified_shard_blocks(td::Status S) { + --pending; + if (S.is_error()) { + fatal_error(S.move_as_error_prefix("failed to verify shard blocks: ")); + return; + } + LOG(DEBUG) << "Verified shard blocks"; + try_validate(); +} + /* * * METHODS CALLED FROM try_validate() stage 1 diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index e42f88548..9c408d12a 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -323,7 +323,7 @@ class ValidateQuery : public td::actor::Actor { void after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res); bool check_one_shard(const block::McShardHash& info, const block::McShardHash* sibling, - const block::WorkchainInfo* wc_info, const block::CatchainValidatorsConfig& ccvc); + const block::WorkchainInfo* wc_info, const block::CatchainValidatorsConfig& ccvc, bool& is_new); bool check_shard_layout(); bool register_shard_block_creators(std::vector creator_list); bool check_cur_validator_set(); @@ -331,6 +331,7 @@ class ValidateQuery : public td::actor::Actor { bool check_utime_lt(); bool prepare_out_msg_queue_size(); void got_out_queue_size(size_t i, td::Result res); + void verified_shard_blocks(td::Status S); bool fix_one_processed_upto(block::MsgProcessedUpto& proc, ton::ShardIdFull owner, bool allow_cur = false); bool fix_processed_upto(block::MsgProcessedUptoCollection& upto, bool allow_cur = false); diff --git a/validator/interfaces/shard-block.h b/validator/interfaces/shard-block.h index e88a970bb..6e31cc136 100644 --- a/validator/interfaces/shard-block.h +++ b/validator/interfaces/shard-block.h @@ -35,6 +35,7 @@ class ShardTopBlockDescription : public td::CntObject { virtual bool after_split() const = 0; virtual bool after_merge() const = 0; virtual CatchainSeqno catchain_seqno() const = 0; + virtual const std::vector& get_chain_blocks() const = 0; virtual UnixTime generated_at() const = 0; // if method returns false this shard block description is discarded diff --git a/validator/interfaces/shard.h b/validator/interfaces/shard.h index 3546f0a39..dff077f4c 100644 --- a/validator/interfaces/shard.h +++ b/validator/interfaces/shard.h @@ -69,7 +69,7 @@ class MasterchainState : virtual public ShardState { virtual td::Ref get_total_validator_set(int next) const = 0; // next = -1 -> prev, next = 0 -> cur virtual bool rotated_all_shards() const = 0; virtual std::vector> get_shards() const = 0; - virtual td::Ref get_shard_from_config(ShardIdFull shard) const = 0; + virtual td::Ref get_shard_from_config(ShardIdFull shard, bool exact = true) const = 0; virtual bool workchain_is_active(WorkchainId workchain_id) const = 0; virtual td::uint32 monitor_min_split_depth(WorkchainId workchain_id) const = 0; virtual td::uint32 min_split_depth(WorkchainId workchain_id) const = 0; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 65f6f48a8..140f01ff2 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -270,6 +270,10 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void add_persistent_state_description(td::Ref desc) = 0; + virtual void wait_verify_shard_blocks(std::vector blocks, td::Promise promise) { + promise.set_result(td::Unit()); + } + static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) { return ts / (1 << 17) != prev_ts / (1 << 17); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 535f638ce..f68cdf941 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -42,6 +42,7 @@ #include "td/utils/JsonBuilder.h" #include "common/delay.h" +#include "td/actor/MultiPromise.h" #include "td/utils/filesystem.h" #include "validator/stats-merger.h" @@ -312,8 +313,7 @@ void ValidatorManagerImpl::get_zero_state(BlockIdExt block_id, td::Promise promise) { - td::actor::send_closure(db_, &Db::get_persistent_state_file_size, block_id, masterchain_block_id, - std::move(promise)); + td::actor::send_closure(db_, &Db::get_persistent_state_file_size, block_id, masterchain_block_id, std::move(promise)); } void ValidatorManagerImpl::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) { @@ -495,7 +495,8 @@ void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { } void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { - if (!is_validator() && !is_shard_collator(block_id.shard_full()) && !cached_block_candidates_.count(block_id)) { + if (!is_validator() && !is_shard_collator(block_id.shard_full()) && !cached_block_candidates_.count(block_id) && + shard_block_retainers_.empty()) { return; } if (!last_masterchain_block_handle_) { @@ -544,6 +545,9 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refmay_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { return; } + for (auto &[_, actor] : shard_block_retainers_) { + td::actor::send_closure(actor, &ShardBlockRetainer::new_shard_block_description, desc); + } auto it = shard_blocks_.find(ShardTopBlockDescriptionId{desc->shard(), desc->catchain_seqno()}); if (it != shard_blocks_.end() && desc->block_id().id.seqno <= it->second.latest_desc->block_id().id.seqno) { VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block broadcast"; @@ -565,9 +569,22 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refblock_id(), 0, td::Timestamp::in(60.0), std::move(P)); } if (validating_masterchain()) { - preload_msg_queue_to_masterchain(desc); + td::MultiPromise mp; + auto ig = mp.init_guard(); + preload_msg_queue_to_masterchain(desc, ig.get_promise()); + wait_verify_shard_blocks({desc->block_id()}, ig.get_promise().wrap([desc](td::Unit) { + VLOG(VALIDATOR_DEBUG) << "verified top shard block " << desc->block_id().to_str(); + return td::Unit{}; + })); + ig.add_promise([SelfId = actor_id(this), desc](td::Result R) mutable { + if (R.is_error()) { + VLOG(VALIDATOR_DEBUG) << "shard block description: " << R.move_as_error(); + return; + } + td::actor::send_closure(SelfId, &ValidatorManagerImpl::set_shard_block_description_ready, std::move(desc)); + }); } - for (auto& [_, collator_node] : collator_nodes_) { + for (auto &[_, collator_node] : collator_nodes_) { if (collator_node.can_collate_shard(desc->shard())) { td::actor::send_closure(collator_node.actor, &CollatorNode::update_validator_group_info, desc->shard(), std::vector{desc->block_id()}, desc->catchain_seqno()); @@ -575,32 +592,48 @@ void ValidatorManagerImpl::add_shard_block_description(td::Ref desc) { +void ValidatorManagerImpl::preload_msg_queue_to_masterchain(td::Ref desc, + td::Promise promise) { + if (!validating_masterchain()) { + promise.set_error(td::Status::Error("not validating masterchain")); + return; + } auto id = ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}; auto it = shard_blocks_.find(id); - if (!validating_masterchain() || it == shard_blocks_.end() || - it->second.latest_desc->block_id() != desc->block_id()) { + if (it == shard_blocks_.end() || it->second.latest_desc->block_id() != desc->block_id()) { + promise.set_error(td::Status::Error("shard block description is outdated")); return; } wait_neighbor_msg_queue_proofs( ShardIdFull{masterchainId}, {desc->block_id()}, td::Timestamp::in(10.0), - [=, SelfId = actor_id(this), - retry_at = td::Timestamp::in(1.0)](td::Result>> R) { + [=, SelfId = actor_id(this), retry_at = td::Timestamp::in(1.0), + promise = std::move(promise)](td::Result>> R) mutable { if (R.is_error()) { delay_action( - [=]() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::preload_msg_queue_to_masterchain, desc); }, + [=, promise = std::move(promise)]() mutable { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::preload_msg_queue_to_masterchain, desc, + std::move(promise)); + }, retry_at); return; } auto res = R.move_as_ok(); auto &queue = res[desc->block_id()]; CHECK(queue.not_null()); - td::actor::send_closure(SelfId, &ValidatorManagerImpl::loaded_msg_queue_to_masterchain, desc, std::move(queue)); + td::actor::send_closure(SelfId, &ValidatorManagerImpl::loaded_msg_queue_to_masterchain, desc, std::move(queue), + std::move(promise)); }); } void ValidatorManagerImpl::loaded_msg_queue_to_masterchain(td::Ref desc, - td::Ref res) { + td::Ref res, + td::Promise promise) { + VLOG(VALIDATOR_DEBUG) << "loaded out msg queue to masterchain from " << desc->block_id().to_str(); + cached_msg_queue_to_masterchain_[desc->block_id()] = std::move(res); + promise.set_value(td::Unit()); +} + +void ValidatorManagerImpl::set_shard_block_description_ready(td::Ref desc) { auto id = ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}; auto it = shard_blocks_.find(id); if (it == shard_blocks_.end()) { @@ -608,12 +641,8 @@ void ValidatorManagerImpl::loaded_msg_queue_to_masterchain(td::Refsecond; if (info.ready_desc.is_null() || info.ready_desc->block_id().seqno() < desc->block_id().seqno()) { - VLOG(VALIDATOR_DEBUG) << "loaded out msg queue to masterchain from " << desc->block_id(); - if (info.ready_desc.not_null()) { - cached_msg_queue_to_masterchain_.erase(info.ready_desc->block_id()); - } + VLOG(VALIDATOR_DEBUG) << "top shard block description is ready: " << desc->block_id().to_str(); info.ready_desc = desc; - cached_msg_queue_to_masterchain_[desc->block_id()] = std::move(res); } } @@ -895,8 +924,7 @@ void ValidatorManagerImpl::wait_block_data_short(BlockIdExt block_id, td::uint32 void ValidatorManagerImpl::wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) { if (last_masterchain_state_.not_null() && !opts_->need_monitor(left_id.shard_full(), last_masterchain_state_)) { - return promise.set_error( - td::Status::Error(PSTRING() << "not monitoring shard " << left_id.shard_full().to_str())); + return promise.set_error(td::Status::Error(PSTRING() << "not monitoring shard " << left_id.shard_full().to_str())); } td::actor::create_actor("merge", left_id, right_id, priority, actor_id(this), timeout, std::move(promise)) @@ -1356,7 +1384,7 @@ void ValidatorManagerImpl::store_persistent_state_file(BlockIdExt block_id, Bloc } void ValidatorManagerImpl::store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_data, + std::function write_data, td::Promise promise) { td::actor::send_closure(db_, &Db::store_persistent_state_file_gen, block_id, masterchain_block_id, std::move(write_data), std::move(promise)); @@ -2142,6 +2170,13 @@ void ValidatorManagerImpl::new_masterchain_block() { for (auto &c : collator_nodes_) { td::actor::send_closure(c.second.actor, &CollatorNode::new_masterchain_block_notification, last_masterchain_state_); } + if (!shard_block_verifier_.empty()) { + td::actor::send_closure(shard_block_verifier_, &ShardBlockVerifier::update_masterchain_state, + last_masterchain_state_); + } + for (auto &[_, actor] : shard_block_retainers_) { + td::actor::send_closure(actor, &ShardBlockRetainer::update_masterchain_state, last_masterchain_state_); + } if (last_masterchain_seqno_ % 1024 == 0) { LOG(WARNING) << "applied masterchain block " << last_masterchain_block_id_; } @@ -2152,9 +2187,9 @@ void ValidatorManagerImpl::update_shard_overlays() { std::set shards_to_monitor; shards_to_monitor.insert(ShardIdFull{masterchainId}); std::set workchains; - for (const auto& shard : last_masterchain_state_->get_shards()) { + for (const auto &shard : last_masterchain_state_->get_shards()) { workchains.insert(shard->shard().workchain); - if (opts_->need_monitor(shard->shard(),last_masterchain_state_)) { + if (opts_->need_monitor(shard->shard(), last_masterchain_state_)) { shards_to_monitor.insert(shard->shard()); } } @@ -2178,7 +2213,7 @@ void ValidatorManagerImpl::update_shards() { td::uint32 threshold = 9407194; bool force_group_id_upgrade = last_masterchain_seqno_ == threshold; auto legacy_opts_hash = opts.get_hash(); - if (last_masterchain_seqno_ >= threshold) { //TODO move to get_consensus_config() + if (last_masterchain_seqno_ >= threshold) { //TODO move to get_consensus_config() opts.proto_version = std::max(opts.proto_version, 1); } auto opts_hash = opts.get_hash(); @@ -2269,7 +2304,6 @@ void ValidatorManagerImpl::update_shards() { auto legacy_val_group_id = get_validator_set_id(shard, val_set, legacy_opts_hash, key_seqno, opts); auto val_group_id = get_validator_set_id(shard, val_set, opts_hash, key_seqno, opts); - auto it = validator_groups_.find(legacy_val_group_id); if (it != validator_groups_.end()) { new_validator_groups_.emplace(val_group_id, std::move(it->second)); @@ -2293,6 +2327,7 @@ void ValidatorManagerImpl::update_shards() { } active_validator_groups_master_ = active_validator_groups_shard_ = 0; + adnl::AdnlNodeIdShort mc_validator_adnl_id = adnl::AdnlNodeIdShort::zero(); if (allow_validate_) { for (auto &desc : new_shards) { auto shard = desc.first; @@ -2344,6 +2379,12 @@ void ValidatorManagerImpl::update_shards() { new_validator_groups_.emplace(val_group_id, ValidatorGroupEntry{std::move(G), shard}); } } + if (shard.is_masterchain()) { + mc_validator_adnl_id = adnl::AdnlNodeIdShort{val_set->get_validator(validator_id.bits256_value())->addr}; + if (mc_validator_adnl_id.is_zero()) { + mc_validator_adnl_id = adnl::AdnlNodeIdShort{validator_id.bits256_value()}; + } + } } } } @@ -2365,6 +2406,12 @@ void ValidatorManagerImpl::update_shards() { val_group_id, ValidatorGroupEntry{ create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_), shard}); } + if (shard.is_masterchain() && mc_validator_adnl_id.is_zero()) { + mc_validator_adnl_id = adnl::AdnlNodeIdShort{val_set->get_validator(validator_id.bits256_value())->addr}; + if (mc_validator_adnl_id.is_zero()) { + mc_validator_adnl_id = adnl::AdnlNodeIdShort{validator_id.bits256_value()}; + } + } } } @@ -2407,6 +2454,7 @@ void ValidatorManagerImpl::update_shards() { td::actor::send_closure(serializer_, &AsyncStateSerializer::auto_disable_serializer, is_validator() && last_masterchain_state_->get_global_id() == -239); // mainnet only } + init_shard_block_verifier(mc_validator_adnl_id); } void ValidatorManagerImpl::written_destroyed_validator_sessions(std::vector> list) { @@ -2416,36 +2464,36 @@ void ValidatorManagerImpl::written_destroyed_validator_sessions(std::vectorsecond; - if (!B.latest_desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { - if (B.ready_desc.not_null()) { - cached_msg_queue_to_masterchain_.erase(B.ready_desc->block_id()); - } - it = shard_blocks_.erase(it); - continue; - } - if (B.ready_desc.not_null() && - !B.ready_desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { - cached_msg_queue_to_masterchain_.erase(B.ready_desc->block_id()); - B.ready_desc = {}; - } - ++it; + for (auto it = shard_blocks_.begin(); it != shard_blocks_.end();) { + auto &B = it->second; + if (!B.latest_desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { + it = shard_blocks_.erase(it); + continue; } + if (B.ready_desc.not_null() && + !B.ready_desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { + B.ready_desc = {}; + } + ++it; } - { - auto it = out_shard_blocks_.begin(); - while (it != out_shard_blocks_.end()) { - auto &B = it->second; - if (!B->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { - auto it2 = it++; - out_shard_blocks_.erase(it2); - } else { - ++it; - } + for (auto it = cached_msg_queue_to_masterchain_.begin(); it != cached_msg_queue_to_masterchain_.end();) { + const BlockIdExt &block_id = it->first; + auto shard = last_masterchain_state_->get_shard_from_config(block_id.shard_full(), false); + if (shard.is_null() || shard->top_block_id().seqno() >= block_id.seqno()) { + it = cached_msg_queue_to_masterchain_.erase(it); + continue; + } + ++it; + } + + for (auto it = out_shard_blocks_.begin(); it != out_shard_blocks_.end();) { + auto &B = it->second; + if (!B->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { + auto it2 = it++; + out_shard_blocks_.erase(it2); + } else { + ++it; } } } @@ -2865,7 +2913,6 @@ bool ValidatorManagerImpl::Collator::can_collate_shard(ShardIdFull shard) const [&](const ShardIdFull &our_shard) { return shard_intersects(shard, our_shard); }); } - void ValidatorManagerImpl::got_next_key_blocks(std::vector r) { if (r.size() == 0) { delay_action([SelfId = actor_id( @@ -2931,9 +2978,8 @@ void ValidatorManagerImpl::prepare_stats(td::Promise R) mutable { @@ -3333,6 +3379,9 @@ void ValidatorManagerImpl::update_options(td::Ref opts) for (auto &[_, c] : collation_managers_) { td::actor::send_closure(c, &CollationManager::update_options, opts); } + if (!shard_block_verifier_.empty()) { + td::actor::send_closure(shard_block_verifier_, &ShardBlockVerifier::update_options, opts); + } opts_ = std::move(opts); init_session_stats(); } @@ -3563,8 +3612,8 @@ void ValidatorManagerImpl::init_validator_telemetry() { return; } std::set processed; - for (auto& key : temp_keys_) { - if (const ValidatorDescr* desc = validator_set->get_validator(key.bits256_value())) { + for (auto &key : temp_keys_) { + if (const ValidatorDescr *desc = validator_set->get_validator(key.bits256_value())) { processed.insert(key); adnl::AdnlNodeIdShort adnl_id; if (desc->addr.is_zero()) { @@ -3572,10 +3621,10 @@ void ValidatorManagerImpl::init_validator_telemetry() { } else { adnl_id = adnl::AdnlNodeIdShort{desc->addr}; } - auto& telemetry = validator_telemetry_[key]; + auto &telemetry = validator_telemetry_[key]; if (telemetry.empty()) { - telemetry = td::actor::create_actor( - "telemetry", key, adnl_id, opts_->zero_block_id().file_hash, actor_id(this)); + telemetry = td::actor::create_actor("telemetry", key, adnl_id, + opts_->zero_block_id().file_hash, actor_id(this)); } } } @@ -3610,7 +3659,7 @@ void ValidatorManagerImpl::init_session_stats() { session_stats_enabled_ = true; } -template +template void ValidatorManagerImpl::write_session_stats(const T &obj) { if (!session_stats_enabled_) { return; @@ -3635,6 +3684,32 @@ void ValidatorManagerImpl::write_session_stats(const T &obj) { } } +void ValidatorManagerImpl::init_shard_block_verifier(adnl::AdnlNodeIdShort local_id) { + if (local_id != shard_block_verifier_local_id_) { + shard_block_verifier_local_id_ = local_id; + if (local_id.is_zero()) { + shard_block_verifier_ = {}; + } else { + shard_block_verifier_ = td::actor::create_actor( + "shardblockverifier", local_id, last_masterchain_state_, opts_, actor_id(this), adnl_, rldp_); + } + } +} + +void ValidatorManagerImpl::wait_verify_shard_blocks(std::vector blocks, td::Promise promise) { + if (shard_block_verifier_.empty()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "shard block verifier not inited")); + return; + } + td::actor::send_closure(shard_block_verifier_, &ShardBlockVerifier::wait_shard_blocks, std::move(blocks), + std::move(promise)); +} + +void ValidatorManagerImpl::add_shard_block_retainer(adnl::AdnlNodeIdShort id) { + shard_block_retainers_[id] = td::actor::create_actor( + "shardblockretainer", id, last_masterchain_state_, opts_, actor_id(this), adnl_, rldp_); +} + } // namespace validator } // namespace ton diff --git a/validator/manager.hpp b/validator/manager.hpp index 20aaf04e5..fce2ceee4 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -37,6 +37,8 @@ #include "validator-telemetry.hpp" #include "impl/candidates-buffer.hpp" #include "collator-node.hpp" +#include "shard-block-verifier.hpp" +#include "shard-block-retainer.hpp" #include #include @@ -235,9 +237,10 @@ class ValidatorManagerImpl : public ValidatorManager { } }; // DATA FOR COLLATOR - // Shard block will not be used until queue is ready (to avoid too long masterchain collation) + // Shard block will not be used until it is confirmed by trusted nodes (see ShardBlockVerifier) and + // msg queue to masterchain is ready (to avoid too long masterchain collation) // latest_desc - latest known block - // ready_desc - block with ready msg queue (may be null) + // ready_desc - block ready to be used (may be null) struct ShardTopBlock { td::Ref latest_desc; td::Ref ready_desc; @@ -562,8 +565,10 @@ class ValidatorManagerImpl : public ValidatorManager { void add_shard_block_description(td::Ref desc); void add_cached_block_candidate(ReceivedBlock block); - void preload_msg_queue_to_masterchain(td::Ref desc); - void loaded_msg_queue_to_masterchain(td::Ref desc, td::Ref res); + void preload_msg_queue_to_masterchain(td::Ref desc, td::Promise promise); + void loaded_msg_queue_to_masterchain(td::Ref desc, td::Ref res, + td::Promise promise); + void set_shard_block_description_ready(td::Ref desc); void register_block_handle(BlockHandle handle); @@ -769,6 +774,10 @@ class ValidatorManagerImpl : public ValidatorManager { std::function>>)> callback) override; void unregister_stats_provider(td::uint64 idx) override; + void wait_verify_shard_blocks(std::vector blocks, td::Promise promise) override; + + void add_shard_block_retainer(adnl::AdnlNodeIdShort id) override; + std::map> validator_telemetry_; void init_validator_telemetry(); @@ -795,6 +804,12 @@ class ValidatorManagerImpl : public ValidatorManager { void init_session_stats(); template void write_session_stats(const T &obj); + + td::actor::ActorOwn shard_block_verifier_; + adnl::AdnlNodeIdShort shard_block_verifier_local_id_ = adnl::AdnlNodeIdShort::zero(); + std::map> shard_block_retainers_; + + void init_shard_block_verifier(adnl::AdnlNodeIdShort local_id); }; } // namespace validator diff --git a/validator/shard-block-retainer.cpp b/validator/shard-block-retainer.cpp new file mode 100644 index 000000000..4fb96502e --- /dev/null +++ b/validator/shard-block-retainer.cpp @@ -0,0 +1,179 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "shard-block-retainer.hpp" + +namespace ton::validator { + +void ShardBlockRetainer::start_up() { + if (last_masterchain_state_.not_null()) { + update_masterchain_state(last_masterchain_state_); + } + + class Callback : public adnl::Adnl::Callback { + public: + explicit Callback(td::actor::ActorId id) : id_(std::move(id)) { + } + void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { + } + void receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + td::actor::send_closure(id_, &ShardBlockRetainer::process_query, src, std::move(data), std::move(promise)); + } + + private: + td::actor::ActorId id_; + }; + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::shardBlockVerifier_subscribe::ID), + std::make_unique(actor_id(this))); + td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); +} + +void ShardBlockRetainer::tear_down() { + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::shardBlockVerifier_subscribe::ID)); +} + +void ShardBlockRetainer::update_masterchain_state(td::Ref state) { + last_masterchain_state_ = state; + for (auto it = confirmed_blocks_.begin(); it != confirmed_blocks_.end();) { + if (is_block_outdated(*it)) { + it = confirmed_blocks_.erase(it); + } else { + ++it; + } + } + if (last_masterchain_state_->is_key_state() || !inited_) { + validator_adnl_ids_.clear(); + for (int next = 0; next <= 1; ++next) { + auto vset = last_masterchain_state_->get_total_validator_set(next); + if (vset.is_null()) { + continue; + } + for (auto& val : vset->export_vector()) { + adnl::AdnlNodeIdShort adnl_id{val.addr}; + if (adnl_id.is_zero()) { + adnl_id = adnl::AdnlNodeIdShort{ValidatorFullId{val.key}.short_id()}; + validator_adnl_ids_.insert(adnl_id); + } + } + } + LOG(INFO) << "Updating validator set: " << validator_adnl_ids_.size() << " adnl ids"; + for (auto it = subscribers_.begin(); it != subscribers_.end();) { + if (it->second.is_in_past()) { + LOG(INFO) << "Unsubscribed " << it->first.first << " for " << it->first.second.to_str() << " (expired)"; + it = subscribers_.erase(it); + continue; + } + if (!validator_adnl_ids_.contains(it->first.first)) { + LOG(INFO) << "Unsubscribed " << it->first.first << " for " << it->first.second.to_str() << " (not a validator)"; + it = subscribers_.erase(it); + continue; + } + ++it; + } + } + inited_ = true; +} + +void ShardBlockRetainer::new_shard_block_description(td::Ref desc) { + if (last_masterchain_state_.is_null() || is_block_outdated(desc->block_id()) || + !opts_->need_monitor(desc->shard(), last_masterchain_state_)) { + return; + } + td::actor::send_closure( + manager_, &ValidatorManager::wait_block_state_short, desc->block_id(), 0, td::Timestamp::in(30.0), + [SelfId = actor_id(this), desc](td::Result> R) { + if (R.is_error()) { + LOG(WARNING) << "Wait block state for " << desc->block_id().to_str() << " : " << R.move_as_error(); + td::actor::send_closure(SelfId, &ShardBlockRetainer::new_shard_block_description, std::move(desc)); + return; + } + td::actor::send_closure(SelfId, &ShardBlockRetainer::confirm_shard_block_description, std::move(desc)); + }); +} + +void ShardBlockRetainer::process_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, + td::Promise promise) { + TRY_RESULT_PROMISE(promise, query, fetch_tl_object(data, true)) + ShardIdFull shard = create_shard_id(query->shard_); + if (!shard.is_valid_ext() || shard.is_masterchain()) { + promise.set_error(td::Status::Error(PSTRING() << "invalid shard " << shard.to_str())); + return; + } + if (!validator_adnl_ids_.contains(src)) { + promise.set_error(td::Status::Error(PSTRING() << "unauthorized src " << src)); + return; + } + td::Timestamp& ttl = subscribers_[{src, shard}]; + if (!ttl) { + std::vector blocks; + for (const BlockIdExt& block : confirmed_blocks_) { + if (shard_intersects(block.shard_full(), shard)) { + blocks.push_back(block); + } + } + LOG(INFO) << "New subscriber " << src << " for " << shard.to_str() << ", sending " << blocks.size() << " blocks"; + send_confirmations(src, std::move(blocks)); + } + ttl = td::Timestamp::in(SUBSCRIPTION_TTL); + promise.set_value(create_serialize_tl_object(0)); +} + +void ShardBlockRetainer::send_confirmations(adnl::AdnlNodeIdShort dst, std::vector blocks) { + for (size_t l = 0; l < blocks.size(); l += MAX_BLOCKS_PER_MESSAGE) { + size_t r = std::min(l + MAX_BLOCKS_PER_MESSAGE, blocks.size()); + auto query = create_tl_object(); + for (size_t i = l; i < r; ++i) { + query->blocks_.push_back(create_tl_block_id(blocks[i])); + } + td::actor::send_closure(rldp_, &rldp::Rldp::send_message, local_id_, dst, serialize_tl_object(query, true)); + } +} + +void ShardBlockRetainer::confirm_shard_block_description(td::Ref desc) { + for (const BlockIdExt& block_id : desc->get_chain_blocks()) { + confirm_block(block_id); + } +} + +void ShardBlockRetainer::confirm_block(BlockIdExt block_id) { + if (is_block_outdated(block_id) || !confirmed_blocks_.insert(block_id).second) { + return; + } + size_t sent = 0; + for (auto it = subscribers_.begin(); it != subscribers_.end();) { + if (it->second.is_in_past()) { + LOG(INFO) << "Unsubscribed " << it->first.first << " for " << it->first.second.to_str() << " (expired)"; + it = subscribers_.erase(it); + continue; + } + if (shard_intersects(it->first.second, block_id.shard_full())) { + ++sent; + send_confirmations(it->first.first, {block_id}); + } + ++it; + } + LOG(INFO) << "Confirmed block " << block_id.to_str() << ", sending " << sent << " confirmations"; +} + +bool ShardBlockRetainer::is_block_outdated(const BlockIdExt& block_id) const { + auto shard_desc = last_masterchain_state_->get_shard_from_config(block_id.shard_full(), false); + return shard_desc.not_null() && shard_desc->top_block_id().seqno() >= block_id.seqno(); +} + +} // namespace ton::validator diff --git a/validator/shard-block-retainer.hpp b/validator/shard-block-retainer.hpp new file mode 100644 index 000000000..14a6cfa8a --- /dev/null +++ b/validator/shard-block-retainer.hpp @@ -0,0 +1,71 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "interfaces/validator-manager.h" +#include "rldp.h" +#include + +namespace ton::validator { + +class ShardBlockRetainer : public td::actor::Actor { + public: + ShardBlockRetainer(adnl::AdnlNodeIdShort local_id, td::Ref last_masterchain_state, + td::Ref opts, td::actor::ActorId manager, + td::actor::ActorId adnl, td::actor::ActorId rldp) + : local_id_(local_id) + , last_masterchain_state_(last_masterchain_state) + , opts_(std::move(opts)) + , manager_(std::move(manager)) + , adnl_(std::move(adnl)) + , rldp_(std::move(rldp)) { + } + + void start_up() override; + void tear_down() override; + void update_masterchain_state(td::Ref state); + void new_shard_block_description(td::Ref desc); + + void update_options(td::Ref opts) { + opts_ = std::move(opts); + } + + private: + adnl::AdnlNodeIdShort local_id_; + td::Ref last_masterchain_state_; + td::Ref opts_; + td::actor::ActorId manager_; + td::actor::ActorId adnl_; + td::actor::ActorId rldp_; + + bool inited_ = false; + std::set validator_adnl_ids_; + std::map, td::Timestamp> subscribers_; + std::set confirmed_blocks_; + + void process_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); + void send_confirmations(adnl::AdnlNodeIdShort dst, std::vector blocks); + + void confirm_shard_block_description(td::Ref desc); + void confirm_block(BlockIdExt block_id); + + bool is_block_outdated(const BlockIdExt& block_id) const; + + static constexpr double SUBSCRIPTION_TTL = 60.0; + static constexpr size_t MAX_BLOCKS_PER_MESSAGE = 8; +}; + +} // namespace ton::validator diff --git a/validator/shard-block-verifier.cpp b/validator/shard-block-verifier.cpp new file mode 100644 index 000000000..5beb1e9a3 --- /dev/null +++ b/validator/shard-block-verifier.cpp @@ -0,0 +1,194 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "shard-block-verifier.hpp" + +#include "td/actor/MultiPromise.h" + +namespace ton::validator { + +void ShardBlockVerifier::start_up() { + update_config(opts_->get_shard_block_verifier_config()); + update_masterchain_state(last_masterchain_state_); + + class Callback : public adnl::Adnl::Callback { + public: + explicit Callback(td::actor::ActorId id) : id_(std::move(id)) { + } + void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { + td::actor::send_closure(id_, &ShardBlockVerifier::process_message, src, std::move(data)); + } + void receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + } + + private: + td::actor::ActorId id_; + }; + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::shardBlockVerifier_confirmBlocks::ID), + std::make_unique(actor_id(this))); + td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); +} + +void ShardBlockVerifier::tear_down() { + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::shardBlockVerifier_confirmBlocks::ID)); +} + +void ShardBlockVerifier::update_masterchain_state(td::Ref state) { + last_masterchain_state_ = std::move(state); + for (auto it = blocks_.begin(); it != blocks_.end();) { + if (is_block_outdated(it->first)) { + it->second.finalize_promises(); + it = blocks_.erase(it); + } else { + ++it; + } + } +} + +void ShardBlockVerifier::wait_shard_blocks(std::vector blocks, td::Promise promise) { + td::MultiPromise mp; + auto ig = mp.init_guard(); + ig.add_promise(std::move(promise)); + for (const BlockIdExt& block_id : blocks) { + BlockInfo* info = get_block_info(block_id); + if (info && !info->confirmed) { + info->promises.push_back(ig.get_promise()); + } + } +} + +void ShardBlockVerifier::update_config(td::Ref new_config) { + auto old_config = std::move(config_); + config_ = std::move(new_config); + auto old_blocks = std::move(blocks_); + blocks_.clear(); + for (auto& [block_id, old_info] : old_blocks) { + BlockInfo* new_info = get_block_info(block_id); + if (new_info == nullptr) { + old_info.finalize_promises(); + continue; + } + new_info->promises = std::move(old_info.promises); + if (new_info->confirmed) { + new_info->finalize_promises(); + } + for (size_t old_src_idx = 0; old_src_idx < old_info.confirmed_by.size(); ++old_src_idx) { + if (old_info.confirmed_by[old_src_idx]) { + set_block_confirmed(old_config->shards[old_info.config_shard_idx].trusted_nodes[old_src_idx], block_id); + } + } + } + + alarm_timestamp().relax(send_subscribe_at_ = td::Timestamp::now()); +} + +void ShardBlockVerifier::alarm() { + if (send_subscribe_at_ && send_subscribe_at_.is_in_past()) { + for (auto& shard_config : config_->shards) { + for (auto& node_id : shard_config.trusted_nodes) { + td::Promise P = [shard = shard_config.shard_id, node_id](td::Result R) { + if (R.is_error()) { + LOG(WARNING) << "Subscribe to " << node_id << " for " << shard.to_str() << " : " << R.move_as_error(); + } + }; + td::actor::send_closure(rldp_, &rldp::Rldp::send_query, local_id_, node_id, "subscribe", std::move(P), + td::Timestamp::in(3.0), + create_serialize_tl_object( + create_tl_shard_id(shard_config.shard_id), 0)); + } + } + send_subscribe_at_ = td::Timestamp::in(SEND_SUBSCRIBE_PERIOD); + } + alarm_timestamp().relax(send_subscribe_at_); +} + +void ShardBlockVerifier::process_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) { + auto r_obj = fetch_tl_object(data, true); + if (r_obj.is_error()) { + return; + } + for (const auto& b : r_obj.ok()->blocks_) { + set_block_confirmed(src, create_block_id(b)); + } +} + +int ShardBlockVerifier::get_config_shard_idx(const ShardIdFull& shard_id) const { + for (size_t i = 0; i < config_->shards.size(); i++) { + if (shard_intersects(shard_id, config_->shards[i].shard_id)) { + return i; + } + } + return -1; +} + +bool ShardBlockVerifier::is_block_outdated(const BlockIdExt& block_id) const { + auto shard_desc = last_masterchain_state_->get_shard_from_config(block_id.shard_full(), false); + return shard_desc.not_null() && shard_desc->top_block_id().seqno() >= block_id.seqno(); +} + +ShardBlockVerifier::BlockInfo* ShardBlockVerifier::get_block_info(const BlockIdExt& block_id) { + auto it = blocks_.find(block_id); + if (it != blocks_.end()) { + return &it->second; + } + int config_shard_idx = get_config_shard_idx(block_id.shard_full()); + if (config_shard_idx < 0 || is_block_outdated(block_id)) { + return nullptr; + } + auto& shard_config = config_->shards[config_shard_idx]; + BlockInfo& info = blocks_[block_id]; + info.config_shard_idx = config_shard_idx; + info.confirmed_by.resize(shard_config.trusted_nodes.size(), false); + info.confirmed = (shard_config.required_confirms == 0); + return &info; +} + +void ShardBlockVerifier::set_block_confirmed(adnl::AdnlNodeIdShort src, BlockIdExt block_id) { + BlockInfo* info = get_block_info(block_id); + if (info == nullptr) { + LOG(INFO) << "Confirm for " << block_id.to_str() << " from " << src << " : ignored"; + return; + } + auto& shard_config = config_->shards[info->config_shard_idx]; + size_t src_idx = 0; + while (src_idx < shard_config.trusted_nodes.size() && shard_config.trusted_nodes[src_idx] != src) { + ++src_idx; + } + if (src_idx == shard_config.trusted_nodes.size()) { + LOG(INFO) << "Confirm for " << block_id.to_str() << " from " << src << " : unknown src"; + return; + } + if (info->confirmed_by[src_idx]) { + LOG(INFO) << "Confirm for " << block_id.to_str() << " from " << src << " #" << src_idx << " : duplicate"; + return; + } + info->confirmed_by[src_idx] = true; + ++info->confirmed_by_cnt; + LOG(INFO) << "Confirm for " << block_id.to_str() << " from " << src << " #" << src_idx << " : accepted (" + << info->confirmed_by_cnt << "/" << shard_config.required_confirms << "/" + << shard_config.trusted_nodes.size() << ")" + << (info->confirmed_by_cnt == shard_config.required_confirms ? ", CONFIRMED" : ""); + if (info->confirmed_by_cnt == shard_config.required_confirms) { + info->confirmed = true; + info->finalize_promises(); + info->promises.clear(); + } +} + +} // namespace ton::validator diff --git a/validator/shard-block-verifier.hpp b/validator/shard-block-verifier.hpp new file mode 100644 index 000000000..82c1e3738 --- /dev/null +++ b/validator/shard-block-verifier.hpp @@ -0,0 +1,90 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "interfaces/validator-manager.h" +#include "rldp.h" +#include + +namespace ton::validator { + +class ShardBlockVerifier : public td::actor::Actor { + public: + ShardBlockVerifier(adnl::AdnlNodeIdShort local_id, td::Ref last_masterchain_state, + td::Ref opts, td::actor::ActorId manager, + td::actor::ActorId adnl, td::actor::ActorId rldp) + : local_id_(local_id) + , last_masterchain_state_(last_masterchain_state) + , opts_(std::move(opts)) + , manager_(std::move(manager)) + , adnl_(std::move(adnl)) + , rldp_(std::move(rldp)) { + } + + void start_up() override; + void tear_down() override; + void update_masterchain_state(td::Ref state); + void wait_shard_blocks(std::vector blocks, td::Promise promise); + + void update_options(td::Ref opts) { + if (config_ != opts->get_shard_block_verifier_config()) { + update_config(opts->get_shard_block_verifier_config()); + } + opts_ = std::move(opts); + } + + void alarm() override; + + private: + adnl::AdnlNodeIdShort local_id_; + td::Ref last_masterchain_state_; + td::Ref opts_; + td::actor::ActorId manager_; + td::actor::ActorId adnl_; + td::actor::ActorId rldp_; + + td::Ref config_; + + td::Timestamp send_subscribe_at_ = td::Timestamp::never(); + + struct BlockInfo { + size_t config_shard_idx = 0; + std::vector confirmed_by; + td::uint32 confirmed_by_cnt = 0; + bool confirmed = false; + std::vector> promises; + + void finalize_promises() { + for (auto& promise : promises) { + promise.set_value(td::Unit()); + } + } + }; + std::map blocks_; + + void update_config(td::Ref new_config); + void process_message(adnl::AdnlNodeIdShort src, td::BufferSlice data); + + int get_config_shard_idx(const ShardIdFull& shard_id) const; + bool is_block_outdated(const BlockIdExt& block_id) const; + BlockInfo* get_block_info(const BlockIdExt& block_id); + + void set_block_confirmed(adnl::AdnlNodeIdShort src, BlockIdExt block_id); + + static constexpr double SEND_SUBSCRIBE_PERIOD = 10.0; +}; + +} // namespace ton::validator diff --git a/validator/validator-options.cpp b/validator/validator-options.cpp index 230a5df46..52047a1d8 100644 --- a/validator/validator-options.cpp +++ b/validator/validator-options.cpp @@ -69,6 +69,34 @@ CollatorsList CollatorsList::default_list() { return list; } +td::Status ShardBlockVerifierConfig::unpack(const ton_api::engine_validator_shardBlockVerifierConfig& obj) { + shards.clear(); + for (const auto& shard_obj : obj.shards_) { + Shard shard; + shard.shard_id = create_shard_id(shard_obj->shard_id_); + if (shard.shard_id.is_masterchain() || !shard.shard_id.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard " << shard.shard_id.to_str()); + } + std::set trusted_nodes; + for (const td::Bits256& id : shard_obj->trusted_nodes_) { + adnl::AdnlNodeIdShort node_id{id}; + if (!trusted_nodes.insert(node_id).second) { + return td::Status::Error(PSTRING() << "duplicate node " << node_id); + } + shard.trusted_nodes.push_back(node_id); + if (shard_obj->required_confirms_ < 0 || shard_obj->required_confirms_ > (int)shard.trusted_nodes.size()) { + return td::Status::Error(PSTRING() + << "invalid required_confirms " << shard_obj->required_confirms_ << " for shard " + << shard.shard_id.to_str() << " (nodes: " << shard.trusted_nodes.size() << ")"); + } + shard.required_confirms = shard_obj->required_confirms_; + shards.push_back(std::move(shard)); + } + } + return td::Status::OK(); +} + + td::Ref ValidatorManagerOptions::create(BlockIdExt zero_block_id, BlockIdExt init_block_id, std::function check_shard, bool allow_blockchain_init, double sync_blocks_before, diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 0c6f93633..607c8890e 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -169,6 +169,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool check_collator_node_whitelist(adnl::AdnlNodeIdShort id) const override { return !collator_node_whitelist_enabled_ || collator_node_whitelist_.contains(id); } + td::Ref get_shard_block_verifier_config() const override { + return shard_block_verifier_config_; + } void set_zero_block_id(BlockIdExt block_id) override { zero_block_id_ = block_id; @@ -286,6 +289,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_collator_node_whitelist_enabled(bool enabled) override { collator_node_whitelist_enabled_ = enabled; } + void set_shard_block_verifier_config(td::Ref config) override { + shard_block_verifier_config_ = std::move(config); + } ValidatorManagerOptionsImpl *make_copy() const override { return new ValidatorManagerOptionsImpl(*this); @@ -345,6 +351,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { td::Ref collators_list_{true, CollatorsList::default_list()}; std::set collator_node_whitelist_; bool collator_node_whitelist_enabled_ = false; + td::Ref shard_block_verifier_config_{true}; }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 2318d53e5..09b790a35 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -95,6 +95,17 @@ struct CollatorsList : public td::CntObject { static CollatorsList default_list(); }; +struct ShardBlockVerifierConfig : public td::CntObject { + struct Shard { + ShardIdFull shard_id; + std::vector trusted_nodes; + td::uint32 required_confirms; + }; + std::vector shards; + + td::Status unpack(const ton_api::engine_validator_shardBlockVerifierConfig& obj); +}; + struct ValidatorManagerOptions : public td::CntObject { public: virtual BlockIdExt zero_block_id() const = 0; @@ -141,6 +152,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual double get_catchain_broadcast_speed_multiplier() const = 0; virtual td::Ref get_collators_list() const = 0; virtual bool check_collator_node_whitelist(adnl::AdnlNodeIdShort id) const = 0; + virtual td::Ref get_shard_block_verifier_config() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; @@ -179,6 +191,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_collators_list(td::Ref list) = 0; virtual void set_collator_node_whitelisted_validator(adnl::AdnlNodeIdShort id, bool add) = 0; virtual void set_collator_node_whitelist_enabled(bool enabled) = 0; + virtual void set_shard_block_verifier_config(td::Ref config) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, @@ -348,6 +361,10 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_collation_manager_stats( td::Promise> promise) = 0; + + virtual void add_shard_block_retainer(adnl::AdnlNodeIdShort id) { + LOG(ERROR) << "Unimplemented add_shard_block_retainer"; + } }; } // namespace validator From ebde55ea343bba9c7b00947c4e6590c2899c5899 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Fri, 25 Apr 2025 13:28:37 +0400 Subject: [PATCH 227/388] [Tolk] Simplify VarDescr, drop _Const enum value Now, only VarDescr::int_const represents whether it's a constant, without being combined (and potentially become unsync) val bit. --- tolk-tester/tests/logical-operators.tolk | 27 +++++++++++++++++++ tolk-tester/tests/smart-cast-tests.tolk | 4 ++- tolk/abscode.cpp | 33 +++++++++++------------- tolk/analyzer.cpp | 4 +-- tolk/tolk.h | 17 +++++++----- 5 files changed, 57 insertions(+), 28 deletions(-) diff --git a/tolk-tester/tests/logical-operators.tolk b/tolk-tester/tests/logical-operators.tolk index cbc82f15e..7e3168c7c 100644 --- a/tolk-tester/tests/logical-operators.tolk +++ b/tolk-tester/tests/logical-operators.tolk @@ -148,6 +148,31 @@ fun testConvertIfToIfnot(x: bool) { return -4; } +@pure +fun get123(): int + asm "123 PUSHINT"; + +@method_id(114) +fun test114() { + val fals = (get123() < 0) as int; + return (fals ? -1 : fals) != 0; +} + +@method_id(115) +fun test115() { + val tru = get123() >= 0; + val fals = get123() < 0; + + if ((true || false) && (false || false)) { + throw 123; + } + if ((tru || fals) && (fals || fals)) { + throw 456; + } + return (tru, fals); +} + + fun main() { } @@ -187,6 +212,8 @@ fun main() { @testcase | 112 | 5 0 | 0 -1 0 -1 -1 @testcase | 112 | 0 -1 | -1 -1 -1 -1 -1 @testcase | 113 | 0 | 1 +@testcase | 114 | | 0 +@testcase | 115 | | -1 0 @fif_codegen """ diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index 62477c135..946c00222 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -148,9 +148,10 @@ fun test10(): int { return x + y; } +@method_id(111) fun test11() { var [x, y] = [getNullableInt(), getNullableInt()]; - if (random()) { return x == null || y == null ? -1 : x + y; } + if (eq(10) < 0) { return x == null || y == null ? -1 : x + y; } if (true && (x == null || y == null) && !!true) { return 0; } return x + y; } @@ -665,6 +666,7 @@ fun main(x: int?): int { /** @testcase | 0 | 1 | 1 +@testcase | 111 | | 10 @testcase | 123 | | 7 @testcase | 124 | 4 | 6 @testcase | 124 | null | 15 diff --git a/tolk/abscode.cpp b/tolk/abscode.cpp index c33fcd89d..90c7fbcf1 100644 --- a/tolk/abscode.cpp +++ b/tolk/abscode.cpp @@ -59,9 +59,6 @@ void VarDescr::show_value(std::ostream& os) const { if (val & _Int) { os << 'i'; } - if (val & _Const) { - os << 'c'; - } if (val & _Zero) { os << '0'; } @@ -114,7 +111,7 @@ void VarDescr::set_const(td::RefInt256 value) { if (!int_const->signed_fits_bits(257)) { int_const.write().invalidate(); } - val = _Const | _Int; + val = _Int; int s = sgn(int_const); if (s < -1) { val |= _Nan | _NonZero; @@ -130,41 +127,41 @@ void VarDescr::set_const(td::RefInt256 value) { } } -void VarDescr::set_const(std::string value) { - str_const = value; - val = _Const; +void VarDescr::set_const(const std::string&) { + int_const.clear(); + val = 0; } void VarDescr::operator|=(const VarDescr& y) { - val &= y.val; - if (is_int_const() && y.is_int_const() && cmp(int_const, y.int_const) != 0) { - val &= ~_Const; - } - if (!(val & _Const)) { - int_const.clear(); + if (is_int_const()) { + bool y_same = y.is_int_const() && *int_const == *y.int_const; + if (!y_same) { + int_const.clear(); + } } + val &= y.val; } void VarDescr::operator&=(const VarDescr& y) { - val |= y.val; - if (y.int_const.not_null() && int_const.is_null()) { + if (y.is_int_const()) { int_const = y.int_const; } + val |= y.val; } void VarDescr::set_value(const VarDescr& y) { - val = y.val; int_const = y.int_const; + val = y.val; } void VarDescr::set_value(VarDescr&& y) { - val = y.val; int_const = std::move(y.int_const); + val = y.val; } void VarDescr::clear_value() { - val = 0; int_const.clear(); + val = 0; } void VarDescrList::show(std::ostream& os) const { diff --git a/tolk/analyzer.cpp b/tolk/analyzer.cpp index 2cd65c419..935c8b02c 100644 --- a/tolk/analyzer.cpp +++ b/tolk/analyzer.cpp @@ -62,10 +62,10 @@ bool operator==(const VarDescrList& x, const VarDescrList& y) { } bool same_values(const VarDescr& x, const VarDescr& y) { - if (x.val != y.val || x.int_const.is_null() != y.int_const.is_null()) { + if (x.val != y.val || x.is_int_const() != y.is_int_const()) { return false; } - if (x.int_const.not_null() && cmp(x.int_const, y.int_const) != 0) { + if (x.is_int_const() && *x.int_const != *y.int_const) { return false; } return true; diff --git a/tolk/tolk.h b/tolk/tolk.h index 3f00d0d4b..74d8fdd9f 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -68,7 +68,6 @@ struct VarDescr { enum { _Last = 1, _Unused = 2 }; int flags; enum { - _Const = 16, _Int = 32, _Zero = 64, _NonZero = 128, @@ -79,16 +78,15 @@ struct VarDescr { _Even = 16384, _Odd = 32768, }; - static constexpr int ConstZero = _Const | _Int | _Zero | _Pos | _Neg | _Finite | _Even; - static constexpr int ConstOne = _Const | _Int | _NonZero | _Pos | _Finite | _Odd; - static constexpr int ConstTrue = _Const | _Int | _NonZero | _Neg | _Finite | _Odd; + static constexpr int ConstZero = _Int | _Zero | _Pos | _Neg | _Finite | _Even; + static constexpr int ConstOne = _Int | _NonZero | _Pos | _Finite | _Odd; + static constexpr int ConstTrue = _Int | _NonZero | _Neg | _Finite | _Odd; static constexpr int ValBit = _Int | _Pos | _Finite; static constexpr int ValBool = _Int | _Neg | _Finite; static constexpr int FiniteInt = _Int | _Finite; static constexpr int FiniteUInt = _Int | _Finite | _Pos; int val; td::RefInt256 int_const; - std::string str_const; explicit VarDescr(var_idx_t _idx = -1, int _flags = 0, int _val = 0) : idx(_idx), flags(_flags), val(_val) { } @@ -120,7 +118,12 @@ struct VarDescr { return val & _Odd; } bool is_int_const() const { - return (val & (_Int | _Const)) == (_Int | _Const) && int_const.not_null(); +#ifdef TOLK_DEBUG + if (int_const.not_null()) { + tolk_assert(val & _Int); + } +#endif + return int_const.not_null(); } bool always_nonpos() const { return val & _Neg; @@ -151,7 +154,7 @@ struct VarDescr { } void set_const(long long value); void set_const(td::RefInt256 value); - void set_const(std::string value); + void set_const(const std::string& value); void operator+=(const VarDescr& y) { flags &= y.flags; } From 9e6c1fc9875e9ca683a5b88e7e824c74dfb95a0a Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Mon, 17 Feb 2025 18:01:23 +0300 Subject: [PATCH 228/388] [Tolk] Implement structures that work like tensors on a stack Support declaring structures with arbitrary fields and creating objects very similar to TS syntax. Structs are actually tensors on a stack, and accessing their fields behave exactly as accessing tensor's elements by index. As a consequence, recurring struct fields doesn't work; it will lead to infinity size. All features of type system (nullables, unions) seamlessly work with structures. A struct with 1 field has no extra overhead, it's just a field on a stack. --- .../tests/allow-post-modification.tolk | 13 +- .../tests/invalid-declaration/err-1585.tolk | 10 + .../tests/invalid-semantics/err-4085.tolk | 22 + .../tests/invalid-semantics/err-4150.tolk | 13 + .../tests/invalid-semantics/err-4193.tolk | 13 + .../tests/invalid-semantics/err-4260.tolk | 15 + .../tests/invalid-semantics/err-4391.tolk | 13 + .../tests/invalid-semantics/err-4440.tolk | 21 + .../tests/invalid-semantics/err-4519.tolk | 12 + .../tests/invalid-semantics/err-4572.tolk | 9 + .../tests/invalid-semantics/err-4682.tolk | 13 + .../tests/invalid-semantics/err-4712.tolk | 10 + .../tests/invalid-semantics/err-4822.tolk | 23 + .../tests/invalid-syntax/err-3034.tolk | 8 + .../tests/invalid-syntax/err-3150.tolk | 9 + .../tests/invalid-syntax/err-3690.tolk | 12 + .../tests/invalid-syntax/err-3802.tolk | 9 + .../tests/invalid-typing/err-6078.tolk | 11 + .../tests/invalid-typing/err-6184.tolk | 11 + .../tests/invalid-typing/err-6230.tolk | 10 + .../tests/invalid-typing/err-6709.tolk | 13 + .../tests/invalid-typing/err-6829.tolk | 6 +- .../tests/invalid-typing/err-6845.tolk | 12 + .../tests/invalid-typing/err-6942.tolk | 19 + tolk-tester/tests/meaningful-1.tolk | 43 ++ tolk-tester/tests/smart-cast-tests.tolk | 52 ++ tolk-tester/tests/struct-tests.tolk | 519 ++++++++++++++++++ tolk-tester/tests/type-aliases-tests.tolk | 7 + tolk-tester/tests/var-apply-tests.tolk | 21 + tolk/abscode.cpp | 9 +- tolk/ast-from-tokens.cpp | 91 ++- tolk/ast-replacer.h | 6 + tolk/ast-replicator.h | 12 + tolk/ast-stringifier.h | 20 + tolk/ast-visitor.h | 6 + tolk/ast.cpp | 20 + tolk/ast.h | 109 +++- tolk/fwd-declarations.h | 4 + tolk/pipe-ast-to-legacy.cpp | 139 +++-- tolk/pipe-calc-rvalue-lvalue.cpp | 19 + tolk/pipe-check-inferred-types.cpp | 13 + tolk/pipe-check-rvalue-lvalue.cpp | 25 + tolk/pipe-infer-types-and-calls.cpp | 129 ++++- tolk/pipe-register-symbols.cpp | 21 + tolk/pipe-resolve-identifiers.cpp | 37 +- tolk/smart-casts-cfg.cpp | 37 +- tolk/symtable.cpp | 25 + tolk/symtable.h | 32 ++ tolk/type-system.cpp | 54 ++ tolk/type-system.h | 23 + 50 files changed, 1719 insertions(+), 61 deletions(-) create mode 100644 tolk-tester/tests/invalid-declaration/err-1585.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4085.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4150.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4193.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4260.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4391.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4440.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4519.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4572.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4682.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4712.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4822.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3034.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3150.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3690.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3802.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6078.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6184.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6230.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6709.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6845.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6942.tolk create mode 100644 tolk-tester/tests/meaningful-1.tolk create mode 100644 tolk-tester/tests/struct-tests.tolk diff --git a/tolk-tester/tests/allow-post-modification.tolk b/tolk-tester/tests/allow-post-modification.tolk index 180654e32..10c243740 100644 --- a/tolk-tester/tests/allow-post-modification.tolk +++ b/tolk-tester/tests/allow-post-modification.tolk @@ -112,6 +112,16 @@ fun test_assign_tensor_global(x: (int, int)) { return (x, x = (20, 30), fs = x.multens((1, 2)), fs.multens(multens(mutate x, (-1, -1))), x, fs); } +struct Point1D { + c: int; +} + +@method_id(25) +fun test_with_single_slot_struct(x: Point1D) { + var result = (x, x.c += 10, [x, x.c += 20, eq(x.c -= 50), x.c], eq2((x.c, x.c *= eq(x.c /= 2)))); + return result; +} + fun main() { } @@ -132,6 +142,7 @@ fun main() { @testcase | 22 | 100 | 100 210 4200 630 @testcase | 23 | 1 1 | 1 1 20 30 20 60 -400 -3600 -20 -60 -400 -3600 @testcase | 24 | 1 1 | 1 1 20 30 20 60 -400 -3600 -20 -60 -400 -3600 +@testcase | 25 | 100 | 100 110 [ 110 130 80 80 ] 80 3200 @fif_codegen """ @@ -147,5 +158,5 @@ fun main() { // x.0 x.1 """ -@code_hash 61280273714870328160131559159866470128402169974050439159015534193532598351244 +@code_hash 6737917279814799680932710799951154408447028229503449454536845752635763933556 */ diff --git a/tolk-tester/tests/invalid-declaration/err-1585.tolk b/tolk-tester/tests/invalid-declaration/err-1585.tolk new file mode 100644 index 000000000..0c1fef332 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1585.tolk @@ -0,0 +1,10 @@ +struct B { a: A } +struct A { b: BAlias } + +type BAlias = B; + +/** +@compilation_should_fail +@stderr struct `B` size is infinity due to recursive fields +@stderr struct B { + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4085.tolk b/tolk-tester/tests/invalid-semantics/err-4085.tolk new file mode 100644 index 000000000..11c403908 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4085.tolk @@ -0,0 +1,22 @@ +struct A { + value: int; +} + +struct B { + data: slice; +} + +fun main() { + var r1: A | B = A { value: 10 }; + var r2: A | null = { value: 10 }; + var r3: A | int | builder = { value: 10 }; + + var r4: A | B = { value: 20 }; +} + +/** +@compilation_should_fail +@stderr can not detect struct name +@stderr use either `var v: StructName = { ... }` or `var v = StructName { ... }` +@stderr { value: 20 } + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4150.tolk b/tolk-tester/tests/invalid-semantics/err-4150.tolk new file mode 100644 index 000000000..d54257872 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4150.tolk @@ -0,0 +1,13 @@ +fun increment(mutate x: int) { + x += 1; +} + +fun checkCantMutateFieldOfImmutableTensor() { + val t = (0, 1); + increment(mutate ((t!)).0); +} + +/** +@compilation_should_fail +@stderr modifying immutable variable `t` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4193.tolk b/tolk-tester/tests/invalid-semantics/err-4193.tolk new file mode 100644 index 000000000..b0b31fa7d --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4193.tolk @@ -0,0 +1,13 @@ +struct User { + id: int; +} + +fun cantInstantiateNonGenericStruct() { + var u = User { id: 3 }; +} + +/** +@compilation_should_fail +@stderr generic T not expected here +@stderr User + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4260.tolk b/tolk-tester/tests/invalid-semantics/err-4260.tolk new file mode 100644 index 000000000..6f174a595 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4260.tolk @@ -0,0 +1,15 @@ +struct User { + id: int; +} + +type UserAlias = User; + +fun cantInstantiateNonGenericStruct() { + var u = UserAlias { id: 3 }; +} + +/** +@compilation_should_fail +@stderr generic T not expected here +@stderr UserAlias + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4391.tolk b/tolk-tester/tests/invalid-semantics/err-4391.tolk new file mode 100644 index 000000000..80baf9b83 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4391.tolk @@ -0,0 +1,13 @@ +struct User { + id: int; + name: slice; +} + +fun main() { + var u: User = { id: 1 }; +} + +/** +@compilation_should_fail +@stderr field `name` missed in initialization of struct `User` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4440.tolk b/tolk-tester/tests/invalid-semantics/err-4440.tolk new file mode 100644 index 000000000..9c59f2c50 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4440.tolk @@ -0,0 +1,21 @@ +struct NumberFields { + `-1`: int; + `0`: int; + `100500`: int; +} + +fun main() { + var b: NumberFields = { `-1`: -1, `0`: 0, `100500`: 100500 }; + b.0; // ok + b.`0`; // ok + b.`-1`; // ok + b.`100500`; // ok + b.100500; // ok + b.123; +} + +/** +@compilation_should_fail +@stderr field `123` doesn't exist in type `NumberFields` +@stderr b.123 + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4519.tolk b/tolk-tester/tests/invalid-semantics/err-4519.tolk new file mode 100644 index 000000000..b0377b719 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4519.tolk @@ -0,0 +1,12 @@ +struct value { id: int; } + +fun main() { + var value = 100; + var obj = value { id: 100 }; +} + +/** +@compilation_should_fail +@stderr `value` does not name a struct +@stderr obj = value + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4572.tolk b/tolk-tester/tests/invalid-semantics/err-4572.tolk new file mode 100644 index 000000000..a62b63a0f --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4572.tolk @@ -0,0 +1,9 @@ +fun checkCantMutateFieldOfImmutableTuple() { + val ks = null as [int, [int, [int]]]?; + ks!.1.1.0 = 10; +} + +/** +@compilation_should_fail +@stderr modifying immutable variable `ks` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4682.tolk b/tolk-tester/tests/invalid-semantics/err-4682.tolk new file mode 100644 index 000000000..054a6a403 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4682.tolk @@ -0,0 +1,13 @@ +struct User { id: int; } + +fun absk(v: int) { return v > 0 ? v : -v; } + +fun main() { + absk(User); +} + +/** +@compilation_should_fail +@stderr struct `User` can not be used as a value +@stderr absk(User) +*/ diff --git a/tolk-tester/tests/invalid-semantics/err-4712.tolk b/tolk-tester/tests/invalid-semantics/err-4712.tolk new file mode 100644 index 000000000..2e210eb4b --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4712.tolk @@ -0,0 +1,10 @@ +fun main() { + var t: tuple = createEmptyTuple(); + t.asdf = 10; +} + +/** +@compilation_should_fail +@stderr field `asdf` doesn't exist in type `tuple` +@stderr in function `main` +*/ diff --git a/tolk-tester/tests/invalid-semantics/err-4822.tolk b/tolk-tester/tests/invalid-semantics/err-4822.tolk new file mode 100644 index 000000000..85cf4073b --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4822.tolk @@ -0,0 +1,23 @@ +struct Point { + x: int; + y: int; +} + +struct Coord2 { + p1: Point?; + p2: Point; +} + +fun increment(mutate x: int) { + x += 1; +} + +fun checkCantMutateFieldOfImmutableTuple(p1: Point?, p2: Point) { + val c2: Coord2 = { p1, p2 }; + (c2!.p1!).x.increment(); +} + +/** +@compilation_should_fail +@modifying immutable variable `c2` + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3034.tolk b/tolk-tester/tests/invalid-syntax/err-3034.tolk new file mode 100644 index 000000000..af02af80a --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3034.tolk @@ -0,0 +1,8 @@ +struct A { + id; +} + +/** +@compilation_should_fail +@stderr expected `: `, got `;` +*/ diff --git a/tolk-tester/tests/invalid-syntax/err-3150.tolk b/tolk-tester/tests/invalid-syntax/err-3150.tolk new file mode 100644 index 000000000..8828fcadf --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3150.tolk @@ -0,0 +1,9 @@ +struct A { + id: int = 3; +} + +/** +@compilation_should_fail +@stderr default values for fields not supported yet +@stderr id: int = 3 +*/ diff --git a/tolk-tester/tests/invalid-syntax/err-3690.tolk b/tolk-tester/tests/invalid-syntax/err-3690.tolk new file mode 100644 index 000000000..934e18541 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3690.tolk @@ -0,0 +1,12 @@ +struct A { + id: int; +} + +fun main() { + A { id: 1, id: 2 }; +} + +/** +@compilation_should_fail +@stderr duplicate field initialization +*/ diff --git a/tolk-tester/tests/invalid-syntax/err-3802.tolk b/tolk-tester/tests/invalid-syntax/err-3802.tolk new file mode 100644 index 000000000..fec840c98 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3802.tolk @@ -0,0 +1,9 @@ +struct A { + id1: int + id2: int +} + +/** +@compilation_should_fail +@stderr expected `;` or `,`, got `id2` +*/ diff --git a/tolk-tester/tests/invalid-typing/err-6078.tolk b/tolk-tester/tests/invalid-typing/err-6078.tolk new file mode 100644 index 000000000..139086643 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6078.tolk @@ -0,0 +1,11 @@ +struct S { v: int; } + +fun cantAssignIncorrectTypeToField() { + var s: S = { v: 0 }; + s.v = createEmptyTuple(); +} + +/** +@compilation_should_fail +@stderr can not assign `tuple` to field of type `int` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6184.tolk b/tolk-tester/tests/invalid-typing/err-6184.tolk new file mode 100644 index 000000000..16ba33862 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6184.tolk @@ -0,0 +1,11 @@ +struct S { v: () -> int; } + +fun cantCreateIncorrectTypeToField() { + var s: S = { v: beginCell }; +} + +/** +@compilation_should_fail +@stderr can not assign `() -> builder` to field of type `() -> int` +@stderr v: beginCell + */ diff --git a/tolk-tester/tests/invalid-typing/err-6230.tolk b/tolk-tester/tests/invalid-typing/err-6230.tolk new file mode 100644 index 000000000..03c80df87 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6230.tolk @@ -0,0 +1,10 @@ +struct S { a: int; b: int; } + +fun cantCastStructToTensor(s: S) { + s as (int, int); +} + +/** +@compilation_should_fail +@stderr type `S` can not be cast to `(int, int)` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6709.tolk b/tolk-tester/tests/invalid-typing/err-6709.tolk new file mode 100644 index 000000000..d1fe41281 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6709.tolk @@ -0,0 +1,13 @@ +struct A { id: int; } +struct B { id: int; } +type AAlias = A; + +fun cantAssignDifferentStructures(a: A) { + var a2: AAlias = a; // ok + var b: B = a; +} + +/** +@compilation_should_fail +@stderr can not assign `A` to variable of type `B` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6829.tolk b/tolk-tester/tests/invalid-typing/err-6829.tolk index fbf462367..d3f2e505c 100644 --- a/tolk-tester/tests/invalid-typing/err-6829.tolk +++ b/tolk-tester/tests/invalid-typing/err-6829.tolk @@ -1,7 +1,9 @@ type Pair2 = (int, int); type Pair2Or3 = Pair2 | (int, int, int); -fun matchDoesntCoverAllCases(a: Pair2Or3 | slice | [[int]] | builder) { +struct A {} + +fun matchDoesntCoverAllCases(a: Pair2Or3 | slice | [[int]] | A) { if (a !is slice) { match (a) { (int, int) => {} @@ -12,5 +14,5 @@ fun matchDoesntCoverAllCases(a: Pair2Or3 | slice | [[int]] | builder) { /** @compilation_should_fail -@stderr `match` does not cover all possible types; missing types are: `(int, int, int)`, `builder` +@stderr `match` does not cover all possible types; missing types are: `(int, int, int)`, `A` */ diff --git a/tolk-tester/tests/invalid-typing/err-6845.tolk b/tolk-tester/tests/invalid-typing/err-6845.tolk new file mode 100644 index 000000000..b739ce52a --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6845.tolk @@ -0,0 +1,12 @@ +struct A { id: int; value: int; } + +fun takeTensor(v: (int, int)) {} + +fun cantPassStructAsTensor() { + takeTensor(A{id: 1, value:2}); +} + +/** +@compilation_should_fail +@stderr can not pass `A` to `(int, int)` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6942.tolk b/tolk-tester/tests/invalid-typing/err-6942.tolk new file mode 100644 index 000000000..0ffe5c218 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6942.tolk @@ -0,0 +1,19 @@ +struct Point { x: int; y: int; } + +fun methodOverNullable(p: Point?) {} +fun method(p: Point) {} + +fun cantAssignNullableObjectField(a: Point?) { + if (a != null) { + a.x = 10; // ok + } + a.methodOverNullable(); // ok + a.method(); // also ok, would have failed on type checking, but doesn't reach type checking + a.y = 20; +} + +/** +@compilation_should_fail +@stderr can not access field `y` of a possibly nullable object `Point?` +@stderr check it via `obj != null` or use non-null assertion `obj!` operator + */ diff --git a/tolk-tester/tests/meaningful-1.tolk b/tolk-tester/tests/meaningful-1.tolk new file mode 100644 index 000000000..19356dc52 --- /dev/null +++ b/tolk-tester/tests/meaningful-1.tolk @@ -0,0 +1,43 @@ +struct CounterReset { + initialValue: int64; +} + +struct CounterIncrement1 { +} + +struct CounterIncrement { + byValue: int32; +} + +type MyMessage = + CounterIncrement + | CounterIncrement1 + | CounterReset; + +fun fakeParseMessage(mode: int): MyMessage { + if (mode == 1) { + return CounterIncrement1 {}; + } else if (mode > 0) { + return CounterIncrement { byValue: mode }; + } else { + return CounterReset { initialValue: 0 }; + } +} + +fun onInternalMessage(curCounter: int, mode: int) { + var m = fakeParseMessage(mode); + __expect_type(m, "MyMessage"); + + val nextCounter: int = match (m) { + CounterReset => m.initialValue, + CounterIncrement => curCounter + m.byValue, + CounterIncrement1 => curCounter + 1, + }; + return nextCounter; +} + +/** +@testcase | 0 | 50 0 | 0 +@testcase | 0 | 50 1 | 51 +@testcase | 0 | 50 9 | 59 + */ diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index 946c00222..77374bdce 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -1,6 +1,17 @@ // the goal of this file is not only to @testcase results — // but to check that this file compiles +struct Point { + x: int; + y: int; +} + +struct PointOfNulls { + x: int?; + y: int?; + sub: Point?; +} + fun getNullableInt(): int? { return 5; } fun getNullableSlice(): slice? { return null; } fun takeNullableInt(a: int?) {} @@ -12,6 +23,7 @@ fun sameTensor(t: (int, int)) { return t; } fun sameTensor2(t: (int?, Pair4N)) { return t; } fun eq(v: T) { return v; } fun getTwo(): X { return 2 as X; } +fun assignNullTo(mutate v: T?) { v = null; } type Pair4N = (slice, slice, slice, builder)?; @@ -659,6 +671,44 @@ fun test60() { } +@method_id(161) +fun test61(p: Point) { + var p2: Point? = p; + __expect_type(p2, "Point"); + if (p.x < 10) { + assignNullTo(mutate p2); + } + __expect_type(p2, "Point?"); + return (p2, p2!); +} + +fun test62() { + var p = PointOfNulls { x: 1, y: null, sub: null }; + __expect_type(p.x, "int?"); + __expect_type(p.sub, "Point?"); + if (p.sub == null) { + p.x! > 10 ? p.sub = { x: 1, y : 2 } : Point { x: 3, y: 4 }; + __expect_type(p.sub, "Point?"); + p.x! > 10 ? p.sub = { x: 1, y : 2 } : p.sub = eq({ x: 3, y: 4 }); + } + __expect_type(p.sub, "Point"); + __expect_type(p!!.sub!!, "Point"); + p.sub.x + p.x! + p.y!; + return random() ? null : p; +} + +fun test63() { + __expect_type(test62, "() -> PointOfNulls?"); + var r = test62(); + __expect_type(r!.sub, "Point?"); + if (r != null) { + r.sub = { x: 1, y: 2 }; + } + __expect_type(r!.sub, "Point?"); + r!.sub = { x: 1, y: 2 }; + __expect_type(r!.sub, "Point"); +} + fun main(x: int?): int { return x == null ? -1 : x; @@ -685,6 +735,8 @@ fun main(x: int?): int { @testcase | 147 | | (null) (null) 100 (null) 100 (null) (null) 0 @testcase | 158 | | 123 10 123 5 @testcase | 160 | | 101 109 +@testcase | 161 | 9 9 | (null) (null) 0 (null) (null) +@testcase | 161 | 19 0 | 19 0 133 19 0 @stderr warning: expression of type `int` can never be `null`, this condition is always true @stderr warning: unreachable code diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk new file mode 100644 index 000000000..91dc00543 --- /dev/null +++ b/tolk-tester/tests/struct-tests.tolk @@ -0,0 +1,519 @@ +struct JustInt { + value: int; +} + +struct JustIntWrapper { + int: JustInt; +} + +type MInt = int; +type JustIntAlias = JustInt; +type JustIntAlias2 = JustIntAlias; + +struct Storage { + owner: User, + lastPoint: Point, +} + +struct Point { + x: int; + y: MInt +} + +struct User { + id: int; + name: slice; +} + +struct OuterStorage { + st: Storage, + stMaybe: Storage? +} + +struct WithTensorInside { + coords: (int, int); + tup: [int, int]; + otherCoords: (int, int)?; + otherTup: tuple; +} + +fun sumCoords(p: Point) { + return p.x + p.y; +} + +fun getStorage1(x: int): Storage { + val owner = User { id: 1, name: "u" }; + return { + owner, + lastPoint: { x, y: 10, }, + }; +} + +fun generatePoint(x: int, y: int): Point { + return { x, y }; +} + +fun assignCoords(mutate self: Point) { + self.x = 10; + self.y = 20; +} + +@method_id(101) +fun test1() { + var i1 = JustInt { value: 1 }; + var i2: JustInt = JustInt { value: 2, }; + var i3: JustInt = { `value`: 3 }; + __expect_type(i3, "JustInt"); + __expect_type(i3!, "JustInt"); + return (i1, i2, i3, [i1, JustInt{value:5}]); +} + +@method_id(102) +fun test2() { + var i1 = JustIntAlias2 { value: 1 }; + var i2: JustInt = JustIntAlias2 { value: 2, }; + var i3: JustIntAlias2 = { `value`: 3 }; + __expect_type(i1.value, "int"); + __expect_type(i1.value + i2.value + i3.value, "int"); + __expect_type(i3, "JustIntAlias2"); + __expect_type([i2, i3], "[JustInt, JustIntAlias2]"); + __expect_type([i2] as [JustIntAlias], "[JustIntAlias]"); + return (i1, i2, i3, [i1, JustIntAlias{value:5}]); +} + +@method_id(103) +fun test3() { + return generatePoint(5, 6); +} + +@method_id(104) +fun test4() { + var p: Point = { x: 10, y: 20 }; + return (p == null, p, p = {x:30,y:40}, p != null); +} + +@method_id(105) +fun test5() { + var b = PointAlias { y: 20, x: 10 }; + var p: PointAlias = Point { x: b.x, y: b.y }; + p.x += 5; + return (sumCoords(p), p.y += 10, p.sumCoords()); +} + +struct BacktickNames { + `my()id`: MInt, + `100500`: int; +} + +@method_id(106) +fun test6() { + val b: BacktickNames = { `100500`: 20, `my()id`: 10 }; + var p: Point = { x: 0, y: 0 }; + p.x += b.`my()id`; + p.y += b.100500; + return p; +} + +@method_id(107) +fun test7() { + var s = getStorage1(5); + s.owner.name = beginCell().storeInt(s.lastPoint.x, 32).endCell().beginParse(); + var s2 = s; + assignCoords(mutate s.lastPoint); + return (s.owner.name.loadInt(32), s2.owner.name.loadInt(32), s.lastPoint, s2.lastPoint.sumCoords()); +} + +struct Empty{} +struct OuterEmpty { nothing: EmptyAlias } +type EmptyAlias = Empty; + +@method_id(108) +fun test8() { + var e1: Empty = {}; + var o1: OuterEmpty = { nothing: e1 }; + var o2 = OuterEmpty { nothing: {} }; + __expect_type(o2.nothing, "EmptyAlias"); + var o3 = Empty{} as Empty?; + var o4 = null as Empty?; + var o5: EmptyAlias? = o3!; + var o6: OuterEmpty? = { nothing: {} }; + var o7 = o3! as EmptyAlias?; + var o8 = o6 as OuterEmpty?; + __expect_type(o3, "Empty?"); + __expect_type(o6, "OuterEmpty"); + __expect_type(o8, "OuterEmpty?"); + return (e1, Empty{}, EmptyAlias{}, o2, o1, 777, o3, 777, o4, 777, o5, 777, o6, 777, o7, 777, o8, 777, o3!, 777); +} + +fun maxCoord(p: Point): int8 { + return p.x > p.y ? p.x : p.y; +} + +@method_id(109) +fun test9() { + var p = Point { x: 80, y: 100 }; + p.assignCoords(); + return (maxCoord({y: 70, x: 30}), maxCoord(generatePoint(30, 20)), maxCoord(p), p.maxCoord(), Point{x:-80,y:-80}.maxCoord()); +} + +@method_id(110) +fun test10(notNull: bool) { + var p: Point? = null; + __expect_type(p, "null"); + if (notNull) { + p = { x: 1, y: 2 }; + __expect_type(p, "Point"); + __expect_type(p.x, "int"); + __expect_type(p.y, "MInt"); + __expect_type(p.x + p.y, "int"); + assignCoords(mutate p); + } + __expect_type(p, "Point?"); + return p; +} + +@method_id(111) +fun test11(notNull: bool) { + var os: OuterStorage = { + st: { owner: { id: 0, name: "" }, lastPoint: { x: 0, y: 0 } }, + stMaybe: notNull ? { owner: { id: 1, name: "" }, lastPoint: { x: 1, y: 1 } } : null, + }; + if (os.stMaybe == null) { + os!.st!.lastPoint!.y = 2; + os.stMaybe = { owner: { id: 2, name: "" }, lastPoint: { x: 2, y: 3 } }; + } else { + os.stMaybe.owner.id += 1; + os.stMaybe.lastPoint = { x: 3, y: 4 }; + } + if (!notNull) { + assignCoords(mutate os.stMaybe.lastPoint); + os.stMaybe.lastPoint.assignCoords(); + } + return (os.st.lastPoint, os.stMaybe.lastPoint, os.stMaybe.lastPoint.sumCoords()); +} + +@method_id(112) +fun test12(notNull: bool): Point? { + return notNull ? { x: 1, y : 2 } : null; +} + +@method_id(113) +fun test13() { + return null as User?; +} + +@method_id(114) +fun test14() { + var p: Point? = { y: 2, x: 1 }; + __expect_type(p, "Point"); + __expect_type(p as Point, "Point"); + __expect_type(p as Point?, "Point?"); + return p; +} + +@method_id(115) +fun test15() { + var os = OuterStorage { + st: { owner: { id: 0, name: "" }, lastPoint: { x: 0, y: 0 } }, + stMaybe: null, + }; + __expect_type(os.stMaybe, "Storage?"); + return os.stMaybe; +} + +fun pushZero(mutate t: tuple) { + t.tuplePush(0); +} + +@method_id(116) +fun test16(): WithTensorInside? { + var wt: WithTensorInside = { + coords: (1, 2), + tup: [3, 4], + otherCoords: null, + otherTup: createEmptyTuple(), + }; + wt.coords = (-1, -2); + wt.coords.1 -= 2; + wt.tup.0 += 5; + wt.otherCoords!.0 = 7; + wt.otherTup.tuplePush(0); + pushZero(mutate wt.otherTup); + (wt.otherTup.1, wt.otherTup.0) = (11, 10); + return wt; +} + +@method_id(117) +fun test17(x: JustInt?) { + var w1: JustIntWrapper = { int: { value: x == null ? -1 : x.value } }; + var w2: JustIntWrapper? = x == null ? x : { int: { value: x.value } }; + return (x, x!, x!.value, w1, w2); +} + +fun sumXY

(point: P) { return point.x + point.y; } @method_id(118) @@ -488,11 +489,11 @@ fun test34() { return o1; } -@pure fun getXPure() { return 1; } -@pure fun getYPure() { return 2; } +@pure @noinline fun getXPure() { return 1; } +@pure @noinline fun getYPure() { return 2; } global t_impure: tuple; -fun getXImpure() { t_impure.push(1); return 1; } -fun getYImpure() { t_impure.push(2); return 2; } +@noinline fun getXImpure() { t_impure.push(1); return 1; } +@noinline fun getYImpure() { t_impure.push(2); return 2; } @method_id(135) fun test35() { diff --git a/tolk-tester/tests/try-catch-tests.tolk b/tolk-tester/tests/try-catch-tests.tolk index 8acb6f2c7..b1878955e 100644 --- a/tolk-tester/tests/try-catch-tests.tolk +++ b/tolk-tester/tests/try-catch-tests.tolk @@ -164,6 +164,7 @@ fun test109(): (int, int) { return (g_reg, l_reg); } +@noinline fun alwaysThrow123(): never { throw 123; } @@ -173,6 +174,7 @@ fun alwaysThrowX(x: int): never { else { throw (x, null); } } +@noinline fun anotherNever(throw123: bool): never { if (throw123) { alwaysThrow123(); } alwaysThrowX(456); @@ -224,9 +226,13 @@ fun test111(b: bool) { } } -fun mySetCode(newCode: slice): void +fun mySetCodeAsm(newCode: slice): void asm "SETCODE"; +fun mySetCode(newCode: slice) { + mySetCodeAsm(newCode); +} + fun testCodegen3(numberId: int, paramVal: cell) { if (numberId == -1000) { var cs = paramVal.beginParse(); @@ -288,7 +294,7 @@ fun main() { @testcase | 112 | 5 | 138 @testcase | 113 | | 2048 -@code_hash 100812963733861852648745958727639570706776197131971804296829155862118445447691 +@code_hash 91375323652563311931660098165979913104421063599680251769401655783153201389325 @fif_codegen """ diff --git a/tolk-tester/tests/union-types-tests.tolk b/tolk-tester/tests/union-types-tests.tolk index 422b08bdb..53e6caefa 100644 --- a/tolk-tester/tests/union-types-tests.tolk +++ b/tolk-tester/tests/union-types-tests.tolk @@ -535,6 +535,7 @@ fun test47() { return (checkSimple(1), checkSimple(beginCell().endCell().beginParse()), checkSimple(100), checkSimple(beginCell())); } +@noinline fun checkGeneric3(a: T1 | T2 | T3): (int, T1 | T2?) { var vv: T1? = match (a) { T1 => a, T2 => null, T3 => null }; return match (a) { diff --git a/tolk/CMakeLists.txt b/tolk/CMakeLists.txt index 2e0bf0cad..4db9233a9 100644 --- a/tolk/CMakeLists.txt +++ b/tolk/CMakeLists.txt @@ -22,6 +22,7 @@ set(TOLK_SOURCE pipe-check-serialized-fields.cpp pipe-constant-folding.cpp pipe-optimize-boolean-expr.cpp + pipe-detect-inline-in-place.cpp pipe-ast-to-legacy.cpp pipe-find-unused-symbols.cpp pipe-generate-fif-output.cpp diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 9a3a4e6fb..5c5953415 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -1347,6 +1347,7 @@ static V parse_annotation(Lexer& lex) { throw ParseError(loc, "unknown annotation " + static_cast(name)); case AnnotationKind::inline_simple: case AnnotationKind::inline_ref: + case AnnotationKind::noinline: case AnnotationKind::pure: if (v_arg) { throw ParseError(v_arg->loc, "arguments aren't allowed for " + static_cast(name)); @@ -1471,13 +1472,17 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vectorkind) { case AnnotationKind::inline_simple: - flags |= FunctionData::flagInline; + inline_mode = FunctionInlineMode::inlineViaFif; // maybe will be replaced by inlineInPlace later break; case AnnotationKind::inline_ref: - flags |= FunctionData::flagInlineRef; + inline_mode = FunctionInlineMode::inlineRef; + break; + case AnnotationKind::noinline: + inline_mode = FunctionInlineMode::noInline; break; case AnnotationKind::pure: flags |= FunctionData::flagMarkedAsPure; @@ -1502,7 +1507,7 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector(loc, v_ident, v_param_list, v_body, receiver_type, ret_type, genericsT_list, std::move(tvm_method_id), flags); + return createV(loc, v_ident, v_param_list, v_body, receiver_type, ret_type, genericsT_list, std::move(tvm_method_id), flags, inline_mode); } static AnyV parse_struct_field(Lexer& lex) { diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index 03a9b0a53..2aabec2b6 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -335,7 +335,8 @@ class ASTReplicator final { clone(v_orig->return_type_node), v_orig->genericsT_list ? clone(v_orig->genericsT_list) : nullptr, v_orig->tvm_method_id, - v_orig->flags + v_orig->flags, + v_orig->inline_mode ); } diff --git a/tolk/ast.cpp b/tolk/ast.cpp index 59a18a163..27c7e4c13 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -60,6 +60,9 @@ AnnotationKind Vertex::parse_kind(std::string_view name) { if (name == "@inline_ref") { return AnnotationKind::inline_ref; } + if (name == "@noinline") { + return AnnotationKind::noinline; + } if (name == "@method_id") { return AnnotationKind::method_id; } diff --git a/tolk/ast.h b/tolk/ast.h index cf82a0a17..400c849b1 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -138,6 +138,7 @@ enum ASTNodeKind { enum class AnnotationKind { inline_simple, inline_ref, + noinline, method_id, pure, deprecated, @@ -1243,6 +1244,7 @@ struct Vertex final : ASTOtherVararg { V genericsT_list; // for non-generics it's nullptr td::RefInt256 tvm_method_id; // specified via @method_id annotation int flags; // from enum in FunctionData + FunctionInlineMode inline_mode; // from annotations like `@inline` or auto-detected "in-place" bool is_asm_function() const { return children.at(2)->kind == ast_asm_body; } bool is_code_function() const { return children.at(2)->kind == ast_block_statement; } @@ -1251,9 +1253,9 @@ struct Vertex final : ASTOtherVararg { Vertex* mutate() const { return const_cast(this); } void assign_fun_ref(FunctionPtr fun_ref); - Vertex(SrcLocation loc, V name_identifier, V parameters, AnyV body, AnyTypeV receiver_type_node, AnyTypeV return_type_node, V genericsT_list, td::RefInt256 tvm_method_id, int flags) + Vertex(SrcLocation loc, V name_identifier, V parameters, AnyV body, AnyTypeV receiver_type_node, AnyTypeV return_type_node, V genericsT_list, td::RefInt256 tvm_method_id, int flags, FunctionInlineMode inline_mode) : ASTOtherVararg(ast_function_declaration, loc, {name_identifier, parameters, body}) - , receiver_type_node(receiver_type_node), return_type_node(return_type_node), genericsT_list(genericsT_list), tvm_method_id(std::move(tvm_method_id)), flags(flags) {} + , receiver_type_node(receiver_type_node), return_type_node(return_type_node), genericsT_list(genericsT_list), tvm_method_id(std::move(tvm_method_id)), flags(flags), inline_mode(inline_mode) {} }; template<> diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 3696d550e..d4db1edec 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -41,14 +41,14 @@ static std::vector define_builtin_parameters(const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const simple_compile_func_t& func, int flags) { - auto* f_sym = new FunctionData(name, {}, "", nullptr, return_type, define_builtin_parameters(params_types, flags), flags, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); + auto* f_sym = new FunctionData(name, {}, "", nullptr, return_type, define_builtin_parameters(params_types, flags), flags, FunctionInlineMode::notCalculated, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); G.symtable.add_function(f_sym); } static void define_builtin_method(const std::string& name, TypePtr receiver_type, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const simple_compile_func_t& func, int flags, std::initializer_list arg_order = {}, std::initializer_list ret_order = {}) { std::string method_name = name.substr(name.find('.') + 1); - auto* f_sym = new FunctionData(name, {}, std::move(method_name), receiver_type, return_type, define_builtin_parameters(params_types, flags), flags, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); + auto* f_sym = new FunctionData(name, {}, std::move(method_name), receiver_type, return_type, define_builtin_parameters(params_types, flags), flags, FunctionInlineMode::notCalculated, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); f_sym->arg_order = arg_order; f_sym->ret_order = ret_order; G.symtable.add_function(f_sym); @@ -1517,6 +1517,9 @@ void define_builtins() { define_builtin_func("__expect_type", {TypeDataUnknown::create(), Slice}, Unit, nullptr, compile_expect_type, FunctionData::flagMarkedAsPure); + define_builtin_func("__expect_inline", {Bool}, Unit, nullptr, + compile_expect_type, + FunctionData::flagMarkedAsPure); define_builtin_method("T.__toTuple", typeT, {typeT}, TypeDataTuple::create(), declReceiverT, compile_any_object_to_tuple, FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); diff --git a/tolk/codegen.cpp b/tolk/codegen.cpp index c635c7483..d392a58b8 100644 --- a/tolk/codegen.cpp +++ b/tolk/codegen.cpp @@ -517,7 +517,7 @@ bool Op::generate_code_step(Stack& stack) { } } } else { - if (f_sym->is_inline() || f_sym->is_inline_ref()) { + if (f_sym->inline_mode == FunctionInlineMode::inlineViaFif || f_sym->inline_mode == FunctionInlineMode::inlineRef) { stack.o << AsmOp::Custom(loc, f_sym->name + " INLINECALLDICT", (int)right.size(), (int)left.size()); } else if (f_sym->is_code_function() && std::get(f_sym->body)->code->require_callxargs) { stack.o << AsmOp::Custom(loc, f_sym->name + (" PREPAREDICT"), 0, 2); diff --git a/tolk/fwd-declarations.h b/tolk/fwd-declarations.h index 97cba7200..981eaa4a6 100644 --- a/tolk/fwd-declarations.h +++ b/tolk/fwd-declarations.h @@ -52,4 +52,12 @@ struct GenericsSubstitutions; struct SrcFile; +enum class FunctionInlineMode { + notCalculated, + inlineViaFif, + inlineRef, + inlineInPlace, + noInline, +}; + } // namespace tolk diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index c2c7a9b54..ae51cf752 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -370,7 +370,7 @@ FunctionPtr instantiate_generic_function(FunctionPtr fun_ref, GenericsSubstituti } TypePtr new_return_type = replace_genericT_with_deduced(fun_ref->declared_return_type, allocatedTs); TypePtr new_receiver_type = replace_genericT_with_deduced(fun_ref->receiver_type, allocatedTs); - FunctionData* new_fun_ref = new FunctionData(new_name, fun_ref->loc, fun_ref->method_name, new_receiver_type, new_return_type, std::move(new_parameters), fun_ref->flags, nullptr, allocatedTs, fun_ref->body, fun_ref->ast_root); + FunctionData* new_fun_ref = new FunctionData(new_name, fun_ref->loc, fun_ref->method_name, new_receiver_type, new_return_type, std::move(new_parameters), fun_ref->flags, fun_ref->inline_mode, nullptr, allocatedTs, fun_ref->body, fun_ref->ast_root); new_fun_ref->arg_order = fun_ref->arg_order; new_fun_ref->ret_order = fun_ref->ret_order; new_fun_ref->base_fun_ref = fun_ref; diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 8a6c2911b..f97d7a689 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -337,6 +337,32 @@ class CheckReorderingForAsmArgOrderIsSafeVisitor final : public ASTVisitorFuncti } }; +// when a call to `f()` was inlined, f's body was processed, leaving some state +// that should be cleared upon next inlining; +// for instance, ir_idx of local variables point to caller (where f was inlined) +class ClearStateAfterInlineInPlace final : public ASTVisitorFunctionBody { + void visit(V v) override { + if (!v->marked_as_redef) { + v->var_ref->mutate()->assign_ir_idx({}); + } + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + tolk_assert(false); + } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + tolk_assert(fun_ref->is_inlined_in_place()); + + for (int i = 0; i < fun_ref->get_num_params(); ++i) { + fun_ref->get_param(i).mutate()->assign_ir_idx({}); + } + + parent::visit(v_function->get_body()); + } +}; + // given `{some_expr}!`, return some_expr static AnyExprV unwrap_not_null_operator(AnyExprV v) { while (auto v_notnull = v->try_as()) { @@ -609,6 +635,54 @@ static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob tolk_assert(false); } +static std::vector gen_inline_fun_call_in_place(CodeBlob& code, TypePtr ret_type, V v_call, const std::vector>& vars_per_arg) { + FunctionPtr fun_ref = v_call->fun_maybe; + tolk_assert(vars_per_arg.size() == fun_ref->parameters.size()); + for (int i = 0; i < fun_ref->get_num_params(); ++i) { + const LocalVarData& param_i = fun_ref->get_param(i); + if (!param_i.is_used_as_lval() && !param_i.is_mutate_parameter()) { + // if param used for reading only, pass the same ir_idx as for an argument + // it decreases number of tmp variables and leads to better optimizations + // (being honest, it's quite strange that copy+LET may lead to more stack permutations) + param_i.mutate()->assign_ir_idx(std::vector(vars_per_arg[i])); + } else { + std::vector ith_param = code.create_var(param_i.declared_type, v_call->loc, param_i.name); + code.emplace_back(v_call->loc, Op::_Let, ith_param, vars_per_arg[i]); + param_i.mutate()->assign_ir_idx(std::move(ith_param)); + } + } + + std::vector rvect_call = code.create_tmp_var(ret_type, v_call->loc, "(inlined-return)"); + std::vector* backup_outer_inline = code.inline_rvect_out; + FunctionPtr backup_cur_fun = code.fun_ref; + bool backup_inline_before_return = code.inlining_before_immediate_return; + code.inline_rvect_out = &rvect_call; + code.inlining_before_immediate_return = stmt_before_immediate_return == v_call; + code.fun_ref = fun_ref; + + auto v_ast_root = fun_ref->ast_root->as(); + auto v_block = v_ast_root->get_body()->as(); + process_any_statement(v_block, code); + + if (fun_ref->has_mutate_params() && fun_ref->inferred_return_type == TypeDataVoid::create()) { + std::vector mutated_vars; + for (const LocalVarData& p_sym: fun_ref->parameters) { + if (p_sym.is_mutate_parameter()) { + mutated_vars.insert(mutated_vars.end(), p_sym.ir_idx.begin(), p_sym.ir_idx.end()); + } + } + code.emplace_back(v_call->loc, Op::_Let, rvect_call, std::move(mutated_vars)); + } + + ClearStateAfterInlineInPlace visitor; + visitor.start_visiting_function(fun_ref, v_ast_root); + + code.fun_ref = backup_cur_fun; + code.inline_rvect_out = backup_outer_inline; + code.inlining_before_immediate_return = backup_inline_before_return; + return rvect_call; +} + // "Transition to target (runtime) type" is the following process. // Imagine `fun analyze(t: (int,int)?)` and a call `analyze((1,2))`. // `(1,2)` (inferred_type) is 2 stack slots, but `t` (target_type) is 3 (one for null-flag). @@ -1468,9 +1542,14 @@ static std::vector process_function_call(V v, Code for (const std::vector& list : vars_per_arg) { args_vars.insert(args_vars.end(), list.cbegin(), list.cend()); } - std::vector rvect_apply = fun_ref->is_compile_time_special_gen() - ? gen_compile_time_code_instead_of_fun_call(code, v->loc, vars_per_arg, fun_ref) - : gen_op_call(code, op_call_type, v->loc, std::move(args_vars), fun_ref, "(fun-call)", arg_order_already_equals_asm); + std::vector rvect_call; + if (fun_ref->is_compile_time_special_gen()) { + rvect_call = gen_compile_time_code_instead_of_fun_call(code, v->loc, vars_per_arg, fun_ref); + } else if (fun_ref->is_inlined_in_place() && fun_ref->is_code_function()) { + rvect_call = gen_inline_fun_call_in_place(code, op_call_type, v, vars_per_arg); + } else { + rvect_call = gen_op_call(code, op_call_type, v->loc, std::move(args_vars), fun_ref, "(fun-call)", arg_order_already_equals_asm); + } if (fun_ref->has_mutate_params()) { LValContext local_lval; @@ -1491,20 +1570,20 @@ static std::vector process_function_call(V v, Code std::vector rvect = code.create_tmp_var(real_ret_type, v->loc, "(fun-call)"); left.insert(left.end(), rvect.begin(), rvect.end()); vars_modification_watcher.trigger_callbacks(left, v->loc); - code.emplace_back(v->loc, Op::_Let, left, rvect_apply); + code.emplace_back(v->loc, Op::_Let, left, rvect_call); local_lval.after_let(std::move(left), code, v->loc); - rvect_apply = rvect; + rvect_call = rvect; } if (obj_leftmost && fun_ref->does_return_self()) { if (obj_leftmost->is_lvalue) { // to handle if obj is global var, potentially re-assigned inside a chain - rvect_apply = pre_compile_expr(obj_leftmost, code, nullptr); + rvect_call = pre_compile_expr(obj_leftmost, code, nullptr); } else { // temporary object, not lvalue, pre_compile_expr - rvect_apply = vars_per_arg[asm_self_idx]; + rvect_call = vars_per_arg[asm_self_idx]; } } - return transition_to_target_type(std::move(rvect_apply), code, target_type, v); + return transition_to_target_type(std::move(rvect_call), code, target_type, v); } static std::vector process_braced_expression(V v, CodeBlob& code, TypePtr target_type) { @@ -1772,6 +1851,7 @@ static void process_block_statement(V v, CodeBlob& code) { FunctionPtr cur_f = code.fun_ref; bool does_f_return_nothing = cur_f->inferred_return_type == TypeDataVoid::create() && !cur_f->does_return_self() && !cur_f->has_mutate_params(); bool is_toplevel_block = v == cur_f->ast_root->as()->get_body(); + bool inlining_doesnt_prevent = code.inline_rvect_out == nullptr || code.inlining_before_immediate_return; // we want to optimize `match` and `if/else`: if it's the last statement, implicitly add "return" to every branch // (to generate IFJMP instead of nested IF ELSE); @@ -1782,11 +1862,11 @@ static void process_block_statement(V v, CodeBlob& code) { AnyV stmt = v->get_item(i); AnyV next_stmt = v->get_item(i + 1); bool next_is_empty_return = next_stmt->kind == ast_return_statement && !next_stmt->as()->has_return_value(); - stmt_before_immediate_return = next_is_empty_return && does_f_return_nothing ? stmt : nullptr; + stmt_before_immediate_return = next_is_empty_return && does_f_return_nothing && inlining_doesnt_prevent ? stmt : nullptr; process_any_statement(stmt, code); } AnyV last_stmt = v->get_item(v->size() - 1); - stmt_before_immediate_return = is_toplevel_block && does_f_return_nothing ? last_stmt : nullptr; + stmt_before_immediate_return = is_toplevel_block && does_f_return_nothing && inlining_doesnt_prevent ? last_stmt : nullptr; process_any_statement(last_stmt, code); stmt_before_immediate_return = backup; } @@ -1941,25 +2021,35 @@ static void process_throw_statement(V v, CodeBlob& code) { } static void process_return_statement(V v, CodeBlob& code) { - TypePtr child_target_type = code.fun_ref->inferred_return_type; - if (code.fun_ref->does_return_self()) { - child_target_type = code.fun_ref->parameters[0].declared_type; + // it's a function we're traversing AST of; + // probably, it's called and inlined into another (outer) function, we handle this below + FunctionPtr fun_ref = code.fun_ref; + + TypePtr child_target_type = fun_ref->inferred_return_type; + if (fun_ref->does_return_self()) { + child_target_type = fun_ref->parameters[0].declared_type; } std::vector return_vars = pre_compile_expr(v->get_return_value(), code, child_target_type); - if (code.fun_ref->does_return_self()) { + if (fun_ref->does_return_self()) { return_vars = {}; } - if (code.fun_ref->has_mutate_params()) { + if (fun_ref->has_mutate_params()) { std::vector mutated_vars; - for (const LocalVarData& p_sym: code.fun_ref->parameters) { + for (const LocalVarData& p_sym: fun_ref->parameters) { if (p_sym.is_mutate_parameter()) { mutated_vars.insert(mutated_vars.end(), p_sym.ir_idx.begin(), p_sym.ir_idx.end()); } } return_vars.insert(return_vars.begin(), mutated_vars.begin(), mutated_vars.end()); } - code.emplace_back(v->loc, Op::_Return, std::move(return_vars)); + + // if fun_ref is called and inlined into a parent, assign a result instead of generating a return statement + if (code.inline_rvect_out) { + code.emplace_back(v->loc, Op::_Let, *code.inline_rvect_out, std::move(return_vars)); + } else { + code.emplace_back(v->loc, Op::_Return, std::move(return_vars)); + } } // append "return" (void) to the end of the function @@ -2123,7 +2213,7 @@ class ConvertASTToLegacyOpVisitor final { static void start_visiting_function(FunctionPtr fun_ref, V) { tolk_assert(fun_ref->is_type_inferring_done()); - if (fun_ref->is_code_function()) { + if (fun_ref->is_code_function() && !fun_ref->is_inlined_in_place()) { convert_function_body_to_CodeBlob(fun_ref, std::get(fun_ref->body)); } else if (fun_ref->is_asm_function()) { convert_asm_body_to_AsmOp(fun_ref, std::get(fun_ref->body)); diff --git a/tolk/pipe-check-rvalue-lvalue.cpp b/tolk/pipe-check-rvalue-lvalue.cpp index be4db2749..075e99982 100644 --- a/tolk/pipe-check-rvalue-lvalue.cpp +++ b/tolk/pipe-check-rvalue-lvalue.cpp @@ -37,11 +37,11 @@ static void fire_error_cannot_be_used_as_lvalue(FunctionPtr cur_f, AnyV v, const } GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_modifying_immutable_variable(FunctionPtr cur_f, AnyExprV v, LocalVarPtr var_ref) { +static void fire_error_modifying_immutable_variable(FunctionPtr cur_f, SrcLocation loc, LocalVarPtr var_ref) { if (var_ref->param_idx == 0 && var_ref->name == "self") { - throw ParseError(cur_f, v->loc, "modifying `self`, which is immutable by default; probably, you want to declare `mutate self`"); + throw ParseError(cur_f, loc, "modifying `self`, which is immutable by default; probably, you want to declare `mutate self`"); } else { - throw ParseError(cur_f, v->loc, "modifying immutable variable `" + var_ref->name + "`"); + throw ParseError(cur_f, loc, "modifying immutable variable `" + var_ref->name + "`"); } } @@ -59,6 +59,13 @@ static void validate_function_used_as_noncall(FunctionPtr cur_f, AnyExprV v, Fun class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { FunctionPtr cur_f = nullptr; + void on_var_used_as_lvalue(SrcLocation loc, LocalVarPtr var_ref) const { + if (var_ref->is_immutable()) { + fire_error_modifying_immutable_variable(cur_f, loc, var_ref); + } + var_ref->mutate()->assign_used_as_lval(); + } + void visit(V v) override { if (v->is_lvalue) { fire_error_cannot_be_used_as_lvalue(cur_f, v, "braced expression"); @@ -162,8 +169,8 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { } if (auto as_ref = leftmost_obj->try_as()) { - if (LocalVarPtr var_ref = as_ref->sym->try_as(); var_ref && var_ref->is_immutable()) { - fire_error_modifying_immutable_variable(cur_f, leftmost_obj, var_ref); + if (LocalVarPtr var_ref = as_ref->sym->try_as()) { + on_var_used_as_lvalue(leftmost_obj->loc, var_ref); } } } @@ -211,8 +218,8 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { if (v->is_lvalue) { tolk_assert(v->sym); - if (LocalVarPtr var_ref = v->sym->try_as(); var_ref && var_ref->is_immutable()) { - fire_error_modifying_immutable_variable(cur_f, v, var_ref); + if (LocalVarPtr var_ref = v->sym->try_as()) { + on_var_used_as_lvalue(v->loc, var_ref); } else if (v->sym->try_as()) { fire(cur_f, v->loc, "modifying immutable constant"); } else if (v->sym->try_as()) { diff --git a/tolk/pipe-detect-inline-in-place.cpp b/tolk/pipe-detect-inline-in-place.cpp new file mode 100644 index 000000000..21001c2aa --- /dev/null +++ b/tolk/pipe-detect-inline-in-place.cpp @@ -0,0 +1,299 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-visitor.h" +#include + +/* + * This pipe detects whether each function can be inlined in-place or not. + * Outcome: call `fun_ref->assign_inline_mode_in_place()` for "lightweight" or "called only once" functions, + * and they will be inlined in-place while converting AST to IR (to Ops), and won't be generated to Fift. + * + * Given AST only, there is no definite algorithm to predict whether a function is "simple" ("lightweight"), + * so that inlining it will do better. There are no correct metrics at AST level that can be mapped onto TVM complexity. + * So, instead of overcomplicating and fine-tuning an algorithm, we're heading a simple way: + * - if a function is tiny, inline it always + * - if a function is called only once, inline it (only there, obviously) + * - if a function is marked `@inline` (intended by the user), inline it in place (if possible) + * - see `should_auto_inline_if_not_prevented()` + * + * What can prevent a function from inlining? Several reasons: + * - it's recursive + * - it's used as non-call (a reference to it is taken) + * - see `is_inlining_prevented_even_if_annotated()` + * + * About `@inline` annotation. It means "user intention", so the compiler tries to inline it in-place + * without considering AST metrics. But anyway, something may prevent inlining (middle returns, for example). + * In this case, the desired flag is just not set; inline_mode remains inlineViaFif, we'll generate `PROCINLINE`. + * + * Besides inline detection, this pipe populates `fun_ref->n_times_called` (while building call graph). + * It's used in Fift output inside comments. + */ + +namespace tolk { + +// to calculate recursions, at first we populate the call graph +// (purpose: functions in recursive call chains can't be inlined) +static std::unordered_map> call_graph; + +// when traversing a function, collect some AST metrics used to detect whether it's lightweight +struct StateWhileTraversingFunction { + FunctionPtr fun_ref; + bool has_returns_in_the_middle = false; + int n_statements = 0; + int n_function_calls = 0; + int n_binary_operators = 0; + int n_control_flow = 0; + int n_globals = 0; + int max_block_depth = 0; + + explicit StateWhileTraversingFunction(FunctionPtr fun_ref) + : fun_ref(fun_ref) {} + + int calculate_ast_cost() const { + return n_function_calls + n_binary_operators + n_statements * 2 + + n_control_flow * 10 + n_globals * 5 + (max_block_depth - 1) * 10; + } + + bool is_inlining_prevented_even_if_annotated() const { + // even if user specified `@inline`, we can't do anything about recursions, for example; + // in this case, in-place inlining won't happen, we'll generate `PROCINLINE` to Fift + bool is_inside_recursion = fun_ref->n_times_called >= 9999; + return has_returns_in_the_middle || is_inside_recursion || fun_ref->is_used_as_noncall() || !fun_ref->is_code_function(); + } + + bool should_auto_inline_if_not_prevented() const { + // if a function is called only once, inline it regardless of its size + // (to prevent this, `@inline_ref` can be used, for example) + if (fun_ref->n_times_called == 1) { + return true; + } + + // if a function is lightweight, inline in regardless of how many times it's called + // (for instance, `Storage.load` is always inlined) + int approx_cost_per_call = calculate_ast_cost(); + if (approx_cost_per_call < 30) { + return true; + } + + // try to _somehow_ detect whether to inline it or not + return approx_cost_per_call * fun_ref->n_times_called < 150; + } +}; + +// traverse the AST, collect metrics, and in the end, probably set the inline flag +class DetectIfToInlineFunctionInPlaceVisitor final : ASTVisitorFunctionBody { + StateWhileTraversingFunction cur_state{nullptr}; + int block_depth = 0; + std::vector> collected_expect_inline; // `__expect_inline()` compiler assertions + +protected: + void visit(V v) override { + if (v->fun_maybe && v->fun_maybe->is_builtin_function() && v->fun_maybe->name == "__expect_inline") { + collected_expect_inline.push_back(v); + } else { + cur_state.n_function_calls++; + } + parent::visit(v); + } + + void visit(V v) override { + if (v->tok == tok_logical_and || v->tok == tok_logical_not) { + cur_state.n_control_flow++; + } else { + cur_state.n_binary_operators++; + } + parent::visit(v); + } + + void visit(V v) override { + if (v->sym->try_as()) { + cur_state.n_globals++; + } + } + + void visit(V v) override { + block_depth++; + cur_state.n_statements += v->size(); + cur_state.max_block_depth = std::max(cur_state.max_block_depth, block_depth); + parent::visit(v); + block_depth--; + } + + void visit(V v) override { + cur_state.n_control_flow++; + parent::visit(v); + } + + void visit(V v) override { + cur_state.n_control_flow++; + parent::visit(v); + } + + void visit(V v) override { + cur_state.n_control_flow++; + parent::visit(v); + } + + void visit(V v) override { + cur_state.n_control_flow++; + parent::visit(v); + } + + void visit(V v) override { + cur_state.n_control_flow++; + parent::visit(v); + } + + void visit(V v) override { + cur_state.n_control_flow++; + parent::visit(v); + } + + void visit(V v) override { + cur_state.n_control_flow++; + parent::visit(v); + } + + void visit(V v) override { + cur_state.n_control_flow++; + parent::visit(v); + } + + void visit(V v) override { + // detect if `return` the last return statement in a function's body + // (currently in-place inlining for functions with returns in the middle is not supported) + auto body_block = cur_state.fun_ref->ast_root->as()->get_body()->as(); + bool is_last_statement = body_block->get_item(body_block->size() - 1) == v; + cur_state.has_returns_in_the_middle |= !is_last_statement; + parent::visit(v); + } + + public: + bool should_visit_function(FunctionPtr fun_ref) override { + // unsupported or no-sense cases + if (fun_ref->is_builtin_function() || fun_ref->is_asm_function() || fun_ref->is_generic_function() || + fun_ref->has_tvm_method_id() || !fun_ref->arg_order.empty() || !fun_ref->ret_order.empty() || + fun_ref->is_used_as_noncall()) { + return false; + } + // disabled by the user + if (fun_ref->inline_mode == FunctionInlineMode::noInline || fun_ref->inline_mode == FunctionInlineMode::inlineRef) { + return false; + } + // okay, start auto-detection; + // for functions marked `@inline` (inlineViaFif), probably we'll change to inlineInPlace + return true; + } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_state = StateWhileTraversingFunction(fun_ref); + collected_expect_inline.clear(); + parent::visit(v_function->get_body()); + + bool prevented_anyway = cur_state.is_inlining_prevented_even_if_annotated(); + bool will_inline = false; + if (fun_ref->inline_mode == FunctionInlineMode::inlineViaFif) { + // if a function is marked `@inline`, so the user requested in to be inlined; + // if it's possible, do it; otherwise, leave it as `PROCINLINE` to Fift + will_inline = !prevented_anyway; + } else { + // a function is not marked `@inline` / `@inline_ref` / etc., so automatically decide + will_inline = !prevented_anyway && cur_state.should_auto_inline_if_not_prevented(); + } + + // handle `__expect_inline(true)` (assertions inside compiler tests) + for (auto v_expect : collected_expect_inline) { + tolk_assert(v_expect->get_num_args() == 1 && v_expect->get_arg(0)->get_expr()->kind == ast_bool_const); + bool expected = v_expect->get_arg(0)->get_expr()->as()->bool_val; + if (expected != will_inline) { + fire(fun_ref, v_expect->loc, "__expect_inline failed"); + } + } + + // okay, this function will be inlined, mark the flag + if (will_inline && fun_ref->n_times_called) { + fun_ref->mutate()->assign_inline_mode_in_place(); + } + } +}; + +// this visitor (called once for a function): +// 1) fills call_graph[cur_f] (all function calls from cur_f) +// 2) increments n_times_called +// as a result of applying it to every function, we get a full call graph and how many times each function was called; +// we'll use this call graph to detect recursive components (functions within recursions can not be inlined) +class CallGraphBuilderVisitor final : ASTVisitorFunctionBody { + FunctionPtr cur_f{nullptr}; + +protected: + void visit(V v) override { + if (FunctionPtr called_f = v->fun_maybe) { + if (called_f->is_code_function()) { + call_graph[cur_f].emplace_back(called_f); + } + called_f->mutate()->n_times_called++; + } + parent::visit(v); + } + + public: + bool should_visit_function(FunctionPtr fun_ref) override { + // don't include asm functions, we don't need them in calculations + return fun_ref->is_code_function() && !fun_ref->is_generic_function(); + } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + call_graph[cur_f] = std::vector{}; + parent::visit(v_function->get_body()); + } +}; + +static void detect_recursive_functions() { + // 1) build call_graph (and calculate n_times_called also) + visit_ast_of_all_functions(); + + // 2) using call_graph, detect cycles (the smallest, non-optimized algorithm, okay for our needs) + for (const auto& it : call_graph) { + FunctionPtr f_start_from = it.first; + std::unordered_set visited; + std::function is_recursive_dfs = [&](FunctionPtr cur) -> bool { + for (FunctionPtr f_called : call_graph[cur]) { + if (f_called == f_start_from) + return true; + if (!visited.insert(f_called).second) + continue; + if (is_recursive_dfs(f_called)) + return true; + } + return false; + }; + if (!it.second.empty() && is_recursive_dfs(f_start_from)) { + f_start_from->mutate()->n_times_called = 9999; // means "recursive" + } + } +} + +void pipeline_detect_inline_in_place() { + detect_recursive_functions(); + visit_ast_of_all_functions(); + call_graph.clear(); +} + +} // namespace tolk diff --git a/tolk/pipe-find-unused-symbols.cpp b/tolk/pipe-find-unused-symbols.cpp index cce03cc90..dad565e0e 100644 --- a/tolk/pipe-find-unused-symbols.cpp +++ b/tolk/pipe-find-unused-symbols.cpp @@ -37,7 +37,7 @@ namespace tolk { static void mark_function_used_dfs(const std::unique_ptr& op); static void mark_function_used(FunctionPtr fun_ref) { - if (!fun_ref->is_code_function() || fun_ref->is_really_used()) { // already handled + if (!fun_ref->is_code_function() || fun_ref->is_really_used() || fun_ref->is_inlined_in_place()) { // already handled return; } diff --git a/tolk/pipe-generate-fif-output.cpp b/tolk/pipe-generate-fif-output.cpp index 6cfd50683..ca82cd3fb 100644 --- a/tolk/pipe-generate-fif-output.cpp +++ b/tolk/pipe-generate-fif-output.cpp @@ -79,13 +79,17 @@ static void generate_output_func(FunctionPtr fun_ref) { std::cerr << "\n---------- resulting code for " << fun_ref->name << " -------------\n"; } const char* modifier = ""; - if (fun_ref->is_inline()) { + if (fun_ref->inline_mode == FunctionInlineMode::inlineViaFif) { modifier = "INLINE"; - } else if (fun_ref->is_inline_ref()) { + } else if (fun_ref->inline_mode == FunctionInlineMode::inlineRef) { modifier = "REF"; } if (G.settings.tolk_src_as_line_comments) { - std::cout << " // " << fun_ref->loc << std::endl; + std::cout << " // " << fun_ref->loc; + if (!fun_ref->n_times_called && !fun_ref->is_used_as_noncall() && !fun_ref->is_entrypoint() && !fun_ref->has_tvm_method_id()) { + std::cout << " (note: function never called!)"; + } + std::cout << std::endl; } std::cout << " " << fun_ref->name << " PROC" << modifier << ":<{"; int mode = 0; @@ -103,10 +107,10 @@ static void generate_output_func(FunctionPtr fun_ref) { if (G.settings.tolk_src_as_line_comments) { mode |= Stack::_LineComments; } - if (fun_ref->is_inline() && code->ops->noreturn()) { + if (fun_ref->inline_mode == FunctionInlineMode::inlineViaFif && code->ops->noreturn()) { mode |= Stack::_InlineFunc; } - if (fun_ref->is_inline() || fun_ref->is_inline_ref()) { + if (fun_ref->inline_mode == FunctionInlineMode::inlineViaFif || fun_ref->inline_mode == FunctionInlineMode::inlineRef) { mode |= Stack::_InlineAny; } code->generate_code(std::cout, mode, 2); @@ -133,11 +137,13 @@ void pipeline_generate_fif_output_to_std_cout() { std::cout << "PROGRAM{\n"; bool has_main_procedure = false; + int n_inlined_in_place = 0; for (FunctionPtr fun_ref : G.all_functions) { - if (!fun_ref->does_need_codegen()) { + if (fun_ref->is_asm_function() || !fun_ref->does_need_codegen()) { if (G.is_verbosity(2) && fun_ref->is_code_function()) { std::cerr << fun_ref->name << ": code not generated, function does not need codegen\n"; } + n_inlined_in_place += fun_ref->is_inlined_in_place(); continue; } @@ -157,6 +163,15 @@ void pipeline_generate_fif_output_to_std_cout() { throw Fatal("the contract has no entrypoint; forgot `fun onInternalMessage(...)`?"); } + if (n_inlined_in_place) { + std::cout << " // " << n_inlined_in_place << " functions inlined in-place:" << "\n"; + for (FunctionPtr fun_ref : G.all_functions) { + if (fun_ref->is_inlined_in_place()) { + std::cout << " // - " << fun_ref->name << " (" << fun_ref->n_times_called << (fun_ref->n_times_called == 1 ? " call" : " calls") << ")\n"; + } + } + } + for (GlobalVarPtr var_ref : G.all_global_vars) { if (!var_ref->is_really_used() && G.settings.remove_unused_functions) { if (G.is_verbosity(2)) { @@ -169,7 +184,7 @@ void pipeline_generate_fif_output_to_std_cout() { } for (FunctionPtr fun_ref : G.all_functions) { - if (!fun_ref->does_need_codegen()) { + if (fun_ref->is_asm_function() || !fun_ref->does_need_codegen()) { continue; } generate_output_func(fun_ref); diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index 7eaa6ac91..b4394a2d8 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -215,7 +215,7 @@ static FunctionPtr register_function(V v, FunctionPtr const GenericsDeclaration* genericTs = nullptr; // at registering it's null; will be assigned after types resolving FunctionBody f_body = v->get_body()->kind == ast_block_statement ? static_cast(new FunctionBodyCode) : static_cast(new FunctionBodyAsm); - FunctionData* f_sym = new FunctionData(std::move(name), v->loc, std::move(method_name), v->receiver_type_node, v->return_type_node, std::move(parameters), 0, genericTs, substitutedTs, f_body, v); + FunctionData* f_sym = new FunctionData(std::move(name), v->loc, std::move(method_name), v->receiver_type_node, v->return_type_node, std::move(parameters), 0, v->inline_mode, genericTs, substitutedTs, f_body, v); f_sym->base_fun_ref = base_fun_ref; // for `f`, here is `f` if (auto v_asm = v->get_body()->try_as()) { diff --git a/tolk/pipeline.h b/tolk/pipeline.h index 4e5beca20..e4448deb1 100644 --- a/tolk/pipeline.h +++ b/tolk/pipeline.h @@ -44,6 +44,7 @@ void pipeline_check_pure_impure_operations(); void pipeline_check_serialized_fields(); void pipeline_constant_folding(); void pipeline_optimize_boolean_expressions(); +void pipeline_detect_inline_in_place(); void pipeline_convert_ast_to_legacy_Expr_Op(); void pipeline_find_unused_symbols(); diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 46d394938..b9b4b1fa8 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -60,6 +60,10 @@ bool FunctionData::does_need_codegen() const { if (is_generic_function()) { return false; } + // if calls to this function were inlined in place, the function itself is omitted from fif + if (is_inlined_in_place()) { + return false; + } // currently, there is no inlining, all functions are codegenerated // (but actually, unused ones are later removed by Fift) // in the future, we may want to implement a true AST inlining for "simple" functions @@ -105,6 +109,10 @@ void FunctionData::assign_is_really_used() { this->flags |= flagReallyUsed; } +void FunctionData::assign_inline_mode_in_place() { + this->inline_mode = FunctionInlineMode::inlineInPlace; +} + void FunctionData::assign_arg_order(std::vector&& arg_order) { this->arg_order = std::move(arg_order); } @@ -129,6 +137,10 @@ void GlobalConstData::assign_init_value(AnyExprV init_value) { this->init_value = init_value; } +void LocalVarData::assign_used_as_lval() { + this->flags |= flagUsedAsLVal; +} + void LocalVarData::assign_ir_idx(std::vector&& ir_idx) { this->ir_idx = std::move(ir_idx); } diff --git a/tolk/symtable.h b/tolk/symtable.h index 15a52d148..243c38a60 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -50,6 +50,7 @@ struct LocalVarData final : Symbol { flagMutateParameter = 1, // parameter was declared with `mutate` keyword flagImmutable = 2, // variable was declared via `val` (not `var`) flagLateInit = 4, // variable was declared via `lateinit` (not assigned at declaration) + flagUsedAsLVal = 8, // variable is assigned or in another way used as lvalue inside a function }; AnyTypeV type_node; // either at declaration `var x:int`, or if omitted, from assigned value `var x=2` @@ -80,9 +81,11 @@ struct LocalVarData final : Symbol { bool is_immutable() const { return flags & flagImmutable; } bool is_lateinit() const { return flags & flagLateInit; } bool is_mutate_parameter() const { return flags & flagMutateParameter; } + bool is_used_as_lval() const { return flags & flagUsedAsLVal; } bool has_default_value() const { return default_value != nullptr; } LocalVarData* mutate() const { return const_cast(this); } + void assign_used_as_lval(); void assign_ir_idx(std::vector&& ir_idx); void assign_resolved_type(TypePtr declared_type); void assign_inferred_type(TypePtr inferred_type); @@ -104,8 +107,6 @@ struct FunctionData final : Symbol { static constexpr int EMPTY_TVM_METHOD_ID = -10; enum { - flagInline = 1, // marked `@inline` - flagInlineRef = 2, // marked `@inline_ref` flagTypeInferringDone = 4, // type inferring step of function's body (all AST nodes assigning v->inferred_type) is done flagUsedAsNonCall = 8, // used not only as `f()`, but as a 1-st class function (assigned to var, pushed to tuple, etc.) flagMarkedAsPure = 16, // declared as `pure`, can't call impure and access globals, unused invocations are optimized out @@ -123,6 +124,8 @@ struct FunctionData final : Symbol { int tvm_method_id = EMPTY_TVM_METHOD_ID; int flags; + FunctionInlineMode inline_mode; + int n_times_called = 0; // calculated while building call graph; 9999 for recursions std::string method_name; // for `fun Container.store` here is "store" AnyTypeV receiver_type_node; // for `fun Container.store` here is `Container` @@ -141,9 +144,10 @@ struct FunctionData final : Symbol { FunctionBody body; AnyV ast_root; // V for user-defined (not builtin) - FunctionData(std::string name, SrcLocation loc, std::string method_name, AnyTypeV receiver_type_node, AnyTypeV return_type_node, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) + FunctionData(std::string name, SrcLocation loc, std::string method_name, AnyTypeV receiver_type_node, AnyTypeV return_type_node, std::vector parameters, int initial_flags, FunctionInlineMode inline_mode, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) : Symbol(std::move(name), loc) , flags(initial_flags) + , inline_mode(inline_mode) , method_name(std::move(method_name)) , receiver_type_node(receiver_type_node) , parameters(std::move(parameters)) @@ -153,9 +157,10 @@ struct FunctionData final : Symbol { , body(body) , ast_root(ast_root) { } - FunctionData(std::string name, SrcLocation loc, std::string method_name, TypePtr receiver_type, TypePtr declared_return_type, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) + FunctionData(std::string name, SrcLocation loc, std::string method_name, TypePtr receiver_type, TypePtr declared_return_type, std::vector parameters, int initial_flags, FunctionInlineMode inline_mode, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) : Symbol(std::move(name), loc) , flags(initial_flags) + , inline_mode(inline_mode) , method_name(std::move(method_name)) , receiver_type_node(nullptr) , receiver_type(receiver_type) @@ -189,8 +194,7 @@ struct FunctionData final : Symbol { bool is_generic_function() const { return genericTs != nullptr; } bool is_instantiation_of_generic_function() const { return substitutedTs != nullptr; } - bool is_inline() const { return flags & flagInline; } - bool is_inline_ref() const { return flags & flagInlineRef; } + bool is_inlined_in_place() const { return inline_mode == FunctionInlineMode::inlineInPlace; } bool is_type_inferring_done() const { return flags & flagTypeInferringDone; } bool is_used_as_noncall() const { return flags & flagUsedAsNonCall; } bool is_marked_as_pure() const { return flags & flagMarkedAsPure; } @@ -218,6 +222,7 @@ struct FunctionData final : Symbol { void assign_is_implicit_return(); void assign_is_type_inferring_done(); void assign_is_really_used(); + void assign_inline_mode_in_place(); void assign_arg_order(std::vector&& arg_order); }; diff --git a/tolk/tolk.cpp b/tolk/tolk.cpp index 6ead2b149..6f8bbba91 100644 --- a/tolk/tolk.cpp +++ b/tolk/tolk.cpp @@ -67,6 +67,7 @@ int tolk_proceed(const std::string &entrypoint_filename) { pipeline_check_serialized_fields(); pipeline_constant_folding(); pipeline_optimize_boolean_expressions(); + pipeline_detect_inline_in_place(); pipeline_convert_ast_to_legacy_Expr_Op(); pipeline_find_unused_symbols(); diff --git a/tolk/tolk.h b/tolk/tolk.h index a329b1d55..3f6dcadd8 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -810,7 +810,7 @@ bool apply_op(StackTransform& trans, const AsmOp& op); */ struct Optimizer { - static constexpr int optimize_depth = 20; + static constexpr int optimize_depth = 30; AsmOpConsList code_; int l_{0}, l2_{0}, p_, pb_, q_, indent_; bool debug_{false}; @@ -1048,6 +1048,8 @@ struct CodeBlob { std::string name; SrcLocation forced_loc; std::vector vars; + std::vector* inline_rvect_out = nullptr; + bool inlining_before_immediate_return = false; std::unique_ptr ops; std::unique_ptr* cur_ops; #ifdef TOLK_DEBUG From 37c9fcd6608ce5dbe9a2569190725c64b8ab9a97 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sun, 15 Jun 2025 12:41:03 +0700 Subject: [PATCH 329/388] [Tolk] Change STSLICER to STSLICE in stdlib In real-world code, it requires fewer stack permutations, because slices are not constant --- crypto/smartcont/tolk-stdlib/common.tolk | 6 +++--- tolk-tester/tests/pack-unpack-6.tolk | 10 ++++------ 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 30aabed3a..86995717f 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -759,12 +759,12 @@ fun builder.storeUint(mutate self, x: int, len: int): self /// Stores a slice into a builder. @pure fun builder.storeSlice(mutate self, s: slice): self - asm "STSLICER"; + asm(s self) "STSLICE"; /// Stores an address into a builder. @pure fun builder.storeAddress(mutate self, addr: address): self - asm "STSLICER"; + asm(addr self) "STSLICE"; /// Stores amount of Toncoins into a builder. @pure @@ -792,7 +792,7 @@ fun builder.storeMaybeRef(mutate self, c: cell?): self /// Concatenates two builders. @pure fun builder.storeBuilder(mutate self, from: builder): self - asm "STBR"; + asm(from self) "STB"; /// Stores a slice representing TL addr_none$00 (two `0` bits). @pure diff --git a/tolk-tester/tests/pack-unpack-6.tolk b/tolk-tester/tests/pack-unpack-6.tolk index 07ec41f85..34e40578c 100644 --- a/tolk-tester/tests/pack-unpack-6.tolk +++ b/tolk-tester/tests/pack-unpack-6.tolk @@ -131,9 +131,9 @@ RETALT @fif_codegen """ test3 PROC:<{ - NEWC x{0109ab} PUSHSLICE - STSLICER + NEWC + STSLICE ENDC CTOS x{01} SDBEGINSQ @@ -168,8 +168,7 @@ IF:<{ 9 THROWIF 8 EQINT 9 THROWIFNOT - SWAP - STSLICER + STSLICE """ @fif_codegen @@ -177,8 +176,7 @@ IF:<{ test6 PROC:<{ x{11} PUSHSLICE NEWC - SWAP - STSLICER + STSLICE ENDC HASHCU """ From bce0e5e797de86f7ef8f243059c13bb10d12f73b Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sun, 15 Jun 2025 22:19:53 +0700 Subject: [PATCH 330/388] [Tolk] The magic `lazy` keyword Lazy loading, partial loading, partial updating See pipe-lazy-load-insertions.cpp for comments --- crypto/smartcont/tolk-stdlib/common.tolk | 34 +- .../tests/invalid-semantics/err-4460.tolk | 14 + .../tests/invalid-serialization/err-7300.tolk | 15 + .../tests/invalid-serialization/err-7301.tolk | 14 + .../tests/invalid-serialization/err-7302.tolk | 16 + .../tests/invalid-serialization/err-7303.tolk | 11 + .../tests/invalid-serialization/err-7304.tolk | 22 + .../tests/invalid-serialization/err-7305.tolk | 19 + .../tests/invalid-serialization/err-7306.tolk | 17 + .../tests/invalid-serialization/err-7307.tolk | 17 + .../tests/invalid-serialization/err-7308.tolk | 18 + .../tests/invalid-serialization/err-7683.tolk | 12 + .../tests/invalid-serialization/err-7684.tolk | 11 + .../tests/invalid-serialization/err-7720.tolk | 15 + .../tests/invalid-serialization/err-7800.tolk | 17 + .../tests/invalid-serialization/err-7923.tolk | 14 + .../tests/invalid-typing/err-6366.tolk | 1 + tolk-tester/tests/lazy-algo-tests.tolk | 1505 +++++++++++++ tolk-tester/tests/lazy-load-tests.tolk | 1981 +++++++++++++++++ tolk-tester/tests/struct-tests.tolk | 4 +- tolk/CMakeLists.txt | 2 + tolk/ast-aux-data.h | 24 + tolk/ast-from-tokens.cpp | 39 +- tolk/ast-replacer.h | 4 + tolk/ast-replicator.h | 8 + tolk/ast-stringifier.h | 4 + tolk/ast-visitor.h | 6 +- tolk/ast.cpp | 8 + tolk/ast.h | 33 +- tolk/builtins.cpp | 6 + tolk/fwd-declarations.h | 2 + tolk/lazy-helpers.cpp | 101 + tolk/lazy-helpers.h | 91 + tolk/lexer.cpp | 1 + tolk/lexer.h | 1 + tolk/pack-unpack-api.cpp | 129 ++ tolk/pack-unpack-api.h | 9 + tolk/pack-unpack-serializers.cpp | 168 +- tolk/pack-unpack-serializers.h | 18 + tolk/pipe-ast-to-legacy.cpp | 247 +- tolk/pipe-calc-rvalue-lvalue.cpp | 14 + tolk/pipe-check-inferred-types.cpp | 4 +- tolk/pipe-check-rvalue-lvalue.cpp | 7 + tolk/pipe-generate-fif-output.cpp | 4 +- tolk/pipe-infer-types-and-calls.cpp | 30 +- tolk/pipe-lazy-load-insertions.cpp | 1032 +++++++++ tolk/pipe-resolve-identifiers.cpp | 2 +- tolk/pipeline.h | 1 + tolk/smart-casts-cfg.cpp | 11 + tolk/smart-casts-cfg.h | 1 + tolk/tolk.cpp | 1 + tolk/tolk.h | 17 +- tolk/type-system.cpp | 9 + tolk/type-system.h | 1 + 54 files changed, 5722 insertions(+), 70 deletions(-) create mode 100644 tolk-tester/tests/invalid-semantics/err-4460.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7300.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7301.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7302.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7303.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7304.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7305.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7306.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7307.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7308.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7683.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7684.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7720.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7800.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7923.tolk create mode 100644 tolk-tester/tests/lazy-algo-tests.tolk create mode 100644 tolk-tester/tests/lazy-load-tests.tolk create mode 100644 tolk/lazy-helpers.cpp create mode 100644 tolk/lazy-helpers.h create mode 100644 tolk/pipe-lazy-load-insertions.cpp diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 86995717f..c2674a4c5 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -233,15 +233,17 @@ struct PackOptions { /// UnpackOptions allows you to control behavior of `MyStruct.fromCell(c)` and similar functions. struct UnpackOptions { - // after finished reading all fields from a cell/slice, call [slice.assertEnd] to ensure no remaining data left; - // it's the default behavior, it ensures that you've fully described data you're reading with a struct; - // example: `struct Point { x: int8; y: int8 }`, input "0102" is ok, "0102FF" will throw excno 9; - // note: setting this to false does not decrease gas (DROP from a stack and ENDS cost the same); - // note: this option controls [T.fromCell] and [T.fromSlice], but is ignored by [slice.loadAny] + /// after finished reading all fields from a cell/slice, call [slice.assertEnd] to ensure no remaining data left; + /// it's the default behavior, it ensures that you've fully described data you're reading with a struct; + /// example: `struct Point { x: int8; y: int8 }`, input "0102" is ok, "0102FF" will throw excno 9; + /// note: setting this to false does not decrease gas (DROP from a stack and ENDS cost the same); + /// note: this option controls [T.fromCell] and [T.fromSlice], but is ignored by [slice.loadAny]; + /// note: `lazy` ignores this option, because it reads fields on demand or even skips them assertEndAfterReading: bool = true, /// this excNo is thrown if a prefix doesn't match, e.g. for `struct (0x01) A` given input "88..."; - /// similarly, for a union type, this is thrown when none of the opcodes match + /// similarly, for a union type, this is thrown when none of the opcodes match; + /// note: `lazy` ignores this option if you have `else` in `match` (you write custom logic there) throwIfOpcodeDoesNotMatch: int = 63, } @@ -327,6 +329,26 @@ fun T.getDeclaredPackPrefix(): int fun T.getDeclaredPackPrefixLen(): int builtin; +/// Forces an object created by `lazy` to load fully. Returns the remaining slice (having read all fields). +/// Since `options.assertEndAfterReading` is ignored by `lazy` (fields are loaded on demand), +/// this method can help you overcome this, if you really need to check input consistency. +/// Example: +/// ``` +/// val msg = lazy CounterMessage.fromSlice(s); +/// match (msg) { // it's a lazy match, without creating a union on the stack +/// CounterIncrement => { +/// ... +/// newCounter = curCounter + msg.incBy; // `incBy` loaded here, on demand +/// msg.forceLoadLazyObject().assertEnd() // the purpose: get remainder +/// } +/// } +/// ``` +/// Note: while [slice.assertEnd] may seem reasonable, these checks are avoided in practice, +/// because the purpose of `lazy` is to auto-detect and load only necessary fields, not up to the end. +@pure +fun T.forceLoadLazyObject(self): slice + builtin; + /// Cell represents a typed cell reference (as opposed to untyped `cell`). /// Example: /// ``` diff --git a/tolk-tester/tests/invalid-semantics/err-4460.tolk b/tolk-tester/tests/invalid-semantics/err-4460.tolk new file mode 100644 index 000000000..1cae8c7a9 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4460.tolk @@ -0,0 +1,14 @@ +struct Point { + x: int8; + y: int8; +} + +fun main() { + var p = lazy Point.fromSlice(stringHexToSlice("0A14")); + p.asdf; +} + +/** +@compilation_should_fail +@stderr field `asdf` doesn't exist in type `Point` + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7300.tolk b/tolk-tester/tests/invalid-serialization/err-7300.tolk new file mode 100644 index 000000000..31930736e --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7300.tolk @@ -0,0 +1,15 @@ +struct A { a1: int7; a2: int8; a3: int8; } +struct B { b1: int7; b2: int8; b3: int8; } +type AOrB = A | B; + +fun f() { + val o = lazy AOrB.fromSlice(""); + __expect_lazy(""); + return (o is A) ? o.a1 : o.b1; +} + +/** +@compilation_should_fail +@stderr `lazy` will not work here, because variable `o` it's used in a non-lazy manner +@stderr hint: lazy union may be used only in `match` statement + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7301.tolk b/tolk-tester/tests/invalid-serialization/err-7301.tolk new file mode 100644 index 000000000..1843bc483 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7301.tolk @@ -0,0 +1,14 @@ +type Int32Or64 = int32 | int64; + +fun f() { + val o = lazy Int32Or64.fromSlice(""); + match (o) { + int32 => return o + 1, + int64 => throw o, + } +} + +/** +@compilation_should_fail +@stderr `lazy` union should contain only structures, but it contains `int32` + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7302.tolk b/tolk-tester/tests/invalid-serialization/err-7302.tolk new file mode 100644 index 000000000..7885f6d0c --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7302.tolk @@ -0,0 +1,16 @@ +struct A{} +type StructAndNot = A | int8; + +fun f() { + var o = lazy StructAndNot.fromSlice(""); // neg5 + __expect_lazy(""); + match (o) { + A => {} + int8 => {} + } +} + +/** +@compilation_should_fail +@stderr `lazy` union should contain only structures, but it contains `int8` + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7303.tolk b/tolk-tester/tests/invalid-serialization/err-7303.tolk new file mode 100644 index 000000000..3fe030217 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7303.tolk @@ -0,0 +1,11 @@ +type PairInt8 = (int8, int8); + +fun f() { + var i = lazy PairInt8.fromSlice(stringHexToSlice("0102")); + return (i.0, i.1); +} + +/** +@compilation_should_fail +@stderr `lazy` is applicable to structs, not to `PairInt8` + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7304.tolk b/tolk-tester/tests/invalid-serialization/err-7304.tolk new file mode 100644 index 000000000..4659caf80 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7304.tolk @@ -0,0 +1,22 @@ +struct Counter7Increment { byValue: int7 } +struct Counter7Decrement { byValue: int7 } +type MsgEitherCounter = Counter7Increment | Counter7Decrement; + +fun f(c: cell) { + // because 2 matches + var msg = lazy MsgEitherCounter.fromCell(c); // neg8 + var t = createEmptyTuple(); + match (msg) { + Counter7Increment => {} + Counter7Decrement => {} + } + match (msg) { + Counter7Increment => {} + Counter7Decrement => {} + } +} + +/** +@compilation_should_fail +@stderr `lazy` will not work here, because variable `msg` it's used in a non-lazy manner + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7305.tolk b/tolk-tester/tests/invalid-serialization/err-7305.tolk new file mode 100644 index 000000000..59fb5ba50 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7305.tolk @@ -0,0 +1,19 @@ +struct Counter7Increment { byValue: int7 } +struct Counter7Decrement { byValue: int7 } +type MsgEitherCounter = Counter7Increment | Counter7Decrement; + +fun f() { + // because .toCell() + var msg = lazy MsgEitherCounter.fromSlice(""); // neg9 + match (msg) { + Counter7Increment => {} + Counter7Decrement => {} + } + return msg.toCell(); +} + +/** +@compilation_should_fail +@stderr `lazy` will not work here, because variable `msg` it's used in a non-lazy manner +@stderr lazy MsgEitherCounter + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7306.tolk b/tolk-tester/tests/invalid-serialization/err-7306.tolk new file mode 100644 index 000000000..d515c7d49 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7306.tolk @@ -0,0 +1,17 @@ +struct Counter7Increment { byValue: int7 } +struct Counter7Decrement { byValue: int7 } +type MsgEitherCounter = Counter7Increment | Counter7Decrement; + +fun f() { + // because in a complex expression + var msg = lazy MsgEitherCounter.fromSlice(""); + match (msg) { + Counter7Increment => 1 as int32, + Counter7Decrement => 2 as int32, + }.toCell() +} + +/** +@compilation_should_fail +@stderr `lazy` will not work here, because variable `msg` it's used in a non-lazy manner + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7307.tolk b/tolk-tester/tests/invalid-serialization/err-7307.tolk new file mode 100644 index 000000000..9265cf1ce --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7307.tolk @@ -0,0 +1,17 @@ +struct Counter7Increment { byValue: int7 } +struct Counter7Decrement { byValue: int7 } +type MsgEitherCounter = Counter7Increment | Counter7Decrement; + +fun f() { + var msg = lazy MsgEitherCounter.fromSlice(""); // neg10 + match (msg) { + Counter7Increment => 1 as int32, + Counter7Decrement => 2 as int32, + else => throw 123, // not allowed in either + } +} + +/** +@compilation_should_fail +@stderr `else` is unreachable, because this `match` has only two options (0/1 prefixes) + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7308.tolk b/tolk-tester/tests/invalid-serialization/err-7308.tolk new file mode 100644 index 000000000..a46964a86 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7308.tolk @@ -0,0 +1,18 @@ +struct (0x01) A { } +struct (0x02) B { } +struct (0x03) C { } +type MyInput = A | B | C; + +fun f() { + var msg = lazy MyInput.fromSlice(""); + match (msg) { + A => {} + B => {} + else => throw 123, + } +} + +/** +@compilation_should_fail +@stderr `match` does not cover all possible types; missing types are: `C` + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7683.tolk b/tolk-tester/tests/invalid-serialization/err-7683.tolk new file mode 100644 index 000000000..f82924c90 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7683.tolk @@ -0,0 +1,12 @@ +struct A { +} + +fun neg4() { + // because compiler doesn't analyze deep assignments + var (a, b) = (1, lazy A.fromSlice("")); +} + +/** +@compilation_should_fail +@stderr error: incorrect `lazy` operator usage, it's not directly assigned to a variable + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7684.tolk b/tolk-tester/tests/invalid-serialization/err-7684.tolk new file mode 100644 index 000000000..cfb49b3ae --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7684.tolk @@ -0,0 +1,11 @@ +struct St { +} + +fun getStorage() { + return lazy St.fromCell(contract.getData()); +} + +/** +@compilation_should_fail +@stderr error: incorrect `lazy` operator usage, it's not directly assigned to a variable + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7720.tolk b/tolk-tester/tests/invalid-serialization/err-7720.tolk new file mode 100644 index 000000000..ec1bc4d35 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7720.tolk @@ -0,0 +1,15 @@ +struct Point { + x: int8; + y: int8; +} + +fun neg11(s: slice) { + // mutating function can't be lazy + var p = lazy s.loadAny(); + return p.x; +} + +/** +@compilation_should_fail +@stderr `lazy` operator can only be used with built-in functions like fromCell/fromSlice or simple wrappers over them +*/ diff --git a/tolk-tester/tests/invalid-serialization/err-7800.tolk b/tolk-tester/tests/invalid-serialization/err-7800.tolk new file mode 100644 index 000000000..1df8e5f5c --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7800.tolk @@ -0,0 +1,17 @@ +struct Point { + x: int32; + y: int32; +} + +fun loadPoint(s: slice): Point { + return Point.fromSlice(s); +} + +fun main() { + var p = lazy loadPoint(stringHexToSlice("0000000100000002")); +} + +/** +@compilation_should_fail +@stderr `lazy` operator can only be used with built-in functions like fromCell/fromSlice + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7923.tolk b/tolk-tester/tests/invalid-serialization/err-7923.tolk new file mode 100644 index 000000000..8b809560d --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7923.tolk @@ -0,0 +1,14 @@ +struct Point { + x: int8; + y: int8; +} + +fun main() { + var p = Point.fromSlice(stringHexToSlice("0102")); + p.forceLoadLazyObject(); +} + +/** +@compilation_should_fail +@stderr this method is applicable to lazy variables only + */ diff --git a/tolk-tester/tests/invalid-typing/err-6366.tolk b/tolk-tester/tests/invalid-typing/err-6366.tolk index cc6052d99..cba32e770 100644 --- a/tolk-tester/tests/invalid-typing/err-6366.tolk +++ b/tolk-tester/tests/invalid-typing/err-6366.tolk @@ -2,6 +2,7 @@ fun main() { match (0 as int | slice) { int => 1, + slice => 2, else => 10, }; } diff --git a/tolk-tester/tests/lazy-algo-tests.tolk b/tolk-tester/tests/lazy-algo-tests.tolk new file mode 100644 index 000000000..363640ed3 --- /dev/null +++ b/tolk-tester/tests/lazy-algo-tests.tolk @@ -0,0 +1,1505 @@ +struct A { a1: int7; a2: int8; a3: int8; } +struct B { b1: int7; b2: int8; b3: int8; } +type AOrB = A | B; + +@noinline +fun A.getA1(self) { + return self.a1; +} + +fun AOrB.doMatch(self) { + __expect_inline(true); + match (self) { + A => {} + B => {} + } +} + +struct Fields9 { + f1: uint8; f2: uint8; f3: uint8; + f4: uint8; f5: uint8; f6: uint8; + f7: uint8; f8: uint8; f9: uint8; +} + +fun uint8.doNothing(self) {} + +@noinline +fun Fields9.method(self) {} + +fun Fields9.useAll(self) { + __expect_inline(true); + if (true) { self.method() } +} + +struct WithEitherABMiddle { + f1: int32; + f2: int32; + ab: A | B; + f4: int32; + f5: int32; + f6: int32; +} + +struct WithEitherABEnd { + f1: int32; + f2: int32; + f3: int32; + f4: int32; + f5: int32; + ab: A | B; +} + +fun WithEitherABEnd.checkF3(self, against: int) { + assert(self.f3 == against, 100); +} + +fun WithEitherABEnd.doMatch(self) { + __expect_inline(true); + match (self.ab) { + A => {} + B => {} + } +} + +fun rand() { return random.uint256() } + +struct Point { + x: int8; + y: int8; +} + +fun Point.assignX(mutate self, x: int) { + self.x = x; +} + +fun Point.getX(self) { + return self.x; +} + +fun Point.getXNoInline(self) { + __expect_inline(false); // due to returns in the middle + if (10>3) { + return self.x; + } + throw 123; +} + +fun Point.getCoord(self, getX: bool) { + return getX ? self.x : self.y; +} + +fun Point.getCoordWrapper(self, getX: bool) { + if (10<3) { throw 123 } + return self.getCoord(getX) +} + +fun getXOfPoint(p: Point) { + return p.x +} + +struct Point3d { + x: int32; + y: int32; + z: int32; +} + +struct Point78ForMaybe { + x: int7; + y: int8; +} + +type OptionalPoint78ForMaybe = Point78ForMaybe?; + +struct ComplexWithPoint { + f1: int8; + p: Point; + f3: int8; +} + +struct ComplexWithMaybePoint { + f1: int8; + p: Point78ForMaybe?; + f3: int8; +} + +struct Counter7Increment { byValue: int7 } +struct Counter7Decrement { byValue: int7 } +type MsgEitherCounter = Counter7Increment | Counter7Decrement; + +struct FieldsAndEitherCounter { + f1: int8; + f2: int8; + c: MsgEitherCounter; +} + +struct WalletStorage { + isSignatureAllowed: bool; + seqno: uint32; + walletId: uint32; + publicKey: uint256; + extensions: dict; +} + +@overflow1023_policy("suppress") +struct NotFixedWidthStorage { + f1: int8; + f2: int8?; + f3: int8; + f4: bool; + f5: cell; + f6: int8 | bool; + f7: address; + f8: int8; + f9: bits200; + f10: (bits400, ()); + f11: coins; +} + +struct WithTensor { + a1: int8; + t: (int8, int8); + a3: int8; +} + +struct WithMaybeTensor { + a1: int8; + a2: int8; + t: (int8, int8)?; +} + +struct MixedDataAndRefs { + i1: int8; + i2: bool; + a1: address; + e: (); + r1: cell; + r2: ((), cell); + i4: Point; + rm: cell?; + r4: Cell?; +} + +struct (0x01) CounterIncrement { byValue: int8 } +struct (0x02) CounterDecrement { byValue: int8 } +struct (0x03) CounterDecrementBy1 { } +struct (0x04) CounterReset { initialValue: int32 } +type MsgFullCounter = CounterIncrement | CounterDecrement | CounterDecrementBy1 | CounterReset; + +struct ItemState { + ownerAddress: address; + content: cell; + auction: cell?; + royaltyParams: cell; +} + +struct HasIncrementEnd { + a: int8; + inc: CounterIncrement; +} + + +fun loadWalletStorage() { + return WalletStorage.fromSlice(stringHexToSlice("8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_")); +} + +fun WalletStorage.load() { + return WalletStorage.fromCell(createEmptyCell()); +} + + + +fun algo1() { + val o = lazy Fields9.fromSlice(""); + if (rand()) { + __expect_lazy("[o] load f1"); + o.f1; + } +} + +fun algo2() { + val o = lazy Fields9.fromSlice(""); + __expect_lazy("[o] load f1"); + if (rand()) { + o.f1; + } else { + o.f1 + o.f1; + } +} + +fun algo3() { + val o = lazy Fields9.fromSlice(""); + if (rand()) { + return; + } + __expect_lazy("[o] load f1"); + o.f1.doNothing(); +} + +fun algo4() { + val o = lazy Fields9.fromSlice(""); + __expect_lazy("[o] load f1 f2"); + if (rand()) { + o.f1; + } else { + o.f2; + } +} + +fun algo5() { + val o = lazy Fields9.fromSlice(""); + __expect_lazy("[o] skip (bits8), load f2, skip (bits8), load f4"); + o.f4; + o.f2; +} + +fun algo6() { + val o = lazy Fields9.fromSlice(""); + __expect_lazy("[o] load f1 f2, skip (bits40), load f8"); + if (rand()) { + o.f8; + } + o.f2; + o.f1 * o.f1 + o.f1 + o.f2 * o.f2; +} + +fun algo7() { + val o = lazy Fields9.fromSlice(""); + if (rand()) { + __expect_lazy("[o] load f1 f2 f3 f4 f5 f6 f7 f8 f9"); + o; + } else { + } +} + +fun algo8() { + if (rand()) { + val o: Fields9 = lazy Fields9.fromSlice(""); + __expect_lazy("[o] load f1 f2 f3 f4 f5 f6 f7 f8 f9"); + o.method(); + } else { + } +} + +fun algo9() { + val p = lazy Point3d.fromCell(createEmptyCell()); + __expect_lazy("[p] load x y z"); + if (p.x && p.y) { + p.x; + p.y; + } else { + p.z; + } +} + +fun algo10() { + val p = lazy Point3d.fromSlice(""); + __expect_lazy("[p] load x, skip (bits32), load z"); + p.z; + if (rand()) { + p.x; + return; + } +} + +fun algo11() { + val p = lazy Point3d.fromSlice(""); + var num = 0; + if (10 > 2) { + num += 1; + } + __expect_lazy("[p] load x"); + rand() + ? p.x as int + : p.x + num; +} + +fun algo12() { + try { + val p = lazy Point3d.fromSlice(""); + try {} + catch {} + assert(10>3, 1); + __expect_lazy("[p] skip (bits32), load y"); + p.y * p.y; + } catch { + } +} + +fun algo13() { + val o = lazy Fields9.fromSlice(""); + if (10 > 3) { + __expect_lazy("[o] load f1"); + assert(o.f1 > 0) throw 400; + } +} + +fun algo14() { + val p = lazy Point.fromSlice(""); + __expect_lazy("[p] load x"); + while (rand()) { + p.x; + } +} + +fun algo15() { + val p = lazy Point.fromSlice(""); + __expect_lazy("[p] skip (bits8), load y"); + do { + p.y; + } while (rand()); +} + +fun algo16() { + val p = lazy Point3d.fromSlice(""); + __expect_lazy("[p] load x, skip (bits32), load z"); + do { + p.x; + } while (p.z); +} + +fun algo17() { + val p = lazy Point.fromSlice(""); + __expect_lazy("[p] load x y"); + repeat(p.x) { + p.y; + } +} + +fun algo18() { + val o = lazy Fields9.fromSlice(""); + __expect_lazy("[o] load f1 f2, skip (bits16), load f5 f6"); + if (o.f1) { + if (o.f2) { + throw o.f6 + } + } + else { o.f5 } +} + +fun algo19() { + val p = lazy Point.fromSlice(""); + __expect_lazy("[p] load x y"); + var p2 = p; + return (p2.x, p2.y); +} + +fun algo20() { + val c = lazy ComplexWithPoint.fromSlice(""); + __expect_lazy("[c] skip (bits8), load p"); + return c.p.x; +} + +fun algo21() { + val c = lazy ComplexWithPoint.fromSlice(""); + __expect_lazy("[c] load f1 p"); + if (c.f1 > 0) { + return c.p.y; + } + throw 0; +} + +fun algo22() { + var w = lazy WithTensor.fromSlice(""); + __expect_lazy("[w] skip (bits8), load t"); + w.t.0 = 10; +} + +fun algo23() { + var w = lazy WithMaybeTensor.fromSlice(""); + __expect_lazy("[w] skip (bits8), load a2 t"); + if (w.t == null) { + return w.a2; + } + return w.t.0; +} + +fun algo24() { + val w = lazy WalletStorage.fromSlice(""); + try { + __expect_lazy("[w] skip (bits1), load seqno"); + return w.seqno as int; + } catch (excno) { + return excno; + } +} + +fun algo25() { + val w = lazy WalletStorage.fromSlice(""); + __expect_lazy("[w] skip (bits1), load seqno"); + try { + return w.seqno as int; + } catch (excno) { + if (excno > 0) { + return w.seqno as int; + } + return excno; + } +} + +fun algo26() { + val w = lazy WalletStorage.fromSlice(""); + __expect_lazy("[w] skip (bits1), load seqno"); + try { + return w.seqno + 1; + } catch (excno) { + } + return w.seqno - 1; +} + +fun algo27() { + val w = lazy WalletStorage.fromSlice(""); + __expect_lazy("[w] skip (bits1), load seqno walletId"); + try { + return w.seqno + 1; + } catch (excno) { + } + if (10 > 3) { + return w.walletId as int; + } + return -1; +} + +fun algo28() { + val w = lazy WalletStorage.fromSlice(""); + try { + try { + return 1; + } catch { + __expect_lazy("[w] skip (bits1), load seqno walletId"); + return w.walletId as int; + w.seqno; + } + } catch (excno) { + return -1; + } +} + +fun algo29() { + val w = lazy WalletStorage.fromSlice(""); + try { + __expect_lazy("[w] skip (bits1), load seqno walletId"); + try { + return 1; + } catch { + return w.walletId as int; + } + w.seqno; + } catch (excno) { + return -1; + } +} + +fun algo30() { + val w = lazy WalletStorage.fromSlice(""); + if (true) { + try { + __expect_lazy("[w] skip (bits1), load seqno walletId"); + try { + return 1; + } catch { + return w.walletId as int; + } + w.seqno; + } catch (excno) { + return -1; + } + } +} + +fun algo31() { + val w = lazy WalletStorage.fromSlice(""); + if (false) {} + else { + __expect_lazy("[w] skip (bits65), load publicKey"); + while (true) { + w.publicKey + } + } +} + +fun algo32() { + val w = lazy WalletStorage.fromSlice(""); + __expect_lazy("[w] load isSignatureAllowed"); + match (true) { + true => { w.isSignatureAllowed } + false => {} + } +} + +fun algo33() { + val p = lazy ComplexWithPoint.fromSlice(""); + try { if(true) {} else { + __expect_lazy("[p] skip (bits8), load p"); + p.p.x; + }} catch{} +} + +fun algo34() { + val p = lazy ComplexWithPoint.fromSlice(""); + __expect_lazy("[p] skip (bits8), load p f3"); + if (p.p.x) { } + else { p.f3 } +} + +fun algo35() { + var p = lazy ComplexWithPoint.fromSlice(""); + try { + __expect_lazy("[p] load f1 p"); + if (p.p.x) { } + else { p.p = {x:1, y:2} } + if (true) { + try {} catch (excno) { match (excno + p.f1) { } } + } + } catch {} +} + +fun algo36() { + var p1 = lazy ComplexWithPoint.fromSlice(""); + var p2 = lazy ComplexWithPoint.fromSlice(""); + __expect_lazy("[p1] skip (bits8), load p"); + try { + __expect_lazy("[p2] load f1 p f3"); + match (true) { + true => { p2.f3 } + else => p2.p.x ? p2.f1 : p1.p.x + } + } catch { p1.p.y -= 1 } +} + +fun algo37() { + var p1 = lazy ComplexWithPoint.fromSlice(""); + var p2 = lazy ComplexWithPoint.fromSlice(""); + try { + if (10>3) { + __expect_lazy("[p2] load f1 p f3"); + var p1 = p2; + } + } catch { + __expect_lazy("[p1] skip (bits8), load p"); + p1.p.y -= 1 + } +} + +fun algo38() { + var p1 = lazy ComplexWithPoint.fromSlice(""); + var p2 = lazy ComplexWithPoint.fromSlice(""); + __expect_lazy("[p1] skip (bits8), load p f3"); + try { + if (10>3) { + __expect_lazy("[p2] skip (bits24), load f3"); + p1.f3 = p2.f3; + } + } catch { + p1.p.y -= 1 + } +} + +fun algo39() { + val p = lazy Point.fromSlice(""); + __expect_lazy("[p] load x"); + return p.getX(); +} + +@method_id(140) +fun algo40() { + val p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x y"); + return p.getCoord(false); +} + +@method_id(141) +fun algo41(getX: bool) { + val p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x y"); + return p.getCoordWrapper(getX); +} + +fun algo42() { + var p = lazy Point.fromSlice(""); + __expect_lazy("[p] load x y"); // calling a mutating method — not available for lazy + p.assignX(10); +} + +fun algo43() { + val p = lazy Point.fromSlice(""); + __expect_lazy("[p] load x y"); // not a method — not available for lazy (not implemented) + return getXOfPoint(p); +} + +fun algo44() { + val p = lazy Point.fromSlice(""); + __expect_lazy("[p] load x y"); // not an inlined method + return p.getXNoInline(); +} + +fun algo46() { + var w = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[w] skip (bits64), load f3, skip (bits64), load ab"); + w.checkF3(0); + w.doMatch(); +} + +fun algo47() { + var f = lazy Fields9.fromSlice(""); + __expect_lazy("[f] load f1 f2 f3 f4 f5 f6 f7 f8 f9"); + f.useAll(); +} + +fun algo48() { + var p = lazy Point.fromSlice(""); + contract.getData(); + __expect_lazy("[p] load x y"); + p.forceLoadLazyObject(); +} + +fun algo49(forceLoad: bool) { + val l = lazy Fields9.fromSlice(""); + if (forceLoad) { + __expect_lazy("[l] load f1 f2 f3 f4 f5 f6 f7 f8 f9"); + try { l.forceLoadLazyObject() } + catch { l.f2 } + } +} + +fun algo50() { + var st = lazy NotFixedWidthStorage.fromSlice(""); + __expect_lazy("[st] skip (bits8) (int8?) (bits9) (int8 | bool) (address) (bits608), load f11"); + st.f11; +} + +fun algo51() { + var w1 = lazy MixedDataAndRefs.fromSlice(""); + var w2 = lazy MixedDataAndRefs.fromSlice(""); + var w3 = lazy MixedDataAndRefs.fromSlice(""); + var w4 = lazy MixedDataAndRefs.fromSlice(""); + var w5 = lazy MixedDataAndRefs.fromSlice(""); + var w6 = lazy MixedDataAndRefs.fromSlice(""); + + __expect_lazy("[w1] load r1"); + w1.r1; + __expect_lazy("[w2] skip (cell), load r2"); + w2.r2; + __expect_lazy("[w3] skip (bits9) (address) (cell) (((), cell)) (bits16), load rm"); + w3.rm; + __expect_lazy("[w4] skip (bits9) (address), load i4"); + w4.i4; + __expect_lazy("[w5] skip (bits8), load i2, skip (address), load e r1 i4"); + w5.i2; w5.e; w5.r1; w5.i4; +} + +fun algo52(cc: Cell?) { + val itemState = lazy cc!.load(); + __expect_lazy("[itemState] skip (address) (cell) (cell?), load royaltyParams"); + return itemState.royaltyParams; +} + +fun algo53() { + val i = lazy ItemState.fromSlice(""); + __expect_lazy("[i] load content"); + return i.content.beginParse(); +} + + +fun algo100() { + var o = lazy AOrB.fromSlice(""); + __expect_lazy("[o] lazy match"); + match (o) { + A => { + __expect_lazy("[o] load a1, skip (bits8), load a3"); + o.a1; + __expect_lazy(""); + o.a3 + } + B => { + __expect_lazy("[o] load b1 b2"); + o.b2; + __expect_lazy(""); + o.b1 + } + } +} + +fun algo101() { + val o = lazy WithEitherABMiddle.fromSlice(""); + __expect_lazy("[o] skip (bits64), load ab, skip (bits32), load f5"); + o.f5; + o.ab; +} + +fun algo102() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits128), load f5 ab"); + o.f5; + o.ab; +} + +fun algo103() { + val o = lazy WithEitherABMiddle.fromSlice(""); + __expect_lazy("[o] skip (bits64), load ab"); + match (o.ab) { + A => {} + B => {} + } +} + +fun algo104() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits160), lazy match ab"); + match (o.ab) { + A => {} + B => {} + } +} + +fun algo105() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] load f1 f2 f3, skip (bits64), lazy match ab"); + match (o.ab) { + A => { + assert(o.f1 > o.f2, o.f3); + // __expect_lazy("[o.ab] load a1 a2"); + return o.ab.a1 + o.ab.a2; + } + B => { + // __expect_lazy("[o.ab] load b1 b2 b3"); + val cp = o.ab; + return cp.b1 + cp.b2; + } + } +} + +fun algo106() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits96), load f4, skip (bits32), lazy match ab"); + if (rand()) { + o.f4; + } else { + match (o.ab) { A => {} B => { + o.ab.b2; + } } + o.f4; + } +} + +fun algo107() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits160), load ab"); + if (o.ab is B) {} + match (o.ab) {} +} + +fun algo108(): int7 | int8 { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits160), lazy match ab"); + return match (o.ab) { + A => { + // __expect_lazy("[o.ab] load a1"); + return o.ab.a1; + } + B => { + // __expect_lazy("[o.ab] skip b1, load b2"); + return o.ab.b2; + } + } +} + +fun algo109(): int { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] load f1 f2, skip (bits32), load f4, skip (bits32), lazy match ab"); + match (o.ab) { + A => { + assert(o.f1 > o.f2, 100); + // __expect_lazy("[o.ab] load a1"); + return o.f1 + o.ab.a1; + } + B => { + return o.f2 + o.f4; + } + } +} + +fun algo110() { + val p = lazy Point3d.fromSlice(""); + __expect_lazy("[p] load x y z"); + match (p.x) { + 1 => p.y, + 2 => p.z, + } +} + +fun algo111() { + val p = lazy Point3d.fromSlice(""); + __expect_lazy("[p] load x y z"); + match (p.x) { + 1 => 1, + else => { var x = p.y + p.y } + } + p.z; +} + +fun algo112() { + val o = lazy Fields9.fromSlice(""); + __expect_lazy("[o] load f1 f2, skip (bits8), load f4 f5 f6"); + match (o.f2) { + uint8 => { + o.f1 + match(o.f4) { 1 => o.f4, else => o.f5 } + } + } + o.f6; +} + +fun algo113() { + val o = lazy Fields9.fromSlice(""); + __expect_lazy("[o] load f1"); + match (o.f1) { + uint8 => { + debug.print(o.f1); + } + } + o.f1; +} + +type PointOrPoint3d = Point | Point3d; + +fun algo114() { + val o = lazy PointOrPoint3d.fromSlice(""); + __expect_lazy("[o] lazy match"); + match (o) { + Point => { + __expect_lazy("[o] load x y"); + return o.x + o.y; + } + Point3d => { + __expect_lazy("[o] load x y z"); + var p = o; + return p.x + p.y; + } + } +} + +fun algo115() { + val o = lazy WithEitherABMiddle.fromSlice(""); + assert(10>3, 1); + __expect_lazy("[o] load f1 f2 ab f4"); + o.f1 + o.f2; + match (o.ab) { + A => { o.f1 + o.ab.a1 } + B => { o.f1 + o.f2 + o.ab.b1 } + } + o.f4; +} + +fun algo116() { + val o = lazy WithEitherABMiddle.fromSlice(""); + __expect_lazy("[o] load f1 f2 ab f4 f5"); + o.f1 + o.f2; + match (o.ab) { + A => { return o.f4 } + B => throw o.f5 + } +} + +fun algo117() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] load f1 f2, skip (bits32), load f4 f5, lazy match ab"); + o.f1 + o.f2; + match (o.ab) { + A => return o.f4, + B => throw o.f5 + } +} + +fun algo118() { + val o = lazy WithEitherABEnd.fromSlice(""); + if (10>3) { + __expect_lazy("[o] load f1 f2, skip (bits96), lazy match ab"); + match (o.ab) { + A => { return o.f1 } + B => { throw o.f2 } + } + } + throw 123; +} + +fun algo119() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] load f1 f2, skip (bits96), lazy match ab"); + match (o.ab) { + A => { + return o.ab.a2 + o.f2; + } + B => { + assert(o.ab.b1) throw o.f1; + } + } + o.f1; + return o.f2 as int; +} + +fun algo120() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] load f1, skip (bits128), lazy match ab"); + match (o.ab) { + A => return o.ab.a2 * 1, + B => return o.ab.b1 << 0, + } + o.f1; +} + +fun algo121() { + val o1 = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o1] load f1 f2 f3 f4 f5, lazy match ab"); + match (o1.ab) { + A => { (o1.f1, o1.f2, o1.f3, o1.f4, o1.f5) } + B => {} + } + val o2 = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o2] load f1 f2 f3 f4 f5 ab"); + match (o2.ab) { + A => { (o2) } + B => {} + } +} + + +type AOrWithEitherABEnd = A | WithEitherABEnd; + +fun algo122() { + val o = lazy AOrWithEitherABEnd.fromSlice(""); + match (o) { + A => { + __expect_lazy("[o] load a1"); + o.a1; + } + WithEitherABEnd => { + __expect_lazy("[o] skip (bits160), load ab"); // not lazy match, nesting not supported + match (o.ab) { + A => { o.ab.a1 } + B => {} + } + } + } +} + +fun algo123() { + val o = lazy AOrB.fromSlice(""); + return match (o) { + A => { + __expect_lazy("[o] load a1"); + throw o.a1; + } + B => { + __expect_lazy("[o] load b1 b2"); + assert(10>o.b2) throw 100; + return o.b1; + } + } +} + +fun algo124() { + val o = lazy AOrB.fromSlice(""); + return match (o) { + A => { + assert(10>3, 100); + assert(10>3, 100); + __expect_lazy("[o] load a1 a2 a3"); + throw (100, o.getA1()); + } + B => { + assert(10>3, 100); + __expect_lazy("[o] load b1 b2 b3"); + return o is B; + } + } +} + +fun algo127() { + val w = lazy WithMaybeTensor.fromSlice(""); + __expect_lazy("[w] skip (bits16), load t"); + match (w.t) { + null => {} + (int8, int8) => {} + } +} + +fun algo128() { + val o = lazy AOrB.fromSlice(""); + __expect_lazy("[o] lazy match"); + match (o) { + A => { + match (o) { A => {} } + } + B => {} + } +} + +fun algo129() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits128), load f5, lazy match ab"); + match (o.ab) { + A => {} + B => { + match (o.ab) { B => { o.f5 } } + } + } +} + +fun algo130() { + var o = lazy AOrB.fromSlice(""); + try { + return 1; + } catch (excno) { + __expect_lazy("[o] lazy match"); + match (o) { + A => { + __expect_lazy("[o] load a1"); + return o.a1 + excno; + } + B => { + __expect_lazy("[o] load b1 b2 b3"); + if (o.b1 > 0) { o = B { b1: 1, b2: 2, b3: 3 } } + return o.b2 as int; + } + } + } +} + +fun algo131() { + val o = lazy AOrB.fromSlice(""); + match (o) { + A => { + try { + __expect_lazy("[o] skip (bits15), load a3"); + return o.a3 as int; + } catch (excno) { return excno } + } + B => { + if (10 > 3) { return -1; } + else { + if (10 > 3) { return -1; } + else { + __expect_lazy("[o] load b1"); + return o.b1 * 0; + } + } + } + } +} + +fun algo132(): int { + val o = lazy AOrB.fromSlice(""); + match (o) { + B => { + __expect_lazy("[o] load b1 b2"); + try { + return o.b1; + } catch { + return o.b2; + } + } + A => { + __expect_lazy("[o] skip (bits7), load a2 a3"); + try { + return o.a3; + } catch { + throw 123; + } + return o.a2; + } + } +} + +fun algo133(): void { + val o = lazy WithEitherABEnd.fromSlice(""); + try {} + catch { + try { + __expect_lazy("[o] load f1, skip (bits128), lazy match ab"); + match (o.ab) { + A => { } + B => { o.f1 } + } + } catch { throw 123 } + } +} + +fun algo134(): void { + val o = lazy WithEitherABEnd.fromSlice(""); + try { try { + __expect_lazy("[o] load f1 f2, skip (bits96), lazy match ab"); + try { + match (o.ab) { + A => { } + B => { } + } + } catch { throw o.f1 + o.f2 } + } catch {} } catch {} +} + +fun algo135() { + val o = lazy AOrB.fromSlice(""); + match (o) { + A => { + __expect_lazy("[o] load a1 a2 a3"); + o.a1; + o.forceLoadLazyObject().assertEnd() + } + B => { + __expect_lazy("[o] load b1 b2"); + if (o.b1) { + o.b2; + } + } + } +} + +fun algo136() { + while (true) { + val obj = lazy AOrB.fromSlice(""); + __expect_lazy("[obj] lazy match"); + match (obj) { + A => { __expect_lazy("[obj] load a1"); obj.a1 } + B => { __expect_lazy("[obj] skip (bits7), load b2"); obj.b2 } + } + } +} + + +fun algo150() { + val o = lazy WithEitherABEnd.fromSlice(""); + var found = false; + __expect_lazy("[o] skip (bits160), load ab"); + while (!found) { + match (o.ab) { + A => {} + B => { found = true } + } + } +} + +fun algo151() { + val o = lazy WithEitherABEnd.fromSlice(""); + var found = false; + if (!found) { + __expect_lazy("[o] skip (bits160), lazy match ab"); + match (o.ab) { + A => {} + B => { found = true } + } + } +} + +fun algo152() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits160), load ab"); + 1 + match (o.ab) { + A => 1, + B => -1, + } +} + +fun algo153() { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits160), load ab"); + 1 && match (o.ab) { + A => 1, + B => -1, + } +} + +fun algo224(): void { + val o = lazy WithEitherABEnd.fromSlice(""); + __expect_lazy("[o] skip (bits32), load f2, skip (bits32), load f4 f5 ab"); + match (o.ab) { + A => {} + B => { o.f2; } + } + match (o.ab) { + A => {} + B => { if (rand()) { o.f4; } return; } + } + if (o.ab is A) { + if (rand()) { + o.f5; + } + } +} + +fun algo225() { + __expect_lazy("[c] lazy match"); + match (val c = lazy MsgFullCounter.fromSlice("")) { + CounterIncrement => { + var d = lazy WalletStorage.fromSlice(""); + __expect_lazy("[d] load (bits1) seqno, save immutable (tail)"); + d.seqno += 1; + d.toCell(); + __expect_lazy("[c] load byValue"); + 1 + c.byValue; + } + CounterDecrement => { + val c = lazy MsgFullCounter.fromSlice(""); + __expect_lazy("[c] lazy match"); + match (c) { + CounterIncrement => {} + CounterDecrement => {} + CounterDecrementBy1 => {} + CounterReset => { + assert(10>3, 101); + __expect_lazy("[c] load initialValue"); + while (10>3) { + if (10>3) { c.initialValue } + } + } + } + } + CounterDecrementBy1 => { + } + CounterReset => { + if (10 > 3) { + __expect_lazy("[c] load initialValue"); + c.initialValue + } + } + else => throw 123, // `else` is allowed in a lazy `match` + } +} + +fun algo226() { + var st = lazy loadWalletStorage(); + __expect_lazy("[st] skip (bits1), load seqno"); + return st.seqno; +} + +fun algo250() { + var f = lazy Fields9.fromSlice(""); + __expect_lazy("[f] skip (bits8), load f2"); + match (f.f2) { + uint8 => {} + } +} + +fun algo251() { + var f = lazy Fields9.fromSlice(""); + __expect_lazy("[f] skip (bits8), load f2"); + match (f.f2) { + 0 => 1, + 1 => 0, + else => throw 0, + } +} + +fun algo252() { + var p = lazy Point.fromSlice(""); + __expect_lazy("[p] skip (bits8), load y"); + match (p.y) { + int8 => {} + } +} + +fun algo253() { + var p = lazy Point.fromSlice(""); + __expect_lazy("[p] load x y"); + match (p.y) { + 0 => p.x, + 1 => 0, + else => throw 0, + } +} + +fun algo254() { + var p = lazy Point.fromSlice(""); + __expect_lazy("[p] lazy match"); + match (p) { + Point => { + return 1; + } + } +} + +fun algo255() { + var p = lazy Point.fromSlice(""); + __expect_lazy("[p] lazy match"); + match (p) { + Point => { + __expect_lazy("[p] load x"); + return p.x; + } + else => throw 123 + } +} + +fun algo256() { + var p = lazy Point.fromSlice(""); + match (p) { + Point => { + __expect_lazy("[p] load x"); + match (p.x) { int8 => {} } + } + } +} + +fun algo257(): int { + val k = lazy HasIncrementEnd.fromSlice(""); + __expect_lazy("[k] skip (bits8), load inc"); + match (k.inc) { + CounterIncrement => { + __expect_lazy(""); + return k.inc.byValue; + } + } +} + + +fun algo300() { + val p = lazy AOrB.fromSlice(stringHexToSlice("8F020304")); + __expect_lazy("[p] lazy match"); + match (p) { + A => { + __expect_lazy("[p] skip (bits7), load a2"); + debug.print(p.a2); + } + B => { + __expect_lazy("[p] load b1 b2"); + debug.print(p.b1 + p.b2); + } + } +} + +@method_id(301) +fun algo301() { + __expect_lazy("[p] lazy match"); + match (val p = lazy AOrB.fromSlice(stringHexToSlice("8F020304"))) { + A => { + __expect_lazy("[p] skip (bits7), load a2"); + return p.a2 as int; + } + B => { + __expect_lazy("[p] load b1 b2"); + return p.b1 + p.b2; + } + } +} + +fun algo302() { + var c = Point{x:1,y:2}.toCell(); + var o = lazy Point.fromCell(c); + __expect_lazy("[o] load x y"); + o.x; + o.y; + return o; +} + +@method_id(303) +fun algo303() { + var c = Point{x:1,y:2}.toCell(); + var o = lazy c.load(); + __expect_lazy("[o] load x y"); + return o; +} + +@method_id(304) +fun algo304() { + var o = lazy Point{x:1,y:2}.toCell().load(); + __expect_lazy("[o] load x y"); + return o; +} + +@method_id(305) +fun algo305() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + assert(10>3, 100); + __expect_lazy("[p] load x y"); + { + assert(p.x == 1, 100); + assert(p.y == 2, 100); + } + return p; +} + +fun algo306() { + var st = lazy WalletStorage.load(); + __expect_lazy("[st] load (bits1) seqno, save immutable (tail)"); + st.seqno += 1; + return st.toCell(); +} + +fun algo307() { + var c = lazy ComplexWithPoint.fromSlice(""); + __expect_lazy("[c] load (bits8) p, save immutable (tail)"); + c.p.assignX(10); + c.toCell(); +} + +fun algo308() { + var c = lazy ComplexWithPoint.fromSlice(""); + __expect_lazy("[c] load (bits8) p, save immutable (tail)"); + c.p.x = 10; + c.toCell(); +} + +fun algo309() { + var c = lazy ComplexWithPoint.fromSlice(""); + __expect_lazy("[c] load f1 p f3"); // object used for writing and toCell, load all for safety + c = { f1: 10, p: {x:0,y:0}, f3: c.f3 }; + c.toCell(); +} + +fun algo310() { + var c = lazy ComplexWithPoint.fromSlice(""); + if (true) { + __expect_lazy("[c] load (bits8) p, save immutable (tail)"); + debug.print(c.p.y += 2); + c.toCell(); + } +} + +fun algo311() { + var st = lazy WalletStorage.load(); + __expect_lazy("[st] load isSignatureAllowed (bits320) extensions"); + st.isSignatureAllowed = true; + st.extensions = null; + return st.toCell(); +} + +fun algo312() { + var st = lazy NotFixedWidthStorage.fromSlice(""); + __expect_lazy("[st] load (bits8) (int8?) (bits9) (cell) (int8 | bool) (address) f8, save immutable (tail)"); + if (rand()) { + st.f8 += 1; + } + st.toCell(); +} + +fun algo313() { + var st = lazy NotFixedWidthStorage.fromSlice(""); + if (rand()) { + __expect_lazy("[st] load (bits8) (int8?) (bits9) f5, save immutable (tail), skip (int8 | bool) (address) (bits8), load f9"); + st.f5 = createEmptyCell(); + st.f9; + st.toCell(); + } +} + + +fun main() { +} + +/** +@testcase | 140 | | 2 +@testcase | 141 | -1 | 1 +@testcase | 301 | | 17 +@testcase | 303 | | 1 2 +@testcase | 304 | | 1 2 +@testcase | 305 | | 1 2 + +@fif_codegen +""" + algo40 PROC:<{ + x{0102} PUSHSLICE + 8 LDI + NIP + 8 LDI + DROP + }> +""" + */ diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk new file mode 100644 index 000000000..36d7491d4 --- /dev/null +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -0,0 +1,1981 @@ +import "@stdlib/tvm-dicts.tolk" + +struct A { a1: int7; a2: uint8; a3: uint8; } +struct B { b1: int7; b2: uint8; b3: uint8; } +type AOrB = A | B; + +@noinline +fun A.getA1(self) { + return self.a1; +} + +struct Fields9 { + f1: uint8; f2: uint8; f3: uint8; + f4: uint8; f5: uint8; f6: uint8; + f7: uint8; f8: uint8; f9: uint8; +} + +@noinline +fun Fields9.method(self) {} + +fun Fields9.getF2(self) { return self.f2 } +fun Fields9.getF5(self) { return self.f5 } + +struct Small { s1: int7; } +struct Big { b1: int7; b2: int8; b3: int8; } +type SmallOrBig = Small | Big; + +struct StructF1F2AB { + f1: int8; + f2: int8; + ab: A | B; +} + +struct StructF1SBF3 { + f1: int8; + sb: Small | Big; + f3: int8; +} + +struct WithEitherABMiddle { + f1: int32; + f2: int32; + ab: A | B; + f4: int32; + f5: int32; + f6: int32; +} + +struct WithEitherABEnd { + f1: int8; + f2: int8; + f3: int8; + f4: int8; + f5: int8; + ab: A | B; +} + +struct Point { + x: int8; + y: int8; +} + +fun Point.getX(self) { + return self.x +} + +fun Point.makeCell(self) { + return self.toCell() +} + +fun Point.makeCellWrapped(self) { + val cc = self.makeCell(); + self.x; + return cc +} + +fun Point.incXAndMakeCell(mutate self) { + __expect_inline(true); + self.x += 1; + return self.toCell() +} + +struct Point3d { + x: int32; + y: int32; + z: int32; +} + +type PointOrPoint3d = Point | Point3d; + +struct Point78ForMaybe { + x: int7; + y: int8; +} + +type OptionalPoint78ForMaybe = Point78ForMaybe?; + +struct ComplexWithPoint { + f1: int8; + p: Point; + f3: int8; +} + +struct ComplexWithMaybePoint { + f1: int8; + p: Point78ForMaybe?; + f3: int8; +} + +struct Counter7Increment { byValue: int7 } +struct Counter7Decrement { byValue: int7 } +type MsgEitherCounter = Counter7Increment | Counter7Decrement; + +struct FieldsAndEitherCounter { + f1: int8; + f2: int8; + c: MsgEitherCounter; +} + +struct WalletStorage { + isSignatureAllowed: bool; + seqno: uint32; + walletId: uint32; + publicKey: uint256; + extensions: dict; +} + +fun WalletStorage.save(self) { + contract.setData(self.toCell()) +} + +@overflow1023_policy("suppress") +struct NotFixedWidthStorage { + f1: int8; + f2: int8?; + f3: int8; + f4: bool; + f5: cell; + f6: int8 | bool; + f7: address; + f8: int8; + f9: bits200; + f10: (bits400, ()); + f11: coins; +} + +struct Inner1 { i1: int8; i2: int8; } +struct Inner2 { i1: int8; i2: int8; } +type Inner2Alias = Inner2; +struct Outer { + inner1: Inner1; + inner2: Inner2Alias; + wa: StructF1F2AB; +} + +struct WithMaybe { + f1: int8; + big: Big?; + f3: int2; +} + +struct WithRefs { + r1: cell?; + f2: int8; + r3: Cell; + f4: int8; +} + +struct (0x01) CounterIncrement { byValue: int8 } +struct (0x02) CounterDecrement { byValue: int8 } +struct (0x03) CounterDecrementBy1 { } +struct (0x04) CounterReset { initialValue: int32 } +type MsgFullCounter = CounterIncrement | CounterDecrement | CounterDecrementBy1 | CounterReset; + +struct(0x12345678) Counter32Increment { byValue: int32; } +struct(0x23456789) Counter32DecrementBy1 {} +struct(0x34567890) Counter32Reset { initial: uint32; f2: int8; f3: int8; f4: int8; f5: int8; } +type MyMsg32 = Counter32Increment | Counter32DecrementBy1 | Counter32Reset; + +struct AutoP1 { f1: int8; f2: int8; } +struct AutoP2 { f1: int8; f2: int8; f3: int8; } +struct AutoP3; +type AutoPrefixedBin2 = AutoP1 | AutoP2 | AutoP3; // 00 / 01 / 10 + +struct(0b0111) ShortP1 { f1: int8; } +struct(0b0100) ShortP2 { f1: int8; f2: int8; } +type ShortPrefixed = ShortP1 | ShortP2; + +struct(0b01) NotEqPrefixedS1 { f: uint6; } +struct(0b0001) NotEqPrefixedS2 { f: uint4; } +struct(0b101) NotEqPrefixedS3 { f: int5; } +type NotEqPrefixed = NotEqPrefixedS1 | NotEqPrefixedS2 | NotEqPrefixedS3; + +struct WithBitsField { + bin16: bits16; + bin8: bits8; +} + +struct WithOneRestField { + i32: int32; + rest: RemainingBitsAndRefs; +} + +struct VestingStorage { + vestingParameters: Cell<(uint64, int32)>; + seqno: uint32; + subwalletId: uint32; + publicKey: uint256; + whitelist: dict; +} + +struct OneIntOneRef { + z: cell? = null; + o: cell; + e: () = (); + i: int32 = 0; + r: cell; +} + + +@noinline +fun contract.getFakeData(seqno: int): Cell { + return WalletStorage { + isSignatureAllowed: true, + seqno: seqno, + walletId: 999, + publicKey: 1 << 100, + extensions: null, + }.toCell() +} + +fun getPointCell(x: int, y: int): Cell { + val p: Point = {x,y}; + return p.toCell(); +} + +fun loadWalletStorage() { + return WalletStorage.fromSlice(stringHexToSlice("8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_")); +} + +fun WalletStorage.load() { + return WalletStorage.fromCell(contract.getFakeData(777)); +} + +@method_id(101) +fun demo101() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x"); + return p.x; +} + +@method_id(102) +fun demo102(takeX: bool) { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + var result: int; + __expect_lazy("[p] load x y"); + if (takeX) { + result = p.x; + } else { + result = p.y; + } + return result; +} + +@method_id(103) +fun demo103() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] skip (bits8), load y"); + return p.y * p.y * p.y; +} + +@method_id(104) +fun demo104() { + var p2 = lazy Point.fromSlice(stringHexToSlice("0A14")); + __expect_lazy("[p2] load x y"); + assert(p2.x < 100) throw 100; + assert(p2.y < 100) throw 100; + + var c = lazy ComplexWithPoint.fromSlice(stringHexToSlice("01020304")); + __expect_lazy("[c] load f1, skip (bits16), load f3"); + return (c.f1, 0 + c.f3); +} + +@method_id(105) +fun demo105(s: slice) { + var c = lazy ComplexWithMaybePoint.fromSlice(s); + __expect_lazy("[c] load f1, skip (Point78ForMaybe?), load f3"); + return (c.f1, c.f3); +} + +@method_id(106) +fun demo106(s: slice) { + var c = lazy ComplexWithMaybePoint.fromSlice(s); + var x: int? = null; + __expect_lazy("[c] skip (bits8), load p"); + if (c.p != null) { + x = c.p.x; + } + return (c.p == null, x); +} + +@method_id(107) +fun demo107() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + return p; +} + +@method_id(108) +fun demo108() { + var o = lazy Fields9.fromSlice(stringHexToSlice("010203040506070809")); + __expect_lazy("[o] load f1 f2 f3 f4 f5 f6 f7 f8 f9"); + { + assert(o.f3 == 3) throw 100; + assert(o.f9 == 9) throw 100; + } + + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x y"); + p.y = 15; + return (p, o); +} + +@method_id(109) +fun demo109() { + var st = lazy WalletStorage.fromSlice(stringHexToSlice("8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_")); + __expect_lazy("[st] skip (bits65), load publicKey"); + return st.publicKey; +} + +@method_id(110) +fun demo110(s: slice) { + var msg = lazy MsgEitherCounter.fromSlice(s); + var t = createEmptyTuple(); + __expect_lazy("[msg] lazy match"); + match (msg) { + Counter7Increment => { + __expect_lazy("[msg] load byValue"); + t.push(+msg.byValue) + } + Counter7Decrement => { + __expect_lazy("[msg] load byValue"); + t.push(-msg.byValue) + } + } + return t; +} + +@method_id(111) +fun demo111(s: slice, x: int) { + var msg = lazy MsgEitherCounter.fromSlice(s); + var t = createEmptyTuple(); + match (msg) { + Counter7Increment => { + assert(x >= 0, 123); + __expect_lazy("[msg] load byValue"); + t.push(+msg.byValue) + } + Counter7Decrement => { + assert(x >= 1, 456); + assert(t.size() == 0, 789); + __expect_lazy("[msg] load byValue"); + t.push(-msg.byValue) + } + } + return t; +} + +@method_id(112) +fun demo112(s: slice) { + var msg = lazy MsgFullCounter.fromSlice(s); + var curX = 0; + __expect_lazy("[msg] lazy match"); + match (msg) { + CounterIncrement => curX += msg.byValue, + CounterDecrement => curX -= msg.byValue, + CounterDecrementBy1 => curX = -1, + CounterReset => curX = msg.initialValue, + } + return curX; +} + +@method_id(113) +fun demo113(s: slice) { + var msg = lazy MsgFullCounter.fromSlice(s); + var curX = 0; + var newX: int = match (msg) { + CounterIncrement => curX + msg.byValue, + CounterDecrement => curX - msg.byValue, + CounterDecrementBy1 => -1, + CounterReset => msg.initialValue, + }; + return newX; +} + +@method_id(114) +fun demo114(s: slice) { + var msg = lazy MsgFullCounter.fromSlice(s); + var curX = 0; + var newX: int | [int] = match (msg) { + CounterDecrementBy1 => -1, + CounterReset => [msg.initialValue as int], + CounterDecrement => [curX - msg.byValue], + CounterIncrement => curX + msg.byValue, + }; + return newX; +} + +@method_id(115) +fun demo115(s: slice): int8 | bool { + var msg = lazy MsgEitherCounter.fromSlice(s); + return match (msg) { + Counter7Decrement => msg.byValue < 5, + Counter7Increment => msg.byValue as int8, + } +} + +@method_id(118) +fun demo118(s: slice) { + match (var msg = lazy MsgEitherCounter.fromSlice(s)) { + Counter7Increment => return +msg.byValue, + Counter7Decrement => { + assert(s.remainingRefsCount() == 0, 456); + __expect_lazy("[msg] load byValue"); + return -msg.byValue + } + } +} + +@method_id(119) +fun demo119(s: slice) { + var o = lazy SmallOrBig.fromSlice(s); + var s1 = -1; + var b1 = -1; + match (o) { + Small => { + __expect_lazy("[o] load s1"); + s1 = o.s1; + } + Big => { + __expect_lazy("[o] skip (bits7), load b2"); + b1 = o.b2; + } + } + return (s1, b1); +} + +@method_id(120) +fun demo120(s: slice) { + var msg = lazy FieldsAndEitherCounter.fromSlice(s); + var t = createEmptyTuple(); + __expect_lazy("[msg] load f1, skip (bits8), lazy match c"); + match (msg.c) { + Counter7Increment => { t.push(+msg.c.byValue) } + Counter7Decrement => { t.push(-msg.c.byValue) } + } + t.push(msg.f1); + return t; +} + +@method_id(121) +fun demo121() { + var c = FieldsAndEitherCounter{f1:1, f2:2, c:Counter7Decrement{byValue:20}}; + var o = lazy FieldsAndEitherCounter.fromCell(c.toCell()); + __expect_lazy("[o] load f1 f2 c"); // load c, not lazy match + return (o.c is Counter7Increment) ? o.f1 + o.c.byValue : o.f2 + o.c.byValue; +} + +@method_id(122) +fun demo122(s: slice): int | slice { + var o = lazy FieldsAndEitherCounter.fromSlice(s); + __expect_lazy("[o] load f1, skip (bits8), lazy match c"); + return match(o.c) { + Counter7Decrement => -o.c.byValue, + Counter7Increment => o.f1 + o.c.byValue, + } +} + +@method_id(123) +fun demo123(s: slice) { + var o = lazy FieldsAndEitherCounter.fromSlice(s); + __expect_lazy("[o] load f1 f2 c"); // not lazy match, used in a complex expression + return o.f1 + match (o.c) { + Counter7Increment => o.f1 as int, + Counter7Decrement => o.f2 * o.f2, + } +} + +@method_id(124) +fun demo124(s: slice) { + var msg = lazy FieldsAndEitherCounter.fromSlice(s); + var t = createEmptyTuple(); + __expect_lazy("[msg] skip (bits8), load f2, lazy match c"); + match (msg.c) { + Counter7Increment => { + var mm = msg.c; + t.push(+mm.byValue) + } + Counter7Decrement => { + try {} + catch {} + var mm = msg.c; + t.push(-mm.byValue); + t.push(msg.f2); + } + } + return t; +} + +@method_id(125) +fun demo125() { + var o = lazy StructF1F2AB.fromSlice(stringHexToSlice("7F200F0203")); + var (a1, a3, b2) = (-1,-1,-1); + __expect_lazy("[o] load f1 f2 ab"); + match (o.ab) { + A => { + a1 = o.ab.a1; + a3 = o.ab.a3; + } + B => { + b2 = o.ab.b2; + } + } + return (o, a1, a3, b2); +} + +@method_id(126) +fun demo126(s: slice) { + var o = lazy StructF1SBF3.fromSlice(s); + var (s1, b2) = (-1, -1); + __expect_lazy("[o] load f1 sb f3"); + match (o.sb) { + Small => { + s1 = o.sb.s1; + } + Big => { + b2 = o.sb.b2; + } + } + return (o, s1, b2); +} + +@method_id(127) +fun demo127() { + val o = lazy Outer.fromSlice(stringHexToSlice("0102030408FF7FFFFF")); + __expect_lazy("[o] load inner1, skip (bits16), load wa"); + var inner1_i1 = o.inner1.i1; + var inner1_i2 = o.inner1.i2; + + var wa_is_A: bool; + var wa_last: int; + match (o.wa.ab) { + A => { wa_is_A = true; __expect_lazy(""); wa_last = o.wa.ab.a3; } + B => { wa_is_A = false; __expect_lazy(""); wa_last = o.wa.ab.b3; } + } + return (inner1_i1, inner1_i2, wa_is_A, wa_last); +} + +@method_id(128) +fun demo128(s: slice) { + var o = lazy WithMaybe.fromSlice(s); + __expect_lazy("[o] load f1 big"); + assert (o.f1 > 0) throw 123; + var isnull = true; + var b3 = -1; + match (o.big) { + null => { + debug.printString("big is null"); + } + Big => { + isnull = false; + assert(o.big.b1 != 0, 123); + b3 = o.big.b3; + } + } + return (isnull, b3); +} + +@method_id(129) +fun demo129() { + var o = lazy AOrB.fromSlice(stringHexToSlice("0F0203")); + var (a1, wasA) = (-1, false); + __expect_lazy("[o] lazy match"); + match (o) { + A => { + __expect_lazy("[o] load a1 a2 a3"); + a1 = o.getA1(); + wasA = true; + } + B => { + __expect_lazy("[o] skip (bits7), load b2"); + a1 = o.b2; + } + } + return (wasA, a1); +} + +@method_id(130) +fun demo130() { + var st = lazy loadWalletStorage(); + __expect_lazy("[st] skip (bits33), load walletId"); + return st.walletId; +} + +@method_id(131) +fun demo131() { + var st = lazy WalletStorage.load(); + __expect_lazy("[st] skip (bits1), load seqno, skip (bits32), load publicKey"); + return (st.seqno, st.publicKey >> 100); +} + +@method_id(132) +fun demo132(incSeqno: bool) { + var st = lazy WalletStorage.fromCell(contract.getFakeData(888)); + __expect_lazy("[st] skip (bits1), load seqno, skip (bits288), load extensions"); + if (incSeqno) { + return (st.seqno + 1, st.extensions == null); + } else { + return (st.seqno as int, st.extensions == null); + } +} + +@method_id(133) +fun demo133() { + var extensions = createEmptyDict(); + extensions.iDictSet(32, 8, stringHexToSlice("12")); + var wCell = WalletStorage { + isSignatureAllowed: true, + seqno: 0, + walletId: 123, + publicKey: 1 << 88, + extensions, + }.toCell(); + if (10 > 3) { + var st = lazy wCell.load(); + __expect_lazy("[st] skip (bits321), load extensions"); + while (!false) { + assert(st.extensions != null, 123); + return st.extensions.iDictGet(32, 8).0!.loadUint(8); + } + } else { + throw 0; + } +} + +@method_id(134) +fun demo134() { + var cc = lazy CounterIncrement.fromSlice(stringHexToSlice("010f"), {throwIfOpcodeDoesNotMatch: 134}); + return cc.byValue; +} + +@method_id(135) +fun demo135(skip: bool) { + var cc = lazy WithBitsField.fromSlice(stringHexToSlice("123456")); + __expect_lazy("[cc] load bin16, save immutable (tail)"); + cc.bin16 = stringHexToSlice("00000000") as bits16; + try { + return cc.toCell({ + skipBitsNFieldsValidation: skip + }).beginParse().remainingBitsCount(); + } catch (excno) { + return excno; + } +} + +@method_id(136) +fun demo136(skip: bool): int { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + if (skip) { + return -1; + } else { + __expect_lazy("[p] skip (bits8), load y"); + return p.y; + } +} + +@method_id(137) +fun demo137(s: slice): int { + val p = lazy Point.fromSlice(s); + try { + __expect_lazy("[p] load x"); + return p.x; // will be ok even if s doesn't contain y + } catch (excno) { + return 100 + excno; + } +} + +@method_id(138) +fun demo138(s: slice): int { + val p = lazy Point.fromSlice(s); + try { + __expect_lazy("[p] skip (bits8), load y"); + return p.y; + } catch (excno) { + return 100 + excno; + } +} + +@method_id(139) +fun demo139(s: slice, skip: bool): int { + var msg = lazy MsgFullCounter.fromSlice(s); + match (msg) { + CounterIncrement => { + try { + if (s.isEnd()) { return 100; } + __expect_lazy("[msg] load byValue"); + return msg.byValue; + } catch (excno) { return excno } + } + CounterReset => throw 123, + CounterDecrementBy1 => throw 456, + CounterDecrement => { + if (!skip) { + __expect_lazy("[msg] load byValue"); + return msg.byValue; + } + return 100; + } + } +} + +@method_id(140) +fun demo140(getF4: bool) { + val o = lazy WithEitherABEnd.fromSlice(stringHexToSlice("010203040580FF00")); + var result: int; + __expect_lazy("[o] skip (bits24), load f4, skip (bits8), lazy match ab"); + if (getF4) { + result = o.f4; + } else { + match (o.ab) { A => throw 123, B => { + result = o.ab.b2; + } } + if (o.f4 < 0) { result *= 100; } + } + return result; +} + +@method_id(141) +fun demo141(eval: bool): int { + val o = lazy WithEitherABEnd.fromSlice(stringHexToSlice("010203040500FF00")); + if (eval) { + try { + __expect_lazy("[o] load f1 f2, skip (bits24), lazy match ab"); + match (o.ab) { + A => { return o.f1 } + B => { throw o.f2 } + } + } catch(excno) { return excno * 100 } + } + throw 123; +} + +@method_id(142) +fun demo142(s: slice, ignore: bool) { + val o = lazy AOrB.fromSlice(s); + match (o) { + A => { + try { + __expect_lazy("[o] skip (bits15), load a3"); + return o.a3 as int; + } catch (excno) { return excno } + } + B => { + if (ignore) { return -1; } + else { + if (ignore) { return -1; } + else { + __expect_lazy("[o] load b1"); + return o.b1 * 1; + } + } + } + } +} + +@method_id(143) +fun demo143(): int { + val o = lazy WithEitherABEnd.fromSlice(stringHexToSlice("0809000000")); // ab missing + try { try { + __expect_lazy("[o] load f1 f2, skip (bits24), lazy match ab"); + try { + match (o.ab) { + A => { return 1 } + B => { return 2 } + } + } catch { throw o.f1 + o.f2 } + } catch(excno) { return excno } } catch { return -1 } +} + +@method_id(144) +fun demo144(dummy: bool) { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + var result = -1; + if (!dummy) { + __expect_lazy("[p] load x"); + result = p.getX(); + } + return result; +} + +@method_id(145) +fun demo145(getF5: bool): int { + var p = lazy Fields9.fromSlice(stringHexToSlice("010203040506070809")); + try { + __expect_lazy("[p] skip (bits8), load f2, skip (bits16), load f5"); + if (getF5) { + return p.getF5(); + } + return 1 * p.getF2(); + } catch { throw 123 } +} + +@method_id(146) +fun demo146() { + val p = lazy WithRefs.fromSlice(stringHexToSlice("0000000")); // no mandatory ref exists, but since we don't access it, it doesn't fail + __expect_lazy("[p] load r1"); + return p.r1 == null; +} + +@method_id(147) +fun demo147() { + val p = lazy WithRefs.fromSlice(stringHexToSlice("0000000")); + if (false) {} else { + try { + __expect_lazy("[p] skip (cell?) (bits8), load f4"); + return (true, p.f4 as int); // even though `r3` (Cell) doesn't exist, it's ignored to reach `f4` + } catch (excno) { + return (false, excno); + }} +} + +@method_id(148) +fun demo148() { + var o = lazy AOrB.fromSlice(stringHexToSlice("0F0203FF")); + match (o) { + A => { + var bits: int; + __expect_lazy("[o] load a1 a2 a3"); + bits = o.forceLoadLazyObject().remainingBitsCount(); + return bits; + } + B => throw 123 + } +} + +@method_id(149) +fun demo149() { + var o = lazy AOrB.fromSlice(stringHexToSlice("8F02")); // corrupted, too small + match (o) { + A => throw 123, + B => { + try { + __expect_lazy("[o] load b1 b2 b3"); + o.forceLoadLazyObject().assertEnd(); + return o.b1 << 8 + } catch (excno) { + return excno + } + } + } +} + +@method_id(150) +fun demo150() { + val o = lazy WithOneRestField.fromSlice(stringHexToSlice("00000001FFFF")); + return (o.i32, o.rest.remainingBitsCount()); +} + +@method_id(151) +fun demo151() { + val c = VestingStorage { + whitelist: createEmptyDict(), + subwalletId: 123, + publicKey: 456, + vestingParameters: (1 as uint64, 2 as int32).toCell(), + seqno: 10, + }.toCell(); + var st = lazy c.load(); + __expect_lazy("[st] skip (Cell<(uint64, int32)>) (bits320), load whitelist"); + return st.whitelist == null; // need to skip prev bits, because dict requires 1 bit +} + +@method_id(152) +fun demo152() { + val c = OneIntOneRef { o: createEmptyCell(), r: (2 as int32).toCell() }.toCell(); + val st = lazy c.load(); + __expect_lazy("[st] skip (cell?) (cell), load r"); + return st.r.beginParse().remainingBitsCount(); +} + +@method_id(153) +fun demo153() { + val c = OneIntOneRef { o: createEmptyCell(), r: (2 as int32).toCell() }.toCell(); + val st = lazy c.load(); + __expect_lazy("[st] skip (cell?), load i"); + return st.i; +} + +global gTup: tuple; + +@noinline +fun pushToGlobalTup(v: T) { gTup.push(v.__toTuple()); } + +@method_id(154) +fun demo154() { + gTup = createEmptyTuple(); + val p = lazy Point.fromSlice(stringHexToSlice("0102")); + pushToGlobalTup(p); + return gTup; +} + +fun inlinedFunctionWithLazy(s: slice) { + __expect_inline(true); + val p = lazy Point.fromSlice(s); + assert(p.y >= 0, 555); +} + +@method_id(155) +fun demo155() { + inlinedFunctionWithLazy(stringHexToSlice("0102")); + inlinedFunctionWithLazy(stringHexToSlice("0304")); + try { + inlinedFunctionWithLazy(stringHexToSlice("03FF")); + } catch (excno) { return excno; } + return -1; +} + +@method_id(156) +fun demo156(s: slice): int { + val d = lazy CounterIncrement.fromSlice(s); + __expect_lazy("[d] lazy match"); + match (d) { + CounterIncrement => { + __expect_lazy("[d] load byValue"); + return d.byValue; + } + else => { + return -100; + } + } +} + + +@method_id(200) +fun demo200() { + var st = lazy WalletStorage.fromCell(contract.getFakeData(888)); + var b = beginCell(); + if (10>3) { + __expect_lazy("[st] load isSignatureAllowed seqno walletId publicKey extensions"); + assert(st.isSignatureAllowed, 123); + b.storeAny(st); + var st2 = lazy WalletStorage.fromCell(b.toCell()); + __expect_lazy("[st2] load isSignatureAllowed"); + if (st2.isSignatureAllowed) { + return 1; + } + } + throw 123; +} + +@method_id(201) +fun demo201() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + var c1 = p.toCell(); + var x = p.x; + var c2 = p.toCell(); + var y = p.y; + var c3 = p.toCell(); + assert(c1.beginParse().bitsEqual(c2.beginParse()), 100); + assert(c1.beginParse().bitsEqual(c3.beginParse()), 100); + return (x, y, p is Point); +} + +@method_id(202) +fun demo202() { + var o = lazy Fields9.fromSlice(stringHexToSlice("010203040506070809")); + __expect_lazy("[o] load f1 f2, save immutable (tail), skip (bits24), load f6"); + if (o.f1 == 1) { + o.f2 = 127; + } + var f6 = o.f6; + var o2 = lazy o.toCell().load(); + __expect_lazy("[o2] skip (bits8), load f2"); + return (o.f2, o2.f2, f6); +} + +@method_id(203) +fun demo203() { + var o = lazy Fields9.fromSlice(stringHexToSlice("010203040506070809")); + if (o.f1 == 1) { + return o.toCell().load(); + } + throw 123; +} + +@method_id(204) +fun demo204() { + var o = lazy Fields9.fromSlice(stringHexToSlice("010203040506070809")); + __expect_lazy("[o] load f1 f2 f3 f4, save immutable (tail)"); + if (o.f1 != 1) { + return o.toCell().load(); + } else { + o.f1 = 10; + } + if (o.f2 != 2) { + return o.toCell().load(); + } else { + o.f2 = 20; + } + if (o.f3 != 3) { + return o.toCell().load(); + } else { + o.f4 = 40; + } + return o.toCell().load(); +} + +@method_id(205) +fun demo205() { + var o = lazy WithMaybe.fromSlice(stringHexToSlice("551_")); + __expect_lazy("[o] save immutable (tail), skip (bits8), load big"); + match (o.big) { null => {} Big => {} } + return o.toCell().beginParse().bitsEqual(stringHexToSlice("551_")); +} + +@method_id(206) +fun demo206() { + var o = lazy Fields9.fromSlice(stringHexToSlice("010203040506070809")); + o.method(); + return (o.toCell() is Cell, o.toCell().beginParse().remainingBitsCount()); +} + +@method_id(207) +fun demo207() { + var o = lazy WithRefs.fromCell(WithRefs { + r1: null, + f2: 123, + r3: getPointCell(10, 20), + f4: 55, + }.toCell()); + o.r1 = getPointCell(60, 70); + o.f2 = 90; + return o.toCell(); +} + +@method_id(208) +fun demo208(s: slice) { + var o = lazy AOrB.fromSlice(s); + __expect_lazy("[o] lazy match"); + match (o) { + A => { + __expect_lazy("[o] load a1 a2 a3"); + var bc = o.toCell().beginParse().remainingBitsCount(); + return (bc, (o as A|B).toCell().beginParse().remainingBitsCount()); + } + B => { + assert(10>3, 101); + __expect_lazy("[o] load b1 b2 b3"); // loaded because of toCell + return (o.b1 as int, o.toCell().hash() as int); + } + } +} + +@method_id(209) +fun demo209() { + var o = lazy StructF1F2AB.fromSlice(stringHexToSlice("7F200F0203")); + __expect_lazy("[o] skip (bits16), lazy match ab"); + match (o.ab) { + A => { + return o.ab.toCell().beginParse().remainingBitsCount(); + } + B => throw 0xFFFF + } +} + +@method_id(210) +fun demo210() { + var o = lazy StructF1F2AB.fromSlice(stringHexToSlice("7F200F0203")); + __expect_lazy("[o] save immutable (tail), skip (bits16), load ab"); // not lazy match because of nested o.toCell() + match (o.ab) { + A => { + return o.toCell().beginParse().remainingBitsCount(); + } + B => throw 0xFFFF + } +} + +@method_id(211) +fun demo211() { + var o = lazy Counter32Reset.fromSlice(stringHexToSlice("34567890000000FF02030405")); + var f3 = -1; + __expect_lazy("[o] save immutable (tail), load initial, skip (bits8), load f3"); + if (o.initial > 100) { + f3 = o.f3; + } + assert(o.toCell().beginParse().bitsEqual(stringHexToSlice("34567890000000FF02030405")), 100); + return o.toCell(); +} + +@method_id(212) +fun demo212() { + var o = lazy Counter32Reset.fromSlice(stringHexToSlice("34567890000000FF02030405")); + __expect_lazy("[o] load initial (bits8) f3, save immutable (tail), load f4"); + if (o.initial > 100 && o.initial < 10000) { + o.f3 = o.f4; + } + return o.toCell().beginParse().bitsEqual(stringHexToSlice("34567890000000FF02040405")); +} + +@method_id(213) +fun demo213(eval: bool) { + var c = lazy ComplexWithPoint.fromSlice(stringHexToSlice("01080903FFFF")); // more than needed + if (eval && 10>3) { + __expect_lazy("[c] load (bits8) p, save immutable (tail)"); + c.p.y += 2; + var s = c.toCell().beginParse(); + return (s.remainingBitsCount(), s.skipBits(8+8).preloadUint(8)); + } else { + throw 123; + } +} + +@method_id(214) +fun demo214() { + val p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] save immutable (tail), load x"); + var s = p.makeCellWrapped().beginParse(); + return (s.loadUint(16), s.remainingBitsCount()); +} + +@method_id(215) +fun demo215() { + val p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x y"); + var s = Point.makeCell(p).beginParse(); + return (s.loadUint(16), s.remainingBitsCount()); +} + +@method_id(216) +fun demo216() { + var st = lazy WalletStorage.fromSlice(stringHexToSlice("8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_")); + __expect_lazy("[st] load (bits1) seqno, save immutable (tail), load walletId"); + st.seqno += 1; + val w = st.walletId; + st.save(); + return (contract.getData().hash() & 0xFFFF, w); +} + +@method_id(217) +fun demo217() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x y"); // because call mutating method + return p.incXAndMakeCell().beginParse().loadUint(16); +} + +@method_id(218) +fun demo218() { + var o = lazy WithRefs.fromCell(WithRefs { + r1: (1 as int32).toCell(), + f2: 2, + r3: Point{x: 10, y: 20}.toCell(), + f4: 4, + }.toCell()); + __expect_lazy("[o] load (cell?) f2, save immutable (tail), load r3"); + val (f2, pointY) = (o.f2, o.r3.load().y); + o.f2 *= 10; + return (f2, pointY, o.toCell().hash() & 0xFFFF, o.f2); +} + + +@method_id(300) +fun demo300() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] save immutable (tail)"); + return p.toCell().beginParse().bitsEqual(stringHexToSlice("0102")); +} + +@method_id(301) +fun demo301() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] save immutable (tail), load x"); + var x = p.x; + return (x, p.toCell().beginParse().bitsEqual(stringHexToSlice("0102"))); +} + +@method_id(302) +fun demo302() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + p.x = 15; + return p.toCell().beginParse().bitsEqual(stringHexToSlice("0F02")); +} + +@method_id(303) +fun demo303() { + var st = lazy WalletStorage.fromSlice(stringHexToSlice("8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_")); + __expect_lazy("[st] load isSignatureAllowed seqno, save immutable (tail)"); + st.isSignatureAllowed = false; + st.seqno += 1; + return st.toCell().hash(); +} + +@method_id(304) +fun demo304() { + var st = lazy WalletStorage.fromSlice(stringHexToSlice("8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_")); + __expect_lazy("[st] load isSignatureAllowed seqno, save immutable (tail), load walletId publicKey extensions"); + var st2 = st; + (st.isSignatureAllowed, st.seqno) = (false, st.seqno + 1); + return (st2, st.toCell().hash()); +} + +@method_id(305) +fun demo305(setX: int?) { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x, save immutable (tail)"); + if (setX != null) { + p.x = setX; + } + return p.toCell().beginParse().loadUint(8); +} + +@method_id(306) +fun demo306(setX: int?) { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x y"); + if (setX != null) { + p.x = setX; + } + return p; +} + +@method_id(307) +fun demo307() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x y"); + var cc = (p as PointOrPoint3d).toCell(); + assert(cc.beginParse().remainingBitsCount() == 1+8+8, 100); + return cc.hash(); +} + +@method_id(308) +fun demo308() { + var p = lazy Point.fromSlice(stringHexToSlice("0102")); + __expect_lazy("[p] load x y"); + var cc = (p as PointOrPoint3d).toCell(); + var ll = lazy cc.load(); + assert(cc.beginParse().remainingBitsCount() == 1+8+8, 100); + __expect_lazy("[ll] lazy match"); + match (ll) { + Point => { + assert(p.x == 1, 100); + p.y += 1; + __expect_lazy("[ll] load x y"); + ll.y += 1; + assert(p.y == ll.y, 101); + ll.y -= 1; + return (p, (ll as PointOrPoint3d).toCell().hash()); + } + Point3d => throw 0 + } +} + +@method_id(309) +fun demo309() { + var st = lazy WalletStorage.load(); + __expect_lazy("[st] load (bits65) publicKey, save immutable (tail)"); + st.publicKey += 1; + return st.toCell(); +} + +@method_id(310) +fun demo310() { + val cell = NotFixedWidthStorage { + f1: 10, + f2: null, + f3: 20, + f4: true, + f5: createEmptyCell(), + f6: 10, + f7: createAddressNone(), + f8: 9, + f9: "" as bits200, + f10: ("" as bits400, ()), + f11: 0, + }.toCell({skipBitsNFieldsValidation: true}); + var st = lazy cell.load(); + var sm = lazy cell.load(); + var sk = lazy cell.load(); + __expect_lazy("[st] load (bits8) (int8?) (bits9) (cell) f6, save immutable (tail), skip (address), load f8"); + match (st.f6) { int8 => st.f6 += 90, bool => throw 123 }; + __expect_lazy("[sm] load f5"); + sm.f5; // previous (data-only) fields are not needed to reach a ref + __expect_lazy("[sk] skip (bits8) (int8?) (bits9) (int8 | bool), load f7"); + sk.f7; // ref (f5) is not needed to be skipped to reach data + return (st.toCell().hash() & 0xFFFFFFFF, st.f8); +} + +@method_id(311) +fun demo311() { + var st = lazy WalletStorage.load(); + __expect_lazy("[st] load isSignatureAllowed (bits320) extensions"); + st.isSignatureAllowed = false; + st.extensions = null; + return st.toCell(); +} + + +@method_id(400) +fun demo400() { + var t = createEmptyTuple(); + t.push(stringHexToSlice("0102")); + t.push(stringHexToSlice("0304")); + t.push(stringHexToSlice("0506")); + var corrX: int = -1; + while (t.size() && corrX == -1) { + var p = lazy Point.fromSlice(t.pop()); + __expect_lazy("[p] load x y"); + if (p.y == 6) { + corrX = p.x; + } + } + if (corrX == -1) { + throw 123; + } + return corrX; +} + +fun helper401(s: slice, fakeHandle: bool): never { + val msg = lazy MsgFullCounter.fromCell(beginCell().storeSlice(s).endCell()); + match (msg) { + CounterDecrementBy1 => throw 0, + CounterReset => throw 0, + CounterDecrement => { + var newX = 100; + __expect_lazy("[msg] load byValue"); + if (!fakeHandle) { + newX -= msg.byValue; + } else { + newX += msg.byValue; + } + throw newX; + } + CounterIncrement => { + var newX = 50; + if (!fakeHandle) { + __expect_lazy("[msg] load byValue"); + newX += msg.byValue; + } + throw newX; + } + else => throw 0xFFFF + } +} + +@method_id(401) +fun demo401(s: slice, fakeHandle: bool) { + try { + helper401(s, fakeHandle); + } catch (excNo) { + return excNo; + } +} + +@method_id(402) +fun demo402(s: slice) { + try { + val msg = lazy MsgFullCounter.fromSlice(s, {throwIfOpcodeDoesNotMatch: 0xFFFF}); + __expect_lazy("[msg] lazy match"); + match (msg) { + CounterDecrementBy1 => {} + CounterReset => {} + CounterDecrement => {} + CounterIncrement => {} + // without else, 0xFFFF is thrown by default + } + return 0; + } catch (excno) { + return excno; + } +} + +@method_id(403) +fun demo403(s: slice) { + val msg = lazy MsgFullCounter.fromSlice(s); + var st = lazy loadWalletStorage(); + var t = createEmptyTuple(); + __expect_lazy("[st] skip (bits1), load seqno walletId"); + match (msg) { + CounterIncrement => t.push(st.seqno), + CounterDecrementBy1 => throw 0, + CounterReset => throw 0, + CounterDecrement => { + t.push(st.walletId); + if (t.size() == 1) { + __expect_lazy("[msg] load byValue"); + t.push(msg.byValue); + } + } + } + return t; +} + +@method_id(404) +fun demo404(s: slice) { + var msg = lazy MyMsg32.fromSlice(s, {throwIfOpcodeDoesNotMatch: 0xFFFF}); + var curCounter = 10; + __expect_lazy("[msg] lazy match"); + match (msg) { + Counter32Increment => { + __expect_lazy("[msg] load byValue"); + curCounter += msg.byValue; + } + Counter32DecrementBy1 => { + curCounter -= 1; + } + Counter32Reset => { + __expect_lazy("[msg] load initial, skip (bits8), load f3"); + assert (msg.initial >= 0) throw 0xFF; + if (msg.initial > 100) { + msg.initial = 100; + } + curCounter = msg.initial; + if (msg.f3 == 100) { + curCounter *= 2; + } + } + } + return curCounter; +} + + +@method_id(405) +fun demo405(s: slice) { + var sum = 0; + try { var msg = lazy AutoPrefixedBin2.fromSlice(s); match (msg) { + AutoP3 => { + sum = -1; + } + AutoP1 => { + __expect_lazy("[msg] load f1 f2"); + sum = msg.f1 + msg.f2; + } + AutoP2 => { + __expect_lazy("[msg] load f1 f2 f3"); + sum = msg.f1; + sum += msg.f2; + if (sum > 0) { + sum += msg.f3; + } + } + } } catch { sum = -2 } + return sum; +} + + +@method_id(406) +fun demo406(s: slice) { + try { + match (val msg = lazy ShortPrefixed.fromSlice(s)) { + ShortP1 => { + return 1; + } + ShortP2 => { + __expect_lazy("[msg] skip (bits8), load f2"); + var sum = 2 + msg.f2; + return sum; + } + else => { throw 9999; } + } + } catch(excno) { return excno } +} + +@method_id(407) +fun demo407(s: slice): int { + match (val msg = lazy NotEqPrefixed.fromSlice(s)) { + NotEqPrefixedS1 => return msg.f, + NotEqPrefixedS2 => return msg.f, + NotEqPrefixedS3 => return msg.f, + } + throw 0xFF; +} + +@method_id(408) +fun demo408() { + var o = lazy Outer.fromSlice(stringHexToSlice("0102030408FF7FFFFF")); + __expect_lazy("[o] skip (bits16), load inner2 wa"); + if (10>3) { + o.inner2.i2 = 10; + } + return o.wa.ab.toCell().beginParse().remainingBitsCount(); +} + +@method_id(409) +fun demo409() { + var o = lazy Counter32Reset.fromSlice(stringHexToSlice("34567890000000FF02030405")); + var f3 = -1; + __expect_lazy("[o] load initial, skip (bits8), load f3"); + if (o.initial > 100) { + f3 = o.f3; + } + return (o.initial, f3); +} + +@method_id(410) +fun demo410(s: slice) { + val msg = lazy MsgFullCounter.fromSlice(s); + // via codegen, check that "return" is implicitly added, and IFJMP is generated + match (msg) { + CounterIncrement => { debug.print(1) } + CounterDecrement => { debug.dumpStack() } + CounterReset => { random.setSeed(10) } + CounterDecrementBy1 => { reserveToncoinsOnBalance(10, 1) } + else => { /* do nothing, "just accept tons" */ } + } +} + +@method_id(411) +fun demo411(s: slice) { + val msg = lazy AOrB.fromSlice(s); + // the same, but for either + match (msg) { + A => { debug.print(1) } + B => { debug.dumpStack() } + } +} + + +fun main() { + val s = stringHexToSlice("0102"); + var p = lazy Point.fromSlice(s); + __expect_lazy("[p] skip (bits8), load y"); + return (p.y, s.remainingBitsCount()); // s not changed +} + + +/** +@testcase | 0 | | 2 16 +@testcase | 101 | | 1 +@testcase | 102 | 0 | 2 +@testcase | 103 | | 8 +@testcase | 104 | | 1 4 +@testcase | 105 | x{01041_} | 1 8 +@testcase | 105 | x{01820304} | 1 4 +@testcase | 106 | x{01041_} | -1 (null) +@testcase | 106 | x{01820304} | 0 2 +@testcase | 107 | | 1 2 +@testcase | 108 | | 1 15 1 2 3 4 5 6 7 8 9 +@testcase | 109 | | 0 +@testcase | 110 | x{08} | [ 8 ] +@testcase | 110 | x{88} | [ -8 ] +@testcase | 111 | x{08} 50 | [ 8 ] +@testcase | 111 | x{88} 50 | [ -8 ] +@testcase | 112 | x{0110} | 16 +@testcase | 112 | x{020f} | -15 +@testcase | 112 | x{03} | -1 +@testcase | 112 | x{03ffff} | -1 +@testcase | 112 | x{04000000FF} | 255 +@testcase | 113 | x{0110} | 16 +@testcase | 113 | x{020f} | -15 +@testcase | 113 | x{03} | -1 +@testcase | 113 | x{03ffff} | -1 +@testcase | 113 | x{04000000FF} | 255 +@testcase | 114 | x{0110} | 16 1 +@testcase | 114 | x{020f} | [ -15 ] typeid-25 +@testcase | 114 | x{03} | -1 1 +@testcase | 114 | x{03ffff} | -1 1 +@testcase | 114 | x{04000000FF} | [ 255 ] typeid-25 +@testcase | 115 | x{08} | 8 42 +@testcase | 115 | x{88} | 0 2 +@testcase | 115 | x{83} | -1 2 +@testcase | 118 | x{08} | 8 +@testcase | 118 | x{88} | -8 +@testcase | 119 | x{0F} | 15 -1 +@testcase | 119 | x{8F0203} | -1 2 +@testcase | 120 | x{010208} | [ 8 1 ] +@testcase | 120 | x{010288} | [ -8 1 ] +@testcase | 121 | | 22 +@testcase | 122 | x{010208} | 9 1 +@testcase | 122 | x{010288} | -8 1 +@testcase | 123 | x{010280} | 5 +@testcase | 124 | x{010283} | [ -3 2 ] +@testcase | 125 | | 127 32 15 2 3 typeid-1 15 3 -1 +@testcase | 126 | x{7F0F20} | 127 (null) (null) 15 typeid-3 32 15 -1 +@testcase | 126 | x{7F8F020320} | 127 15 2 3 typeid-4 32 -1 2 +@testcase | 127 | | 1 2 -1 255 +@testcase | 128 | x{5500} | -1 -1 +@testcase | 128 | x{558F01024} | 0 2 +@testcase | 129 | | -1 15 +@testcase | 130 | | 456 +@testcase | 131 | | 777 1 +@testcase | 132 | -1 | 889 -1 +@testcase | 132 | 0 | 888 -1 +@testcase | 133 | | 18 +@testcase | 134 | | 15 +@testcase | 135 | -1 | 40 +@testcase | 135 | 0 | 9 +@testcase | 136 | -1 | -1 +@testcase | 136 | 0 | 2 +@testcase | 137 | x{0102} | 1 +@testcase | 137 | x{03} | 3 +@testcase | 137 | x{1} | 109 +@testcase | 138 | x{0102} | 2 +@testcase | 138 | x{03} | 109 +@testcase | 139 | x{0105} 0 | 5 +@testcase | 139 | x{01} 0 | 9 +@testcase | 139 | x{0230} 0 | 48 +@testcase | 139 | x{0230} -1 | 100 +@testcase | 140 | -1 | 4 +@testcase | 140 | 0 | 255 +@testcase | 141 | -1 | 1 +@testcase | 142 | x{010203} 0 | 3 +@testcase | 142 | x{880203} 0 | 8 +@testcase | 142 | x{0102} 0 | 9 +@testcase | 143 | | 17 +@testcase | 144 | 0 | 1 +@testcase | 145 | 0 | 2 +@testcase | 145 | -1 | 5 +@testcase | 146 | | -1 +@testcase | 147 | | -1 0 +@testcase | 148 | | 8 +@testcase | 149 | | 9 +@testcase | 150 | | 1 16 +@testcase | 151 | | -1 +@testcase | 152 | | 32 +@testcase | 153 | | 0 +@testcase | 154 | | [ [ 1 2 ] ] +@testcase | 155 | | 555 +@testcase | 156 | x{0110} | 16 +@testcase | 156 | x{FF10} | -100 + +@testcase | 200 | | 1 +@testcase | 201 | | 1 2 -1 +@testcase | 202 | | 127 127 6 +@testcase | 203 | | 1 2 3 4 5 6 7 8 9 +@testcase | 204 | | 10 20 3 40 5 6 7 8 9 +@testcase | 205 | | -1 +@testcase | 206 | | -1 72 +@testcase | 207 | | C{11A18FA6FB377116ABE1827A4469944C29484C8B00B33BDC4E560D405D5BFDAD} +@testcase | 208 | x{0F0203} | 23 24 +@testcase | 209 | | 23 +@testcase | 210 | | 40 +@testcase | 211 | | C{32B95044B7193DA671CFD5822D3CAA23DF5055A7E75622EBCB16D31C23C50AA8} +@testcase | 212 | | -1 +@testcase | 213 | -1 | 48 11 +@testcase | 214 | | 258 0 +@testcase | 215 | | 258 0 +@testcase | 216 | | 12826 456 +@testcase | 217 | | 514 +@testcase | 218 | | 2 20 11636 20 + +@testcase | 300 | | -1 +@testcase | 301 | | 1 -1 +@testcase | 302 | | -1 +@testcase | 303 | | 31704026347210867689619328142399241654002690318751313036781135835273797831937 +@testcase | 304 | | -1 123 456 0 (null) 31704026347210867689619328142399241654002690318751313036781135835273797831937 +@testcase | 305 | null | 1 +@testcase | 305 | 88 | 88 +@testcase | 306 | null | 1 2 +@testcase | 306 | 88 | 88 2 +@testcase | 307 | | 79635787059049818071080046066850883641967849619803682012164346688156057263776 +@testcase | 308 | | 1 3 79635787059049818071080046066850883641967849619803682012164346688156057263776 +@testcase | 309 | | C{EE7C06B69DDCEFEBED507FE35F603A9019D4AE79A13F670FE4E08928AF484838} +@testcase | 310 | | 3332695883 9 +@testcase | 311 | | C{16A4757FA412AD5E89418F1263904CD57FA4DF0E73EE8A53DC60F862C72BCD9D} + +@testcase | 400 | | 5 +@testcase | 401 | x{0110} -1 | 50 +@testcase | 401 | x{020f} 0 | 85 +@testcase | 402 | x{020f} | 0 +@testcase | 402 | x{ffffffff} | 65535 +@testcase | 403 | x{0110} | [ 123 ] +@testcase | 403 | x{020f} | [ 456 15 ] +@testcase | 404 | x{1234567800000010} | 26 +@testcase | 404 | x{23456789} | 9 +@testcase | 404 | x{345678900000000002030405} | 0 +@testcase | 405 | x{01c220_} | 15 +@testcase | 405 | x{41c20260_} | 24 +@testcase | 405 | x{a0_} | -1 +@testcase | 405 | x{ffff} | -2 +@testcase | 406 | x{7} | 1 +@testcase | 406 | x{40102} | 4 +@testcase | 406 | x{99} | 9999 +@testcase | 407 | x{7F} | 63 +@testcase | 407 | x{1F} | 15 +@testcase | 407 | x{BF} | -1 +@testcase | 408 | | 24 +@testcase | 409 | | 255 3 + +@fif_codegen +""" + demo101 PROC:<{ // + x{0102} PUSHSLICE // lazyS + 8 LDI // '9 lazyS + DROP // p.x + }> +""" + +@fif_codegen +""" + demo103 PROC:<{ // + x{0102} PUSHSLICE // lazyS + 8 PUSHINT + SDSKIPFIRST // lazyS + 8 LDI // '10 lazyS + DROP // p.y +""" + +@fif_codegen +""" + demo106 PROC:<{ // s + // lazyS + PUSHNULL // lazyS x + SWAP // x lazyS + 8 PUSHINT + SDSKIPFIRST // x lazyS + 1 LDU // x '18 lazyS +""" + +@fif_codegen +""" + demo109 PROC:<{ // + x{8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_} PUSHSLICE // lazyS + 65 PUSHINT + SDSKIPFIRST // lazyS + 256 LDU // '13 lazyS + DROP // st.publicKey + }> +""" + +@fif_codegen +""" + demo113 PROC:<{ + // lazyS + x{01} SDBEGINSQ + IF:<{ + 8 LDI + DROP + }>ELSE<{ + x{02} SDBEGINSQ + IF:<{ + 8 LDI + DROP + NEGATE + }>ELSE<{ + x{03} SDBEGINSQ + IF:<{ + DROP + -1 PUSHINT + }>ELSE<{ + x{04} SDBEGINSQ + IFNOTJMP:<{ + 63 THROW + }> + 32 LDI + DROP + }> + }> + }> + }> +""" + +@fif_codegen +""" + demo134 PROC:<{ // + x{010f} PUSHSLICE // lazyS + x{01} SDBEGINSQ // lazyS '6 + 134 THROWIFNOT // lazyS + 8 LDI // '11 lazyS + DROP // cc + }> +""" + +@fif_codegen +""" + demo136 PROC:<{ + x{0102} PUSHSLICE + SWAP + IFJMP:<{ + DROP + -1 PUSHINT + }> + 8 PUSHINT + SDSKIPFIRST + 8 LDI + DROP + }> +""" + +@fif_codegen +""" + demo146 PROC:<{ + x{0000000} PUSHSLICE + LDOPTREF + DROP + ISNULL + }> +""" + +@fif_codegen +""" + demo201 PROC:<{ + x{0102} PUSHSLICE + DUP + 8 LDI + 8 LDI + DROP + s2 PUSH + NEWC + STSLICE + ENDC + s3 PUSH + NEWC + STSLICE + ENDC + s0 s4 XCHG + NEWC + STSLICE + ENDC +""" + + +@fif_codegen +""" + demo203 PROC:<{ // + x{010203040506070809} PUSHSLICE // lazyS + DUP // '14 lazyS + 8 LDU // '14 '16 lazyS + DROP // '14 o.f1 + 1 EQINT // '14 '18 + IFJMP:<{ // '14 + NEWC // '14 b + STSLICE // b + ENDC // '21 + CTOS // s +""" + +@fif_codegen +""" + demo206 PROC:<{ // + x{010203040506070809} PUSHSLICE // lazyS + DUP // '14 lazyS + 8 LDU // '14 o.f1 lazyS + 8 LDU // '14 o.f1 o.f2 lazyS + 8 LDU // '14 o.f1 o.f2 o.f3 lazyS + 8 LDU // '14 o.f1 o.f2 o.f3 o.f4 lazyS + 8 LDU // '14 o.f1 o.f2 o.f3 o.f4 o.f5 lazyS + 8 LDU // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 lazyS + 8 LDU // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 o.f7 lazyS + 8 LDU // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 o.f7 o.f8 lazyS + 8 LDU // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 o.f7 o.f8 '32 lazyS + DROP // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 o.f7 o.f8 o.f9 +""" + +@fif_codegen +""" + demo215 PROC:<{ + x{0102} PUSHSLICE + 8 LDI + 8 LDI + DROP + SWAP + NEWC + 8 STI + 8 STI + ENDC +""" + +@fif_codegen +""" + demo302 PROC:<{ + x{0102} PUSHSLICE + 8 LDI + NIP + 15 PUSHINT + NEWC + 8 STI + STSLICE +""" + +@fif_codegen +""" + demo311 PROC:<{ // + 777 PUSHINT // '6=777 + contract.getFakeData CALLDICT // '7 + CTOS // lazyS + 1 LDI // '13 lazyS + NIP // lazyS + 320 PUSHINT // lazyS '14=320 + LDSLICEX // '15 lazyS + DROP // '15 + PUSHNULL // '15 st.extensions + 0 PUSHINT + NEWC + 1 STU // '15 st.extensions b + s1 s2 XCHG // st.extensions '15 b + STSLICE // st.extensions b + STOPTREF // b + ENDC // '21 + }> +""" + +@fif_codegen +""" + demo404 PROC:<{ + // lazyS + x{12345678} SDBEGINSQ + IF:<{ + 32 LDI + DROP + 10 ADDCONST + }>ELSE<{ + x{23456789} SDBEGINSQ + IF:<{ + DROP + 9 PUSHINT + }>ELSE<{ + x{34567890} SDBEGINSQ + IFNOTJMP:<{ + 16 PUSHPOW2DEC + THROWANY + }> + 32 LDU + 8 PUSHINT + SDSKIPFIRST + 8 LDI + DROP + OVER + 100 GTINT + IF:<{ + 100 PUSHINT + s2 POP + }> + 100 EQINT + IF:<{ + 1 LSHIFT# + }> + }> + }> + }> +""" + +@fif_codegen +""" + demo410 PROC:<{ + // lazyS + x{01} SDBEGINSQ + IFJMP:<{ + ... + }> + x{02} SDBEGINSQ + IFJMP:<{ + DROP + DUMPSTK + }> + x{04} SDBEGINSQ + IFJMP:<{ + ... + }> + x{03} SDBEGINSQ + NIP + IFJMP:<{ + ... + RAWRESERVE + }> + }> +""" + +@fif_codegen +""" + demo411 PROC:<{ // s + // lazyS + 1 LDU // '10 lazyS + DROP // '10 + IFJMP:<{ // + DUMPSTK // + }> // + 1 PUSHINT // '11=1 + s0 DUMP DROP // + }> +""" + +@fif_codegen DECLPROC pushToGlobalTup + */ diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index 4a9a431e7..7c224bab4 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -404,8 +404,8 @@ fun test28(getEmpty: bool) { return (t1, t1 is Empty, t1 is (int, int), t1 is builder, 777, t2, t2 is Empty, t2 is Has2EmptyNullable, t2 == null); } -struct Empty1 {} -struct Empty2 {} +struct Empty1; +struct Empty2; @method_id(129) fun test29() { diff --git a/tolk/CMakeLists.txt b/tolk/CMakeLists.txt index 4db9233a9..0705876f3 100644 --- a/tolk/CMakeLists.txt +++ b/tolk/CMakeLists.txt @@ -23,12 +23,14 @@ set(TOLK_SOURCE pipe-constant-folding.cpp pipe-optimize-boolean-expr.cpp pipe-detect-inline-in-place.cpp + pipe-lazy-load-insertions.cpp pipe-ast-to-legacy.cpp pipe-find-unused-symbols.cpp pipe-generate-fif-output.cpp type-system.cpp smart-casts-cfg.cpp generics-helpers.cpp + lazy-helpers.cpp abscode.cpp analyzer.cpp asmops.cpp diff --git a/tolk/ast-aux-data.h b/tolk/ast-aux-data.h index 59006b1fc..37cbcfa84 100644 --- a/tolk/ast-aux-data.h +++ b/tolk/ast-aux-data.h @@ -17,6 +17,7 @@ #pragma once #include "ast.h" +#include "lazy-helpers.h" /* * This file contains a schema of aux_data inside ast_artificial_aux_vertex @@ -35,4 +36,27 @@ struct AuxData_ForceFiftLocation final : ASTAuxData { } }; +// AuxData_LazyObjectLoadFields is a special auto-inserted vertex to load fields of a lazy struct; +// example: `var p = lazy Point.fromSlice(s); aux "load x"; return p.x` +struct AuxData_LazyObjectLoadFields final : ASTAuxData { + LocalVarPtr var_ref; // comes from `lazy` + TypePtr union_variant; // not just `o` but `match(o) { V1 => here }` + StructFieldPtr field_ref; // not just `o` but `match(o.field) { V1 => here }` + LazyStructLoadInfo load_info; // instructions, which fields to load, which to skip, etc. + + AuxData_LazyObjectLoadFields(LocalVarPtr var_ref, TypePtr union_variant, StructFieldPtr field_ref, LazyStructLoadInfo load_info) + : var_ref(var_ref), union_variant(union_variant), field_ref(field_ref), load_info(std::move(load_info)) { + } +}; + +// AuxData_LazyMatchForUnion wraps `match(lazy_var)` or its field +struct AuxData_LazyMatchForUnion final : ASTAuxData { + LocalVarPtr var_ref; // comes from `lazy` + StructFieldPtr field_ref; // not `match(o)`, but `match(o.field)` + + AuxData_LazyMatchForUnion(LocalVarPtr var_ref, StructFieldPtr field_ref) + : var_ref(var_ref), field_ref(field_ref) { + } +}; + } // namespace tolk diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 5c5953415..4ad6ea954 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -690,7 +690,7 @@ static V parse_match_arm(Lexer& lex) { } lex.expect(tok_double_arrow, "`=>`"); - AnyExprV body; + V body = nullptr; if (lex.tok() == tok_opbrace) { // pattern => { ... } AnyV v_block = parse_statement(lex); body = createV(v_block->loc, v_block); @@ -704,7 +704,9 @@ static V parse_match_arm(Lexer& lex) { AnyV v_block = createV(v_return->loc, v_return->loc, {v_return}); body = createV(v_block->loc, v_block); } else { - body = parse_expr(lex); + AnyExprV unbraced_expr = parse_expr(lex); + AnyV v_block = createV(unbraced_expr->loc, unbraced_expr->loc, {createV(unbraced_expr->loc, unbraced_expr)}); + body = createV(unbraced_expr->loc, v_block); } if (pattern_expr == nullptr) { // for match by type / default case, empty vertex, not nullptr @@ -731,13 +733,14 @@ static V parse_match_expression(Lexer& lex) { // after `pattern => { ... }` comma is optional, after `pattern => expr` mandatory bool was_comma = lex.tok() == tok_comma; // trailing comma is allowed always + bool was_unbraced = v_arm->get_body()->get_block_statement()->size() == 1 && v_arm->get_body()->get_block_statement()->get_item(0)->kind == ast_braced_yield_result; if (was_comma) { lex.next(); } if (lex.tok() == tok_clbrace) { break; } - if (!was_comma && v_arm->get_body()->kind != ast_braced_expression) { + if (!was_comma && was_unbraced) { lex.unexpected("`,`"); } } @@ -745,6 +748,14 @@ static V parse_match_expression(Lexer& lex) { return createV(loc, std::move(subject_and_arms)); } +static V parse_lazy_operator(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_lazy, "`lazy`"); + + AnyExprV expr = parse_expr(lex); + return createV(loc, expr); +} + // parse (expr) / [expr] / identifier / number static AnyExprV parse_expr100(Lexer& lex) { SrcLocation loc = lex.cur_location(); @@ -852,6 +863,8 @@ static AnyExprV parse_expr100(Lexer& lex) { return createV(loc, nullptr, parse_object_body(lex)); case tok_match: return parse_match_expression(lex); + case tok_lazy: + return parse_lazy_operator(lex); default: lex.unexpected(""); } @@ -1529,18 +1542,20 @@ static AnyV parse_struct_field(Lexer& lex) { static V parse_struct_body(Lexer& lex) { SrcLocation loc = lex.cur_location(); - lex.expect(tok_opbrace, "`{`"); - std::vector fields; - while (lex.tok() != tok_clbrace) { - fields.push_back(parse_struct_field(lex)); - if (lex.tok() == tok_comma || lex.tok() == tok_semicolon) { - lex.next(); - } else if (lex.tok() != tok_clbrace) { - lex.unexpected("`;` or `,`"); + + if (lex.tok() != tok_semicolon) { // `struct A;` equal to `struct A {}` + lex.expect(tok_opbrace, "`{`"); + while (lex.tok() != tok_clbrace) { + fields.push_back(parse_struct_field(lex)); + if (lex.tok() == tok_comma || lex.tok() == tok_semicolon) { + lex.next(); + } else if (lex.tok() != tok_clbrace) { + lex.unexpected("`;` or `,`"); + } } + lex.expect(tok_clbrace, "`}`"); } - lex.expect(tok_clbrace, "`}`"); return createV(loc, std::move(fields)); } diff --git a/tolk/ast-replacer.h b/tolk/ast-replacer.h index 03e7d4320..581babee3 100644 --- a/tolk/ast-replacer.h +++ b/tolk/ast-replacer.h @@ -95,6 +95,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } @@ -118,6 +119,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } @@ -140,6 +142,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { case ast_empty_expression: return replace(v->as()); case ast_parenthesized_expression: return replace(v->as()); case ast_braced_expression: return replace(v->as()); + case ast_braced_yield_result: return replace(v->as()); case ast_artificial_aux_vertex: return replace(v->as()); case ast_tensor: return replace(v->as()); case ast_bracket_tuple: return replace(v->as()); @@ -163,6 +166,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { case ast_cast_as_operator: return replace(v->as()); case ast_is_type_operator: return replace(v->as()); case ast_not_null_operator: return replace(v->as()); + case ast_lazy_operator: return replace(v->as()); case ast_match_expression: return replace(v->as()); case ast_match_arm: return replace(v->as()); case ast_object_field: return replace(v->as()); diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index 2aabec2b6..19956e467 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -84,6 +84,9 @@ class ASTReplicator final { static V clone(V v) { return createV(v->loc, clone(v->get_block_statement())); } + static V clone(V v) { + return createV(v->loc, clone(v->get_expr())); + } static V clone(V v) { return createV(v->loc, clone(v->get_wrapped_expr()), v->aux_data, v->inferred_type); } @@ -153,6 +156,9 @@ class ASTReplicator final { static V clone(V v) { return createV(v->loc, clone(v->get_expr())); } + static V clone(V v) { + return createV(v->loc, clone(v->get_expr())); + } static V clone(V v) { return createV(v->loc, clone(v->get_all_children())); } @@ -273,6 +279,7 @@ class ASTReplicator final { case ast_empty_expression: return clone(v->as()); case ast_parenthesized_expression: return clone(v->as()); case ast_braced_expression: return clone(v->as()); + case ast_braced_yield_result: return clone(v->as()); case ast_artificial_aux_vertex: return clone(v->as()); case ast_tensor: return clone(v->as()); case ast_bracket_tuple: return clone(v->as()); @@ -296,6 +303,7 @@ class ASTReplicator final { case ast_cast_as_operator: return clone(v->as()); case ast_is_type_operator: return clone(v->as()); case ast_not_null_operator: return clone(v->as()); + case ast_lazy_operator: return clone(v->as()); case ast_match_expression: return clone(v->as()); case ast_match_arm: return clone(v->as()); case ast_object_field: return clone(v->as()); diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index 395f95681..b9ee06979 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -45,6 +45,7 @@ class ASTStringifier final : public ASTVisitor { {ast_empty_expression, "ast_empty_expression"}, {ast_parenthesized_expression, "ast_parenthesized_expression"}, {ast_braced_expression, "ast_braced_expression"}, + {ast_braced_yield_result, "ast_braced_yield_result"}, {ast_artificial_aux_vertex, "ast_artificial_aux_vertex"}, {ast_tensor, "ast_tensor"}, {ast_bracket_tuple, "ast_bracket_tuple"}, @@ -68,6 +69,7 @@ class ASTStringifier final : public ASTVisitor { {ast_cast_as_operator, "ast_cast_as_operator"}, {ast_is_type_operator, "ast_is_type_operator"}, {ast_not_null_operator, "ast_not_null_operator"}, + {ast_lazy_operator, "ast_lazy_operator"}, {ast_match_expression, "ast_match_expression"}, {ast_match_arm, "ast_match_arm"}, {ast_object_field, "ast_object_field"}, @@ -302,6 +304,7 @@ class ASTStringifier final : public ASTVisitor { case ast_empty_expression: return handle_vertex(v->as()); case ast_parenthesized_expression: return handle_vertex(v->as()); case ast_braced_expression: return handle_vertex(v->as()); + case ast_braced_yield_result: return handle_vertex(v->as()); case ast_artificial_aux_vertex: return handle_vertex(v->as()); case ast_tensor: return handle_vertex(v->as()); case ast_bracket_tuple: return handle_vertex(v->as()); @@ -325,6 +328,7 @@ class ASTStringifier final : public ASTVisitor { case ast_cast_as_operator: return handle_vertex(v->as()); case ast_is_type_operator: return handle_vertex(v->as()); case ast_not_null_operator: return handle_vertex(v->as()); + case ast_lazy_operator: return handle_vertex(v->as()); case ast_match_expression: return handle_vertex(v->as()); case ast_match_arm: return handle_vertex(v->as()); case ast_object_field: return handle_vertex(v->as()); diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h index 926ec5808..6e38ba792 100644 --- a/tolk/ast-visitor.h +++ b/tolk/ast-visitor.h @@ -67,7 +67,7 @@ class ASTVisitor { } GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTExprBlockOfStatements* v) { - visit_children(v->child_block_statement->as()); + visit(v->child_block_statement->as()); } GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTStatementUnary* v) { @@ -104,6 +104,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } @@ -127,6 +128,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } @@ -150,6 +152,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { case ast_empty_expression: return visit(v->as()); case ast_parenthesized_expression: return visit(v->as()); case ast_braced_expression: return visit(v->as()); + case ast_braced_yield_result: return visit(v->as()); case ast_artificial_aux_vertex: return visit(v->as()); case ast_tensor: return visit(v->as()); case ast_bracket_tuple: return visit(v->as()); @@ -173,6 +176,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { case ast_cast_as_operator: return visit(v->as()); case ast_is_type_operator: return visit(v->as()); case ast_not_null_operator: return visit(v->as()); + case ast_lazy_operator: return visit(v->as()); case ast_match_expression: return visit(v->as()); case ast_match_arm: return visit(v->as()); case ast_object_field: return visit(v->as()); diff --git a/tolk/ast.cpp b/tolk/ast.cpp index 27c7e4c13..6e3b3b890 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -152,6 +152,10 @@ void Vertex::assign_is_negated(bool is_negated) { this->is_negated = is_negated; } +void Vertex::assign_dest_var_ref(LocalVarPtr dest_var_ref) { + this->dest_var_ref = dest_var_ref; +} + void Vertex::assign_resolved_pattern(MatchArmKind pattern_kind, AnyExprV pattern_expr) { this->pattern_type_node = nullptr; this->pattern_kind = pattern_kind; @@ -190,6 +194,10 @@ void Vertex::assign_first_unreachable(AnyV first_unreachabl this->first_unreachable = first_unreachable; } +void Vertex::assign_new_children(std::vector&& children) { + this->children = std::move(children); +} + void Vertex::assign_target(const DotTarget& target) { this->target = target; } diff --git a/tolk/ast.h b/tolk/ast.h index 400c849b1..1da1d6e6c 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -75,6 +75,7 @@ enum ASTNodeKind { ast_empty_expression, ast_parenthesized_expression, ast_braced_expression, + ast_braced_yield_result, ast_artificial_aux_vertex, ast_tensor, ast_bracket_tuple, @@ -98,6 +99,7 @@ enum ASTNodeKind { ast_cast_as_operator, ast_is_type_operator, ast_not_null_operator, + ast_lazy_operator, ast_match_expression, ast_match_arm, ast_object_field, @@ -499,6 +501,7 @@ template<> // it can contain arbitrary statements inside // it can occur only in special places within the input code, not anywhere // example: `match (intV) { 0 => { ... } }` rhs of 0 is braced expression +// example: `match (intV) { 0 => 1 }` rhs is implicitly wrapped to a braced expression with `1` yielded struct Vertex final : ASTExprBlockOfStatements { auto get_block_statement() const { return child_block_statement->as(); } @@ -506,6 +509,16 @@ struct Vertex final : ASTExprBlockOfStatements { : ASTExprBlockOfStatements(ast_braced_expression, loc, child_block_statement) {} }; +template<> +// ast_braced_yield_result is a special vertex to return a value from a braced expression +// example: `match (intV) { 0 => 1 }` rhs of 0 is implicitly wrapped to a braced expression with `1` yielded +struct Vertex final : ASTExprUnary { + AnyExprV get_expr() const { return child; } + + Vertex(SrcLocation loc, AnyExprV expr) + : ASTExprUnary(ast_braced_yield_result, loc, expr) {} +}; + template<> // ast_artificial_aux_vertex is a compiler-inserted vertex that can't occur in source code // example: special vertex to force location in Fift output at constant usage @@ -880,6 +893,21 @@ struct Vertex final : ASTExprUnary { : ASTExprUnary(ast_not_null_operator, loc, expr) {} }; +template<> +// ast_lazy_operator is `lazy (loading-expr)` for lazy/partial loading +// example: `lazy Storage.fromCell(contract.getData())` +struct Vertex final : ASTExprUnary { + LocalVarPtr dest_var_ref = nullptr; // `var st = lazy Storage.load()` + + AnyExprV get_expr() const { return child; } + + Vertex* mutate() const { return const_cast(this); } + void assign_dest_var_ref(LocalVarPtr dest_var_ref); + + Vertex(SrcLocation loc, AnyExprV expr) + : ASTExprUnary(ast_lazy_operator, loc, expr) {} +}; + template<> // ast_match_expression is `match (subject) { ... arms ... }`, used either as a statement or as an expression // example: `match (intOrSliceVar) { int => 1, slice => 2 }` @@ -907,12 +935,12 @@ struct Vertex final : ASTExprBinary { AnyTypeV pattern_type_node; // for MatchArmKind::exact_type; otherwise nullptr AnyExprV get_pattern_expr() const { return lhs; } - AnyExprV get_body() const { return rhs; } // remember, it may be V + auto get_body() const { return rhs->as(); } Vertex* mutate() const { return const_cast(this); } void assign_resolved_pattern(MatchArmKind pattern_kind, AnyExprV pattern_expr); - Vertex(SrcLocation loc, MatchArmKind pattern_kind, AnyTypeV pattern_type_node, AnyExprV pattern_expr, AnyExprV body) + Vertex(SrcLocation loc, MatchArmKind pattern_kind, AnyTypeV pattern_type_node, AnyExprV pattern_expr, V body) : ASTExprBinary(ast_match_arm, loc, pattern_expr, body) , pattern_kind(pattern_kind), pattern_type_node(pattern_type_node) {} }; @@ -999,6 +1027,7 @@ struct Vertex final : ASTStatementVararg { Vertex* mutate() const { return const_cast(this); } void assign_first_unreachable(AnyV first_unreachable); + void assign_new_children(std::vector&& children); Vertex(SrcLocation loc, SrcLocation loc_end, std::vector&& items) : ASTStatementVararg(ast_block_statement, loc, std::move(items)) diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index d4db1edec..f0d8de7cb 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1491,6 +1491,9 @@ void define_builtins() { define_builtin_method("T.getDeclaredPackPrefixLen", typeT, {}, Int, declReceiverT, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal | FunctionData::flagAllowAnyWidthT); + define_builtin_method("T.forceLoadLazyObject", typeT, {typeT}, Slice, declReceiverT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); define_builtin_method("Cell.load", CellT, {CellT, UnpackOptions}, typeT, declReceiverT, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); @@ -1520,6 +1523,9 @@ void define_builtins() { define_builtin_func("__expect_inline", {Bool}, Unit, nullptr, compile_expect_type, FunctionData::flagMarkedAsPure); + define_builtin_func("__expect_lazy", {Slice}, Unit, nullptr, + compile_expect_type, + FunctionData::flagMarkedAsPure); define_builtin_method("T.__toTuple", typeT, {typeT}, TypeDataTuple::create(), declReceiverT, compile_any_object_to_tuple, FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); diff --git a/tolk/fwd-declarations.h b/tolk/fwd-declarations.h index 981eaa4a6..f46def4ac 100644 --- a/tolk/fwd-declarations.h +++ b/tolk/fwd-declarations.h @@ -60,4 +60,6 @@ enum class FunctionInlineMode { noInline, }; +typedef int var_idx_t; + } // namespace tolk diff --git a/tolk/lazy-helpers.cpp b/tolk/lazy-helpers.cpp new file mode 100644 index 000000000..22ef5fff2 --- /dev/null +++ b/tolk/lazy-helpers.cpp @@ -0,0 +1,101 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "lazy-helpers.h" +#include "tolk.h" +#include "type-system.h" + +namespace tolk { + +/* + * This file contains "lazy" state across multiple files. + * They all are used after `lazy` operators have been processed and "load xxx" vertices have been inserted. + * Particularly, while transforming AST to Ops. + * For comments about laziness, see pipe-lazy-load-insertions.cpp. + */ + +LazyVariableLoadedState::LazyVariableLoadedState(TypePtr declared_type, std::vector&& ir_slice, std::vector&& ir_options) + : declared_type(declared_type) + , ir_slice(std::move(ir_slice)) + , ir_options(std::move(ir_options)) + , loaded_state(declared_type->unwrap_alias()->try_as() ? declared_type->unwrap_alias()->try_as()->struct_ref : nullptr) { + // fill loaded_variants: variants of a lazy union or the last field of a struct if it's a union + const TypeDataUnion* t_union = declared_type->unwrap_alias()->try_as(); + if (is_struct() && loaded_state.original_struct->get_num_fields()) { + t_union = loaded_state.original_struct->fields.back()->declared_type->unwrap_alias()->try_as(); + } + if (t_union) { + variants_state.reserve(t_union->size()); + for (TypePtr variant : t_union->variants) { + const TypeDataStruct* variant_struct = variant->unwrap_alias()->try_as(); + variants_state.emplace_back(variant_struct ? variant_struct->struct_ref : nullptr); + } + } +} + +const LazyStructLoadedState* LazyVariableLoadedState::get_struct_state(StructPtr original_struct) const { + if (loaded_state.original_struct == original_struct) { + return &loaded_state; + } + for (const LazyStructLoadedState& struct_state : variants_state) { + if (struct_state.original_struct == original_struct) { + return &struct_state; + } + } + return nullptr; +} + +void LazyVariableLoadedState::assert_field_loaded(StructPtr original_struct, StructFieldPtr original_field) const { + // on field access `point.x`, ensure that it's loaded, so the value on a stack is not an occasional null + const LazyStructLoadedState* struct_state = get_struct_state(original_struct); + tolk_assert(struct_state); + tolk_assert(struct_state->was_loaded_once()); + StructFieldPtr hidden_field = struct_state->hidden_struct->find_field(original_field->name); + tolk_assert(hidden_field); + tolk_assert(struct_state->ith_field_was_loaded[hidden_field->field_idx]); +} + +void LazyStructLoadedState::on_started_loading(StructPtr hidden_struct) { + this->hidden_struct = hidden_struct; + this->ith_field_was_loaded.resize(hidden_struct->get_num_fields()); // initially false +} + +void LazyStructLoadedState::on_original_field_loaded(StructFieldPtr hidden_field) { + this->ith_field_was_loaded[hidden_field->field_idx] = true; + // for example, `var p = lazy Point; aux "load x"; return p.x`; + // we are at "load x", it exists in Point, here just save it was loaded (for assertions and debugging); + // apart from saving, stack is also updated when loading, `p` becomes `valueX null` +} + +void LazyStructLoadedState::on_aside_field_loaded(StructFieldPtr hidden_field, std::vector&& ir_field_gap) { + this->ith_field_was_loaded[hidden_field->field_idx] = true; + this->aside_gaps_and_tail.emplace_back(hidden_field, std::move(ir_field_gap)); + // for example, `var st = lazy Storage; aux "load gap, load seqno"; st.seqno += 1; st.toCell()`; + // we are at "load gap", it does not exist in Storage, so save loaded value separately +} + +std::vector LazyStructLoadedState::get_ir_loaded_aside_field(StructFieldPtr hidden_field) const { + // for example, `var st = lazy Storage; aux "load gap, load seqno"; st.seqno += 1; st.toCell()`; + // we are at "st.toCell()" that stores immutable gap before modified "seqno" + for (const auto& [gap_field, ir_field_gap] : this->aside_gaps_and_tail) { + if (gap_field == hidden_field) { + return ir_field_gap; + } + } + tolk_assert(false); +} + +} // namespace tolk diff --git a/tolk/lazy-helpers.h b/tolk/lazy-helpers.h new file mode 100644 index 000000000..8fea95b7b --- /dev/null +++ b/tolk/lazy-helpers.h @@ -0,0 +1,91 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "fwd-declarations.h" +#include + +namespace tolk { + +// LazyStructLoadInfo describes how to load a struct: which fields to load, which to skip. +// It's calculated based on variable usages and passed through the pipeline carried by auxiliary AST vertices. +// Based on it, lazy loading Ops are generated in pack-unpack api. +// To understand `hidden_struct`, read pipe-lazy-load-insertions.cpp. +struct LazyStructLoadInfo { + enum ActionWithField { + LoadField, + SkipField, + LazyMatchField, + SaveImmutableTail, + }; + + StructPtr original_struct; // original (e.g. `Point`) + StructPtr hidden_struct; // "lazy Point" — only requested fields, matching binary shape + std::vector ith_field_action; // each for corresponding field of a struct + + LazyStructLoadInfo(StructPtr original_struct, StructPtr hidden_struct, std::vector&& ith_field_action) + : original_struct(original_struct) + , hidden_struct(hidden_struct) + , ith_field_action(std::move(ith_field_action)) { + } +}; + +// LazyStructLoadedState represents state (which fields were already loaded) while generating AST to Ops. +// For example, variable `var p = lazy Point.fromSlice(s); aux "load x"; return p.x` is initially "nothing loaded", +// and after "load x" ith_field_action[0] becomes true (and `p` is updated on a stack and becomes `valueX null`). +struct LazyStructLoadedState { + StructPtr original_struct; // original (e.g. `Point`) + StructPtr hidden_struct = nullptr; // "lazy Point" — only requested fields, matching binary shape + std::vector ith_field_was_loaded; // each for corresponding field of hidden_struct + std::vector>> aside_gaps_and_tail; + + explicit LazyStructLoadedState(StructPtr original_struct) + : original_struct(original_struct) {} + + void on_started_loading(StructPtr hidden_struct); + void on_original_field_loaded(StructFieldPtr hidden_field); + void on_aside_field_loaded(StructFieldPtr hidden_field, std::vector&& ir_field_gap); + + bool was_loaded_once() const { return hidden_struct != nullptr; } + std::vector get_ir_loaded_aside_field(StructFieldPtr hidden_field) const; + + LazyStructLoadedState* mutate() const { return const_cast(this); } +}; + +// LazyVariableLoadedState contains a state of a whole lazy variable while generating AST to Ops. +// For example, `var p = lazy Point.fromSlice(s)` contains one struct. +// But `var msg = lazy MyMsgUnion.fromSlice(s)` contains N variants, each with own state, but common lazy slice `s`. +// When inlining a function, like `p.getX()`, `self` of `getX` also becomes a lazy variable pointing to the same state. +struct LazyVariableLoadedState { + TypePtr declared_type; + std::vector ir_slice; // filled by `lazy` operator + std::vector ir_options; // same, comes from `lazy T.fromSlice(s, options)` + LazyStructLoadedState loaded_state; // for struct: filled; for union: empty + std::vector variants_state; // variants of a lazy union or the last field if it's a union + + LazyVariableLoadedState(TypePtr declared_type, std::vector&& ir_slice, std::vector&& ir_options); + + bool is_struct() const { return loaded_state.original_struct != nullptr; } + bool is_union() const { return loaded_state.original_struct == nullptr; } + + const LazyStructLoadedState* get_struct_state(StructPtr original_struct) const; + void assert_field_loaded(StructPtr original_struct, StructFieldPtr original_field) const; +}; + + +} // namespace tolk + diff --git a/tolk/lexer.cpp b/tolk/lexer.cpp index 94620dc25..44d166768 100644 --- a/tolk/lexer.cpp +++ b/tolk/lexer.cpp @@ -354,6 +354,7 @@ struct ChunkIdentifierOrKeyword final : ChunkLexerBase { if (str == "self") return tok_self; if (str == "tolk") return tok_tolk; if (str == "type") return tok_type; + if (str == "lazy") return tok_lazy; if (str == "enum") return tok_enum; break; case 5: diff --git a/tolk/lexer.h b/tolk/lexer.h index dbe0f34b7..63f0e6a30 100644 --- a/tolk/lexer.h +++ b/tolk/lexer.h @@ -118,6 +118,7 @@ enum TokenType { tok_if, tok_else, tok_match, + tok_lazy, tok_arrow, tok_double_arrow, diff --git a/tolk/pack-unpack-api.cpp b/tolk/pack-unpack-api.cpp index 107130d59..62ef8d7bb 100644 --- a/tolk/pack-unpack-api.cpp +++ b/tolk/pack-unpack-api.cpp @@ -16,6 +16,7 @@ */ #include "pack-unpack-api.h" #include "generics-helpers.h" +#include "lazy-helpers.h" #include "type-system.h" #include @@ -182,6 +183,14 @@ bool check_struct_can_be_packed_or_unpacked(TypePtr any_type, bool is_pack, std: return true; } +static int calc_offset_on_stack(StructPtr struct_ref, int field_idx) { + int stack_offset = 0; + for (int i = 0; i < field_idx; ++i) { + stack_offset += struct_ref->get_field(i)->declared_type->get_width_on_stack(); + } + return stack_offset; +} + // -------------------------------------------- // high-level API for outer code @@ -254,6 +263,126 @@ std::vector generate_skip_struct_in_slice(CodeBlob& code, SrcLocation return ir_slice; // return mutated slice } +void generate_lazy_struct_from_slice(CodeBlob& code, SrcLocation loc, const LazyVariableLoadedState* lazy_variable, const LazyStructLoadInfo& load_info, const std::vector& ir_obj) { + StructPtr original_struct = load_info.original_struct; + StructPtr hidden_struct = load_info.hidden_struct; + tolk_assert(hidden_struct->fields.size() == load_info.ith_field_action.size()); + + const LazyStructLoadedState* loaded_state = lazy_variable->get_struct_state(original_struct); + tolk_assert(loaded_state && !loaded_state->was_loaded_once()); + loaded_state->mutate()->on_started_loading(hidden_struct); + + UnpackContext ctx(code, loc, lazy_variable->ir_slice, lazy_variable->ir_options); + + if (hidden_struct->opcode.exists()) { + ctx.loadAndCheckOpcode(hidden_struct->opcode); + } + + for (int field_idx = 0; field_idx < hidden_struct->get_num_fields(); ++field_idx) { + StructFieldPtr hidden_field = hidden_struct->get_field(field_idx); + tolk_assert(!loaded_state->ith_field_was_loaded[field_idx]); + + // note that as opposed to regular loading, lazy loading doesn't return rvect, it fills stack slots (ir_obj) instead + switch (load_info.ith_field_action[field_idx]) { + case LazyStructLoadInfo::LoadField: { + if (StructFieldPtr original_field = original_struct->find_field(hidden_field->name)) { + tolk_assert(hidden_field->declared_type == original_field->declared_type); + std::vector ir_field = ctx.generate_unpack_any(hidden_field->declared_type); + int stack_offset = calc_offset_on_stack(original_struct, original_field->field_idx); + int stack_width = hidden_field->declared_type->get_width_on_stack(); + code.emplace_back(loc, Op::_Let, std::vector(ir_obj.begin() + stack_offset, ir_obj.begin() + stack_offset + stack_width), std::move(ir_field)); + loaded_state->mutate()->on_original_field_loaded(hidden_field); + } else { + tolk_assert(hidden_field->name == "(gap)"); + std::vector ir_gap = ctx.generate_unpack_any(hidden_field->declared_type); + loaded_state->mutate()->on_aside_field_loaded(hidden_field, std::move(ir_gap)); + } + break; + } + case LazyStructLoadInfo::SkipField: { + ctx.generate_skip_any(hidden_field->declared_type); + break; + } + case LazyStructLoadInfo::LazyMatchField: { + StructFieldPtr original_field = original_struct->find_field(hidden_field->name); + tolk_assert(original_field && hidden_field->declared_type == original_field->declared_type); + loaded_state->mutate()->on_original_field_loaded(hidden_field); + break; + } + case LazyStructLoadInfo::SaveImmutableTail: { + std::vector ir_immutable_tail = code.create_tmp_var(TypeDataSlice::create(), loc, "(lazy-tail-slice)"); + code.emplace_back(loc, Op::_Let, ir_immutable_tail, lazy_variable->ir_slice); + loaded_state->mutate()->on_aside_field_loaded(hidden_field, std::move(ir_immutable_tail)); + break; + } + } + } + + // options.assertEndAfterReading is ignored by `lazy`, because tail fields may be skipped, it's okay +} + +std::vector generate_lazy_struct_to_cell(CodeBlob& code, SrcLocation loc, const LazyStructLoadedState* loaded_state, std::vector&& ir_obj, const std::vector& ir_options) { + StructPtr original_struct = loaded_state->original_struct; + StructPtr hidden_struct = loaded_state->hidden_struct; + + std::vector rvect_builder = code.create_var(TypeDataBuilder::create(), loc, "b"); + code.emplace_back(loc, Op::_Call, rvect_builder, std::vector{}, lookup_function("beginCell")); + + PackContext ctx(code, loc, rvect_builder, ir_options); + + if (hidden_struct->opcode.exists()) { + ctx.storeUint(code.create_int(loc, hidden_struct->opcode.pack_prefix, "(struct-prefix)"), hidden_struct->opcode.prefix_len); + } + + for (int field_idx = 0; field_idx < hidden_struct->get_num_fields(); ++field_idx) { + StructFieldPtr hidden_field = hidden_struct->get_field(field_idx); + tolk_assert(loaded_state->ith_field_was_loaded[field_idx]); + + if (StructFieldPtr original_field = original_struct->find_field(hidden_field->name)) { + int stack_offset = calc_offset_on_stack(original_struct, original_field->field_idx); + int stack_width = hidden_field->declared_type->get_width_on_stack(); + std::vector ir_field(ir_obj.begin() + stack_offset, ir_obj.begin() + stack_offset + stack_width); + ctx.generate_pack_any(hidden_field->declared_type, std::move(ir_field)); + } else { + std::vector ir_gap_or_tail = loaded_state->get_ir_loaded_aside_field(hidden_field); + if (hidden_field->declared_type->unwrap_alias()->try_as()) { + ctx.storeSlice(ir_gap_or_tail[0]); + } else { + ctx.generate_pack_any(hidden_field->declared_type, std::move(ir_gap_or_tail)); + } + if (hidden_field->name == "(tail)") { + break; + } + } + } + + std::vector rvect_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(cell)"); + code.emplace_back(loc, Op::_Call, rvect_cell, std::move(rvect_builder), lookup_function("builder.endCell")); + + return rvect_cell; +} + +std::vector generate_lazy_match_for_union(CodeBlob& code, SrcLocation loc, TypePtr union_type, const LazyVariableLoadedState* lazy_variable, const LazyMatchOptions& options) { + tolk_assert(lazy_variable->ir_options.size() == 2); + UnpackContext ctx(code, loc, lazy_variable->ir_slice, lazy_variable->ir_options); + std::vector rvect_match = ctx.generate_lazy_match_any(union_type, options); + + return rvect_match; +} + +std::vector generate_lazy_object_finish_loading(CodeBlob& code, SrcLocation loc, const LazyVariableLoadedState* lazy_variable, std::vector&& ir_obj) { + tolk_assert(lazy_variable->ir_slice.size() == 1); + + // the call to `lazy_var.forceLoadLazyObject()` does not do anything: at the moment of analyzing, + // it had marked all the object as "used", all fields where loaded, and the slice points after the last field; + // so, just return the held slice + static_cast(code); + static_cast(loc); + static_cast(ir_obj); + + return lazy_variable->ir_slice; +} + PackSize estimate_serialization_size(TypePtr any_type) { EstimateContext ctx; return ctx.estimate_any(any_type); diff --git a/tolk/pack-unpack-api.h b/tolk/pack-unpack-api.h index 7737125d4..7b4d02f66 100644 --- a/tolk/pack-unpack-api.h +++ b/tolk/pack-unpack-api.h @@ -32,4 +32,13 @@ std::vector generate_skip_struct_in_slice(CodeBlob& code, SrcLocation PackSize estimate_serialization_size(TypePtr any_type); std::vector generate_estimate_size_call(CodeBlob& code, SrcLocation loc, TypePtr any_type); +struct LazyStructLoadInfo; +struct LazyStructLoadedState; +struct LazyVariableLoadedState; + +void generate_lazy_struct_from_slice(CodeBlob& code, SrcLocation loc, const LazyVariableLoadedState* lazy_variable, const LazyStructLoadInfo& load_info, const std::vector& ir_obj); +std::vector generate_lazy_struct_to_cell(CodeBlob& code, SrcLocation loc, const LazyStructLoadedState* loaded_state, std::vector&& ir_obj, const std::vector& ir_options); +std::vector generate_lazy_match_for_union(CodeBlob& code, SrcLocation loc, TypePtr union_type, const LazyVariableLoadedState* lazy_variable, const LazyMatchOptions& options); +std::vector generate_lazy_object_finish_loading(CodeBlob& code, SrcLocation loc, const LazyVariableLoadedState* lazy_variable, std::vector&& ir_obj); + } // namespace tolk diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index d139a4526..770ec56b9 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -36,7 +36,8 @@ namespace tolk { - +class LValContext; +std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr target_type = nullptr, LValContext* lval_ctx = nullptr); std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, TypePtr cmp_type, const std::vector& expr_ir_idx, SrcLocation loc, const char* debug_desc); std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc); @@ -182,6 +183,34 @@ void UnpackContext::assertEndIfOption() const { } } +void UnpackContext::throwInvalidOpcode() const { + std::vector args_throw = { option_throwIfOpcodeDoesNotMatch }; + Op& op_throw = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_throw), lookup_function("__throw")); + op_throw.set_impure_flag(); +} + +const LazyMatchOptions::MatchBlock* LazyMatchOptions::find_match_block(TypePtr variant) const { + for (const MatchBlock& b : match_blocks) { + if (b.arm_variant->get_type_id() == variant->get_type_id()) { + return &b; + } + } + tolk_assert(false); +} + +void LazyMatchOptions::save_match_result_on_arm_end(CodeBlob& code, SrcLocation loc, const MatchBlock* arm_block, std::vector&& ir_arm_result, const std::vector& ir_match_expr_result) const { + if (!is_statement) { + // if it's `match` expression (not statement), then every arm has a result, assigned to a whole `match` result + ir_arm_result = transition_to_target_type(std::move(ir_arm_result), code, arm_block->block_expr_type, match_expr_type, loc); + code.emplace_back(loc, Op::_Let, ir_match_expr_result, std::move(ir_arm_result)); + } else if (add_return_to_all_arms) { + // if it's `match` statement, even if an arm is an expression, it's void, actually + // moreover, if it's the last statement in a function, add implicit "return" to all match cases to produce IFJMP + code.emplace_back(loc, Op::_Return); + } +} + + // -------------------------------------------- // serializers with pack/unpack/skip/estimate // @@ -638,6 +667,34 @@ struct S_Either final : ISerializer { return ir_result; } + std::vector lazy_match(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc, const LazyMatchOptions& options) const { + for (const LazyMatchOptions::MatchBlock& m : options.match_blocks) { + if (m.arm_variant == nullptr) { // `else => ...` not allowed for Either + // it's not the best place to fire an error, but let it be + throw ParseError(loc, "`else` is unreachable, because this `match` has only two options (0/1 prefixes)"); + } + } + tolk_assert(options.match_blocks.size() == 2); + std::vector ir_result = code.create_tmp_var(options.match_expr_type, loc, "(match-expression)"); + std::vector ir_is_right = ctx->loadUint(1, "(eitherBit)"); + Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_is_right)); + { + code.push_set_cur(if_op.block0); + const LazyMatchOptions::MatchBlock* m_block = options.find_match_block(t_right); + std::vector ith_result = pre_compile_expr(m_block->v_body, code); + options.save_match_result_on_arm_end(code, loc, m_block, std::move(ith_result), ir_result); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_op.block1); + const LazyMatchOptions::MatchBlock* m_block = options.find_match_block(t_left); + std::vector ith_result = pre_compile_expr(m_block->v_body, code); + options.save_match_result_on_arm_end(code, loc, m_block, std::move(ith_result), ir_result); + code.close_pop_cur(loc); + } + return ir_result; + } + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { std::vector ir_is_right = ctx->loadUint(1, "(eitherBit)"); Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_is_right)); @@ -714,12 +771,54 @@ struct S_MultipleConstructors final : ISerializer { } // we're inside last ELSE - std::vector args_throw = { ctx->option_throwIfOpcodeDoesNotMatch }; - Op& op_throw = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_throw), lookup_function("__throw")); - op_throw.set_impure_flag(); + ctx->throwInvalidOpcode(); + for (int j = 0; j < t_union->size(); ++j) { + code.close_pop_cur(loc); // close all outer IFs + } + return ir_result; + } + + std::vector lazy_match(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc, const LazyMatchOptions& options) const { + std::vector opcodes_order_mapping(t_union->size(), -1); + const LazyMatchOptions::MatchBlock* else_block = nullptr; + for (int i = 0; i < static_cast(options.match_blocks.size()); ++i) { + if (options.match_blocks[i].arm_variant) { + int variant_idx = t_union->get_variant_idx(options.match_blocks[i].arm_variant); + tolk_assert(variant_idx != -1); + opcodes_order_mapping[i] = variant_idx; + } else { + tolk_assert(else_block == nullptr); + else_block = &options.match_blocks[i]; + } + } + + FunctionPtr f_tryStripPrefix = lookup_function("slice.tryStripPrefix"); + + std::vector ir_result = code.create_tmp_var(options.match_expr_type, loc, "(match-expression)"); + std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(prefix-eq)"); + + for (int i = 0; i < t_union->size(); ++i) { + StructData::PackOpcode opcode = opcodes[opcodes_order_mapping[i]]; + std::vector args = { ctx->ir_slice0, code.create_int(loc, opcode.pack_prefix, "(pack-prefix)"), code.create_int(loc, opcode.prefix_len, "(prefix-len)") }; + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_prefix_eq[0]}, std::move(args), f_tryStripPrefix); + Op& if_op = code.emplace_back(loc, Op::_If, ir_prefix_eq); + code.push_set_cur(if_op.block0); + std::vector ith_result = pre_compile_expr(options.match_blocks[i].v_body, code); + options.save_match_result_on_arm_end(code, loc, &options.match_blocks[i], std::move(ith_result), ir_result); + code.close_pop_cur(loc); + code.push_set_cur(if_op.block1); // open ELSE + } + + if (else_block) { + std::vector else_result = pre_compile_expr(else_block->v_body, code); + options.save_match_result_on_arm_end(code, loc, else_block, std::move(else_result), ir_result); + } else { + ctx->throwInvalidOpcode(); + } for (int j = 0; j < t_union->size(); ++j) { code.close_pop_cur(loc); // close all outer IFs } + return ir_result; } @@ -739,9 +838,7 @@ struct S_MultipleConstructors final : ISerializer { } // we're inside last ELSE - std::vector args_throw = { ctx->option_throwIfOpcodeDoesNotMatch }; - Op& op_throw = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_throw), lookup_function("__throw")); - op_throw.set_impure_flag(); + ctx->throwInvalidOpcode(); for (int j = 0; j < t_union->size(); ++j) { code.close_pop_cur(loc); // close all outer IFs } @@ -842,6 +939,49 @@ struct S_CustomStruct final : ISerializer { return ir_struct; } + std::vector lazy_match(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc, const LazyMatchOptions& options) const { + const LazyMatchOptions::MatchBlock* when_block = nullptr; // Point => ... + const LazyMatchOptions::MatchBlock* else_block = nullptr; // else => ... + for (const LazyMatchOptions::MatchBlock& match_block : options.match_blocks) { + if (match_block.arm_variant) { + tolk_assert(match_block.arm_variant->equal_to(TypeDataStruct::create(struct_ref))); + when_block = &match_block; + } else { + else_block = &match_block; + } + } + + std::vector ir_result = code.create_tmp_var(options.match_expr_type, loc, "(match-expression)"); + std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(prefix-eq)"); + + StructData::PackOpcode opcode = struct_ref->opcode; + if (opcode.exists()) { // it's `match` over a struct (makes sense for a struct with prefix and `else` branch) + std::vector args = { ctx->ir_slice0, code.create_int(loc, opcode.pack_prefix, "(pack-prefix)"), code.create_int(loc, opcode.prefix_len, "(prefix-len)") }; + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_prefix_eq[0]}, std::move(args), lookup_function("slice.tryStripPrefix")); + } else { + code.emplace_back(loc, Op::_Let, ir_prefix_eq, std::vector{code.create_int(loc, -1, "(true)")}); + } + Op& if_op = code.emplace_back(loc, Op::_If, ir_prefix_eq); + { + code.push_set_cur(if_op.block0); + std::vector when_result = pre_compile_expr(when_block->v_body, code); + options.save_match_result_on_arm_end(code, loc, when_block, std::move(when_result), ir_result); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_op.block1); + if (else_block) { + std::vector else_result = pre_compile_expr(else_block->v_body, code); + options.save_match_result_on_arm_end(code, loc, else_block, std::move(else_result), ir_result); + } else { + ctx->throwInvalidOpcode(); + } + code.close_pop_cur(loc); + } + + return ir_result; + } + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { if (struct_ref->opcode.exists() && ctx->get_prefix_mode() == PrefixReadMode::LoadAndCheck) { ctx->loadAndCheckOpcode(struct_ref->opcode); @@ -1048,6 +1188,20 @@ void UnpackContext::generate_skip_any(TypePtr any_type, PrefixReadMode prefix_mo this->prefix_mode = backup; } +std::vector UnpackContext::generate_lazy_match_any(TypePtr any_type, const LazyMatchOptions& options) const { + std::unique_ptr serializer = get_serializer_for_type(any_type); + if (auto* s = dynamic_cast(serializer.get())) { + return s->lazy_match(this, code, loc, options); + } + if (auto* s = dynamic_cast(serializer.get())) { + return s->lazy_match(this, code, loc, options); + } + if (auto* s = dynamic_cast(serializer.get())) { + return s->lazy_match(this, code, loc, options); + } + tolk_assert(false); +} + PackSize EstimateContext::estimate_any(TypePtr any_type, PrefixEstimateMode prefix_mode) const { PrefixEstimateMode backup = this->prefix_mode; this->prefix_mode = prefix_mode; diff --git a/tolk/pack-unpack-serializers.h b/tolk/pack-unpack-serializers.h index 06163153b..0e664b9a1 100644 --- a/tolk/pack-unpack-serializers.h +++ b/tolk/pack-unpack-serializers.h @@ -91,6 +91,22 @@ enum class PrefixReadMode { DoNothingAlreadyLoaded, }; +struct LazyMatchOptions { + struct MatchBlock { + TypePtr arm_variant; // left of `V => ...`; nullptr for `else => ...` + AnyExprV v_body; // right of `V => ...` + TypePtr block_expr_type; // for match expression, if `V => expr`, it's expr's inferred_type + }; + + TypePtr match_expr_type; // type of `match` expression, `void` for statement + bool is_statement; // it's `match` statement, not expression, so it does not return any result + bool add_return_to_all_arms; // it's the last statement in a function, add "return" to its cases for better Fift code + std::vector match_blocks; + + const MatchBlock* find_match_block(TypePtr variant) const; + void save_match_result_on_arm_end(CodeBlob& code, SrcLocation loc, const MatchBlock* arm_block, std::vector&& ir_arm_result, const std::vector& ir_match_expr_result) const; +}; + class UnpackContext { CodeBlob& code; SrcLocation loc; @@ -115,9 +131,11 @@ class UnpackContext { void skipBits(int len) const; void skipBits_var(var_idx_t ir_len) const; void assertEndIfOption() const; + void throwInvalidOpcode() const; std::vector generate_unpack_any(TypePtr any_type, PrefixReadMode prefix_mode = PrefixReadMode::LoadAndCheck) const; void generate_skip_any(TypePtr any_type, PrefixReadMode prefix_mode = PrefixReadMode::LoadAndCheck) const; + std::vector generate_lazy_match_any(TypePtr any_type, const LazyMatchOptions& options) const; }; diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index f97d7a689..a6d84a30c 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -111,10 +111,10 @@ static int calc_offset_on_stack(const TypeDataTensor* t_tensor, int index_at) { return stack_offset; } -static int calc_offset_on_stack(const TypeDataStruct* t_struct, int field_idx) { +static int calc_offset_on_stack(StructPtr struct_ref, int field_idx) { int stack_offset = 0; for (int i = 0; i < field_idx; ++i) { - stack_offset += t_struct->struct_ref->fields[i]->declared_type->get_width_on_stack(); + stack_offset += struct_ref->get_field(i)->declared_type->get_width_on_stack(); } return stack_offset; } @@ -179,7 +179,7 @@ class LValContext { stack_offset = calc_offset_on_stack(t_tensor, index_at); } else if (const TypeDataStruct* t_struct = obj_type->try_as()) { stack_width = t_struct->struct_ref->get_field(index_at)->declared_type->get_width_on_stack(); - stack_offset = calc_offset_on_stack(t_struct, index_at); + stack_offset = calc_offset_on_stack(t_struct->struct_ref, index_at); } else { tolk_assert(false); } @@ -363,6 +363,28 @@ class ClearStateAfterInlineInPlace final : public ASTVisitorFunctionBody { } }; + +// CodeBlob has a mapping [st => ptr] +const LazyVariableLoadedState* CodeBlob::get_lazy_variable(LocalVarPtr var_ref) const { + for (const LazyVarRefAtCodegen& stored : lazy_variables) { + if (stored.var_ref == var_ref) { + return stored.var_state; + } + } + return nullptr; +} + +// detect `st` by vertex "st" +const LazyVariableLoadedState* CodeBlob::get_lazy_variable(AnyExprV v) const { + if (auto as_ref = v->try_as()) { + if (LocalVarPtr var_ref = as_ref->sym->try_as()) { + return get_lazy_variable(var_ref); + } + } + return nullptr; +} + + // given `{some_expr}!`, return some_expr static AnyExprV unwrap_not_null_operator(AnyExprV v) { while (auto v_notnull = v->try_as()) { @@ -567,11 +589,21 @@ static std::vector gen_op_call(CodeBlob& code, TypePtr ret_type, SrcL return rvect; } -static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob& code, SrcLocation loc, const std::vector>& vars_per_arg, FunctionPtr called_f) { - if (called_f->is_instantiation_of_generic_function()) { +static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob& code, V v_call, const std::vector>& vars_per_arg) { + SrcLocation loc = v_call->loc; + FunctionPtr called_f = v_call->fun_maybe; + + if (called_f->is_method() && called_f->is_instantiation_of_generic_function()) { std::string_view f_name = called_f->base_fun_ref->name; TypePtr typeT = called_f->substitutedTs->typeT_at(0); + const LazyVariableLoadedState* lazy_variable = v_call->dot_obj_is_self ? code.get_lazy_variable(v_call->get_self_obj()) : nullptr; + + if (f_name == "T.toCell" && lazy_variable && lazy_variable->is_struct()) { + // in: object Lazy (partially loaded), out: Cell + std::vector ir_obj = vars_per_arg[0]; // = lazy_var_ref->ir_idx + return generate_lazy_struct_to_cell(code, loc, &lazy_variable->loaded_state, std::move(ir_obj), vars_per_arg[1]); + } if (f_name == "T.toCell") { // in: object T, out: Cell (just a cell, wrapped) std::vector ir_obj = vars_per_arg[0]; @@ -615,6 +647,19 @@ static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob if (f_name == "T.estimatePackSize") { return generate_estimate_size_call(code, loc, typeT); } + if (f_name == "T.forceLoadLazyObject") { + // in: object T, out: slice (same slice that a lazy variable holds, after loading/skipping all its fields) + if (!lazy_variable) { + fire(code.fun_ref, v_call->loc, "this method is applicable to lazy variables only"); + } + std::vector ir_obj = vars_per_arg[0]; + return generate_lazy_object_finish_loading(code, loc, lazy_variable, std::move(ir_obj)); + } + } + + if (called_f->is_instantiation_of_generic_function()) { + std::string_view f_name = called_f->base_fun_ref->name; + TypePtr typeT = called_f->substitutedTs->typeT_at(0); if (f_name == "createMessage") { std::vector ir_msg_params = vars_per_arg[0]; @@ -635,51 +680,59 @@ static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob tolk_assert(false); } -static std::vector gen_inline_fun_call_in_place(CodeBlob& code, TypePtr ret_type, V v_call, const std::vector>& vars_per_arg) { - FunctionPtr fun_ref = v_call->fun_maybe; - tolk_assert(vars_per_arg.size() == fun_ref->parameters.size()); - for (int i = 0; i < fun_ref->get_num_params(); ++i) { - const LocalVarData& param_i = fun_ref->get_param(i); +std::vector gen_inline_fun_call_in_place(CodeBlob& code, TypePtr ret_type, SrcLocation loc, FunctionPtr f_inlined, AnyExprV self_obj, bool is_before_immediate_return, const std::vector>& vars_per_arg) { + tolk_assert(vars_per_arg.size() == f_inlined->parameters.size()); + for (int i = 0; i < f_inlined->get_num_params(); ++i) { + const LocalVarData& param_i = f_inlined->get_param(i); if (!param_i.is_used_as_lval() && !param_i.is_mutate_parameter()) { // if param used for reading only, pass the same ir_idx as for an argument // it decreases number of tmp variables and leads to better optimizations // (being honest, it's quite strange that copy+LET may lead to more stack permutations) param_i.mutate()->assign_ir_idx(std::vector(vars_per_arg[i])); } else { - std::vector ith_param = code.create_var(param_i.declared_type, v_call->loc, param_i.name); - code.emplace_back(v_call->loc, Op::_Let, ith_param, vars_per_arg[i]); + std::vector ith_param = code.create_var(param_i.declared_type, loc, param_i.name); + code.emplace_back(loc, Op::_Let, ith_param, vars_per_arg[i]); param_i.mutate()->assign_ir_idx(std::move(ith_param)); } } - std::vector rvect_call = code.create_tmp_var(ret_type, v_call->loc, "(inlined-return)"); + std::vector rvect_call = code.create_tmp_var(ret_type, loc, "(inlined-return)"); std::vector* backup_outer_inline = code.inline_rvect_out; FunctionPtr backup_cur_fun = code.fun_ref; bool backup_inline_before_return = code.inlining_before_immediate_return; + auto backup_lazy_variables = code.lazy_variables; code.inline_rvect_out = &rvect_call; - code.inlining_before_immediate_return = stmt_before_immediate_return == v_call; - code.fun_ref = fun_ref; + code.inlining_before_immediate_return = is_before_immediate_return; + code.fun_ref = f_inlined; + // specially handle `point.getX()` if point is a lazy var: to make `self.toCell()` work and `self.x` asserted; + // (only methods preserve lazy, `getXOf(point)` does not, though theoretically can be done) + const LazyVariableLoadedState* lazy_receiver = self_obj ? code.get_lazy_variable(self_obj) : nullptr; + if (lazy_receiver) { + LocalVarPtr self_var_ref = &f_inlined->parameters[0]; // `self` becomes lazy while inlining + code.lazy_variables.emplace_back(self_var_ref, lazy_receiver); // (points to the same slice, immutable tail, etc.) + } - auto v_ast_root = fun_ref->ast_root->as(); + auto v_ast_root = f_inlined->ast_root->as(); auto v_block = v_ast_root->get_body()->as(); process_any_statement(v_block, code); - if (fun_ref->has_mutate_params() && fun_ref->inferred_return_type == TypeDataVoid::create()) { + if (f_inlined->has_mutate_params() && f_inlined->inferred_return_type == TypeDataVoid::create()) { std::vector mutated_vars; - for (const LocalVarData& p_sym: fun_ref->parameters) { + for (const LocalVarData& p_sym: f_inlined->parameters) { if (p_sym.is_mutate_parameter()) { mutated_vars.insert(mutated_vars.end(), p_sym.ir_idx.begin(), p_sym.ir_idx.end()); } } - code.emplace_back(v_call->loc, Op::_Let, rvect_call, std::move(mutated_vars)); + code.emplace_back(loc, Op::_Let, rvect_call, std::move(mutated_vars)); } ClearStateAfterInlineInPlace visitor; - visitor.start_visiting_function(fun_ref, v_ast_root); + visitor.start_visiting_function(f_inlined, v_ast_root); code.fun_ref = backup_cur_fun; code.inline_rvect_out = backup_outer_inline; code.inlining_before_immediate_return = backup_inline_before_return; + code.lazy_variables = std::move(backup_lazy_variables); return rvect_call; } @@ -1283,6 +1336,58 @@ static std::vector process_not_null_operator(V return transition_to_target_type(std::move(rvect), code, target_type, v); } +static std::vector process_lazy_operator(V v, CodeBlob& code, TypePtr target_type) { + // `lazy Storage.fromSlice(s)` does not load anything here, it only saves a slice for future loads; + // "future loads" are special auxiliary AST vertices "load x" that were inserted in pipe-lazy-load-insertions.cpp + auto v_call = v->get_expr()->try_as(); + tolk_assert(v_call && v_call->fun_maybe); + + FunctionPtr called_f = v_call->fun_maybe; + if (called_f->is_code_function()) { // `lazy loadStorage()` is allowed, it contains just `return ...`, inline it here + auto f_body = called_f->ast_root->as()->get_body()->as(); + tolk_assert(f_body->size() == 1 && f_body->get_item(0)->kind == ast_return_statement); + auto f_returns = f_body->get_item(0)->as(); + v_call = f_returns->get_return_value()->try_as(); + tolk_assert(v_call && v_call->fun_maybe && v_call->fun_maybe->is_builtin_function()); + called_f = v_call->fun_maybe; + } + + // only predefined built-in functions are allowed for lazy loading + tolk_assert(called_f->is_builtin_function() && called_f->is_instantiation_of_generic_function()); + std::string_view f_name = called_f->base_fun_ref->name; + std::vector ir_slice = code.create_var(TypeDataSlice::create(), v->loc, "lazyS"); + bool has_passed_options = false; + if (f_name == "T.fromSlice") { + std::vector passed_slice = pre_compile_expr(v_call->get_arg(0)->get_expr(), code); + code.emplace_back(v->loc, Op::_Let, ir_slice, std::move(passed_slice)); + has_passed_options = v_call->get_num_args() == 2; + } else if (f_name == "T.fromCell") { + std::vector ir_cell = pre_compile_expr(v_call->get_arg(0)->get_expr(), code); + code.emplace_back(v->loc, Op::_Call, ir_slice, ir_cell, lookup_function("cell.beginParse")); + has_passed_options = v_call->get_num_args() == 2; + } else if (f_name == "Cell.load") { + std::vector ir_cell = pre_compile_expr(v_call->get_callee()->try_as()->get_obj(), code); + code.emplace_back(v->loc, Op::_Call, ir_slice, ir_cell, lookup_function("cell.beginParse")); + has_passed_options = v_call->get_num_args() == 1; + } else { + tolk_assert(false); + } + + // on `var p = lazy Point.fromSlice(s, options)`, save s and options (lazy_variable) + AnyExprV v_options = has_passed_options ? v_call->get_arg(v_call->get_num_args() - 1)->get_expr() : called_f->parameters.back().default_value; + std::vector ir_options = pre_compile_expr(v_options, code, called_f->parameters[1].declared_type); + const LazyVariableLoadedState* lazy_variable = new LazyVariableLoadedState(v->dest_var_ref->declared_type, std::move(ir_slice), std::move(ir_options)); + code.lazy_variables.emplace_back(v->dest_var_ref, lazy_variable); + + // initially, all contents of `p` is filled by nulls, but before `p.x` or any other field usages, + // they will be loaded by separate AST aux vertices; + // same for unions: `val msg = lazy MyMsgUnion`, msg is N+1 nulls, but next lazy `match` will transition slots, + // which will be filled by loads + std::vector ir_null = gen_op_call(code, TypeDataNullLiteral::create(), v->loc, {}, lookup_function("__null"), "(init-null)"); + std::vector ir_initial_nulls(v->dest_var_ref->ir_idx.size(), ir_null[0]); + return transition_to_target_type(std::move(ir_initial_nulls), code, target_type, v); +} + static std::vector process_match_expression(V v, CodeBlob& code, TypePtr target_type) { TypePtr lhs_type = v->get_subject()->inferred_type->unwrap_alias(); @@ -1297,10 +1402,18 @@ static std::vector process_match_expression(V v // it's either `match` by type (all arms patterns are types) or `match` by expression bool is_match_by_type = v->get_arm(0)->pattern_kind == MatchArmKind::exact_type; + bool last_else_branch = v->get_arm(n_arms - 1)->pattern_kind == MatchArmKind::else_branch; // detect whether `match` is exhaustive bool is_exhaustive = is_match_by_type // match by type covers all cases, checked earlier || !v->is_statement() // match by expression is guaranteely exhaustive, checked earlier - || v->get_arm(n_arms - 1)->pattern_kind == MatchArmKind::else_branch; + || last_else_branch; + + // `else` is not allowed in `match` by type; this was not fired at type checking, + // because it might turned out to be a lazy match, where `else` is allowed; + // if we are here, it's not a lazy match, it's a regular one (the lazy one is handled specially, in aux vertex) + if (is_match_by_type && last_else_branch) { + throw ParseError(v->get_arm(n_arms - 1)->loc, "`else` is not allowed in `match` by type; you should cover all possible types"); + } // example 1 (exhaustive): `match (v) { int => ... slice => ... builder => ... }` // construct nested IFs: IF is int { ... } ELSE { IF is slice { ... } ELSE { ... } } @@ -1374,9 +1487,14 @@ static std::vector process_dot_access(V v, CodeBlob& return lval_ir_idx; } } + // handle `lazyPoint.x`, assert that slot for "x" is loaded (ensure lazy-loading correctness); + // same for `val msg = lazy MyMsgUnion; match(...) msg.field` inside a specific variant (struct_ref) + if (const LazyVariableLoadedState* lazy_variable = code.get_lazy_variable(v->get_obj())) { + lazy_variable->assert_field_loaded(t_struct->struct_ref, field_ref); + } std::vector lhs_vars = pre_compile_expr(v->get_obj(), code, nullptr, lval_ctx); int stack_width = field_ref->declared_type->get_width_on_stack(); - int stack_offset = calc_offset_on_stack(t_struct, field_ref->field_idx); + int stack_offset = calc_offset_on_stack(t_struct->struct_ref, field_ref->field_idx); std::vector rvect{lhs_vars.begin() + stack_offset, lhs_vars.begin() + stack_offset + stack_width}; // an object field might be smart cast at this point, for example we're in `if (user.t != null)` // it means that we must drop the null flag (if `user.t` is a tensor), or maybe perform other stack transformations @@ -1544,9 +1662,9 @@ static std::vector process_function_call(V v, Code } std::vector rvect_call; if (fun_ref->is_compile_time_special_gen()) { - rvect_call = gen_compile_time_code_instead_of_fun_call(code, v->loc, vars_per_arg, fun_ref); + rvect_call = gen_compile_time_code_instead_of_fun_call(code, v, vars_per_arg); } else if (fun_ref->is_inlined_in_place() && fun_ref->is_code_function()) { - rvect_call = gen_inline_fun_call_in_place(code, op_call_type, v, vars_per_arg); + rvect_call = gen_inline_fun_call_in_place(code, op_call_type, v->loc, v->fun_maybe, v->get_self_obj(), v == stmt_before_immediate_return, vars_per_arg); } else { rvect_call = gen_op_call(code, op_call_type, v->loc, std::move(args_vars), fun_ref, "(fun-call)", arg_order_already_equals_asm); } @@ -1587,12 +1705,18 @@ static std::vector process_function_call(V v, Code } static std::vector process_braced_expression(V v, CodeBlob& code, TypePtr target_type) { - // `{ ... }` used as an expression can not return a value currently (there is no syntax in a language) - // that's why it can appear only in special places, and its usage correctness has been checked - tolk_assert(v->inferred_type == TypeDataVoid::create() || v->inferred_type == TypeDataNever::create()); - process_any_statement(v->get_block_statement(), code); - static_cast(target_type); - return {}; + // generally, `{ ... }` is a block statement not returning a value; it's used to represent `match` braced arms; + // unless it's a special vertex "braced expression" (currently, only `match` arms) + std::vector implicit_rvect; + for (AnyV item : v->get_block_statement()->get_items()) { + if (auto v_return = item->try_as()) { + tolk_assert(implicit_rvect.empty()); + implicit_rvect = pre_compile_expr(v_return->get_expr(), code); + } else { + process_any_statement(item, code); + } + } + return transition_to_target_type(std::move(implicit_rvect), code, target_type, v); } static std::vector process_tensor(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { @@ -1780,6 +1904,67 @@ static std::vector process_artificial_aux_vertex(V(v->aux_data)) { + const LazyVariableLoadedState* lazy_variable = code.get_lazy_variable(data->var_ref); + tolk_assert(lazy_variable); + + std::vector ir_obj = data->var_ref->ir_idx; // loading will update stack slots of `p` + TypePtr t_orig = data->var_ref->declared_type; + + if (data->field_ref) { // extract a field from a whole lazy variable + tolk_assert(lazy_variable->is_struct()); + int stack_offset = calc_offset_on_stack(lazy_variable->loaded_state.original_struct, data->field_ref->field_idx); + int stack_width = data->field_ref->declared_type->get_width_on_stack(); + ir_obj = std::vector(ir_obj.begin() + stack_offset, ir_obj.begin() + stack_offset + stack_width); + t_orig = data->field_ref->declared_type; + } + + if (data->union_variant) { // extract a variant from a union (a union variable or a union field of a struct) + ir_obj = transition_to_target_type(std::move(ir_obj), code, t_orig, data->union_variant, v->loc); + } + + // `load_info` contains instructions to skip, load, save tail, etc.; + // it generates LETs to ir_obj, so stack slots of lazy_variable will contain loaded data + generate_lazy_struct_from_slice(code, v->loc, lazy_variable, data->load_info, ir_obj); + return transition_to_target_type({}, code, target_type, v); + } + + // aux "match(lazyUnion)" / aux "match(obj.lastUnionField)" + if (const auto* data = dynamic_cast(v->aux_data)) { + V v_match = v->get_wrapped_expr()->as(); + pre_compile_expr(v_match->get_subject(), code, nullptr); + + const LazyVariableLoadedState* lazy_variable = code.get_lazy_variable(data->var_ref); + tolk_assert(lazy_variable); + TypePtr t_union = data->field_ref ? data->field_ref->declared_type : data->var_ref->declared_type; + + std::vector match_blocks; + match_blocks.reserve(v_match->get_arms_count()); + for (int i = 0; i < v_match->get_arms_count(); ++i) { + auto v_arm = v_match->get_arm(i); + TypePtr arm_variant = nullptr; + if (v_arm->pattern_kind == MatchArmKind::exact_type) { + arm_variant = v_arm->pattern_type_node->resolved_type->unwrap_alias(); + } else { + tolk_assert(v_arm->pattern_kind == MatchArmKind::else_branch); // `else` allowed in a lazy match + } + match_blocks.emplace_back(LazyMatchOptions::MatchBlock{arm_variant, v_arm->get_body(), v_arm->get_body()->inferred_type}); + } + + LazyMatchOptions options = { + .match_expr_type = v->inferred_type, + .is_statement = v_match->is_statement(), + .add_return_to_all_arms = v == stmt_before_immediate_return, + .match_blocks = std::move(match_blocks), + }; + + // it will generate match by a slice prefix, and for each `match` arm, invoke pre_compile_expr(), + // which contains "aux load" particularly + std::vector ir_match = generate_lazy_match_for_union(code, v->loc, t_union, lazy_variable, options); + return transition_to_target_type(std::move(ir_match), code, target_type, v); + } + tolk_assert(false); } @@ -1803,6 +1988,8 @@ std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr targ return process_is_type_operator(v->as(), code, target_type); case ast_not_null_operator: return process_not_null_operator(v->as(), code, target_type, lval_ctx); + case ast_lazy_operator: + return process_lazy_operator(v->as(), code, target_type); case ast_match_expression: return process_match_expression(v->as(), code, target_type); case ast_dot_access: diff --git a/tolk/pipe-calc-rvalue-lvalue.cpp b/tolk/pipe-calc-rvalue-lvalue.cpp index d451a7c60..dc269a342 100644 --- a/tolk/pipe-calc-rvalue-lvalue.cpp +++ b/tolk/pipe-calc-rvalue-lvalue.cpp @@ -88,6 +88,13 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { restore_state(saved); } + void visit(V v) override { + mark_vertex(v); + MarkingState saved = enter_rvalue_if_none(); + parent::visit(v); + restore_state(saved); + } + void visit(V v) override { mark_vertex(v); MarkingState saved = enter_rvalue_if_none(); @@ -213,6 +220,13 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { parent::visit(v->get_expr()); // leave lvalue state unchanged, for `mutate x!` both `x!` and `x` are lvalue } + void visit(V v) override { + mark_vertex(v); + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + void visit(V v) override { mark_vertex(v); MarkingState saved = enter_state(MarkingState::RValue); diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index ba1582559..b47513ee4 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -587,7 +587,9 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { fire(cur_f, v_arm->loc, "duplicated `else` branch"); } if (has_type_arm) { - fire(cur_f, v_arm->loc, "`else` is not allowed in `match` by type; you should cover all possible types"); + // `else` is not allowed in `match` by type, but we don't fire an error here, + // because it might turn out to be a lazy `match`, where `else` is allowed; + // if it's not lazy, an error is fired later } has_else_arm = true; } diff --git a/tolk/pipe-check-rvalue-lvalue.cpp b/tolk/pipe-check-rvalue-lvalue.cpp index 075e99982..cb8067628 100644 --- a/tolk/pipe-check-rvalue-lvalue.cpp +++ b/tolk/pipe-check-rvalue-lvalue.cpp @@ -125,6 +125,13 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { parent::visit(v->get_expr()); } + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(cur_f, v, "lazy expression"); + } + parent::visit(v->get_expr()); + } + void visit(V v) override { if (v->is_lvalue) { fire_error_cannot_be_used_as_lvalue(cur_f, v, "literal"); diff --git a/tolk/pipe-generate-fif-output.cpp b/tolk/pipe-generate-fif-output.cpp index ca82cd3fb..f0bd7c5a3 100644 --- a/tolk/pipe-generate-fif-output.cpp +++ b/tolk/pipe-generate-fif-output.cpp @@ -47,7 +47,7 @@ static void generate_output_func(FunctionPtr fun_ref) { CodeBlob* code = std::get(fun_ref->body)->code; if (G.is_verbosity(3)) { - code->print(std::cerr, 9); + code->print(std::cerr, 0); } code->prune_unreachable_code(); if (G.is_verbosity(5)) { @@ -73,7 +73,7 @@ static void generate_output_func(FunctionPtr fun_ref) { } code->mark_noreturn(); if (G.is_verbosity(3)) { - code->print(std::cerr, 15); + // code->print(std::cerr, 15); } if (G.is_verbosity(2)) { std::cerr << "\n---------- resulting code for " << fun_ref->name << " -------------\n"; diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 0345ac0aa..2d396762c 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -376,10 +376,12 @@ class InferTypesAndCallsAndFieldsVisitor final { return infer_is_type_operator(v->as(), std::move(flow), used_as_condition); case ast_not_null_operator: return infer_not_null_operator(v->as(), std::move(flow), used_as_condition); + case ast_lazy_operator: + return infer_lazy_operator(v->as(), std::move(flow), used_as_condition); case ast_parenthesized_expression: return infer_parenthesized(v->as(), std::move(flow), used_as_condition, hint); case ast_braced_expression: - return infer_braced_expression(v->as(), std::move(flow), used_as_condition); + return infer_braced_expression(v->as(), std::move(flow), used_as_condition, hint); case ast_reference: return infer_reference(v->as(), std::move(flow), used_as_condition, hint); case ast_dot_access: @@ -830,16 +832,34 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(after_expr.out_flow), true); } + ExprFlow infer_lazy_operator(V v, FlowContext&& flow, bool used_as_condition) { + ExprFlow lazy_expr = infer_any_expr(v->get_expr(), std::move(flow), used_as_condition); + assign_inferred_type(v, v->get_expr()); // there is no Lazy, so `lazy expr` is just typeof expr + return lazy_expr; + } + ExprFlow infer_parenthesized(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), used_as_condition, hint); assign_inferred_type(v, v->get_expr()); return after_expr; } - ExprFlow infer_braced_expression(V v, FlowContext&& flow, bool used_as_condition) { - // `{ ... }` used as an expression can not return a value currently (there is no syntax in a language) - flow = process_any_statement(v->get_block_statement(), std::move(flow)); - assign_inferred_type(v, flow.is_unreachable() ? TypeDataNever::create() : TypeDataVoid::create()); + ExprFlow infer_braced_expression(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + // generally, `{ ... }` is a block statement not returning a value; it's used to represent `match` braced arms; + // unless it's a special vertex (used to represent a non-braced `match` arm) + AnyExprV implicit_result = nullptr; + for (AnyV item : v->get_block_statement()->get_items()) { + if (auto v_return = item->try_as()) { + tolk_assert(implicit_result == nullptr); + implicit_result = v_return; + flow = infer_any_expr(v_return->get_expr(), std::move(flow), false, hint).out_flow; + assign_inferred_type(v_return, v_return->get_expr()); + } else { + flow = process_any_statement(item, std::move(flow)); + } + } + + assign_inferred_type(v, flow.is_unreachable() ? TypeDataNever::create() : implicit_result ? implicit_result->inferred_type : TypeDataVoid::create()); return ExprFlow(std::move(flow), used_as_condition); } diff --git a/tolk/pipe-lazy-load-insertions.cpp b/tolk/pipe-lazy-load-insertions.cpp new file mode 100644 index 000000000..e7510023a --- /dev/null +++ b/tolk/pipe-lazy-load-insertions.cpp @@ -0,0 +1,1032 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "lazy-helpers.h" +#include "ast.h" +#include "ast-aux-data.h" +#include "ast-visitor.h" +#include "ast-replacer.h" +#include "type-system.h" +#include "smart-casts-cfg.h" +#include "pack-unpack-api.h" + +namespace tolk { + +/* + * This pipe finds `lazy` operators and inserts loading points to load only required fields just before being used. + * It happens after type inferring/checking. While inferring, `lazy expr` was inferred just as expr. + * There is no dedicated `Lazy` type in the type system. All the magic of laziness is calculated here. + * + * This is the second version of the algorithm. The first one (which didn't reach production) attempted to + * calculate precise loading locations right before every field usage. + * E.g., `assert(obj.f1 > 0); assert(obj.f2 > 0)` — it detected "load f1" + assert1 + "load f2" + assert2. + * However, it turned out to increase gas consumption. In practice, when a structure has only a few fields (99% cases), + * it's better to load them ALL AT ONCE rather than on demand: this results in fewer stack manipulations. + * + * `lazy` is not about lazy/partial loading, but also about partial updating. + * As opposed to non-lazy `var st = Storage.fromCell(c); st.xxx = yyy; st.toCell()`, which loads and writes all fields, + * partial updating detects immutable portions of a struct, saves them separately, and reuses on toCell(). + * + * All in all, the algorithm focuses on: + * - Identifying which fields are used for a lazy variable. + * - Loading all required fields at once (and skipping unused ones). + * - Doing lazy matching for unions, to avoid constructing heavy union types on the stack. + * - Calculating and loading used fields at the right place for every union variant. + * - Analyzing modification and gathering immutable portions at loading to be reused on saving. + * + * Example "lazy union": + * > val msg = lazy MyMessage.fromSlice(msgBody); // doesn't construct a union, actually + * > match (msg) { // not a match by type, but a lazy match by a slice prefix + * > CounterReset => { + * > assert(senderAddress == storage.owner) throw 403; + * > // <-- here "load msg.initial" is inserted + * > storage.counter = msg.initial; + * > } + * > } + * + * Example "skip unused": + * > get seqno() { + * > val storage = lazy Storage.fromCell(contract.getData()); + * > // <-- here "skip all fields before seqno, load seqno" is inserted + * > return storage.seqno; + * > } + * + * Example "nesting into try/if": + * > val st = lazy Storage.fromSlice(s); + * > try { + * > // <-- here "load necessary fields" is inserted, if they are used only in `try` body + * > st.someField + * > } + * + * Example "bubbling to closest (lca) statement": + * > val st = lazy Storage.fromSlice(s); + * > // <-- here "load necessary fields" is inserted, because they are used both in `try` and after + * > try { ... st.someField } catch {} + * > st.anotherField + * + * Example "modification and immutable tail": + * > val st = lazy loadStorage(); + * > // <-- here "load f1 f2, save immutable tail, load rest" is inserted + * > ... read all fields + * > st.f2 = newValue; // only f2 is modified, others are only read + * > contract.setData(st.toCell()); // st.toCell() writes f1 f2 and immutable tail + * + * Example "modification and immutable gap": + * > struct Storage { a: int32; b: int32; c: int32; seqno: int32; } + * > var st = lazy loadStorage(); + * > // <-- here "load 96 bits, load seqno" is inserted (note: "abc" are grouped into "96 bits") + * > st.seqno += 1; // only seqno is accessed, "abc" not, that's why they are grouped + * > st.toCell(); // writes 96 bits (grouped "abc") and seqno + * + * Implementation: "original struct" and "hidden struct". + * To group fields for loading/skipping, for every lazy variable, "hidden struct" is created, containing: + * - fields from original struct that are used + * - gaps and artificial fields that are not used, to match binary representation + * + * Example 1: + * | struct Point { x: int8, y: int8 } | hidden: struct lazyPoint { gap: bits8; y: int8 } + * | val p = lazy Point | p is initially `null null` on a stack + * | <-- "skip 8 bits, load y" | field gap skipped, y loaded AND mapped onto a stack to match p.y + * | p.y | p is now `null yValue` + * + * Example 2: + * | struct St { a,b,c; seqno; ... } | hidden: struct lazyStorage { gap: bits96; seqno: int32; tail: slice } + * | val st = lazy St | st is initially `null null null null` + * | <-- "load 96 bits, seqno, save tail" | field gap loaded, seqno loaded, tail (rest fields) saved + * | st.seqno += 1 | st is now `null null null seqno`, "gap" and "tail" kept aside st's ir_idx + * | st.toCell() | writes gap (96 bits, grouped "abc") + modified seqno + storeSlice tail + * + * In a similar way, it works for unions: `lazy UnionType` is represented exactly as `UnionType` on a stack, + * that's why type transitions and methods inlining work natively (when transforming AST to Ops). + * But for each variant, its own "lazyVariant" hidden_struct is created. Used fields are loaded and placed + * into correct placed on a stack, gaps are skipped or placed aside. + * + * Some highlights and considerations: + * - `lazy A.fromSlice(s)` does NOT read from slice immediately; instead, it saves the slice pointer and reads on demand + * (at "loading points" inserted by the compiler in the current pipe). + * - Options can be passed: `lazy A.fromSlice(s, {...})`, but `assertEndAfterReading` is ignored, it doesn't make sense + * (because fields are read later, only required ones, and the last is preloaded rather than loaded). + * There is a special method `a.forceLoadLazyObject()`, can be used inside `match` to load the variant fully. + * - The compiler detects which properties are accessed and inserts "load x y z" as close to the first usage as possible; + * it does NOT split loading into multiple instructions (first x y, then z somewhere below): it has a negative effect. + * - Inlined methods preserve laziness (e.g. `point.getX()` / `storage.save()`): the compiler analyzes the bodies of + * those methods to detect usages of `self`, and marks used fields to be loaded in advance. + * - Perfectly works for unions if a union is used only in `match`; then a union is not even created on a stack: + * instead, this `match` becomes lazy and works but cutting a slice prefix. + * - Effective toCell(): the compiler tracks which fields are modified and reuses an immutable tail slice on writing. + * - Has optimization "to reach a ref, no need to load preceding bits". + * E.g. `struct St { v: int32; content: cell; }` and `st = lazy St; st.content` does only LDREF, no "skip 32 bits". + * - The last requested field is preloaded, not loaded. + * + * There are some drawbacks (probable possible enhancements): + * - When a union is used in some way except match, lazy is inapplicable (compilation error). + * - When a union has some primitives besides structures, lazy is inapplicable (for `T?`, particularly). + * Possible enhancement: handle unions where structs are mixed with primitives. + * - When a struct has a union field, it can be lazily matched only if it's the last. + * Possible enhancement: allow lazy match for union fields in the middle. + * - When `match` is used inside complex expressions, it's not lazy for safety. + * Possible enhancement: `cond && match(...)` is unsafe to be lazy, but `1 + match(...)` could be lazy. + * - Only methods preserve laziness (`p.getX()`), functions do not. + * Possible enhancement: `getXOfPoint(p)` could also be lazy for inlined functions (now `p` is read as a whole). + */ + +// Given fun_ref = "A.fromSlice" from `lazy A.fromSlice(s)` check it's correct to be inside the `lazy` operator. +static bool does_function_satisfy_for_lazy_operator(FunctionPtr fun_ref, bool allow_wrapper = true) { + if (!fun_ref) { + return false; + } + // allow `lazy SomeStruct.fromSlice(s)`; these functions are also handled while transforming AST to Ops + if (fun_ref->is_builtin_function() && fun_ref->is_instantiation_of_generic_function()) { + std::string_view f_name = fun_ref->base_fun_ref->name; + return f_name == "T.fromSlice" || f_name == "T.fromCell" || f_name == "Cell.load"; + } + // allow `lazy loadData()`, where loadData() is a simple wrapper like + // `fun loadData() { return SomeStruct.fromCell(contract.getData()) }` + if (allow_wrapper && fun_ref->is_code_function() && fun_ref->get_num_params() == 0) { + auto f_body = fun_ref->ast_root->as()->get_body()->as(); + if (f_body->size() == 1) { + if (auto f_returns = f_body->get_item(0)->try_as(); f_returns && f_returns->has_return_value()) { + if (auto f_returns_call = f_returns->get_return_value()->try_as()) { + return does_function_satisfy_for_lazy_operator(f_returns_call->fun_maybe, false); + } + } + } + } + return false; +} + +// Currently, only `A | B | ...` (only structures) can be lazily loaded; later steps rely on StructPtr. +// For example, `(int32, int32) | ...` or `T?` are incompatible with `lazy`. +// If structs don't have prefixes, a prefix tree is built for a union, it also works. +static TypePtr is_union_type_prevented_from_lazy_loading(const TypeDataUnion* t_union) { + for (TypePtr variant : t_union->variants) { + bool is_struct = variant->unwrap_alias()->try_as(); + if (!is_struct) { + return variant; + } + } + return nullptr; +} + +// Given `lazy `, check that expr is correct: a valid function call with valid types. +// If not, fire an error. +static void check_lazy_operator_used_correctly(FunctionPtr cur_f, V v) { + bool is_ok_call = v->get_expr()->kind == ast_function_call + && does_function_satisfy_for_lazy_operator(v->get_expr()->as()->fun_maybe); + if (!is_ok_call) { + fire(cur_f, v->loc, "`lazy` operator can only be used with built-in functions like fromCell/fromSlice or simple wrappers over them"); + } + + // it should be either a struct or a union of structs + TypePtr expr_type = v->inferred_type; + if (expr_type->unwrap_alias()->try_as()) { + return; + } + if (const TypeDataUnion* expr_union = expr_type->unwrap_alias()->try_as()) { + if (TypePtr wrong_variant = is_union_type_prevented_from_lazy_loading(expr_union)) { + fire(cur_f, v->loc, "`lazy` union should contain only structures, but it contains `" + wrong_variant->as_human_readable() + "`"); + } + return; + } + fire(cur_f, v->loc, "`lazy` is applicable to structs, not to `" + expr_type->as_human_readable() + "`"); +} + +// Given `storage.save()` for a lazy `storage` variable, check if `self` inside should gain laziness. +// If yes, the body of the method is also traversed to detect usages. +// If no, it's assumed that all fields of `storage` are used (an object used "as a whole"). +static bool can_method_be_inlined_preserving_lazy(FunctionPtr method_ref) { + if (method_ref->is_builtin_function() && method_ref->is_instantiation_of_generic_function()) { + std::string_view f_name = method_ref->base_fun_ref->name; + return f_name == "T.toCell" || f_name == "T.forceLoadLazyObject"; + } + + return method_ref->is_inlined_in_place() && // only AST-inlined methods can be lazy + !method_ref->has_mutate_params() && + !method_ref->does_return_self(); +} + +// The first stage of an algorithm is to collect every lazy expression, every field, every union variant. +// This "collecting" is done inside a block, considering all nested statements. +// As a result, we know how many times a variable (and every field independently) is used for reading, writing, etc. +struct ExprUsagesWhileCollecting { + std::string name_str; // "v" / "v.field" / "v.field.nested"; for debugging only + TypePtr expr_type; // either type of variable/field or its narrowed type inside `match` + StructPtr struct_ref; // if it's a struct, otherwise, nullptr + + int used_for_reading = 0; + int used_for_writing = 0; + int used_for_matching = 0; + int used_for_toCell = 0; + int total_usages_with_fields = 0; + std::vector needed_above_stmt; + std::vector> used_as_match_subj; + + std::vector fields; // for struct: every field; otherwise: empty + std::vector variants; // for union: every variant; otherwise: itself (for match over non-union) + + ExprUsagesWhileCollecting(std::string name_for_debugging, TypePtr expr_type, bool is_variant_of_itself = false) + : name_str(std::move(name_for_debugging)) + , expr_type(expr_type) + , struct_ref(nullptr) { + if (const TypeDataUnion* expr_union = expr_type->unwrap_alias()->try_as()) { + variants.reserve(expr_union->size()); + for (int i = 0; i < expr_union->size(); ++i) { + variants.emplace_back(name_str + "(#" + std::to_string(i) + ")", expr_union->variants[i]); + } + return; + } + if (const TypeDataStruct* t_struct = expr_type->unwrap_alias()->try_as()) { + struct_ref = t_struct->struct_ref; + fields.reserve(struct_ref->get_num_fields()); + for (int i = 0; i < struct_ref->get_num_fields(); ++i) { + StructFieldPtr field_ref = struct_ref->get_field(i); + fields.emplace_back(name_str + "." + field_ref->name, field_ref->declared_type); + } + } + // to allow code like + // > val msg = lazy Counter.fromSlice(s) <-- struct! not union + // > match (msg) { Counter => {} else => {} } + // we track `msg` inside `match` as a single variant — not over union, but over itself + if (!is_variant_of_itself) { + variants.emplace_back(name_str, expr_type, true); + } + } + + void merge_with_sub_block(const ExprUsagesWhileCollecting& rhs) { + tolk_assert(expr_type->equal_to(rhs.expr_type) && struct_ref == rhs.struct_ref); + used_for_reading += rhs.used_for_reading; + used_for_writing += rhs.used_for_writing; + used_for_matching += rhs.used_for_matching; + used_for_toCell += rhs.used_for_toCell; + total_usages_with_fields += rhs.total_usages_with_fields; + needed_above_stmt.insert(needed_above_stmt.end(), rhs.needed_above_stmt.begin(), rhs.needed_above_stmt.end()); + used_as_match_subj.insert(used_as_match_subj.end(), rhs.used_as_match_subj.begin(), rhs.used_as_match_subj.end()); + for (int i = 0; i < static_cast(fields.size()); ++i) { + fields[i].merge_with_sub_block(rhs.fields[i]); + } + for (int i = 0; i < static_cast(variants.size()); ++i) { + variants[i].merge_with_sub_block(rhs.variants[i]); + } + } + + void on_used_rw(bool is_lvalue) { + is_lvalue ? used_for_writing++ : used_for_reading++; + total_usages_with_fields++; + } + + void on_used_toCell() { + used_for_toCell++; + total_usages_with_fields++; + } + + void on_used_as_match_subj(V v_match) { + used_as_match_subj.push_back(v_match); + used_for_matching++; + total_usages_with_fields++; + } + + bool is_self_or_field_used_for_reading() const { + if (used_for_reading) { + return true; + } + bool any = false; + for (const ExprUsagesWhileCollecting& field_usages : fields) { + any |= field_usages.is_self_or_field_used_for_reading() || field_usages.is_self_or_child_used_for_matching(); + } + return any; + } + + bool is_self_or_field_used_for_toCell() const { + if (used_for_toCell) { + return true; + } + bool any = false; + for (const ExprUsagesWhileCollecting& field_usages : fields) { + any |= field_usages.is_self_or_field_used_for_toCell(); + } + return any; + } + + bool is_self_or_child_used_for_writing() const { + if (used_for_writing) { + return true; + } + bool any = false; + for (const ExprUsagesWhileCollecting& field_usages : fields) { + any |= field_usages.is_self_or_child_used_for_writing(); + } + for (const ExprUsagesWhileCollecting& variant_usages : variants) { + any |= variant_usages.is_self_or_child_used_for_writing(); + } + return any; + } + + bool is_self_or_child_used_for_matching() const { + if (used_for_matching) { + return true; + } + bool any = false; + for (const ExprUsagesWhileCollecting& field_usages : fields) { + any |= field_usages.is_self_or_child_used_for_matching(); + } + for (const ExprUsagesWhileCollecting& variant_usages : variants) { + any |= variant_usages.is_self_or_child_used_for_matching(); + } + return any; + } + + void treat_match_like_read() { + if (used_for_matching) { + used_for_reading++; + total_usages_with_fields++; + } + for (ExprUsagesWhileCollecting& field_usages : fields) { + field_usages.treat_match_like_read(); + } + for (ExprUsagesWhileCollecting& variant_usages : variants) { + variant_usages.treat_match_like_read(); + } + } + + LazyStructLoadInfo generate_hidden_struct_load_all(SrcLocation loc, bool is_variant_of_union) const { + tolk_assert(struct_ref); + + StructPtr hidden_struct = new StructData( + "(lazy)" + struct_ref->name, + loc, + std::vector(struct_ref->fields), + is_variant_of_union ? StructData::PackOpcode(0, 0) : struct_ref->opcode, + struct_ref->overflow1023_policy, + nullptr, + nullptr, + struct_ref->ast_root + ); + std::vector all_fields_load_actions(struct_ref->get_num_fields(), LazyStructLoadInfo::LoadField); + + return LazyStructLoadInfo(struct_ref, hidden_struct, std::move(all_fields_load_actions)); + } + + // for every field of a struct, after calculating all usages, determine: which fields to load, and which to skip + LazyStructLoadInfo calculate_hidden_struct(SrcLocation loc, bool is_variant_of_union) const { + tolk_assert(struct_ref); + + struct FutureField { + LazyStructLoadInfo::ActionWithField action; + std::string_view field_name; + TypePtr field_type; + PackSize pack_size; + + FutureField(LazyStructLoadInfo::ActionWithField action, std::string_view field_name, TypePtr field_type) + : action(action), field_name(field_name), field_type(field_type) + , pack_size(estimate_serialization_size(field_type)) {} + }; + + std::vector future_fields; + + bool object_used_as_a_whole = used_for_reading || used_for_writing || (used_for_toCell && is_variant_of_union); + + // if used as toCell(), detect last modified field_idx: after it, immutable tail can be saved + bool need_immutable_tail = used_for_toCell && !is_variant_of_union; + int last_modified_field_idx = -1; + for (int field_idx = struct_ref->get_num_fields() - 1; field_idx >= 0; --field_idx) { + if (used_for_writing || fields[field_idx].is_self_or_child_used_for_writing()) { + last_modified_field_idx = field_idx; + break; + } + } + + // fill future_fields + for (int field_idx = 0; field_idx < struct_ref->get_num_fields(); ++field_idx) { + StructFieldPtr orig_field = struct_ref->get_field(field_idx); + TypePtr field_type = orig_field->declared_type; + const ExprUsagesWhileCollecting& field_usages = fields[field_idx]; + bool used_anyhow_but_match = field_usages.is_self_or_field_used_for_reading() || field_usages.is_self_or_child_used_for_writing() || field_usages.is_self_or_field_used_for_toCell(); + + if (need_immutable_tail && field_idx == last_modified_field_idx + 1) { + future_fields.emplace_back(LazyStructLoadInfo::SaveImmutableTail, "(tail)", TypeDataSlice::create()); + } + + if (field_usages.used_for_matching == 1 && !used_anyhow_but_match && !object_used_as_a_whole && !used_for_toCell && !is_variant_of_union && field_idx == struct_ref->get_num_fields() - 1 && + field_type->unwrap_alias()->try_as() && !is_union_type_prevented_from_lazy_loading(field_type->unwrap_alias()->try_as())) { + future_fields.emplace_back(LazyStructLoadInfo::LazyMatchField, orig_field->name, orig_field->declared_type); + continue; + } + if (used_anyhow_but_match || field_usages.is_self_or_child_used_for_matching() || object_used_as_a_whole) { + future_fields.emplace_back(LazyStructLoadInfo::LoadField, orig_field->name, orig_field->declared_type); + continue; + } + + // okay, this field is not needed; we should skip it; + // try to merge "skip 8 bits" + "skip 16 bits" into a single "skip 24 bits" + if (!future_fields.empty() && future_fields.back().action == LazyStructLoadInfo::SkipField) { + if (const TypeDataBytesN* last_bitsN = future_fields.back().field_type->try_as()) { + PackSize cur_size = estimate_serialization_size(field_type); + if (cur_size.min_bits == cur_size.max_bits && cur_size.max_refs == 0) { + TypePtr total_bitsN = TypeDataBytesN::create(true, last_bitsN->n_width + cur_size.max_bits); + future_fields.back().field_type = total_bitsN; + continue; + } + } + } + + // generate "skip 8 bits" instead of "skip int8" (it's more effective, and it can be merged with next) + TypePtr skip_type = field_type; + PackSize skip_size = estimate_serialization_size(field_type); + if (skip_size.min_bits == skip_size.max_bits && skip_size.max_refs == 0) { + skip_type = TypeDataBytesN::create(true, skip_size.max_bits); + } + future_fields.emplace_back(LazyStructLoadInfo::SkipField, "(gap)", skip_type); + } + + // if we need tail, we should load all fields before it (even if they aren't used) + if (need_immutable_tail) { + for (FutureField& f : future_fields) { + if (f.action == LazyStructLoadInfo::SaveImmutableTail) { + break; + } + f.action = LazyStructLoadInfo::LoadField; + } + } + + // here we drop "skip field" if we actually don't need even to skip it, just ignore, like it does not exist; + // example: unused fields in the end `load a; skip b; skip c` -> `load a`; + // example: `skip bits8; load ref` - `load ref`, because to reach a ref, no need to skip preceding bits; + for (size_t i = future_fields.size(); i-- > 0; ) { + if (FutureField f = future_fields[i]; f.action == LazyStructLoadInfo::SkipField) { + PackSize s_cur = f.pack_size; + PackSize s_after(0); + for (size_t j = i + 1; j < future_fields.size(); ++j) { + s_after = EstimateContext::sum(s_after, future_fields[j].pack_size); + } + bool ignore = (s_after.max_bits == 0 && s_after.max_refs == 0) // nothing is loaded after — no need to skip cur + || (s_after.max_bits == 0 && s_cur.max_refs == 0) // no reach ref, no need to skip bits + || (s_after.max_refs == 0 && s_cur.max_bits == 0) // and vice versa: no reach data, to need to skip refs + || (s_cur.max_bits == 0 && s_cur.max_refs == 0); // empty struct/tensor, no need "bits0 skip" + if (ignore) { + future_fields.erase(future_fields.begin() + static_cast(i)); + } + } + } + + // okay, we're done calculating; transform future_fields to hidden_struct + std::vector hidden_fields; + std::vector ith_field_action; + hidden_fields.reserve(future_fields.size()); + ith_field_action.reserve(future_fields.size()); + for (int field_idx = 0; field_idx < static_cast(future_fields.size()); ++field_idx) { + FutureField f = future_fields[field_idx]; + StructFieldPtr created = new StructFieldData(static_cast(f.field_name), {}, field_idx, nullptr, nullptr); + created->mutate()->assign_resolved_type(f.field_type); + hidden_fields.push_back(created); + ith_field_action.push_back(f.action); + } + + StructPtr hidden_struct = new StructData( + "(lazy)" + struct_ref->name, + loc, + std::move(hidden_fields), + is_variant_of_union ? StructData::PackOpcode(0, 0) : struct_ref->opcode, + struct_ref->overflow1023_policy, + nullptr, + nullptr, + struct_ref->ast_root + ); + + return LazyStructLoadInfo(struct_ref, hidden_struct, std::move(ith_field_action)); + } +}; + +// After collecting all vars/fields/variants usages, we should store, where exactly (in AST) which fields to load. +// Every insertion point is represented as this class, it's transformed to an AST auxiliary vertex by a replacer. +struct OneLoadingInsertionPoint { + std::vector all_stmts_where_used; + TypePtr union_variant; + StructFieldPtr field_ref; + LazyStructLoadInfo load_info; + mutable bool was_inserted_to_ast = false; + + OneLoadingInsertionPoint(std::vector&& all_stmts_where_used, TypePtr union_variant, StructFieldPtr field_ref, LazyStructLoadInfo&& load_info) + : all_stmts_where_used(std::move(all_stmts_where_used)) + , union_variant(union_variant) + , field_ref(field_ref) + , load_info(std::move(load_info)) {} + + void mark_inserted_to_ast() const { + was_inserted_to_ast = true; + } + + bool is_mentioned_in_stmt(AnyV stmt) const { + return std::find(all_stmts_where_used.begin(), all_stmts_where_used.end(), stmt) != all_stmts_where_used.end(); + } +}; + +// Every `lazy` operator must be assigned to a variable: `var st = lazy getStorage()`. +// Then `st` is a lazy variable, for which all calculations are done, after which it's stored as this class. +struct LazyVarInFunction { + LocalVarPtr var_ref; + V created_by_lazy_op; + V v_lazy_match_var_itself = nullptr; // lazy `match` for the variable itself + V v_lazy_match_last_field = nullptr; // lazy `match` for the last field of a struct + std::vector load_points; // a set of points where AST should be updated + + // convert already calculated usages of "st" variable and all its fields to a final immutable representation + LazyVarInFunction(FunctionPtr cur_f, LocalVarPtr var_ref, V created_by_lazy_op, ExprUsagesWhileCollecting&& var_usages) + : var_ref(var_ref) + , created_by_lazy_op(created_by_lazy_op) { + + // handle if `msg` is used only in `match (msg) { ... }` + // (it may even be not a union, just a struct with opcode, and `match` with `else`) + bool used_only_as_match = var_usages.used_for_matching == 1 && !var_usages.used_for_reading && !var_usages.used_for_toCell && !var_usages.used_for_writing; + if (used_only_as_match) { + v_lazy_match_var_itself = var_usages.used_as_match_subj.front(); + load_points.reserve(var_usages.variants.size()); + for (ExprUsagesWhileCollecting& variant_usages : var_usages.variants) { + LazyStructLoadInfo load_info = variant_usages.calculate_hidden_struct(created_by_lazy_op->loc, true); + load_points.emplace_back(std::move(variant_usages.needed_above_stmt), variant_usages.expr_type, nullptr, std::move(load_info)); + } + return; + } + + // okay, variable is used not only as `match`; + // prohibit this to a union: lazy union may only be matched, nothing more (`msg is A` etc. don't work) + const TypeDataStruct* t_struct = var_ref->declared_type->unwrap_alias()->try_as(); + bool is_union = t_struct == nullptr; + if (is_union) { + fire(cur_f, created_by_lazy_op->loc, "`lazy` will not work here, because variable `" + var_ref->name + "` it's used in a non-lazy manner\nhint: lazy union may be used only in `match` statement"); + } + + // so, it's just a struct, `lazy Point`; we've already calculated all statements where its fields are used + LazyStructLoadInfo load_info = var_usages.calculate_hidden_struct(created_by_lazy_op->loc, false); + bool is_lazy_match_last_field = !load_info.ith_field_action.empty() && load_info.ith_field_action.back() == LazyStructLoadInfo::LazyMatchField; + load_points.emplace_back(std::move(var_usages.needed_above_stmt), nullptr, nullptr, std::move(load_info)); + + // but probably, there is `match (lazyObj.lastField)` which is lazy; + if (is_lazy_match_last_field) { + StructFieldPtr field_ref = t_struct->struct_ref->fields.back(); + const ExprUsagesWhileCollecting& last_field_usages = var_usages.fields.back(); + v_lazy_match_last_field = last_field_usages.used_as_match_subj.front(); + // inside `match` over a field, loading locations were not detected: insert "load all fields" into every arm + for (int i = 0; i < v_lazy_match_last_field->get_arms_count(); ++i) { + if (auto v_arm = v_lazy_match_last_field->get_arm(i); v_arm->pattern_kind == MatchArmKind::exact_type) { + TypePtr union_variant = v_arm->pattern_type_node->resolved_type; + auto v_arm_body = v_arm->get_body()->get_block_statement(); + if (!v_arm_body->empty()) { + const TypeDataUnion* t_union = field_ref->declared_type->unwrap_alias()->try_as(); + int variant_idx = t_union->get_variant_idx(union_variant); + LazyStructLoadInfo load_all = last_field_usages.variants[variant_idx].generate_hidden_struct_load_all(created_by_lazy_op->loc, true); + load_points.emplace_back(std::vector{v_arm_body->get_item(0)}, union_variant, field_ref, std::move(load_all)); + } + } + } + } + } +}; + +static std::unordered_map> functions_with_lazy_vars; + + +static ExprUsagesWhileCollecting collect_expr_usages_in_block(std::string name_for_debugging, SinkExpression s_expr, TypePtr expr_type, V v_block); + +// This visitor finds usages of "v" / "v.field" / etc. in ONE statement or expression and populates lazy_expr data. +// For every struct, all its fields are also populated; for a union — all its variants. +// Since AST vertices don't have "parent_node", we need to remember some details while traversing top-down. +class CollectUsagesInStatementVisitor final : ASTVisitorFunctionBody { + AnyV cur_stmt; + SinkExpression s_expr; + ExprUsagesWhileCollecting* lazy_expr; + V parent_dot = nullptr; + + CollectUsagesInStatementVisitor(AnyV cur_stmt, SinkExpression s_expr, ExprUsagesWhileCollecting* lazy_expr) + : cur_stmt(cur_stmt), s_expr(s_expr), lazy_expr(lazy_expr) {} + +protected: + void visit(V v) override { + if (extract_sink_expression_from_vertex(v) == s_expr) { + bool is_subj_of_dot = parent_dot && parent_dot->is_target_struct_field() && parent_dot->get_obj() == v; + if (!is_subj_of_dot) { + lazy_expr->on_used_rw(v->is_lvalue); + } + } + } + + void visit(V v) override { + if (extract_sink_expression_from_vertex(v) == s_expr) { + bool is_subj_of_dot = parent_dot && parent_dot->is_target_struct_field() && parent_dot->get_obj() == v; + if (!is_subj_of_dot) { + lazy_expr->on_used_rw(v->is_lvalue); + } + } + auto backup = parent_dot; + parent_dot = v; + parent::visit(v); + parent_dot = backup; + } + + void visit(V v) override { + FunctionPtr fun_ref = v->fun_maybe; + if (fun_ref && fun_ref->does_accept_self() && can_method_be_inlined_preserving_lazy(fun_ref)) { + AnyExprV dot_obj = v->get_callee()->as()->get_obj(); + if (extract_sink_expression_from_vertex(dot_obj) == s_expr) { + // handle built-in functions specially + if (fun_ref->is_builtin_function() && fun_ref->base_fun_ref->name == "T.toCell") { + lazy_expr->on_used_toCell(); + return; + } + if (fun_ref->is_builtin_function() && fun_ref->base_fun_ref->name == "T.forceLoadLazyObject") { + lazy_expr->on_used_rw(false); // just use the object as a whole, slice will point to its end + return; + } + if (s_expr.index_path) { // obj.f.method(), mark obj.f as used anyway + lazy_expr->on_used_rw(false); + } + tolk_assert(fun_ref->is_code_function()); + + // okay, we have `st.save()` / `p.getX()`, which will be inlined when transforming to IR; + // dig into that method's body to fetch used fields `self.x` etc. + auto v_body_block = fun_ref->ast_root->try_as()->get_body()->try_as(); + ExprUsagesWhileCollecting inner_usages = collect_expr_usages_in_block(lazy_expr->name_str + "(=self)", SinkExpression(&fun_ref->parameters[0]), lazy_expr->expr_type, v_body_block); + inner_usages.treat_match_like_read(); // nested lazy match in inlined functions doesn't work, it's not wrapped into aux vertex + lazy_expr->merge_with_sub_block(inner_usages); + return; + } + } + + parent::visit(v); + } + + void visit(V v) override { + AnyExprV subj = v->get_subject(); + bool is_match_by_cur = extract_sink_expression_from_vertex(subj) == s_expr; + + // `match` statement over current expression is okay (it will be lazy if it's the only, and other conditions satisfy); + // `match` expression, generally, is not safe to be lazy, e.g. `return cond && match(...)`, + // but simply `return match(...)` / `var result = match(...)` is okay + bool is_safe = false; + if (v->is_statement()) { + is_safe = cur_stmt == v; + } else if (auto v_return = cur_stmt->try_as()) { + is_safe = v_return->get_return_value() == v; + } else if (auto v_assign = cur_stmt->try_as()) { + is_safe = v_assign->get_rhs() == v; + } else if (auto v_set_assign = cur_stmt->try_as()) { + is_safe = v_set_assign->get_rhs() == v; + } + + if (is_match_by_cur && is_safe) { + lazy_expr->on_used_as_match_subj(v); + const TypeDataUnion* expr_as_union = lazy_expr->expr_type->unwrap_alias()->try_as(); + for (int i = 0; i < v->get_arms_count(); ++i) { + if (auto v_arm = v->get_arm(i); v_arm->pattern_kind == MatchArmKind::exact_type) { + TypePtr exact_type = v_arm->pattern_type_node->resolved_type; + auto v_block = v_arm->get_body()->get_block_statement(); + ExprUsagesWhileCollecting variant_usages = collect_expr_usages_in_block(lazy_expr->name_str, s_expr, exact_type, v_block); + int variant_idx = expr_as_union ? expr_as_union->get_variant_idx(exact_type) : 0; // match over non-union is ok + lazy_expr->variants[variant_idx].merge_with_sub_block(variant_usages); + } + } + return; + } + + parent::visit(v); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + tolk_assert(false); + } + + static void collect_usages_in_expression(ExprUsagesWhileCollecting* out, SinkExpression s_expr, AnyV v_expr) { + CollectUsagesInStatementVisitor visitor(v_expr, s_expr, out); + visitor.parent::visit(v_expr); + if (out->struct_ref) { + for (int field_idx = 0; field_idx < out->struct_ref->get_num_fields(); ++field_idx) { + collect_usages_in_expression(&out->fields[field_idx], s_expr.get_child_s_expr(field_idx), v_expr); + out->total_usages_with_fields += out->fields[field_idx].total_usages_with_fields; + } + } + } +}; + +// This visitor analyzes A WHOLE BLOCK, statement by statement, and detects all statements where lazy_expr is used. +// It takes care of nested try/catch, etc. +// Ideally, it should calculate the only "lca" AST vertex of all usages, but it's not as easy as it seems. +// Instead, `lazy_expr->needed_above_stmt` contains all statements where expr is "mentioned" (and needs to be loaded before). +// And later, traversing top-down, the first occurrence is taken, inserting an AST aux vertex right before it. +class CollectUsagesInBlockBottomUp { + ExprUsagesWhileCollecting* lazy_expr; + SinkExpression s_expr; + + CollectUsagesInBlockBottomUp(ExprUsagesWhileCollecting* lazy_expr, SinkExpression s_expr) + : lazy_expr(lazy_expr), s_expr(s_expr) {} + + void visit_try_catch_statement(V v) const { + ExprUsagesWhileCollecting u_try = visit_sub_block(v->get_try_body()); + ExprUsagesWhileCollecting u_catch = visit_sub_block(v->get_catch_body()); + + if (u_try.total_usages_with_fields && u_catch.total_usages_with_fields) { + lazy_expr->needed_above_stmt.push_back(v); + } + if (u_try.total_usages_with_fields || u_catch.total_usages_with_fields) { + if (lazy_expr->total_usages_with_fields) { + lazy_expr->needed_above_stmt.push_back(v); + } + } + + lazy_expr->merge_with_sub_block(u_try); + lazy_expr->merge_with_sub_block(u_catch); + } + + void visit_if_statement(V v) const { + ExprUsagesWhileCollecting u_cond = visit_other(v->get_cond()); + ExprUsagesWhileCollecting u_then = visit_sub_block(v->get_if_body()); + ExprUsagesWhileCollecting u_else = visit_sub_block(v->get_else_body()); + + if (u_cond.total_usages_with_fields) { + lazy_expr->needed_above_stmt.push_back(v); + } + if (u_then.total_usages_with_fields && u_else.total_usages_with_fields) { + lazy_expr->needed_above_stmt.push_back(v); + } + if (u_then.total_usages_with_fields || u_else.total_usages_with_fields) { + if (lazy_expr->total_usages_with_fields) { + lazy_expr->needed_above_stmt.push_back(v); + } + } + + lazy_expr->merge_with_sub_block(u_cond); + lazy_expr->merge_with_sub_block(u_then); + lazy_expr->merge_with_sub_block(u_else); + } + + void visit_block_statement(V v) const { + for (int i = v->size() - 1; i >= 0; --i) { + AnyV ith_statement = v->get_item(i); + switch (ith_statement->kind) { + case ast_try_catch_statement: + visit_try_catch_statement(ith_statement->as()); + continue; + case ast_if_statement: + visit_if_statement(ith_statement->as()); + continue; + default: + break; + } + + ExprUsagesWhileCollecting u_ith = visit_other(ith_statement); + if (u_ith.total_usages_with_fields) { + u_ith.needed_above_stmt.push_back(ith_statement); + } + lazy_expr->merge_with_sub_block(u_ith); + } + } + + ExprUsagesWhileCollecting visit_sub_block(V v_block) const { + return visit_block_bottom_up(lazy_expr->name_str, s_expr, lazy_expr->expr_type, v_block); + } + + ExprUsagesWhileCollecting visit_other(AnyV v) const { + ExprUsagesWhileCollecting result(lazy_expr->name_str, lazy_expr->expr_type); + CollectUsagesInStatementVisitor::collect_usages_in_expression(&result, s_expr, v); + return result; + } + +public: + static ExprUsagesWhileCollecting visit_block_bottom_up(std::string name_for_debugging, SinkExpression s_expr, TypePtr expr_type, V v_block) { + ExprUsagesWhileCollecting lazy_expr(std::move(name_for_debugging), expr_type); + CollectUsagesInBlockBottomUp visitor(&lazy_expr, s_expr); + visitor.visit_block_statement(v_block); + return lazy_expr; + } +}; + +static ExprUsagesWhileCollecting collect_expr_usages_in_block(std::string name_for_debugging, SinkExpression s_expr, TypePtr expr_type, V v_block) { + return CollectUsagesInBlockBottomUp::visit_block_bottom_up(std::move(name_for_debugging), s_expr, expr_type, v_block); +} + +// Step 1: +// This visitor finds `var st = lazy expr`, launches finding usages for `st`, +// and adds `st` as LazyVarInFunction to a global list. +class CollectAllLazyObjectsAndFieldsVisitor final : public ASTVisitorFunctionBody { + FunctionPtr cur_f = nullptr; + V parent_block = nullptr; + +protected: + void visit(V v) override { + auto backup = parent_block; + parent_block = v; + parent::visit(v); + parent_block = backup; + } + + // `var st = lazy ...` + void visit(V v) override { + if (auto rhs_lazy = v->get_rhs()->try_as()) { + check_lazy_operator_used_correctly(cur_f, rhs_lazy); + + if (auto lhs_var_decl = v->get_lhs()->try_as()) { + auto lhs_var = lhs_var_decl->get_expr()->try_as(); + if (!lhs_var->marked_as_redef) { + // collect usages of a lazy var inside the same block statement where it's declared + LocalVarPtr var_ref = lhs_var->var_ref; + ExprUsagesWhileCollecting var_usages = collect_expr_usages_in_block(var_ref->name, SinkExpression(var_ref), var_ref->declared_type, parent_block); + LazyVarInFunction lazy_var(cur_f, var_ref, rhs_lazy, std::move(var_usages)); + functions_with_lazy_vars[cur_f].emplace_back(std::move(lazy_var)); + } + } + } + + parent::visit(v); + } + + // check that `lazy` operator used in a correct pattern with a correct expression + void visit(V v) override { + for (const LazyVarInFunction& lazy_var : functions_with_lazy_vars[cur_f]) { + if (lazy_var.created_by_lazy_op == v) { + parent::visit(v); + return; + } + } + + // for `return lazy ...` and other cases except allowed + fire(cur_f, v->loc, "incorrect `lazy` operator usage, it's not directly assigned to a variable\nhint: use `lazy` like this:\n> var st = lazy MyStorage.fromSlice(...)"); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && !fun_ref->is_generic_function(); + } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + parent::visit(v_function->get_body()); + } +}; + +// Step 2: +// After visiting all functions and finding all lazy variables, this replacer updates AST, +// inserting (already calculated) load vertices. They are auxiliary vertices holding special data. +// They are handled later when transforming AST to Ops. +class LazyLoadInsertionsReplacer final : public ASTReplacerInFunctionBody { + FunctionPtr cur_f = nullptr; + V f_body = nullptr; + +protected: + // `var st = lazy expr` -> save "st" (it will be used in codegen to assert "st.x" that "x" is loaded) + AnyExprV replace(V v) override { + for (const LazyVarInFunction& lazy_var : functions_with_lazy_vars[cur_f]) { + if (lazy_var.created_by_lazy_op == v) { + v->mutate()->assign_dest_var_ref(lazy_var.var_ref); + return parent::replace(v); + } + } + + tolk_assert(false); // all `lazy` operators where detected and handled + } + + // `{ ... }` -> `{ ... load ... }` + AnyV replace(V v) override { + std::vector new_children; // since we don't have "parent_node" and "next_child" in AST, + new_children.reserve(v->size()); // traverse every block statement and insert "load" in the middle + + for (AnyV stmt : v->get_items()) { + for (const LazyVarInFunction& lazy_var : functions_with_lazy_vars[cur_f]) { + for (const OneLoadingInsertionPoint& ins : lazy_var.load_points) { + if (!ins.was_inserted_to_ast && ins.is_mentioned_in_stmt(stmt)) { + ASTAuxData* aux_data = new AuxData_LazyObjectLoadFields(lazy_var.var_ref, ins.union_variant, ins.field_ref, ins.load_info); + new_children.push_back(createV(stmt->loc, createV(stmt->loc), aux_data, TypeDataVoid::create())); + ins.mark_inserted_to_ast(); + } + } + } + new_children.push_back(parent::replace(stmt)); + } + + v->mutate()->assign_new_children(std::move(new_children)); + return v; + } + + // `match (lazy_obj)` / `match (lazy_obj.field)` -> wrap with aux + AnyExprV replace(V v) override { + for (const LazyVarInFunction& lazy_var : functions_with_lazy_vars[cur_f]) { + bool is_lazy_match_for_union = lazy_var.v_lazy_match_var_itself == v; + if (is_lazy_match_for_union) { + ASTAuxData* aux_data = new AuxData_LazyMatchForUnion(lazy_var.var_ref, nullptr); + return createV(v->loc, parent::replace(v), aux_data, v->inferred_type); + } + + bool is_lazy_match_for_last_field = lazy_var.v_lazy_match_last_field == v; + if (is_lazy_match_for_last_field) { + StructPtr struct_ref = lazy_var.var_ref->declared_type->unwrap_alias()->try_as()->struct_ref; + ASTAuxData* aux_data = new AuxData_LazyMatchForUnion(lazy_var.var_ref, struct_ref->fields.back()); + return createV(v->loc, parent::replace(v), aux_data, v->inferred_type); + } + } + + return parent::replace(v); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && functions_with_lazy_vars.contains(fun_ref); + } + + void start_replacing_in_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + f_body = v_function->get_body()->as(); + parent::replace(v_function->get_body()); + } +}; + +// Step 3: +// After modifying AST (inserting loads, lazy match, etc.), +// check __expect_lazy() calls, used in compiler tests as assertions. +class CheckExpectLazyAssertionsVisitor final : public ASTVisitorFunctionBody { + FunctionPtr cur_f = nullptr; + + static std::string stringify_lazy_load_above_stmt(const AuxData_LazyObjectLoadFields* aux_load) { + static const char* action_to_str[] = {"load", "skip", "lazy match", "save immutable"}; + + const LazyStructLoadInfo& load_info = aux_load->load_info; + StructPtr struct_ref = load_info.hidden_struct; + std::string_view last_action; + + std::string result = "[" + aux_load->var_ref->name + "] "; + for (int i = 0; i < struct_ref->get_num_fields(); ++i) { + std::string field_name = struct_ref->get_field(i)->name; + if (field_name == "(gap)") { + field_name = "(" + struct_ref->get_field(i)->declared_type->as_human_readable() + ")"; + } + std::string_view action = action_to_str[load_info.ith_field_action[i]]; + if (action != last_action) { + if (result[result.size() - 2] != ']') { + result += ", "; + } + result += action; + last_action = action; + } + result += " "; + result += field_name; + } + return result; + } + + void visit(V v) override { + // again, given "__expect_lazy(...)", we have no "next sibling", so traverse block statements + for (int i = 0; i < v->size(); ++i) { + AnyV cur_stmt = v->get_item(i); + if (auto v_call = cur_stmt->try_as()) { + if (v_call->fun_maybe && v_call->fun_maybe->is_builtin_function() && v_call->fun_maybe->name == "__expect_lazy") { + AnyV next_stmt = v->get_item(i + 1); + std::string_view expected = v_call->get_arg(0)->get_expr()->as()->str_val; + std::string actual; + if (auto next_aux = next_stmt->try_as()) { + if (const auto* aux_load = dynamic_cast(next_aux->aux_data)) { + actual = stringify_lazy_load_above_stmt(aux_load); + } + if (const auto* aux_match = dynamic_cast(next_aux->aux_data)) { + actual = "[" + aux_match->var_ref->name + "] " + "lazy match"; + } + } + + if (actual != expected) { + throw ParseError(cur_stmt->loc, "__expect_lazy failed: actual \"" + actual + "\""); + } + } + } + parent::visit(cur_stmt); + } + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && functions_with_lazy_vars.contains(fun_ref); + } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + parent::visit(v_function->get_body()); + } +}; + +void pipeline_lazy_load_insertions() { + visit_ast_of_all_functions(); + replace_ast_of_all_functions(); + visit_ast_of_all_functions(); + functions_with_lazy_vars.clear(); +} + +} // namespace tolk diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 2f6694469..fad233c7f 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -211,7 +211,7 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { // resolve identifiers after => at first - parent::visit(v->get_body()); + visit(v->get_body()); // because handling lhs of => is comprehensive switch (v->pattern_kind) { diff --git a/tolk/pipeline.h b/tolk/pipeline.h index e4448deb1..001228bee 100644 --- a/tolk/pipeline.h +++ b/tolk/pipeline.h @@ -45,6 +45,7 @@ void pipeline_check_serialized_fields(); void pipeline_constant_folding(); void pipeline_optimize_boolean_expressions(); void pipeline_detect_inline_in_place(); +void pipeline_lazy_load_insertions(); void pipeline_convert_ast_to_legacy_Expr_Op(); void pipeline_find_unused_symbols(); diff --git a/tolk/smart-casts-cfg.cpp b/tolk/smart-casts-cfg.cpp index 31b19558f..6597efe95 100644 --- a/tolk/smart-casts-cfg.cpp +++ b/tolk/smart-casts-cfg.cpp @@ -113,6 +113,17 @@ std::string SinkExpression::to_string() const { return result; } +SinkExpression SinkExpression::get_child_s_expr(int field_idx) const { + uint64_t new_index_path = index_path; // if we have c.1 (index_path = 2) and construct c.1.N, calc (N<<8 + 2) + for (int empty_byte = 0; empty_byte < 8; ++empty_byte) { + if ((index_path & (0xFF << (empty_byte*8))) == 0) { + new_index_path += (field_idx + 1) << (empty_byte*8); + break; + } + } + return SinkExpression(var_ref, new_index_path); +} + static std::string to_string(SignState s) { static const char* txt[6 + 1] = {"sign=unknown", ">0", "<0", "=0", ">=0", "<=0", "sign=never"}; return txt[static_cast(s)]; diff --git a/tolk/smart-casts-cfg.h b/tolk/smart-casts-cfg.h index 1e24a7b69..b579e8f31 100644 --- a/tolk/smart-casts-cfg.h +++ b/tolk/smart-casts-cfg.h @@ -71,6 +71,7 @@ struct SinkExpression { explicit operator bool() const { return var_ref != nullptr; } std::string to_string() const; + SinkExpression get_child_s_expr(int field_idx) const; }; // UnreachableKind is a reason of why control flow is unreachable or interrupted diff --git a/tolk/tolk.cpp b/tolk/tolk.cpp index 6f8bbba91..cbad23fe3 100644 --- a/tolk/tolk.cpp +++ b/tolk/tolk.cpp @@ -68,6 +68,7 @@ int tolk_proceed(const std::string &entrypoint_filename) { pipeline_constant_folding(); pipeline_optimize_boolean_expressions(); pipeline_detect_inline_in_place(); + pipeline_lazy_load_insertions(); pipeline_convert_ast_to_legacy_Expr_Op(); pipeline_find_unused_symbols(); diff --git a/tolk/tolk.h b/tolk/tolk.h index 3f6dcadd8..9bd21c99d 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -46,7 +46,6 @@ inline void fire(FunctionPtr cur_f, SrcLocation loc, const std::string& message) * */ -typedef int var_idx_t; typedef int const_idx_t; struct TmpVar { @@ -1042,12 +1041,26 @@ struct FunctionBodyAsm { void compile(AsmOpList& dest, SrcLocation loc) const; }; +struct LazyVariableLoadedState; + +// LazyVarRefAtCodegen is a mutable state of a variable assigned by `lazy` operator: +// > var p = lazy Point.fromSlice(s) +// When inlining a method `p.getX()`, `self` also becomes lazy, pointing to the same state. +struct LazyVarRefAtCodegen { + LocalVarPtr var_ref; + const LazyVariableLoadedState* var_state; + + LazyVarRefAtCodegen(LocalVarPtr var_ref, const LazyVariableLoadedState* var_state) + : var_ref(var_ref), var_state(var_state) {} +}; + struct CodeBlob { int var_cnt, in_var_cnt; FunctionPtr fun_ref; std::string name; SrcLocation forced_loc; std::vector vars; + std::vector lazy_variables; std::vector* inline_rvect_out = nullptr; bool inlining_before_immediate_return = false; std::unique_ptr ops; @@ -1101,6 +1114,8 @@ struct CodeBlob { close_blk(location); pop_cur(); } + const LazyVariableLoadedState* get_lazy_variable(LocalVarPtr var_ref) const; + const LazyVariableLoadedState* get_lazy_variable(AnyExprV v) const; void prune_unreachable_code(); void fwd_analyze(); void mark_noreturn(); diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index d37a65778..b6ffd6022 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -1316,6 +1316,15 @@ bool TypeDataUnion::has_all_variants_of(const TypeDataUnion* rhs_type) const { return true; } +int TypeDataUnion::get_variant_idx(TypePtr lookup_variant) const { + for (int i = 0; i < size(); ++i) { + if (variants[i]->equal_to(lookup_variant)) { + return i; + } + } + return -1; +} + // given this = `T1 | T2 | ...` and rhs_type, find the only (not ambiguous) T_i that can accept it TypePtr TypeDataUnion::calculate_exact_variant_to_fit_rhs(TypePtr rhs_type) const { // primitive 1-slot nullable don't store type_id, they can be assigned less strict, like `int?` to `int16?` diff --git a/tolk/type-system.h b/tolk/type-system.h index c1f9cbb6d..62841893a 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -569,6 +569,7 @@ class TypeDataUnion final : public TypeData { TypePtr calculate_exact_variant_to_fit_rhs(TypePtr rhs_type) const; bool has_all_variants_of(const TypeDataUnion* rhs_type) const; + int get_variant_idx(TypePtr lookup_variant) const; int get_width_on_stack() const override; int get_type_id() const override; From dc328acd348b380c44e4a823a93527f4b7827f5b Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 12 Jun 2025 13:23:48 +0700 Subject: [PATCH 331/388] [Tolk] Deprecate and comment some functions in stdlib --- crypto/smartcont/tolk-stdlib/common.tolk | 55 +++++++++++++++++------- tolk-tester/tests/cells-slices.tolk | 5 --- tolk-tester/tests/send-msg-1.tolk | 4 +- tolk-tester/tests/send-msg-2.tolk | 2 +- tolk-tester/tests/send-msg-3.tolk | 2 +- tolk/builtins.cpp | 10 ++--- tolk/send-message-api.cpp | 2 +- 7 files changed, 50 insertions(+), 30 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index c2674a4c5..3eefbbc5f 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -388,7 +388,7 @@ fun Cell.beginParse(self): slice /// Returns hash of a typed cell, same as [cell.hash]. @pure -fun Cell.hash(self): slice +fun Cell.hash(self): uint256 asm "HASHCU"; /// RemainingBitsAndRefs is a special built-in type to get "all the rest" slice tail on reading. @@ -458,20 +458,20 @@ fun stringToBase256(constString: slice): int /// Computes the representation hash of a `cell` and returns it as a 256-bit unsigned integer `x`. /// Useful for signing and checking signatures of arbitrary entities represented by a tree of cells. @pure -fun cell.hash(self): int +fun cell.hash(self): uint256 asm "HASHCU"; /// Computes the hash of a `slice` and returns it as a 256-bit unsigned integer `x`. /// The result is the same as if an ordinary cell containing only data and references from `s` had been created /// and its hash computed by [cell.hash]. @pure -fun slice.hash(self): int +fun slice.hash(self): uint256 asm "HASHSU"; /// Computes sha256 of the data bits of a `slice`. If the bit length of `s` is not divisible by eight, /// throws a cell underflow exception. The hash value is returned as a 256-bit unsigned integer `x`. @pure -fun slice.bitsHash(self): int +fun slice.bitsHash(self): uint256 asm "SHA256U"; /// Checks the Ed25519-`signature` of a `hash` (a 256-bit unsigned integer, usually computed as the hash of some data) @@ -1119,7 +1119,7 @@ struct (0x00) ExtOutLogBucket { /// Options for creating an external outgoing message. /// Consider [createExternalLogMessage] for examples. -struct createExternalLogMessageOptions { +struct CreateExternalLogMessageOptions { /// destination is either an external address or a pattern to calculate it dest: address | // either some valid external/none address (not internal!) builder | // ... or a manually constructed builder with a valid external address @@ -1143,7 +1143,7 @@ struct createExternalLogMessageOptions { /// (if body is small, it will be inlined without an expensive cell creation) /// (if body is large, the compiler will automatically wrap it into a cell) @pure -fun createExternalLogMessage(options: createExternalLogMessageOptions): OutMessage +fun createExternalLogMessage(options: CreateExternalLogMessageOptions): OutMessage builtin; /// UnsafeBodyNoRef is used to prevent default behavior: when message body is potentially large, @@ -1157,13 +1157,13 @@ fun createExternalLogMessage(options: createExternalLogMessageOptions { - bodyForceNoRef: T; + forceInline: T; } /// OutMessage is a result of [createMessage]. @@ -1202,6 +1202,13 @@ struct StateInit { } /// Calculates a hash of StateInit if only code+data are set. +/// Example: +/// ``` +/// val addrHash = StateInit.calcHashCodeData(codeCell, dataCell); +/// createMessage({ +/// dest: (workchain, addrHash), +/// ... +/// ``` @pure fun StateInit.calcHashCodeData(code: cell, data: cell): uint256 asm """ // code data @@ -1304,23 +1311,41 @@ fun sendRawMessage(msg: cell, mode: int): void Receiving and handling messages. */ +/// Skip 0xFFFFFFFF prefix (when a message is bounced). +@pure +fun slice.skipBouncedPrefix(mutate self): self + asm "32 LDU" "NIP"; + /// Load msgFlags from incoming message body (4 bits). @pure +@deprecated("use `onInternalMessage(in: InMessage)` and `in.isBounced` instead of parsing msgCell manually") fun slice.loadMessageFlags(mutate self): int asm( -> 1 0) "4 LDU"; -/// Skip 0xFFFFFFFF prefix (when a message is bounced). -@pure -fun slice.skipBouncedPrefix(mutate self): self - asm "32 PUSHINT" "SDSKIPFIRST"; - -/// The guideline recommends to start the body of an internal message with uint32 `op` and uint64 `queryId`. +/// Loads a message opcode (32-bit integer) when parsing message body manually. @pure fun slice.loadMessageOp(mutate self): int asm( -> 1 0) "32 LDU"; -/// The guideline recommends that uint64 `queryId` should follow uint32 `op`. +/// Stores a message opcode (32-bit integer) when composing an output message manually. +@pure +@deprecated("use `createMessage` instead of composing messages manually") +fun builder.storeMessageOp(mutate self, op: int): self + asm(op self) "32 STU"; + +/// Loads a message queryId (64-bit integer, immediately following the opcode) when parsing message body manually. @pure +@deprecated("use structures and lazy match instead of parsing messages manually") fun slice.loadMessageQueryId(mutate self): int asm( -> 1 0) "64 LDU"; +@pure +@deprecated("use structures and lazy loading instead of parsing messages manually") +fun slice.skipMessageQueryId(mutate self): self + asm "64 LDU" "NIP"; + +@pure +@deprecated("use `createMessage` instead of composing messages manually") +fun builder.storeMessageQueryId(mutate self, queryId: int): self + asm(queryId self) "64 STU"; + diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index 2cbb78647..d8e62bc6c 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -1,11 +1,6 @@ fun builder.store_u32(mutate self, value: int): self { return self.storeUint(value, 32); } -fun builder.storeMessageOp(mutate self, op: int): self - asm(op self) "32 STU"; -fun builder.storeMessageQueryId(mutate self, queryId: int): self - asm(queryId self) "64 STU"; - fun slice.load_u32(mutate self): int { return self.loadUint(32); } diff --git a/tolk-tester/tests/send-msg-1.tolk b/tolk-tester/tests/send-msg-1.tolk index a7be4ec05..3a889dcf4 100644 --- a/tolk-tester/tests/send-msg-1.tolk +++ b/tolk-tester/tests/send-msg-1.tolk @@ -643,7 +643,7 @@ fun test22(op: uint32, queryId: uint64) { dest: address("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff"), value: ton("0.08"), body: UnsafeBodyNoRef { - bodyForceNoRef: bodyB + forceInline: bodyB }, }); assert(b.hash() == test22_manual(bodyB).hash(), 122); @@ -690,7 +690,7 @@ fun test23(state32: uint32) { var b = createMessage({ bounce: true, body: UnsafeBodyNoRef { - bodyForceNoRef: bd, + forceInline: bd, }, dest: { workchain: 9, stateInit: init }, value: ton("0.10009"), diff --git a/tolk-tester/tests/send-msg-2.tolk b/tolk-tester/tests/send-msg-2.tolk index 1d11752db..050cd8432 100644 --- a/tolk-tester/tests/send-msg-2.tolk +++ b/tolk-tester/tests/send-msg-2.tolk @@ -294,7 +294,7 @@ fun test5() { closeTo: pivot, } }, - body: UnsafeBodyNoRef { bodyForceNoRef: body }, + body: UnsafeBodyNoRef { forceInline: body }, value: ton("0.1"), }); return b.hash() diff --git a/tolk-tester/tests/send-msg-3.tolk b/tolk-tester/tests/send-msg-3.tolk index 07265ec96..8db47589d 100644 --- a/tolk-tester/tests/send-msg-3.tolk +++ b/tolk-tester/tests/send-msg-3.tolk @@ -171,7 +171,7 @@ fun test9(topic: int) { }; var b = createExternalLogMessage({ body: UnsafeBodyNoRef { - bodyForceNoRef: bd, + forceInline: bd, }, dest: ExtOutLogBucket { topic: topic }, }); diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index f0d8de7cb..d15288564 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1248,7 +1248,7 @@ void define_builtins() { TypePtr PackOptions = TypeDataUnknown::create(); TypePtr UnpackOptions = TypeDataUnknown::create(); TypePtr CreateMessageOptions = TypeDataUnknown::create(); - TypePtr createExternalLogMessageOptions = TypeDataUnknown::create(); + TypePtr CreateExternalLogMessageOptions = TypeDataUnknown::create(); TypePtr OutMessage = TypeDataUnknown::create(); TypePtr AddressShardingOptions = TypeDataUnknown::create(); const GenericsDeclaration* declTBody = new GenericsDeclaration(std::vector{{"TBody", nullptr}}, 0); @@ -1510,7 +1510,7 @@ void define_builtins() { define_builtin_func("createMessage", {CreateMessageOptions}, OutMessage, declTBody, compile_time_only_function, FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); - define_builtin_func("createExternalLogMessage", {createExternalLogMessageOptions}, OutMessage, declTBody, + define_builtin_func("createExternalLogMessage", {CreateExternalLogMessageOptions}, OutMessage, declTBody, compile_time_only_function, FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); @@ -1584,15 +1584,15 @@ void patch_builtins_after_stdlib_loaded() { lookup_function("builder.storeAny")->mutate()->parameters[2].default_value = v_empty_PackOptions; StructPtr struct_ref_CreateMessageOptions = lookup_global_symbol("CreateMessageOptions")->try_as(); - StructPtr struct_ref_createExternalLogMessageOptions = lookup_global_symbol("createExternalLogMessageOptions")->try_as(); + StructPtr struct_ref_CreateExternalLogMessageOptions = lookup_global_symbol("CreateExternalLogMessageOptions")->try_as(); StructPtr struct_ref_OutMessage = lookup_global_symbol("OutMessage")->try_as(); TypePtr CreateMessageOptions = TypeDataGenericTypeWithTs::create(struct_ref_CreateMessageOptions, nullptr, {TypeDataGenericT::create("TBody")}); - TypePtr createExternalLogMessageOptions = TypeDataGenericTypeWithTs::create(struct_ref_createExternalLogMessageOptions, nullptr, {TypeDataGenericT::create("TBody")}); + TypePtr CreateExternalLogMessageOptions = TypeDataGenericTypeWithTs::create(struct_ref_CreateExternalLogMessageOptions, nullptr, {TypeDataGenericT::create("TBody")}); TypePtr OutMessage = TypeDataStruct::create(struct_ref_OutMessage); lookup_function("createMessage")->mutate()->parameters[0].declared_type = CreateMessageOptions; lookup_function("createMessage")->mutate()->declared_return_type = OutMessage; - lookup_function("createExternalLogMessage")->mutate()->parameters[0].declared_type = createExternalLogMessageOptions; + lookup_function("createExternalLogMessage")->mutate()->parameters[0].declared_type = CreateExternalLogMessageOptions; lookup_function("createExternalLogMessage")->mutate()->declared_return_type = OutMessage; } diff --git a/tolk/send-message-api.cpp b/tolk/send-message-api.cpp index 4fd8ab846..6d0b53524 100644 --- a/tolk/send-message-api.cpp +++ b/tolk/send-message-api.cpp @@ -329,7 +329,7 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T } std::vector generate_createExternalLogMessage(CodeBlob& code, SrcLocation loc, TypePtr bodyT, std::vector&& rvect) { - StructPtr s_Options = lookup_global_symbol("createExternalLogMessageOptions")->try_as(); + StructPtr s_Options = lookup_global_symbol("CreateExternalLogMessageOptions")->try_as(); StructPtr s_ExtOutLogBucket = lookup_global_symbol("ExtOutLogBucket")->try_as(); const TypeDataUnion* t_dest = s_Options->find_field("dest")->declared_type->try_as(); From d9bb6bb0c09f11712495a2b2600d819c9bbb7ba6 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Wed, 18 Jun 2025 17:04:17 +0300 Subject: [PATCH 332/388] [Tolk] Use parenthesis `()` as a suffix for Fift generation For every function f, name it `f()` instead of `f` in Fift output. > DECLPROC f() > f() CALLDICT This avoids collisions with Fift reserved names like execute/swap/etc. For global vars, `$` prefix is now used: > DECLGLOBVAR $name --- .../tests/allow-post-modification.tolk | 6 +-- tolk-tester/tests/asm-arg-order.tolk | 24 +++++----- tolk-tester/tests/assignment-tests.tolk | 2 +- tolk-tester/tests/bit-operators.tolk | 14 +++--- tolk-tester/tests/cells-slices.tolk | 16 +++---- tolk-tester/tests/codegen-check-demo.tolk | 8 ++-- tolk-tester/tests/comments-tests.tolk | 2 +- tolk-tester/tests/generics-1.tolk | 22 ++++----- tolk-tester/tests/generics-2.tolk | 6 +-- tolk-tester/tests/generics-3.tolk | 2 +- tolk-tester/tests/generics-4.tolk | 14 +++--- tolk-tester/tests/if-else-tests.tolk | 46 +++++++++---------- tolk-tester/tests/indexed-access.tolk | 24 +++++----- tolk-tester/tests/inline-tests.tolk | 16 +++---- tolk-tester/tests/intN-tests.tolk | 8 ++-- tolk-tester/tests/lazy-algo-tests.tolk | 2 +- tolk-tester/tests/lazy-load-tests.tolk | 38 +++++++-------- tolk-tester/tests/logical-operators.tolk | 16 +++---- tolk-tester/tests/match-by-expr-tests.tolk | 20 ++++---- tolk-tester/tests/methods-tests.tolk | 6 +-- tolk-tester/tests/mutate-methods.tolk | 14 +++--- tolk-tester/tests/no-spaces.tolk | 10 ++-- tolk-tester/tests/null-keyword.tolk | 6 +-- tolk-tester/tests/nullable-tensors.tolk | 6 +-- tolk-tester/tests/op-priority.tolk | 6 +-- tolk-tester/tests/pack-unpack-1.tolk | 2 +- tolk-tester/tests/pack-unpack-4.tolk | 2 +- tolk-tester/tests/pack-unpack-6.tolk | 10 ++-- tolk-tester/tests/parse-address.tolk | 4 +- .../tests/remove-unused-functions.tolk | 16 +++---- tolk-tester/tests/self-keyword.tolk | 28 +++++------ tolk-tester/tests/smart-cast-tests.tolk | 6 +-- tolk-tester/tests/some-tests-2.tolk | 24 +++++----- tolk-tester/tests/special-fun-names.tolk | 32 +++++++++++-- tolk-tester/tests/struct-tests.tolk | 24 +++++----- tolk-tester/tests/try-catch-tests.tolk | 10 ++-- tolk-tester/tests/type-aliases-tests.tolk | 2 +- tolk-tester/tests/union-types-tests.tolk | 8 ++-- tolk-tester/tests/use-before-declare.tolk | 4 +- tolk-tester/tests/var-apply-tests.tolk | 13 ++++++ tolk/codegen.cpp | 12 ++--- tolk/pipe-generate-fif-output.cpp | 10 ++-- 42 files changed, 289 insertions(+), 252 deletions(-) diff --git a/tolk-tester/tests/allow-post-modification.tolk b/tolk-tester/tests/allow-post-modification.tolk index 0f8603cc6..a74b2e1ec 100644 --- a/tolk-tester/tests/allow-post-modification.tolk +++ b/tolk-tester/tests/allow-post-modification.tolk @@ -154,14 +154,14 @@ fun main() { @fif_codegen """ - int.~inc PROC:<{ // self y - inc CALLDICT // self newY + int.~inc() PROC:<{ // self y + inc() CALLDICT // self newY }> """ @fif_codegen """ - test_assign_tensor_global PROC:<{ // x.0 x.1 + test_assign_tensor_global() PROC:<{ // x.0 x.1 """ @code_hash 6737917279814799680932710799951154408447028229503449454536845752635763933556 diff --git a/tolk-tester/tests/asm-arg-order.tolk b/tolk-tester/tests/asm-arg-order.tolk index b8622060a..1558a3c72 100644 --- a/tolk-tester/tests/asm-arg-order.tolk +++ b/tolk-tester/tests/asm-arg-order.tolk @@ -219,7 +219,7 @@ fun main() { @fif_codegen """ - test27 PROC:<{ + test27() PROC:<{ 10 PUSHINT 2 PUSHINT 1 ADDCONST MUL @@ -228,18 +228,18 @@ fun main() { @fif_codegen """ - test28 PROC:<{ - get10Pure CALLDICT - get2Pure CALLDICT + test28() PROC:<{ + get10Pure() CALLDICT + get2Pure() CALLDICT 1 ADDCONST MUL }> """ @fif_codegen """ - test29 PROC:<{ - get2Impure CALLDICT - get10Impure CALLDICT + test29() PROC:<{ + get2Impure() CALLDICT + get10Impure() CALLDICT SWAP 1 ADDCONST MUL }> @@ -247,17 +247,17 @@ fun main() { @fif_codegen """ - test30 PROC:<{ + test30() PROC:<{ ... - g10 GETGLOB - g2 GETGLOB + $g10 GETGLOB + $g2 GETGLOB 1 ADDCONST MUL }> """ @fif_codegen """ - test32 PROC:<{ + test32() PROC:<{ 10 PUSHINT 2 PUSHINT 1 ADDCONST MUL @@ -266,7 +266,7 @@ fun main() { @fif_codegen """ - test34 PROC:<{ + test34() PROC:<{ x{020a} PUSHSLICE 8 LDU 8 LDU diff --git a/tolk-tester/tests/assignment-tests.tolk b/tolk-tester/tests/assignment-tests.tolk index e53020d05..bd9b28d9b 100644 --- a/tolk-tester/tests/assignment-tests.tolk +++ b/tolk-tester/tests/assignment-tests.tolk @@ -283,7 +283,7 @@ fun main(value: int, ) { @fif_codegen """ - test116 PROC:<{ // + test116() PROC:<{ // 1 PUSHINT // '10=1 2 PUSHINT // '10=1 '11=2 3 PUSHINT // '10=1 '11=2 '12=3 diff --git a/tolk-tester/tests/bit-operators.tolk b/tolk-tester/tests/bit-operators.tolk index 692275bf6..c734740df 100644 --- a/tolk-tester/tests/bit-operators.tolk +++ b/tolk-tester/tests/bit-operators.tolk @@ -126,7 +126,7 @@ fun testBoolCompareOptimized(x: bool) { @fif_codegen """ - boolWithBitwiseConst PROC:<{ + boolWithBitwiseConst() PROC:<{ 0 PUSHINT // '3 -1 PUSHINT // '3 '5 0 PUSHINT // '3 '5 '7 @@ -136,7 +136,7 @@ fun testBoolCompareOptimized(x: bool) { @fif_codegen """ - testDoUntilCodegen PROC:<{ // i n + testDoUntilCodegen() PROC:<{ // i n 0 PUSHINT // i n cnt=0 UNTIL:<{ INC // i n cnt @@ -162,7 +162,7 @@ fun testBoolCompareOptimized(x: bool) { @fif_codegen """ - testConstNegateCodegen PROC:<{ + testConstNegateCodegen() PROC:<{ TRUE // '0 FALSE // '0 '1 FALSE // '0 '1 '2 @@ -174,7 +174,7 @@ fun testBoolCompareOptimized(x: bool) { @fif_codegen """ - testBoolNegateOptimized PROC:<{ // x + testBoolNegateOptimized() PROC:<{ // x DUP // x x NOT // x '1 OVER // x '1 x @@ -186,14 +186,14 @@ fun testBoolCompareOptimized(x: bool) { @fif_codegen """ - testBoolCompareOptimized PROC:<{ // x + testBoolCompareOptimized() PROC:<{ // x DUP // x x NOT // x '1 OVER // x '1 x - eqX CALLDICT // x '1 '2 + eqX() CALLDICT // x '1 '2 NOT // x '1 '3 s2 PUSH // x '1 '3 x - eqX CALLDICT // x '1 '3 '4 + eqX() CALLDICT // x '1 '3 '4 s3 PUSH // x '1 '3 '4 x }> """ diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index d8e62bc6c..fa96bd529 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -352,7 +352,7 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ - test6 PROC:<{ + test6() PROC:<{ 18446744082299486211 PUSHINT NEWC 96 STU // '2 @@ -361,7 +361,7 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ - test17 PROC:<{ + test17() PROC:<{ 4219 PUSHINT NEWC 16 STU @@ -369,7 +369,7 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ - test18 PROC:<{ + test18() PROC:<{ 421 PUSHINT NEWC 26 STU @@ -377,7 +377,7 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ - test19 PROC:<{ + test19() PROC:<{ NEWC 123 PUSHINT SWAP @@ -392,7 +392,7 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ - test20 PROC:<{ + test20() PROC:<{ 40976 PUSHINT NEWC 16 STU @@ -400,7 +400,7 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ - test21 PROC:<{ + test21() PROC:<{ 14 PUSHINT SDSKIPFIRST }> @@ -408,7 +408,7 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ - test22 PROC:<{ + test22() PROC:<{ NEWC // '2 NEWC // b1 b2 SWAP // b2 b1 @@ -421,7 +421,7 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ - test23 PROC:<{ + test23() PROC:<{ NEWC SWAP IF:<{ diff --git a/tolk-tester/tests/codegen-check-demo.tolk b/tolk-tester/tests/codegen-check-demo.tolk index ff2ca3c2c..dec38a885 100644 --- a/tolk-tester/tests/codegen-check-demo.tolk +++ b/tolk-tester/tests/codegen-check-demo.tolk @@ -33,7 +33,7 @@ Below, I just give examples of @fif_codegen tag: @fif_codegen """ -main PROC:<{ // s +main() PROC:<{ // s 17 PUSHINT // s '3=17 OVER // s z=17 t WHILE:<{ @@ -49,7 +49,7 @@ main PROC:<{ // s @fif_codegen """ -main PROC:<{ +main() PROC:<{ ... WHILE:<{ ... @@ -76,13 +76,13 @@ main PROC:<{ @fif_codegen """ -test1 PROC:<{ +test1() PROC:<{ FALSE }> """ @fif_codegen NOT // '8 -@fif_codegen main PROC:<{ +@fif_codegen main() PROC:<{ @fif_codegen_avoid PROCINLINE @fif_codegen_avoid END c diff --git a/tolk-tester/tests/comments-tests.tolk b/tolk-tester/tests/comments-tests.tolk index 3d00a0b1a..96caefe92 100644 --- a/tolk-tester/tests/comments-tests.tolk +++ b/tolk-tester/tests/comments-tests.tolk @@ -27,7 +27,7 @@ fun main() { @fif_codegen_enable_comments @fif_codegen """ - demo_fields_def PROC:<{ // x + demo_fields_def() PROC:<{ // x // 13: f1: x + C_20 20 ADDCONST // '6 10 PUSHINT // '6 '7=10 diff --git a/tolk-tester/tests/generics-1.tolk b/tolk-tester/tests/generics-1.tolk index 39ddb2cdf..b92f27a6e 100644 --- a/tolk-tester/tests/generics-1.tolk +++ b/tolk-tester/tests/generics-1.tolk @@ -234,19 +234,19 @@ fun main(x: int): (int, [Tup2Int]) { @testcase | 112 | 0 0 | 0 0 -1 @testcase | 113 | 5 1 | 5 1 -100 (null) 5 1 -100 9 -100 (null) -100 (null) 5 1 -100 (null) (null) (null) 0 -@fif_codegen DECLPROC eq1 -@fif_codegen DECLPROC eq1 -@fif_codegen DECLPROC eq1<(int,int)> -@fif_codegen DECLPROC eq1<[int,int]> -@fif_codegen DECLPROC eq1 -@fif_codegen DECLPROC eq1 -@fif_codegen DECLPROC eq1>> +@fif_codegen DECLPROC eq1() +@fif_codegen DECLPROC eq1() +@fif_codegen DECLPROC eq1<(int,int)>() +@fif_codegen DECLPROC eq1<[int,int]>() +@fif_codegen DECLPROC eq1() +@fif_codegen DECLPROC eq1() +@fif_codegen DECLPROC eq1>>() // was inlined -@fif_codegen_avoid DECLPROC getTwo +@fif_codegen_avoid DECLPROC getTwo() @fif_codegen_avoid getTwo -@fif_codegen_avoid DECLPROC eq1 -@fif_codegen_avoid DECLPROC eq2 -@fif_codegen_avoid DECLPROC eq3 +@fif_codegen_avoid DECLPROC eq1() +@fif_codegen_avoid DECLPROC eq2() +@fif_codegen_avoid DECLPROC eq3() */ diff --git a/tolk-tester/tests/generics-2.tolk b/tolk-tester/tests/generics-2.tolk index 65d76cebf..2a6684f0d 100644 --- a/tolk-tester/tests/generics-2.tolk +++ b/tolk-tester/tests/generics-2.tolk @@ -475,12 +475,12 @@ fun main(c: Wrapper, d: WrappedInt) { @fif_codegen """ - test1 PROC:<{ + test1() PROC:<{ 23 PUSHINT // c1=23 DUP // c1=23 c2=23 }> """ -@fif_codegen DECLPROC getWrappervalue1> -@fif_codegen DECLPROC getWrappervalue2>> +@fif_codegen DECLPROC getWrappervalue1>() +@fif_codegen DECLPROC getWrappervalue2>>() */ diff --git a/tolk-tester/tests/generics-3.tolk b/tolk-tester/tests/generics-3.tolk index 691b87835..494e48ba2 100644 --- a/tolk-tester/tests/generics-3.tolk +++ b/tolk-tester/tests/generics-3.tolk @@ -136,7 +136,7 @@ fun main() { @fif_codegen """ - test3 PROC:<{ + test3() PROC:<{ 10 PUSHINT // r.USlot1=10 -1 PUSHINT // r.USlot1=10 '22=-1 FALSE // r.USlot1=10 '22=-1 '24 diff --git a/tolk-tester/tests/generics-4.tolk b/tolk-tester/tests/generics-4.tolk index 2869ad1f5..c5f3e038c 100644 --- a/tolk-tester/tests/generics-4.tolk +++ b/tolk-tester/tests/generics-4.tolk @@ -180,16 +180,16 @@ fun main() { @testcase | 107 | | 1 10 110 117 @testcase | 108 | | 10 (null) 20 -@fif_codegen DECLPROC eqUnusedT -@fif_codegen DECLPROC eqUnusedU -@fif_codegen DECLPROC mySend -@fif_codegen DECLPROC mySend -@fif_codegen DECLPROC mySend -@fif_codegen DECLPROC mySend +@fif_codegen DECLPROC eqUnusedT() +@fif_codegen DECLPROC eqUnusedU() +@fif_codegen DECLPROC mySend() +@fif_codegen DECLPROC mySend() +@fif_codegen DECLPROC mySend() +@fif_codegen DECLPROC mySend() @fif_codegen """ - test6 PROC:<{ // + test6() PROC:<{ // NEWC // '8 ENDC // p.init.USlot2 -1 PUSHINT // p.init.USlot2 '11=-1 diff --git a/tolk-tester/tests/if-else-tests.tolk b/tolk-tester/tests/if-else-tests.tolk index d34dc178b..d510dd4a4 100644 --- a/tolk-tester/tests/if-else-tests.tolk +++ b/tolk-tester/tests/if-else-tests.tolk @@ -256,7 +256,7 @@ fun main() { @fif_codegen """ - test3 PROC:<{ // x + test3() PROC:<{ // x DUP // x x 20 NEQINT // x '2 IFNOTJMP:<{ // x @@ -270,7 +270,7 @@ fun main() { @fif_codegen """ - withNoElse PROC:<{ + withNoElse() PROC:<{ DUP 123 EQINT IFJMP:<{ @@ -295,7 +295,7 @@ fun main() { @fif_codegen """ - withElse PROC:<{ + withElse() PROC:<{ DUP 123 EQINT IFJMP:<{ @@ -320,7 +320,7 @@ fun main() { @fif_codegen """ - withMatch PROC:<{ + withMatch() PROC:<{ DUP 123 EQINT IFJMP:<{ @@ -345,59 +345,59 @@ fun main() { @fif_codegen """ - demo9_1 PROC:<{ + demo9_1() PROC:<{ 0 GTINT IFJMP:<{ - t GETGLOB + $t GETGLOB 123 PUSHINT TPUSH - t SETGLOB + $t SETGLOB }> - t GETGLOB + $t GETGLOB 456 PUSHINT TPUSH - t SETGLOB + $t SETGLOB }> """ @fif_codegen """ - demo9_2 PROC:<{ + demo9_2() PROC:<{ 0 GTINT IFJMP:<{ - t GETGLOB + $t GETGLOB 123 PUSHINT TPUSH - t SETGLOB + $t SETGLOB }> }> """ @fif_codegen """ - demo9_3 PROC:<{ + demo9_3() PROC:<{ 0 GTINT IFJMP:<{ - t GETGLOB + $t GETGLOB 123 PUSHINT TPUSH - t SETGLOB + $t SETGLOB }> }> """ @fif_codegen """ - demo9_4 PROC:<{ + demo9_4() PROC:<{ DUP -100 NEQINT IFJMP:<{ 0 GTINT IFJMP:<{ - t GETGLOB + $t GETGLOB 123 PUSHINT TPUSH - t SETGLOB + $t SETGLOB }> }> DROP @@ -406,22 +406,22 @@ fun main() { @fif_codegen """ - demo10_1 PROC:<{ + demo10_1() PROC:<{ DUP 0 EQINT IFJMP:<{ DROP - t GETGLOB + $t GETGLOB 123 PUSHINT TPUSH - t SETGLOB + $t SETGLOB }> 1 EQINT IFJMP:<{ - t GETGLOB + $t GETGLOB 456 PUSHINT TPUSH - t SETGLOB + $t SETGLOB }> }> """ diff --git a/tolk-tester/tests/indexed-access.tolk b/tolk-tester/tests/indexed-access.tolk index 8cd4a74d1..dd0400c28 100644 --- a/tolk-tester/tests/indexed-access.tolk +++ b/tolk-tester/tests/indexed-access.tolk @@ -307,7 +307,7 @@ fun main(){} @fif_codegen """ - testCodegenSimple PROC:<{ + testCodegenSimple() PROC:<{ 1 PUSHINT // '2=1 SINGLE // t1 2 PUSHINT // t1 '3=2 @@ -327,43 +327,43 @@ fun main(){} STRDUMP DROP 1 PUSHINT // t1 t2 '20=1 SINGLE // t1 t2 '18 - gTup SETGLOB + $gTup SETGLOB 2 PUSHINT // t1 t2 '21=2 - gTup GETGLOB // t1 t2 '21=2 g_gTup + $gTup GETGLOB // t1 t2 '21=2 g_gTup SWAP // t1 t2 g_gTup '21=2 0 SETINDEX // t1 t2 g_gTup - gTup SETGLOB + $gTup SETGLOB x{} PUSHSLICE // t1 t2 '25 STRDUMP DROP 1 PUSHINT // t1 t2 '28=1 2 PUSHINT // t1 t2 '26=1 '27=2 PAIR - gTens SETGLOB + $gTens SETGLOB 4 PUSHINT // t1 t2 g_gTens.1=4 - gTens GETGLOB + $gTens GETGLOB UNPAIR // t1 t2 g_gTens.1=4 g_gTens.0 g_gTens.1 DROP // t1 t2 g_gTens.1=4 g_gTens.0 SWAP // t1 t2 g_gTens.0 g_gTens.1=4 PAIR - gTens SETGLOB + $gTens SETGLOB x{} PUSHSLICE // t1 t2 '36 STRDUMP DROP - gTup GETGLOB // t1 t2 g_gTup - gTens GETGLOB + $gTup GETGLOB // t1 t2 g_gTup + $gTens GETGLOB UNPAIR // t1 t2 g_gTup g_gTens.0 g_gTens.1 }> """ @fif_codegen """ - testCodegenNoPureIndexedAccess PROC:<{ + testCodegenNoPureIndexedAccess() PROC:<{ 0 PUSHINT // '8=0 }> """ @fif_codegen """ - testCodegenIndexPostfix1 PROC:<{ // x.0 x.1 + testCodegenIndexPostfix1() PROC:<{ // x.0 x.1 // ab.1 ab.0 SWAP // ab.0 ab.1 }> @@ -371,7 +371,7 @@ fun main(){} @fif_codegen """ - testCodegenIndexPostfix2 PROC:<{ // x.0 x.1.0 x.1.1 x.2 + testCodegenIndexPostfix2() PROC:<{ // x.0 x.1.0 x.1.1 x.2 s2 POP // y.0 y.2 y.1.1 s1 s2 XCHG // y.2 y.0 y.1.1 }> diff --git a/tolk-tester/tests/inline-tests.tolk b/tolk-tester/tests/inline-tests.tolk index 0645546a6..243652b46 100644 --- a/tolk-tester/tests/inline-tests.tolk +++ b/tolk-tester/tests/inline-tests.tolk @@ -450,7 +450,7 @@ fun usedIn10ButDeclaredBelow(x: int) { @fif_codegen_avoid Storage.save @fif_codegen """ - test5 PROC:<{ // + test5() PROC:<{ // c4 PUSH // '7 CTOS // s LDMSGADDR // '11 s @@ -458,7 +458,7 @@ fun usedIn10ButDeclaredBelow(x: int) { @fif_codegen """ - test6 PROC:<{ + test6() PROC:<{ 11 PUSHINT DUP }> @@ -466,7 +466,7 @@ fun usedIn10ButDeclaredBelow(x: int) { @fif_codegen """ - test8 PROC:<{ // + test8() PROC:<{ // 5 PUSHINT NEWC 32 STU // '3 @@ -480,7 +480,7 @@ fun usedIn10ButDeclaredBelow(x: int) { @fif_codegen """ - test9 PROC:<{ // x + test9() PROC:<{ // x 0 PUSHINT // p1.x p1.y=0 s0 s1 PUSH2 // p1.x p1.y=0 p2.x=0 p2.y NIL // x p1.y=0 p2.x=0 p2.y self @@ -506,7 +506,7 @@ fun usedIn10ButDeclaredBelow(x: int) { @fif_codegen """ - test10 PROC:<{ + test10() PROC:<{ 5 PUSHINT // '3=5 DUP // '3=5 y=5 5 PUSHINT @@ -518,7 +518,7 @@ fun usedIn10ButDeclaredBelow(x: int) { @fif_codegen """ - test12 PROC:<{ // x + test12() PROC:<{ // x 40 PUSHINT // x r1 SWAP // r1 x 2 ADDCONST // r1 y @@ -527,7 +527,7 @@ fun usedIn10ButDeclaredBelow(x: int) { @fif_codegen """ - wrapperNotInlined PROC:<{ // x + wrapperNotInlined() PROC:<{ // x DUP // x x ISNULL // x '1 IFNOTJMP:<{ // x @@ -538,7 +538,7 @@ fun usedIn10ButDeclaredBelow(x: int) { @fif_codegen """ - wrap16 PROC:<{ + wrap16() PROC:<{ IFJMP:<{ 139 PUSHINT OVER diff --git a/tolk-tester/tests/intN-tests.tolk b/tolk-tester/tests/intN-tests.tolk index bf3981112..9cdc46988 100644 --- a/tolk-tester/tests/intN-tests.tolk +++ b/tolk-tester/tests/intN-tests.tolk @@ -249,13 +249,13 @@ fun main() { @testcase | 114 | 5 | (null) (null) 0 @testcase | 114 | 15 | 15 2 typeid-2 -@fif_codegen DECLPROC assign0 -@fif_codegen DECLPROC assign0 -@fif_codegen DECLPROC assign0 +@fif_codegen DECLPROC assign0() +@fif_codegen DECLPROC assign0() +@fif_codegen DECLPROC assign0() @fif_codegen """ - test12 PROC:<{ + test12() PROC:<{ 100000000 PUSHINT // '4=100000000 }> """ diff --git a/tolk-tester/tests/lazy-algo-tests.tolk b/tolk-tester/tests/lazy-algo-tests.tolk index 363640ed3..27214077b 100644 --- a/tolk-tester/tests/lazy-algo-tests.tolk +++ b/tolk-tester/tests/lazy-algo-tests.tolk @@ -1494,7 +1494,7 @@ fun main() { @fif_codegen """ - algo40 PROC:<{ + algo40() PROC:<{ x{0102} PUSHSLICE 8 LDI NIP diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk index 36d7491d4..b7eba14fa 100644 --- a/tolk-tester/tests/lazy-load-tests.tolk +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -1686,7 +1686,7 @@ fun main() { @fif_codegen """ - demo101 PROC:<{ // + demo101() PROC:<{ // x{0102} PUSHSLICE // lazyS 8 LDI // '9 lazyS DROP // p.x @@ -1695,7 +1695,7 @@ fun main() { @fif_codegen """ - demo103 PROC:<{ // + demo103() PROC:<{ // x{0102} PUSHSLICE // lazyS 8 PUSHINT SDSKIPFIRST // lazyS @@ -1705,7 +1705,7 @@ fun main() { @fif_codegen """ - demo106 PROC:<{ // s + demo106() PROC:<{ // s // lazyS PUSHNULL // lazyS x SWAP // x lazyS @@ -1716,7 +1716,7 @@ fun main() { @fif_codegen """ - demo109 PROC:<{ // + demo109() PROC:<{ // x{8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_} PUSHSLICE // lazyS 65 PUSHINT SDSKIPFIRST // lazyS @@ -1727,7 +1727,7 @@ fun main() { @fif_codegen """ - demo113 PROC:<{ + demo113() PROC:<{ // lazyS x{01} SDBEGINSQ IF:<{ @@ -1759,7 +1759,7 @@ fun main() { @fif_codegen """ - demo134 PROC:<{ // + demo134() PROC:<{ // x{010f} PUSHSLICE // lazyS x{01} SDBEGINSQ // lazyS '6 134 THROWIFNOT // lazyS @@ -1770,7 +1770,7 @@ fun main() { @fif_codegen """ - demo136 PROC:<{ + demo136() PROC:<{ x{0102} PUSHSLICE SWAP IFJMP:<{ @@ -1786,7 +1786,7 @@ fun main() { @fif_codegen """ - demo146 PROC:<{ + demo146() PROC:<{ x{0000000} PUSHSLICE LDOPTREF DROP @@ -1796,7 +1796,7 @@ fun main() { @fif_codegen """ - demo201 PROC:<{ + demo201() PROC:<{ x{0102} PUSHSLICE DUP 8 LDI @@ -1819,7 +1819,7 @@ fun main() { @fif_codegen """ - demo203 PROC:<{ // + demo203() PROC:<{ // x{010203040506070809} PUSHSLICE // lazyS DUP // '14 lazyS 8 LDU // '14 '16 lazyS @@ -1834,7 +1834,7 @@ fun main() { @fif_codegen """ - demo206 PROC:<{ // + demo206() PROC:<{ // x{010203040506070809} PUSHSLICE // lazyS DUP // '14 lazyS 8 LDU // '14 o.f1 lazyS @@ -1851,7 +1851,7 @@ fun main() { @fif_codegen """ - demo215 PROC:<{ + demo215() PROC:<{ x{0102} PUSHSLICE 8 LDI 8 LDI @@ -1865,7 +1865,7 @@ fun main() { @fif_codegen """ - demo302 PROC:<{ + demo302() PROC:<{ x{0102} PUSHSLICE 8 LDI NIP @@ -1877,9 +1877,9 @@ fun main() { @fif_codegen """ - demo311 PROC:<{ // + demo311() PROC:<{ // 777 PUSHINT // '6=777 - contract.getFakeData CALLDICT // '7 + contract.getFakeData() CALLDICT // '7 CTOS // lazyS 1 LDI // '13 lazyS NIP // lazyS @@ -1899,7 +1899,7 @@ fun main() { @fif_codegen """ - demo404 PROC:<{ + demo404() PROC:<{ // lazyS x{12345678} SDBEGINSQ IF:<{ @@ -1939,7 +1939,7 @@ fun main() { @fif_codegen """ - demo410 PROC:<{ + demo410() PROC:<{ // lazyS x{01} SDBEGINSQ IFJMP:<{ @@ -1965,7 +1965,7 @@ fun main() { @fif_codegen """ - demo411 PROC:<{ // s + demo411() PROC:<{ // s // lazyS 1 LDU // '10 lazyS DROP // '10 @@ -1977,5 +1977,5 @@ fun main() { }> """ -@fif_codegen DECLPROC pushToGlobalTup +@fif_codegen DECLPROC pushToGlobalTup() */ diff --git a/tolk-tester/tests/logical-operators.tolk b/tolk-tester/tests/logical-operators.tolk index c925b8c1a..8242d5ff4 100644 --- a/tolk-tester/tests/logical-operators.tolk +++ b/tolk-tester/tests/logical-operators.tolk @@ -217,7 +217,7 @@ fun main() { @fif_codegen """ - simpleAllConst PROC:<{ + simpleAllConst() PROC:<{ TRUE 0 PUSHINT TRUE @@ -232,7 +232,7 @@ fun main() { @fif_codegen """ - compileTimeEval1 PROC:<{ // x + compileTimeEval1() PROC:<{ // x DUP // x x 0 EQINT // x '1 FALSE // x '1 '4 @@ -248,7 +248,7 @@ fun main() { @fif_codegen """ - withIfNot PROC:<{ // x y + withIfNot() PROC:<{ // x y c2 SAVE SAMEALTSAVE OVER // x y x @@ -268,7 +268,7 @@ fun main() { @fif_codegen """ - testAndConstCodegen PROC:<{ + testAndConstCodegen() PROC:<{ FALSE 0 PUSHINT DUP @@ -290,7 +290,7 @@ fun main() { @fif_codegen """ - testOrConstCodegen PROC:<{ + testOrConstCodegen() PROC:<{ -1 PUSHINT TRUE FALSE @@ -315,7 +315,7 @@ For example, `a && b` can be expressed without IFs. These are moments of future optimizations. For now, it's more than enough. @fif_codegen """ - testAndSimpleCodegen PROC:<{ // a b + testAndSimpleCodegen() PROC:<{ // a b SWAP // b a IF:<{ // b 0 NEQINT // '2 @@ -328,7 +328,7 @@ These are moments of future optimizations. For now, it's more than enough. @fif_codegen """ - testOrSimpleCodegen PROC:<{ // a b + testOrSimpleCodegen() PROC:<{ // a b SWAP // b a 0 GTINT // b '3 IF:<{ // b @@ -343,7 +343,7 @@ These are moments of future optimizations. For now, it's more than enough. @fif_codegen """ - testConvertIfToIfnot PROC:<{ // x + testConvertIfToIfnot() PROC:<{ // x DUP // x x 100 THROWIF DUP // x x diff --git a/tolk-tester/tests/match-by-expr-tests.tolk b/tolk-tester/tests/match-by-expr-tests.tolk index ca3e21d83..489cdf748 100644 --- a/tolk-tester/tests/match-by-expr-tests.tolk +++ b/tolk-tester/tests/match-by-expr-tests.tolk @@ -181,7 +181,7 @@ fun main() { @fif_codegen """ - test1 PROC:<{ // x + test1() PROC:<{ // x DUP // x x 1 EQINT // x '3 IF:<{ // x @@ -207,16 +207,16 @@ fun main() { @fif_codegen """ - test2 PROC:<{ + test2() PROC:<{ 300 PUSHINT }> """ @fif_codegen """ - test3 PROC:<{ // init + test3() PROC:<{ // init DUP // init init - isGt10 CALLDICT // init '2 + isGt10() CALLDICT // init '2 -1 EQINT // init '5 IF:<{ // init 10 PUSHINT // init '3=10 @@ -227,7 +227,7 @@ fun main() { @fif_codegen """ - test8 PROC:<{ // x + test8() PROC:<{ // x DROP // 30 PUSHINT // r1 }> @@ -235,29 +235,29 @@ fun main() { @fif_codegen """ - helper12 PROC:<{ + helper12() PROC:<{ DUP 1 EQINT IFJMP:<{ DROP 1 PUSHINT - g12 SETGLOB + $g12 SETGLOB }> DUP 2 EQINT IFJMP:<{ DROP 2 PUSHINT - g12 SETGLOB + $g12 SETGLOB }> DUP 3 EQINT IFJMP:<{ DROP 3 PUSHINT - g12 SETGLOB + $g12 SETGLOB }> - g12 SETGLOB + $g12 SETGLOB }> """ diff --git a/tolk-tester/tests/methods-tests.tolk b/tolk-tester/tests/methods-tests.tolk index 83118ef36..db3c3e9cd 100644 --- a/tolk-tester/tests/methods-tests.tolk +++ b/tolk-tester/tests/methods-tests.tolk @@ -309,7 +309,7 @@ fun main() {} @testcase | 117 | | 3 @testcase | 118 | | 10 -@fif_codegen DECLPROC Pair.createFrom -@fif_codegen DECLPROC Pair.compareWith -@fif_codegen DECLPROC CounterMsg.onInternalMessage +@fif_codegen DECLPROC Pair.createFrom() +@fif_codegen DECLPROC Pair.compareWith() +@fif_codegen DECLPROC CounterMsg.onInternalMessage() */ diff --git a/tolk-tester/tests/mutate-methods.tolk b/tolk-tester/tests/mutate-methods.tolk index a53b2c15c..892a625ce 100644 --- a/tolk-tester/tests/mutate-methods.tolk +++ b/tolk-tester/tests/mutate-methods.tolk @@ -326,26 +326,26 @@ fun main(){} @fif_codegen """ - int.incrementInPlace PROC:<{ // self byValue + int.incrementInPlace() PROC:<{ // self byValue ADD // self }> """ @fif_codegen """ - testIncrement2 PROC:<{ + testIncrement2() PROC:<{ ... - incrementTwoInPlace CALLDICT // x y sum1 + incrementTwoInPlace() CALLDICT // x y sum1 -ROT 10 PUSHINT // sum1 x y '11=10 - int.incrementTwoInPlace CALLDICT // sum1 x y sum2 + int.incrementTwoInPlace() CALLDICT // sum1 x y sum2 s1 s3 s0 XCHG3 // x y sum1 sum2 }> """ @fif_codegen """ - load_next PROC:<{ // cs + load_next() PROC:<{ // cs 32 LDI // '4 cs SWAP // cs '4 }> @@ -353,14 +353,14 @@ fun main(){} @fif_codegen """ - testStoreUintPureUnusedResult PROC:<{ + testStoreUintPureUnusedResult() PROC:<{ 0 PUSHINT // '11=0 }> """ @fif_codegen """ - testStoreUintImpureUnusedResult PROC:<{ + testStoreUintImpureUnusedResult() PROC:<{ NEWC // b STIX // '2 DROP // diff --git a/tolk-tester/tests/no-spaces.tolk b/tolk-tester/tests/no-spaces.tolk index 016cf478a..732de8011 100644 --- a/tolk-tester/tests/no-spaces.tolk +++ b/tolk-tester/tests/no-spaces.tolk @@ -80,14 +80,14 @@ fun`main`(){} @fif_codegen """ - get_+-+1 PROC:<{ + get_+-+1() PROC:<{ -1 PUSHINT }> """ @fif_codegen """ - unary+bitwise-constant PROC:<{ + unary+bitwise-constant() PROC:<{ -4 PUSHINT 6 PUSHINT -4 PUSHINT @@ -97,14 +97,14 @@ fun`main`(){} @fif_codegen """ - unary_const_check PROC:<{ + unary_const_check() PROC:<{ -1 PUSHINT // fst1=-1 DUP // fst1=-1 snd1=-1 2 PUSHINT // fst1=-1 snd1=-1 trd1=2 s1 s1 s0 PUSH3 // fst1=-1 snd1=-1 trd1=2 fst2=-1 snd2=-1 trd2=2 - add3 CALLDICT // fst1=-1 snd1=-1 trd1=2 '13 + add3() CALLDICT // fst1=-1 snd1=-1 trd1=2 '13 3 -ROLL // '13 fst1=-1 snd1=-1 trd1=2 - add3 CALLDICT // '13 '14 + add3() CALLDICT // '13 '14 PAIR // '12 }> """ diff --git a/tolk-tester/tests/null-keyword.tolk b/tolk-tester/tests/null-keyword.tolk index 98b90811f..c36a66c26 100644 --- a/tolk-tester/tests/null-keyword.tolk +++ b/tolk-tester/tests/null-keyword.tolk @@ -92,7 +92,7 @@ fun main() { @fif_codegen """ - test1 PROC:<{ + test1() PROC:<{ PUSHNULL // numbers 1 PUSHINT // numbers '2=1 SWAP // '2=1 numbers @@ -113,14 +113,14 @@ fun main() { @fif_codegen """ - main PROC:<{ + main() PROC:<{ 1 PUSHINT // '3=1 }> """ @fif_codegen """ - test7 PROC:<{ + test7() PROC:<{ ... LDOPTREF // b '9 '8 DROP // b c diff --git a/tolk-tester/tests/nullable-tensors.tolk b/tolk-tester/tests/nullable-tensors.tolk index 56e111b82..5b0394542 100644 --- a/tolk-tester/tests/nullable-tensors.tolk +++ b/tolk-tester/tests/nullable-tensors.tolk @@ -555,7 +555,7 @@ fun main(){} @fif_codegen """ - isTensorNull PROC:<{ // t.0 t.1 t.UTag + isTensorNull() PROC:<{// t.0 t.1 t.UTag 2 1 BLKDROP2 // t.UTag 0 EQINT // '3 }> @@ -563,14 +563,14 @@ fun main(){} @fif_codegen """ - test113 PROC:<{ + test113() PROC:<{ 1 PUSHINT // '2=1 PUSHNULL // '2=1 '3 PAIR // t 1 INDEX // '5 PUSHNULL // '5 '6 0 PUSHINT // '5 '6 '7=0 - isTensorNull CALLDICT // '8 + isTensorNull() CALLDICT // '8 }> """ */ diff --git a/tolk-tester/tests/op-priority.tolk b/tolk-tester/tests/op-priority.tolk index 27b30c988..e635d5423 100644 --- a/tolk-tester/tests/op-priority.tolk +++ b/tolk-tester/tests/op-priority.tolk @@ -95,21 +95,21 @@ fun main() { @fif_codegen """ - unary_minus_1 PROC:<{ // a b c + unary_minus_1() PROC:<{ // a b c -ROT // c a b ADD // c '3 NEGATE // c '4 SWAP // '4 c MUL // '5 }> - unary_minus_2 PROC:<{ // a b c + unary_minus_2() PROC:<{ // a b c -ROT // c a b ADD // c '3 NEGATE // c '4 SWAP // '4 c MUL // '5 }> - unary_minus_3 PROC:<{ // a b c + unary_minus_3() PROC:<{ // a b c -ROT // c a b ADD // c '3 SWAP // '3 c diff --git a/tolk-tester/tests/pack-unpack-1.tolk b/tolk-tester/tests/pack-unpack-1.tolk index 379055c46..68ce0bc10 100644 --- a/tolk-tester/tests/pack-unpack-1.tolk +++ b/tolk-tester/tests/pack-unpack-1.tolk @@ -106,7 +106,7 @@ fun main(c: cell) { @fif_codegen """ - test3 PROC:<{ + test3() PROC:<{ 42949672980 PUSHINT NEWC 64 STI // b diff --git a/tolk-tester/tests/pack-unpack-4.tolk b/tolk-tester/tests/pack-unpack-4.tolk index d2bcfb618..6d913f584 100644 --- a/tolk-tester/tests/pack-unpack-4.tolk +++ b/tolk-tester/tests/pack-unpack-4.tolk @@ -124,7 +124,7 @@ fun main() {} @fif_codegen """ - test6 PROC:<{ // + test6() PROC:<{ // b{010000000000001111} PUSHSLICE // s b{00} SDBEGINSQ // s '8 IF:<{ // s diff --git a/tolk-tester/tests/pack-unpack-6.tolk b/tolk-tester/tests/pack-unpack-6.tolk index 34e40578c..51c1f7c13 100644 --- a/tolk-tester/tests/pack-unpack-6.tolk +++ b/tolk-tester/tests/pack-unpack-6.tolk @@ -130,7 +130,7 @@ RETALT @fif_codegen """ - test3 PROC:<{ + test3() PROC:<{ x{0109ab} PUSHSLICE NEWC STSLICE @@ -160,7 +160,7 @@ IF:<{ @fif_codegen """ - test5 PROC:<{ + test5() PROC:<{ x{11} PUSHSLICE NEWC OVER @@ -173,7 +173,7 @@ IF:<{ @fif_codegen """ - test6 PROC:<{ + test6() PROC:<{ x{11} PUSHSLICE NEWC STSLICE @@ -183,7 +183,7 @@ IF:<{ @fif_codegen """ - test8 PROC:<{ // + test8() PROC:<{ // x{010f} PUSHSLICE // '0 16 PUSHPOW2DEC // s '2=65535 SWAP // '2=65535 s @@ -197,7 +197,7 @@ IF:<{ @fif_codegen """ - test9 PROC:<{ + test9() PROC:<{ x{010f} PUSHSLICE x{01} SDBEGINSQ IF:<{ diff --git a/tolk-tester/tests/parse-address.tolk b/tolk-tester/tests/parse-address.tolk index 36fb2ec2b..c489527ad 100644 --- a/tolk-tester/tests/parse-address.tolk +++ b/tolk-tester/tests/parse-address.tolk @@ -241,7 +241,7 @@ fun main() { @fif_codegen """ - check0 PROC:<{ // addr + check0() PROC:<{ // addr REWRITESTDADDR DROP // '2 0 EQINT // '4 @@ -251,7 +251,7 @@ fun main() { @fif_codegen """ - codegenAddrEq PROC:<{ // a b + codegenAddrEq() PROC:<{ // a b 2DUP // a b a b SDEQ // a b '2 IFJMP:<{ // a b diff --git a/tolk-tester/tests/remove-unused-functions.tolk b/tolk-tester/tests/remove-unused-functions.tolk index cd72fd08c..a6b910009 100644 --- a/tolk-tester/tests/remove-unused-functions.tolk +++ b/tolk-tester/tests/remove-unused-functions.tolk @@ -32,18 +32,18 @@ fun main(): (int, int, int) { @testcase | 0 | | 3 10 20 -@fif_codegen DECLPROC used_as_noncall1 -@fif_codegen DECLGLOBVAR used_gv +@fif_codegen DECLPROC used_as_noncall1() +@fif_codegen DECLGLOBVAR $used_gv -@fif_codegen_avoid DECLPROC unused1 -@fif_codegen_avoid DECLPROC unused2 -@fif_codegen_avoid DECLPROC unused3 -@fif_codegen_avoid DECLGLOBVAR unused_gv +@fif_codegen_avoid DECLPROC unused1() +@fif_codegen_avoid DECLPROC unused2() +@fif_codegen_avoid DECLPROC unused3() +@fif_codegen_avoid DECLGLOBVAR $unused_gv Note, that `usedButOptimizedOut()` (a pure function which result is unused) is currently codegenerated, since it's formally reachable. This is because optimizing code is a moment of codegen for now (later than marking unused symbols). -@fif_codegen DECLPROC usedButOptimizedOut -@fif_codegen_avoid usedButOptimizedOut CALLDICT +@fif_codegen DECLPROC usedButOptimizedOut() +@fif_codegen_avoid usedButOptimizedOut() CALLDICT */ diff --git a/tolk-tester/tests/self-keyword.tolk b/tolk-tester/tests/self-keyword.tolk index e5941dc81..5fa5538dd 100644 --- a/tolk-tester/tests/self-keyword.tolk +++ b/tolk-tester/tests/self-keyword.tolk @@ -226,27 +226,29 @@ fun main() { } @fif_codegen """ - int.incChained PROC:<{ // self - INC // self + int.incChained() PROC:<{ // self + INC // self }> - int.incChained2 PROC:<{ // self - int.incChained CALLDICT // self + int.incChained2() PROC:<{ // self + int.incChained() CALLDICT // self }> - int.incChained3 PROC:<{ // self - int.incChained CALLDICT // self + int.incChained3() PROC:<{ // self + int.incChained() CALLDICT // self }> - int.incChained4 PROC:<{ // self - int.incChained CALLDICT // self + int.incChained4() PROC:<{ // self + int.incChained() CALLDICT // self }> """ @fif_codegen """ - testIncChainedCodegen PROC:<{ // x - int.incChained CALLDICT // x - int.incChained2 CALLDICT // x - int.incChained3 CALLDICT // x - int.incChained4 CALLDICT // x + testIncChainedCodegen() PROC:<{ // x + int.incChained() CALLDICT // x + int.incChained2() CALLDICT // x + int.incChained3() CALLDICT // x + int.incChained4() CALLDICT // x }> """ + +@fif_codegen int.checkNotEqChainableMutateAnother() PREPAREDICT */ diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index 247431eb0..1a0f96a0f 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -749,12 +749,12 @@ fun main(x: int?): int { @stderr warning: unreachable code @stderr var t2 redef = getNullableInt()!; -@fif_codegen eq55 PROC:<{ -@fif_codegen eq55 PROC:<{ +@fif_codegen eq55() PROC:<{ +@fif_codegen eq55() PROC:<{ @fif_codegen """ - test60 PROC:<{ + test60() PROC:<{ 101 PUSHINT // cc1 109 PUSHINT // cc1 cc2 }> diff --git a/tolk-tester/tests/some-tests-2.tolk b/tolk-tester/tests/some-tests-2.tolk index 3568da56c..9a4cb8cd0 100644 --- a/tolk-tester/tests/some-tests-2.tolk +++ b/tolk-tester/tests/some-tests-2.tolk @@ -114,14 +114,14 @@ fun test96(p: Point) { @fif_codegen """ - touchCodegen2 PROC:<{ - get10 CALLDICT // f + touchCodegen2() PROC:<{ + get10() CALLDICT // f }> """ @fif_codegen """ - testDumpDontPolluteStack PROC:<{ + testDumpDontPolluteStack() PROC:<{ ... DUMPSTK x{6d79} PUSHSLICE // f s '5 @@ -132,7 +132,7 @@ fun test96(p: Point) { @fif_codegen """ - testStartBalanceCodegen1 PROC:<{ + testStartBalanceCodegen1() PROC:<{ BALANCE // t FIRST // first }> @@ -140,7 +140,7 @@ fun test96(p: Point) { @fif_codegen """ - testStartBalanceCodegen2 PROC:<{ + testStartBalanceCodegen2() PROC:<{ BALANCE FIRST // first }> @@ -148,24 +148,24 @@ fun test96(p: Point) { @fif_codegen """ - test95 PROC:<{ + test95() PROC:<{ ... - next GETGLOB // g_next + $next GETGLOB // g_next 3 PUSHINT // g_next '14=3 4 PUSHINT // g_next '14=3 '15=4 5 PUSHINT // g_next '14=3 '15=4 '16=5 TRIPLE // '10 '11 SWAP - cur SETGLOB - next SETGLOB - cur GETGLOB // g_cur - next GETGLOB // g_cur g_next + $cur SETGLOB + $next SETGLOB + $cur GETGLOB // g_cur + $next GETGLOB // g_cur g_next }> """ @fif_codegen """ - test96 PROC:<{ + test96() PROC:<{ s1 DUMP s0 DUMP 2 BLKDROP }> """ diff --git a/tolk-tester/tests/special-fun-names.tolk b/tolk-tester/tests/special-fun-names.tolk index 8fae6d5db..11c34f98d 100644 --- a/tolk-tester/tests/special-fun-names.tolk +++ b/tolk-tester/tests/special-fun-names.tolk @@ -4,6 +4,23 @@ fun onRunTickTock() { return -2; } fun onSplitPrepare() { return -3; } fun onSplitInstall() { return -4; } +// 'execute' and many other names are reserved words in Fift, +// but since they are codegenerated like `execute()` (it's a valid Fift name!), no collisions +@noinline +fun execute() { return 123 } +@noinline +fun `PUSHINT`() { return 456 } +@noinline +fun `@havebits`() { return 789 } + +global swap: int; + +@method_id(101) +fun test101() { + swap = 1; + return execute() + `PUSHINT`() + `@havebits`() + swap; +} + /** @experimental_options remove-unused-functions @@ -12,13 +29,18 @@ fun onSplitInstall() { return -4; } @testcase | -2 | | -2 @testcase | -3 | | -3 @testcase | -4 | | -4 +@testcase | 101 | | 1369 @fif_codegen """ - 0 DECLMETHOD onInternalMessage - -1 DECLMETHOD onExternalMessage - -2 DECLMETHOD onRunTickTock - -3 DECLMETHOD onSplitPrepare - -4 DECLMETHOD onSplitInstall + 0 DECLMETHOD onInternalMessage() + -1 DECLMETHOD onExternalMessage() + -2 DECLMETHOD onRunTickTock() + -3 DECLMETHOD onSplitPrepare() + -4 DECLMETHOD onSplitInstall() """ + +@fif_codegen DECLPROC execute() +@fif_codegen execute() CALLDICT +@fif_codegen DECLGLOBVAR $swap */ diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index 7c224bab4..18d596c34 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -627,7 +627,7 @@ type PointAlias = Point; @fif_codegen """ - test6 PROC:<{ + test6() PROC:<{ 10 PUSHINT // p.x 20 PUSHINT // p.x p.y }> @@ -635,7 +635,7 @@ type PointAlias = Point; @fif_codegen """ - sumXY PROC:<{ // point.x point.y + sumXY() PROC:<{ // point.x point.y ADD // '2 }> """ @@ -643,7 +643,7 @@ type PointAlias = Point; @fif_codegen """ - test23 PROC:<{ // p.x p.y + test23() PROC:<{ // p.x p.y 2DROP 20 PUSHINT // '10=20 }> @@ -651,27 +651,27 @@ type PointAlias = Point; @fif_codegen """ - test29 PROC:<{ + test29() PROC:<{ PUSHNULL // '5 }> """ @fif_codegen """ - test35 PROC:<{ // - getXPure CALLDICT // '2 - getYPure CALLDICT // p.x p.y + test35() PROC:<{ // + getXPure() CALLDICT // '2 + getYPure() CALLDICT // p.x p.y }> """ @fif_codegen """ - test36 PROC:<{ // + test36() PROC:<{ // NIL // '0 - t_impure SETGLOB // - getYImpure CALLDICT // '4 - getXImpure CALLDICT // p.y p.x - t_impure GETGLOB // p.y p.x g_t_impure + $t_impure SETGLOB // + getYImpure() CALLDICT // '4 + getXImpure() CALLDICT // p.y p.x + $t_impure GETGLOB // p.y p.x g_t_impure s1 s2 XCHG // p.x p.y g_t_impure }> """ diff --git a/tolk-tester/tests/try-catch-tests.tolk b/tolk-tester/tests/try-catch-tests.tolk index b1878955e..04e32531c 100644 --- a/tolk-tester/tests/try-catch-tests.tolk +++ b/tolk-tester/tests/try-catch-tests.tolk @@ -298,7 +298,7 @@ fun main() { @fif_codegen """ - testCodegen1 PROC:<{ // x + testCodegen1() PROC:<{ // x DUP // x x 10 GTINT // x '2 IFJMP:<{ // x @@ -316,17 +316,17 @@ fun main() { @fif_codegen """ - testCodegen2 PROC:<{ // x + testCodegen2() PROC:<{ // x DUP // x x 10 GTINT // x '2 IFJMP:<{ // x DROP // - alwaysThrow123 CALLDICT + alwaysThrow123() CALLDICT }> // x 10 LESSINT // '5 IFJMP:<{ // FALSE // '6 - anotherNever CALLDICT + anotherNever() CALLDICT }> // 0 PUSHINT // '8=0 }> @@ -334,7 +334,7 @@ fun main() { @fif_codegen """ - testCodegen3 PROC:<{ // numberId paramVal + testCodegen3() PROC:<{ // numberId paramVal SWAP -1000 PUSHINT // paramVal numberId '2=-1000 EQUAL // paramVal '3 diff --git a/tolk-tester/tests/type-aliases-tests.tolk b/tolk-tester/tests/type-aliases-tests.tolk index 5d725ecf5..3fc534505 100644 --- a/tolk-tester/tests/type-aliases-tests.tolk +++ b/tolk-tester/tests/type-aliases-tests.tolk @@ -212,7 +212,7 @@ fun main(x: MInt, y: MInt?) { @fif_codegen """ - test9 PROC:<{ // b + test9() PROC:<{ // b DUP // b b IFNOTJMP:<{ // b NOT // '2 diff --git a/tolk-tester/tests/union-types-tests.tolk b/tolk-tester/tests/union-types-tests.tolk index 53e6caefa..5dc3c9fca 100644 --- a/tolk-tester/tests/union-types-tests.tolk +++ b/tolk-tester/tests/union-types-tests.tolk @@ -854,14 +854,14 @@ fun main() { @fif_codegen """ - test26 PROC:<{ + test26() PROC:<{ 42 PUSHINT // result }> """ @fif_codegen """ - checkSimple PROC:<{ // a.USlot1 a.UTag + checkSimple() PROC:<{ // a.USlot1 a.UTag NIP // a.UTag DUP // a.UTag a.UTag 1 EQINT // a.UTag '3 @@ -881,7 +881,7 @@ fun main() { @fif_codegen """ - test61 PROC:<{ + test61() PROC:<{ NIP DUP 46 EQINT @@ -897,6 +897,6 @@ fun main() { }> """ -@fif_codegen DECLPROC checkGeneric3 +@fif_codegen DECLPROC checkGeneric3() */ diff --git a/tolk-tester/tests/use-before-declare.tolk b/tolk-tester/tests/use-before-declare.tolk index 42afe8b00..0c59c6715 100644 --- a/tolk-tester/tests/use-before-declare.tolk +++ b/tolk-tester/tests/use-before-declare.tolk @@ -48,14 +48,14 @@ const first = 1; @fif_codegen """ - test1 PROC:<{ + test1() PROC:<{ 30 PUSHINT // '10 }> """ @fif_codegen """ - test2 PROC:<{ + test2() PROC:<{ 2 PUSHINT // '2 }> """ diff --git a/tolk-tester/tests/var-apply-tests.tolk b/tolk-tester/tests/var-apply-tests.tolk index d1361bd0c..702c9fddc 100644 --- a/tolk-tester/tests/var-apply-tests.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -325,6 +325,18 @@ fun testSavingFunWithDefaults() { return cc(1, 2, 3); } +@noinline +fun tupleCreator() { return createEmptyTuple() } +@noinline +fun getTupleCreator() { return tupleCreator } + +@method_id(120) +fun testNoInlineSaving() { + var creator = getTupleCreator(); + __expect_type(creator, "() -> tuple"); + return creator(); +} + fun main() {} @@ -355,4 +367,5 @@ fun main() {} @testcase | 117 | | 6 8 @testcase | 118 | | 0 10 -1 (null) @testcase | 119 | | 6 +@testcase | 120 | | [] */ diff --git a/tolk/codegen.cpp b/tolk/codegen.cpp index d392a58b8..f5ef89de3 100644 --- a/tolk/codegen.cpp +++ b/tolk/codegen.cpp @@ -338,7 +338,7 @@ bool Op::generate_code_step(Stack& stack) { if (!used || disabled()) { return true; } - stack.o << AsmOp::Custom(loc, g_sym->name + " GETGLOB", 0, 1); + stack.o << AsmOp::Custom(loc, "$" + g_sym->name + " GETGLOB", 0, 1); if (left.size() != 1) { tolk_assert(left.size() <= 15); stack.o << AsmOp::UnTuple(loc, (int)left.size()); @@ -376,7 +376,7 @@ bool Op::generate_code_step(Stack& stack) { std::get(f_sym->body)->compile(stack.o, res, args0, loc); // compile res := f (args0) } } else { - stack.o << AsmOp::Custom(loc, f_sym->name + " CALLDICT", (int)right.size(), (int)left.size()); + stack.o << AsmOp::Custom(loc, f_sym->name + "() CALLDICT", (int)right.size(), (int)left.size()); } stack.o.undent(); stack.o << AsmOp::Custom({}, "}>"); @@ -518,12 +518,12 @@ bool Op::generate_code_step(Stack& stack) { } } else { if (f_sym->inline_mode == FunctionInlineMode::inlineViaFif || f_sym->inline_mode == FunctionInlineMode::inlineRef) { - stack.o << AsmOp::Custom(loc, f_sym->name + " INLINECALLDICT", (int)right.size(), (int)left.size()); + stack.o << AsmOp::Custom(loc, f_sym->name + "() INLINECALLDICT", (int)right.size(), (int)left.size()); } else if (f_sym->is_code_function() && std::get(f_sym->body)->code->require_callxargs) { - stack.o << AsmOp::Custom(loc, f_sym->name + (" PREPAREDICT"), 0, 2); + stack.o << AsmOp::Custom(loc, f_sym->name + ("() PREPAREDICT"), 0, 2); exec_callxargs((int)right.size() + 1, (int)left.size()); } else { - stack.o << AsmOp::Custom(loc, f_sym->name + " CALLDICT", (int)right.size(), (int)left.size()); + stack.o << AsmOp::Custom(loc, f_sym->name + "() CALLDICT", (int)right.size(), (int)left.size()); } } stack.modified(); @@ -551,7 +551,7 @@ bool Op::generate_code_step(Stack& stack) { stack.o << AsmOp::Tuple(loc, (int)right.size()); } if (!right.empty()) { - stack.o << AsmOp::Custom(loc, g_sym->name + " SETGLOB", 1, 0); + stack.o << AsmOp::Custom(loc, "$" + g_sym->name + " SETGLOB", 1, 0); stack.modified(); } stack.s.resize(k); diff --git a/tolk/pipe-generate-fif-output.cpp b/tolk/pipe-generate-fif-output.cpp index f0bd7c5a3..237649f45 100644 --- a/tolk/pipe-generate-fif-output.cpp +++ b/tolk/pipe-generate-fif-output.cpp @@ -91,13 +91,13 @@ static void generate_output_func(FunctionPtr fun_ref) { } std::cout << std::endl; } - std::cout << " " << fun_ref->name << " PROC" << modifier << ":<{"; + std::cout << " " << fun_ref->name << "() PROC" << modifier << ":<{"; int mode = 0; if (G.settings.stack_layout_comments) { mode |= Stack::_StackComments; size_t len = 2 + fun_ref->name.size() + 5 + std::strlen(modifier) + 3; while (len < 28) { // a bit weird, but okay for now: - std::cout << ' '; // insert space after "xxx PROC" before `// stack state` + std::cout << ' '; // insert space after "xxx() PROC" before `// stack state` len++; // (the first AsmOp-comment that will be code generated) } // space is the same as used to align comments in asmops.cpp std::cout << '\t'; @@ -153,9 +153,9 @@ void pipeline_generate_fif_output_to_std_cout() { std::cout << " "; if (fun_ref->has_tvm_method_id()) { - std::cout << fun_ref->tvm_method_id << " DECLMETHOD " << fun_ref->name << "\n"; + std::cout << fun_ref->tvm_method_id << " DECLMETHOD " << fun_ref->name << "()\n"; } else { - std::cout << "DECLPROC " << fun_ref->name << "\n"; + std::cout << "DECLPROC " << fun_ref->name << "()\n"; } } @@ -180,7 +180,7 @@ void pipeline_generate_fif_output_to_std_cout() { continue; } - std::cout << " " << "DECLGLOBVAR " << var_ref->name << "\n"; + std::cout << " " << "DECLGLOBVAR $" << var_ref->name << "\n"; } for (FunctionPtr fun_ref : G.all_functions) { From 6354018b27c979c9597fcf6ff7880f84bd58758a Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sun, 22 Jun 2025 09:25:12 +0300 Subject: [PATCH 333/388] [Tolk] Better handle nested unknown types at assignment Fix an issue with a ternary operator that returns (and is assigned to) multiple variables --- tolk-tester/tests/dicts-demo.tolk | 13 +++++++++++++ tolk/smart-casts-cfg.cpp | 18 +++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/tolk-tester/tests/dicts-demo.tolk b/tolk-tester/tests/dicts-demo.tolk index 1ff4cd8c3..f53debb7f 100644 --- a/tolk-tester/tests/dicts-demo.tolk +++ b/tolk-tester/tests/dicts-demo.tolk @@ -93,6 +93,17 @@ fun test104() { ); } +@method_id(105) +fun test105(takeNext: bool) { + var dict = createEmptyDict(); + dict.uDictSet(32, 0, stringHexToSlice("01")); + dict.uDictSet(32, 8, stringHexToSlice("02")); + var (next, data, found) = takeNext ? dict.uDictGetNext(32, -1) : dict.uDictGetPrev(32, 9); + __expect_type(found, "bool"); + assert(found, 10); + return data!.loadUint(8); +} + fun main() {} /** @@ -102,4 +113,6 @@ fun main() {} @testcase | 102 | | [ [ 4 104 ] [ 3 103 ] [ 2 102 ] [ 1 101 ] ] @testcase | 103 | | 1 1 2 1 3 (null) @testcase | 104 | | 12 34 56 78 0 +@testcase | 105 | -1 | 1 +@testcase | 105 | 0 | 2 */ diff --git a/tolk/smart-casts-cfg.cpp b/tolk/smart-casts-cfg.cpp index 6597efe95..7dd60ee97 100644 --- a/tolk/smart-casts-cfg.cpp +++ b/tolk/smart-casts-cfg.cpp @@ -218,6 +218,22 @@ static TypePtr calculate_type_lca(TypePtr a, TypePtr b, bool* became_union = nul return resulting_union; } +// when `var v = rhs`, `v` is `unknown` before assignment (before rhs->inferred_type is assigned to it); +// when `var (v1,v2,v3) = rhs`, left side is `(unknown,unknown,unknown)` +static bool is_type_unknown_from_var_lhs_decl(TypePtr t) { + if (t == TypeDataUnknown::create()) { + return true; + } + if (const auto* t_tensor = t->try_as()) { + bool all_unknown = true; + for (TypePtr item : t_tensor->items) { + all_unknown &=is_type_unknown_from_var_lhs_decl(item); + } + return all_unknown; + } + return false; +} + // merge (unify) of two sign states: what sign do we definitely have // it's used on data flow rejoin // example: `if (x > 0) ... else ...`; lca(Positive, NonPositive) = Unknown @@ -263,7 +279,7 @@ BoolState calculate_bool_lca(BoolState a, BoolState b) { void TypeInferringUnifyStrategy::unify_with(TypePtr next, TypePtr dest_hint) { // example: `var r = ... ? int8 : int16`, will be inferred as `int8 | int16` (via unification) // but `var r: int = ... ? int8 : int16`, will be inferred as `int` (it's dest_hint) - if (dest_hint && dest_hint != TypeDataUnknown::create() && !dest_hint->unwrap_alias()->try_as()) { + if (dest_hint && !is_type_unknown_from_var_lhs_decl(dest_hint) && !dest_hint->unwrap_alias()->try_as()) { if (dest_hint->can_rhs_be_assigned(next)) { next = dest_hint; } From 1774c05ac670ec90a712f9f84e2b0b71cacdb8b9 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Mon, 14 Apr 2025 22:03:13 +0300 Subject: [PATCH 334/388] [Tolk] Modern `onInternalMessage` and `onBouncedMessage` Instead of "FunC-style" parsing msg_cell, use `in.senderAddress`, `in.originalForwardFee`, that are mapped onto TVM-11 instructions --- crypto/smartcont/tolk-stdlib/common.tolk | 44 +++++- .../smartcont/tolk-stdlib/gas-payments.tolk | 11 +- tolk-tester/tests/cells-slices.tolk | 4 +- tolk-tester/tests/handle-msg-1.tolk | 134 ++++++++++++++++++ tolk-tester/tests/handle-msg-2.tolk | 23 +++ tolk-tester/tests/handle-msg-3.tolk | 32 +++++ tolk-tester/tests/handle-msg-4.tolk | 40 ++++++ tolk-tester/tests/handle-msg-5.tolk | 39 +++++ tolk-tester/tests/handle-msg-6.tolk | 47 ++++++ tolk-tester/tests/handle-msg-7.tolk | 22 +++ .../tests/invalid-declaration/err-1906.tolk | 9 ++ .../tests/invalid-declaration/err-1907.tolk | 8 ++ .../tests/invalid-declaration/err-1908.tolk | 12 ++ .../tests/invalid-semantics/err-4618.tolk | 11 ++ .../tests/invalid-semantics/err-4790.tolk | 11 ++ .../tests/invalid-semantics/err-4890.tolk | 11 ++ tolk-tester/tests/lazy-load-tests.tolk | 2 +- tolk-tester/tests/meaningful-1.tolk | 6 + tolk-tester/tolk-tester.py | 2 + tolk/CMakeLists.txt | 2 + tolk/ast-aux-data.h | 13 ++ tolk/ast-from-tokens.cpp | 18 ++- tolk/ast.cpp | 3 + tolk/ast.h | 2 + tolk/builtins.cpp | 24 ++++ tolk/gen-entrypoints.cpp | 120 ++++++++++++++++ tolk/gen-entrypoints.h | 28 ++++ tolk/pipe-ast-to-legacy.cpp | 11 ++ tolk/pipe-detect-inline-in-place.cpp | 4 +- tolk/pipe-infer-types-and-calls.cpp | 8 ++ tolk/pipe-register-symbols.cpp | 3 + tolk/pipe-transform-on-message.cpp | 119 ++++++++++++++++ tolk/pipeline.h | 1 + tolk/symtable.cpp | 9 ++ tolk/symtable.h | 5 +- tolk/tolk.cpp | 1 + 36 files changed, 825 insertions(+), 14 deletions(-) create mode 100644 tolk-tester/tests/handle-msg-1.tolk create mode 100644 tolk-tester/tests/handle-msg-2.tolk create mode 100644 tolk-tester/tests/handle-msg-3.tolk create mode 100644 tolk-tester/tests/handle-msg-4.tolk create mode 100644 tolk-tester/tests/handle-msg-5.tolk create mode 100644 tolk-tester/tests/handle-msg-6.tolk create mode 100644 tolk-tester/tests/handle-msg-7.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1906.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1907.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1908.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4618.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4790.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4890.tolk create mode 100644 tolk/gen-entrypoints.cpp create mode 100644 tolk/gen-entrypoints.h create mode 100644 tolk/pipe-transform-on-message.cpp diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 3eefbbc5f..9c57bf18f 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -843,7 +843,7 @@ fun slice.remainingBitsAndRefsCount(self): (int, int) /// Checks whether a slice is empty (i.e., contains no bits of data and no cell references). @pure -fun slice.isEnd(self): bool +fun slice.isEmpty(self): bool asm "SEMPTY"; /// Checks whether a slice has no bits of data. @@ -1311,6 +1311,45 @@ fun sendRawMessage(msg: cell, mode: int): void Receiving and handling messages. */ +/// InMessage is an input for `onInternalMessage` — when your contract accepts a non-bounced message. +/// Internally, some data exist on the stack, some can be acquired with TVM instructions. +/// The compiler replaces `in.someField` and gets a value correctly. +/// Example: +/// ``` +/// fun onInternalMessage(in: InMessage) { +/// in.senderAddress // actually calls `INMSG_SRC` asm instruction +/// in.body // actually gets from a TVM stack +/// ``` +struct InMessage { + senderAddress: address; // an internal address from which the message arrived + valueCoins: coins; // ton amount attached to an incoming message + valueExtra: dict; // extra currencies attached to an incoming message + originalForwardFee: coins; // fee that was paid by the sender + createdLt: uint64; // logical time when a message was created + createdAt: uint32; // unixtime when a message was created + body: slice; // message body, parse it with `lazy AllowedMsg.fromSlice(in.body)` +} + +/// InMessageBounced is an input for `onBouncedMessage`. +/// Very similar to a non-bounced input [InMessage]. +/// Note, that `bouncedBody` is currently 256 bits: 0xFFFFFFFF + 224 bits from the originally sent message. +/// Parse it with care! +/// Example: +/// ``` +/// fun onBouncedMessage(in: InMessageBounced) { +/// in.bouncedBody.skipBouncedPrefix(); +/// val originalOpcode = in.bouncedBody.loadUint(32); +/// ``` +struct InMessageBounced { + senderAddress: address; // an internal address from which the message was bounced + valueCoins: coins; // ton amount attached to a message + valueExtra: dict; // extra currencies attached to a message + originalForwardFee: coins; // comission that the sender has payed to send this message + createdLt: uint64; // logical time when a message was created (and bounced) + createdAt: uint32; // unixtime when a message was created (and bounced) + bouncedBody: slice; // currently 256 bits: 0xFFFFFFFF + 224 bits sent originally +} + /// Skip 0xFFFFFFFF prefix (when a message is bounced). @pure fun slice.skipBouncedPrefix(mutate self): self @@ -1318,7 +1357,7 @@ fun slice.skipBouncedPrefix(mutate self): self /// Load msgFlags from incoming message body (4 bits). @pure -@deprecated("use `onInternalMessage(in: InMessage)` and `in.isBounced` instead of parsing msgCell manually") +@deprecated("use `onBouncedMessage` handler instead of parsing msgCell flags manually") fun slice.loadMessageFlags(mutate self): int asm( -> 1 0) "4 LDU"; @@ -1349,3 +1388,4 @@ fun slice.skipMessageQueryId(mutate self): self fun builder.storeMessageQueryId(mutate self, queryId: int): self asm(queryId self) "64 STU"; + diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk index 21de803ca..5aafedbe1 100644 --- a/crypto/smartcont/tolk-stdlib/gas-payments.tolk +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -45,16 +45,17 @@ fun calculateGasFeeWithoutFlatPrice(workchain: int8, gasUsed: coins): coins fun calculateStorageFee(workchain: int8, seconds: int, bits: int, cells: int): coins asm(cells bits seconds workchain) "GETSTORAGEFEE"; -/// Calculates amount of nanotoncoins you should pay to send a message of specified size. -fun calculateMessageFee(workchain: int8, bits: int, cells: int): coins +/// Calculates amount of nanotoncoins you should pay to send a message of a specified size. +fun calculateForwardFee(workchain: int8, bits: int, cells: int): coins asm(cells bits workchain) "GETFORWARDFEE"; -/// Same as [calculateMessageFee], but without lump price (you have supposed to read https://docs.ton.org/develop/howto/fees-low-level) -fun calculateMessageFeeWithoutLumpPrice(workchain: int8, bits: int, cells: int): coins +/// Same as [calculateForwardFee], but without lump price (you have supposed to read https://docs.ton.org/develop/howto/fees-low-level) +fun calculateForwardFeeWithoutLumpPrice(workchain: int8, bits: int, cells: int): coins asm(cells bits workchain) "GETFORWARDFEESIMPLE"; /// Calculates fee that was paid by the sender of an incoming internal message. -fun calculateOriginalMessageFee(workchain: int8, incomingFwdFee: coins): coins +@deprecated("use modern `onInternalMessage` and access `in.originalForwardFee` directly") +fun calculateOriginalForwardFee(workchain: int8, incomingFwdFee: coins): coins asm(incomingFwdFee workchain) "GETORIGINALFWDFEE"; /// Returns the amount of nanotoncoins current contract debts for storage. ("due" and "debt" are synonyms) diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index fa96bd529..8fee091b8 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -112,11 +112,11 @@ fun slice.sumNumbersInSlice(mutate self): int { fun test10() { var ref = beginCell().storeInt(100, 32).endCell(); var s: slice = beginCell().storeInt(1, 32).storeInt(2, 32).storeRef(ref).endCell().beginParse(); - var result = (s.remainingBitsCount(), s.sumNumbersInSlice(), s.remainingBitsCount(), s.isEnd(), s.isEndOfBits(), s.isEndOfRefs()); + var result = (s.remainingBitsCount(), s.sumNumbersInSlice(), s.remainingBitsCount(), s.isEmpty(), s.isEndOfBits(), s.isEndOfRefs()); var ref2: cell = s.loadRef(); var s2: slice = ref2.beginParse(); s.assertEnd(); - return (result, s2.loadInt(32), s2.isEnd()); + return (result, s2.loadInt(32), s2.isEmpty()); } @method_id(111) diff --git a/tolk-tester/tests/handle-msg-1.tolk b/tolk-tester/tests/handle-msg-1.tolk new file mode 100644 index 000000000..1e08082d5 --- /dev/null +++ b/tolk-tester/tests/handle-msg-1.tolk @@ -0,0 +1,134 @@ + +fun setTvmRegisterC7(c7: [tuple]): void + asm "c7 POP"; + +@noinline +fun emulateC7PresentForTests(bounced: bool, senderAddress: address, fwdFee: coins, createdLt: int, createdAt: int, valueCoins: coins, valueExtra: dict) { + var c7_inner = createEmptyTuple(); + repeat (17) { + c7_inner.push(null); + } + + var inmsgparams = createEmptyTuple(); + inmsgparams.push(null); // bounce (not present in InMessage) + inmsgparams.push(bounced); + inmsgparams.push(senderAddress); + inmsgparams.push(fwdFee); + inmsgparams.push(createdLt); + inmsgparams.push(createdAt); + inmsgparams.push(null); // orig value (not present in InMessage) + inmsgparams.push(valueCoins); // in onInternalMessage, from a stack; in onBouncedMessage, from TVM + inmsgparams.push(valueExtra); + inmsgparams.push(null); // state init (not present in InMessage) + c7_inner.push(inmsgparams); + + setTvmRegisterC7([c7_inner]); +} + +fun invokeTest(body: slice, method_id: int): void + asm "c3 PUSH" "EXECUTE"; // stack: body + +@method_id(101) +fun handle1(in: InMessage) { + return in.senderAddress.getWorkchain(); +} + +@method_id(201) +fun entrypoint1() { + emulateC7PresentForTests(true, address("EQCtrtTXEAoSpoERmiqOnICe9LHxn2N89N4BH9qdHlrG-U0i"), 100, 20, 30, 900, null); + invokeTest("", 101); +} + +@method_id(102) +fun handle2(in: InMessage) { + return (in.valueCoins, in.senderAddress.isInternal()); +} + +@method_id(202) +fun entrypoint2() { + emulateC7PresentForTests(false, address("EQCtrtTXEAoSpoERmiqOnICe9LHxn2N89N4BH9qdHlrG-U0i"), 100, 20, 30, 888, null); + invokeTest("", 102); +} + +@method_id(103) +fun handle3(`in()`: InMessage) { + return ( + `in()`.senderAddress.getWorkchainAndHash(), + `in()`.valueCoins, + `in()`.valueExtra, + `in()`.createdLt, + `in()`.createdAt, + `in()`.body.isEmpty(), + ) +} + +@method_id(203) +fun entrypoint3() { + emulateC7PresentForTests(true, address("-1:000000000000000000000000000000000000000000000000000000000000FFFF"), 100, 20, 30, 999, null); + invokeTest("00", 103); +} + +fun onInternalMessage(in: InMessage) { + __expect_type(in.body, "slice"); + __expect_type(in.senderAddress, "address"); + __expect_type(in.valueCoins, "coins"); + return in.originalForwardFee; +} + +/** +@testcase | 201 | | 0 +@testcase | 202 | | 888 -1 +@testcase | 203 | | -1 65535 999 (null) 20 30 0 + +@fif_codegen +""" + handle1() PROC:<{ // in.body + DROP // + INMSG_SRC // '1 + REWRITESTDADDR + DROP // '3 + }> +""" + +@fif_codegen +""" + handle2() PROC:<{ // in.body + DROP // + INMSG_VALUE // '1 + INMSG_SRC // '1 '3 + b{10} SDBEGINSQ + NIP // '1 '5 + }> +""" + +@fif_codegen +""" + handle3() PROC:<{ + INMSG_SRC + REWRITESTDADDR + INMSG_VALUE + INMSG_VALUEEXTRA + INMSG_LT + INMSG_UTIME + s0 s6 XCHG + SEMPTY + s5 s6 XCHG + s4 s5 XCHG + s3 s4 XCHG + s1 s3 s0 XCHG3 + }> +""" + +@fif_codegen +""" + onInternalMessage() PROC:<{ // in.body + DROP + INMSG_BOUNCED + 0 THROWIF + INMSG_FWDFEE + 0 PUSHINT + GETORIGINALFWDFEE + }> +""" + + */ diff --git a/tolk-tester/tests/handle-msg-2.tolk b/tolk-tester/tests/handle-msg-2.tolk new file mode 100644 index 000000000..2f8a8b29f --- /dev/null +++ b/tolk-tester/tests/handle-msg-2.tolk @@ -0,0 +1,23 @@ + +@method_id(101) +fun test1() { + return 0; +} + +fun onInternalMessage(in: InMessage) { + return in.valueExtra; +} + +/** +@testcase | 101 | | 0 + +@fif_codegen +""" + onInternalMessage() PROC:<{ + DROP + INMSG_BOUNCED + 0 THROWIF + INMSG_VALUEEXTRA + }> +""" +*/ \ No newline at end of file diff --git a/tolk-tester/tests/handle-msg-3.tolk b/tolk-tester/tests/handle-msg-3.tolk new file mode 100644 index 000000000..78bb301dd --- /dev/null +++ b/tolk-tester/tests/handle-msg-3.tolk @@ -0,0 +1,32 @@ + +@method_id(101) +fun test1() { + return 0; +} + +fun onBouncedMessage(in: InMessageBounced) { + in.bouncedBody.skipBouncedPrefix(); + throw in.bouncedBody.loadUint(32); +} + +fun onInternalMessage(in: InMessage) { + return in.body; +} + +/** +@testcase | 101 | | 0 + +@fif_codegen +""" + onInternalMessage() PROC:<{ + INMSG_BOUNCED + IFJMP:<{ + 32 LDU + NIP + 32 LDU + DROP + THROWANY + }> // in.body + }> +""" +*/ \ No newline at end of file diff --git a/tolk-tester/tests/handle-msg-4.tolk b/tolk-tester/tests/handle-msg-4.tolk new file mode 100644 index 000000000..ac772992d --- /dev/null +++ b/tolk-tester/tests/handle-msg-4.tolk @@ -0,0 +1,40 @@ + +@method_id(101) +fun test1() { + return 0; +} + +@inline_ref +fun onBouncedMessage(in: InMessageBounced) { + contract.setData(createEmptyCell()); +} + +fun onInternalMessage(in: InMessage) { + return 123; +} + +/** +@testcase | 101 | | 0 + +@fif_codegen +""" + onBouncedMessage() PROCREF:<{ // in.body + DROP + PUSHREF + c4 POP + }> +""" + +@fif_codegen +""" + onInternalMessage() PROC:<{ // in.body + INMSG_BOUNCED // in.body '1 + IFJMP:<{ // in.body + onBouncedMessage() INLINECALLDICT + }> + DROP + 123 PUSHINT + }> +""" + + */ diff --git a/tolk-tester/tests/handle-msg-5.tolk b/tolk-tester/tests/handle-msg-5.tolk new file mode 100644 index 000000000..d5a30f8f9 --- /dev/null +++ b/tolk-tester/tests/handle-msg-5.tolk @@ -0,0 +1,39 @@ + +@method_id(101) +fun test1() { + return 0; +} + +@noinline +fun onBouncedMessage(in: InMessageBounced) { + throw in.valueCoins; +} + +fun onInternalMessage(in: InMessage) { + return in.valueCoins; +} + +/** +@testcase | 101 | | 0 + +@fif_codegen +""" + onBouncedMessage() PROC:<{ + DROP + INMSG_VALUE + THROWANY + }> +""" + +@fif_codegen +""" + onInternalMessage() PROC:<{ // in.body + INMSG_BOUNCED // in.body '1 + IFJMP:<{ // in.body + onBouncedMessage() CALLDICT // + }> // in.body + DROP // + INMSG_VALUE // '3 + }> +""" + */ diff --git a/tolk-tester/tests/handle-msg-6.tolk b/tolk-tester/tests/handle-msg-6.tolk new file mode 100644 index 000000000..0d349f9cc --- /dev/null +++ b/tolk-tester/tests/handle-msg-6.tolk @@ -0,0 +1,47 @@ + +@method_id(101) +fun test1() { + return 0; +} + +fun onBouncedMessage(in: InMessageBounced) { + if (in.valueExtra != null) { + return; + } + throw in.originalForwardFee; +} + +fun onInternalMessage(in: InMessage) { + return in.senderAddress; +} + +/** +@testcase | 101 | | 0 + +@fif_codegen +""" + onBouncedMessage() PROC:<{ + DROP + INMSG_VALUEEXTRA + ISNULL + IFNOTJMP:<{ + }> + INMSG_FWDFEE + 0 PUSHINT + GETORIGINALFWDFEE + THROWANY + }> +""" + +@fif_codegen +""" + onInternalMessage() PROC:<{ + INMSG_BOUNCED + IFJMP:<{ + onBouncedMessage() CALLDICT + }> + DROP + INMSG_SRC + }> +""" + */ diff --git a/tolk-tester/tests/handle-msg-7.tolk b/tolk-tester/tests/handle-msg-7.tolk new file mode 100644 index 000000000..8410d6f3d --- /dev/null +++ b/tolk-tester/tests/handle-msg-7.tolk @@ -0,0 +1,22 @@ + +@method_id(101) +fun test1() { + return 0; +} + +@on_bounced_policy("manual") +fun onInternalMessage(in: InMessage) { + return in.valueExtra; +} + +/** +@testcase | 101 | | 0 + +@fif_codegen +""" + onInternalMessage() PROC:<{ + DROP + INMSG_VALUEEXTRA + }> +""" +*/ \ No newline at end of file diff --git a/tolk-tester/tests/invalid-declaration/err-1906.tolk b/tolk-tester/tests/invalid-declaration/err-1906.tolk new file mode 100644 index 000000000..2b7e458cf --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1906.tolk @@ -0,0 +1,9 @@ + +fun onBouncedMessage(in: InMessageBounced) { + return in.valueExtra; +} + +/** +@compilation_should_fail +@stderr `onBouncedMessage` should return `void` + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1907.tolk b/tolk-tester/tests/invalid-declaration/err-1907.tolk new file mode 100644 index 000000000..945a8bf48 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1907.tolk @@ -0,0 +1,8 @@ + +fun onBouncedMessage() { +} + +/** +@compilation_should_fail +@stderr `onBouncedMessage` should have one parameter `InMessageBounced` + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1908.tolk b/tolk-tester/tests/invalid-declaration/err-1908.tolk new file mode 100644 index 000000000..390441813 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1908.tolk @@ -0,0 +1,12 @@ + +fun handle(in: InMessage) { +} + +fun onInternalMessage(in: InMessage) { + return handle(in); +} + +/** +@compilation_should_fail +@stderr using `in` as an object is prohibited, because `InMessage` is a built-in struct, its fields are mapped to TVM instructions + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4618.tolk b/tolk-tester/tests/invalid-semantics/err-4618.tolk new file mode 100644 index 000000000..6fb6419a8 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4618.tolk @@ -0,0 +1,11 @@ +fun onInternalMessage() { + onExternalMessage(""); +} + +fun onExternalMessage(body: slice) { +} + +/** +@compilation_should_fail +@stderr onExternalMessage is a special entrypoint, it can not be called as a regular function + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4790.tolk b/tolk-tester/tests/invalid-semantics/err-4790.tolk new file mode 100644 index 000000000..910d74114 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4790.tolk @@ -0,0 +1,11 @@ +fun onInternalMessage() { + var f = onExternalMessage; +} + +fun onExternalMessage(body: slice) { +} + +/** +@compilation_should_fail +@stderr can not get reference to this function, it's a special entrypoint + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4890.tolk b/tolk-tester/tests/invalid-semantics/err-4890.tolk new file mode 100644 index 000000000..d913652a0 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4890.tolk @@ -0,0 +1,11 @@ + +fun onInternalMessage(in: InMessage) { + in.valueCoins = 123; + return in.valueCoins; +} + +/** +@compilation_should_fail +@stderr modifying an immutable variable +@stderr hint: fields of InMessage can be used for reading only + */ diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk index b7eba14fa..63c40f152 100644 --- a/tolk-tester/tests/lazy-load-tests.tolk +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -701,7 +701,7 @@ fun demo139(s: slice, skip: bool): int { match (msg) { CounterIncrement => { try { - if (s.isEnd()) { return 100; } + if (s.isEmpty()) { return 100; } __expect_lazy("[msg] load byValue"); return msg.byValue; } catch (excno) { return excno } diff --git a/tolk-tester/tests/meaningful-1.tolk b/tolk-tester/tests/meaningful-1.tolk index 19356dc52..45278ad64 100644 --- a/tolk-tester/tests/meaningful-1.tolk +++ b/tolk-tester/tests/meaningful-1.tolk @@ -40,4 +40,10 @@ fun onInternalMessage(curCounter: int, mode: int) { @testcase | 0 | 50 0 | 0 @testcase | 0 | 50 1 | 51 @testcase | 0 | 50 9 | 59 + +@fif_codegen +""" + onInternalMessage() PROC:<{ // curCounter mode + fakeParseMessage() CALLDICT // curCounter m.USlot1 m.UTag +""" */ diff --git a/tolk-tester/tolk-tester.py b/tolk-tester/tolk-tester.py index f39d2d3ab..da81f87fe 100644 --- a/tolk-tester/tolk-tester.py +++ b/tolk-tester/tolk-tester.py @@ -116,6 +116,8 @@ def __init__(self, method_id_str: str, input_str: str, output_str: str): continue elif in_arg.startswith("x{") or TolkTestCaseInputOutput.reJustNumber.fullmatch(in_arg): processed_inputs.append(in_arg) + elif in_arg.startswith("cell{"): + processed_inputs.append("") elif TolkTestCaseInputOutput.reMathExpr.fullmatch(in_arg): processed_inputs.append(str(eval(in_arg))) elif in_arg == "null": diff --git a/tolk/CMakeLists.txt b/tolk/CMakeLists.txt index 0705876f3..bc8a6daa0 100644 --- a/tolk/CMakeLists.txt +++ b/tolk/CMakeLists.txt @@ -6,6 +6,7 @@ set(TOLK_SOURCE ast.cpp ast-from-tokens.cpp constant-evaluator.cpp + gen-entrypoints.cpp pack-unpack-api.cpp pack-unpack-serializers.cpp send-message-api.cpp @@ -24,6 +25,7 @@ set(TOLK_SOURCE pipe-optimize-boolean-expr.cpp pipe-detect-inline-in-place.cpp pipe-lazy-load-insertions.cpp + pipe-transform-on-message.cpp pipe-ast-to-legacy.cpp pipe-find-unused-symbols.cpp pipe-generate-fif-output.cpp diff --git a/tolk/ast-aux-data.h b/tolk/ast-aux-data.h index 37cbcfa84..36a806a09 100644 --- a/tolk/ast-aux-data.h +++ b/tolk/ast-aux-data.h @@ -18,6 +18,7 @@ #include "ast.h" #include "lazy-helpers.h" +#include "tolk.h" /* * This file contains a schema of aux_data inside ast_artificial_aux_vertex @@ -59,4 +60,16 @@ struct AuxData_LazyMatchForUnion final : ASTAuxData { } }; +struct AuxData_OnInternalMessage_getField final : ASTAuxData { + FunctionPtr f_onInternalMessage; + const std::string_view field_name; + + AuxData_OnInternalMessage_getField(FunctionPtr f_onInternalMessage, std::string_view field_name) + : f_onInternalMessage(f_onInternalMessage) + , field_name(field_name) { + } + + std::vector generate_get_InMessage_field(CodeBlob& code, SrcLocation loc) const; +}; + } // namespace tolk diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 4ad6ea954..98d4e546e 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -1376,7 +1376,8 @@ static V parse_annotation(Lexer& lex) { throw ParseError(loc, "expecting `(number)` after " + static_cast(name)); } break; - case AnnotationKind::overflow1023_policy: { + case AnnotationKind::overflow1023_policy: + case AnnotationKind::on_bounced_policy: { if (!v_arg || v_arg->size() != 1 || v_arg->get_item(0)->kind != ast_string_const) { throw ParseError(loc, "expecting `(\"policy_name\")` after " + static_cast(name)); } @@ -1410,7 +1411,8 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vectorintval; break; } + case AnnotationKind::on_bounced_policy: { + std::string_view str = v_annotation->get_arg()->get_item(0)->as()->str_val; + if (str == "manual") { + flags |= FunctionData::flagManualOnBounce; + } else { + v_annotation->error("incorrect value for " + static_cast(v_annotation->name)); + } + if (f_name != "onInternalMessage") { + v_annotation->error("this annotation is applicable only to onInternalMessage()"); + } + break; + } case AnnotationKind::deprecated: case AnnotationKind::custom: break; diff --git a/tolk/ast.cpp b/tolk/ast.cpp index 6e3b3b890..b22f9d31f 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -75,6 +75,9 @@ AnnotationKind Vertex::parse_kind(std::string_view name) { if (name == "@overflow1023_policy") { return AnnotationKind::overflow1023_policy; } + if (name == "@on_bounced_policy") { + return AnnotationKind::on_bounced_policy; + } return AnnotationKind::unknown; } diff --git a/tolk/ast.h b/tolk/ast.h index 1da1d6e6c..8836d2a06 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -146,6 +146,7 @@ enum class AnnotationKind { deprecated, custom, overflow1023_policy, + on_bounced_policy, unknown, }; @@ -522,6 +523,7 @@ struct Vertex final : ASTExprUnary { template<> // ast_artificial_aux_vertex is a compiler-inserted vertex that can't occur in source code // example: special vertex to force location in Fift output at constant usage +// example: `msg.isBounced` / `msg.xxx` in onInternalMessage are handled specially struct Vertex final : ASTExprUnary { const ASTAuxData* aux_data; // custom payload, see ast-aux-data.h diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index d15288564..59c07da95 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -975,6 +975,24 @@ static AsmOp compile_throw_if_unless(std::vector& res, std::vector& res, std::vector& args, SrcLocation loc) { + return exec_op(loc, "GETORIGINALFWDFEE", 2); +} + +static AsmOp compile_calc_InMessage_getInMsgParam(std::vector& res, std::vector& args, SrcLocation loc) { + // instead of "0 INMSGPARAM", generate "INMSG_BOUNCE", etc. — these are aliases in Asm.fif + static const char* aliases[] = { + "INMSG_BOUNCE", "INMSG_BOUNCED", "INMSG_SRC", "INMSG_FWDFEE", "INMSG_LT", "INMSG_UTIME", "INMSG_ORIGVALUE", "INMSG_VALUE", "INMSG_VALUEEXTRA", "INMSG_STATEINIT", + }; + tolk_assert(res.size() == 1 && args.size() == 1 && args[0].is_int_const()); + args[0].unused(); + size_t idx = args[0].int_const->to_long(); + if (idx < std::size(aliases)) { + return exec_op(loc, aliases[idx], 0); + } + return exec_arg_op(loc, "INMSGPARAM", args[0].int_const, 1); +} + static AsmOp compile_throw_arg(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.empty() && args.size() == 2); VarDescr &x = args[1]; @@ -1364,6 +1382,12 @@ void define_builtins() { define_builtin_func("__throw_if_unless", ParamsInt3, Unit, nullptr, compile_throw_if_unless, 0); + define_builtin_func("__InMessage.originalForwardFee", ParamsInt2, Int, nullptr, + compile_calc_InMessage_originalForwardFee, + 0); + define_builtin_func("__InMessage.getInMsgParam", ParamsInt1, Int, nullptr, + compile_calc_InMessage_getInMsgParam, + 0); // compile-time only functions, evaluated essentially at compile-time, no runtime implementation // they are placed in stdlib and marked as `builtin` diff --git a/tolk/gen-entrypoints.cpp b/tolk/gen-entrypoints.cpp new file mode 100644 index 000000000..196b3dd16 --- /dev/null +++ b/tolk/gen-entrypoints.cpp @@ -0,0 +1,120 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "gen-entrypoints.h" +#include "type-system.h" + +/* + * This module is responsible for `onInternalMessage` at IR generation. + * In FunC, `recv_internal()` entrypoint was declared in such a way: + * > () recv_internal(int my_balance, int msg_value, cell in_msg_full, slice in_msg_body) + * Whenever the user wanted to check whether the message is bounced, he had to parse the cell manually. + * + * In Tolk: + * > fun onInternalMessage(in: InMessage) + * And to use `in.senderAddress`, `in.body`, `in.originalForwardFee`, etc. in the function. + * Under the hood, `in.senderAddress` is transformed into `INMSG_SRC`, and so on. + * + * Also, if `onBouncedMessage` exists, it's embedded directly, like + * > if (INMSG_BOUNCED) { onBouncedMessage(in.body); return; } + */ + +namespace tolk { + +// implemented in ast-from-legacy.cpp +std::vector gen_inline_fun_call_in_place(CodeBlob& code, TypePtr ret_type, SrcLocation loc, FunctionPtr f_inlined, AnyExprV self_obj, bool is_before_immediate_return, const std::vector>& vars_per_arg); + + +// check for "modern" `fun onInternalMessage(in: InMessage)`, +// because a "FunC-style" (msgCell: cell, msgBody: slice) also works; +// after transformation `in.xxx` to TVM aux vertices, the last parameter is named `in.body` +static bool is_modern_onInternalMessage(FunctionPtr f_onInternalMessage) { + return f_onInternalMessage->get_num_params() == 1 && f_onInternalMessage->get_param(0).name == "in.body"; +} + +std::vector AuxData_OnInternalMessage_getField::generate_get_InMessage_field(CodeBlob& code, SrcLocation loc) const { + if (field_name == "body") { // take `in.body` from a stack + return f_onInternalMessage->find_param("in.body")->ir_idx; + } + if (field_name == "bouncedBody") { // we're in onBouncedMessage() + return f_onInternalMessage->find_param("in.body")->ir_idx; + } + + int idx = -1; + if (field_name == "isBounced") idx = 1; + else if (field_name == "senderAddress") idx = 2; + else if (field_name == "originalForwardFee") idx = 3; + else if (field_name == "createdLt") idx = 4; + else if (field_name == "createdAt") idx = 5; + else if (field_name == "valueCoins") idx = 7; + else if (field_name == "valueExtra") idx = 8; + tolk_assert(idx != -1); + + std::vector ir_msgparam = code.create_tmp_var(TypeDataInt::create(), loc, field_name.data()); + code.emplace_back(loc, Op::_Call, ir_msgparam, std::vector{code.create_int(loc, idx, "(param-idx)")}, lookup_function("__InMessage.getInMsgParam")); + + if (field_name == "originalForwardFee") { + code.emplace_back(loc, Op::_Call, ir_msgparam, std::vector{ir_msgparam[0], code.create_int(loc, 0, "(basechain)")}, lookup_function("__InMessage.originalForwardFee")); + } + + return ir_msgparam; +} + +void handle_onInternalMessage_codegen_start(FunctionPtr f_onInternalMessage, const std::vector& rvect_params, CodeBlob& code, SrcLocation loc) { + // ignore FunC-style `onInternalMessage(msgCell, msgBody)` + if (!is_modern_onInternalMessage(f_onInternalMessage)) { + return; + } + // ignore `@on_bounced_policy("manual")`, don't insert "if (isBounced) return" + if (f_onInternalMessage->is_manual_on_bounce()) { + return; + } + + const Symbol* sym = lookup_global_symbol("onBouncedMessage"); + FunctionPtr f_onBouncedMessage = sym ? sym->try_as() : nullptr; + + AuxData_OnInternalMessage_getField get_isBounced(f_onInternalMessage, "isBounced"); + std::vector ir_isBounced = get_isBounced.generate_get_InMessage_field(code, loc); + + if (f_onBouncedMessage) { + // generate: `if (isBounced) { onBouncedMessage(); return; } + tolk_assert(f_onBouncedMessage->inferred_return_type->get_width_on_stack() == 0); + Op& if_isBounced = code.emplace_back(loc, Op::_If, ir_isBounced); + { + code.push_set_cur(if_isBounced.block0); + std::vector ir_bodySlice(rvect_params.end() - 1, rvect_params.end()); + if (f_onBouncedMessage->is_inlined_in_place()) { + gen_inline_fun_call_in_place(code, TypeDataVoid::create(), loc, f_onBouncedMessage, nullptr, true, {ir_bodySlice}); + } else { + Op& op_call = code.emplace_back(loc, Op::_Call, std::vector{}, ir_bodySlice, f_onBouncedMessage); + op_call.set_impure_flag(); + } + code.emplace_back(loc, Op::_Return, std::vector{}); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_isBounced.block1); + code.close_pop_cur(loc); + } + } else { + // generate: `assert (!isBounced) throw 0` + std::vector args_throw0if = { code.create_int(loc, 0, "(exit-0)"), ir_isBounced[0], code.create_int(loc, 1, "") }; + Op& op_assert = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_throw0if), lookup_function("__throw_if_unless")); + op_assert.set_impure_flag(); + } +} + +} // namespace tolk diff --git a/tolk/gen-entrypoints.h b/tolk/gen-entrypoints.h new file mode 100644 index 000000000..90f62b9bc --- /dev/null +++ b/tolk/gen-entrypoints.h @@ -0,0 +1,28 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "ast-aux-data.h" +#include "fwd-declarations.h" +#include "tolk.h" + +namespace tolk { + +void handle_onInternalMessage_codegen_start(FunctionPtr f_onInternalMessage, const std::vector& rvect_params, CodeBlob& code, SrcLocation loc); +std::vector generate_get_requested_field_parsing_on_demand(const AuxData_OnInternalMessage_getField* aux_data, CodeBlob& code, SrcLocation loc); + +} // namespace tolk diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index a6d84a30c..aa2e2dd3a 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -23,8 +23,10 @@ #include "common/refint.h" #include "smart-casts-cfg.h" #include "pack-unpack-api.h" +#include "gen-entrypoints.h" #include "generics-helpers.h" #include "send-message-api.h" +#include "gen-entrypoints.h" /* * This pipe is the last one operating AST: it transforms AST to IR. @@ -1965,6 +1967,11 @@ static std::vector process_artificial_aux_vertex(V(v->aux_data)) { + std::vector rvect = data->generate_get_InMessage_field(code, v->loc); + return transition_to_target_type(std::move(rvect), code, target_type, v); + } + tolk_assert(false); } @@ -2304,6 +2311,10 @@ static void convert_function_body_to_CodeBlob(FunctionPtr fun_ref, FunctionBodyC blob->in_var_cnt = blob->var_cnt; tolk_assert(blob->var_cnt == total_arg_width); + if (fun_ref->name == "onInternalMessage") { + handle_onInternalMessage_codegen_start(fun_ref, rvect_import, *blob, fun_ref->loc); + } + process_block_statement(v_body, *blob); append_implicit_return_statement(v_body->loc_end, *blob); diff --git a/tolk/pipe-detect-inline-in-place.cpp b/tolk/pipe-detect-inline-in-place.cpp index 21001c2aa..4221347f9 100644 --- a/tolk/pipe-detect-inline-in-place.cpp +++ b/tolk/pipe-detect-inline-in-place.cpp @@ -227,7 +227,9 @@ class DetectIfToInlineFunctionInPlaceVisitor final : ASTVisitorFunctionBody { } // okay, this function will be inlined, mark the flag - if (will_inline && fun_ref->n_times_called) { + bool is_called = fun_ref->n_times_called + || fun_ref->name == "onBouncedMessage"; // implicitly called by the compiler from onInternalMessage() + if (will_inline && is_called) { fun_ref->mutate()->assign_inline_mode_in_place(); } } diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 2d396762c..432802279 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -953,6 +953,9 @@ class InferTypesAndCallsAndFieldsVisitor final { if (fun_ref->is_compile_time_const_val() || fun_ref->is_compile_time_special_gen()) { fire(cur_f, v->loc, "can not get reference to this function, it's compile-time only"); } + if (fun_ref->is_entrypoint()) { + fire(cur_f, v->loc, "can not get reference to this function, it's a special entrypoint"); + } fun_ref->mutate()->assign_is_used_as_noncall(); get_or_infer_return_type(fun_ref); assign_inferred_type(v, fun_ref->inferred_full_type); @@ -1159,6 +1162,11 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(flow), used_as_condition); } + // prevent calling `onBouncedMessage()` and other special functions directly + if (fun_ref->is_entrypoint()) { + fire(cur_f, v->loc, fun_ref->name + " is a special entrypoint, it can not be called as a regular function"); + } + // so, we have a call `f(args)` or `obj.f(args)`, f is fun_ref (function / method) (code / asm / builtin) // we're going to iterate over passed arguments, and (if generic) infer substitutedTs // at first, check argument count diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index b4394a2d8..917722946 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -52,6 +52,9 @@ static int calculate_tvm_method_id_for_entrypoint(std::string_view func_name) { if (func_name == "onSplitInstall") { return -4; } + if (func_name == "onBouncedMessage") { + return FunctionData::EMPTY_TVM_METHOD_ID; + } tolk_assert(false); } diff --git a/tolk/pipe-transform-on-message.cpp b/tolk/pipe-transform-on-message.cpp new file mode 100644 index 000000000..68aff5060 --- /dev/null +++ b/tolk/pipe-transform-on-message.cpp @@ -0,0 +1,119 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-aux-data.h" +#include "ast-replacer.h" +#include "type-system.h" + +/* + * This pipe analyzes the body of + * > fun onInternalMessage(in: InMessage) + * and replaces `in.senderAddress` / etc. with aux AST vertices (handled specially at IR generation). + * + * This function is transformed to + * > fun onInternalMessage(in.body) + * so, + * - accessing `in.body` actually will take an element from a stack + * - accessing `in.senderAddress` will emit `INMSG_SRC` TVM instruction. + */ + +namespace tolk { + +// handle all functions having a prototype `fun f(var: InMessage)` (for testing purposes) +static bool is_onInternalMessage(FunctionPtr fun_ref) { + if (fun_ref->is_entrypoint() || fun_ref->has_tvm_method_id()) { + if (fun_ref->get_num_params() == 1) { + const auto* t_param = fun_ref->get_param(0).declared_type->try_as(); + return t_param && t_param->struct_ref->name == "InMessage"; + } + } + return false; +} + +// `onBouncedMessage` is only one, it's automatically embedded into `onInternalMessage` if exists +static bool is_onBouncedMessage(FunctionPtr fun_ref) { + return fun_ref->is_entrypoint() && fun_ref->name == "onBouncedMessage"; +} + + +struct TransformOnInternalMessageReplacer final : ASTReplacerInFunctionBody { + FunctionPtr cur_f = nullptr; + LocalVarPtr param_ref = nullptr; // `in` for `fun onInternalMessage(in: InMessage)` + + static void validate_onBouncedMessage(FunctionPtr f) { + if (f->inferred_return_type != TypeDataVoid::create() && f->inferred_return_type != TypeDataNever::create()) { + fire(f, f->loc, "`onBouncedMessage` should return `void`"); + } + if (f->get_num_params() != 1) { + fire(f, f->loc, "`onBouncedMessage` should have one parameter `InMessageBounced`"); + } + const auto* t_struct = f->get_param(0).declared_type->try_as(); + if (!t_struct || t_struct->struct_ref->name != "InMessageBounced") { + fire(f, f->loc, "`onBouncedMessage` should have one parameter `InMessageBounced`"); + } + } + +protected: + AnyExprV replace(V v) override { + // don't allow `var v = in` or passing `in` to another function (only `in.someField` is allowed) + if (v->sym == param_ref) { + fire(cur_f, v->loc, "using `" + param_ref->name + "` as an object is prohibited, because `InMessage` is a built-in struct, its fields are mapped to TVM instructions\nhint: use `" + param_ref->name + ".senderAddress` and other fields directly"); + } + return parent::replace(v); + } + + AnyExprV replace(V v) override { + // replace `in.senderAddress` / `in.valueCoins` with an aux vertex + if (v->get_obj()->kind == ast_reference && v->get_obj()->as()->sym == param_ref) { + if (v->is_lvalue && v->get_field_name() != "body" && v->get_field_name() != "bouncedBody") { + fire(cur_f, v->loc, "modifying an immutable variable\nhint: fields of InMessage can be used for reading only"); + } + + ASTAuxData* aux_getField = new AuxData_OnInternalMessage_getField(cur_f, v->get_field_name()); + return createV(v->loc, v, aux_getField, v->inferred_type); + } + + return parent::replace(v); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return is_onInternalMessage(fun_ref) || is_onBouncedMessage(fun_ref); + } + + void start_replacing_in_function(FunctionPtr fun_ref, V v_function) override { + if (fun_ref->name == "onBouncedMessage") { + validate_onBouncedMessage(fun_ref); + } + + cur_f = fun_ref; + param_ref = &fun_ref->parameters[0]; + + parent::replace(v_function->get_body()); + + std::vector new_parameters; + new_parameters.emplace_back("in.body", fun_ref->loc, TypeDataSlice::create(), nullptr, 0, 0); + fun_ref->mutate()->parameters = std::move(new_parameters); + } +}; + +void pipeline_transform_onInternalMessage() { + replace_ast_of_all_functions(); +} + +} // namespace tolk diff --git a/tolk/pipeline.h b/tolk/pipeline.h index 001228bee..eb1c153b4 100644 --- a/tolk/pipeline.h +++ b/tolk/pipeline.h @@ -46,6 +46,7 @@ void pipeline_constant_folding(); void pipeline_optimize_boolean_expressions(); void pipeline_detect_inline_in_place(); void pipeline_lazy_load_insertions(); +void pipeline_transform_onInternalMessage(); void pipeline_convert_ast_to_legacy_Expr_Op(); void pipeline_find_unused_symbols(); diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index b9b4b1fa8..03d0cda13 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -42,6 +42,15 @@ std::string StructData::as_human_readable() const { return name + genericTs->as_human_readable(); } +LocalVarPtr FunctionData::find_param(std::string_view name) const { + for (const LocalVarData& param_data : parameters) { + if (param_data.name == name) { + return ¶m_data; + } + } + return nullptr; +} + bool FunctionData::does_need_codegen() const { // when a function is declared, but not referenced from code in any way, don't generate its body if (!is_really_used() && G.settings.remove_unused_functions) { diff --git a/tolk/symtable.h b/tolk/symtable.h index 243c38a60..50ab271be 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -112,7 +112,7 @@ struct FunctionData final : Symbol { flagMarkedAsPure = 16, // declared as `pure`, can't call impure and access globals, unused invocations are optimized out flagImplicitReturn = 32, // control flow reaches end of function, so it needs implicit return at the end flagContractGetter = 64, // was declared via `get func(): T`, tvm_method_id is auto-assigned - flagIsEntrypoint = 128, // it's `main` / `onExternalMessage` / etc. + flagIsEntrypoint = 128, // it's `main` / `onExternalMessage` / etc. flagHasMutateParams = 256, // has parameters declared as `mutate` flagAcceptsSelf = 512, // is a member function (has `self` first parameter) flagReturnsSelf = 1024, // return type is `self` (returns the mutated 1st argument), calls can be chainable @@ -120,6 +120,7 @@ struct FunctionData final : Symbol { flagCompileTimeVal = 4096, // calculated only at compile-time for constant arguments: `ton("0.05")`, `stringCrc32`, and others flagCompileTimeGen = 8192, // at compile-time it's handled specially, not as a regular function: `T.toCell`, etc. flagAllowAnyWidthT = 16384, // for built-in generic functions that is not restricted to be 1-slot type + flagManualOnBounce = 32768, // for onInternalMessage, don't insert "if (isBounced) return" }; int tvm_method_id = EMPTY_TVM_METHOD_ID; @@ -184,6 +185,7 @@ struct FunctionData final : Symbol { int get_num_params() const { return static_cast(parameters.size()); } const LocalVarData& get_param(int idx) const { return parameters[idx]; } + LocalVarPtr find_param(std::string_view name) const; bool is_code_function() const { return std::holds_alternative(body); } bool is_asm_function() const { return std::holds_alternative(body); } @@ -210,6 +212,7 @@ struct FunctionData final : Symbol { bool is_compile_time_const_val() const { return flags & flagCompileTimeVal; } bool is_compile_time_special_gen() const { return flags & flagCompileTimeGen; } bool is_variadic_width_T_allowed() const { return flags & flagAllowAnyWidthT; } + bool is_manual_on_bounce() const { return flags & flagManualOnBounce; } bool does_need_codegen() const; diff --git a/tolk/tolk.cpp b/tolk/tolk.cpp index cbad23fe3..63a2dd7f2 100644 --- a/tolk/tolk.cpp +++ b/tolk/tolk.cpp @@ -69,6 +69,7 @@ int tolk_proceed(const std::string &entrypoint_filename) { pipeline_optimize_boolean_expressions(); pipeline_detect_inline_in_place(); pipeline_lazy_load_insertions(); + pipeline_transform_onInternalMessage(); pipeline_convert_ast_to_legacy_Expr_Op(); pipeline_find_unused_symbols(); From 72e0ae29a2fa2e947738c8e4910217c376150581 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 26 Jun 2025 19:45:55 +0300 Subject: [PATCH 335/388] [Tolk] Correctly handle `import` without ".tolk" extension Also, output abs paths in error messages. --- tolk-tester/tests/imports-tests.tolk | 18 ++++++ tolk-tester/tests/imports/has-tinf-err.tolk | 4 ++ tolk-tester/tests/imports/some-helpers.tolk | 1 + tolk-tester/tests/imports/some-storage.tolk | 1 + .../{use-dicts.tolk => use-dicts.ext.tolk} | 0 .../tests/invalid-symbol/err-2205.tolk | 2 +- .../tests/invalid-symbol/err-2539.tolk | 6 +- .../tests/invalid-symbol/err-2540.tolk | 11 ++++ .../tests/invalid-typing/err-6722.tolk | 7 +++ tolk-tester/tests/logical-operators.tolk | 2 +- tolk/ast-stringifier.h | 2 +- tolk/compiler-state.h | 2 +- tolk/pipe-discover-parse-sources.cpp | 5 +- tolk/pipe-generate-fif-output.cpp | 4 +- tolk/pipe-resolve-identifiers.cpp | 20 +------ tolk/pipe-resolve-types.cpp | 3 + tolk/src-file.cpp | 57 ++++++++++++------- tolk/src-file.h | 19 ++++--- tolk/symtable.cpp | 13 +++++ tolk/symtable.h | 2 + tolk/tolk-main.cpp | 20 +++---- tolk/tolk-wasm.cpp | 2 +- 22 files changed, 131 insertions(+), 70 deletions(-) create mode 100644 tolk-tester/tests/imports-tests.tolk create mode 100644 tolk-tester/tests/imports/has-tinf-err.tolk create mode 100644 tolk-tester/tests/imports/some-helpers.tolk create mode 100644 tolk-tester/tests/imports/some-storage.tolk rename tolk-tester/tests/imports/{use-dicts.tolk => use-dicts.ext.tolk} (100%) create mode 100644 tolk-tester/tests/invalid-symbol/err-2540.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6722.tolk diff --git a/tolk-tester/tests/imports-tests.tolk b/tolk-tester/tests/imports-tests.tolk new file mode 100644 index 000000000..c4dc93478 --- /dev/null +++ b/tolk-tester/tests/imports-tests.tolk @@ -0,0 +1,18 @@ +import "imports/some-math" +import "./imports/use-dicts.ext" + +fun main() { + prepareDict_3_30_4_40_5_x(4); + return someAdd(1, 2); +} + +/** +@testcase | 0 | | 3 + +@fif_codegen +""" + main() PROC:<{ + 3 PUSHINT + }> +""" + */ diff --git a/tolk-tester/tests/imports/has-tinf-err.tolk b/tolk-tester/tests/imports/has-tinf-err.tolk new file mode 100644 index 000000000..bc9e5c8b1 --- /dev/null +++ b/tolk-tester/tests/imports/has-tinf-err.tolk @@ -0,0 +1,4 @@ + +fun asdf() { + var i: int = ""; +} diff --git a/tolk-tester/tests/imports/some-helpers.tolk b/tolk-tester/tests/imports/some-helpers.tolk new file mode 100644 index 000000000..a5caa2515 --- /dev/null +++ b/tolk-tester/tests/imports/some-helpers.tolk @@ -0,0 +1 @@ +import "./some-storage" diff --git a/tolk-tester/tests/imports/some-storage.tolk b/tolk-tester/tests/imports/some-storage.tolk new file mode 100644 index 000000000..93a692987 --- /dev/null +++ b/tolk-tester/tests/imports/some-storage.tolk @@ -0,0 +1 @@ +struct SomeStorage {} diff --git a/tolk-tester/tests/imports/use-dicts.tolk b/tolk-tester/tests/imports/use-dicts.ext.tolk similarity index 100% rename from tolk-tester/tests/imports/use-dicts.tolk rename to tolk-tester/tests/imports/use-dicts.ext.tolk diff --git a/tolk-tester/tests/invalid-symbol/err-2205.tolk b/tolk-tester/tests/invalid-symbol/err-2205.tolk index 3c9d72204..c3d47a672 100644 --- a/tolk-tester/tests/invalid-symbol/err-2205.tolk +++ b/tolk-tester/tests/invalid-symbol/err-2205.tolk @@ -3,6 +3,6 @@ import "../imports/invalid-no-import.tolk"; /** @compilation_should_fail -@stderr ../imports/invalid-no-import.tolk:2:13 +@stderr tests/imports/invalid-no-import.tolk:2:13 @stderr Using a non-imported symbol `someAdd` */ diff --git a/tolk-tester/tests/invalid-symbol/err-2539.tolk b/tolk-tester/tests/invalid-symbol/err-2539.tolk index f7c66c258..65c02115b 100644 --- a/tolk-tester/tests/invalid-symbol/err-2539.tolk +++ b/tolk-tester/tests/invalid-symbol/err-2539.tolk @@ -1,9 +1,9 @@ import "@stdlib/tvm-dicts" -import "../imports/use-dicts-err.tolk" +import "../imports/use-dicts-err" /** @compilation_should_fail -@stderr ../imports/use-dicts-err.tolk:2:22 +@stderr tests/imports/use-dicts-err.tolk:2:22 @stderr Using a non-imported symbol `createEmptyDict` -@stderr Forgot to import "@stdlib/tvm-dicts"? +@stderr hint: forgot to import "@stdlib/tvm-dicts"? */ diff --git a/tolk-tester/tests/invalid-symbol/err-2540.tolk b/tolk-tester/tests/invalid-symbol/err-2540.tolk new file mode 100644 index 000000000..0b14df1b8 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2540.tolk @@ -0,0 +1,11 @@ +import "../imports/some-helpers" + +fun usingNonImported() { + var s: SomeStorage = {}; +} + +/** +@compilation_should_fail +@stderr Using a non-imported symbol `SomeStorage` +@stderr forgot to import "some-storage.tolk"? + */ diff --git a/tolk-tester/tests/invalid-typing/err-6722.tolk b/tolk-tester/tests/invalid-typing/err-6722.tolk new file mode 100644 index 000000000..854e21167 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6722.tolk @@ -0,0 +1,7 @@ +import "../imports/./has-tinf-err" + +/** +@compilation_should_fail +@stderr imports/has-tinf-err.tolk:3:18 +@stderr can not assign `slice` to variable of type `int` + */ diff --git a/tolk-tester/tests/logical-operators.tolk b/tolk-tester/tests/logical-operators.tolk index 8242d5ff4..c184da72c 100644 --- a/tolk-tester/tests/logical-operators.tolk +++ b/tolk-tester/tests/logical-operators.tolk @@ -1,4 +1,4 @@ -import "imports/use-dicts.tolk" +import "imports/use-dicts.ext.tolk" fun simpleAllConst() { return (!0, !!0 & !false, !!!0, !1, !!1, !-1, !!-1, (!5 as int == 0) == !0, !0 == true); diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index b9ee06979..fd19401e9 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -247,7 +247,7 @@ class ASTStringifier final : public ASTVisitor { case ast_import_directive: return static_cast(v->as()->get_file_leaf()->str_val); case ast_tolk_file: - return v->as()->file->rel_filename; + return v->as()->file->realpath; default: return {}; } diff --git a/tolk/compiler-state.h b/tolk/compiler-state.h index 2193503a2..6685e092a 100644 --- a/tolk/compiler-state.h +++ b/tolk/compiler-state.h @@ -56,7 +56,7 @@ struct CompilerSettings { std::string output_filename; std::string boc_output_filename; - std::string stdlib_folder; // a path to tolk-stdlib/; files imported via @stdlib/xxx are there + std::string stdlib_folder; // path to tolk-stdlib/; note: from tolk-js it's empty! tolk-js reads files via js callback FsReadCallback read_callback; diff --git a/tolk/pipe-discover-parse-sources.cpp b/tolk/pipe-discover-parse-sources.cpp index 2f7c02d6d..1cadf07f4 100644 --- a/tolk/pipe-discover-parse-sources.cpp +++ b/tolk/pipe-discover-parse-sources.cpp @@ -52,10 +52,9 @@ void pipeline_discover_and_parse_sources(const std::string& stdlib_filename, con for (AnyV v_toplevel : file->ast->as()->get_toplevel_declarations()) { if (auto v_import = v_toplevel->try_as()) { std::string imported_str = v_import->get_file_name(); - size_t cur_slash_pos = file->rel_filename.rfind('/'); - std::string rel_filename = cur_slash_pos == std::string::npos || imported_str[0] == '@' + std::string rel_filename = imported_str[0] == '@' ? std::move(imported_str) - : file->rel_filename.substr(0, cur_slash_pos + 1) + imported_str; + : file->extract_dirname() + imported_str; const SrcFile* imported = G.all_src_files.locate_and_register_source_file(rel_filename, v_import->loc); file->imports.push_back(SrcFile::ImportDirective{imported}); diff --git a/tolk/pipe-generate-fif-output.cpp b/tolk/pipe-generate-fif-output.cpp index 237649f45..1dc513f05 100644 --- a/tolk/pipe-generate-fif-output.cpp +++ b/tolk/pipe-generate-fif-output.cpp @@ -125,11 +125,11 @@ void pipeline_generate_fif_output_to_std_cout() { std::cout << "// automatically generated from "; bool need_comma = false; for (const SrcFile* file : G.all_src_files) { - if (!file->is_stdlib_file()) { + if (!file->is_stdlib_file) { if (need_comma) { std::cout << ", "; } - std::cout << file->rel_filename; + std::cout << file->extract_short_name(); need_comma = true; } } diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index fad233c7f..46710cb63 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -64,22 +64,6 @@ static void fire_error_type_used_as_symbol(FunctionPtr cur_f, V } } -static void check_import_exists_when_using_sym(FunctionPtr cur_f, AnyV v_usage, const Symbol* used_sym) { - SrcLocation sym_loc = used_sym->loc; - if (!v_usage->loc.is_symbol_from_same_or_builtin_file(sym_loc)) { - const SrcFile* declared_in = sym_loc.get_src_file(); - bool has_import = false; - for (const SrcFile::ImportDirective& import : v_usage->loc.get_src_file()->imports) { - if (import.imported_file == declared_in) { - has_import = true; - } - } - if (!has_import) { - throw ParseError(cur_f, v_usage->loc, "Using a non-imported symbol `" + used_sym->name + "`. Forgot to import \"" + declared_in->rel_filename + "\"?"); - } - } -} - struct NameAndScopeResolver { std::vector> scopes; @@ -180,7 +164,9 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { // for global functions, global vars and constants, `import` must exist if (!sym->try_as()) { - check_import_exists_when_using_sym(cur_f, v, sym); + if (!v->loc.is_symbol_from_same_or_builtin_file(sym->loc)) { + sym->check_import_exists_when_used_from(cur_f, v->loc); + } } } diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 194cc4998..2a9bb2028 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -168,6 +168,9 @@ class TypeNodesVisitorResolver { } if (const Symbol* sym = lookup_global_symbol(text)) { if (TypePtr custom_type = try_resolve_user_defined_type(cur_f, loc, sym, allow_without_type_arguments)) { + if (!v->loc.is_symbol_from_same_or_builtin_file(sym->loc)) { + sym->check_import_exists_when_used_from(cur_f, v->loc); + } return custom_type; } } diff --git a/tolk/src-file.cpp b/tolk/src-file.cpp index ab54ce7b0..07db0eb88 100644 --- a/tolk/src-file.cpp +++ b/tolk/src-file.cpp @@ -24,41 +24,44 @@ namespace tolk { static_assert(sizeof(SrcLocation) == 8); -const SrcFile* AllRegisteredSrcFiles::find_file(const std::string& abs_filename) const { +const SrcFile* AllRegisteredSrcFiles::find_file(const std::string& realpath) const { + // files with the same realpath are considered equal for (const SrcFile* file : all_src_files) { - if (file->abs_filename == abs_filename) { + if (file->realpath == realpath) { return file; } } return nullptr; } -const SrcFile* AllRegisteredSrcFiles::locate_and_register_source_file(const std::string& rel_filename, SrcLocation included_from) { - td::Result path = G.settings.read_callback(CompilerSettings::FsReadCallbackKind::Realpath, rel_filename.c_str()); +const SrcFile* AllRegisteredSrcFiles::locate_and_register_source_file(const std::string& filename, SrcLocation included_from) { + bool is_stdlib = filename.size() > 8 && filename.starts_with("@stdlib/"); + + td::Result path = G.settings.read_callback(CompilerSettings::FsReadCallbackKind::Realpath, filename.c_str()); if (path.is_error()) { if (included_from.is_defined()) { throw ParseError(included_from, "Failed to import: " + path.move_as_error().message().str()); } - throw Fatal("Failed to locate " + rel_filename + ": " + path.move_as_error().message().str()); + throw Fatal("Failed to locate " + filename + ": " + path.move_as_error().message().str()); } - std::string abs_filename = path.move_as_ok(); - if (const SrcFile* file = find_file(abs_filename)) { + std::string realpath = path.move_as_ok(); + if (const SrcFile* file = find_file(realpath)) { return file; } - td::Result text = G.settings.read_callback(CompilerSettings::FsReadCallbackKind::ReadFile, abs_filename.c_str()); + td::Result text = G.settings.read_callback(CompilerSettings::FsReadCallbackKind::ReadFile, realpath.c_str()); if (text.is_error()) { if (included_from.is_defined()) { throw ParseError(included_from, "Failed to import: " + text.move_as_error().message().str()); } - throw Fatal("Failed to read " + rel_filename + ": " + text.move_as_error().message().str()); + throw Fatal("Failed to read " + realpath + ": " + text.move_as_error().message().str()); } int file_id = static_cast(all_src_files.size()); // SrcFile::file_id is the index in all files - SrcFile* created = new SrcFile(file_id, rel_filename, std::move(abs_filename), text.move_as_ok()); + SrcFile* created = new SrcFile(file_id, is_stdlib, std::move(realpath), text.move_as_ok()); if (G.is_verbosity(1)) { - std::cerr << "register file_id " << created->file_id << " " << created->abs_filename << std::endl; + std::cerr << "register file_id " << created->file_id << " " << created->realpath << std::endl; } all_src_files.push_back(created); return created; @@ -72,11 +75,6 @@ SrcFile* AllRegisteredSrcFiles::get_next_unparsed_file() { return const_cast(all_src_files[++last_parsed_file_id]); } -bool SrcFile::is_stdlib_file() const { - std::string_view rel(rel_filename); - return rel.size() > 10 && rel.substr(0, 8) == "@stdlib/"; // common.tolk, tvm-dicts.tolk, etc -} - bool SrcFile::is_offset_valid(int offset) const { return offset >= 0 && offset < static_cast(text.size()); } @@ -116,9 +114,30 @@ SrcFile::SrcPosition SrcFile::convert_offset(int offset) const { return SrcPosition{offset, line_idx + 1, char_idx + 1, line_str}; } +std::string SrcFile::extract_short_name() const { + size_t last_slash = realpath.find_last_of("/\\"); + if (last_slash == std::string::npos) { + return realpath; + } + std::string short_name = realpath.substr(last_slash + 1); // "file.tolk" (no path) + + if (is_stdlib_file) { // not "common.tolk", but "@stdlib/common" + return "@stdlib/" + short_name.substr(0, short_name.size() - 5); + } + return short_name; +} + +std::string SrcFile::extract_dirname() const { + size_t last_slash = realpath.find_last_of("/\\"); + if (last_slash == std::string::npos) { + return ""; + } + return realpath.substr(0, last_slash + 1); +} + std::ostream& operator<<(std::ostream& os, const SrcFile* src_file) { - return os << (src_file ? src_file->rel_filename : "unknown-location"); + return os << (src_file ? src_file->realpath : "unknown-location"); } std::ostream& operator<<(std::ostream& os, const Fatal& fatal) { @@ -233,7 +252,7 @@ void ParseError::show(std::ostream& os) const { // print "location: line1 \n (spaces) line2 \n ..." std::string_view message = this->message; std::string loc_text = loc.to_string(); - std::string loc_spaces(std::min(static_cast(loc_text.size()), 30), ' '); + std::string loc_spaces(std::min(static_cast(loc_text.size()), 9), ' '); size_t start = 0, end; os << loc_text << ": error: "; while ((end = message.find('\n', start)) != std::string::npos) { @@ -248,7 +267,7 @@ void ParseError::show(std::ostream& os) const { } } if (current_function) { - os << " // in function `" << current_function->as_human_readable() << "`" << std::endl; + os << std::endl << " // in function `" << current_function->as_human_readable() << "`" << std::endl; } loc.show_context(os); } diff --git a/tolk/src-file.h b/tolk/src-file.h index 1f3eb030e..9b39a0acc 100644 --- a/tolk/src-file.h +++ b/tolk/src-file.h @@ -35,25 +35,26 @@ struct SrcFile { }; int file_id; // an incremental counter through all parsed files - std::string rel_filename; // relative to cwd - std::string abs_filename; // absolute from root + bool is_stdlib_file; // is a part of Tolk distribution, imported via "@stdlib/..." + std::string realpath; // what "realpath" returned to locate (either abs path or what tolk-js returns) std::string text; // file contents loaded into memory, every Token::str_val points inside it AnyV ast = nullptr; // when a file has been parsed, its ast_tolk_file is kept here std::vector imports; // to check strictness (can't use a symbol without importing its file) - SrcFile(int file_id, std::string rel_filename, std::string abs_filename, std::string&& text) + SrcFile(int file_id, bool is_stdlib_file, std::string realpath, std::string&& text) : file_id(file_id) - , rel_filename(std::move(rel_filename)) - , abs_filename(std::move(abs_filename)) + , is_stdlib_file(is_stdlib_file) + , realpath(std::move(realpath)) , text(std::move(text)) { } SrcFile(const SrcFile& other) = delete; SrcFile &operator=(const SrcFile&) = delete; - bool is_stdlib_file() const; - bool is_offset_valid(int offset) const; SrcPosition convert_offset(int offset) const; + + std::string extract_short_name() const; + std::string extract_dirname() const; }; @@ -102,9 +103,9 @@ class AllRegisteredSrcFiles { public: const SrcFile* get_file(int file_id) const { return all_src_files.at(file_id); } - const SrcFile* find_file(const std::string& abs_filename) const; + const SrcFile* find_file(const std::string& realpath) const; - const SrcFile* locate_and_register_source_file(const std::string& rel_filename, SrcLocation included_from); + const SrcFile* locate_and_register_source_file(const std::string& filename, SrcLocation included_from); SrcFile* get_next_unparsed_file(); auto begin() const { return all_src_files.begin(); } diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 03d0cda13..348fac839 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -21,6 +21,19 @@ namespace tolk { +void Symbol::check_import_exists_when_used_from(FunctionPtr cur_f, SrcLocation used_loc) const { + const SrcFile* declared_in = loc.get_src_file(); + bool has_import = false; + for (const SrcFile::ImportDirective& import : used_loc.get_src_file()->imports) { + if (import.imported_file == declared_in) { + has_import = true; + } + } + if (!has_import) { + throw ParseError(cur_f, used_loc, "Using a non-imported symbol `" + name + "`\nhint: forgot to import \"" + declared_in->extract_short_name() + "\"?"); + } +} + std::string FunctionData::as_human_readable() const { if (!is_generic_function()) { return name; // if it's generic instantiation like `f`, its name is "f", not "f" diff --git a/tolk/symtable.h b/tolk/symtable.h index 50ab271be..268b474d3 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -43,6 +43,8 @@ struct Symbol { #endif return dynamic_cast(this); } + + void check_import_exists_when_used_from(FunctionPtr cur_f, SrcLocation used_loc) const; }; struct LocalVarData final : Symbol { diff --git a/tolk/tolk-main.cpp b/tolk/tolk-main.cpp index 4e9c1f4d8..17a2b4c84 100644 --- a/tolk/tolk-main.cpp +++ b/tolk/tolk-main.cpp @@ -149,19 +149,15 @@ static std::string auto_discover_stdlib_folder() { td::Result fs_read_callback(CompilerSettings::FsReadCallbackKind kind, const char* query) { switch (kind) { case CompilerSettings::FsReadCallbackKind::Realpath: { - td::Result res_realpath; - if (query[0] == '@' && strlen(query) > 8 && !strncmp(query, "@stdlib/", 8)) { - // import "@stdlib/filename" or import "@stdlib/filename.tolk" - std::string path = G.settings.stdlib_folder + static_cast(query + 7); - if (strncmp(path.c_str() + path.size() - 5, ".tolk", 5) != 0) { - path += ".tolk"; - } - res_realpath = td::realpath(td::CSlice(path.c_str())); - } else { - // import "relative/to/cwd/path.tolk" - res_realpath = td::realpath(td::CSlice(query)); - } + bool is_stdlib = query[0] == '@' && strlen(query) > 8 && !strncmp(query, "@stdlib/", 8); + std::string path = is_stdlib + ? G.settings.stdlib_folder + static_cast(query + 7) + : static_cast(query); + if (strncmp(path.c_str() + path.size() - 5, ".tolk", 5) != 0) { + path += ".tolk"; + } + td::Result res_realpath = td::realpath(td::CSlice(path.c_str())); if (res_realpath.is_error()) { // note, that for non-existing files, `realpath()` on Linux/Mac returns an error, // whereas on Windows, it returns okay, but fails after, on reading, with a message "cannot open file" diff --git a/tolk/tolk-wasm.cpp b/tolk/tolk-wasm.cpp index 3c8cb87dd..b5149481b 100644 --- a/tolk/tolk-wasm.cpp +++ b/tolk/tolk-wasm.cpp @@ -57,7 +57,7 @@ static td::Result compile_internal(char *config_json) { std::cerr.rdbuf(errs.rdbuf()); int exit_code = tolk_proceed(entrypoint_filename); if (exit_code != 0) { - return td::Status::Error("Tolk compilation error: " + errs.str()); + return td::Status::Error(errs.str()); } TRY_RESULT(fift_res, fift::compile_asm_program(outs.str(), "/fiftlib/")); From 2bb89b461ffc5ef193429823ee1c9e17dab75818 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sun, 29 Jun 2025 21:38:10 +0300 Subject: [PATCH 336/388] [Tolk] Correctly handle variadic ints at serialization Occasionally, `varint32` was not handled at packing to/from cells and worked like a regular `int32`. This MR fixes the issue, and also: - adds missing varintN types - fix intN/uintN limits - rename bytesN to bitsN, since "bits" are more frequent in practice --- crypto/smartcont/tolk-stdlib/common.tolk | 4 +- tolk-tester/tests/intN-tests.tolk | 4 +- tolk-tester/tests/lazy-load-tests.tolk | 17 ++++++- tolk-tester/tests/pack-unpack-2.tolk | 47 +++++++++++++++++++ tolk-tester/tests/pack-unpack-5.tolk | 3 +- tolk-tester/tests/pack-unpack-6.tolk | 4 +- tolk/builtins.cpp | 45 ++++++++++++++++++ tolk/pack-unpack-api.cpp | 4 +- tolk/pack-unpack-serializers.cpp | 60 +++++++++++++++++++----- tolk/pack-unpack-serializers.h | 2 +- tolk/pipe-ast-to-legacy.cpp | 8 ++-- tolk/pipe-lazy-load-insertions.cpp | 6 +-- tolk/pipe-resolve-types.cpp | 36 +++++++------- tolk/type-system.cpp | 32 ++++++------- tolk/type-system.h | 30 ++++++------ 15 files changed, 223 insertions(+), 79 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 9c57bf18f..7cf61f562 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -227,8 +227,8 @@ struct PackOptions { /// however, if you guarantee that a slice is valid (for example, it comes from trusted sources), /// set this option to true to disable runtime checks; /// note: `int32` and other are always validated for overflow without any extra gas, - /// so this flag controls only rarely used `bytesN` / `bitsN` types - skipBitsNFieldsValidation: bool = false, + /// so this flag controls only rarely used `bitsN` type + skipBitsNValidation: bool = false, } /// UnpackOptions allows you to control behavior of `MyStruct.fromCell(c)` and similar functions. diff --git a/tolk-tester/tests/intN-tests.tolk b/tolk-tester/tests/intN-tests.tolk index 9cdc46988..cd557092e 100644 --- a/tolk-tester/tests/intN-tests.tolk +++ b/tolk-tester/tests/intN-tests.tolk @@ -80,7 +80,7 @@ fun test3(op: int32, qid: uint64) { if (op == qid) {} if ((op as int32?)! == qid) {} if (op == (qid as uint64)) {} - if ((op as uint257?)! != (qid as int256?)!) {} + if ((op as int257?)! != (qid as uint256?)!) {} __expect_type(op << qid, "int"); takeAnyInt(op); @@ -227,7 +227,7 @@ fun main() { assign0(mutate t.1 as uint16); assign0(mutate t.2 as int); __expect_type(t.0 as int1, "int1"); - __expect_type(t.0 as uint257, "uint257"); + __expect_type(t.0 as int257, "int257"); __expect_type(0 as int32, "int32"); __expect_type((0 as int32) as uint64?, "uint64?"); __expect_type(null as (int8, [uint16?])?, "(int8, [uint16?])?"); diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk index 63c40f152..208c89241 100644 --- a/tolk-tester/tests/lazy-load-tests.tolk +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -217,6 +217,11 @@ struct OneIntOneRef { r: cell; } +struct WithVariadicInts { + i1: varuint32; + i2: varuint32; +} + @noinline fun contract.getFakeData(seqno: int): Cell { @@ -655,7 +660,7 @@ fun demo135(skip: bool) { cc.bin16 = stringHexToSlice("00000000") as bits16; try { return cc.toCell({ - skipBitsNFieldsValidation: skip + skipBitsNValidation: skip }).beginParse().remainingBitsCount(); } catch (excno) { return excno; @@ -939,6 +944,13 @@ fun demo156(s: slice): int { } } +@method_id(157) +fun demo157() { + var c: Cell = WithVariadicInts { i1: 1 << 100, i2: 1 << 200 }.toCell(); + val d = lazy c.load(); + return d.i2 == (1 << 200); +} + @method_id(200) fun demo200() { @@ -1280,7 +1292,7 @@ fun demo310() { f9: "" as bits200, f10: ("" as bits400, ()), f11: 0, - }.toCell({skipBitsNFieldsValidation: true}); + }.toCell({skipBitsNValidation: true}); var st = lazy cell.load(); var sm = lazy cell.load(); var sk = lazy cell.load(); @@ -1625,6 +1637,7 @@ fun main() { @testcase | 155 | | 555 @testcase | 156 | x{0110} | 16 @testcase | 156 | x{FF10} | -100 +@testcase | 157 | | -1 @testcase | 200 | | 1 @testcase | 201 | | 1 2 -1 diff --git a/tolk-tester/tests/pack-unpack-2.tolk b/tolk-tester/tests/pack-unpack-2.tolk index 51432d0b4..cb01fbfeb 100644 --- a/tolk-tester/tests/pack-unpack-2.tolk +++ b/tolk-tester/tests/pack-unpack-2.tolk @@ -334,6 +334,30 @@ struct DifferentMix3 { pairm: (int32, int64)?; } +/* + ui16: VarUInteger 16 + i16: VarInteger 16 + ui32: VarUInteger 32 + i32: VarInteger 32 + */ + +struct WithVariadicInts { + ui16: varuint16; + i16: varint16; + ui32: varuint32; + i32: varint32; +} + +/* + Test MIN_INT and MAX_INT + */ + +struct EdgeCaseInts { + maxUint: uint256 = +115792089237316195423570985008687907853269984665640564039457584007913129639935; + maxInt: int257 = +115792089237316195423570985008687907853269984665640564039457584007913129639935; + minInt: int257 = -115792089237316195423570985008687907853269984665640564039457584007913129639935 - 1; +} + /* Test: write with builder, read with other struct (type `builder` is available for writing, but not for reading) @@ -670,6 +694,27 @@ fun test_mutatingRemainder() { return obj.rest.remainingBitsCount(); } +@method_id(228) +fun test_VariadicIntegers() { + var t: WithVariadicInts = { + ui16: (1 << 120) - 1, + i16: -(1 << 119), + ui32: (1 << 248) - 1, + i32: -(1 << 247), + }; + run(t, stringHexToSlice("ffffffffffffffffffffffffffffffff800000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe000000000000000000000000000000000000000000000000000000000000020_")); + run({ui16: 0, i16: 0, ui32: 0, i32: 0}, stringHexToSlice("000020_")); + + var c = t.toCell().load(); + return (c.ui16 == t.ui16) & (c.i16 == t.i16) & (c.ui32 == t.ui32) & (c.i32 == t.i32) +} + +@method_id(229) +fun test_EdgeCaseIntegers() { + var edge: EdgeCaseInts = {}; + var manual = beginCell().storeUint(edge.maxUint, 256).storeInt(edge.maxInt, 257).storeInt(edge.minInt, 257); + return edge.toCell().hash() == manual.endCell().hash() +} fun main() { var t: JustInt32 = { value: 10 }; @@ -709,4 +754,6 @@ fun main() { @testcase | 225 | | 5 40 65535 8 50000000 @testcase | 226 | | 1 16 0 @testcase | 227 | | 16 +@testcase | 228 | | -1 +@testcase | 229 | | -1 */ diff --git a/tolk-tester/tests/pack-unpack-5.tolk b/tolk-tester/tests/pack-unpack-5.tolk index 46b2a77b5..af397181e 100644 --- a/tolk-tester/tests/pack-unpack-5.tolk +++ b/tolk-tester/tests/pack-unpack-5.tolk @@ -22,6 +22,7 @@ fun test1() { coins.estimatePackSize(), bits6.estimatePackSize(), bytes8.estimatePackSize(), + varint32.estimatePackSize(), ); } @@ -204,7 +205,7 @@ fun main() { } /** -@testcase | 101 | | [ 32 32 0 0 ] [ 64 64 0 0 ] [ 1 1 0 0 ] [ 1 1 0 0 ] [ 0 9999 0 4 ] [ 4 124 0 0 ] [ 6 6 0 0 ] [ 64 64 0 0 ] +@testcase | 101 | | [ 32 32 0 0 ] [ 64 64 0 0 ] [ 1 1 0 0 ] [ 1 1 0 0 ] [ 0 9999 0 4 ] [ 4 124 0 0 ] [ 6 6 0 0 ] [ 64 64 0 0 ] [ 5 253 0 0 ] @testcase | 102 | | [ 32 32 0 0 ] [ 1 33 0 0 ] [ 1 33 0 0 ] [ 17 33 0 0 ] @testcase | 103 | | [ 67 199 0 0 ] [ 69 599 0 0 ] @testcase | 104 | | [ 4 534 0 0 ] [ 2 536 0 0 ] [ 6 1070 0 0 ] diff --git a/tolk-tester/tests/pack-unpack-6.tolk b/tolk-tester/tests/pack-unpack-6.tolk index 51c1f7c13..71f6ab0b4 100644 --- a/tolk-tester/tests/pack-unpack-6.tolk +++ b/tolk-tester/tests/pack-unpack-6.tolk @@ -50,13 +50,13 @@ fun test5() { @method_id(106) fun test6() { - return SomeBytesFields { f1: stringHexToSlice("11") as bytes1 }.toCell({skipBitsNFieldsValidation: true}).hash() & 0xFFFF; + return SomeBytesFields { f1: stringHexToSlice("11") as bytes1 }.toCell({skipBitsNValidation: true}).hash() & 0xFFFF; } @method_id(107) fun test7(believe: bool) { try { - return SomeBytesFields { f1: stringHexToSlice("ffff") as bytes1 }.toCell({skipBitsNFieldsValidation: believe}).hash() & 0xFFFF; + return SomeBytesFields { f1: stringHexToSlice("ffff") as bytes1 }.toCell({skipBitsNValidation: believe}).hash() & 0xFFFF; } catch (excno) { return excno; } diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 59c07da95..cce907beb 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1036,6 +1036,25 @@ static AsmOp compile_fetch_int(std::vector& res, std::vector return exec_op(loc, (fetch ? "LD"s : "PLD"s) + (sgnd ? "IX" : "UX"), 2, 1 + (unsigned)fetch); } +// fun slice.__loadVarInt(mutate self, bits: int, unsigned: bool): int +static AsmOp compile_fetch_varint(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(args.size() == 3 && res.size() == 2); + // it's a hidden function for auto-serialization (not exposed to stdlib), to bits/unsigned are not dynamic + tolk_assert(args[1].is_int_const() && args[2].is_int_const()); + long n_bits = args[1].int_const->to_long(); + long is_unsigned = args[2].int_const->to_long(); + + args[1].unused(); + args[2].unused(); + if (n_bits == 16) { + return exec_op(loc, is_unsigned ? "LDVARUINT16" : "LDVARINT16", 1, 2); + } + if (n_bits == 32) { + return exec_op(loc, is_unsigned ? "LDVARUINT32" : "LDVARINT32", 1, 2); + } + tolk_assert(false); +} + // fun builder.storeInt (mutate self, x: int, len: int): self asm(x b len) "STIX"; // fun builder.storeUint (mutate self, x: int, len: int): self asm(x b len) "STUX"; static AsmOp compile_store_int(std::vector& res, std::vector& args, SrcLocation loc, bool sgnd) { @@ -1063,6 +1082,25 @@ static AsmOp compile_store_int(std::vector& res, std::vector return exec_op(loc, sgnd ? "STIX" : "STUX", 3, 1); } +// fun builder.__storeVarInt (mutate self, x: int, bits: int, unsigned: bool): self +static AsmOp compile_store_varint(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(args.size() == 4 && res.size() == 1); + // it's a hidden function for auto-serialization (not exposed to stdlib), to bits/unsigned are not dynamic + tolk_assert(args[2].is_int_const() && args[3].is_int_const()); + long n_bits = args[2].int_const->to_long(); + long is_unsigned = args[3].int_const->to_long(); + + args[2].unused(); + args[3].unused(); + if (n_bits == 16) { + return exec_op(loc, is_unsigned ? "STVARUINT16" : "STVARINT16", 2, 1); + } + if (n_bits == 32) { + return exec_op(loc, is_unsigned ? "STVARUINT32" : "STVARINT32", 2, 1); + } + tolk_assert(false); +} + // fun builder.storeBool(mutate self, value: bool): self asm( -> 1 0) "1 STI"; static AsmOp compile_store_bool(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(args.size() == 2 && res.size() == 1); @@ -1388,6 +1426,13 @@ void define_builtins() { define_builtin_func("__InMessage.getInMsgParam", ParamsInt1, Int, nullptr, compile_calc_InMessage_getInMsgParam, 0); + define_builtin_method("builder.__storeVarInt", Builder, {Builder, Int, Int, Bool}, Unit, nullptr, + compile_store_varint, // not exposed to stdlib, used in auto-serialization + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf); + define_builtin_method("slice.__loadVarInt", Slice, {Slice, Int, Bool}, Int, nullptr, + compile_fetch_varint, // not exposed to stdlib, used in auto-serialization + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, + {}, {1, 0}); // compile-time only functions, evaluated essentially at compile-time, no runtime implementation // they are placed in stdlib and marked as `builtin` diff --git a/tolk/pack-unpack-api.cpp b/tolk/pack-unpack-api.cpp index 62ef8d7bb..083537710 100644 --- a/tolk/pack-unpack-api.cpp +++ b/tolk/pack-unpack-api.cpp @@ -58,7 +58,7 @@ class PackUnpackAvailabilityChecker { if (any_type->try_as()) { return {}; } - if (any_type->try_as()) { + if (any_type->try_as()) { return {}; } if (any_type == TypeDataCoins::create()) { @@ -345,7 +345,7 @@ std::vector generate_lazy_struct_to_cell(CodeBlob& code, SrcLocation ctx.generate_pack_any(hidden_field->declared_type, std::move(ir_field)); } else { std::vector ir_gap_or_tail = loaded_state->get_ir_loaded_aside_field(hidden_field); - if (hidden_field->declared_type->unwrap_alias()->try_as()) { + if (hidden_field->declared_type->unwrap_alias()->try_as()) { ctx.storeSlice(ir_gap_or_tail[0]); } else { ctx.generate_pack_any(hidden_field->declared_type, std::move(ir_gap_or_tail)); diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index 770ec56b9..f38f5d211 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -65,7 +65,7 @@ PackContext::PackContext(CodeBlob& code, SrcLocation loc, std::vector , f_storeUint(lookup_function("builder.storeUint")) , ir_builder(std::move(ir_builder)) , ir_builder0(this->ir_builder[0]) - , option_skipBitsNFieldsValidation(ir_options[0]) { + , option_skipBitsNValidation(ir_options[0]) { } void PackContext::storeInt(var_idx_t ir_idx, int len) const { @@ -264,16 +264,51 @@ struct S_IntN final : ISerializer { } }; -struct S_BytesN final : ISerializer { +struct S_VariadicIntN final : ISerializer { + const int n_bits; // only 16 and 32 available + const bool is_unsigned; + + explicit S_VariadicIntN(int n_bits, bool is_unsigned) + : n_bits(n_bits), is_unsigned(is_unsigned) {} + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + FunctionPtr f_storeVarInt = lookup_function("builder.__storeVarInt"); + std::vector args = { ctx->ir_builder0, rvect[0], code.create_int(loc, n_bits, "(n-bits)"), code.create_int(loc, is_unsigned, "(is-unsigned)") }; + code.emplace_back(loc, Op::_Call, ctx->ir_builder, std::move(args), f_storeVarInt); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_loadVarInt = lookup_function("slice.__loadVarInt"); + std::vector args = { ctx->ir_slice0, code.create_int(loc, n_bits, "(n-bits)"), code.create_int(loc, is_unsigned, "(is-unsigned)") }; + std::vector result = code.create_tmp_var(TypeDataInt::create(), loc, "(loaded-varint)"); + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, result[0]}, std::move(args), f_loadVarInt); + return result; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + // no TVM instruction to skip, just load but don't use the result + unpack(ctx, code, loc); + } + + PackSize estimate(const EstimateContext* ctx) override { + if (n_bits == 32) { + return PackSize(5, 253); + } else { + return PackSize(4, 124); // same as `coins` + } + } +}; + +struct S_BitsN final : ISerializer { const int n_bits; - explicit S_BytesN(int n_width, bool is_bits) + explicit S_BitsN(int n_width, bool is_bits) : n_bits(is_bits ? n_width : n_width * 8) {} void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { tolk_assert(rvect.size() == 1); - Op& if_disabled_by_user = code.emplace_back(loc, Op::_If, std::vector{ctx->option_skipBitsNFieldsValidation}); + Op& if_disabled_by_user = code.emplace_back(loc, Op::_If, std::vector{ctx->option_skipBitsNValidation}); { code.push_set_cur(if_disabled_by_user.block0); code.close_pop_cur(loc); @@ -400,10 +435,8 @@ struct S_Coins final : ISerializer { } void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { - FunctionPtr f_loadCoins = lookup_function("slice.loadCoins"); - std::vector args = ctx->ir_slice; - std::vector dummy_loaded = code.create_tmp_var(TypeDataInt::create(), loc, "(loaded-coins)"); - code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, dummy_loaded[0]}, std::move(args), f_loadCoins); + // no TVM instruction to skip, just load but don't use the result + unpack(ctx, code, loc); } PackSize estimate(const EstimateContext* ctx) override { @@ -428,9 +461,7 @@ struct S_Address final : ISerializer { // we can't do just // ctx->skipBits(2 + 1 + 8 + 256); // because it may be addr_none or addr_extern; there is no "skip address" in TVM, so just load it - FunctionPtr f_loadAddress = lookup_function("slice.loadAddress"); - std::vector ir_address = code.create_tmp_var(TypeDataSlice::create(), loc, "(tmp-addr)"); - code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_address[0]}, ctx->ir_slice, f_loadAddress); + unpack(ctx, code, loc); } PackSize estimate(const EstimateContext* ctx) override { @@ -1088,10 +1119,13 @@ std::vector auto_generate_opcodes_for_union(TypePtr union_type, std: static std::unique_ptr get_serializer_for_type(TypePtr any_type) { if (const auto* t_intN = any_type->try_as()) { + if (t_intN->is_variadic) { + return std::make_unique(t_intN->n_bits, t_intN->is_unsigned); + } return std::make_unique(t_intN->n_bits, t_intN->is_unsigned); } - if (const auto* t_bytesN = any_type->try_as()) { - return std::make_unique(t_bytesN->n_width, t_bytesN->is_bits); + if (const auto* t_bitsN = any_type->try_as()) { + return std::make_unique(t_bitsN->n_width, t_bitsN->is_bits); } if (any_type == TypeDataCoins::create()) { return std::make_unique(); diff --git a/tolk/pack-unpack-serializers.h b/tolk/pack-unpack-serializers.h index 0e664b9a1..b3427953b 100644 --- a/tolk/pack-unpack-serializers.h +++ b/tolk/pack-unpack-serializers.h @@ -64,7 +64,7 @@ class PackContext { public: const std::vector ir_builder; const var_idx_t ir_builder0; - const var_idx_t option_skipBitsNFieldsValidation; + const var_idx_t option_skipBitsNValidation; PackContext(CodeBlob& code, SrcLocation loc, std::vector ir_builder, const std::vector& ir_options); diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index aa2e2dd3a..38a7c492a 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -1026,20 +1026,20 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as()) { + // no changes in rvect, since bitsN is slice at TVM level + if (target_type == TypeDataSlice::create() && original_type->try_as()) { return rvect; } // pass `slice` to `bytes32` // same as above - if (original_type == TypeDataSlice::create() && target_type->try_as()) { + if (original_type == TypeDataSlice::create() && target_type->try_as()) { return rvect; } // pass `bytes32` to `bytes64` / `bits128` to `bytes16` // no changes in rvect - if (original_type->try_as() && target_type->try_as()) { + if (original_type->try_as() && target_type->try_as()) { return rvect; } diff --git a/tolk/pipe-lazy-load-insertions.cpp b/tolk/pipe-lazy-load-insertions.cpp index e7510023a..64438645b 100644 --- a/tolk/pipe-lazy-load-insertions.cpp +++ b/tolk/pipe-lazy-load-insertions.cpp @@ -433,10 +433,10 @@ struct ExprUsagesWhileCollecting { // okay, this field is not needed; we should skip it; // try to merge "skip 8 bits" + "skip 16 bits" into a single "skip 24 bits" if (!future_fields.empty() && future_fields.back().action == LazyStructLoadInfo::SkipField) { - if (const TypeDataBytesN* last_bitsN = future_fields.back().field_type->try_as()) { + if (const TypeDataBitsN* last_bitsN = future_fields.back().field_type->try_as()) { PackSize cur_size = estimate_serialization_size(field_type); if (cur_size.min_bits == cur_size.max_bits && cur_size.max_refs == 0) { - TypePtr total_bitsN = TypeDataBytesN::create(true, last_bitsN->n_width + cur_size.max_bits); + TypePtr total_bitsN = TypeDataBitsN::create(last_bitsN->n_width + cur_size.max_bits, true); future_fields.back().field_type = total_bitsN; continue; } @@ -447,7 +447,7 @@ struct ExprUsagesWhileCollecting { TypePtr skip_type = field_type; PackSize skip_size = estimate_serialization_size(field_type); if (skip_size.min_bits == skip_size.max_bits && skip_size.max_refs == 0) { - skip_type = TypeDataBytesN::create(true, skip_size.max_bits); + skip_type = TypeDataBitsN::create(skip_size.max_bits, true); } future_fields.emplace_back(LazyStructLoadInfo::SkipField, "(gap)", skip_type); } diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 2a9bb2028..7ca035cfd 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -71,24 +71,24 @@ static void fire_error_generic_type_used_without_T(FunctionPtr cur_f, SrcLocatio fire(cur_f, loc, "type `" + type_name_with_Ts + "` is generic, you should provide type arguments"); } -static TypePtr parse_intN(std::string_view strN, bool is_unsigned) { +static TypePtr parse_intN_uintN(std::string_view strN, bool is_unsigned) { int n; - auto result = std::from_chars(strN.data() + 3 + static_cast(is_unsigned), strN.data() + strN.size(), n); + auto result = std::from_chars(strN.data(), strN.data() + strN.size(), n); bool parsed = result.ec == std::errc() && result.ptr == strN.data() + strN.size(); - if (!parsed || n <= 0 || n > 256 + static_cast(is_unsigned)) { + if (!parsed || n <= 0 || n > 257 - static_cast(is_unsigned)) { return nullptr; // `int1000`, maybe it's user-defined alias, let it be unresolved } - return TypeDataIntN::create(is_unsigned, false, n); + return TypeDataIntN::create(n, is_unsigned, false); } -static TypePtr parse_bytesN(std::string_view strN, bool is_bits) { +static TypePtr parse_bytesN_bitsN(std::string_view strN, bool is_bits) { int n; - auto result = std::from_chars(strN.data() + 5 - static_cast(is_bits), strN.data() + strN.size(), n); + auto result = std::from_chars(strN.data(), strN.data() + strN.size(), n); bool parsed = result.ec == std::errc() && result.ptr == strN.data() + strN.size(); if (!parsed || n <= 0 || n > 1024) { return nullptr; // `bytes9999`, maybe it's user-defined alias, let it be unresolved } - return TypeDataBytesN::create(is_bits, n); + return TypeDataBitsN::create(n, is_bits); } static TypePtr try_parse_predefined_type(std::string_view str) { @@ -113,8 +113,12 @@ static TypePtr try_parse_predefined_type(std::string_view str) { if (str == "address") return TypeDataAddress::create(); break; case 8: - if (str == "varint16") return TypeDataIntN::create(false, true, 16); - if (str == "varint32") return TypeDataIntN::create(false, true, 32); + if (str == "varint16") return TypeDataIntN::create(16, false, true); + if (str == "varint32") return TypeDataIntN::create(32, false, true); + break; + case 9: + if (str == "varuint16") return TypeDataIntN::create(16, true, true); + if (str == "varuint32") return TypeDataIntN::create(32, true, true); break; case 12: if (str == "continuation") return TypeDataContinuation::create(); @@ -124,22 +128,22 @@ static TypePtr try_parse_predefined_type(std::string_view str) { } if (str.starts_with("int")) { - if (TypePtr intN = parse_intN(str, false)) { + if (TypePtr intN = parse_intN_uintN(str.substr(3), false)) { return intN; } } - if (str.size() > 4 && str.starts_with("uint")) { - if (TypePtr uintN = parse_intN(str, true)) { + if (str.starts_with("uint")) { + if (TypePtr uintN = parse_intN_uintN(str.substr(4), true)) { return uintN; } } - if (str.size() > 4 && str.starts_with("bits")) { - if (TypePtr bitsN = parse_bytesN(str, true)) { + if (str.starts_with("bits")) { + if (TypePtr bitsN = parse_bytesN_bitsN(str.substr(4), true)) { return bitsN; } } - if (str.size() > 5 && str.starts_with("bytes")) { - if (TypePtr bytesN = parse_bytesN(str, false)) { + if (str.starts_with("bytes")) { + if (TypePtr bytesN = parse_bytesN_bitsN(str.substr(5), false)) { return bytesN; } } diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index b6ffd6022..6253537e2 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -295,27 +295,27 @@ TypePtr TypeDataBrackets::create(std::vector&& items) { return hash.register_unique(new TypeDataBrackets(hash.children_flags(), std::move(items))); } -TypePtr TypeDataIntN::create(bool is_unsigned, bool is_variadic, int n_bits) { +TypePtr TypeDataIntN::create(int n_bits, bool is_unsigned, bool is_variadic) { TypeDataHasherForUnique hash(1678330938771108027ULL); + hash.feed_hash(n_bits); hash.feed_hash(is_unsigned); hash.feed_hash(is_variadic); - hash.feed_hash(n_bits); if (TypePtr existing = hash.get_existing()) { return existing; } - return hash.register_unique(new TypeDataIntN(is_unsigned, is_variadic, n_bits)); + return hash.register_unique(new TypeDataIntN(n_bits, is_unsigned, is_variadic)); } -TypePtr TypeDataBytesN::create(bool is_bits, int n_width) { +TypePtr TypeDataBitsN::create(int n_width, bool is_bits) { TypeDataHasherForUnique hash(7810988137199333041ULL); - hash.feed_hash(is_bits); hash.feed_hash(n_width); + hash.feed_hash(is_bits); if (TypePtr existing = hash.get_existing()) { return existing; } - return hash.register_unique(new TypeDataBytesN(is_bits, n_width)); + return hash.register_unique(new TypeDataBitsN(n_width, is_bits)); } TypePtr TypeDataUnion::create(std::vector&& variants) { @@ -504,7 +504,7 @@ int TypeDataIntN::get_type_id() const { } } -int TypeDataBytesN::get_type_id() const { +int TypeDataBitsN::get_type_id() const { return TypeIdCalculation::assign_type_id(this); } @@ -591,9 +591,9 @@ std::string TypeDataIntN::as_human_readable() const { return s_int + std::to_string(n_bits); } -std::string TypeDataBytesN::as_human_readable() const { - std::string s_bytes = is_bits ? "bits" : "bytes"; - return s_bytes + std::to_string(n_width); +std::string TypeDataBitsN::as_human_readable() const { + std::string s_bits = is_bits ? "bits" : "bytes"; + return s_bits + std::to_string(n_width); } std::string TypeDataUnion::as_human_readable() const { @@ -739,7 +739,7 @@ bool TypeDataSlice::can_rhs_be_assigned(TypePtr rhs) const { if (const TypeDataAlias* rhs_alias = rhs->try_as()) { return can_rhs_be_assigned(rhs_alias->underlying_type); } - return rhs == TypeDataNever::create(); // note, that bytesN/address is NOT automatically cast to slice without `as` operator + return rhs == TypeDataNever::create(); // note, that bitsN/address is NOT automatically cast to slice without `as` operator } bool TypeDataBuilder::can_rhs_be_assigned(TypePtr rhs) const { @@ -881,8 +881,8 @@ bool TypeDataIntN::can_rhs_be_assigned(TypePtr rhs) const { return rhs == TypeDataNever::create(); // `int8` is NOT assignable to `int32` without `as` } -bool TypeDataBytesN::can_rhs_be_assigned(TypePtr rhs) const { - // `slice` is NOT assignable to bytesN without `as` +bool TypeDataBitsN::can_rhs_be_assigned(TypePtr rhs) const { + // `slice` is NOT assignable to bitsN without `as` // `bytes32` is NOT assignable to `bytes256` and even to `bits256` without `as` if (rhs == this) { return true; @@ -1010,7 +1010,7 @@ bool TypeDataCell::can_be_casted_with_as_operator(TypePtr cast_to) const { } bool TypeDataSlice::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (cast_to->try_as()) { // `slice` to `bytes32` / `slice` to `bits8` + if (cast_to->try_as()) { // `slice` to `bytes32` / `slice` to `bits8` return true; } if (cast_to == TypeDataAddress::create()) { @@ -1176,8 +1176,8 @@ bool TypeDataIntN::can_be_casted_with_as_operator(TypePtr cast_to) const { return cast_to == TypeDataInt::create() || cast_to == TypeDataCoins::create(); } -bool TypeDataBytesN::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (cast_to->try_as()) { // `bytes256` as `bytes512`, `bits1` as `bytes8` +bool TypeDataBitsN::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (cast_to->try_as()) { // `bytes256` as `bytes512`, `bits1` as `bytes8` return true; } if (const TypeDataUnion* to_union = cast_to->try_as()) { // `bytes8` as `slice?` diff --git a/tolk/type-system.h b/tolk/type-system.h index 62841893a..e9a63c213 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -458,18 +458,18 @@ class TypeDataBrackets final : public TypeData { * intN is smoothly cast from/to plain int, mathematical operators on intN also "fall back" to general int. */ class TypeDataIntN final : public TypeData { - TypeDataIntN(bool is_unsigned, bool is_variadic, int n_bits) + TypeDataIntN(int n_bits, bool is_unsigned, bool is_variadic) : TypeData(0) + , n_bits(n_bits) , is_unsigned(is_unsigned) - , is_variadic(is_variadic) - , n_bits(n_bits) {} + , is_variadic(is_variadic) {} public: + const int n_bits; const bool is_unsigned; const bool is_variadic; - const int n_bits; - static TypePtr create(bool is_unsigned, bool is_variadic, int n_bits); + static TypePtr create(int n_bits, bool is_unsigned, bool is_variadic); int get_type_id() const override; std::string as_human_readable() const override; @@ -497,22 +497,22 @@ class TypeDataCoins final : public TypeData { }; /* - * `bytes256`, `bits512`, `bytes8` are TypeDataBytesN. At TVM level, it's just slice. - * The purpose of bytesN is to be used in struct fields, describing the way of serialization (n bytes / n bits). - * In this essence, bytesN is very similar to intN. - * Note, that unlike intN automatically cast to/from int, bytesN does NOT auto cast to slice (without `as`). + * `bits512`, `bytes8`, `bits9` are TypeDataBitsN. At TVM level, it's just slice. + * The purpose of bitsN is to be used in struct fields, describing the way of serialization (n bytes / n bits). + * In this essence, bitsN is very similar to intN. + * Note that unlike intN automatically cast to/from int, bitsN does NOT auto cast to slice (without `as`). */ -class TypeDataBytesN final : public TypeData { - TypeDataBytesN(bool is_bits, int n_width) +class TypeDataBitsN final : public TypeData { + TypeDataBitsN(int n_width, bool is_bits) : TypeData(0) - , is_bits(is_bits) - , n_width(n_width) {} + , n_width(n_width) + , is_bits(is_bits) {} public: + const int n_width; // either in bits, or in bytes const bool is_bits; - const int n_width; - static TypePtr create(bool is_bits, int n_width); + static TypePtr create(int n_width, bool is_bits); int get_type_id() const override; std::string as_human_readable() const override; From 8d650aa174ac592ccad1225a0919f06e041990a3 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sun, 29 Jun 2025 23:18:13 +0300 Subject: [PATCH 337/388] [Tolk] Support custom pack/unpack serializers for any T For any type alias, one can now specify > fun T.packToBuilder > fun T.unpackFromSlice That will be called whenever that type is serialized. Perfect to express custom TL/B behavior, like > len: (## 8) > data: (bits (len*8)) --- .../tests/invalid-serialization/err-7750.tolk | 22 +++ .../tests/invalid-serialization/err-7751.tolk | 20 ++ .../tests/invalid-serialization/err-7752.tolk | 20 ++ .../tests/invalid-serialization/err-7753.tolk | 19 ++ .../tests/invalid-serialization/err-7754.tolk | 22 +++ tolk-tester/tests/lazy-load-tests.tolk | 31 +++ tolk-tester/tests/pack-unpack-7.tolk | 182 ++++++++++++++++++ tolk/CMakeLists.txt | 2 +- tolk/pack-unpack-api.cpp | 40 ++++ tolk/pack-unpack-serializers.cpp | 55 ++++++ tolk/pack-unpack-serializers.h | 1 + tolk/pipe-check-serialized-fields.cpp | 6 +- tolk/pipe-detect-inline-in-place.cpp | 16 +- tolk/pipe-infer-types-and-calls.cpp | 2 +- tolk/pipeline.h | 2 +- tolk/tolk.cpp | 2 +- 16 files changed, 433 insertions(+), 9 deletions(-) create mode 100644 tolk-tester/tests/invalid-serialization/err-7750.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7751.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7752.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7753.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7754.tolk create mode 100644 tolk-tester/tests/pack-unpack-7.tolk diff --git a/tolk-tester/tests/invalid-serialization/err-7750.tolk b/tolk-tester/tests/invalid-serialization/err-7750.tolk new file mode 100644 index 000000000..fcc5329ab --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7750.tolk @@ -0,0 +1,22 @@ +type CustomInt = int; + +fun CustomInt.packToBuilder(self, mutate b: builder) { +} + +struct WithC { + a: int32; + b: CustomInt; +} + +fun main() { + WithC.fromSlice(""); +} + +/** +@compilation_should_fail +@stderr auto-serialization via fromSlice() is not available for type `WithC` +@stderr because field `WithC.b` of type `CustomInt` can't be serialized +@stderr because type `CustomInt` defines a custom pack function, but does not define unpack +@stderr hint: declare unpacker like this: +@stderr fun CustomInt.unpackFromSlice(mutate s: slice): CustomInt + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7751.tolk b/tolk-tester/tests/invalid-serialization/err-7751.tolk new file mode 100644 index 000000000..41ddd142e --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7751.tolk @@ -0,0 +1,20 @@ +type CustomInt = int; + +fun CustomInt.packToBuilder(self, b: builder) { +} + +fun CustomInt.unpackFromSlice(mutate s: slice) { +} + + +fun main() { + (5 as CustomInt).toCell() +} + +/** +@compilation_should_fail +@stderr auto-serialization via toCell() is not available for type `CustomInt` +@stderr because `CustomInt.packToBuilder()` is declared incorrectly +@stderr hint: it must accept 2 parameters and return nothing: +@stderr fun CustomInt.packToBuilder(self, mutate b: builder) + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7752.tolk b/tolk-tester/tests/invalid-serialization/err-7752.tolk new file mode 100644 index 000000000..b8dc0b9ec --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7752.tolk @@ -0,0 +1,20 @@ +type CustomInt = int; + +fun CustomInt.packToBuilder(self, mutate b: builder) { +} + +fun CustomInt.unpackFromSlice(self, mutate s: slice) { +} + + +fun main() { + (5 as CustomInt).toCell() +} + +/** +@compilation_should_fail +@stderr auto-serialization via toCell() is not available for type `CustomInt` +@stderr because `CustomInt.unpackFromSlice()` is declared incorrectly +@stderr hint: it must accept 1 parameter and return an object: +@stderr fun CustomInt.unpackFromSlice(mutate s: slice) + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7753.tolk b/tolk-tester/tests/invalid-serialization/err-7753.tolk new file mode 100644 index 000000000..6e30b474b --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7753.tolk @@ -0,0 +1,19 @@ +type CustomInt = int; + +fun CustomInt.packToBuilder(self, mutate b: builder) { +} + +fun CustomInt.unpackFromSlice(mutate s: slice) { + return (1, 2, 3) +} + +fun main() { + (5 as CustomInt).toCell() +} + +/** +@compilation_should_fail +@stderr auto-serialization via toCell() is not available for type `CustomInt` +@stderr because `CustomInt.unpackFromSlice()` is declared incorrectly +@stderr hint: it must accept 1 parameter and return an object: + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7754.tolk b/tolk-tester/tests/invalid-serialization/err-7754.tolk new file mode 100644 index 000000000..21127dd47 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7754.tolk @@ -0,0 +1,22 @@ +type CustomInt = int; + +fun CustomInt.packToBuilder(self, mutate b: builder) { +} + +fun CustomInt.unpackFromSlice(mutate s: slice) { + if (s.remainingBitsCount() > 10) { + return 123; + } + return 600; +} + +fun main() { + var s = ""; + s.loadAny(); +} + +/** +@compilation_should_fail +@stderr auto-serialization via loadAny() is not available for type `CustomInt` +@stderr because `CustomInt.unpackFromSlice()` can't be inlined; probably, it contains `return` in the middle + */ diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk index 208c89241..7a5643a54 100644 --- a/tolk-tester/tests/lazy-load-tests.tolk +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -222,6 +222,25 @@ struct WithVariadicInts { i2: varuint32; } +global gModByCustom: int; + +type MagicGlobalModifier = (); + +fun MagicGlobalModifier.packToBuilder(self, mutate b: builder) { + b.storeUint(gModByCustom, 8); +} + +fun MagicGlobalModifier.unpackFromSlice(mutate s: slice) { + gModByCustom = s.loadUint(8); + return (); +} + +struct WithGlobalModifier { + a: int8; + g: MagicGlobalModifier = (); + n: int8; +} + @noinline fun contract.getFakeData(seqno: int): Cell { @@ -951,6 +970,17 @@ fun demo157() { return d.i2 == (1 << 200); } +@method_id(158) +fun demo158() { + val m1 = lazy WithGlobalModifier.fromSlice(stringHexToSlice("01FF")); + __expect_lazy("[m1] skip (bits8), load g"); + m1.g; // modifies gModByCustom by a custom unpack function + val after1 = gModByCustom; + val m2 = lazy WithGlobalModifier.fromSlice(stringHexToSlice("010A02")); + __expect_lazy("[m2] skip (bits8) (MagicGlobalModifier), load n"); + return (m2.n, gModByCustom, after1); // also modifies by skipping (unpack called) +} + @method_id(200) fun demo200() { @@ -1638,6 +1668,7 @@ fun main() { @testcase | 156 | x{0110} | 16 @testcase | 156 | x{FF10} | -100 @testcase | 157 | | -1 +@testcase | 158 | | 2 10 255 @testcase | 200 | | 1 @testcase | 201 | | 1 2 -1 diff --git a/tolk-tester/tests/pack-unpack-7.tolk b/tolk-tester/tests/pack-unpack-7.tolk new file mode 100644 index 000000000..07407ff91 --- /dev/null +++ b/tolk-tester/tests/pack-unpack-7.tolk @@ -0,0 +1,182 @@ +// encoded as TL/B `len: (## 8) data: (bits (len*8))` +type TelegramString = slice; + +fun TelegramString.packToBuilder(self, mutate b: builder) { + val bytes = self.remainingBitsCount() / 8; + b.storeUint(bytes, 8); + b.storeSlice(self); +} + +fun TelegramString.unpackFromSlice(mutate s: slice) { + val bytes = s.loadUint(8); + return s.loadBits(bytes * 8); +} + +type Custom8 = int; + +fun Custom8.packToBuilder(self, mutate b: builder) { + b.storeUint(self, 8) +} + +fun Custom8.unpackFromSlice(mutate s: slice) { + return s.loadUint(8) +} + +struct StorWithStr { + a: int32; + str: TelegramString; + b: int32; +} + +struct PointWithCustomInt { + a: Custom8; + b: int8; +} + + +type MyBorderedInt = int; + +fun MyBorderedInt.packToBuilder(self, mutate b: builder) { + if (self > 10) { b.storeUint(1, 4) } + else if (self > 0) { b.storeUint(2, 4) } + else { b.storeUint(3, 4) } +} + +fun MyBorderedInt.unpackFromSlice(mutate s: slice) { + return match (s.loadUint(4)) { + 1 => 10, + 2 => 0, + 3 => -1, + else => throw 123 + } +} + +struct WithMyBorder { + a: int8; + b: MyBorderedInt; +} + +type MyCustomNothing = (); + +struct WithFakeWriter { + a: int8; + fake: MyCustomNothing = (); + b: int8; +} + +fun MyCustomNothing.packToBuilder(self, mutate b: builder) { + b.storeUint(123, 32); + b.storeRef(createEmptyCell()); +} + +global gModByCustom: int; + +type MagicGlobalModifier = (); + +fun MagicGlobalModifier.packToBuilder(self, mutate b: builder) { + b.storeUint(gModByCustom, 8); +} + +fun MagicGlobalModifier.unpackFromSlice(mutate s: slice) { + gModByCustom = s.loadUint(8); + return (); +} + +struct WithGlobalModifier { + a: int8; + g: MagicGlobalModifier = (); + n: int8; +} + +type Tensor3Skipping1 = (int, int, int); + +fun Tensor3Skipping1.unpackFromSlice(mutate s: slice): Tensor3Skipping1 { + val e0 = s.loadUint(8); + val e2 = s.loadUint(8); + return (e0, 0, e2); +} + +fun Tensor3Skipping1.packToBuilder(self, mutate b: builder) { + b.storeUint(self.0, 8).storeUint(self.2, 8); +} + + +@method_id(101) +fun test1() { + var t: StorWithStr = { a: 10, str: "abc", b: 20 }; + var c = t.toCell(); + var back = c.load(); + return ((t.b == back.b) & (t.str.bitsEqual(back.str)), back.str.remainingBitsCount(), c.hash() & 0xFFFF); +} + +@method_id(102) +fun test2() { + var s = ("" as TelegramString, "" as TelegramString); + return beginCell().storeAny(s).endCell().beginParse().remainingBitsCount(); +} + +@method_id(103) +fun test3() { + return PointWithCustomInt.fromSlice(stringHexToSlice("0102")); +} + +@method_id(104) +fun test4(initialInt: int) { + var c = WithMyBorder { a: 0, b: initialInt }.toCell(); + return c.load().b; +} + +@method_id(105) +fun test5() { + var f: WithFakeWriter = { a: 10, b: 20 }; + var s = beginCell().storeAny(f).endCell().beginParse(); + val size = s.remainingBitsAndRefsCount(); + return (s.skipBits(8).loadUint(32), size.1); +} + +@method_id(106) +fun test6() { + gModByCustom = 6; + val m1: WithGlobalModifier = { a: 8, n: 16 }; + val c1 = m1.toCell(); + val r2 = WithGlobalModifier.fromSlice(stringHexToSlice("01FF02")); + val gAfter2 = gModByCustom; + WithGlobalModifier.fromSlice(stringHexToSlice("010002")); // not deleted, sets 0 + val gAfter3 = gModByCustom; + var s = stringHexToSlice("010902FF"); + s.skipAny(); // custom loader also called + return (c1.beginParse().skipBits(8).loadUint(8), r2.n, gAfter2, gAfter3, s.remainingBitsCount(), gModByCustom); +} + +@method_id(107) +fun test7() { + val t = (1, 2, 3) as Tensor3Skipping1; + __expect_type(t.toCell(), "Cell"); + return t.toCell().load(); +} + +fun main() { + __expect_type("" as TelegramString, "TelegramString"); +} + +/** +@testcase | 101 | | -1 24 5203 +@testcase | 102 | | 16 +@testcase | 103 | | 1 2 +@testcase | 104 | 55 | 10 +@testcase | 104 | 8 | 0 +@testcase | 104 | -5 | -1 +@testcase | 105 | | 123 1 +@testcase | 106 | | 6 2 255 0 8 9 +@testcase | 107 | | 1 0 3 + +@fif_codegen +""" + test3() PROC:<{ + x{0102} PUSHSLICE + 8 LDU + 8 LDI + DROP + }> +""" + */ diff --git a/tolk/CMakeLists.txt b/tolk/CMakeLists.txt index bc8a6daa0..033456e7f 100644 --- a/tolk/CMakeLists.txt +++ b/tolk/CMakeLists.txt @@ -20,10 +20,10 @@ set(TOLK_SOURCE pipe-refine-lvalue-for-mutate.cpp pipe-check-rvalue-lvalue.cpp pipe-check-pure-impure.cpp - pipe-check-serialized-fields.cpp pipe-constant-folding.cpp pipe-optimize-boolean-expr.cpp pipe-detect-inline-in-place.cpp + pipe-check-serialized-fields.cpp pipe-lazy-load-insertions.cpp pipe-transform-on-message.cpp pipe-ast-to-legacy.cpp diff --git a/tolk/pack-unpack-api.cpp b/tolk/pack-unpack-api.cpp index 083537710..abab637e9 100644 --- a/tolk/pack-unpack-api.cpp +++ b/tolk/pack-unpack-api.cpp @@ -53,6 +53,26 @@ struct CantSerializeBecause { class PackUnpackAvailabilityChecker { std::vector already_checked; + static bool check_declared_packToBuilder(TypePtr receiver_type, FunctionPtr f_pack) { + if (!f_pack->does_accept_self() || f_pack->does_mutate_self() || f_pack->get_num_params() != 2) { + return false; + } + if (f_pack->get_param(1).declared_type != TypeDataBuilder::create() || !f_pack->has_mutate_params()) { + return false; + } + return f_pack->inferred_return_type->get_width_on_stack() == 0; + } + + static bool check_declared_unpackFromSlice(TypePtr receiver_type, FunctionPtr f_unpack) { + if (f_unpack->does_accept_self() || f_unpack->get_num_params() != 1) { + return false; + } + if (f_unpack->get_param(0).declared_type != TypeDataSlice::create() || !f_unpack->has_mutate_params()) { + return false; + } + return f_unpack->inferred_return_type->equal_to(receiver_type); + } + public: std::optional detect_why_cant_serialize(TypePtr any_type, bool is_pack) { if (any_type->try_as()) { @@ -137,6 +157,26 @@ class PackUnpackAvailabilityChecker { if (t_alias->alias_ref->name == "RemainingBitsAndRefs") { // it's built-in RemainingBitsAndRefs (slice) return {}; } + if (FunctionPtr f_pack = get_custom_pack_unpack_function(t_alias, true)) { + std::string receiver_name = t_alias->alias_ref->as_human_readable(); + if (!check_declared_packToBuilder(t_alias, f_pack)) { + return CantSerializeBecause("because `" + receiver_name + ".packToBuilder()` is declared incorrectly\nhint: it must accept 2 parameters and return nothing:\n> fun " + receiver_name + ".packToBuilder(self, mutate b: builder)"); + } + if (!f_pack->is_inlined_in_place()) { + return CantSerializeBecause("because `" + receiver_name + ".packToBuilder()` can't be inlined; probably, it contains `return` in the middle"); + } + if (FunctionPtr f_unpack = get_custom_pack_unpack_function(t_alias, false)) { + if (!check_declared_unpackFromSlice(t_alias, f_unpack)) { + return CantSerializeBecause("because `" + receiver_name + ".unpackFromSlice()` is declared incorrectly\nhint: it must accept 1 parameter and return an object:\n> fun " + receiver_name + ".unpackFromSlice(mutate s: slice): " + receiver_name); + } + if (!f_unpack->is_inlined_in_place()) { + return CantSerializeBecause("because `" + receiver_name + ".unpackFromSlice()` can't be inlined; probably, it contains `return` in the middle"); + } + } else if (!is_pack) { + return CantSerializeBecause("because type `" + receiver_name + "` defines a custom pack function, but does not define unpack\nhint: declare unpacker like this:\n> fun " + receiver_name + ".unpackFromSlice(mutate s: slice): " + receiver_name); + } + return {}; + } if (auto why = detect_why_cant_serialize(t_alias->underlying_type, is_pack)) { return CantSerializeBecause("because alias `" + t_alias->as_human_readable() + "` expands to `" + t_alias->underlying_type->as_human_readable() + "`", why.value()); } diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index f38f5d211..ecad40502 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -40,6 +40,7 @@ class LValContext; std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr target_type = nullptr, LValContext* lval_ctx = nullptr); std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, TypePtr cmp_type, const std::vector& expr_ir_idx, SrcLocation loc, const char* debug_desc); std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc); +std::vector gen_inline_fun_call_in_place(CodeBlob& code, TypePtr ret_type, SrcLocation loc, FunctionPtr f_inlined, AnyExprV self_obj, bool is_before_immediate_return, const std::vector>& vars_per_arg); bool is_type_cellT(TypePtr any_type) { if (const TypeDataStruct* t_struct = any_type->try_as()) { @@ -49,6 +50,24 @@ bool is_type_cellT(TypePtr any_type) { return false; } +// For any type alias, one can declare custom pack/unpack functions: +// > type TelegramString = slice +// > fun TelegramString.packToBuilder(self, mutate b: builder) { ... } +// > fun TelegramString.unpackFromSlice(mutate s: slice): TelegramString { ... } +// It's externally checked in advance that it's declared correctly. +FunctionPtr get_custom_pack_unpack_function(TypePtr receiver_type, bool is_pack) { + if (const TypeDataAlias* t_alias = receiver_type->try_as()) { + if (t_alias->alias_ref->is_instantiation_of_generic_alias()) { + // does not work for generic aliases currently, because `MyAlias.pack` was not instantiated earlier + return nullptr; + } + std::string receiver_name = t_alias->alias_ref->as_human_readable(); + if (const Symbol* sym = lookup_global_symbol(receiver_name + (is_pack ? ".packToBuilder" : ".unpackFromSlice"))) { + return sym->try_as(); + } + } + return nullptr; +} // -------------------------------------------- // options, context, common helpers @@ -1037,6 +1056,39 @@ struct S_CustomStruct final : ISerializer { } }; +struct S_CustomReceiverForPackUnpack final : ISerializer { + TypePtr receiver_type; + + explicit S_CustomReceiverForPackUnpack(TypePtr receiver_type) + : receiver_type(receiver_type) {} + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + FunctionPtr f_pack = get_custom_pack_unpack_function(receiver_type, true); + tolk_assert(f_pack && f_pack->does_accept_self() && f_pack->inferred_return_type->get_width_on_stack() == 0); + std::vector vars_per_arg = { std::move(rvect), ctx->ir_builder }; + std::vector ir_mutated_builder = gen_inline_fun_call_in_place(code, TypeDataBuilder::create(), loc, f_pack, nullptr, false, vars_per_arg); + code.emplace_back(loc, Op::_Let, ctx->ir_builder, std::move(ir_mutated_builder)); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_unpack = get_custom_pack_unpack_function(receiver_type, false); + tolk_assert(f_unpack && f_unpack->inferred_return_type->get_width_on_stack() == receiver_type->get_width_on_stack()); + TypePtr ret_type = TypeDataTensor::create({TypeDataSlice::create(), receiver_type}); + std::vector ir_slice_and_res = gen_inline_fun_call_in_place(code, ret_type, loc, f_unpack, nullptr, false, {ctx->ir_slice}); + code.emplace_back(loc, Op::_Let, ctx->ir_slice, std::vector{ir_slice_and_res.front()}); + return std::vector(ir_slice_and_res.begin() + 1, ir_slice_and_res.end()); + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + // just load and ignore the result + unpack(ctx, code, loc); + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize::unpredictable_infinity(); + } +}; + // -------------------------------------------- // automatically generate opcodes @@ -1192,6 +1244,9 @@ static std::unique_ptr get_serializer_for_type(TypePtr any_type) { if (t_alias->alias_ref->name == "RemainingBitsAndRefs") { return std::make_unique(); } + if (get_custom_pack_unpack_function(t_alias, true)) { + return std::make_unique(t_alias); + } return get_serializer_for_type(t_alias->underlying_type); } diff --git a/tolk/pack-unpack-serializers.h b/tolk/pack-unpack-serializers.h index b3427953b..ab11733aa 100644 --- a/tolk/pack-unpack-serializers.h +++ b/tolk/pack-unpack-serializers.h @@ -163,6 +163,7 @@ class EstimateContext { bool is_type_cellT(TypePtr any_type); +FunctionPtr get_custom_pack_unpack_function(TypePtr receiver_type, bool is_pack); std::vector auto_generate_opcodes_for_union(TypePtr union_type, std::string& because_msg); } // namespace tolk diff --git a/tolk/pipe-check-serialized-fields.cpp b/tolk/pipe-check-serialized-fields.cpp index a13032e37..f8b9b9cd0 100644 --- a/tolk/pipe-check-serialized-fields.cpp +++ b/tolk/pipe-check-serialized-fields.cpp @@ -82,10 +82,10 @@ class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody TypePtr serialized_type = nullptr; bool is_pack = false; if (f_name == "Cell.load" || f_name == "T.fromSlice" || f_name == "T.fromCell" || f_name == "T.toCell" || - f_name == "T.loadAny" || f_name == "slice.skipAny" || f_name == "slice.storeAny" || f_name == "T.estimatePackSize" || + f_name == "T.loadAny" || f_name == "slice.skipAny" || f_name == "slice.loadAny" || f_name == "builder.storeAny" || f_name == "T.estimatePackSize" || f_name == "createMessage" || f_name == "createExternalLogMessage") { serialized_type = fun_ref->substitutedTs->typeT_at(0); - is_pack = f_name == "T.toCell" || f_name == "slice.storeAny" || f_name == "T.estimatePackSize" || f_name == "createMessage" || f_name == "createExternalLogMessage"; + is_pack = f_name == "T.toCell" || f_name == "builder.storeAny" || f_name == "T.estimatePackSize" || f_name == "createMessage" || f_name == "createExternalLogMessage"; } else { return; // not a serialization function } @@ -111,7 +111,7 @@ class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody }; void pipeline_check_serialized_fields() { - visit_ast_of_all_functions(); + visit_ast_of_all_functions(); } } // namespace tolk diff --git a/tolk/pipe-detect-inline-in-place.cpp b/tolk/pipe-detect-inline-in-place.cpp index 4221347f9..7e892b687 100644 --- a/tolk/pipe-detect-inline-in-place.cpp +++ b/tolk/pipe-detect-inline-in-place.cpp @@ -51,6 +51,19 @@ namespace tolk { // (purpose: functions in recursive call chains can't be inlined) static std::unordered_map> call_graph; +static bool is_called_implicitly_by_compiler(FunctionPtr f) { + if (f->name == "onBouncedMessage") { + return true; + } + if (f->is_method() && f->method_name == "packToBuilder") { + return f->does_accept_self() && !f->does_mutate_self() && f->get_num_params() == 2 && f->has_mutate_params(); + } + if (f->is_method() && f->method_name == "unpackFromSlice") { + return !f->does_accept_self() && f->get_num_params() == 1 && f->has_mutate_params(); + } + return false; +} + // when traversing a function, collect some AST metrics used to detect whether it's lightweight struct StateWhileTraversingFunction { FunctionPtr fun_ref; @@ -227,8 +240,7 @@ class DetectIfToInlineFunctionInPlaceVisitor final : ASTVisitorFunctionBody { } // okay, this function will be inlined, mark the flag - bool is_called = fun_ref->n_times_called - || fun_ref->name == "onBouncedMessage"; // implicitly called by the compiler from onInternalMessage() + bool is_called = fun_ref->n_times_called || is_called_implicitly_by_compiler(fun_ref); if (will_inline && is_called) { fun_ref->mutate()->assign_inline_mode_in_place(); } diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 432802279..1253cdffc 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -1073,7 +1073,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // check for method (`t.size` / `user.getId`); even `i.0()` can be here if `fun int.0(self)` exists // for `T.copy` / `Container.create`, substitution for T is also returned if (!fun_ref) { - std::tie(fun_ref, substitutedTs) = choose_only_method_to_call(cur_f, dot_obj->loc, obj_type, field_name); + std::tie(fun_ref, substitutedTs) = choose_only_method_to_call(cur_f, dot_obj->loc, dot_obj->inferred_type, field_name); } // not a field, not a method — fire an error diff --git a/tolk/pipeline.h b/tolk/pipeline.h index eb1c153b4..641beb06b 100644 --- a/tolk/pipeline.h +++ b/tolk/pipeline.h @@ -41,10 +41,10 @@ void pipeline_check_inferred_types(); void pipeline_refine_lvalue_for_mutate_arguments(); void pipeline_check_rvalue_lvalue(); void pipeline_check_pure_impure_operations(); -void pipeline_check_serialized_fields(); void pipeline_constant_folding(); void pipeline_optimize_boolean_expressions(); void pipeline_detect_inline_in_place(); +void pipeline_check_serialized_fields(); void pipeline_lazy_load_insertions(); void pipeline_transform_onInternalMessage(); void pipeline_convert_ast_to_legacy_Expr_Op(); diff --git a/tolk/tolk.cpp b/tolk/tolk.cpp index 63a2dd7f2..69b03f20a 100644 --- a/tolk/tolk.cpp +++ b/tolk/tolk.cpp @@ -64,10 +64,10 @@ int tolk_proceed(const std::string &entrypoint_filename) { pipeline_refine_lvalue_for_mutate_arguments(); pipeline_check_rvalue_lvalue(); pipeline_check_pure_impure_operations(); - pipeline_check_serialized_fields(); pipeline_constant_folding(); pipeline_optimize_boolean_expressions(); pipeline_detect_inline_in_place(); + pipeline_check_serialized_fields(); pipeline_lazy_load_insertions(); pipeline_transform_onInternalMessage(); pipeline_convert_ast_to_legacy_Expr_Op(); From 63670d2449991b99be40bb5fb6119036e817770b Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Mon, 30 Jun 2025 20:52:55 +0300 Subject: [PATCH 338/388] [Tolk] Use of `AutoDeployAddress` for calculation aside createMessage New functions in stdlib: > AutoDeployAddress.buildAddress > AutoDeployAddress.addressMatches Based on the same philosophy as createMessage --- crypto/smartcont/tolk-stdlib/common.tolk | 28 +- tolk-tester/tests/build-addr-tests.tolk | 310 +++++++++++++++++++++++ tolk-tester/tests/parse-address.tolk | 91 ------- tolk/builtins.cpp | 13 + tolk/pipe-ast-to-legacy.cpp | 9 + tolk/send-message-api.cpp | 253 +++++++++++++++--- tolk/send-message-api.h | 3 + 7 files changed, 576 insertions(+), 131 deletions(-) create mode 100644 tolk-tester/tests/build-addr-tests.tolk diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 7cf61f562..da52bfb98 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -1056,17 +1056,39 @@ struct AddressShardingOptions { /// }, /// ... /// ``` -/// It's often called "deployment", but it's inaccurate. TON Blockchain doesn't have a dedicated -/// deployment mechanism. You just send a message to some address — and if it doesn't exist, but you've -/// attached a way to initialize it (code+data) — it's initialized immediately, and accepts your message. /// You just provide code+data, and the compiler automatically calculates the destination address, /// because in TON, the address of a contract, by definition, is a hash of its initial state. +/// You can also use it without sending a message. See [buildAddress] and [addressMatches]. struct AutoDeployAddress { workchain: int8 = BASECHAIN; stateInit: ContractState | cell; toShard: AddressShardingOptions? = null; } +/// Constructs an address that a deployed contract will have. +/// For example, from a jetton minter, you want to calculate an address of a jetton wallet: +/// ``` +/// val jwDeployed = calcDeployedJettonWallet(...); +/// val jwAddrBuilt = jwDeployed.buildAddress(); +/// ``` +/// Just instead of `dest` for [createMessage], you use it is to calculate a jetton/nft address. +/// Note: returns `builder`, not `address`! It's cheap. +/// If you really need `address`, use `address.fromValidBuilder(res)`. +@pure +fun AutoDeployAddress.buildAddress(self): builder + builtin; + +/// Checks that an address matches a deployed contract. +/// For example, from a jetton minter, you're checking whether a message is from a jetton wallet: +/// ``` +/// val jwDeployed = calcDeployedJettonWallet(...); +/// val fromJW = jwDeployed.addressMatches(senderAddress); +/// ``` +/// Just instead of `dest` for [createMessage], you use it is to check whether a sender is a jetton/nft. +@pure +fun AutoDeployAddress.addressMatches(self, addr: address): bool + builtin; + /// Options for creating an outgoing message. /// Consider [createMessage] for examples. struct CreateMessageOptions { diff --git a/tolk-tester/tests/build-addr-tests.tolk b/tolk-tester/tests/build-addr-tests.tolk new file mode 100644 index 000000000..788450a55 --- /dev/null +++ b/tolk-tester/tests/build-addr-tests.tolk @@ -0,0 +1,310 @@ +const SHARD_DEPTH = 8; + +fun getMyAddressDev(): address + asm "x{80194DC6438F99D3D9DBE151944925D90B2492954BF6B9C070FBFF2DDED5F30547D_} PUSHSLICE"; + +struct WalletStorage { + balance: coins; + ownerAddress: address; + minterAddress: address; +} + +fun WalletStorage.generateEmptyData(ownerAddress: address, minterAddress: address) { + val emptyWalletStorage: WalletStorage = { + balance: 0, + ownerAddress, + minterAddress, + }; + return emptyWalletStorage.toCell(); +} + +@inline_ref +fun buildAddrInShard_manual(a: address, options: AddressShardingOptions) { + var sb = options.closeTo as slice; + sb.skipBits(3); // addr_std$10 + anycast 0 + val wc_b = sb.loadInt(8); + val shardPrefix = sb.loadUint(options.fixedPrefixLength); + + var sa = a as slice; + sa.skipBits(3 + 8 + options.fixedPrefixLength); + + return beginCell() + .storeUint(0b100, 3) // addr_std$10 + anycast 0 + .storeInt(wc_b, 8) + .storeUint(shardPrefix, options.fixedPrefixLength) + .storeSlice(sa); +} + +@method_id(101) +fun test1() { + val a = address("0:00000000000000000000000000000000000000000000000000000000000000FF"); + val b = address("1:1100000000000000000000000000000000000000000000000000000000000000"); + val dd1 = buildAddrInShard_manual(a, {fixedPrefixLength: 8, closeTo: b}); + val dd2 = a.buildSameAddressInAnotherShard({fixedPrefixLength: 8, closeTo: b}); + assert(dd1.endCell().hash() == dd2.endCell().hash(), 400); + return dd1.endCell().beginParse() as address + == address("1:11000000000000000000000000000000000000000000000000000000000000FF"); +} + +@method_id(102) +fun test2() { + val a = address("0:1234aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + val b = address("0:FFFFbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); + val dd1 = buildAddrInShard_manual(a, {closeTo: b, fixedPrefixLength: 16}); + val dd2 = a.buildSameAddressInAnotherShard({fixedPrefixLength: 16, closeTo: b}); + assert((a as slice).remainingBitsCount() == 267, 267); + assert((b as slice).remainingBitsCount() == 267, 267); + assert(dd1.endCell().hash() == dd2.endCell().hash(), 400); + return address.fromValidBuilder(dd2) + == address("0:FFFFaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); +} + +@method_id(103) +fun test3() { + var t1_a = address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"); + var t1_b = address("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"); + var t2_a = address("EQCe4AYIBce1pAk2qJJPSs1OzyZRlKjkfq8zuC8D7erv6DUP"); + var t2_b = address("EQCtrtTXEAoSpoERmiqOnICe9LHxn2N89N4BH9qdHlrG-U0i"); + var t3_a = address("EQAUTbQiM522Y_XJ_T98QPhPhTmb4nV--VSPiha8kC6kRfPO"); + var t3_b = address("EQBlqsm144Dq6SjbPI4jjZvA1hqTIP3CvHovbIfW_t-SCALE"); + return ( + t1_a.buildSameAddressInAnotherShard({fixedPrefixLength: 8, closeTo: t1_b}).endCell().hash() & 0xFFFF, + t1_a.buildSameAddressInAnotherShard({fixedPrefixLength: 4, closeTo: t1_b}).endCell().hash() & 0xFFFF, + t2_a.buildSameAddressInAnotherShard({fixedPrefixLength: 2, closeTo: t2_b}).endCell().hash() & 0xFFFF, + t3_a.buildSameAddressInAnotherShard({fixedPrefixLength: 30, closeTo: t2_b}).endCell().hash() & 0xFFFF, + t2_a.buildSameAddressInAnotherShard({fixedPrefixLength: 1, closeTo: t3_b}).endCell().hash() & 0xFFFF, + t2_a.buildSameAddressInAnotherShard({fixedPrefixLength: 0, closeTo: t3_b}).endCell().hash() & 0xFFFF, + ) +} + +@method_id(104) +fun test4(shardDepth: int) { + var a = address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"); + var b = address("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"); + return a.buildSameAddressInAnotherShard({fixedPrefixLength: shardDepth, closeTo: b}).endCell().hash() & 0xFFFF; +} + +@method_id(105) +fun test5() { + var b = beginCell() + .storeUint(0b100, 3) // std addr no anycast + .storeInt(MASTERCHAIN, 8) + .storeUint(0xFFFF, 256); + val a = address("-1:000000000000000000000000000000000000000000000000000000000000FFFF"); + return address.fromValidBuilder(b) == a; +} + +@method_id(106) +fun test6() { + val a = address("0:00000000000000000000000000000000000000000000000000000000000000FF"); + val b = address("1:1100000000000000000000000000000000000000000000000000000000000000"); + return address.fromValidBuilder(a.buildSameAddressInAnotherShard({fixedPrefixLength: 8, closeTo: b})) == + address("1:11000000000000000000000000000000000000000000000000000000000000FF"); +} + + +fun manual_buildAddressOfJettonWallet_plain(ownerAddress: address, minterAddress: address, jettonWalletCode: cell): builder { + val stateInitHash = StateInit.calcHashCodeData( + jettonWalletCode, + WalletStorage.generateEmptyData(ownerAddress, minterAddress) + ); + return beginCell() + .storeUint(0b100, 3) + .storeUint(BASECHAIN, 8) + .storeUint(stateInitHash, 256) +} + +fun manual_buildAddressOfJettonWallet_sharded(ownerAddress: address, minterAddress: address, jettonWalletCode: cell): builder { + val stateInitHash = StateInit.calcHashPrefixCodeData( + SHARD_DEPTH, + jettonWalletCode, + WalletStorage.generateEmptyData(ownerAddress, minterAddress) + ); + var mask = stateInitHash & ((1 << (256 - SHARD_DEPTH)) - 1); + val shard_prefix = (ownerAddress as slice).getMiddleBits(3 + 8, SHARD_DEPTH); + return beginCell() + .storeUint(0b100, 3) + .storeUint(BASECHAIN, 8) + .storeSlice(shard_prefix) + .storeUint( mask, 256 - SHARD_DEPTH); +} + +fun address.manual_isAddressOfJettonWallet_plain(self, ownerAddress: address, minterAddress: address, jettonWalletCode: cell) { + val stateInitHash = StateInit.calcHashCodeData( + jettonWalletCode, + WalletStorage.generateEmptyData(ownerAddress, minterAddress) + ); + val (wc, hash) = self.getWorkchainAndHash(); + return (stateInitHash == hash) & (BASECHAIN == wc); +} + +fun address.manual_isAddressOfJettonWallet_sharded(self, ownerAddress: address, minterAddress: address, jettonWalletCode: cell) { + var stateInitHash = StateInit.calcHashPrefixCodeData( + SHARD_DEPTH, + jettonWalletCode, + WalletStorage.generateEmptyData(ownerAddress, minterAddress) + ); + val mask = (1 << (256 - SHARD_DEPTH)) - 1; + stateInitHash = stateInitHash & mask; + var (wc, hash) = self.getWorkchainAndHash(); + hash = hash & mask; + return (stateInitHash == hash) & (BASECHAIN == wc); +} + +@method_id(110) +fun test10() { + val jwCode: cell = beginCell().storeInt(0x273849723892, 94).endCell(); + val ownerAddress = address("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"); + val minterAddress = getMyAddressDev(); + + val b1 = manual_buildAddressOfJettonWallet_plain(ownerAddress, minterAddress, jwCode); + val si2: AutoDeployAddress = { + stateInit: { code: jwCode, data: WalletStorage.generateEmptyData(ownerAddress, minterAddress) } + }; + val b2 = si2.buildAddress(); + assert (b1.endCell().hash() == b2.endCell().hash()) throw 123; + + val checkedAddr = b1.endCell().beginParse() as address; + assert (checkedAddr.manual_isAddressOfJettonWallet_plain(ownerAddress, minterAddress, jwCode)) throw 123; + assert (si2.addressMatches(checkedAddr)) throw 123; + + return ( + si2.addressMatches(ownerAddress), + AutoDeployAddress { + workchain: -1, + stateInit: { code: jwCode, data: WalletStorage.generateEmptyData(ownerAddress, minterAddress) } + }.addressMatches(checkedAddr) + ) +} + +@method_id(111) +fun test11() { + val jwCode: cell = beginCell().storeInt(0x273849723892, 94).endCell(); + val ownerAddress = address("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"); + val minterAddress = getMyAddressDev(); + + val b1 = manual_buildAddressOfJettonWallet_sharded(ownerAddress, minterAddress, jwCode); + val si2: AutoDeployAddress = { + stateInit: { code: jwCode, data: WalletStorage.generateEmptyData(ownerAddress, minterAddress) }, + toShard: { fixedPrefixLength: SHARD_DEPTH, closeTo: ownerAddress } + }; + val b2 = si2.buildAddress(); + assert (b1.endCell().hash() == b2.endCell().hash()) throw 123; + + val checkedAddr = b1.endCell().beginParse() as address; + assert (checkedAddr.manual_isAddressOfJettonWallet_sharded(ownerAddress, minterAddress, jwCode)) throw 123; + assert (si2.addressMatches(checkedAddr)) throw 123; + + return ( + AutoDeployAddress { + stateInit: { code: jwCode, data: WalletStorage.generateEmptyData(ownerAddress, minterAddress) }, + }.addressMatches(checkedAddr), + si2.addressMatches(ownerAddress), + AutoDeployAddress { + workchain: -1, + stateInit: { code: jwCode, data: WalletStorage.generateEmptyData(ownerAddress, minterAddress) }, + toShard: { fixedPrefixLength: SHARD_DEPTH, closeTo: ownerAddress } + }.addressMatches(checkedAddr) + ) +} + +@method_id(112) +fun test12() { + val jwCode: cell = beginCell().storeInt(0x273849723892, 94).endCell(); + val ownerAddress = address("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"); + var i = 1; + while (i < 10) { + var params: AutoDeployAddress = { + workchain: -1, + stateInit: { code: jwCode, data: createEmptyCell() }, + toShard: { fixedPrefixLength: i, closeTo: ownerAddress } + }; + val addrBuilt = params.buildAddress(); + val addr = address.fromValidBuilder(addrBuilt); + assert (params.addressMatches(addr)) throw 123; + params.workchain = 0; + assert (!params.addressMatches(addr)) throw 123; + params.workchain = -1; + params.toShard!.fixedPrefixLength += 1; + assert (!params.addressMatches(addr)) throw 123; + i += 1 + } + return i; +} + +@method_id(113) +fun test13() { + val stateInitCell = StateInit { + fixedPrefixLength: null, + code: createEmptyCell(), + data: beginCell().storeUint(123, 32).endCell(), + special: null, + library: null, + }.toCell(); + + val manualBuilt = beginCell().storeUint(0b100, 3).storeUint(0, 8) + .storeUint(stateInitCell.hash(), 256); + val si: AutoDeployAddress = { + stateInit: stateInitCell + }; + val addrBuilt = si.buildAddress(); + assert (manualBuilt.endCell().hash() == addrBuilt.endCell().hash()) throw 123; + + val addr = addrBuilt.endCell().beginParse() as address; + return ( + si.addressMatches(addr), + AutoDeployAddress {workchain: 99, stateInit: stateInitCell}.addressMatches(addr), + ) +} + +@method_id(114) +fun test14() { + val ownerAddress = address("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"); + val stateInitCell = StateInit { + fixedPrefixLength: 10, + code: createEmptyCell(), + data: beginCell().storeUint(123, 32).endCell(), + special: null, + library: null, + }.toCell(); + + val manualBuilt = beginCell().storeUint(0b100, 3).storeUint(9, 8) + .storeSlice((ownerAddress as slice).getMiddleBits(3+8, 10)) + .storeUint(stateInitCell.hash() & ((1 << 246) - 1), 246); + val si: AutoDeployAddress = { + workchain: 9, + stateInit: stateInitCell, + toShard: { fixedPrefixLength: 10, closeTo: ownerAddress } + }; + val addrBuilt = si.buildAddress(); + val addr = addrBuilt.endCell().beginParse() as address; + + val stateInitHash = stateInitCell.hash(); + val (wc, hash) = addr.getWorkchainAndHash(); + val manual_isAddrOfContract = ((stateInitHash & ((1 << 246) - 1)) == (hash & ((1 << 246) - 1))) & (9 == wc); + + return ( + manualBuilt.endCell().hash() == addrBuilt.endCell().hash(), + manual_isAddrOfContract, + si.addressMatches(addr), + ) +} + + + +fun main() {} + +/** +@testcase | 101 | | -1 +@testcase | 102 | | -1 +@testcase | 103 | | 39876 24338 15241 50719 11252 15241 +@testcase | 104 | 2 | 24338 +@testcase | 104 | 9 | 39876 +@testcase | 105 | | -1 +@testcase | 106 | | -1 +@testcase | 110 | | 0 0 +@testcase | 111 | | 0 0 0 +@testcase | 112 | | 10 +@testcase | 113 | | -1 0 +@testcase | 114 | | -1 -1 -1 + */ diff --git a/tolk-tester/tests/parse-address.tolk b/tolk-tester/tests/parse-address.tolk index c489527ad..0079871a6 100644 --- a/tolk-tester/tests/parse-address.tolk +++ b/tolk-tester/tests/parse-address.tolk @@ -36,90 +36,6 @@ fun codegenAddrEq(a: address, b: address) { return 3; } -@inline_ref -fun buildAddrInShard_manual(a: address, options: AddressShardingOptions) { - var sb = options.closeTo as slice; - sb.skipBits(3); // addr_std$10 + anycast 0 - val wc_b = sb.loadInt(8); - val shardPrefix = sb.loadUint(options.fixedPrefixLength); - - var sa = a as slice; - sa.skipBits(3 + 8 + options.fixedPrefixLength); - - return beginCell() - .storeUint(0b100, 3) // addr_std$10 + anycast 0 - .storeInt(wc_b, 8) - .storeUint(shardPrefix, options.fixedPrefixLength) - .storeSlice(sa); -} - -@method_id(101) -fun test1() { - val a = address("0:00000000000000000000000000000000000000000000000000000000000000FF"); - val b = address("1:1100000000000000000000000000000000000000000000000000000000000000"); - val dd1 = buildAddrInShard_manual(a, {fixedPrefixLength: 8, closeTo: b}); - val dd2 = a.buildSameAddressInAnotherShard({fixedPrefixLength: 8, closeTo: b}); - assert(dd1.endCell().hash() == dd2.endCell().hash(), 400); - return dd1.endCell().beginParse() as address - == address("1:11000000000000000000000000000000000000000000000000000000000000FF"); -} - -@method_id(102) -fun test2() { - val a = address("0:1234aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); - val b = address("0:FFFFbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); - val dd1 = buildAddrInShard_manual(a, {closeTo: b, fixedPrefixLength: 16}); - val dd2 = a.buildSameAddressInAnotherShard({fixedPrefixLength: 16, closeTo: b}); - assert((a as slice).remainingBitsCount() == 267, 267); - assert((b as slice).remainingBitsCount() == 267, 267); - assert(dd1.endCell().hash() == dd2.endCell().hash(), 400); - return address.fromValidBuilder(dd2) - == address("0:FFFFaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); -} - -@method_id(103) -fun test3() { - var t1_a = address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"); - var t1_b = address("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"); - var t2_a = address("EQCe4AYIBce1pAk2qJJPSs1OzyZRlKjkfq8zuC8D7erv6DUP"); - var t2_b = address("EQCtrtTXEAoSpoERmiqOnICe9LHxn2N89N4BH9qdHlrG-U0i"); - var t3_a = address("EQAUTbQiM522Y_XJ_T98QPhPhTmb4nV--VSPiha8kC6kRfPO"); - var t3_b = address("EQBlqsm144Dq6SjbPI4jjZvA1hqTIP3CvHovbIfW_t-SCALE"); - return ( - t1_a.buildSameAddressInAnotherShard({fixedPrefixLength: 8, closeTo: t1_b}).endCell().hash() & 0xFFFF, - t1_a.buildSameAddressInAnotherShard({fixedPrefixLength: 4, closeTo: t1_b}).endCell().hash() & 0xFFFF, - t2_a.buildSameAddressInAnotherShard({fixedPrefixLength: 2, closeTo: t2_b}).endCell().hash() & 0xFFFF, - t3_a.buildSameAddressInAnotherShard({fixedPrefixLength: 30, closeTo: t2_b}).endCell().hash() & 0xFFFF, - t2_a.buildSameAddressInAnotherShard({fixedPrefixLength: 1, closeTo: t3_b}).endCell().hash() & 0xFFFF, - t2_a.buildSameAddressInAnotherShard({fixedPrefixLength: 0, closeTo: t3_b}).endCell().hash() & 0xFFFF, - ) -} - -@method_id(104) -fun test4(shardDepth: int) { - var a = address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"); - var b = address("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"); - return a.buildSameAddressInAnotherShard({fixedPrefixLength: shardDepth, closeTo: b}).endCell().hash() & 0xFFFF; -} - -@method_id(105) -fun test5() { - var b = beginCell() - .storeUint(0b100, 3) // std addr no anycast - .storeInt(MASTERCHAIN, 8) - .storeUint(0xFFFF, 256); - val a = address("-1:000000000000000000000000000000000000000000000000000000000000FFFF"); - return address.fromValidBuilder(b) == a; -} - -@method_id(106) -fun test6() { - val a = address("0:00000000000000000000000000000000000000000000000000000000000000FF"); - val b = address("1:1100000000000000000000000000000000000000000000000000000000000000"); - return address.fromValidBuilder(a.buildSameAddressInAnotherShard({fixedPrefixLength: 8, closeTo: b})) == - address("1:11000000000000000000000000000000000000000000000000000000000000FF"); -} - fun main() { __expect_type(cc1, "address"); @@ -231,13 +147,6 @@ fun main() { /** @testcase | 0 | | -1 0 -1 0 -1 -@testcase | 101 | | -1 -@testcase | 102 | | -1 -@testcase | 103 | | 39876 24338 15241 50719 11252 15241 -@testcase | 104 | 2 | 24338 -@testcase | 104 | 9 | 39876 -@testcase | 105 | | -1 -@testcase | 106 | | -1 @fif_codegen """ diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index cce907beb..22d743753 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1307,6 +1307,7 @@ void define_builtins() { TypePtr CreateExternalLogMessageOptions = TypeDataUnknown::create(); TypePtr OutMessage = TypeDataUnknown::create(); TypePtr AddressShardingOptions = TypeDataUnknown::create(); + TypePtr AutoDeployAddress = TypeDataUnknown::create(); const GenericsDeclaration* declTBody = new GenericsDeclaration(std::vector{{"TBody", nullptr}}, 0); // builtin operators @@ -1582,6 +1583,12 @@ void define_builtins() { define_builtin_func("createExternalLogMessage", {CreateExternalLogMessageOptions}, OutMessage, declTBody, compile_time_only_function, FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); + define_builtin_method("AutoDeployAddress.buildAddress", AutoDeployAddress, {AutoDeployAddress}, Builder, nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagCompileTimeGen); + define_builtin_method("AutoDeployAddress.addressMatches", AutoDeployAddress, {AutoDeployAddress, Address}, Bool, nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagCompileTimeGen); // functions not presented in stdlib at all // used in tolk-tester to check/expose internal compiler state @@ -1614,9 +1621,15 @@ void patch_builtins_after_stdlib_loaded() { lookup_function("debug.dumpStack")->mutate()->receiver_type = debug; StructPtr struct_ref_AddressShardingOptions = lookup_global_symbol("AddressShardingOptions")->try_as(); + StructPtr struct_ref_AutoDeployAddress = lookup_global_symbol("AutoDeployAddress")->try_as(); TypePtr AddressShardingOptions = TypeDataStruct::create(struct_ref_AddressShardingOptions); + TypePtr AutoDeployAddress = TypeDataStruct::create(struct_ref_AutoDeployAddress); lookup_function("address.buildSameAddressInAnotherShard")->mutate()->parameters[1].declared_type = AddressShardingOptions; + lookup_function("AutoDeployAddress.buildAddress")->mutate()->receiver_type = AutoDeployAddress; + lookup_function("AutoDeployAddress.buildAddress")->mutate()->parameters[0].declared_type = AutoDeployAddress; + lookup_function("AutoDeployAddress.addressMatches")->mutate()->receiver_type = AutoDeployAddress; + lookup_function("AutoDeployAddress.addressMatches")->mutate()->parameters[0].declared_type = AutoDeployAddress; StructPtr struct_ref_CellT = lookup_global_symbol("Cell")->try_as(); StructPtr struct_ref_PackOptions = lookup_global_symbol("PackOptions")->try_as(); diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 38a7c492a..3f525287b 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -678,6 +678,15 @@ static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob std::vector ir_shard_options = vars_per_arg[1]; return generate_address_buildInAnotherShard(code, loc, std::move(ir_self_address), std::move(ir_shard_options)); } + if (called_f->name == "AutoDeployAddress.buildAddress") { + std::vector ir_self = vars_per_arg[0]; + return generate_AutoDeployAddress_buildAddress(code, loc, std::move(ir_self)); + } + if (called_f->name == "AutoDeployAddress.addressMatches") { + std::vector ir_self = vars_per_arg[0]; + std::vector ir_address = vars_per_arg[1]; + return generate_AutoDeployAddress_addressMatches(code, loc, std::move(ir_self), std::move(ir_address)); + } tolk_assert(false); } diff --git a/tolk/send-message-api.cpp b/tolk/send-message-api.cpp index 6d0b53524..2b4d80be8 100644 --- a/tolk/send-message-api.cpp +++ b/tolk/send-message-api.cpp @@ -43,6 +43,49 @@ static std::vector create_default_PackOptions(CodeBlob& code, SrcLoca return ir_options; } +// calculate `addrHash &= mask` where mask = `(1 << (256 - SHARD_DEPTH)) - 1` +static void append_bitwise_and_shard_mask(CodeBlob& code, SrcLocation loc, var_idx_t ir_addr_hash, var_idx_t ir_shard_depth) { + var_idx_t ir_one = code.create_int(loc, 1, "(one)"); + std::vector ir_mask = code.create_tmp_var(TypeDataInt::create(), loc, "(mask)"); + code.emplace_back(loc, Op::_Call, ir_mask, std::vector{code.create_int(loc, 256, ""), ir_shard_depth}, lookup_function("_-_")); + code.emplace_back(loc, Op::_Call, ir_mask, std::vector{ir_one, ir_mask[0]}, lookup_function("_<<_")); + code.emplace_back(loc, Op::_Call, ir_mask, std::vector{ir_mask[0], ir_one}, lookup_function("_-_")); + code.emplace_back(loc, Op::_Call, std::vector{ir_addr_hash}, std::vector{ir_addr_hash, ir_mask[0]}, lookup_function("_&_")); +} + +// struct AutoDeployAddress { workchain: int8; stateInit: ContractState | cell; toShard: AddressShardingOptions?; } +struct IR_AutoDeployAddress { + std::vector is_ContractState, // stateInit is ContractState + is_AddressSharding; // toShard is not null + var_idx_t workchain, // workchain + stateInitCode, stateInitData, stateInitCell, // stateInit + ir_shardDepth, ir_closeTo; // toShard + + IR_AutoDeployAddress(CodeBlob& code, SrcLocation loc, const std::vector& ir_vars) { + StructPtr s_AutoDeployAddress = lookup_global_symbol("AutoDeployAddress")->try_as(); + const TypeDataUnion* t_stateInit = s_AutoDeployAddress->find_field("stateInit")->declared_type->try_as(); + const TypeDataUnion* t_toShard = s_AutoDeployAddress->find_field("toShard")->declared_type->try_as(); + tolk_assert(ir_vars.size() == 1 + 3 + 3); + tolk_assert(t_stateInit && t_stateInit->get_width_on_stack() == (2+1) && t_stateInit->size() == 2); + tolk_assert(t_toShard && t_toShard->get_width_on_stack() == (2+1) && t_toShard->or_null); + + workchain = ir_vars[0]; + + std::vector ir_stateInitUnion(ir_vars.begin() + 1, ir_vars.begin() + 1 + 3); + is_ContractState = pre_compile_is_type(code, t_stateInit, t_stateInit->variants[0], ir_stateInitUnion, loc, "(is-ContractState)"); + std::vector ir_ContractState = transition_to_target_type(std::vector(ir_stateInitUnion), code, t_stateInit, t_stateInit->variants[0], loc); + stateInitCode = ir_ContractState[0]; + stateInitData = ir_ContractState[1]; + stateInitCell = transition_to_target_type(std::vector(ir_stateInitUnion), code, t_stateInit, t_stateInit->variants[1], loc)[0]; + + std::vector ir_toShardOrNull(ir_vars.begin() + 1 + 3, ir_vars.begin() + 1 + 3 + 3); + is_AddressSharding = pre_compile_is_type(code, t_toShard, t_toShard->or_null, ir_toShardOrNull, loc, "(is-AddressSharding)"); + std::vector ir_AddressSharding = transition_to_target_type(std::vector(ir_toShardOrNull), code, t_toShard, t_toShard->or_null, loc); + ir_shardDepth = ir_AddressSharding[0]; + ir_closeTo = ir_AddressSharding[1]; + } +}; + std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, TypePtr bodyT, std::vector&& rvect) { StructPtr s_Options = lookup_global_symbol("CreateMessageOptions")->try_as(); StructPtr s_AutoDeployAddress = lookup_global_symbol("AutoDeployAddress")->try_as(); @@ -50,13 +93,9 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T const TypeDataBool* t_bounce = s_Options->find_field("bounce")->declared_type->try_as(); const TypeDataUnion* t_dest = s_Options->find_field("dest")->declared_type->try_as(); const TypeDataUnion* t_value = s_Options->find_field("value")->declared_type->try_as(); - const TypeDataUnion* t_stateInit = s_AutoDeployAddress->find_field("stateInit")->declared_type->try_as(); - const TypeDataUnion* t_toShard = s_AutoDeployAddress->find_field("toShard")->declared_type->try_as(); tolk_assert(t_bounce); tolk_assert(t_dest && t_dest->get_width_on_stack() == (1+3+3+1) && t_dest->size() == 4); tolk_assert(t_value && t_value->get_width_on_stack() == (2+1) && t_value->size() == 2); - tolk_assert(t_stateInit && t_stateInit->get_width_on_stack() == (2+1) && t_stateInit->size() == 2); - tolk_assert(t_toShard && t_toShard->get_width_on_stack() == (2+1) && t_toShard->or_null); int offset = 0; auto next_slice = [&rvect, &offset](int width) -> std::vector { @@ -79,17 +118,7 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T std::vector ir_dest_is_AutoDeploy = pre_compile_is_type(code, t_dest, TypeDataStruct::create(s_AutoDeployAddress), ir_dest, loc, "(is-address)"); std::vector ir_dest_is_builder = pre_compile_is_type(code, t_dest, TypeDataBuilder::create(), ir_dest, loc, "(is-builder)"); std::vector ir_dest_AutoDeployAddress = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataStruct::create(s_AutoDeployAddress), loc); - // dest.workchain - std::vector ir_dest_workchain(ir_dest_AutoDeployAddress.begin(), ir_dest_AutoDeployAddress.begin() + 1); - // dest.stateInit (code+data or cell) - std::vector ir_dest_stateInitUnion(ir_dest_AutoDeployAddress.begin() + 1, ir_dest_AutoDeployAddress.begin() + 1 + 3); - std::vector ir_is_ContractState = pre_compile_is_type(code, t_stateInit, t_stateInit->variants[0], ir_dest_stateInitUnion, loc, "(is-ContractState)"); - std::vector ir_ContractState = transition_to_target_type(std::vector(ir_dest_stateInitUnion), code, t_stateInit, t_stateInit->variants[0], loc); - std::vector ir_StateInitCell = transition_to_target_type(std::vector(ir_dest_stateInitUnion), code, t_stateInit, t_stateInit->variants[1], loc); - // dest.toShard (shardPrefix+closeTo or null) - std::vector ir_dest_toShardOrNull(ir_dest_AutoDeployAddress.begin() + 1 + 3, ir_dest_AutoDeployAddress.begin() + 1 + 3 + 3); - std::vector ir_is_AddressSharding = pre_compile_is_type(code, t_toShard, t_toShard->or_null, ir_dest_toShardOrNull, loc, "(is-AddressSharding)"); - std::vector ir_AddressSharding = transition_to_target_type(std::vector(ir_dest_toShardOrNull), code, t_toShard, t_toShard->or_null, loc); + IR_AutoDeployAddress ir_dest_ad(code, loc, ir_dest_AutoDeployAddress); // currently, there is no way to pass PackOptions, defaults are used std::vector ir_options = create_default_PackOptions(code, loc); @@ -154,26 +183,27 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T // and, if toShard, take first D bits from dest.toShard.closeTo and mix with 256-D bits of hash code.push_set_cur(if_AutoDeploy.block0); ctx.storeUint(code.create_int(loc, 0b100, "(addr-prefix)"), 3); // addr_std$10 + 0 anycast - ctx.storeInt(ir_dest_workchain[0], 8); + ctx.storeInt(ir_dest_ad.workchain, 8); std::vector ir_hash = code.create_tmp_var(TypeDataInt::create(), loc, "(addr-hash)"); - Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_is_ContractState); + Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_dest_ad.is_ContractState); { // input is `dest: { ... stateInit: { code, data } }` code.push_set_cur(if_ContractState.block0); - Op& if_sharded = code.emplace_back(loc, Op::_If, ir_is_AddressSharding); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_dest_ad.is_AddressSharding); { // input is `dest: { ... stateInit: { code, data }, toShard: { fixedPrefixLength, closeTo } }; // then stateInitHash = (hash of StateInit = 0b1(depth)0110 (prefix + code + data)) code.push_set_cur(if_sharded.block0); - std::vector ir_args_depth_code_data = { ir_AddressSharding[0], ir_ContractState[0], ir_ContractState[1] }; - code.emplace_back(loc, Op::_Call, ir_hash, std::move(ir_args_depth_code_data), lookup_function("StateInit.calcHashPrefixCodeData")); + std::vector args = { ir_dest_ad.ir_shardDepth, ir_dest_ad.stateInitCode, ir_dest_ad.stateInitData }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashPrefixCodeData")); code.close_pop_cur(loc); } { // input is: `dest: { ... stateInit: { code, data } }` (toShard is null); // then hash = (hash of StateInit = 0b00110 (only code + data)) code.push_set_cur(if_sharded.block1); - code.emplace_back(loc, Op::_Call, ir_hash, ir_ContractState, lookup_function("StateInit.calcHashCodeData")); + std::vector args = { ir_dest_ad.stateInitCode, ir_dest_ad.stateInitData }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashCodeData")); code.close_pop_cur(loc); } code.close_pop_cur(loc); @@ -181,10 +211,11 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T { // input is `dest: { ... stateInit: cell }` code.push_set_cur(if_ContractState.block1); - code.emplace_back(loc, Op::_Call, ir_hash, ir_StateInitCell, lookup_function("cell.hash")); + std::vector args = { ir_dest_ad.stateInitCell }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("cell.hash")); code.close_pop_cur(loc); } - Op& if_sharded = code.emplace_back(loc, Op::_If, ir_is_AddressSharding); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_dest_ad.is_AddressSharding); { // input is `dest: { ... toShard: { fixedPrefixLength, closeTo } }` // we already calculated stateInitHash (ir_hash): either cell.hash() or based on prefix+code+data; @@ -196,17 +227,14 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T // | hash (result) | 01010101...yyy | // remember, that closeTo is addr_std$10 + 0 + workchain + xxx...xxx, so skip 11 bits and read 8 code.push_set_cur(if_sharded.block0); + append_bitwise_and_shard_mask(code, loc, ir_hash[0], ir_dest_ad.ir_shardDepth); + std::vector ir_lowerD = code.create_tmp_var(TypeDataInt::create(), loc, "(lowerD)"); + code.emplace_back(loc, Op::_Call, ir_lowerD, std::vector{code.create_int(loc, 256, ""), ir_dest_ad.ir_shardDepth}, lookup_function("_-_")); std::vector ir_shardPrefix = code.create_tmp_var(TypeDataSlice::create(), loc, "(shardPrefix)"); - std::vector args_subslice = { ir_AddressSharding[1], code.create_int(loc, 3+8, ""), ir_AddressSharding[0] }; + std::vector args_subslice = { ir_dest_ad.ir_closeTo, code.create_int(loc, 3+8, ""), ir_dest_ad.ir_shardDepth }; code.emplace_back(loc, Op::_Call, ir_shardPrefix, std::move(args_subslice), lookup_function("slice.getMiddleBits")); ctx.storeSlice(ir_shardPrefix[0]); // first 8 bits of closeTo hash - std::vector ir_mask = code.create_tmp_var(TypeDataInt::create(), loc, "(mask)"); - code.emplace_back(loc, Op::_Call, ir_mask, std::vector{code.create_int(loc, 256, ""), ir_AddressSharding[0]}, lookup_function("_-_")); - code.emplace_back(loc, Op::_Call, ir_mask, std::vector{ir_one, ir_mask[0]}, lookup_function("_<<_")); - code.emplace_back(loc, Op::_Call, ir_mask, std::vector{ir_mask[0], ir_one}, lookup_function("_-_")); - code.emplace_back(loc, Op::_Call, ir_hash, std::vector{ir_hash[0], ir_mask[0]}, lookup_function("_&_")); - code.emplace_back(loc, Op::_Call, ir_mask, std::vector{code.create_int(loc, 256, ""), ir_AddressSharding[0]}, lookup_function("_-_")); - ctx.storeUint_var(ir_hash[0], ir_mask[0]); // 248 STU (stateInitHash & mask) + ctx.storeUint_var(ir_hash[0], ir_lowerD[0]); // 248 STU (stateInitHash & mask) code.close_pop_cur(loc); } { @@ -275,17 +303,17 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T } { code.push_set_cur(if_no_init.block0); - Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_is_ContractState); + Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_dest_ad.is_ContractState); { // input is `dest: { ... stateInit: { code, data } }` and need to compose TL/B StateInit; // it's either just code+data OR (if `toShard: { ... }` is set) fixedPrefixLength+code+data code.push_set_cur(if_ContractState.block0); - Op& if_sharded = code.emplace_back(loc, Op::_If, ir_is_AddressSharding); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_dest_ad.is_AddressSharding); { // 1 (maybe true) + 0 (either left) + 1 (maybe true of StateInit) + fixedPrefixLength + 0110 + body ref or not code.push_set_cur(if_sharded.block0); ctx.storeUint(code.create_int(loc, 0b101, ""), 1 + 1 + 1); - ctx.storeUint(ir_AddressSharding[0], 5); // fixedPrefixLength (shard depth) + ctx.storeUint(ir_dest_ad.ir_shardDepth, 5); // fixedPrefixLength (shard depth) ctx.storeUint(code.create_int(loc, 0b01100 + body_store_as_ref, ""), 4 + 1); code.close_pop_cur(loc); // also, we used dest.toShard to fill CommonMsgInfoRelaxed.dest.address (with a mask for stateInitHash, see above) @@ -297,8 +325,8 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T ctx.storeUint(ir_rest_bits, 1 + 1 + 5 + 1); code.close_pop_cur(loc); } - ctx.storeRef(ir_ContractState[0]); // dest.stateInit.code - ctx.storeRef(ir_ContractState[1]); // dest.stateInit.data + ctx.storeRef(ir_dest_ad.stateInitCode); + ctx.storeRef(ir_dest_ad.stateInitData); code.close_pop_cur(loc); } { @@ -307,7 +335,7 @@ std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, T code.push_set_cur(if_ContractState.block1); var_idx_t ir_rest_bits = code.create_int(loc, 0b110 + body_store_as_ref, "(rest-bits)"); ctx.storeUint(ir_rest_bits, 1 + 1 + 1); - ctx.storeRef(ir_StateInitCell[0]); + ctx.storeRef(ir_dest_ad.stateInitCell); code.close_pop_cur(loc); } code.close_pop_cur(loc); @@ -493,4 +521,155 @@ std::vector generate_address_buildInAnotherShard(CodeBlob& code, SrcL return ir_builder; } +std::vector generate_AutoDeployAddress_buildAddress(CodeBlob& code, SrcLocation loc, std::vector&& ir_auto_deploy) { + IR_AutoDeployAddress ir_self(code, loc, ir_auto_deploy); + + std::vector ir_builder = code.create_tmp_var(TypeDataSlice::create(), loc, "(addr-b)"); + // important! unlike `createMessage()`, we calculate hash and shard prefix BEFORE creating a cell + // (for fewer stack manipulations) + + // calculate stateInitHash = (hash of StateInit cell would be, but without constructing a cell) + std::vector ir_hash = code.create_tmp_var(TypeDataInt::create(), loc, "(addr-hash)"); + Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_self.is_ContractState); + { + // called `{ ... stateInit: { code, data } }` + code.push_set_cur(if_ContractState.block0); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); + { + // called `{ ... stateInit: { code, data }, toShard: { fixedPrefixLength, closeTo } } + code.push_set_cur(if_sharded.block0); + std::vector args = { ir_self.ir_shardDepth, ir_self.stateInitCode, ir_self.stateInitData }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashPrefixCodeData")); + code.close_pop_cur(loc); + } + { + // called `{ ... stateInit: { code, data } }` (toShard is null) + code.push_set_cur(if_sharded.block1); + std::vector args = { ir_self.stateInitCode, ir_self.stateInitData }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashCodeData")); + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + { + // called `{ ... stateInit: cell }` + code.push_set_cur(if_ContractState.block1); + std::vector args = { ir_self.stateInitCell }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("cell.hash")); + code.close_pop_cur(loc); + } + + // now, if toShard, perform bitwise calculations with hashes (order on a stack matters) + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); + { + // called `{ ... toShard: { fixedPrefixLength, closeTo } }` + // we already calculated stateInitHash (ir_hash): either cell.hash() or based on prefix+code+data; + // keep hash = (last 256-D bits from stateInitHash) = `hash & mask` + code.push_set_cur(if_sharded.block0); + append_bitwise_and_shard_mask(code, loc, ir_hash[0], ir_self.ir_shardDepth); + std::vector ir_lowerD = code.create_tmp_var(TypeDataInt::create(), loc, "(lowerD)"); + code.emplace_back(loc, Op::_Call, ir_lowerD, std::vector{code.create_int(loc, 256, ""), ir_self.ir_shardDepth}, lookup_function("_-_")); + + // calculate shard_prefix = (first D bits from dest.toShard.closeTo) + std::vector ir_shardPrefix = code.create_tmp_var(TypeDataSlice::create(), loc, "(shardPrefix)"); + std::vector args_subslice = { ir_self.ir_closeTo, code.create_int(loc, 3+8, ""), ir_self.ir_shardDepth }; + code.emplace_back(loc, Op::_Call, ir_shardPrefix, std::move(args_subslice), lookup_function("slice.getMiddleBits")); + + // on a stack: stateInitHash & mask; shard prefix; create a cell and store all + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, lookup_function("beginCell")); + PackContext ctx(code, loc, ir_builder, create_default_PackOptions(code, loc)); + ctx.storeUint(code.create_int(loc, 0b100, "(addr-prefix)"), 3); // addr_std$10 + 0 anycast + ctx.storeInt(ir_self.workchain, 8); + ctx.storeSlice(ir_shardPrefix[0]); // first 8 bits of closeTo hash + ctx.storeUint_var(ir_hash[0], ir_lowerD[0]); // 248 STU (stateInitHash & mask) + code.close_pop_cur(loc); + } + { + // called `{ workchain, stateInit }` (toShard is null); + // on a stack: hash (already calculated); create a cell and store all + code.push_set_cur(if_sharded.block1); + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, lookup_function("beginCell")); + PackContext ctx(code, loc, ir_builder, create_default_PackOptions(code, loc)); + ctx.storeUint(code.create_int(loc, 0b100, "(addr-prefix)"), 3); // addr_std$10 + 0 anycast + ctx.storeInt(ir_self.workchain, 8); + ctx.storeUint(ir_hash[0], 256); + code.close_pop_cur(loc); + } + + return ir_builder; +} + +std::vector generate_AutoDeployAddress_addressMatches(CodeBlob& code, SrcLocation loc, std::vector&& ir_auto_deploy, std::vector&& ir_address) { + IR_AutoDeployAddress ir_self(code, loc, ir_auto_deploy); + + // at first, calculate stateInitHash = (hash of StateInit cell would be, but without constructing a cell) + std::vector ir_hash = code.create_tmp_var(TypeDataInt::create(), loc, "(addr-hash)"); + Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_self.is_ContractState); + { + // called `{ ... stateInit: { code, data } }` + code.push_set_cur(if_ContractState.block0); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); + { + // called `{ ... stateInit: { code, data }, toShard: { fixedPrefixLength, closeTo } } + code.push_set_cur(if_sharded.block0); + std::vector args = { ir_self.ir_shardDepth, ir_self.stateInitCode, ir_self.stateInitData }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashPrefixCodeData")); + code.close_pop_cur(loc); + } + { + // called `{ ... stateInit: { code, data } }` (toShard is null) + code.push_set_cur(if_sharded.block1); + std::vector args = { ir_self.stateInitCode, ir_self.stateInitData }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("StateInit.calcHashCodeData")); + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + { + // called `{ ... stateInit: cell }` + code.push_set_cur(if_ContractState.block1); + std::vector args = { ir_self.stateInitCell }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(args), lookup_function("cell.hash")); + code.close_pop_cur(loc); + } + + // now calculate `stateInitHash &= mask` where mask = `(1 << (256 - SHARD_DEPTH)) - 1` + Op& if_sharded1 = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); + { + code.push_set_cur(if_sharded1.block0); + append_bitwise_and_shard_mask(code, loc, ir_hash[0], ir_self.ir_shardDepth); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_sharded1.block1); + code.close_pop_cur(loc); + } + + // now do `(wc, hash) = addr.getWorkchainAndHash()` + std::vector ir_addr_wc_hash = code.create_tmp_var(TypeDataTensor::create({TypeDataInt::create(), TypeDataInt::create()}), loc, "(self-wc-hash)"); + code.emplace_back(loc, Op::_Call, ir_addr_wc_hash, ir_address, lookup_function("address.getWorkchainAndHash")); + + // now calculate `hash &= mask` (the same as we did earlier for stateInitHash) + Op& if_sharded2 = code.emplace_back(loc, Op::_If, ir_self.is_AddressSharding); + { + code.push_set_cur(if_sharded2.block0); + append_bitwise_and_shard_mask(code, loc, ir_addr_wc_hash[1], ir_self.ir_shardDepth); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_sharded2.block1); + code.close_pop_cur(loc); + } + + // finally, eval `(hash == stateInitHash) & (wc == workchain)` + std::vector ir_eq_hash = code.create_tmp_var(TypeDataInt::create(), loc, "(eq-hash)"); + code.emplace_back(loc, Op::_Call, ir_eq_hash, std::vector{ir_addr_wc_hash[1], ir_hash[0]}, lookup_function("_==_")); + std::vector ir_eq_wc = code.create_tmp_var(TypeDataInt::create(), loc, "(eq-wc)"); + code.emplace_back(loc, Op::_Call, ir_eq_wc, std::vector{ir_addr_wc_hash[0], ir_self.workchain}, lookup_function("_==_")); + + std::vector ir_bool_result = code.create_tmp_var(TypeDataBool::create(), loc, "(is-addr-result)"); + code.emplace_back(loc, Op::_Call, ir_bool_result, std::vector{ir_eq_hash[0], ir_eq_wc[0]}, lookup_function("_&_")); + return ir_bool_result; +} + } // namespace tolk diff --git a/tolk/send-message-api.h b/tolk/send-message-api.h index 1af7389d4..f86be5446 100644 --- a/tolk/send-message-api.h +++ b/tolk/send-message-api.h @@ -25,4 +25,7 @@ std::vector generate_createExternalLogMessage(CodeBlob& code, SrcLoca std::vector generate_address_buildInAnotherShard(CodeBlob& code, SrcLocation loc, std::vector&& ir_self_address, std::vector&& ir_shard_options); +std::vector generate_AutoDeployAddress_buildAddress(CodeBlob& code, SrcLocation loc, std::vector&& ir_auto_deploy); +std::vector generate_AutoDeployAddress_addressMatches(CodeBlob& code, SrcLocation loc, std::vector&& ir_auto_deploy, std::vector&& ir_address); + } // namespace tolk From 6f5fb744606ef09a7fcfa5e4b192f62040bb09ef Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 3 Jul 2025 18:46:40 +0300 Subject: [PATCH 339/388] [Tolk] Make a semicolon optional for top-level declarations After declaring type aliases, constants, etc. semicolon is now optional. Struct fields can also be separated with a newline only. --- crypto/smartcont/tolk-stdlib/common.tolk | 461 ++++++++---------- .../smartcont/tolk-stdlib/gas-payments.tolk | 24 +- crypto/smartcont/tolk-stdlib/lisp-lists.tolk | 10 +- crypto/smartcont/tolk-stdlib/tvm-dicts.tolk | 138 +++--- .../smartcont/tolk-stdlib/tvm-lowlevel.tolk | 8 +- tolk-tester/tests/assignment-tests.tolk | 8 +- tolk-tester/tests/constants-tests.tolk | 13 +- tolk-tester/tests/indexed-access.tolk | 4 +- .../tests/invalid-declaration/err-1043.tolk | 6 + .../tests/invalid-declaration/err-1794.tolk | 2 +- .../tests/invalid-syntax/err-3618.tolk | 2 +- .../tests/invalid-syntax/err-3802.tolk | 9 - tolk-tester/tests/methods-tests.tolk | 8 +- tolk-tester/tests/mutate-methods.tolk | 4 +- tolk-tester/tests/no-spaces.tolk | 6 +- tolk-tester/tests/pack-unpack-3.tolk | 13 +- tolk-tester/tests/pack-unpack-5.tolk | 8 +- tolk-tester/tests/smart-cast-tests.tolk | 2 +- tolk-tester/tests/strings-tests.tolk | 10 +- tolk-tester/tests/struct-tests.tolk | 26 +- tolk-tester/tests/type-aliases-tests.tolk | 18 +- tolk-tester/tests/union-types-tests.tolk | 22 +- tolk/ast-from-tokens.cpp | 38 +- 23 files changed, 402 insertions(+), 438 deletions(-) create mode 100644 tolk-tester/tests/invalid-declaration/err-1043.tolk delete mode 100644 tolk-tester/tests/invalid-syntax/err-3802.tolk diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index da52bfb98..a72718246 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -7,7 +7,7 @@ tolk 0.99 /// Currently, working with dictionaries is still low-level, with raw cells. /// But just for clarity, we use "dict" instead of a "cell?" where a cell-dictionary is assumed. /// Every dictionary object can be null. TVM NULL is essentially "empty dictionary". -type dict = cell?; +type dict = cell? /** Tuple manipulation primitives. @@ -18,47 +18,47 @@ type dict = cell?; /// Creates a tuple with zero elements. @pure fun createEmptyTuple(): tuple - asm "NIL"; + asm "NIL" /// Appends a value to tuple, resulting in `Tuple t' = (x1, ..., xn, value)`. /// If its size exceeds 255, throws a type check exception. @pure fun tuple.push(mutate self, value: T): void - asm "TPUSH"; + asm "TPUSH" /// Returns the first element of a non-empty tuple. /// `t.0` is actually the same as `t.first()` @pure fun tuple.first(self): T - asm "FIRST"; + asm "FIRST" /// Returns the [`index`]-th element of a tuple. /// `t.i` is actually the same as `t.get(i)` @pure fun tuple.get(self, index: int): T - builtin; + builtin /// Sets the [`index`]-th element of a tuple to a specified value /// (element with this index must already exist, a new element isn't created). /// `t.i = value` is actually the same as `t.set(value, i)` @pure fun tuple.set(mutate self, value: T, index: int): void - builtin; + builtin /// Returns the size of a tuple (elements count in it). @pure fun tuple.size(self): int - asm "TLEN"; + asm "TLEN" /// Returns the last element of a non-empty tuple. @pure fun tuple.last(self): T - asm "LAST"; + asm "LAST" /// Pops and returns the last element of a non-empty tuple. @pure fun tuple.pop(mutate self): T - asm "TPOP"; + asm "TPOP" /** @@ -70,64 +70,64 @@ fun tuple.pop(mutate self): T /// Note, that `ton()` requires a constant string; `ton(some_var)` is an error @pure fun ton(floatString: slice): coins - builtin; + builtin /// Computes the minimum of two integers. @pure fun min(x: int, y: int): int - asm "MIN"; + asm "MIN" /// Computes the maximum of two integers. @pure fun max(x: int, y: int): int - asm "MAX"; + asm "MAX" /// Sorts two integers. /// Example: `minMax(x, y)` with (x=20, y=10) and with (x=10, y=20) returns (10, 20) @pure fun minMax(x: int, y: int): (int, int) - asm "MINMAX"; + asm "MINMAX" /// Computes the absolute value of an integer. @pure fun abs(x: int): int - asm "ABS"; + asm "ABS" /// Returns the sign of an integer: `-1` if x < 0, `0` if x == 0, `1` if x > 0. @pure fun sign(x: int): int - asm "SGN"; + asm "SGN" /// Computes the quotient and remainder of [x] / [y]. Example: divMod(112,3) = (37,1) @pure fun divMod(x: int, y: int): (int, int) - asm "DIVMOD"; + asm "DIVMOD" /// Computes the remainder and quotient of [x] / [y]. Example: modDiv(112,3) = (1,37) @pure fun modDiv(x: int, y: int): (int, int) - asm(-> 1 0) "DIVMOD"; + asm(-> 1 0) "DIVMOD" /// Computes multiple-then-divide: floor([x] * [y] / [z]). /// The intermediate result is stored in a 513-bit integer to prevent precision loss. @pure fun mulDivFloor(x: int, y: int, z: int): int - builtin; + builtin /// Similar to `mulDivFloor`, but rounds the result: round([x] * [y] / [z]). @pure fun mulDivRound(x: int, y: int, z: int): int - builtin; + builtin /// Similar to `mulDivFloor`, but ceils the result: ceil([x] * [y] / [z]). @pure fun mulDivCeil(x: int, y: int, z: int): int - builtin; + builtin /// Computes the quotient and remainder of ([x] * [y] / [z]). Example: mulDivMod(112,3,10) = (33,6) @pure fun mulDivMod(x: int, y: int, z: int): (int, int) - builtin; + builtin /** @@ -136,83 +136,83 @@ fun mulDivMod(x: int, y: int, z: int): (int, int) /// `contract` is a built-in struct, it has only static methods. /// Example: `contract.getCode()` and other methods. -struct contract {} +struct contract /// Returns the internal address of the current smart contract. /// If necessary, it can be parsed further using [address.getWorkchain] and others. @pure fun contract.getAddress(): address - asm "MYADDR"; + asm "MYADDR" /// Returns the balance (in nanotoncoins) of the smart contract at the start of Computation Phase. /// Note that RAW primitives such as [sendMessage] do not update this field. @pure fun contract.getOriginalBalance(): coins - asm "BALANCE" "FIRST"; + asm "BALANCE" "FIRST" /// Same as [contract.getOriginalBalance], but returns a tuple: /// `int` — balance in nanotoncoins; /// `dict` — a dictionary with 32-bit keys representing the balance of "extra currencies". @pure fun contract.getOriginalBalanceWithExtraCurrencies(): [coins, dict] - asm "BALANCE"; + asm "BALANCE" /// Returns the persistent contract storage cell. It can be parsed or modified with slice and builder primitives later. @pure fun contract.getData(): cell - asm "c4 PUSH"; + asm "c4 PUSH" /// Sets `cell` [c] as persistent contract data. You can update persistent contract storage with this primitive. fun contract.setData(c: cell): void - asm "c4 POP"; + asm "c4 POP" /// Retrieves code of smart-contract from c7 @pure fun contract.getCode(): cell - asm "MYCODE"; + asm "MYCODE" /// Creates an output action that would change this smart contract code to that given by cell [newCode]. /// Notice that this change will take effect only after the successful termination of the current run of the smart contract. fun contract.setCodePostponed(newCode: cell): void - asm "SETCODE"; + asm "SETCODE" /** Global getters of current blockchain (environment) state. */ -const MASTERCHAIN = -1; -const BASECHAIN = 0; +const MASTERCHAIN = -1 +const BASECHAIN = 0 /// `blockchain` is a built-in struct, it has only static methods. /// Example: `blockchain.configParam(16)` and other methods. -struct blockchain {} +struct blockchain /// Returns current Unix timestamp (in seconds). @pure fun blockchain.now(): int - asm "NOW"; + asm "NOW" /// Returns the logical time of the current transaction. @pure fun blockchain.logicalTime(): int - asm "LTIME"; + asm "LTIME" /// Returns the starting logical time of the current block. @pure fun blockchain.currentBlockLogicalTime(): int - asm "BLOCKLT"; + asm "BLOCKLT" /// Returns the value of the global configuration parameter with integer index `i` as a `cell` or `null` value. @pure fun blockchain.configParam(x: int): cell? - asm "CONFIGOPTPARAM"; + asm "CONFIGOPTPARAM" /// Commits the current state of registers `c4` (“persistent data”) and `c5` (“actions”) /// so that the current execution is considered “successful” with the saved values even if an exception /// in Computation Phase is thrown later. fun commitContractDataAndActions(): void - asm "COMMIT"; + asm "COMMIT" /** @@ -257,7 +257,7 @@ struct UnpackOptions { /// (beginCell() + serialize fields + endCell()). @pure fun T.toCell(self, options: PackOptions = {}): Cell - builtin; + builtin /// Parse anything from a cell (most likely, you'll call it for structures). /// Example: @@ -268,7 +268,7 @@ fun T.toCell(self, options: PackOptions = {}): Cell /// (packedCell.beginParse() + read from slice). @pure fun T.fromCell(packedCell: cell, options: UnpackOptions = {}): T - builtin; + builtin /// Parse anything from a slice (most likely, you'll call it for structures). /// Example: @@ -281,7 +281,7 @@ fun T.fromCell(packedCell: cell, options: UnpackOptions = {}): T /// If you need to mutate it, like `cs.loadInt()`, consider calling `cs.loadAny()`. @pure fun T.fromSlice(rawSlice: slice, options: UnpackOptions = {}): T - builtin; + builtin /// Parse anything from a slice, shifting its internal pointer. /// Similar to `slice.loadUint()` and others, but allows loading structures. @@ -294,7 +294,7 @@ fun T.fromSlice(rawSlice: slice, options: UnpackOptions = {}): T /// to read data from the middle. @pure fun slice.loadAny(mutate self, options: UnpackOptions = {}): T - builtin; + builtin /// Skip anything in a slice, shifting its internal pointer. /// Similar to `slice.skipBits()` and others, but allows skipping structures. @@ -305,7 +305,7 @@ fun slice.loadAny(mutate self, options: UnpackOptions = {}): T /// ``` @pure fun slice.skipAny(mutate self, options: UnpackOptions = {}): self - builtin; + builtin /// Store anything to a builder. /// Similar to `builder.storeUint()` and others, but allows storing structures. @@ -315,19 +315,19 @@ fun slice.skipAny(mutate self, options: UnpackOptions = {}): self /// ``` @pure fun builder.storeAny(mutate self, v: T, options: PackOptions = {}): self - builtin; + builtin /// Returns serialization prefix of a struct. Works at compile-time. /// Example: for `struct (0xF0) AssetRegular { ... }` will return `240`. @pure fun T.getDeclaredPackPrefix(): int - builtin; + builtin /// Returns serialization prefix length of a struct. Works at compile-time. /// Example: for `struct (0xF0) AssetRegular { ... }` will return `16`. @pure fun T.getDeclaredPackPrefixLen(): int - builtin; + builtin /// Forces an object created by `lazy` to load fully. Returns the remaining slice (having read all fields). /// Since `options.assertEndAfterReading` is ignored by `lazy` (fields are loaded on demand), @@ -347,7 +347,7 @@ fun T.getDeclaredPackPrefixLen(): int /// because the purpose of `lazy` is to auto-detect and load only necessary fields, not up to the end. @pure fun T.forceLoadLazyObject(self): slice - builtin; + builtin /// Cell represents a typed cell reference (as opposed to untyped `cell`). /// Example: @@ -365,7 +365,7 @@ fun T.forceLoadLazyObject(self): slice /// Note, that `st = MyStorage.fromSlice(s)` does NOT deep-load any refs; `st.extra` is `Cell`, not `T`; /// you should manually call `st.extra.load()` to get T (ExtraData in this example). struct Cell { - tvmCell: cell; + tvmCell: cell } /// Parse data from already loaded cell reference. @@ -379,17 +379,17 @@ struct Cell { /// ``` @pure fun Cell.load(self, options: UnpackOptions = {}): T - builtin; + builtin /// Converts a typed cell into a slice. @pure fun Cell.beginParse(self): slice - asm "CTOS"; + asm "CTOS" /// Returns hash of a typed cell, same as [cell.hash]. @pure fun Cell.hash(self): uint256 - asm "HASHCU"; + asm "HASHCU" /// RemainingBitsAndRefs is a special built-in type to get "all the rest" slice tail on reading. /// Example: @@ -400,19 +400,19 @@ fun Cell.hash(self): uint256 /// } /// ``` /// When you deserialize JettonMessage, forwardPayload contains "everything left after reading fields above". -type RemainingBitsAndRefs = slice; +type RemainingBitsAndRefs = slice /// Creates a cell with zero bits and references. /// Equivalent to `beginCell().endCell()` but cheaper. @pure fun createEmptyCell(): cell - asm " PUSHREF"; + asm " PUSHREF" /// Creates a slice with zero remaining bits and references. /// Equivalent to `beginCell().endCell().beginParse()` but cheaper. @pure fun createEmptySlice(): slice - asm "x{} PUSHSLICE"; + asm "x{} PUSHSLICE" /** @@ -424,14 +424,14 @@ fun createEmptySlice(): slice /// Note: stringCrc32(slice_var) does not work! It accepts a constant string and works at compile-time. @pure fun stringCrc32(constString: slice): int - builtin; + builtin /// Compile-time function that calculates crc16 (XMODEM) of a constant string. /// Example: `const op = stringCrc16("some_str")` = 53407 = 0xD09F /// Note: stringCrc16(slice_var) does not work! It accepts a constant string and works at compile-time. @pure fun stringCrc16(constString: slice): int - builtin; + builtin /// Compile-time function that calculates sha256 of a constant string and returns 256-bit integer. /// Example: `const hash = stringSha256("some_crypto_key")` @@ -439,40 +439,40 @@ fun stringCrc16(constString: slice): int /// Use `sliceBitsHash` or `sliceHash` (declared below) to hash a slice without/with its refs at runtime. @pure fun stringSha256(constString: slice): int - builtin; + builtin /// Compile-time function that calculates sha256 of a constant string and takes the first 32 bits. /// Example: `const minihash = stringSha256_32("some_crypto_key")` /// Note: stringSha256_32(slice_var) does not work! It accepts a constant string and works at compile-time. @pure fun stringSha256_32(constString: slice): int - builtin; + builtin /// Compile-time function that takes N-chars ascii string and interprets it as a number in base 256. /// Example: `const value = stringToBase256("AB")` = 16706 (65*256 + 66) /// Note: stringToBase256(slice_var) does not work! It accepts a constant string and works at compile-time. @pure fun stringToBase256(constString: slice): int - builtin; + builtin /// Computes the representation hash of a `cell` and returns it as a 256-bit unsigned integer `x`. /// Useful for signing and checking signatures of arbitrary entities represented by a tree of cells. @pure fun cell.hash(self): uint256 - asm "HASHCU"; + asm "HASHCU" /// Computes the hash of a `slice` and returns it as a 256-bit unsigned integer `x`. /// The result is the same as if an ordinary cell containing only data and references from `s` had been created /// and its hash computed by [cell.hash]. @pure fun slice.hash(self): uint256 - asm "HASHSU"; + asm "HASHSU" /// Computes sha256 of the data bits of a `slice`. If the bit length of `s` is not divisible by eight, /// throws a cell underflow exception. The hash value is returned as a 256-bit unsigned integer `x`. @pure fun slice.bitsHash(self): uint256 - asm "SHA256U"; + asm "SHA256U" /// Checks the Ed25519-`signature` of a `hash` (a 256-bit unsigned integer, usually computed as the hash of some data) /// using [publicKey] (also represented by a 256-bit unsigned integer). @@ -483,7 +483,7 @@ fun slice.bitsHash(self): uint256 /// the second hashing occurring inside `CHKSIGNS`. @pure fun isSignatureValid(hash: int, signature: slice, publicKey: int): bool - asm "CHKSIGNU"; + asm "CHKSIGNU" /// Checks whether [signature] is a valid Ed25519-signature of the data portion of `slice data` using `publicKey`, /// similarly to [isSignatureValid]. @@ -492,40 +492,40 @@ fun isSignatureValid(hash: int, signature: slice, publicKey: int): bool /// with sha256 used to reduce [data] to the 256-bit number that is actually signed. @pure fun isSliceSignatureValid(data: slice, signature: slice, publicKey: int): bool - asm "CHKSIGNS"; + asm "CHKSIGNS" /// `random` is a built-in struct, it has only static methods. /// Example: `random.uint256()` and other methods. -struct random {} +struct random /// Generates a new pseudo-random unsigned 256-bit integer x. /// Ensure you've called [random.initialize] in advance to make it unpredictable! fun random.uint256(): uint256 - asm "RANDU256"; + asm "RANDU256" /// Generates a new pseudo-random integer z in the range 0..limit−1 (or limit..−1, if upto < 0). /// More precisely, an unsigned random value x is generated as in random; then z := x * limit / 2^256 is computed. /// Ensure you've called [random.initialize] in advance to make it unpredictable! fun random.range(limit: int): int - asm "RAND"; + asm "RAND" /// Returns the current random seed used to generate pseudo-random numbers. @pure fun random.getSeed(): uint256 - asm "RANDSEED"; + asm "RANDSEED" /// Sets the random seed to the provided value. fun random.setSeed(seed: uint256): void - asm "SETRAND"; + asm "SETRAND" /// Initializes (mixes) random seed with the provided value. fun random.initializeBy(mixSeedWith: uint256): void - asm "ADDRAND"; + asm "ADDRAND" /// Initializes random seed with current time to make random generation unpredictable. /// Typically, you call this function once before calling [random.uint256] / [random.range]. fun random.initialize(): void - asm "LTIME" "ADDRAND"; + asm "LTIME" "ADDRAND" /** @@ -544,22 +544,22 @@ fun random.initialize(): void /// a zero flag is returned to indicate failure. If [c] is `null`, returns `x = y = z = 0`. @pure fun cell.calculateSize(self, maxCells: int): (int, int, int, bool) - asm "CDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT"; + asm "CDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT" /// Similar to [cell.calculateSize], but accepting a `slice` [s] instead of a `cell`. /// The returned value of `x` does not take into account the cell that contains the `slice` [s] itself; /// however, the data bits and the cell references of [s] are accounted for in `y` and `z`. @pure fun slice.calculateSize(self, maxCells: int): (int, int, int, bool) - asm "SDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT"; + asm "SDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT" /// A non-quiet version of [cell.calculateSize] that throws a cell overflow exception (`8`) on failure. fun cell.calculateSizeStrict(self, maxCells: int): (int, int, int) - asm "CDATASIZE"; + asm "CDATASIZE" /// A non-quiet version of [cell.calculateSize] that throws a cell overflow exception (`8`) on failure. fun slice.calculateSizeStrict(self, maxCells: int): (int, int, int) - asm "SDATASIZE"; + asm "SDATASIZE" /// Returns the depth of a `cell`. /// If [c] has no references, then return `0`; @@ -567,21 +567,21 @@ fun slice.calculateSizeStrict(self, maxCells: int): (int, int, int) /// If [c] is a `null` instead of a cell, returns zero. @pure fun cell?.depth(self): int - asm "CDEPTH"; + asm "CDEPTH" /// Returns the depth of a `slice`. /// If [s] has no references, then returns `0`; /// otherwise the returned value is one plus the maximum of depths of cells referred to from [s]. @pure fun slice.depth(self): int - asm "SDEPTH"; + asm "SDEPTH" /// Returns the depth of a `builder`. /// If no cell references are stored in [b], then returns 0; /// otherwise the returned value is one plus the maximum of depths of cells referred to from [b]. @pure fun builder.depth(self): int - asm "BDEPTH"; + asm "BDEPTH" /// Returns the number of stack slots anyVariable occupies (works at compile-time). /// Example: sizeof(nullableInt) = 1, because `int?` is 1 TVM slot holding either NULL or a value. @@ -589,7 +589,7 @@ fun builder.depth(self): int /// Useful for debugging or when preparing stack contents for RUNVM. @pure fun sizeof(anyVariable: T): int - builtin; + builtin /** @@ -599,19 +599,19 @@ fun sizeof(anyVariable: T): int /// `debug` is a built-in struct, it has only static methods. /// Example: `debug.print(v)` and other methods. -struct debug {} +struct debug /// Dump a variable [x] to the debug log. fun debug.print(x: T): void - builtin; + builtin /// Dump a string [x] to the debug log. fun debug.printString(x: T): void - builtin; + builtin /// Dumps the stack (at most the top 255 values) and shows the total stack depth. fun debug.dumpStack(): void - builtin; + builtin /** @@ -625,126 +625,126 @@ fun debug.dumpStack(): void /// Note: stringHexToSlice(slice_var) does not work! It accepts a constant string and works at compile-time. @pure fun stringHexToSlice(constStringBytesHex: slice): slice - builtin; + builtin /// Converts a `cell` into a `slice`. Notice that [c] must be either an ordinary cell, /// or an exotic cell (see [TVM.pdf](https://ton-blockchain.github.io/docs/tvm.pdf), 3.1.2) /// which is automatically loaded to yield an ordinary cell `c'`, converted into a `slice` afterwards. @pure fun cell.beginParse(self): slice - asm "CTOS"; + asm "CTOS" /// Checks if slice is empty. If not, throws an exception with code 9. fun slice.assertEnd(self): void - asm "ENDS"; + asm "ENDS" /// Loads the next reference from the slice. @pure fun slice.loadRef(mutate self): cell - asm( -> 1 0) "LDREF"; + asm( -> 1 0) "LDREF" /// Preloads the next reference from the slice. @pure fun slice.preloadRef(self): cell - asm "PLDREF"; + asm "PLDREF" /// Loads a signed [len]-bit integer from a slice. @pure fun slice.loadInt(mutate self, len: int): int - builtin; + builtin /// Loads an unsigned [len]-bit integer from a slice. @pure fun slice.loadUint(mutate self, len: int): int - builtin; + builtin /// Loads the first `0 ≤ len ≤ 1023` bits from slice [s] into a separate slice `s''`. @pure fun slice.loadBits(mutate self, len: int): slice - builtin; + builtin /// Preloads a signed [len]-bit integer from a slice. @pure fun slice.preloadInt(self, len: int): int - builtin; + builtin /// Preloads an unsigned [len]-bit integer from a slice. @pure fun slice.preloadUint(self, len: int): int - builtin; + builtin /// Preloads the first `0 ≤ len ≤ 1023` bits from slice [s] into a separate slice. @pure fun slice.preloadBits(self, len: int): slice - builtin; + builtin /// Loads serialized amount of Toncoins (any unsigned integer up to `2^120 - 1`). @pure fun slice.loadCoins(mutate self): coins - asm( -> 1 0) "LDGRAMS"; + asm( -> 1 0) "LDGRAMS" /// Loads bool (-1 or 0) from a slice @pure fun slice.loadBool(mutate self): bool - asm( -> 1 0) "1 LDI"; + asm( -> 1 0) "1 LDI" /// Shifts a slice pointer to [len] bits forward, mutating the slice. @pure fun slice.skipBits(mutate self, len: int): self - builtin; + builtin /// Returns the first `0 ≤ len ≤ 1023` bits of a slice. @pure fun slice.getFirstBits(self, len: int): slice - asm "SDCUTFIRST"; + asm "SDCUTFIRST" /// Returns all but the last `0 ≤ len ≤ 1023` bits of a slice. @pure fun slice.removeLastBits(mutate self, len: int): self - asm "SDSKIPLAST"; + asm "SDSKIPLAST" /// Returns the last `0 ≤ len ≤ 1023` bits of a slice. @pure fun slice.getLastBits(self, len: int): slice - asm "SDCUTLAST"; + asm "SDCUTLAST" /// Returns `0 ≤ len ≤ 1023` bits of a slice starting from `0 ≤ offset ≤ 1023`. /// (in other words, extracts a bit substring `[offset, len)` out of the slice) @pure fun slice.getMiddleBits(self, offset: int, len: int): slice - asm "SDSUBSTR"; + asm "SDSUBSTR" /// Loads a dictionary (TL HashMapE structure, represented as TVM cell) from a slice. /// Returns `null` if `nothing` constructor is used. @pure fun slice.loadDict(mutate self): dict - asm( -> 1 0) "LDDICT"; + asm( -> 1 0) "LDDICT" /// Preloads a dictionary (cell) from a slice. @pure fun slice.preloadDict(self): dict - asm "PLDDICT"; + asm "PLDDICT" /// Loads a dictionary as [slice.loadDict], but returns only the remainder of the slice. @pure fun slice.skipDict(mutate self): self - asm "SKIPDICT"; + asm "SKIPDICT" /// Loads (Maybe ^Cell) from a slice. /// In other words, loads 1 bit: if it's true, loads the first ref, otherwise returns `null`. @pure fun slice.loadMaybeRef(mutate self): cell? - asm( -> 1 0) "LDOPTREF"; + asm( -> 1 0) "LDOPTREF" /// Preloads (Maybe ^Cell) from a slice. @pure fun slice.preloadMaybeRef(self): cell? - asm "PLDOPTREF"; + asm "PLDOPTREF" /// Loads (Maybe ^Cell), but returns only the remainder of the slice. @pure fun slice.skipMaybeRef(mutate self): self - asm "SKIPOPTREF"; + asm "SKIPOPTREF" /** Builder primitives: constructing cells. @@ -756,70 +756,70 @@ fun slice.skipMaybeRef(mutate self): self /// Creates a new empty builder. @pure fun beginCell(): builder - asm "NEWC"; + asm "NEWC" /// Converts a builder into an ordinary `cell`. @pure fun builder.endCell(self): cell - asm "ENDC"; + asm "ENDC" /// Stores a reference to a cell into a builder. @pure fun builder.storeRef(mutate self, c: cell): self - asm(c self) "STREF"; + asm(c self) "STREF" /// Stores a signed [len]-bit integer into a builder (`0 ≤ len ≤ 257`). @pure fun builder.storeInt(mutate self, x: int, len: int): self - builtin; + builtin /// Stores an unsigned [len]-bit integer into a builder (`0 ≤ len ≤ 256`). @pure fun builder.storeUint(mutate self, x: int, len: int): self - builtin; + builtin /// Stores a slice into a builder. @pure fun builder.storeSlice(mutate self, s: slice): self - asm(s self) "STSLICE"; + asm(s self) "STSLICE" /// Stores an address into a builder. @pure fun builder.storeAddress(mutate self, addr: address): self - asm(addr self) "STSLICE"; + asm(addr self) "STSLICE" /// Stores amount of Toncoins into a builder. @pure fun builder.storeCoins(mutate self, x: coins): self - builtin; + builtin /// Stores bool (-1 or 0) into a builder. /// Attention: true value is `-1`, not 1! If you pass `1` here, TVM will throw an exception. @pure fun builder.storeBool(mutate self, x: bool): self - builtin; + builtin /// Stores dictionary (represented by TVM `cell` or `null`) into a builder. /// In other words, stores a `1`-bit and a reference to [c] if [c] is not `null` and `0`-bit otherwise. @pure fun builder.storeDict(mutate self, c: dict): self - asm(c self) "STDICT"; + asm(c self) "STDICT" /// Stores (Maybe ^Cell) into a builder. /// In other words, if cell is `null`, store '0' bit; otherwise, store '1' and a ref to [c]. @pure fun builder.storeMaybeRef(mutate self, c: cell?): self - asm(c self) "STOPTREF"; + asm(c self) "STOPTREF" /// Concatenates two builders. @pure fun builder.storeBuilder(mutate self, from: builder): self - asm(from self) "STB"; + asm(from self) "STB" /// Stores a slice representing TL addr_none$00 (two `0` bits). @pure fun builder.storeAddressNone(mutate self): self - asm "b{00} STSLICECONST"; + asm "b{00} STSLICECONST" /** @@ -829,84 +829,51 @@ fun builder.storeAddressNone(mutate self): self /// Returns the number of references in a slice. @pure fun slice.remainingRefsCount(self): int - asm "SREFS"; + asm "SREFS" /// Returns the number of data bits in a slice. @pure fun slice.remainingBitsCount(self): int - asm "SBITS"; + asm "SBITS" /// Returns both the number of data bits and the number of references in a slice. @pure fun slice.remainingBitsAndRefsCount(self): (int, int) - asm "SBITREFS"; + asm "SBITREFS" /// Checks whether a slice is empty (i.e., contains no bits of data and no cell references). @pure fun slice.isEmpty(self): bool - asm "SEMPTY"; + asm "SEMPTY" /// Checks whether a slice has no bits of data. @pure fun slice.isEndOfBits(self): bool - asm "SDEMPTY"; + asm "SDEMPTY" /// Checks whether a slice has no references. @pure fun slice.isEndOfRefs(self): bool - asm "SREMPTY"; + asm "SREMPTY" /// Checks whether data parts of two slices coinside. @pure fun slice.bitsEqual(self, b: slice): bool - asm "SDEQ"; + asm "SDEQ" /// Returns the number of cell references already stored in a builder. @pure fun builder.refsCount(self): int - asm "BREFS"; + asm "BREFS" /// Returns the number of data bits already stored in a builder. @pure fun builder.bitsCount(self): int - asm "BBITS"; + asm "BBITS" /** Address manipulation primitives. - The address manipulation primitives listed below serialize and deserialize values according to the following TL-B scheme: - ```TL-B - addr_none$00 = MsgAddressExt; - addr_extern$01 len:(## 8) external_address:(bits len) - = MsgAddressExt; - anycast_info$_ depth:(#<= 30) { depth >= 1 } - rewrite_pfx:(bits depth) = Anycast; - addr_std$10 anycast:(Maybe Anycast) - workchain_id:int8 address:bits256 = MsgAddressInt; - addr_var$11 anycast:(Maybe Anycast) addr_len:(## 9) - workchain_id:int32 address:(bits addr_len) = MsgAddressInt; - _ _:MsgAddressInt = MsgAddress; - _ _:MsgAddressExt = MsgAddress; - - int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool - src:MsgAddress dest:MsgAddressInt - value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams - created_lt:uint64 created_at:uint32 = CommonMsgInfoRelaxed; - ext_out_msg_info$11 src:MsgAddress dest:MsgAddressExt - created_lt:uint64 created_at:uint32 = CommonMsgInfoRelaxed; - ``` - A deserialized `MsgAddress` is represented by a tuple `t` as follows: - - - `addr_none` is represented by `t = (0)`, - i.e., a tuple containing exactly one integer equal to zero. - - `addr_extern` is represented by `t = (1, s)`, - where slice `s` contains the field `external_address`. In other words, ` - t` is a pair (a tuple consisting of two entries), containing an integer equal to one and slice `s`. - - `addr_std` is represented by `t = (2, u, x, s)`, - where `u` is either a `null` (if `anycast` is absent) or a slice `s'` containing `rewrite_pfx` (if anycast is present). - Next, integer `x` is the `workchain_id`, and slice `s` contains the address. - - `addr_var` is represented by `t = (3, u, x, s)`, - where `u`, `x`, and `s` have the same meaning as for `addr_std`. */ /// Compile-time function that parses a valid contract address. @@ -915,19 +882,19 @@ fun builder.bitsCount(self): int /// Returns `address`, which can be stored in a builder, compared with `==`, etc. @pure fun address(stdAddress: slice): address - builtin; + builtin /// Creates a slice representing "none address" (TL addr_none$00 — two zero bits). @pure fun createAddressNone(): address - asm "b{00} PUSHSLICE"; + asm "b{00} PUSHSLICE" /// Returns if it's an empty address. /// Don't confuse it with null! Empty address is a slice with two `0` bits. /// In TL/B, it's addr_none$00. @pure fun address.isNone(self): bool - asm "b{00} SDBEGINSQ" "NIP"; + asm "b{00} SDBEGINSQ" "NIP" /// Returns if it's a standard (internal) address. Such addresses contain workchain (8 bits) and hash (256 bits). /// All contract addresses are internal, so it's the most practical use case. @@ -935,25 +902,25 @@ fun address.isNone(self): bool /// For internal addresses, you can call [address.getWorkchain] and [address.getWorkchainAndHash]. @pure fun address.isInternal(self): bool - asm "b{10} SDBEGINSQ" "NIP"; + asm "b{10} SDBEGINSQ" "NIP" /// Returns if it's an external address, used to communication with the outside world. /// In TL/B it's addr_extern$01. @pure fun address.isExternal(self): bool - asm "b{01} SDBEGINSQ" "NIP"; + asm "b{01} SDBEGINSQ" "NIP" /// Extracts workchain and hash from a standard (internal) address. /// If the address is not internal, throws a cell deserialization exception. @pure fun address.getWorkchainAndHash(self): (int8, uint256) - asm "REWRITESTDADDR"; + asm "REWRITESTDADDR" /// Extracts workchain from a standard (internal) address. /// If the address is not internal, throws a cell deserialization exception. @pure fun address.getWorkchain(self): int8 - asm "REWRITESTDADDR" "DROP"; + asm "REWRITESTDADDR" "DROP" /// Checks whether two addresses are equal. Equivalent to `a == b`. /// Deprecated! Left for smoother transition from FunC, where you used `slice` everywhere. @@ -961,12 +928,12 @@ fun address.getWorkchain(self): int8 @pure @deprecated("use `senderAddress == ownerAddress`, not `senderAddress.bitsEqual(ownerAddress)`") fun address.bitsEqual(self, b: address): bool - asm "SDEQ"; + asm "SDEQ" /// Loads from slice [s] a valid `MsgAddress` (none/internal/external). @pure fun slice.loadAddress(mutate self): address - asm( -> 1 0) "LDMSGADDR"; + asm( -> 1 0) "LDMSGADDR" /** @@ -974,27 +941,27 @@ fun slice.loadAddress(mutate self): address */ /// mode = 0: Reserve exact amount of nanotoncoins -const RESERVE_MODE_EXACT_AMOUNT = 0; +const RESERVE_MODE_EXACT_AMOUNT = 0 /// +1: Actually reserves all but amount, meaning `currentContractBalance - amount` -const RESERVE_MODE_ALL_BUT_AMOUNT = 1; +const RESERVE_MODE_ALL_BUT_AMOUNT = 1 /// +2: Actually set `min(amount, currentContractBalance)` (without this mode, if amount is greater, the action will fail) -const RESERVE_MODE_AT_MOST = 2; +const RESERVE_MODE_AT_MOST = 2 /// +4: [amount] is increased by the _original_ balance of the current account (before the compute phase). -const RESERVE_MODE_INCREASE_BY_ORIGINAL_BALANCE = 4; +const RESERVE_MODE_INCREASE_BY_ORIGINAL_BALANCE = 4 /// +8: Actually sets `amount = -amount` before performing any further actions. -const RESERVE_MODE_NEGATE_AMOUNT = 8; +const RESERVE_MODE_NEGATE_AMOUNT = 8 /// +16: If this action fails, the transaction will be bounced. -const RESERVE_MODE_BOUNCE_ON_ACTION_FAIL = 16; +const RESERVE_MODE_BOUNCE_ON_ACTION_FAIL = 16 /// Creates an output action which would reserve Toncoins on balance. /// For [reserveMode] consider constants above. fun reserveToncoinsOnBalance(nanoTonCoins: coins, reserveMode: int): void - asm "RAWRESERVE"; + asm "RAWRESERVE" /// Similar to [reserveToncoinsOnBalance], but also accepts a dictionary extraAmount (represented by a cell or null) /// with extra currencies. In this way currencies other than Toncoin can be reserved. fun reserveExtraCurrenciesOnBalance(nanoTonCoins: coins, extraAmount: dict, reserveMode: int): void - asm "RAWRESERVEX"; + asm "RAWRESERVEX" /** @@ -1007,39 +974,39 @@ fun reserveExtraCurrenciesOnBalance(nanoTonCoins: coins, extraAmount: dict, rese */ /// mode = 0 is used for ordinary messages; the gas fees are deducted from the senging amount; action phaes should NOT be ignored. -const SEND_MODE_REGULAR = 0; +const SEND_MODE_REGULAR = 0 /// +1 means that the sender wants to pay transfer fees separately. -const SEND_MODE_PAY_FEES_SEPARATELY = 1; +const SEND_MODE_PAY_FEES_SEPARATELY = 1 /// +2 means that any errors arising while processing this message during the action phase should be ignored. -const SEND_MODE_IGNORE_ERRORS = 2; +const SEND_MODE_IGNORE_ERRORS = 2 /// in the case of action fail - bounce transaction. No effect if SEND_MODE_IGNORE_ERRORS (+2) is used. TVM UPGRADE 2023-07. https://docs.ton.org/learn/tvm-instructions/tvm-upgrade-2023-07#sending-messages -const SEND_MODE_BOUNCE_ON_ACTION_FAIL = 16; +const SEND_MODE_BOUNCE_ON_ACTION_FAIL = 16 /// mode = 32 means that the current account must be destroyed if its resulting balance is zero. -const SEND_MODE_DESTROY = 32; +const SEND_MODE_DESTROY = 32 /// mode = 64 is used for messages that carry all the remaining value of the inbound message in addition to the value initially indicated in the new message. -const SEND_MODE_CARRY_ALL_REMAINING_MESSAGE_VALUE = 64; +const SEND_MODE_CARRY_ALL_REMAINING_MESSAGE_VALUE = 64 /// mode = 128 is used for messages that are to carry all the remaining balance of the current smart contract (instead of the value originally indicated in the message). -const SEND_MODE_CARRY_ALL_BALANCE = 128; +const SEND_MODE_CARRY_ALL_BALANCE = 128 /// do not create an action, only estimate fee. TVM UPGRADE 2023-07. https://docs.ton.org/learn/tvm-instructions/tvm-upgrade-2023-07#sending-messages -const SEND_MODE_ESTIMATE_FEE_ONLY = 1024; +const SEND_MODE_ESTIMATE_FEE_ONLY = 1024 /// Other modes affect the fee calculation as follows: /// +64 substitutes the entire balance of the incoming message as an outcoming value (slightly inaccurate, gas expenses that cannot be estimated before the computation is completed are not taken into account). /// +128 substitutes the value of the entire balance of the contract before the start of the computation phase (slightly inaccurate, since gas expenses that cannot be estimated before the completion of the computation phase are not taken into account). -type ExtraCurrenciesDict = dict; +type ExtraCurrenciesDict = dict /// ContractState is "code + data" of a contract. /// Used in outgoing messages (StateInit) to initialize a destination contract. struct ContractState { - code: cell; - data: cell; + code: cell + data: cell } /// AddressShardingOptions provides settings to calculate an address in another shard. /// Consider [createMessage] and [address.buildSameAddressInAnotherShard] for usage. struct AddressShardingOptions { - fixedPrefixLength: uint5; // shard depth, formerly splitDepth - closeTo: address; + fixedPrefixLength: uint5 // shard depth, formerly splitDepth + closeTo: address } /// AutoDeployAddress is a destination that initializes a receiver contract if it does not exist yet. @@ -1060,9 +1027,9 @@ struct AddressShardingOptions { /// because in TON, the address of a contract, by definition, is a hash of its initial state. /// You can also use it without sending a message. See [buildAddress] and [addressMatches]. struct AutoDeployAddress { - workchain: int8 = BASECHAIN; - stateInit: ContractState | cell; - toShard: AddressShardingOptions? = null; + workchain: int8 = BASECHAIN + stateInit: ContractState | cell + toShard: AddressShardingOptions? = null } /// Constructs an address that a deployed contract will have. @@ -1076,7 +1043,7 @@ struct AutoDeployAddress { /// If you really need `address`, use `address.fromValidBuilder(res)`. @pure fun AutoDeployAddress.buildAddress(self): builder - builtin; + builtin /// Checks that an address matches a deployed contract. /// For example, from a jetton minter, you're checking whether a message is from a jetton wallet: @@ -1087,22 +1054,22 @@ fun AutoDeployAddress.buildAddress(self): builder /// Just instead of `dest` for [createMessage], you use it is to check whether a sender is a jetton/nft. @pure fun AutoDeployAddress.addressMatches(self, addr: address): bool - builtin; + builtin /// Options for creating an outgoing message. /// Consider [createMessage] for examples. struct CreateMessageOptions { /// whether a message will bounce back on error - bounce: bool; + bounce: bool /// message value: attached tons (or tons + extra currencies) - value: coins | (coins, ExtraCurrenciesDict); + value: coins | (coins, ExtraCurrenciesDict) /// destination is either a provided address, or is auto-calculated by stateInit - dest: address | // either just send a message to some address - builder | // ... or a manually constructed builder with a valid address - (int8, uint256) | // ... or to workchain + hash (also known as accountID) - AutoDeployAddress; // ... or "send to stateInit" aka deploy (address auto-calculated) + dest: address // either just send a message to some address + | builder // ... or a manually constructed builder with a valid address + | (int8, uint256) // ... or to workchain + hash (also known as accountID) + | AutoDeployAddress // ... or "send to stateInit" aka deploy (address auto-calculated) /// body is any serializable object (or just miss this field for empty body) - body: TBody; + body: TBody } /// Creates a message (`OutMessage`) — a well-formatted message cell. @@ -1123,7 +1090,7 @@ struct CreateMessageOptions { /// If you need an empty body, just miss the field `body`, fill other 3 fields. @pure fun createMessage(options: CreateMessageOptions): OutMessage - builtin; + builtin /// ExtOutLogBucket is a variant of a custom external address for emitting logs "to the outer world". @@ -1136,18 +1103,18 @@ fun createMessage(options: CreateMessageOptions): OutMessage /// > 0x01 ExtOutOffchainContract { adnl: uint256 | bits256 } /// Serialization details: '01' (addr_extern) + 256 (len) + 0x00 (prefix) + 248 bits = 267 in total struct (0x00) ExtOutLogBucket { - topic: uint248 | bits248; + topic: uint248 | bits248 } /// Options for creating an external outgoing message. /// Consider [createExternalLogMessage] for examples. struct CreateExternalLogMessageOptions { /// destination is either an external address or a pattern to calculate it - dest: address | // either some valid external/none address (not internal!) - builder | // ... or a manually constructed builder with a valid external address - ExtOutLogBucket; // ... or encode topic/eventID in destination + dest: address // either some valid external/none address (not internal!) + | builder // ... or a manually constructed builder with a valid external address + | ExtOutLogBucket // ... or encode topic/eventID in destination /// body is any serializable object (or just miss this field for empty body) - body: TBody; + body: TBody } /// Creates an external message (`OutMessage`) — a well-formatted message cell. @@ -1166,7 +1133,7 @@ struct CreateExternalLogMessageOptions { /// (if body is large, the compiler will automatically wrap it into a cell) @pure fun createExternalLogMessage(options: CreateExternalLogMessageOptions): OutMessage - builtin; + builtin /// UnsafeBodyNoRef is used to prevent default behavior: when message body is potentially large, /// it's packed into a separate ref. Wrapping body with this struct tells the compiler: @@ -1185,30 +1152,30 @@ fun createExternalLogMessage(options: CreateExternalLogMessageOptions { - forceInline: T; + forceInline: T } /// OutMessage is a result of [createMessage]. /// Essentially, it's a composed message cell, ready to be sent. struct OutMessage { - messageCell: cell; + messageCell: cell } /// Sends a ready message cell. /// For `sendMode`, see the constants above (SEND_MODE_*). fun OutMessage.send(self, sendMode: int): void - asm "SENDRAWMSG"; + asm "SENDRAWMSG" fun OutMessage.sendAndEstimateFee(self, sendMode: int): coins - asm "SENDMSG"; + asm "SENDMSG" @pure fun OutMessage.estimateFeeWithoutSending(self, sendMode: int): coins - asm "10 PUSHPOW2" "OR" "SENDMSG"; + asm "10 PUSHPOW2" "OR" "SENDMSG" @pure fun OutMessage.hash(self): uint256 - asm "HASHCU"; + asm "HASHCU" /// StateInit is a "canonical TL/B representation from block.tlb" of a contract initial state. @@ -1216,11 +1183,11 @@ fun OutMessage.hash(self): uint256 /// To represent code+data, consider [ContractState]. /// To represent sharding (fixedPrefixLength, formerly splitDepth), consider [AutoDeployAddress]. struct StateInit { - fixedPrefixLength: uint5?; - special: (bool, bool)?; - code: cell?; - data: cell?; - library: cell?; + fixedPrefixLength: uint5? + special: (bool, bool)? + code: cell? + data: cell? + library: cell? } /// Calculates a hash of StateInit if only code+data are set. @@ -1251,7 +1218,7 @@ fun StateInit.calcHashCodeData(code: cell, data: cell): uint256 asm 256 STU // store dataHash ONE HASHEXT_SHA256 -"""; +""" /// Calculates a hash of StateInit if fixedPrefixLength+code+data are set. @pure @@ -1280,7 +1247,7 @@ fun StateInit.calcHashPrefixCodeData(fixedPrefixLength: uint5, code: cell, data: ONE HASHEXT_SHA256 // pDepth hash NIP -"""; +""" /// Given an internal address A="aaaa...a" returns "bbaa...a" (D bits from address B, 256-D from A). @@ -1299,7 +1266,7 @@ fun StateInit.calcHashPrefixCodeData(fixedPrefixLength: uint5, code: cell, data: /// If you really need `address`, use `address.fromValidBuilder(resBuilder)`. @pure fun address.buildSameAddressInAnotherShard(self, options: AddressShardingOptions): builder - builtin; + builtin /// Converts a builder containing a valid address (internal/external/none) to `address` (slice under the hood). /// Gas-expensive! Because or a cell creation: essentially, it's `b.endCell().beginParse()`. @@ -1319,13 +1286,13 @@ fun address.buildSameAddressInAnotherShard(self, options: AddressShardingOptions /// ``` @pure fun address.fromValidBuilder(b: builder): address - asm "ENDC" "CTOS"; + asm "ENDC" "CTOS" /// Sends a raw message — a correctly serialized TL object `Message X`. /// In practice, you'll use a high-level wrapper [createMessage]. fun sendRawMessage(msg: cell, mode: int): void - asm "SENDRAWMSG"; + asm "SENDRAWMSG" @@ -1343,13 +1310,13 @@ fun sendRawMessage(msg: cell, mode: int): void /// in.body // actually gets from a TVM stack /// ``` struct InMessage { - senderAddress: address; // an internal address from which the message arrived - valueCoins: coins; // ton amount attached to an incoming message - valueExtra: dict; // extra currencies attached to an incoming message - originalForwardFee: coins; // fee that was paid by the sender - createdLt: uint64; // logical time when a message was created - createdAt: uint32; // unixtime when a message was created - body: slice; // message body, parse it with `lazy AllowedMsg.fromSlice(in.body)` + senderAddress: address // an internal address from which the message arrived + valueCoins: coins // ton amount attached to an incoming message + valueExtra: dict // extra currencies attached to an incoming message + originalForwardFee: coins // fee that was paid by the sender + createdLt: uint64 // logical time when a message was created + createdAt: uint32 // unixtime when a message was created + body: slice // message body, parse it with `lazy AllowedMsg.fromSlice(in.body)` } /// InMessageBounced is an input for `onBouncedMessage`. @@ -1363,51 +1330,51 @@ struct InMessage { /// val originalOpcode = in.bouncedBody.loadUint(32); /// ``` struct InMessageBounced { - senderAddress: address; // an internal address from which the message was bounced - valueCoins: coins; // ton amount attached to a message - valueExtra: dict; // extra currencies attached to a message - originalForwardFee: coins; // comission that the sender has payed to send this message - createdLt: uint64; // logical time when a message was created (and bounced) - createdAt: uint32; // unixtime when a message was created (and bounced) - bouncedBody: slice; // currently 256 bits: 0xFFFFFFFF + 224 bits sent originally + senderAddress: address // an internal address from which the message was bounced + valueCoins: coins // ton amount attached to a message + valueExtra: dict // extra currencies attached to a message + originalForwardFee: coins // comission that the sender has payed to send this message + createdLt: uint64 // logical time when a message was created (and bounced) + createdAt: uint32 // unixtime when a message was created (and bounced) + bouncedBody: slice // currently 256 bits: 0xFFFFFFFF + 224 bits sent originally } /// Skip 0xFFFFFFFF prefix (when a message is bounced). @pure fun slice.skipBouncedPrefix(mutate self): self - asm "32 LDU" "NIP"; + asm "32 LDU" "NIP" /// Load msgFlags from incoming message body (4 bits). @pure @deprecated("use `onBouncedMessage` handler instead of parsing msgCell flags manually") fun slice.loadMessageFlags(mutate self): int - asm( -> 1 0) "4 LDU"; + asm( -> 1 0) "4 LDU" /// Loads a message opcode (32-bit integer) when parsing message body manually. @pure fun slice.loadMessageOp(mutate self): int - asm( -> 1 0) "32 LDU"; + asm( -> 1 0) "32 LDU" /// Stores a message opcode (32-bit integer) when composing an output message manually. @pure @deprecated("use `createMessage` instead of composing messages manually") fun builder.storeMessageOp(mutate self, op: int): self - asm(op self) "32 STU"; + asm(op self) "32 STU" /// Loads a message queryId (64-bit integer, immediately following the opcode) when parsing message body manually. @pure @deprecated("use structures and lazy match instead of parsing messages manually") fun slice.loadMessageQueryId(mutate self): int - asm( -> 1 0) "64 LDU"; + asm( -> 1 0) "64 LDU" @pure @deprecated("use structures and lazy loading instead of parsing messages manually") fun slice.skipMessageQueryId(mutate self): self - asm "64 LDU" "NIP"; + asm "64 LDU" "NIP" @pure @deprecated("use `createMessage` instead of composing messages manually") fun builder.storeMessageQueryId(mutate self, queryId: int): self - asm(queryId self) "64 STU"; + asm(queryId self) "64 STU" diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk index 5aafedbe1..977fa6f2d 100644 --- a/crypto/smartcont/tolk-stdlib/gas-payments.tolk +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -7,7 +7,7 @@ tolk 0.99 /// Returns amount of gas (in gas units) consumed in current Computation Phase. fun getGasConsumedAtTheMoment(): int - asm "GASCONSUMED"; + asm "GASCONSUMED" /// This function is required to be called when you process an external message (from an outer world) /// and "accept" it to blockchain. @@ -15,14 +15,14 @@ fun getGasConsumedAtTheMoment(): int /// As an effect, the current smart contract agrees to buy some gas to finish the current transaction. /// For more details, check [accept_message effects](https://ton.org/docs/#/smart-contracts/accept). fun acceptExternalMessage(): void - asm "ACCEPT"; + asm "ACCEPT" /// When processing an internal message, by default, the limit of gas consumption is determined by incoming message. /// Functions [setGasLimit] and [setGasLimitToMaximum] allow you to change this behavior. /// Sets current gas limit `gl` to its maximal allowed value `gm`, and resets the gas credit `gc` to zero, /// decreasing the value of `gr` by `gc` in the process. fun setGasLimitToMaximum(): void - asm "ACCEPT"; + asm "ACCEPT" /// When processing an internal message, by default, the limit of gas consumption is determined by incoming message. /// Functions [setGasLimit] and [setGasLimitToMaximum] allow you to change this behavior. @@ -30,41 +30,41 @@ fun setGasLimitToMaximum(): void /// If the gas consumed so far (including the present instruction) exceeds the resulting value of `gl`, /// an (unhandled) out of gas exception is thrown before setting new gas limits. fun setGasLimit(limit: int): void - asm "SETGASLIMIT"; + asm "SETGASLIMIT" /// Calculates fee (amount in nanotoncoins to be paid) for a transaction which consumed [gasUsed] gas units. fun calculateGasFee(workchain: int8, gasUsed: int): coins - asm(gasUsed workchain) "GETGASFEE"; + asm(gasUsed workchain) "GETGASFEE" /// Same as [calculateGasFee], but without flat price (you have supposed to read https://docs.ton.org/develop/howto/fees-low-level) fun calculateGasFeeWithoutFlatPrice(workchain: int8, gasUsed: coins): coins - asm(gasUsed workchain) "GETGASFEESIMPLE"; + asm(gasUsed workchain) "GETGASFEESIMPLE" /// Calculates amount of nanotoncoins you should pay for storing a contract of provided size for [seconds]. /// [bits] and [cells] represent contract state (code + data). fun calculateStorageFee(workchain: int8, seconds: int, bits: int, cells: int): coins - asm(cells bits seconds workchain) "GETSTORAGEFEE"; + asm(cells bits seconds workchain) "GETSTORAGEFEE" /// Calculates amount of nanotoncoins you should pay to send a message of a specified size. fun calculateForwardFee(workchain: int8, bits: int, cells: int): coins - asm(cells bits workchain) "GETFORWARDFEE"; + asm(cells bits workchain) "GETFORWARDFEE" /// Same as [calculateForwardFee], but without lump price (you have supposed to read https://docs.ton.org/develop/howto/fees-low-level) fun calculateForwardFeeWithoutLumpPrice(workchain: int8, bits: int, cells: int): coins - asm(cells bits workchain) "GETFORWARDFEESIMPLE"; + asm(cells bits workchain) "GETFORWARDFEESIMPLE" /// Calculates fee that was paid by the sender of an incoming internal message. @deprecated("use modern `onInternalMessage` and access `in.originalForwardFee` directly") fun calculateOriginalForwardFee(workchain: int8, incomingFwdFee: coins): coins - asm(incomingFwdFee workchain) "GETORIGINALFWDFEE"; + asm(incomingFwdFee workchain) "GETORIGINALFWDFEE" /// Returns the amount of nanotoncoins current contract debts for storage. ("due" and "debt" are synonyms) /// If it has no debt, `0` is returned. fun contract.getStorageDuePayment(): coins - asm "DUEPAYMENT"; + asm "DUEPAYMENT" /// Returns the amount of nanotoncoins charged for storage. /// (during storage phase preceeding to current computation phase) @pure fun contract.getStoragePaidPayment(): coins - asm "STORAGEFEES"; + asm "STORAGEFEES" diff --git a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk index 13625c5ca..2853abef0 100644 --- a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk +++ b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk @@ -9,25 +9,25 @@ tolk 0.99 @pure fun createEmptyList(): tuple - asm "PUSHNULL"; + asm "PUSHNULL" /// Adds an element to the beginning of lisp-style list. /// Note, that it does not mutate the list: instead, it returns a new one (it's a lisp pattern). @pure fun listPrepend(head: X, tail: tuple?): tuple - asm "CONS"; + asm "CONS" /// Extracts the head and the tail of lisp-style list. @pure fun listSplit(list: tuple): (X, tuple?) - asm "UNCONS"; + asm "UNCONS" /// Returns the head of lisp-style list. @pure fun listGetHead(list: tuple): X - asm "CAR"; + asm "CAR" /// Returns the tail of lisp-style list. @pure fun listGetTail(list: tuple): tuple? - asm "CDR"; + asm "CDR" diff --git a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk index dc2d3fd6b..14723682a 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk @@ -14,289 +14,289 @@ tolk 0.99 */ /// In @stdlib/common.tolk, there is a type alias: -/// `type dict = cell?;` +/// `type dict = cell?` /// For clarity, we use "dict" instead of a "cell?" where a cell-dictionary is assumed. /// Creates an empty dictionary, which is actually a null value. Equivalent to PUSHNULL @pure fun createEmptyDict(): dict - asm "NEWDICT"; + asm "NEWDICT" /// Checks whether a dictionary is empty. @pure fun dict.dictIsEmpty(self): bool - asm "DICTEMPTY"; + asm "DICTEMPTY" @pure fun dict.iDictGet(self, keyLen: int, key: int): (slice?, bool) - asm(key self keyLen) "DICTIGET" "NULLSWAPIFNOT"; + asm(key self keyLen) "DICTIGET" "NULLSWAPIFNOT" @pure fun dict.uDictGet(self, keyLen: int, key: int): (slice?, bool) - asm(key self keyLen) "DICTUGET" "NULLSWAPIFNOT"; + asm(key self keyLen) "DICTUGET" "NULLSWAPIFNOT" @pure fun dict.sDictGet(self, keyLen: int, key: slice): (slice?, bool) - asm(key self keyLen) "DICTGET" "NULLSWAPIFNOT"; + asm(key self keyLen) "DICTGET" "NULLSWAPIFNOT" @pure fun dict.iDictSet(mutate self, keyLen: int, key: int, value: slice): void - asm(value key self keyLen) "DICTISET"; + asm(value key self keyLen) "DICTISET" @pure fun dict.uDictSet(mutate self, keyLen: int, key: int, value: slice): void - asm(value key self keyLen) "DICTUSET"; + asm(value key self keyLen) "DICTUSET" @pure fun dict.sDictSet(mutate self, keyLen: int, key: slice, value: slice): void - asm(value key self keyLen) "DICTSET"; + asm(value key self keyLen) "DICTSET" @pure fun dict.iDictSetRef(mutate self, keyLen: int, key: int, value: cell): void - asm(value key self keyLen) "DICTISETREF"; + asm(value key self keyLen) "DICTISETREF" @pure fun dict.uDictSetRef(mutate self, keyLen: int, key: int, value: cell): void - asm(value key self keyLen) "DICTUSETREF"; + asm(value key self keyLen) "DICTUSETREF" @pure fun dict.sDictSetRef(mutate self, keyLen: int, key: slice, value: cell): void - asm(value key self keyLen) "DICTSETREF"; + asm(value key self keyLen) "DICTSETREF" @pure fun dict.iDictSetIfNotExists(mutate self, keyLen: int, key: int, value: slice): bool - asm(value key self keyLen) "DICTIADD"; + asm(value key self keyLen) "DICTIADD" @pure fun dict.uDictSetIfNotExists(mutate self, keyLen: int, key: int, value: slice): bool - asm(value key self keyLen) "DICTUADD"; + asm(value key self keyLen) "DICTUADD" @pure fun dict.iDictSetIfExists(mutate self, keyLen: int, key: int, value: slice): bool - asm(value key self keyLen) "DICTIREPLACE"; + asm(value key self keyLen) "DICTIREPLACE" @pure fun dict.uDictSetIfExists(mutate self, keyLen: int, key: int, value: slice): bool - asm(value key self keyLen) "DICTUREPLACE"; + asm(value key self keyLen) "DICTUREPLACE" @pure fun dict.iDictGetRef(self, keyLen: int, key: int): (dict, bool) - asm(key self keyLen) "DICTIGETREF" "NULLSWAPIFNOT"; + asm(key self keyLen) "DICTIGETREF" "NULLSWAPIFNOT" @pure fun dict.uDictGetRef(self, keyLen: int, key: int): (dict, bool) - asm(key self keyLen) "DICTUGETREF" "NULLSWAPIFNOT"; + asm(key self keyLen) "DICTUGETREF" "NULLSWAPIFNOT" @pure fun dict.sDictGetRef(self, keyLen: int, key: slice): (dict, bool) - asm(key self keyLen) "DICTGETREF" "NULLSWAPIFNOT"; + asm(key self keyLen) "DICTGETREF" "NULLSWAPIFNOT" @pure fun dict.iDictGetRefOrNull(self, keyLen: int, key: int): dict - asm(key self keyLen) "DICTIGETOPTREF"; + asm(key self keyLen) "DICTIGETOPTREF" @pure fun dict.uDictGetRefOrNull(self, keyLen: int, key: int): dict - asm(key self keyLen) "DICTUGETOPTREF"; + asm(key self keyLen) "DICTUGETOPTREF" @pure fun dict.sDictGetRefOrNull(self, keyLen: int, key: slice): dict - asm(key self keyLen) "DICTGETOPTREF"; + asm(key self keyLen) "DICTGETOPTREF" @pure fun dict.iDictDelete(mutate self, keyLen: int, key: int): bool - asm(key self keyLen) "DICTIDEL"; + asm(key self keyLen) "DICTIDEL" @pure fun dict.uDictDelete(mutate self, keyLen: int, key: int): bool - asm(key self keyLen) "DICTUDEL"; + asm(key self keyLen) "DICTUDEL" @pure fun dict.sDictDelete(mutate self, keyLen: int, key: slice): bool - asm(key self keyLen) "DICTDEL"; + asm(key self keyLen) "DICTDEL" @pure fun dict.iDictSetAndGet(mutate self, keyLen: int, key: int, value: slice): (slice?, bool) - asm(value key self keyLen) "DICTISETGET" "NULLSWAPIFNOT"; + asm(value key self keyLen) "DICTISETGET" "NULLSWAPIFNOT" @pure fun dict.uDictSetAndGet(mutate self, keyLen: int, key: int, value: slice): (slice?, bool) - asm(value key self keyLen) "DICTUSETGET" "NULLSWAPIFNOT"; + asm(value key self keyLen) "DICTUSETGET" "NULLSWAPIFNOT" @pure fun dict.sDictSetAndGet(mutate self, keyLen: int, key: slice, value: slice): (slice?, bool) - asm(value key self keyLen) "DICTSETGET" "NULLSWAPIFNOT"; + asm(value key self keyLen) "DICTSETGET" "NULLSWAPIFNOT" @pure fun dict.iDictSetAndGetRefOrNull(mutate self, keyLen: int, key: int, value: cell): dict - asm(value key self keyLen) "DICTISETGETOPTREF"; + asm(value key self keyLen) "DICTISETGETOPTREF" @pure fun dict.uDictSetAndGetRefOrNull(mutate self, keyLen: int, key: int, value: cell): dict - asm(value key self keyLen) "DICTUSETGETOPTREF"; + asm(value key self keyLen) "DICTUSETGETOPTREF" @pure fun dict.iDictDeleteAndGet(mutate self, keyLen: int, key: int): (slice?, bool) - asm(key self keyLen) "DICTIDELGET" "NULLSWAPIFNOT"; + asm(key self keyLen) "DICTIDELGET" "NULLSWAPIFNOT" @pure fun dict.uDictDeleteAndGet(mutate self, keyLen: int, key: int): (slice?, bool) - asm(key self keyLen) "DICTUDELGET" "NULLSWAPIFNOT"; + asm(key self keyLen) "DICTUDELGET" "NULLSWAPIFNOT" @pure fun dict.sDictDeleteAndGet(mutate self, keyLen: int, key: slice): (slice?, bool) - asm(key self keyLen) "DICTDELGET" "NULLSWAPIFNOT"; + asm(key self keyLen) "DICTDELGET" "NULLSWAPIFNOT" @pure fun dict.iDictSetBuilder(mutate self, keyLen: int, key: int, value: builder): void - asm(value key self keyLen) "DICTISETB"; + asm(value key self keyLen) "DICTISETB" @pure fun dict.uDictSetBuilder(mutate self, keyLen: int, key: int, value: builder): void - asm(value key self keyLen) "DICTUSETB"; + asm(value key self keyLen) "DICTUSETB" @pure fun dict.sDictSetBuilder(mutate self, keyLen: int, key: slice, value: builder): void - asm(value key self keyLen) "DICTSETB"; + asm(value key self keyLen) "DICTSETB" @pure fun dict.iDictSetBuilderIfNotExists(mutate self, keyLen: int, key: int, value: builder): bool - asm(value key self keyLen) "DICTIADDB"; + asm(value key self keyLen) "DICTIADDB" @pure fun dict.uDictSetBuilderIfNotExists(mutate self, keyLen: int, key: int, value: builder): bool - asm(value key self keyLen) "DICTUADDB"; + asm(value key self keyLen) "DICTUADDB" @pure fun dict.iDictSetBuilderIfExists(mutate self, keyLen: int, key: int, value: builder): bool - asm(value key self keyLen) "DICTIREPLACEB"; + asm(value key self keyLen) "DICTIREPLACEB" @pure fun dict.uDictSetBuilderIfExists(mutate self, keyLen: int, key: int, value: builder): bool - asm(value key self keyLen) "DICTUREPLACEB"; + asm(value key self keyLen) "DICTUREPLACEB" @pure fun dict.iDictDeleteFirstAndGet(mutate self, keyLen: int): (int?, slice?, bool) - asm(-> 0 2 1 3) "DICTIREMMIN" "NULLSWAPIFNOT2"; + asm(-> 0 2 1 3) "DICTIREMMIN" "NULLSWAPIFNOT2" @pure fun dict.uDictDeleteFirstAndGet(mutate self, keyLen: int): (int?, slice?, bool) - asm(-> 0 2 1 3) "DICTUREMMIN" "NULLSWAPIFNOT2"; + asm(-> 0 2 1 3) "DICTUREMMIN" "NULLSWAPIFNOT2" @pure fun dict.sDictDeleteFirstAndGet(mutate self, keyLen: int): (slice?, slice?, bool) - asm(-> 0 2 1 3) "DICTREMMIN" "NULLSWAPIFNOT2"; + asm(-> 0 2 1 3) "DICTREMMIN" "NULLSWAPIFNOT2" @pure fun dict.iDictDeleteLastAndGet(mutate self, keyLen: int): (int?, slice?, bool) - asm(-> 0 2 1 3) "DICTIREMMAX" "NULLSWAPIFNOT2"; + asm(-> 0 2 1 3) "DICTIREMMAX" "NULLSWAPIFNOT2" @pure fun dict.uDictDeleteLastAndGet(mutate self, keyLen: int): (int?, slice?, bool) - asm(-> 0 2 1 3) "DICTUREMMAX" "NULLSWAPIFNOT2"; + asm(-> 0 2 1 3) "DICTUREMMAX" "NULLSWAPIFNOT2" @pure fun dict.sDictDeleteLastAndGet(mutate self, keyLen: int): (slice?, slice?, bool) - asm(-> 0 2 1 3) "DICTREMMAX" "NULLSWAPIFNOT2"; + asm(-> 0 2 1 3) "DICTREMMAX" "NULLSWAPIFNOT2" @pure fun dict.iDictGetFirst(self, keyLen: int): (int?, slice?, bool) - asm (-> 1 0 2) "DICTIMIN" "NULLSWAPIFNOT2"; + asm (-> 1 0 2) "DICTIMIN" "NULLSWAPIFNOT2" @pure fun dict.uDictGetFirst(self, keyLen: int): (int?, slice?, bool) - asm (-> 1 0 2) "DICTUMIN" "NULLSWAPIFNOT2"; + asm (-> 1 0 2) "DICTUMIN" "NULLSWAPIFNOT2" @pure fun dict.sDictGetFirst(self, keyLen: int): (slice?, slice?, bool) - asm (-> 1 0 2) "DICTMIN" "NULLSWAPIFNOT2"; + asm (-> 1 0 2) "DICTMIN" "NULLSWAPIFNOT2" @pure fun dict.iDictGetFirstAsRef(self, keyLen: int): (int?, dict, bool) - asm (-> 1 0 2) "DICTIMINREF" "NULLSWAPIFNOT2"; + asm (-> 1 0 2) "DICTIMINREF" "NULLSWAPIFNOT2" @pure fun dict.uDictGetFirstAsRef(self, keyLen: int): (int?, dict, bool) - asm (-> 1 0 2) "DICTUMINREF" "NULLSWAPIFNOT2"; + asm (-> 1 0 2) "DICTUMINREF" "NULLSWAPIFNOT2" @pure fun dict.sDictGetFirstAsRef(self, keyLen: int): (slice?, dict, bool) - asm (-> 1 0 2) "DICTMINREF" "NULLSWAPIFNOT2"; + asm (-> 1 0 2) "DICTMINREF" "NULLSWAPIFNOT2" @pure fun dict.iDictGetLast(self, keyLen: int): (int?, slice?, bool) - asm (-> 1 0 2) "DICTIMAX" "NULLSWAPIFNOT2"; + asm (-> 1 0 2) "DICTIMAX" "NULLSWAPIFNOT2" @pure fun dict.uDictGetLast(self, keyLen: int): (int?, slice?, bool) - asm (-> 1 0 2) "DICTUMAX" "NULLSWAPIFNOT2"; + asm (-> 1 0 2) "DICTUMAX" "NULLSWAPIFNOT2" @pure fun dict.sDictGetLast(self, keyLen: int): (slice?, slice?, bool) - asm (-> 1 0 2) "DICTMAX" "NULLSWAPIFNOT2"; + asm (-> 1 0 2) "DICTMAX" "NULLSWAPIFNOT2" @pure fun dict.iDictGetLastAsRef(self, keyLen: int): (int?, dict, bool) - asm (-> 1 0 2) "DICTIMAXREF" "NULLSWAPIFNOT2"; + asm (-> 1 0 2) "DICTIMAXREF" "NULLSWAPIFNOT2" @pure fun dict.uDictGetLastAsRef(self, keyLen: int): (int?, dict, bool) - asm (-> 1 0 2) "DICTUMAXREF" "NULLSWAPIFNOT2"; + asm (-> 1 0 2) "DICTUMAXREF" "NULLSWAPIFNOT2" @pure fun dict.sDictGetLastAsRef(self, keyLen: int): (slice?, dict, bool) - asm (-> 1 0 2) "DICTMAXREF" "NULLSWAPIFNOT2"; + asm (-> 1 0 2) "DICTMAXREF" "NULLSWAPIFNOT2" @pure fun dict.iDictGetNext(self, keyLen: int, pivot: int): (int?, slice?, bool) - asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXT" "NULLSWAPIFNOT2"; + asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXT" "NULLSWAPIFNOT2" @pure fun dict.uDictGetNext(self, keyLen: int, pivot: int): (int?, slice?, bool) - asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXT" "NULLSWAPIFNOT2"; + asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXT" "NULLSWAPIFNOT2" @pure fun dict.iDictGetNextOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) - asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXTEQ" "NULLSWAPIFNOT2"; + asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXTEQ" "NULLSWAPIFNOT2" @pure fun dict.uDictGetNextOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) - asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXTEQ" "NULLSWAPIFNOT2"; + asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXTEQ" "NULLSWAPIFNOT2" @pure fun dict.iDictGetPrev(self, keyLen: int, pivot: int): (int?, slice?, bool) - asm(pivot self keyLen -> 1 0 2) "DICTIGETPREV" "NULLSWAPIFNOT2"; + asm(pivot self keyLen -> 1 0 2) "DICTIGETPREV" "NULLSWAPIFNOT2" @pure fun dict.uDictGetPrev(self, keyLen: int, pivot: int): (int?, slice?, bool) - asm(pivot self keyLen -> 1 0 2) "DICTUGETPREV" "NULLSWAPIFNOT2"; + asm(pivot self keyLen -> 1 0 2) "DICTUGETPREV" "NULLSWAPIFNOT2" @pure fun dict.iDictGetPrevOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) - asm(pivot self keyLen -> 1 0 2) "DICTIGETPREVEQ" "NULLSWAPIFNOT2"; + asm(pivot self keyLen -> 1 0 2) "DICTIGETPREVEQ" "NULLSWAPIFNOT2" @pure fun dict.uDictGetPrevOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) - asm(pivot self keyLen -> 1 0 2) "DICTUGETPREVEQ" "NULLSWAPIFNOT2"; + asm(pivot self keyLen -> 1 0 2) "DICTUGETPREVEQ" "NULLSWAPIFNOT2" /** @@ -305,12 +305,12 @@ fun dict.uDictGetPrevOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool @pure fun dict.prefixDictGet(self, keyLen: int, key: slice): (slice, slice?, slice?, bool) - asm(key self keyLen) "PFXDICTGETQ" "NULLSWAPIFNOT2"; + asm(key self keyLen) "PFXDICTGETQ" "NULLSWAPIFNOT2" @pure fun dict.prefixDictSet(mutate self, keyLen: int, key: slice, value: slice): bool - asm(value key self keyLen) "PFXDICTSET"; + asm(value key self keyLen) "PFXDICTSET" @pure fun dict.prefixDictDelete(mutate self, keyLen: int, key: slice): bool - asm(key self keyLen) "PFXDICTDEL"; + asm(key self keyLen) "PFXDICTDEL" diff --git a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk index 724464b70..18a0e159f 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk @@ -5,21 +5,21 @@ tolk 0.99 /// The primitive returns the current value of `c3`. @pure fun getTvmRegisterC3(): continuation - asm "c3 PUSH"; + asm "c3 PUSH" /// Updates the current value of `c3`. Usually, it is used for updating smart contract code in run-time. /// Note that after execution of this primitive the current code /// (and the stack of recursive function calls) won't change, /// but any other function call will use a function from the new code. fun setTvmRegisterC3(c: continuation): void - asm "c3 POP"; + asm "c3 POP" /// Transforms a `slice` [s] into a simple ordinary continuation `c`, with `c.code = s` and an empty stack and savelist. @pure fun transformSliceToContinuation(s: slice): continuation - asm "BLESS"; + asm "BLESS" /// Moves a variable or a value [x] to the top of the stack. @pure fun T.stackMoveToTop(mutate self): void - asm "NOP"; + asm "NOP" diff --git a/tolk-tester/tests/assignment-tests.tolk b/tolk-tester/tests/assignment-tests.tolk index bd9b28d9b..5b8c2094c 100644 --- a/tolk-tester/tests/assignment-tests.tolk +++ b/tolk-tester/tests/assignment-tests.tolk @@ -1,5 +1,5 @@ fun extractFromTypedTuple(params: [int]) { - var [payload: int] = params; + var [payload: int, ] = params; return payload + 10; } @@ -111,7 +111,7 @@ global t107: (int, int); @method_id(107) fun test107() { - ((t107 = (1, 2)).0, (t107 = (3, 4)).1) = (5, 6); + ((t107 = (1, 2)).0, (t107 = (3, 4)).1, ) = (5, 6, ); return t107; } @@ -168,7 +168,7 @@ fun test112() { @method_id(113) fun test113() { - var (a, t, z) = (1, [2,3], (-1,-1)); + var (a, t, z, ) = (1, [2,3], (-1,-1)); (a, t, a, z, t.1, z.1) = (10, [a,12], 13, (a, t.1), 14, t.1); return (a, t, z); } @@ -252,7 +252,7 @@ fun test118(x: int) { fun main(value: int, ) { - var (x: int?, y) = (autoInferIntNull(value), autoInferIntNull(value * 2)); + var (x: int?, y,) = (autoInferIntNull(value), autoInferIntNull(value * 2)); if (x == null && y == null) { return null; } return x == null || y == null ? -1 : x! + y!; } diff --git a/tolk-tester/tests/constants-tests.tolk b/tolk-tester/tests/constants-tests.tolk index 607cccf0a..cafd2ffde 100644 --- a/tolk-tester/tests/constants-tests.tolk +++ b/tolk-tester/tests/constants-tests.tolk @@ -1,8 +1,8 @@ -type MInt = int; -type MSlice = slice; +type MInt = int +type MSlice = slice -const int1 = 1; -const int2 = 2; +const int1 = 1 +const int2 = 2 const int101: int = 101; const int111: MInt = 111; @@ -24,9 +24,8 @@ const strange_minus_1: MInt = (!0 as int); const addr1 = address("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"); -const true1 = true; -const true2 = !!true; -const true3 = true1 && true2; +// since `;` is not mandatory, this is correct from a syntax point of view +const true1 = true const true2 = !!true const true3 = true1 && true2 const false1 = !true; const false2 = false1 || false; diff --git a/tolk-tester/tests/indexed-access.tolk b/tolk-tester/tests/indexed-access.tolk index dd0400c28..af6888df7 100644 --- a/tolk-tester/tests/indexed-access.tolk +++ b/tolk-tester/tests/indexed-access.tolk @@ -41,8 +41,8 @@ fun plus(mutate x: int, y: int): int { fun eq(v: X): X { return v; } -global gTup: [int]; -global gTens: (int, int); +global gTup: [int] +global gTens: (int, int) @method_id(100) fun testCodegenSimple() { diff --git a/tolk-tester/tests/invalid-declaration/err-1043.tolk b/tolk-tester/tests/invalid-declaration/err-1043.tolk new file mode 100644 index 000000000..4ef7d292f --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1043.tolk @@ -0,0 +1,6 @@ +get id() { return 123; } + +/** +@compilation_should_fail +@stderr expected `fun`, got `id` + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1794.tolk b/tolk-tester/tests/invalid-declaration/err-1794.tolk index a064a633c..3505572cf 100644 --- a/tolk-tester/tests/invalid-declaration/err-1794.tolk +++ b/tolk-tester/tests/invalid-declaration/err-1794.tolk @@ -1,4 +1,4 @@ -get int.seqno(self) { +get fun int.seqno(self) { return 0; } diff --git a/tolk-tester/tests/invalid-syntax/err-3618.tolk b/tolk-tester/tests/invalid-syntax/err-3618.tolk index 7751c4a95..0c3fa7d3f 100644 --- a/tolk-tester/tests/invalid-syntax/err-3618.tolk +++ b/tolk-tester/tests/invalid-syntax/err-3618.tolk @@ -2,5 +2,5 @@ const x = 0b0012; /** @compilation_should_fail -@stderr expected `;`, got `2` +@stderr expected top-level declaration, got `2` */ diff --git a/tolk-tester/tests/invalid-syntax/err-3802.tolk b/tolk-tester/tests/invalid-syntax/err-3802.tolk deleted file mode 100644 index fec840c98..000000000 --- a/tolk-tester/tests/invalid-syntax/err-3802.tolk +++ /dev/null @@ -1,9 +0,0 @@ -struct A { - id1: int - id2: int -} - -/** -@compilation_should_fail -@stderr expected `;` or `,`, got `id2` -*/ diff --git a/tolk-tester/tests/methods-tests.tolk b/tolk-tester/tests/methods-tests.tolk index db3c3e9cd..4b09a53d6 100644 --- a/tolk-tester/tests/methods-tests.tolk +++ b/tolk-tester/tests/methods-tests.tolk @@ -207,12 +207,12 @@ fun test12() { __expect_type((null as slice?).arbitraryMethod(), "never"); } -type BuilderOrInt = builder|int; -fun (int|builder).isInt(self) { return self is int; } +type BuilderOrInt = |builder|int +fun|int|builder.isInt(self) { return self is int; } @method_id(113) fun test13() { - var u1 = beginCell() as builder|int; + var u1 = beginCell() as|builder|int; var u2 = 10 as int|builder; var u3 = 20 as BuilderOrInt; return (u1.isInt(), u2.isInt(), u3.isInt(), 10.isInt(), beginCell().isInt(), (8 as int8).isInt()); @@ -224,7 +224,7 @@ fun (int|builder|T).isBuilder(self) { return self is builder; } fun test14() { var b1 = beginCell() as BuilderOrInt | slice; var b2 = beginCell() as builder|slice|int|Point; - var i1 = 5 as int|builder|continuation|(int, int); + var i1 = 5 as |int|builder|continuation|(int, int); var c = beginCell().endCell() as cell?|builder|int; var cn = null as cell?|int|int8|builder; return (b1.isBuilder(), b2.isBuilder(), i1.isBuilder(), c.isBuilder(), cn.isBuilder()); diff --git a/tolk-tester/tests/mutate-methods.tolk b/tolk-tester/tests/mutate-methods.tolk index 892a625ce..f3e93b694 100644 --- a/tolk-tester/tests/mutate-methods.tolk +++ b/tolk-tester/tests/mutate-methods.tolk @@ -138,8 +138,8 @@ fun updateTwoItems(mutate pair: Pair2, byValue: int) { pair = (first + byValue, second + byValue); } -global t107_1: int; -global t107_2: int; +global t107_1: int +global t107_2: int @method_id(107) fun testMutableTensor() { diff --git a/tolk-tester/tests/no-spaces.tolk b/tolk-tester/tests/no-spaces.tolk index 732de8011..3a703dc7c 100644 --- a/tolk-tester/tests/no-spaces.tolk +++ b/tolk-tester/tests/no-spaces.tolk @@ -1,8 +1,10 @@ -const int10:int=10; +const int10:int=10 const dd=1 fun just10(): int { return int10; } fun eq(v: int): int { return`v` } +const ONE = 1struct Point{x:int y:int} + @method_id(101,) fun `get_-1` (): int {return-1} @method_id(102,) fun `get_--1` (): int {return- -1} @method_id(103,) fun `get_---1`(): int {return-(-(-1))} @@ -13,7 +15,7 @@ global `some()var`:int; @method_id(110) fun `some_math`(): int { `some()var`=- -6; - return 1*-2*-3*-4*just10()*-5+-`some()var`+-(-`some()var`)-(-(-`some()var`)); + return ONE*-2*-3*-4*just10()*-5+-`some()var`+-(-`some()var`)-(-(-`some()var`)); } @method_id(111) fun `negative_nums`(a:int):int { diff --git a/tolk-tester/tests/pack-unpack-3.tolk b/tolk-tester/tests/pack-unpack-3.tolk index 310a3ed6c..793f714b9 100644 --- a/tolk-tester/tests/pack-unpack-3.tolk +++ b/tolk-tester/tests/pack-unpack-3.tolk @@ -114,11 +114,10 @@ struct(0x00184300) CounterResetTo { } type MsgCounter1 = - CounterIncrement | - CounterDecrement | - CounterReset0 | - CounterResetTo | -; + | CounterIncrement + | CounterDecrement + | CounterReset0 + | CounterResetTo /* bodyPayload1$001 @@ -146,7 +145,7 @@ struct(0b01) BodyPayload2 { owner_address: address; } -type BodyPayload = BodyPayload1 | BodyPayload2; +type BodyPayload = BodyPayload1 | BodyPayload2 /* sayHiAndGoodbye#89 @@ -172,7 +171,7 @@ struct(0x0013) SayStoreInChain { contents: Cell; } -type MsgExternal1 = SayHiAndGoodbye | SayStoreInChain; +type MsgExternal1 = SayHiAndGoodbye | SayStoreInChain /* transferParams1#794 diff --git a/tolk-tester/tests/pack-unpack-5.tolk b/tolk-tester/tests/pack-unpack-5.tolk index af397181e..1baa24064 100644 --- a/tolk-tester/tests/pack-unpack-5.tolk +++ b/tolk-tester/tests/pack-unpack-5.tolk @@ -118,11 +118,11 @@ fun test6() { return (Test6_1.estimatePackSize(), Test6_2.estimatePackSize(), Test6_or.estimatePackSize()); } -struct(0x1020) Test7_1 { } -struct(0x1030) Test7_2 { } -struct(0x1040) Test7_3 { } +struct(0x1020) Test7_1; +struct(0x1030) Test7_2; +struct(0x1040) Test7_3; -type Test7_or = Test7_1 | Test7_2 | Test7_3 |; +type Test7_or = | Test7_1 | Test7_2 | Test7_3 @method_id(107) fun test7() { diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index 1a0f96a0f..90f1ae644 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -168,7 +168,7 @@ fun test10(): int { @method_id(111) fun test11() { - var [x, y] = [getNullableInt(), getNullableInt()]; + var [x, y,] = [getNullableInt(), getNullableInt()]; if (eq(10) < 0) { return x == null || y == null ? -1 : x + y; } if (true && (x == null || y == null) && !!true) { return 0; } return x + y; diff --git a/tolk-tester/tests/strings-tests.tolk b/tolk-tester/tests/strings-tests.tolk index ab2facf8a..863a8aeff 100644 --- a/tolk-tester/tests/strings-tests.tolk +++ b/tolk-tester/tests/strings-tests.tolk @@ -1,20 +1,20 @@ -get ascii_slice(): slice { +get fun ascii_slice(): slice { return"string"; } -get raw_slice(): slice { +get fun raw_slice(): slice { return stringHexToSlice("abcdef"); } -get addr_slice(): address { +get fun addr_slice(): address { return address("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"); } -get string_hex(): int { +get fun string_hex(): int { return stringToBase256("ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"); } -get fun string_minihash(): int { // 'get' and 'get fun' both possible +get fun string_minihash(): int { return stringSha256_32("transfer(slice, int)"); } diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index 18d596c34..8eba3b427 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -6,8 +6,8 @@ struct JustIntWrapper { int: JustInt; } -type MInt = int; -type JustIntAlias = JustInt; +type MInt = int +type JustIntAlias = JustInt type JustIntAlias2 = JustIntAlias; struct Storage { @@ -16,13 +16,13 @@ struct Storage { } struct Point { - x: int; + x: int y: MInt } struct User { - id: int; - name: slice; + id: int + name: slice } struct OuterStorage { @@ -31,10 +31,10 @@ struct OuterStorage { } struct WithTensorInside { - coords: (int, int); - tup: [int, int]; - otherCoords: (int, int)?; - otherTup: tuple; + coords: (int, int) + tup: [int, int] + otherCoords: (int, int)? + otherTup: tuple } fun sumCoords(p: Point) { @@ -133,9 +133,12 @@ fun test7() { return (s.owner.name.loadInt(32), s2.owner.name.loadInt(32), s.lastPoint, s2.lastPoint.sumCoords()); } -struct Empty{} +struct Empty struct OuterEmpty { nothing: EmptyAlias } -type EmptyAlias = Empty; +type EmptyAlias = Empty + +struct EmptyGeneric1; +struct EmptyGeneric2 @method_id(108) fun test8() { @@ -152,6 +155,7 @@ fun test8() { __expect_type(o3, "Empty?"); __expect_type(o6, "OuterEmpty"); __expect_type(o8, "OuterEmpty?"); + __expect_type(EmptyGeneric2{}, "EmptyGeneric2"); return (e1, Empty{}, EmptyAlias{}, o2, o1, 777, o3, 777, o4, 777, o5, 777, o6, 777, o7, 777, o8, 777, o3!, 777); } diff --git a/tolk-tester/tests/type-aliases-tests.tolk b/tolk-tester/tests/type-aliases-tests.tolk index 3fc534505..3bc0e5114 100644 --- a/tolk-tester/tests/type-aliases-tests.tolk +++ b/tolk-tester/tests/type-aliases-tests.tolk @@ -1,13 +1,13 @@ import "@stdlib/tvm-dicts.tolk" -type MIntN = MInt?; -type MInt = int; -type MInt_v2 = int; -type MVoid = void; -type Pair2_v1 = (int, int); -type Pair2_v2 = (MInt, MInt); -type MBool = bool; -type Tuple2Int = [int, int]; +type MIntN = MInt? +type MInt = int +type MInt_v2 = int +type MVoid = void +type Pair2_v1 = | (int, int) +type Pair2_v2 = (MInt, MInt) +type MBool = | bool +type Tuple2Int = [int, int] struct Point { x: int; y: int } type PointAlias = Point; @@ -160,7 +160,7 @@ fun test10() { var (a2, b2) = x2; __expect_type(a2, "MInt"); var x3: Tuple2Int = [9, 10]; - var [a3, b3] = x3; + var [a3, b3, ] = x3; return a1 + a2 + a3 + b1 + b2 + b3; } diff --git a/tolk-tester/tests/union-types-tests.tolk b/tolk-tester/tests/union-types-tests.tolk index 5dc3c9fca..7b9d52b61 100644 --- a/tolk-tester/tests/union-types-tests.tolk +++ b/tolk-tester/tests/union-types-tests.tolk @@ -1,10 +1,10 @@ -type MInt = int; -type MNull = null; -type MSlice = slice; -type MIntN = int | MNull; -type IntOrSlice = MInt | slice; -type IntOrSliceN = null | slice | int; -type IntOrSliceOrBuilder = int | slice | builder; +type MInt = int +type MNull = | null +type MSlice = slice +type MIntN = int | MNull +type IntOrSlice = MInt | slice +type IntOrSliceN = null | slice | int +type IntOrSliceOrBuilder = | int | slice | builder type Pair2 = (int, int); type Pair2Or3 = Pair2 | (int, int, int); @@ -18,8 +18,8 @@ fun testFlatten() { __expect_type(0 as (int | int | slice | builder | slice), "int | slice | builder"); __expect_type(0 as (int? | int), "int?"); __expect_type(0 as (slice | int? | slice? | cell), "slice | int | null | cell"); - __expect_type(0 as (int | (int | int)), "int"); - __expect_type(0 as (slice | (slice | (slice | int) | (cell | slice?))), "slice | int | cell | null"); + __expect_type(0 as (| int | (int | int)), "int"); + __expect_type(0 as (slice | (slice | (slice | int) | (| cell | slice?))), "slice | int | cell | null"); __expect_type(0 as (MInt | MNull), "MInt?"); __expect_type(0 as (MInt | slice | int), "MInt | slice"); @@ -43,7 +43,7 @@ fun testFlatten() { __expect_type(0 as (int?)?, "int?"); __expect_type(0 as (int?)? | (int)?, "int?"); - __expect_type(0 is builder, "bool"); + __expect_type(0 is (|builder), "bool"); __expect_type(0 !is builder, "bool"); __expect_type(0!!is builder, "bool"); @@ -80,7 +80,7 @@ fun test2() { } @method_id(103) -fun test3(case: int): int | slice | null { +fun test3(case: int): | int | slice | null { var a: int | slice | null = 4; if (case == 0) { a = getSlice32() diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 98d4e546e..39cef55b7 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -236,6 +236,9 @@ static AnyTypeV parse_type_nullable(Lexer& lex) { } static AnyTypeV parse_type_expression(Lexer& lex) { + if (lex.tok() == tok_bitwise_or) { // allow leading `|`, like in TypeScript + lex.next(); + } AnyTypeV result = parse_type_nullable(lex); if (lex.tok() == tok_bitwise_or) { // `int | slice`, `Pair2 | (Pair3 | null)` @@ -243,9 +246,6 @@ static AnyTypeV parse_type_expression(Lexer& lex) { items.emplace_back(result); while (lex.tok() == tok_bitwise_or) { lex.next(); - if (lex.tok() == tok_clpar || lex.tok() == tok_clbracket || lex.tok() == tok_semicolon) { - break; // allow trailing `|` (not leading, like in TypeScript, because of tree-sitter) - } items.emplace_back(parse_type_nullable(lex)); } result = createV(result->loc, std::move(items)); @@ -368,7 +368,6 @@ static AnyV parse_global_var_declaration(Lexer& lex, const std::vectorkind) { @@ -399,7 +398,6 @@ static AnyV parse_constant_declaration(Lexer& lex, const std::vectorkind) { @@ -428,7 +426,6 @@ static AnyV parse_type_alias_declaration(Lexer& lex, const std::vectorkind) { @@ -455,6 +452,9 @@ static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable, bool al std::vector args(1, first); while (lex.tok() == tok_comma) { lex.next(); + if (lex.tok() == tok_clpar) { // trailing comma + break; + } args.push_back(parse_var_declaration_lhs(lex, is_immutable, false)); } lex.expect(tok_clpar, "`)`"); @@ -465,6 +465,9 @@ static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable, bool al std::vector args(1, parse_var_declaration_lhs(lex, is_immutable, false)); while (lex.tok() == tok_comma) { lex.next(); + if (lex.tok() == tok_clbracket) { // trailing comma + break; + } args.push_back(parse_var_declaration_lhs(lex, is_immutable, false)); } lex.expect(tok_clbracket, "`]`"); @@ -1327,7 +1330,6 @@ static AnyV parse_asm_func_body(Lexer& lex, V param_list) { asm_commands.push_back(createV(lex.cur_location(), asm_command)); lex.next(); } - lex.expect(tok_semicolon, "`;`"); return createV(loc, std::move(arg_order), std::move(ret_order), std::move(asm_commands)); } @@ -1388,13 +1390,9 @@ static V parse_annotation(Lexer& lex) { return createV(loc, name, kind, v_arg); } -static AnyV parse_function_declaration(Lexer& lex, const std::vector>& annotations) { +static AnyV parse_function_declaration(Lexer& lex, const std::vector>& annotations, bool is_contract_getter) { SrcLocation loc = lex.cur_location(); - bool is_contract_getter = lex.cur_str() == "get"; - lex.next(); - if (is_contract_getter && lex.tok() == tok_fun) { - lex.next(); // 'get f()' and 'get fun f()' both correct - } + lex.expect(tok_fun, "`fun`"); AnyTypeV receiver_type = nullptr; auto backup = lex.save_parsing_position(); @@ -1460,7 +1458,6 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector(lex.cur_location()); lex.next(); - lex.expect(tok_semicolon, "`;`"); } else if (lex.tok() == tok_opbrace) { v_body = parse_func_body(lex); } else if (lex.tok() == tok_asm) { @@ -1558,14 +1555,12 @@ static V parse_struct_body(Lexer& lex) { SrcLocation loc = lex.cur_location(); std::vector fields; - if (lex.tok() != tok_semicolon) { // `struct A;` equal to `struct A {}` - lex.expect(tok_opbrace, "`{`"); + if (lex.tok() == tok_opbrace) { // `struct A` equal to `struct A {}` + lex.next(); while (lex.tok() != tok_clbrace) { fields.push_back(parse_struct_field(lex)); if (lex.tok() == tok_comma || lex.tok() == tok_semicolon) { lex.next(); - } else if (lex.tok() != tok_clbrace) { - lex.unexpected("`;` or `,`"); } } lex.expect(tok_clbrace, "`}`"); @@ -1700,7 +1695,7 @@ AnyV parse_src_file_to_ast(const SrcFile* file) { annotations.clear(); break; case tok_fun: - toplevel_declarations.push_back(parse_function_declaration(lex, annotations)); + toplevel_declarations.push_back(parse_function_declaration(lex, annotations, false)); annotations.clear(); break; case tok_struct: @@ -1715,8 +1710,9 @@ AnyV parse_src_file_to_ast(const SrcFile* file) { lex.error("`" + static_cast(lex.cur_str()) +"` is not supported yet"); case tok_identifier: - if (lex.cur_str() == "get") { // tok-level "get", contract getter - toplevel_declarations.push_back(parse_function_declaration(lex, annotations)); + if (lex.cur_str() == "get") { // top-level "get", contract getter + lex.next(); + toplevel_declarations.push_back(parse_function_declaration(lex, annotations, true)); annotations.clear(); break; } From 06e0544376a1866a7befd96617757c737e27a11d Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Fri, 4 Jul 2025 14:38:41 +0300 Subject: [PATCH 340/388] [Tolk] Gas optimization: replace ternary with CONDSEL when it's safe --- tolk-tester/tests/ternary-tests.tolk | 367 +++++++++++++++++++++++++++ tolk/builtins.cpp | 20 ++ tolk/pipe-ast-to-legacy.cpp | 28 ++ 3 files changed, 415 insertions(+) create mode 100644 tolk-tester/tests/ternary-tests.tolk diff --git a/tolk-tester/tests/ternary-tests.tolk b/tolk-tester/tests/ternary-tests.tolk new file mode 100644 index 000000000..a1e88a8e8 --- /dev/null +++ b/tolk-tester/tests/ternary-tests.tolk @@ -0,0 +1,367 @@ +const TWO = 2; +global gTup: tuple; + +fun get1() { gTup.push(1); return 1; } +fun get2() { gTup.push(2); return 2; } + +struct Point { + x: int; + y: int; +} + +@noinline +fun Point.create0NoInline(): Point { + return { x: 0, y: 0 } +} + +fun Point.create0(): Point { + return { x: 0, y: 0 } +} + +@method_id(101) +fun test101(x: int) { + return (x ? 2 : 1, x == 5 ? TWO : 1); +} + +@method_id(102) +fun test102(x: int?) { + return x == null ? (0!) : ((x)); +} + +@method_id(103) +fun test103(x: bool) { + if (x ? -1 : +1) { + return x ? -2 : +2; + } + throw 123; +} + +@method_id(104) +fun test104(x: (int, int)) { + return x.0 ? true : null; +} + +@method_id(105) +fun test105(getX: bool) { + var p: Point = { x: 10, y: 20 }; + return getX ? p.x : p!.y; +} + +@noinline +fun helper106(glob: bool, t: tuple) { + return glob ? gTup : t; +} + +@method_id(106) +fun test106() { + gTup = createEmptyTuple(); + gTup.push(1); + return helper106(true, createEmptyTuple()); +} + +@noinline +fun helper107(t: (int, Point)) { + return t.0 ? t.0 : t.1.y; +} + +@method_id(107) +fun test107() { + return (helper107((1, {x: 10, y:20})), helper107((0, {x: 10, y: 20}))); +} + +@method_id(108) +fun test108(calcMin: bool) { + var cb = calcMin ? min : max; + return cb(10, 20); +} + +@method_id(109) +fun test109(bigCost: bool) { + return bigCost ? ton("0.05") : (ton("0.001")); +} + +@method_id(110) +fun test110(k: int) { + val c: [int, int, int] = [k, 6, 7]; + return c.0 > 0 ? c.1 : c.2; +} + +@method_id(111) +fun test111(g: bool) { + return g ? (1, 2) : (3, 4); +} + + +@method_id(200) +fun test200(x: int) { + return 10 > 3 ? 200 : 100; +} + +@method_id(201) +fun test201(x: int) { + return (x is slice) ? 200 : x; +} + +@method_id(202) +fun test202(x: int) { + x > 10 ? x : 0; + x < 10 ? x! : -10; + return x; +} + +@method_id(203) +fun test203() { + var xx = false; + if (!xx) { + return xx ? 100 : 200; + } + return 300; +} + + +@method_id(220) +fun test220(p: Point?, getNull: bool) { + if (p == null) { + return getNull ? p : 20; + } + return p.y; +} + +@method_id(221) +fun test221(c: (int, (int,int)?)) { + if (c.1 != null) { + return c.0 > 5 ? c.1.0 : c.1.1; + } + return c.0 > 5 ? 0 : c.1; +} + + +@method_id(300) +fun test300(x: int): tuple | int { + // no condsel, because more that 1 slot + return x > 0 ? x : gTup; +} + +@method_id(301) +fun test301(first: bool) { + var opt1 = (10, 20); + var opt2 = (30, 40); + // no condsel, because not 1 slot arguments + return first ? opt1 : opt2; +} + +@method_id(302) +fun test302(x: int?) { + // no condsel, because not trivial (unary operator) + return x == null ? 0 : -x; +} + +@method_id(303) +fun test303(first: bool) { + gTup = createEmptyTuple(); + // no condsel, because not trivial + return (first ? get1() : get2(), gTup); +} + +@method_id(304) +fun test304(y: bool) { + return y ? Point.create0NoInline().y : 5; +} + +@method_id(305) +fun test305(y: bool) { + return y ? Point.create0().y : 5; +} + + +fun main() {} + +/** +@testcase | 101 | 0 | 1 1 +@testcase | 101 | 5 | 2 2 +@testcase | 101 | 9 | 2 1 +@testcase | 102 | 8 | 8 +@testcase | 102 | null | 0 +@testcase | 103 | -1 | -2 +@testcase | 104 | 1 2 | -1 +@testcase | 104 | 0 2 | (null) +@testcase | 105 | -1 | 10 +@testcase | 106 | | [ 1 ] +@testcase | 107 | | 1 20 +@testcase | 108 | 0 | 20 +@testcase | 109 | -1 | 50000000 +@testcase | 110 | 5 | 6 +@testcase | 110 | -5 | 7 +@testcase | 111 | -1 | 1 2 +@testcase | 111 | 0 | 3 4 + +@testcase | 200 | 20 | 200 +@testcase | 201 | 5 | 5 +@testcase | 202 | 5 | 5 +@testcase | 203 | | 200 + +@testcase | 220 | null null 0 -1 | (null) +@testcase | 220 | null null 0 0 | 20 +@testcase | 220 | 9 8 123 5 | 8 +@testcase | 221 | 10 null null 0 | 0 +@testcase | 221 | 3 null null 0 | (null) +@testcase | 221 | 10 3 4 100 | 3 +@testcase | 221 | 3 3 4 100 | 4 + +@testcase | 300 | 5 | 5 1 +@testcase | 301 | -1 | 10 20 +@testcase | 302 | null | 0 +@testcase | 303 | -1 | 1 [ 1 ] +@testcase | 304 | -1 | 0 +@testcase | 304 | 0 | 5 +@testcase | 305 | 0 | 5 +@testcase | 305 | 0 | 5 + +@fif_codegen +""" + test101() PROC:<{ + DUP + 2 PUSHINT + 1 PUSHINT + CONDSEL + SWAP + 5 EQINT + 2 PUSHINT + 1 PUSHINT + CONDSEL + }> +""" + +@fif_codegen +""" + test102() PROC:<{ // x + DUP // x x + ISNULL // x '1 + 0 PUSHINT + ROT // '1 '3=0 x + CONDSEL // '2 + }> +""" + +@fif_codegen +""" + test103() PROC:<{ + -2 PUSHINT + 2 PUSHINT + CONDSEL + }> +""" + +@fif_codegen +""" + test105() PROC:<{ // getX + 10 PUSHINT // getX '3=10 + 20 PUSHINT // getX p.x=10 p.y=20 + CONDSEL // '5 + }> +""" + +@fif_codegen +""" + helper106() PROC:<{ + $gTup GETGLOB + SWAP + CONDSEL + }> +""" + +@fif_codegen +""" + helper107() PROC:<{ + NIP + s1 s(-1) PUXC + CONDSEL + }> +""" + +@fif_codegen +""" + test108() PROC:<{ + CONT:<{ + MIN + }> + CONT:<{ + MAX + }> + CONDSEL + 10 PUSHINT + 20 PUSHINT + ROT + 2 1 CALLXARGS + }> +""" + +@fif_codegen +""" + test109() PROC:<{ + 50000000 PUSHINT + 1000000 PUSHINT + CONDSEL + }> +""" + +@fif_codegen +""" + test200() PROC:<{ + DROP + 200 PUSHINT + }> +""" + +@fif_codegen +""" + test201() PROC:<{ + // '2 + }> +""" + +@fif_codegen +""" + test202() PROC:<{ + }> +""" + +@fif_codegen +""" + test203() PROC:<{ + 200 PUSHINT + }> +""" + +@fif_codegen +""" + test220() PROC:<{ + s3 POP + 0 EQINT + IFJMP:<{ + 20 PUSHINT + CONDSEL + }> + NIP + }> +""" + +@fif_codegen +""" + test304() PROC:<{ + IF:<{ + Point.create0NoInline() CALLDICT +""" + +@fif_codegen +""" + test305() PROC:<{ + IF:<{ + 0 PUSHINT + }>ELSE<{ + 5 PUSHINT + }> + }> +""" + +*/ diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 22d743753..dc2795c89 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1004,6 +1004,23 @@ static AsmOp compile_throw_arg(std::vector& res, std::vector } } +// `x ? y : z` can be compiled as `CONDSEL` asm instruction if y and z are don't require evaluation +static AsmOp compile_ternary_as_condsel(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 1 && args.size() == 3); + VarDescr& cond = args[0]; // args = [ cond, when_true, when_false ] + if (cond.always_true()) { + cond.unused(); + args[2].unused(); + return AsmOp::Nop(loc); + } + if (cond.always_false()) { + cond.unused(); + args[1].unused(); + return AsmOp::Nop(loc); + } + return exec_op(loc, "CONDSEL", 3, 1); +} + static AsmOp compile_bool_const(std::vector& res, std::vector& args, SrcLocation loc, bool val) { tolk_assert(res.size() == 1 && args.empty()); VarDescr& r = res[0]; @@ -1434,6 +1451,9 @@ void define_builtins() { compile_fetch_varint, // not exposed to stdlib, used in auto-serialization FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, {}, {1, 0}); + define_builtin_func("__condsel", ParamsInt3, Int, nullptr, + compile_ternary_as_condsel, + 0); // compile-time only functions, evaluated essentially at compile-time, no runtime implementation // they are placed in stdlib and marked as `builtin` diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 3f525287b..732dc3e18 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -409,6 +409,29 @@ static V calc_sink_leftmost_obj(V v) { return leftmost_obj->kind == ast_reference ? leftmost_obj->as() : nullptr; } +// ternary `x ? y : z` can be optimized to asm `CONDSEL` (not IF/ELSE), if y and z don't require evaluation; +// example when can: `cond ? 2 : null`, `x == null ? some_var : obj.field`; +// example when not: `cond ? f() : g()` and other non-trivial arguments +static bool is_ternary_arg_trivial_for_condsel(AnyExprV v, bool require_1slot = true) { + if (require_1slot && v->inferred_type->get_width_on_stack() != 1) { + return false; + } + if (v->kind == ast_int_const || v->kind == ast_string_const || v->kind == ast_bool_const || + v->kind == ast_null_keyword || v->kind == ast_reference) { + return true; + } + if (auto v_par = v->try_as()) { + return is_ternary_arg_trivial_for_condsel(v_par->get_expr(), require_1slot); + } + if (auto v_dot = v->try_as()) { + return is_ternary_arg_trivial_for_condsel(v_dot->get_obj(), false); + } + if (auto v_cast = v->try_as()) { + return is_ternary_arg_trivial_for_condsel(v_cast->get_expr(), require_1slot); + } + return false; +} + static std::vector> pre_compile_tensor_inner(CodeBlob& code, const std::vector& args, const TypeDataTensor* tensor_target_type, LValContext* lval_ctx) { @@ -1303,6 +1326,11 @@ static std::vector process_ternary_operator(V v code.emplace_back(v->get_when_true()->loc, Op::_Let, rvect, pre_compile_expr(v->get_when_true(), code, v->inferred_type)); } else if (v->get_cond()->is_always_false) { code.emplace_back(v->get_when_false()->loc, Op::_Let, rvect, pre_compile_expr(v->get_when_false(), code, v->inferred_type)); + } else if (v->inferred_type->get_width_on_stack() == 1 && is_ternary_arg_trivial_for_condsel(v->get_when_true()) && is_ternary_arg_trivial_for_condsel(v->get_when_false())) { + std::vector ir_true = pre_compile_expr(v->get_when_true(), code, v->inferred_type); + std::vector ir_false = pre_compile_expr(v->get_when_false(), code, v->inferred_type); + std::vector condsel_args = { cond[0], ir_true[0], ir_false[0] }; + code.emplace_back(v->loc, Op::_Call, rvect, std::move(condsel_args), lookup_function("__condsel")); } else { Op& if_op = code.emplace_back(v->loc, Op::_If, cond); code.push_set_cur(if_op.block0); From a045893afb8bc2a2d034dd0fe0eb537407540d11 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Fri, 4 Jul 2025 13:13:40 +0300 Subject: [PATCH 341/388] [Tolk] Peephole optimization: SDSKIPFIRST to LDU+NIP It's a bit better for gas usage, always --- tolk-tester/tests/cells-slices.tolk | 4 ++-- tolk-tester/tests/lazy-load-tests.tolk | 20 ++++++++++---------- tolk-tester/tests/pack-unpack-1.tolk | 4 ++-- tolk/optimize.cpp | 9 +++++++-- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index 8fee091b8..6a298d234 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -401,8 +401,8 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ test21() PROC:<{ - 14 PUSHINT - SDSKIPFIRST + 14 LDU + NIP }> """ diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk index 7a5643a54..72083fc85 100644 --- a/tolk-tester/tests/lazy-load-tests.tolk +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -1741,8 +1741,8 @@ fun main() { """ demo103() PROC:<{ // x{0102} PUSHSLICE // lazyS - 8 PUSHINT - SDSKIPFIRST // lazyS + 8 LDU + NIP // lazyS 8 LDI // '10 lazyS DROP // p.y """ @@ -1753,8 +1753,8 @@ fun main() { // lazyS PUSHNULL // lazyS x SWAP // x lazyS - 8 PUSHINT - SDSKIPFIRST // x lazyS + 8 LDU + NIP // x lazyS 1 LDU // x '18 lazyS """ @@ -1762,8 +1762,8 @@ fun main() { """ demo109() PROC:<{ // x{8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_} PUSHSLICE // lazyS - 65 PUSHINT - SDSKIPFIRST // lazyS + 65 LDU + NIP // lazyS 256 LDU // '13 lazyS DROP // st.publicKey }> @@ -1821,8 +1821,8 @@ fun main() { DROP -1 PUSHINT }> - 8 PUSHINT - SDSKIPFIRST + 8 LDU + NIP 8 LDI DROP }> @@ -1962,8 +1962,8 @@ fun main() { THROWANY }> 32 LDU - 8 PUSHINT - SDSKIPFIRST + 8 LDU + NIP 8 LDI DROP OVER diff --git a/tolk-tester/tests/pack-unpack-1.tolk b/tolk-tester/tests/pack-unpack-1.tolk index 68ce0bc10..406307cfd 100644 --- a/tolk-tester/tests/pack-unpack-1.tolk +++ b/tolk-tester/tests/pack-unpack-1.tolk @@ -112,8 +112,8 @@ fun main(c: cell) { 64 STI // b ENDC // c CTOS // s - 64 PUSHINT - SDSKIPFIRST // s + 64 LDU + NIP // s SBITS // '17 }> """ diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp index c8fd8fc96..dab931d84 100644 --- a/tolk/optimize.cpp +++ b/tolk/optimize.cpp @@ -231,8 +231,13 @@ bool Optimizer::detect_rewrite_MY_skip_bits() { p_ = n_merged; q_ = 2; - oq_[0] = std::make_unique(AsmOp::IntConst(op_[0]->loc, td::make_refint(total_skip_bits))); - oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->loc, "SDSKIPFIRST")); + if (total_skip_bits <= 256) { + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, std::to_string(total_skip_bits) + " LDU")); + oq_[1] = std::make_unique(AsmOp::Pop(op_[0]->loc, 1)); + } else { + oq_[0] = std::make_unique(AsmOp::IntConst(op_[0]->loc, td::make_refint(total_skip_bits))); + oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->loc, "SDSKIPFIRST")); + } return true; } From b297b6e48724b860172acff497e16045b50514fa Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Fri, 4 Jul 2025 16:33:03 +0300 Subject: [PATCH 342/388] [Tolk] Peephole optimization: LD+DROP to PLD Peephole optimizations: - `N LDU` + `DROP` -> `N PLDU` - same for LDI -> PLDI, LDREF -> PLDREF, etc. --- tolk-tester/tests/asm-arg-order.tolk | 5 +-- tolk-tester/tests/handle-msg-3.tolk | 3 +- tolk-tester/tests/lazy-algo-tests.tolk | 3 +- tolk-tester/tests/lazy-load-tests.tolk | 51 +++++++++----------------- tolk-tester/tests/null-keyword.tolk | 3 +- tolk-tester/tests/pack-unpack-6.tolk | 3 +- tolk-tester/tests/pack-unpack-7.tolk | 3 +- tolk-tester/tests/some-tests-3.tolk | 2 +- tolk/optimize.cpp | 35 ++++++++++++++++++ tolk/tolk.h | 1 + 10 files changed, 61 insertions(+), 48 deletions(-) diff --git a/tolk-tester/tests/asm-arg-order.tolk b/tolk-tester/tests/asm-arg-order.tolk index 1558a3c72..c42c1f958 100644 --- a/tolk-tester/tests/asm-arg-order.tolk +++ b/tolk-tester/tests/asm-arg-order.tolk @@ -269,12 +269,11 @@ fun main() { test34() PROC:<{ x{020a} PUSHSLICE 8 LDU - 8 LDU - DROP + 8 PLDU SWAP 1 ADDCONST MUL }> """ -@code_hash 88343225778124778743440967192570181022747908733235053033508198051414837052492 +@code_hash 38717859169035491454494966803813683158800226121601078320213761069394202390827 */ diff --git a/tolk-tester/tests/handle-msg-3.tolk b/tolk-tester/tests/handle-msg-3.tolk index 78bb301dd..41d3c1734 100644 --- a/tolk-tester/tests/handle-msg-3.tolk +++ b/tolk-tester/tests/handle-msg-3.tolk @@ -23,8 +23,7 @@ fun onInternalMessage(in: InMessage) { IFJMP:<{ 32 LDU NIP - 32 LDU - DROP + 32 PLDU THROWANY }> // in.body }> diff --git a/tolk-tester/tests/lazy-algo-tests.tolk b/tolk-tester/tests/lazy-algo-tests.tolk index 27214077b..ac8e9fcd5 100644 --- a/tolk-tester/tests/lazy-algo-tests.tolk +++ b/tolk-tester/tests/lazy-algo-tests.tolk @@ -1498,8 +1498,7 @@ fun main() { x{0102} PUSHSLICE 8 LDI NIP - 8 LDI - DROP + 8 PLDI }> """ */ diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk index 72083fc85..3e6a67bef 100644 --- a/tolk-tester/tests/lazy-load-tests.tolk +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -1732,8 +1732,7 @@ fun main() { """ demo101() PROC:<{ // x{0102} PUSHSLICE // lazyS - 8 LDI // '9 lazyS - DROP // p.x + 8 PLDI // p.x }> """ @@ -1743,8 +1742,7 @@ fun main() { x{0102} PUSHSLICE // lazyS 8 LDU NIP // lazyS - 8 LDI // '10 lazyS - DROP // p.y + 8 PLDI // p.y """ @fif_codegen @@ -1764,8 +1762,7 @@ fun main() { x{8000003d800000e4000000000000000000000000000000000000000000000000000000000000000020_} PUSHSLICE // lazyS 65 LDU NIP // lazyS - 256 LDU // '13 lazyS - DROP // st.publicKey + 256 PLDU // st.publicKey }> """ @@ -1775,13 +1772,11 @@ fun main() { // lazyS x{01} SDBEGINSQ IF:<{ - 8 LDI - DROP + 8 PLDI }>ELSE<{ x{02} SDBEGINSQ IF:<{ - 8 LDI - DROP + 8 PLDI NEGATE }>ELSE<{ x{03} SDBEGINSQ @@ -1793,8 +1788,7 @@ fun main() { IFNOTJMP:<{ 63 THROW }> - 32 LDI - DROP + 32 PLDI }> }> }> @@ -1807,8 +1801,7 @@ fun main() { x{010f} PUSHSLICE // lazyS x{01} SDBEGINSQ // lazyS '6 134 THROWIFNOT // lazyS - 8 LDI // '11 lazyS - DROP // cc + 8 PLDI // cc }> """ @@ -1823,8 +1816,7 @@ fun main() { }> 8 LDU NIP - 8 LDI - DROP + 8 PLDI }> """ @@ -1832,8 +1824,7 @@ fun main() { """ demo146() PROC:<{ x{0000000} PUSHSLICE - LDOPTREF - DROP + PLDOPTREF ISNULL }> """ @@ -1844,8 +1835,7 @@ fun main() { x{0102} PUSHSLICE DUP 8 LDI - 8 LDI - DROP + 8 PLDI s2 PUSH NEWC STSLICE @@ -1866,8 +1856,7 @@ fun main() { demo203() PROC:<{ // x{010203040506070809} PUSHSLICE // lazyS DUP // '14 lazyS - 8 LDU // '14 '16 lazyS - DROP // '14 o.f1 + 8 PLDU // '14 o.f1 1 EQINT // '14 '18 IFJMP:<{ // '14 NEWC // '14 b @@ -1889,8 +1878,7 @@ fun main() { 8 LDU // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 lazyS 8 LDU // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 o.f7 lazyS 8 LDU // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 o.f7 o.f8 lazyS - 8 LDU // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 o.f7 o.f8 '32 lazyS - DROP // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 o.f7 o.f8 o.f9 + 8 PLDU // '14 o.f1 o.f2 o.f3 o.f4 o.f5 o.f6 o.f7 o.f8 o.f9 """ @fif_codegen @@ -1898,8 +1886,7 @@ fun main() { demo215() PROC:<{ x{0102} PUSHSLICE 8 LDI - 8 LDI - DROP + 8 PLDI SWAP NEWC 8 STI @@ -1928,8 +1915,7 @@ fun main() { 1 LDI // '13 lazyS NIP // lazyS 320 PUSHINT // lazyS '14=320 - LDSLICEX // '15 lazyS - DROP // '15 + PLDSLICEX // '15 PUSHNULL // '15 st.extensions 0 PUSHINT NEWC @@ -1947,8 +1933,7 @@ fun main() { // lazyS x{12345678} SDBEGINSQ IF:<{ - 32 LDI - DROP + 32 PLDI 10 ADDCONST }>ELSE<{ x{23456789} SDBEGINSQ @@ -1964,8 +1949,7 @@ fun main() { 32 LDU 8 LDU NIP - 8 LDI - DROP + 8 PLDI OVER 100 GTINT IF:<{ @@ -2011,8 +1995,7 @@ fun main() { """ demo411() PROC:<{ // s // lazyS - 1 LDU // '10 lazyS - DROP // '10 + 1 PLDU // '10 IFJMP:<{ // DUMPSTK // }> // diff --git a/tolk-tester/tests/null-keyword.tolk b/tolk-tester/tests/null-keyword.tolk index c36a66c26..c21da1d0e 100644 --- a/tolk-tester/tests/null-keyword.tolk +++ b/tolk-tester/tests/null-keyword.tolk @@ -122,8 +122,7 @@ fun main() { """ test7() PROC:<{ ... - LDOPTREF // b '9 '8 - DROP // b c + PLDOPTREF // b c ISNULL // b '11 10 MULCONST // b '13 SWAP // '13 b diff --git a/tolk-tester/tests/pack-unpack-6.tolk b/tolk-tester/tests/pack-unpack-6.tolk index 71f6ab0b4..47c0b062d 100644 --- a/tolk-tester/tests/pack-unpack-6.tolk +++ b/tolk-tester/tests/pack-unpack-6.tolk @@ -138,8 +138,7 @@ RETALT CTOS x{01} SDBEGINSQ 63 THROWIFNOT - 8 LDI - DROP + 8 PLDI }> """ diff --git a/tolk-tester/tests/pack-unpack-7.tolk b/tolk-tester/tests/pack-unpack-7.tolk index 07407ff91..60e9c2b5c 100644 --- a/tolk-tester/tests/pack-unpack-7.tolk +++ b/tolk-tester/tests/pack-unpack-7.tolk @@ -175,8 +175,7 @@ fun main() { test3() PROC:<{ x{0102} PUSHSLICE 8 LDU - 8 LDI - DROP + 8 PLDI }> """ */ diff --git a/tolk-tester/tests/some-tests-3.tolk b/tolk-tester/tests/some-tests-3.tolk index 8c53e33bf..5c99c4123 100644 --- a/tolk-tester/tests/some-tests-3.tolk +++ b/tolk-tester/tests/some-tests-3.tolk @@ -31,5 +31,5 @@ fun f(cs: slice) { @testcase | 101 | x{000102030405060708090a0b0c0d0e0f10111213} | 6 @testcase | 0 | x{000102030405060708090a0b0c0d0e0f10111213} | 0 1 2 3 -@code_hash 58474889199998908444151060994149070836199913191952040273624197630531731101157 +@code_hash 110094925152266492480367791654603701827254965953641340034522027582587727126150 */ diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp index dab931d84..732d04baf 100644 --- a/tolk/optimize.cpp +++ b/tolk/optimize.cpp @@ -264,6 +264,40 @@ bool Optimizer::detect_rewrite_NEWC_PUSH_STUR() { return true; } +// pattern `N LDU` + `DROP` -> `N PLDU` (common after loading the last field manually or by `lazy`); +// the same for LDI -> PLDI, LDREF -> PLDREF, etc. +bool Optimizer::detect_rewrite_LDxx_DROP() { + bool second_drop = pb_ > 1 && op_[1]->is_pop() && op_[1]->a == 0; + if (!second_drop || !op_[0]->is_custom()) { + return false; + } + + static const char* ends_with[] = { " LDI", " LDU", " LDBITS"}; + static const char* repl_with[] = {" PLDI", " PLDU", " PLDBITS"}; + static const char* equl_to[] = { "LDREF", "LDDICT", "LDOPTREF", "LDSLICEX"}; + static const char* repl_to[] = {"PLDREF", "PLDDICT", "PLDOPTREF", "PLDSLICEX"}; + + std::string_view f = op_[0]->op; + for (size_t i = 0; i < std::size(ends_with); ++i) { + if (f.ends_with(ends_with[i])) { + p_ = 2; + q_ = 1; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, op_[0]->op.substr(0, f.rfind(' ')) + repl_with[i], 0, 1)); + return true; + } + } + for (size_t i = 0; i < std::size(equl_to); ++i) { + if (f == equl_to[i]) { + p_ = 2; + q_ = 1; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, repl_to[i], 0, 1)); + return true; + } + } + + return false; +} + bool Optimizer::is_push_const(int* i, int* c) const { return pb_ >= 3 && pb_ <= l2_ && tr_[pb_ - 1].is_push_const(i, c); @@ -685,6 +719,7 @@ bool Optimizer::find_at_least(int pb) { (is_xchg_xchg(&i, &j, &k, &l) && rewrite(AsmOp::Xchg(loc, i, j), AsmOp::Xchg(loc, k, l))) || detect_rewrite_big_THROW() || detect_rewrite_MY_store_int() || detect_rewrite_MY_skip_bits() || detect_rewrite_NEWC_PUSH_STUR() || + detect_rewrite_LDxx_DROP() || (!(mode_ & 1) && ((is_rot() && rewrite(AsmOp::Custom(loc, "ROT", 3, 3))) || (is_rotrev() && rewrite(AsmOp::Custom(loc, "-ROT", 3, 3))) || (is_2dup() && rewrite(AsmOp::Custom(loc, "2DUP", 2, 4))) || diff --git a/tolk/tolk.h b/tolk/tolk.h index 9bd21c99d..45b553d8a 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -902,6 +902,7 @@ struct Optimizer { bool detect_rewrite_MY_store_int(); bool detect_rewrite_MY_skip_bits(); bool detect_rewrite_NEWC_PUSH_STUR(); + bool detect_rewrite_LDxx_DROP(); AsmOpConsList extract_code(); }; From 55c2530c826f8f6fcc620297b1f7ae158b8875a1 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Fri, 4 Jul 2025 17:56:01 +0300 Subject: [PATCH 343/388] [Tolk] Peephole optimization: SWAP+any to any/anyR - SWAP + EQUAL -> EQUAL - same for other symmetric operators (NEQ, MUL, etc.) - SWAP + xxx PUSHINT + 32 STUR -> xxx PUSHINT + ROT + 32 STU - SWAP + STSLICER => STSLICE and vice versa - same for other storing operators (STU, STB, STREF, etc.) - SWAP + LESS/LEQ => GREATER/GEQ --- tolk-tester/tests/a-tests.tolk | 2 +- .../tests/allow-post-modification.tolk | 2 +- tolk-tester/tests/calls-tests.tolk | 63 ++++++ tolk-tester/tests/cells-slices.tolk | 213 +++++++++++++++++- tolk-tester/tests/inline-tests.tolk | 4 +- tolk-tester/tests/op-priority.tolk | 9 +- tolk-tester/tests/pack-unpack-6.tolk | 1 - tolk-tester/tests/some-tests-3.tolk | 2 +- tolk-tester/tests/strings-tests.tolk | 2 +- tolk-tester/tests/try-catch-tests.tolk | 2 +- tolk/asmops.cpp | 2 + tolk/optimize.cpp | 77 +++++++ tolk/tolk.h | 3 + 13 files changed, 362 insertions(+), 20 deletions(-) diff --git a/tolk-tester/tests/a-tests.tolk b/tolk-tester/tests/a-tests.tolk index a122ca39d..fb553f7e5 100644 --- a/tolk-tester/tests/a-tests.tolk +++ b/tolk-tester/tests/a-tests.tolk @@ -79,5 +79,5 @@ fun main(): int { method_id | in | out @testcase | 0 | | 31415926535897932384626433832795028841971693993751058209749445923078164 -@code_hash 103119459254804391561326789266022754176632268076661379738543746273668564328534 +@code_hash 85818713521853656486584648797214567489479452958868213061669627117427586818086 */ diff --git a/tolk-tester/tests/allow-post-modification.tolk b/tolk-tester/tests/allow-post-modification.tolk index a74b2e1ec..f825d12f8 100644 --- a/tolk-tester/tests/allow-post-modification.tolk +++ b/tolk-tester/tests/allow-post-modification.tolk @@ -164,5 +164,5 @@ fun main() { test_assign_tensor_global() PROC:<{ // x.0 x.1 """ -@code_hash 6737917279814799680932710799951154408447028229503449454536845752635763933556 +@code_hash 77620375659834063567341916666636335765114427483375544778177892407598637223147 */ diff --git a/tolk-tester/tests/calls-tests.tolk b/tolk-tester/tests/calls-tests.tolk index a03cd5a30..9405c9ead 100644 --- a/tolk-tester/tests/calls-tests.tolk +++ b/tolk-tester/tests/calls-tests.tolk @@ -159,6 +159,26 @@ fun test9() { return int(10) + int(5).plus1() + int.create0() + int.create0().plus1(); } +@noinline fun cmp1(a: int, b: int) { return a == b } +@noinline fun cmp2(a: int, b: int) { return b == a } +@noinline fun cmp3(a: int, b: int) { return a != b } +@noinline fun cmp4(a: int, b: int) { return b != a } + +@method_id(110) +fun test10(a: int, b: int) { + return b - a +} + +@noinline fun leq1(a: int, b: int) { return b < a } +@noinline fun leq2(a: int, b: int) { return b <= a } +@noinline fun leq3(a: int, b: int) { return b > a } +@noinline fun leq4(a: int, b: int) { return b >= a } + +@method_id(111) +fun test11(a: int, b: int) { + return ((b < a), (b <= a), (b > a), (b >= a)); +} + fun main() {} /** @@ -171,4 +191,47 @@ fun main() {} @testcase | 107 | | 16 5 @testcase | 108 | | [ 1 2 3 ] [ 5 8 1 ] @testcase | 109 | | 17 +@testcase | 110 | 5 8 | 3 +@testcase | 111 | 5 8 | 0 0 -1 -1 +@testcase | 111 | 7 7 | 0 -1 0 -1 + +@fif_codegen +""" + cmp1() PROC:<{ + EQUAL + }> + cmp2() PROC:<{ + EQUAL + }> + cmp3() PROC:<{ + NEQ + }> + cmp4() PROC:<{ + NEQ + }> +""" + +@fif_codegen +""" + test10() PROC:<{ + SUBR + }> +""" + +@fif_codegen +""" + leq1() PROC:<{ + GREATER + }> + leq2() PROC:<{ + GEQ + }> + leq3() PROC:<{ + LESS + }> + leq4() PROC:<{ + LEQ + }> +""" + */ diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index 6a298d234..2963177de 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -318,6 +318,121 @@ fun test24(uns: bool) { } } +fun builder.my_storeRef(mutate self, value: cell): self + asm(self value) "STREFR"; + +fun builder.my_storeAddress(mutate self, value: address): self + asm(value self) "STSLICE"; + +fun builder.my_storeBuilder(mutate self, value: builder): self + asm(value self) "STB"; + +fun my_PUSHINT_SWAP_STU(mutate b: builder): void + asm "5 PUSHINT" "SWAP" "8 STU"; + + +@noinline +fun demo30(value: cell, dest: builder) { + // SWAP + STREFR => STREF + return dest.my_storeRef(value); +} + +@noinline +fun demo31(dest: builder, value: cell) { + // SWAP + STREF => STREFR + return dest.storeRef(value); +} + +@noinline +fun demo32(dest: builder, value: builder) { + // SWAP + STB => STBR + return dest.my_storeBuilder(value); +} + +@noinline +fun demo33(value: builder, dest: builder) { + // SWAP + STBR => STB + return dest.storeBuilder(value); +} + +@noinline +fun demo34(dest: builder, value: address) { + // SWAP + STSLICE => STSLICER + return dest.my_storeAddress(value); +} + +@noinline +fun demo35(value: slice, dest: builder) { + // SWAP + STSLICER => STSLICE + return dest.storeSlice(value); +} + +@noinline +fun demo36(x: int) { + var b = beginCell(); + if (x > 0) { + // inside IF: N PUSHINT + SWAP + L STU => N PUSHINT + L STUR + my_PUSHINT_SWAP_STU(mutate b); + } + return b; +} + +@noinline +fun demo37(op: int32, u: int8 | int256) { + var b = beginCell(); + // b.storeUint(input.op, 32); + if (u is int8) { + // inside IF: SWAP + 0 PUSHINT + 3 STUR => 0 PUSHINT + 3 STU + b.storeUint(0, 3); + b.storeUint(u, 8); + } + return b; +} + +@method_id(130) +fun test30() { + return (demo37(10, 88 as int8), demo36(50)); +} + +@method_id(131) +fun test31() { + var s = createAddressNone(); + var dest = beginCell().storeUint(3, 32); + assert(demo35(s as slice, dest).endCell().hash() == demo34(dest, s).endCell().hash()) throw 300; + var r1 = demo35(s as slice, dest).endCell().hash(); + var b = beginCell().storeUint(0xFF, 16); + assert(demo33(b, dest).endCell().hash() == demo32(dest, b).endCell().hash()) throw 301; + var r2 = demo33(b, dest).endCell().hash(); + var r = createEmptyCell(); + assert(demo30(r, dest).endCell().hash() == demo31(dest, r).endCell().hash()) throw 302; + var r3 = demo30(r, dest).endCell().hash(); + return [r1 & 0xFFFFFFFF, r2 & 0xFFFFFFFF, r3 & 0xFFFFFFFF]; +} + +@method_id(132) +fun test32(p: int) { + var b = beginCell(); + var x = p + 10; + b.storeUint(x, 32); // SWAP + n STU => n STUR + return b; +} + +@method_id(133) +fun test33(p: int) { + var b = beginCell(); + var x = p + 10; + b.storeInt(x, 8); // SWAP + n STI => n STIR + return b; +} + +@method_id(134) +fun test34(p: int, n: int) { + var b = beginCell(); + var x = p + 10; + b.storeInt(x, n); // no changes, it's STIX + return b; +} + fun main(): int { return 0; } @@ -347,6 +462,11 @@ fun main(): int { @testcase | 123 | 0 | 255 @testcase | 124 | -1 | 8 @testcase | 124 | 0 | 10 +@testcase | 130 | | BC{00030b10} BC{000205} +@testcase | 131 | | [ 225345949 901620178 1560646286 ] +@testcase | 132 | 0 | BC{00080000000a} +@testcase | 133 | 0 | BC{00020a} +@testcase | 134 | 0 8 | BC{00020a} We test that consequtive storeInt/storeUint with constants are joined into a single number @@ -378,15 +498,13 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ test19() PROC:<{ - NEWC 123 PUSHINT - SWAP + NEWC 4 STI 16 PUSHPOW2DEC 16 STUR -1 PUSHINT - SWAP - 8 STI + 8 STIR }> """ @@ -411,9 +529,9 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin test22() PROC:<{ NEWC // '2 NEWC // b1 b2 - SWAP // b2 b1 2056 PUSHINT - 24 STUR // b2 b1 + ROT + 24 STU // b2 b1 SWAP // b1 b2 10141514286835656557042350424064 PUSHINTX 132 STUR // b1 b2 @@ -437,4 +555,87 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin }> """ +@fif_codegen +""" + demo30() PROC:<{ // value dest + STREF // dest + }> + demo31() PROC:<{ // dest value + STREFR // dest + }> + demo32() PROC:<{ // dest value + STBR // dest + }> + demo33() PROC:<{ // value dest + STB // dest + }> + demo34() PROC:<{ // dest value + STSLICER // dest + }> + demo35() PROC:<{ // value dest + STSLICE // dest + }> +""" + +@fif_codegen +""" + demo36() PROC:<{ // x + NEWC // x b + SWAP // b x + 0 GTINT // b '4 + IF:<{ // b + 5 PUSHINT + 8 STUR // b + }> // b + }> +""" + +@fif_codegen +""" + demo37() PROC:<{ + NEWC + s3 POP + 42 EQINT + IF:<{ + 0 PUSHINT + ROT + 3 STU + 8 STU // b + }>ELSE<{ + DROP // b + }> + }> +""" + +@fif_codegen +""" + test32() PROC:<{ // p + NEWC // p b + SWAP // b p + 10 ADDCONST // b x + 32 STUR // b + }> +""" + +@fif_codegen +""" + test33() PROC:<{ // p + NEWC // p b + SWAP // b p + 10 ADDCONST // b x + 8 STIR // b + }> +""" + +@fif_codegen +""" + test34() PROC:<{ + NEWC + s0 s2 XCHG + 10 ADDCONST + -ROT + STIX + }> +""" + */ diff --git a/tolk-tester/tests/inline-tests.tolk b/tolk-tester/tests/inline-tests.tolk index 243652b46..fc48ddaa3 100644 --- a/tolk-tester/tests/inline-tests.tolk +++ b/tolk-tester/tests/inline-tests.tolk @@ -471,9 +471,9 @@ fun usedIn10ButDeclaredBelow(x: int) { NEWC 32 STU // '3 25 PUSHINT // '3 '9 - SWAP // x self 107374182425 PUSHINT - 64 STUR // x '3 + ROT + 64 STU // x '3 SWAP // '3 x }> """ diff --git a/tolk-tester/tests/op-priority.tolk b/tolk-tester/tests/op-priority.tolk index e635d5423..79c837693 100644 --- a/tolk-tester/tests/op-priority.tolk +++ b/tolk-tester/tests/op-priority.tolk @@ -96,23 +96,20 @@ fun main() { @fif_codegen """ unary_minus_1() PROC:<{ // a b c - -ROT // c a b + s0 s2 XCHG ADD // c '3 NEGATE // c '4 - SWAP // '4 c MUL // '5 }> unary_minus_2() PROC:<{ // a b c - -ROT // c a b + s0 s2 XCHG ADD // c '3 NEGATE // c '4 - SWAP // '4 c MUL // '5 }> unary_minus_3() PROC:<{ // a b c - -ROT // c a b + s0 s2 XCHG ADD // c '3 - SWAP // '3 c MUL // '4 NEGATE // '5 }> diff --git a/tolk-tester/tests/pack-unpack-6.tolk b/tolk-tester/tests/pack-unpack-6.tolk index 47c0b062d..364d026a2 100644 --- a/tolk-tester/tests/pack-unpack-6.tolk +++ b/tolk-tester/tests/pack-unpack-6.tolk @@ -212,7 +212,6 @@ IF:<{ 140 PUSHINT }> 139 PUSHINT - SWAP EQUAL }> """ diff --git a/tolk-tester/tests/some-tests-3.tolk b/tolk-tester/tests/some-tests-3.tolk index 5c99c4123..7208295fe 100644 --- a/tolk-tester/tests/some-tests-3.tolk +++ b/tolk-tester/tests/some-tests-3.tolk @@ -31,5 +31,5 @@ fun f(cs: slice) { @testcase | 101 | x{000102030405060708090a0b0c0d0e0f10111213} | 6 @testcase | 0 | x{000102030405060708090a0b0c0d0e0f10111213} | 0 1 2 3 -@code_hash 110094925152266492480367791654603701827254965953641340034522027582587727126150 +@code_hash 110494322917670257289501183090765859207486616122530510808569169535563021169176 */ diff --git a/tolk-tester/tests/strings-tests.tolk b/tolk-tester/tests/strings-tests.tolk index 863a8aeff..69d0fb2d5 100644 --- a/tolk-tester/tests/strings-tests.tolk +++ b/tolk-tester/tests/strings-tests.tolk @@ -78,5 +78,5 @@ fun test1() { @testcase | 0 | | 0 @testcase | 101 | | [ 65 66 67 68 ] -@code_hash 15952581052628192793433894459656689089299796554419234117286198843831925928625 +@code_hash 33596158548631231081906594921712427436857221384530839008514203474433205250299 */ diff --git a/tolk-tester/tests/try-catch-tests.tolk b/tolk-tester/tests/try-catch-tests.tolk index 04e32531c..136c606d2 100644 --- a/tolk-tester/tests/try-catch-tests.tolk +++ b/tolk-tester/tests/try-catch-tests.tolk @@ -294,7 +294,7 @@ fun main() { @testcase | 112 | 5 | 138 @testcase | 113 | | 2048 -@code_hash 91375323652563311931660098165979913104421063599680251769401655783153201389325 +@code_hash 3924051084946061509165180039638830343498397714643311802900659572522552334228 @fif_codegen """ diff --git a/tolk/asmops.cpp b/tolk/asmops.cpp index 44887633b..3dc39239b 100644 --- a/tolk/asmops.cpp +++ b/tolk/asmops.cpp @@ -223,6 +223,8 @@ AsmOp AsmOp::Parse(SrcLocation loc, const std::string& custom_op) { return AsmOp::Push(loc, 0); } else if (custom_op == "OVER") { return AsmOp::Push(loc, 1); + } else if (custom_op.ends_with(" PUSHINT") && custom_op[0] >= '1' && custom_op[0] <= '9' && custom_op.find(' ') == custom_op.rfind(' ')) { + return AsmOp::IntConst(loc, td::string_to_int256(custom_op.substr(0, custom_op.find(' ')))); } else { return AsmOp::Custom(loc, custom_op); } diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp index 732d04baf..d9e7608be 100644 --- a/tolk/optimize.cpp +++ b/tolk/optimize.cpp @@ -298,6 +298,82 @@ bool Optimizer::detect_rewrite_LDxx_DROP() { return false; } +// pattern `SWAP` + `EQUAL` -> `EQUAL` +// and other symmetric operators: NEQ, MUL, etc. +bool Optimizer::detect_rewrite_SWAP_symmetric() { + bool first_swap = op_[0]->is_swap(); + if (!first_swap || pb_ < 2 || !op_[1]->is_custom()) { + return false; + } + std::string_view n = op_[1]->op; + bool next_symmetric = n == "EQUAL" || n == "NEQ" || n == "SDEQ" || n == "AND" || n == "OR" + || n == "ADD" || n == "MUL" || n == "MIN" || n == "MAX"; + if (!next_symmetric) { + return false; + } + + p_ = 2; + q_ = 1; + oq_[0] = std::move(op_[1]); + return true; +} + +// pattern `SWAP` + `xxx PUSHINT` + `32 STUR` -> `xxx PUSHINT` + `ROT` + `32 STU` +bool Optimizer::detect_rewrite_SWAP_PUSH_STUR() { + bool first_swap = op_[0]->is_swap(); + if (!first_swap || pb_ < 3) { + return false; + } + bool next_push = op_[1]->is_const() && op_[1]->op.ends_with(" PUSHINT"); + if (!next_push) { + return false; + } + bool next_stu_r = op_[2]->is_custom() && (op_[2]->op.ends_with(" STUR") || op_[2]->op.ends_with(" STIR")); + if (!next_stu_r) { + return false; + } + + p_ = 3; + q_ = 3; + oq_[0] = std::move(op_[1]); + oq_[1] = std::make_unique(AsmOp::BlkSwap(oq_[0]->loc, 1, 2)); // ROT + oq_[2] = std::make_unique(AsmOp::Custom(oq_[0]->loc, op_[2]->op.substr(0, op_[2]->op.size() - 1), 1, 1)); + return true; +} + +// pattern `SWAP` + `STSLICER` -> `STSLICE` and vice versa: `SWAP` + `STSLICE` => `STSLICER`; +// same for `STB` / `STREF` / `n STU` / `n STI` +bool Optimizer::detect_rewrite_SWAP_STxxxR() { + bool first_swap = op_[0]->is_swap(); + if (!first_swap || pb_ < 2 || !op_[1]->is_custom()) { + return false; + } + + static const char* ends_with[] = {" STU", " STI", " STUR", " STIR"}; + static const char* repl_with[] = {" STUR", " STIR", " STU", " STI"}; + static const char* equl_to[] = {"STSLICE", "STSLICER", "STB", "STBR", "SUB", "SUBR", "STREF", "STREFR", "LESS", "LEQ", "GREATER", "GEQ"}; + static const char* repl_to[] = {"STSLICER", "STSLICE", "STBR", "STB", "SUBR", "SUB", "STREFR", "STREF", "GREATER", "GEQ", "LESS", "LEQ"}; + + std::string_view f = op_[1]->op; + for (size_t i = 0; i < std::size(ends_with); ++i) { + if (f.ends_with(ends_with[i])) { + p_ = 2; + q_ = 1; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, op_[1]->op.substr(0, f.rfind(' ')) + repl_with[i], 1, 1)); + return true; + } + } + for (size_t i = 0; i < std::size(equl_to); ++i) { + if (f == equl_to[i]) { + p_ = 2; + q_ = 1; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, repl_to[i], 0, 1)); + return true; + } + } + + return false; +} bool Optimizer::is_push_const(int* i, int* c) const { return pb_ >= 3 && pb_ <= l2_ && tr_[pb_ - 1].is_push_const(i, c); @@ -720,6 +796,7 @@ bool Optimizer::find_at_least(int pb) { detect_rewrite_big_THROW() || detect_rewrite_MY_store_int() || detect_rewrite_MY_skip_bits() || detect_rewrite_NEWC_PUSH_STUR() || detect_rewrite_LDxx_DROP() || + detect_rewrite_SWAP_symmetric() || detect_rewrite_SWAP_PUSH_STUR() || detect_rewrite_SWAP_STxxxR() || (!(mode_ & 1) && ((is_rot() && rewrite(AsmOp::Custom(loc, "ROT", 3, 3))) || (is_rotrev() && rewrite(AsmOp::Custom(loc, "-ROT", 3, 3))) || (is_2dup() && rewrite(AsmOp::Custom(loc, "2DUP", 2, 4))) || diff --git a/tolk/tolk.h b/tolk/tolk.h index 45b553d8a..32c6843f8 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -903,6 +903,9 @@ struct Optimizer { bool detect_rewrite_MY_skip_bits(); bool detect_rewrite_NEWC_PUSH_STUR(); bool detect_rewrite_LDxx_DROP(); + bool detect_rewrite_SWAP_symmetric(); + bool detect_rewrite_SWAP_PUSH_STUR(); + bool detect_rewrite_SWAP_STxxxR(); AsmOpConsList extract_code(); }; From eb8cdfb5414ab8794342f23e7abe453102be42d3 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Fri, 4 Jul 2025 23:20:56 +0300 Subject: [PATCH 344/388] [Tolk] Peephole optimization: PUSHINT+STU to STSLICECONST Now, `b.storeInt(123, 32)` generates not `123 PUSHINT; SWAP; STI`, but `x{...} STSLICECONST`. Although, for certain conditions, still generate STUR/STUR targeting smaller bytecode. --- tolk-tester/tests/cells-slices.tolk | 34 ++++++++-------------- tolk-tester/tests/constants-tests.tolk | 2 +- tolk-tester/tests/inline-tests.tolk | 11 +++---- tolk-tester/tests/lazy-load-tests.tolk | 6 ++-- tolk-tester/tests/pack-unpack-1.tolk | 3 +- tolk-tester/tests/strings-tests.tolk | 2 +- tolk/optimize.cpp | 40 ++++++++++++++++++++------ 7 files changed, 53 insertions(+), 45 deletions(-) diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index 2963177de..57a92f03b 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -473,26 +473,23 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ test6() PROC:<{ - 18446744082299486211 PUSHINT NEWC - 96 STU // '2 + x{000000010000000200000003} STSLICECONST ENDC // '11 """ @fif_codegen """ test17() PROC:<{ - 4219 PUSHINT NEWC - 16 STU + x{107b} STSLICECONST """ @fif_codegen """ test18() PROC:<{ - 421 PUSHINT NEWC - 26 STU + b{00000000000000000110100101} STSLICECONST """ @fif_codegen @@ -501,8 +498,7 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin 123 PUSHINT NEWC 4 STI - 16 PUSHPOW2DEC - 16 STUR + x{ffff} STSLICECONST -1 PUSHINT 8 STIR }> @@ -511,9 +507,8 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin @fif_codegen """ test20() PROC:<{ - 40976 PUSHINT NEWC - 16 STU + x{a010} STSLICECONST """ @fif_codegen @@ -529,12 +524,10 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin test22() PROC:<{ NEWC // '2 NEWC // b1 b2 - 2056 PUSHINT - ROT - 24 STU // b2 b1 + SWAP // b2 b1 + x{000808} STSLICECONST // b2 b1 SWAP // b1 b2 - 10141514286835656557042350424064 PUSHINTX - 132 STUR // b1 b2 + x{000000080010000000000000000000000} STSLICECONST // b1 b2 """ @fif_codegen @@ -543,13 +536,11 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin NEWC SWAP IF:<{ - 91343852333181432387730302044911803916571639814 PUSHINT - 256 STUR + x{0000000000000000000000001000000000000000000000000200000000000006} STSLICECONST 8 PUSHINT 256 STUR }>ELSE<{ - 56539106072908298546665520023773392506479484700019806659963456035401760775 PUSHINT - 255 STIR + b{000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000111} STSLICECONST }> BBITS }> @@ -597,9 +588,8 @@ We test that consequtive storeInt/storeUint with constants are joined into a sin s3 POP 42 EQINT IF:<{ - 0 PUSHINT - ROT - 3 STU + SWAP + b{000} STSLICECONST 8 STU // b }>ELSE<{ DROP // b diff --git a/tolk-tester/tests/constants-tests.tolk b/tolk-tester/tests/constants-tests.tolk index cafd2ffde..5a386cd4b 100644 --- a/tolk-tester/tests/constants-tests.tolk +++ b/tolk-tester/tests/constants-tests.tolk @@ -126,5 +126,5 @@ fun main() { @testcase | 105 | | -1 0 7 48 @testcase | 106 | | -1 0 0 -1 -@code_hash 39394999917297360167950120811797040756088634052063904555277102239104815314650 +@code_hash 32362412747322136329528616455651783746542516198110452861733590440068294458753 */ diff --git a/tolk-tester/tests/inline-tests.tolk b/tolk-tester/tests/inline-tests.tolk index fc48ddaa3..d85c5543b 100644 --- a/tolk-tester/tests/inline-tests.tolk +++ b/tolk-tester/tests/inline-tests.tolk @@ -467,13 +467,11 @@ fun usedIn10ButDeclaredBelow(x: int) { @fif_codegen """ test8() PROC:<{ // - 5 PUSHINT NEWC - 32 STU // '3 + x{00000005} STSLICECONST // '3 25 PUSHINT // '3 '9 - 107374182425 PUSHINT - ROT - 64 STU // x '3 + SWAP // x self + x{0000001900000019} STSLICECONST // x '3 SWAP // '3 x }> """ @@ -509,9 +507,8 @@ fun usedIn10ButDeclaredBelow(x: int) { test10() PROC:<{ 5 PUSHINT // '3=5 DUP // '3=5 y=5 - 5 PUSHINT NEWC - 32 STU // a=5 b=5 c + x{00000005} STSLICECONST // a=5 b=5 c BBITS // a=5 b=5 '18 }> """ diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk index 3e6a67bef..4d097a215 100644 --- a/tolk-tester/tests/lazy-load-tests.tolk +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -1900,9 +1900,8 @@ fun main() { x{0102} PUSHSLICE 8 LDI NIP - 15 PUSHINT NEWC - 8 STI + x{0f} STSLICECONST STSLICE """ @@ -1917,9 +1916,8 @@ fun main() { 320 PUSHINT // lazyS '14=320 PLDSLICEX // '15 PUSHNULL // '15 st.extensions - 0 PUSHINT NEWC - 1 STU // '15 st.extensions b + b{0} STSLICECONST // '15 st.extensions b s1 s2 XCHG // st.extensions '15 b STSLICE // st.extensions b STOPTREF // b diff --git a/tolk-tester/tests/pack-unpack-1.tolk b/tolk-tester/tests/pack-unpack-1.tolk index 406307cfd..9c9acf821 100644 --- a/tolk-tester/tests/pack-unpack-1.tolk +++ b/tolk-tester/tests/pack-unpack-1.tolk @@ -107,9 +107,8 @@ fun main(c: cell) { @fif_codegen """ test3() PROC:<{ - 42949672980 PUSHINT NEWC - 64 STI // b + x{0000000a00000014} STSLICECONST // b ENDC // c CTOS // s 64 LDU diff --git a/tolk-tester/tests/strings-tests.tolk b/tolk-tester/tests/strings-tests.tolk index 69d0fb2d5..99a422161 100644 --- a/tolk-tester/tests/strings-tests.tolk +++ b/tolk-tester/tests/strings-tests.tolk @@ -78,5 +78,5 @@ fun test1() { @testcase | 0 | | 0 @testcase | 101 | | [ 65 66 67 68 ] -@code_hash 33596158548631231081906594921712427436857221384530839008514203474433205250299 +@code_hash 52991558142486159683575375539594116045551936710713942466413832119934760926753 */ diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp index d9e7608be..605329bcd 100644 --- a/tolk/optimize.cpp +++ b/tolk/optimize.cpp @@ -164,8 +164,8 @@ bool Optimizer::detect_rewrite_big_THROW() { return true; } -// purpose 1: for one constant b.storeInt(123, 32) generate not "123 PUSHINT; SWAP; STI", but "123 PUSHINT; STIR" -// purpose 2: consecutive b.storeUint(ff, 16).storeUint(ff, 16) generate one "00ff00ff" STU +// purpose 1: for b.storeInt(123, 32) generate not "123 PUSHINT; SWAP; STI", but "x{...} STSLICECONST" +// purpose 2: consecutive b.storeUint(ff, 16).storeUint(ff, 16) generate one "x{00ff00ff} STSLICECONST" // (since it works at IR level, it also works for const variables and auto-serialization) bool Optimizer::detect_rewrite_MY_store_int() { bool first_my_store = op_[0]->is_custom() && op_[0]->op.starts_with("MY_store_int"); @@ -198,14 +198,38 @@ bool Optimizer::detect_rewrite_MY_store_int() { n_merged++; } - p_ = n_merged; - q_ = 2; - oq_[0] = std::make_unique(AsmOp::IntConst(op_[0]->loc, total_number)); - if (total_number == 0 && total_len == 4 && first_unsigned) { // "STGRAMS" stores four 0-bits cheaper than "4 STUR" - oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->loc, "STGRAMS", 1, 1)); - } else { + // we do not want to always use STSLICECONST; for example, storing "0" 64-bit via x{00...} is more effective + // for a single operation, but in practice, total bytecode becomes larger, which has a cumulative negative effect; + // here is a heuristic "when to use STSLICECONST, when leave PUSHINT + STUR", based on real contracts measurements + bool use_stsliceconst = total_len <= 32 || (total_len <= 48 && total_number >= 256) || (total_len <= 64 && total_number >= 65536) + || (total_len <= 96 && total_number >= (1ULL<<32)) || (total_number > (1ULL<<62)); + if (!use_stsliceconst) { + p_ = n_merged; + q_ = 2; + oq_[0] = std::make_unique(AsmOp::IntConst(op_[0]->loc, total_number)); oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->loc, std::to_string(total_len) + (first_unsigned ? " STUR" : " STIR"), 1, 1)); + return true; } + + p_ = n_merged; + q_ = 1; + + // output "x{...}" or "b{...}" (if length not divisible by 4) + const td::RefInt256 base = td::make_refint(total_len % 4 == 0 ? 16 : 2); + const int s_len = base == 16 ? total_len / 4 : total_len; + const char* digits = "0123456789abcdef"; + + std::string result(s_len + 3, '0'); + result[0] = base == 16 ? 'x' : 'b'; + result[1] = '{'; + result[s_len + 3 - 1] = '}'; + for (int i = s_len - 1; i >= 0 && total_number != 0; --i) { + result[2 + i] = digits[(total_number % base)->to_long()]; + total_number /= base; + } + + result += " STSLICECONST"; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, result, 0, 1)); return true; } From 83ca804ea68a56b2af6092599ae293e7550f29ee Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Fri, 4 Jul 2025 23:52:15 +0300 Subject: [PATCH 345/388] [Tolk] Peephole optimization: NOT+THROWIFNOT to THROWIF --- tolk-tester/tests/logical-operators.tolk | 6 +++--- tolk/optimize.cpp | 24 ++++++++++++++++++++++++ tolk/pipe-ast-to-legacy.cpp | 17 ++++------------- tolk/tolk.h | 1 + 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/tolk-tester/tests/logical-operators.tolk b/tolk-tester/tests/logical-operators.tolk index c184da72c..9b70ae40c 100644 --- a/tolk-tester/tests/logical-operators.tolk +++ b/tolk-tester/tests/logical-operators.tolk @@ -351,15 +351,15 @@ These are moments of future optimizations. For now, it's more than enough. DUP // x x IFNOTJMP:<{ // x DROP // - 1 PUSHINT // '5=1 + 1 PUSHINT // '7=1 }> // x DUP // x x IFNOTJMP:<{ // x DROP // - 1 PUSHINT // '6=1 + 1 PUSHINT // '8=1 }> // x 100 THROWIFNOT - -4 PUSHINT // '9=-4 + -4 PUSHINT // '11=-4 }> """ diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp index 605329bcd..93c6e2067 100644 --- a/tolk/optimize.cpp +++ b/tolk/optimize.cpp @@ -399,6 +399,29 @@ bool Optimizer::detect_rewrite_SWAP_STxxxR() { return false; } +// pattern `NOT` + `123 THROWIFNOT` -> `123 THROWIF` (and THROWIF -> THROWIFNOT) +bool Optimizer::detect_rewrite_NOT_THROWIF() { + bool first_not = op_[0]->is_custom() && op_[0]->op == "NOT"; + if (!first_not || pb_ < 2 || !op_[1]->is_custom()) { + return false; + } + + static const char* ends_with[] = {" THROWIF", " THROWIFNOT"}; + static const char* repl_with[] = {" THROWIFNOT", " THROWIF"}; + + std::string_view f = op_[1]->op; + for (size_t i = 0; i < std::size(ends_with); ++i) { + if (f.ends_with(ends_with[i])) { + p_ = 2; + q_ = 1; + oq_[0] = std::make_unique(AsmOp::Custom(op_[0]->loc, op_[1]->op.substr(0, f.rfind(' ')) + repl_with[i], 1, 0)); + return true; + } + } + + return false; +} + bool Optimizer::is_push_const(int* i, int* c) const { return pb_ >= 3 && pb_ <= l2_ && tr_[pb_ - 1].is_push_const(i, c); } @@ -821,6 +844,7 @@ bool Optimizer::find_at_least(int pb) { detect_rewrite_MY_store_int() || detect_rewrite_MY_skip_bits() || detect_rewrite_NEWC_PUSH_STUR() || detect_rewrite_LDxx_DROP() || detect_rewrite_SWAP_symmetric() || detect_rewrite_SWAP_PUSH_STUR() || detect_rewrite_SWAP_STxxxR() || + detect_rewrite_NOT_THROWIF() || (!(mode_ & 1) && ((is_rot() && rewrite(AsmOp::Custom(loc, "ROT", 3, 3))) || (is_rotrev() && rewrite(AsmOp::Custom(loc, "-ROT", 3, 3))) || (is_2dup() && rewrite(AsmOp::Custom(loc, "2DUP", 2, 4))) || diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 732dc3e18..7d1d2e1b6 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -2103,21 +2103,12 @@ static void process_block_statement(V v, CodeBlob& code) { } static void process_assert_statement(V v, CodeBlob& code) { - std::vector args(3); - if (auto v_not = v->get_cond()->try_as(); v_not && v_not->tok == tok_logical_not) { - args[0] = v->get_thrown_code(); - args[1] = v->get_cond()->as()->get_rhs(); - args[2] = createV(v->loc, true); - args[2]->mutate()->assign_inferred_type(TypeDataInt::create()); - } else { - args[0] = v->get_thrown_code(); - args[1] = v->get_cond(); - args[2] = createV(v->loc, false); - args[2]->mutate()->assign_inferred_type(TypeDataInt::create()); - } + std::vector ir_thrown_code = pre_compile_expr(v->get_thrown_code(), code); + std::vector ir_cond = pre_compile_expr(v->get_cond(), code); + tolk_assert(ir_cond.size() == 1 && ir_thrown_code.size() == 1); + std::vector args_vars = { ir_thrown_code[0], ir_cond[0], code.create_int(v->loc, 0, "(assert-0)") }; FunctionPtr builtin_sym = lookup_function("__throw_if_unless"); - std::vector args_vars = pre_compile_tensor(code, args); gen_op_call(code, TypeDataVoid::create(), v->loc, std::move(args_vars), builtin_sym, "(throw-call)"); } diff --git a/tolk/tolk.h b/tolk/tolk.h index 32c6843f8..9bef3c4df 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -906,6 +906,7 @@ struct Optimizer { bool detect_rewrite_SWAP_symmetric(); bool detect_rewrite_SWAP_PUSH_STUR(); bool detect_rewrite_SWAP_STxxxR(); + bool detect_rewrite_NOT_THROWIF(); AsmOpConsList extract_code(); }; From 38a95cb9e38e8f759f38b9c19441788da9dc2729 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sat, 5 Jul 2025 16:33:28 +0300 Subject: [PATCH 346/388] [Tolk] Better error messages on type mismatch For example, when passing `bits8` to `slice`, the compiler now suggests to use `as` operator. --- tolk-tester/tests/assignment-tests.tolk | 9 ++++ .../tests/invalid-typing/err-6010.tolk | 12 +++++ .../tests/invalid-typing/err-6087.tolk | 1 + .../tests/invalid-typing/err-6135.tolk | 1 + .../tests/invalid-typing/err-6249.tolk | 1 + .../tests/invalid-typing/err-6390.tolk | 8 +++ .../tests/invalid-typing/err-6595.tolk | 10 ++++ .../tests/invalid-typing/err-6603.tolk | 14 +++++ .../tests/invalid-typing/err-6633.tolk | 11 ++++ .../tests/invalid-typing/err-6716.tolk | 7 +++ .../tests/invalid-typing/err-6737.tolk | 1 + .../tests/invalid-typing/err-6803.tolk | 1 + .../tests/invalid-typing/err-6839.tolk | 1 + .../tests/invalid-typing/err-6937.tolk | 1 + tolk/pipe-ast-to-legacy.cpp | 5 ++ tolk/pipe-check-inferred-types.cpp | 54 +++++++++++++++---- tolk/pipe-infer-types-and-calls.cpp | 6 +-- tolk/type-system.cpp | 3 ++ 18 files changed, 132 insertions(+), 14 deletions(-) create mode 100644 tolk-tester/tests/invalid-typing/err-6010.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6390.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6595.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6603.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6633.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6716.tolk diff --git a/tolk-tester/tests/assignment-tests.tolk b/tolk-tester/tests/assignment-tests.tolk index 5b8c2094c..0343bdf50 100644 --- a/tolk-tester/tests/assignment-tests.tolk +++ b/tolk-tester/tests/assignment-tests.tolk @@ -249,6 +249,14 @@ fun test118(x: int) { return (i1, i2, st.owner.remainingBitsCount(), i3.0); } +@method_id(119) +fun test119(a: int, b: int) { + var t: tuple = [a + 1, b * 2] as tuple; + val l = t.size(); + val [c,d:int] = [t.0 as int, t.1]; + return (l, c, d); +} + fun main(value: int, ) { @@ -279,6 +287,7 @@ fun main(value: int, ) { @testcase | 116 | | 1 2 3 4 [ 1 2 3 4 ] @testcase | 117 | | [ 20 ] @testcase | 118 | 3 | 10 3 0 1 +@testcase | 119 | 1 2 | 2 2 4 @fif_codegen diff --git a/tolk-tester/tests/invalid-typing/err-6010.tolk b/tolk-tester/tests/invalid-typing/err-6010.tolk new file mode 100644 index 000000000..ece391173 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6010.tolk @@ -0,0 +1,12 @@ +fun analyze(t: slice) {} + +fun main(t: slice?) { + analyze(t); +} + +/** +@compilation_should_fail +@stderr can not pass `slice?` to `slice` +@stderr hint: probably, you should check on null +@stderr hint: alternatively, use `!` operator to bypass nullability checks: `!` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6087.tolk b/tolk-tester/tests/invalid-typing/err-6087.tolk index aabd956ea..dfc3ee62f 100644 --- a/tolk-tester/tests/invalid-typing/err-6087.tolk +++ b/tolk-tester/tests/invalid-typing/err-6087.tolk @@ -6,4 +6,5 @@ fun cantAutoCastBytesNToSlice() { /** @compilation_should_fail @stderr method `loadInt` not found for type `bits32` +@stderr but it exists for type `slice` */ diff --git a/tolk-tester/tests/invalid-typing/err-6135.tolk b/tolk-tester/tests/invalid-typing/err-6135.tolk index f55c6dd48..7bf580871 100644 --- a/tolk-tester/tests/invalid-typing/err-6135.tolk +++ b/tolk-tester/tests/invalid-typing/err-6135.tolk @@ -8,4 +8,5 @@ fun cantAssignIntNToCoins(n: int8, c: coins) { @compilation_should_fail @stderr can not assign `coins` to variable of type `int8` @stderr n = c; +@stderr hint: use `as` operator for unsafe casting: ` as int8` */ diff --git a/tolk-tester/tests/invalid-typing/err-6249.tolk b/tolk-tester/tests/invalid-typing/err-6249.tolk index 1553bf75e..f5f4722d9 100644 --- a/tolk-tester/tests/invalid-typing/err-6249.tolk +++ b/tolk-tester/tests/invalid-typing/err-6249.tolk @@ -7,4 +7,5 @@ fun cantUnifyBytesNAndSlice(n: slice, c: bytes16) { /** @compilation_should_fail @stderr types of ternary branches are incompatible: `slice` and `bytes16` +@stderr hint: maybe, you should use ` as ` to make them identical */ diff --git a/tolk-tester/tests/invalid-typing/err-6390.tolk b/tolk-tester/tests/invalid-typing/err-6390.tolk new file mode 100644 index 000000000..969cfd843 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6390.tolk @@ -0,0 +1,8 @@ +fun main(k: (int, slice)) { + var (a, b, c: int?) = k; +} + +/** +@compilation_should_fail +@stderr can not assign `(int, slice)`, sizes mismatch + */ diff --git a/tolk-tester/tests/invalid-typing/err-6595.tolk b/tolk-tester/tests/invalid-typing/err-6595.tolk new file mode 100644 index 000000000..c016ae187 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6595.tolk @@ -0,0 +1,10 @@ +fun main() { + blockchain.configParam(true); +} + +/** +@compilation_should_fail +@stderr can not pass `bool` to `int` +@stderr hint: use `as` operator for unsafe casting: ` as int` +@stderr caution! in TVM, bool TRUE is -1, not 1 + */ diff --git a/tolk-tester/tests/invalid-typing/err-6603.tolk b/tolk-tester/tests/invalid-typing/err-6603.tolk new file mode 100644 index 000000000..5f6275c35 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6603.tolk @@ -0,0 +1,14 @@ +struct A { + kk: tuple + dd: [int, int] +} + +fun main(a: A) { + a.kk = a.dd +} + +/** +@compilation_should_fail +@stderr can not assign `[int, int]` to field of type `tuple` +@stderr hint: use `as` operator for unsafe casting: ` as tuple` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6633.tolk b/tolk-tester/tests/invalid-typing/err-6633.tolk new file mode 100644 index 000000000..d8ed9c298 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6633.tolk @@ -0,0 +1,11 @@ +fun main(c: cell) { + var sender_address: slice = c.beginParse().loadAddress(); +} + +/** +@compilation_should_fail +@stderr can not assign `address` to variable of type `slice` +@stderr hint: unlike FunC, Tolk has a special type `address` (which is slice at the TVM level); +@stderr most likely, you just need `address` everywhere +@stderr hint: alternatively, use `as` operator for unsafe casting: ` as slice` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6716.tolk b/tolk-tester/tests/invalid-typing/err-6716.tolk new file mode 100644 index 000000000..134fb9d14 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6716.tolk @@ -0,0 +1,7 @@ +fun f(d: bits8 = "") {} + +/** +@compilation_should_fail +@stderr can not assign `slice` to `bits8` +@stderr hint: use `as` operator for unsafe casting: ` as bits8` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6737.tolk b/tolk-tester/tests/invalid-typing/err-6737.tolk index cc4bc2f59..86934f3c8 100644 --- a/tolk-tester/tests/invalid-typing/err-6737.tolk +++ b/tolk-tester/tests/invalid-typing/err-6737.tolk @@ -9,4 +9,5 @@ fun testAssignBetweenDifferentIntN(op: int32, qid: uint64) { @compilation_should_fail @stderr can not assign `uint64` to variable of type `int32` @stderr op = qid; +@stderr hint: use `as` operator for unsafe casting: ` as int32` */ diff --git a/tolk-tester/tests/invalid-typing/err-6803.tolk b/tolk-tester/tests/invalid-typing/err-6803.tolk index b04e8d974..1e9592a68 100644 --- a/tolk-tester/tests/invalid-typing/err-6803.tolk +++ b/tolk-tester/tests/invalid-typing/err-6803.tolk @@ -9,4 +9,5 @@ fun cantPassBytesNToSlice(c: bits16) { @compilation_should_fail @stderr can not pass `bits16` to `slice` @stderr takeAnySlice(c); +@stderr hint: use `as` operator for unsafe casting: ` as slice` */ diff --git a/tolk-tester/tests/invalid-typing/err-6839.tolk b/tolk-tester/tests/invalid-typing/err-6839.tolk index 91cca9c4f..e8a4fd7c0 100644 --- a/tolk-tester/tests/invalid-typing/err-6839.tolk +++ b/tolk-tester/tests/invalid-typing/err-6839.tolk @@ -16,4 +16,5 @@ fun cantMixDifferentThis() { /** @compilation_should_fail @stderr method `appendBuilder` not found for type `int` +@stderr (but it exists for type `builder`) */ diff --git a/tolk-tester/tests/invalid-typing/err-6937.tolk b/tolk-tester/tests/invalid-typing/err-6937.tolk index d380279a9..6038ea656 100644 --- a/tolk-tester/tests/invalid-typing/err-6937.tolk +++ b/tolk-tester/tests/invalid-typing/err-6937.tolk @@ -9,4 +9,5 @@ fun cantAssignSliceToBytesN() { @compilation_should_fail @stderr can not assign `slice` to variable of type `bytes16` @stderr var c2: bytes16 = getAnySlice(); +@stderr hint: use `as` operator for unsafe casting: ` as bytes16` */ diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 7d1d2e1b6..f22b0ab44 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -1075,6 +1075,11 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as() && target_type->try_as()) { + return rvect; + } + // pass something to `unknown` // probably, it comes from `_ = rhs`, type of `_` is unknown, it's target_type of rhs // no changes in rvect diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index b47513ee4..97a953f4f 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -49,6 +49,38 @@ static std::string expression_as_string(AnyExprV v) { return "expression"; } +// fire a general error on type mismatch; for example, "can not assign `cell` to `slice`"; +// for instance, if `as` operator is applicable, compiler will suggest it +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_type_mismatch(FunctionPtr cur_f, SrcLocation loc, const char* text_tpl, TypePtr src, TypePtr dst) { +#ifdef TOLK_DEBUG + tolk_assert(!dst->can_rhs_be_assigned(src)); +#endif + std::string message = text_tpl; + message.replace(message.find("{src}"), 5, to_string(src)); + message.replace(message.find("{dst}"), 5, to_string(dst)); + if (src->can_be_casted_with_as_operator(dst)) { + bool suggest_as = !dst->try_as() && !dst->try_as(); + if ((src == TypeDataAddress::create() || src == TypeDataSlice::create()) && (dst == TypeDataAddress::create() || dst == TypeDataSlice::create())) { + message += "\nhint: unlike FunC, Tolk has a special type `address` (which is slice at the TVM level);"; + message += "\n most likely, you just need `address` everywhere"; + message += "\nhint: alternatively, use `as` operator for unsafe casting: ` as " + dst->as_human_readable() + "`"; + } else if (suggest_as) { + message += "\nhint: use `as` operator for unsafe casting: ` as " + dst->as_human_readable() + "`"; + } + if (src == TypeDataBool::create() && dst == TypeDataInt::create()) { + message += "\ncaution! in TVM, bool TRUE is -1, not 1"; + } + } + if (const TypeDataUnion* src_nullable = src->try_as(); src_nullable && src_nullable->or_null) { + if (dst->can_rhs_be_assigned(src_nullable->or_null)) { + message += "\nhint: probably, you should check on null"; + message += "\nhint: alternatively, use `!` operator to bypass nullability checks: `!`"; + } + } + fire(cur_f, loc, message); +} + // fire an error on `!cell` / `+slice` GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_cannot_apply_operator(FunctionPtr cur_f, SrcLocation loc, std::string_view operator_name, AnyExprV unary_expr) { @@ -78,7 +110,7 @@ static void check_function_argument_passed(FunctionPtr cur_f, TypePtr param_type if (is_obj_of_dot_call) { fire(cur_f, ith_arg->loc, "can not call method for " + to_string(param_type) + " with object of type " + to_string(ith_arg)); } else { - fire(cur_f, ith_arg->loc, "can not pass " + to_string(ith_arg) + " to " + to_string(param_type)); + fire_error_type_mismatch(cur_f, ith_arg->loc, "can not pass {src} to {dst}", ith_arg->inferred_type, param_type); } } } @@ -349,7 +381,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { auto arg_i = v->get_arg(i)->get_expr(); TypePtr param_type = f_callable->params_types[i]; if (!param_type->can_rhs_be_assigned(arg_i->inferred_type)) { - fire(cur_f, arg_i->loc, "can not pass " + to_string(arg_i) + " to " + to_string(param_type)); + fire_error_type_mismatch(cur_f, arg_i->loc, "can not pass {src} to {dst}", arg_i->inferred_type, param_type); } } return; @@ -411,7 +443,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } if (declared_type) { if (!declared_type->can_rhs_be_assigned(rhs_type)) { - fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to variable of type " + to_string(declared_type)); + fire_error_type_mismatch(cur_f, err_loc->loc, "can not assign {src} to variable of type {dst}", rhs_type, declared_type); } } else { if (rhs_type == TypeDataNullLiteral::create()) { @@ -469,11 +501,11 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // for strange lhs like `f() = rhs` type checking will pass, but will fail lvalue check later if (!lhs->inferred_type->can_rhs_be_assigned(rhs_type)) { if (lhs->try_as()) { - fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to variable of type " + to_string(lhs)); + fire_error_type_mismatch(cur_f, err_loc->loc, "can not assign {src} to variable of type {dst}", rhs_type, lhs->inferred_type); } else if (lhs->try_as()) { - fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to field of type " + to_string(lhs)); + fire_error_type_mismatch(cur_f, err_loc->loc, "can not assign {src} to field of type {dst}", rhs_type, lhs->inferred_type); } else { - fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to " + to_string(lhs)); + fire_error_type_mismatch(cur_f, err_loc->loc, "can not assign {src} to {dst}", rhs_type, lhs->inferred_type); } } } @@ -490,7 +522,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { TypePtr expr_type = v->get_return_value()->inferred_type; if (!cur_f->inferred_return_type->can_rhs_be_assigned(expr_type)) { - fire(cur_f, v->get_return_value()->loc, "can not convert type " + to_string(expr_type) + " to return type " + to_string(cur_f->inferred_return_type)); + fire_error_type_mismatch(cur_f, v->get_return_value()->loc, "can not convert type {src} to return type {dst}", expr_type, cur_f->inferred_return_type); } } @@ -626,7 +658,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { parent::visit(v->get_init_val()); if (!v->field_ref->declared_type->can_rhs_be_assigned(v->get_init_val()->inferred_type)) { - fire(cur_f, v->get_init_val()->loc, "can not assign " + to_string(v->get_init_val()) + " to field of type " + to_string(v->field_ref->declared_type)); + fire_error_type_mismatch(cur_f, v->get_init_val()->loc, "can not assign {src} to field of type {dst}", v->get_init_val()->inferred_type, v->field_ref->declared_type); } } @@ -740,7 +772,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { TypePtr inferred_type = param_ref->default_value->inferred_type; if (!param_ref->declared_type->can_rhs_be_assigned(inferred_type)) { - throw ParseError(param_ref->loc, "can not assign " + to_string(inferred_type) + " to " + to_string(param_ref->declared_type)); + fire_error_type_mismatch(fun_ref, param_ref->loc, "can not assign {src} to {dst}", inferred_type, param_ref->declared_type); } } } @@ -756,7 +788,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (const_ref->declared_type) { // `const a: int = ...` TypePtr inferred_type = const_ref->init_value->inferred_type; if (!const_ref->declared_type->can_rhs_be_assigned(inferred_type)) { - throw ParseError(const_ref->loc, "can not assign " + to_string(inferred_type) + " to " + to_string(const_ref->declared_type)); + fire_error_type_mismatch(nullptr, const_ref->loc, "can not assign {src} to {dst}", inferred_type, const_ref->declared_type); } } } @@ -767,7 +799,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { TypePtr inferred_type = field_ref->default_value->inferred_type; if (!field_ref->declared_type->can_rhs_be_assigned(inferred_type)) { - throw ParseError(field_ref->loc, "can not assign " + to_string(inferred_type) + " to " + to_string(field_ref->declared_type)); + fire_error_type_mismatch(nullptr, field_ref->loc, "can not assign {src} to {dst}", inferred_type, field_ref->declared_type); } } }; diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 1253cdffc..1bca8908a 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -536,7 +536,7 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(lhs_var, rhs_type); assign_inferred_type(lhs_var->var_ref, rhs_type); } - TypePtr smart_casted_type = declared_type ? calc_smart_cast_type_on_assignment(declared_type, rhs_type) : rhs_type; + TypePtr smart_casted_type = declared_type && rhs_type != TypeDataUnknown::create() ? calc_smart_cast_type_on_assignment(declared_type, rhs_type) : rhs_type; out_flow.register_known_type(SinkExpression(lhs_var->var_ref), smart_casted_type); return; } @@ -740,7 +740,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // example: `var v = ternary`, show an inference error // do NOT show an error for `var v: T = ternary` (T is hint); it will be checked by type checker later if (hint == nullptr || hint == TypeDataUnknown::create() || hint->has_genericT_inside()) { - fire(cur_f, v->loc, "types of ternary branches are incompatible: " + to_string(v->get_when_true()) + " and " + to_string(v->get_when_false())); + fire(cur_f, v->loc, "types of ternary branches are incompatible: " + to_string(v->get_when_true()) + " and " + to_string(v->get_when_false()) + "\nhint: maybe, you should use ` as ` to make them identical"); } } assign_inferred_type(v, branches_unifier.get_result()); @@ -1081,7 +1081,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // as a special case, handle accessing fields of nullable objects, to show a more precise error if (const TypeDataUnion* as_union = obj_type->try_as(); as_union && as_union->or_null) { if (const TypeDataStruct* n_struct = as_union->or_null->try_as(); n_struct && n_struct->struct_ref->find_field(field_name)) { - fire(cur_f, v_ident->loc, "can not access field `" + static_cast(field_name) + "` of a possibly nullable object " + to_string(dot_obj) + "\ncheck it via `obj != null` or use non-null assertion `obj!` operator"); + fire(cur_f, v_ident->loc, "can not access field `" + static_cast(field_name) + "` of a possibly nullable object " + to_string(dot_obj) + "\nhint: check it via `obj != null` or use non-null assertion `obj!` operator"); } } if (out_f_called) { diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index 6253537e2..cf89edb0b 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -1146,6 +1146,9 @@ bool TypeDataTensor::can_be_casted_with_as_operator(TypePtr cast_to) const { } bool TypeDataBrackets::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (cast_to->try_as()) { // `[int, int]` as `tuple` + return true; + } if (const auto* to_tuple = cast_to->try_as(); to_tuple && to_tuple->size() == size()) { for (int i = 0; i < size(); ++i) { if (!items[i]->can_be_casted_with_as_operator(to_tuple->items[i])) { From 705a419edafadb5e821f83eb29844234e6ceb77f Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Mon, 7 Jul 2025 03:49:08 +0300 Subject: [PATCH 347/388] [Tolk] More correctly generate a serialization prefix tree for nullable unions Now, treat `T?` as `Maybe` even if T is a union. Example: A|B|C|D|null => 0 | 100+A | 101+B | 110+C | 111+D. If no `null`, just distribute sequentially: A|B|C => 00+A | 01+B | 10+C --- tolk-tester/tests/lazy-load-tests.tolk | 32 ++++++++++++++++++++++++-- tolk-tester/tests/pack-unpack-4.tolk | 16 ++++++------- tolk-tester/tests/pack-unpack-5.tolk | 4 ++-- tolk/pack-unpack-serializers.cpp | 13 +++++++---- 4 files changed, 48 insertions(+), 17 deletions(-) diff --git a/tolk-tester/tests/lazy-load-tests.tolk b/tolk-tester/tests/lazy-load-tests.tolk index 4d097a215..f686fe67c 100644 --- a/tolk-tester/tests/lazy-load-tests.tolk +++ b/tolk-tester/tests/lazy-load-tests.tolk @@ -241,6 +241,18 @@ struct WithGlobalModifier { n: int8; } +struct WithN5 { n: uint5 } +type U111 = int8 | int16 | WithN5 | bits100 +struct WithU111 { + a: U111 + b: U111? + bit: bool +} +fun WithU111.getSlice_a8_bnull(): slice asm "b{000000000101} PUSHSLICE" +fun WithU111.getSlice_a16_b8(): slice asm "b{010000000000000001100000000011} PUSHSLICE" +fun WithU111.getSlice_a8_b5(): slice asm "b{0000000001110111111} PUSHSLICE" +fun WithU111.getSlice_a8_b5_nobit(): slice asm "b{000000000111011111} PUSHSLICE" + @noinline fun contract.getFakeData(seqno: int): Cell { @@ -981,6 +993,21 @@ fun demo158() { return (m2.n, gModByCustom, after1); // also modifies by skipping (unpack called) } +@method_id(159) +fun demo159() { + val p1 = lazy WithU111.fromSlice(WithU111.getSlice_a8_bnull()); + val p2 = lazy WithU111.fromSlice(WithU111.getSlice_a16_b8()); + val p3 = lazy WithU111.fromSlice(WithU111.getSlice_a8_b5()); + val p4 = lazy WithU111.fromSlice(WithU111.getSlice_a8_b5_nobit()); + + val p1_bit = p1.bit; + val p2_bit = p2.bit; + val p3_bit = p3.bit; + val p4_b = p4.b; + + return (p1_bit, p2_bit, (p3.b is WithN5) ? p3.b.n as int : 777, p3_bit, (p4_b is WithN5) ? p4_b.n as int: 777); +} + @method_id(200) fun demo200() { @@ -1601,10 +1628,10 @@ fun main() { @testcase | 113 | x{03ffff} | -1 @testcase | 113 | x{04000000FF} | 255 @testcase | 114 | x{0110} | 16 1 -@testcase | 114 | x{020f} | [ -15 ] typeid-25 +@testcase | 114 | x{020f} | [ -15 ] typeid-27 @testcase | 114 | x{03} | -1 1 @testcase | 114 | x{03ffff} | -1 1 -@testcase | 114 | x{04000000FF} | [ 255 ] typeid-25 +@testcase | 114 | x{04000000FF} | [ 255 ] typeid-27 @testcase | 115 | x{08} | 8 42 @testcase | 115 | x{88} | 0 2 @testcase | 115 | x{83} | -1 2 @@ -1669,6 +1696,7 @@ fun main() { @testcase | 156 | x{FF10} | -100 @testcase | 157 | | -1 @testcase | 158 | | 2 10 255 +@testcase | 159 | | -1 -1 31 -1 31 @testcase | 200 | | 1 @testcase | 201 | | 1 2 -1 diff --git a/tolk-tester/tests/pack-unpack-4.tolk b/tolk-tester/tests/pack-unpack-4.tolk index 6d913f584..08a1b4328 100644 --- a/tolk-tester/tests/pack-unpack-4.tolk +++ b/tolk-tester/tests/pack-unpack-4.tolk @@ -38,10 +38,10 @@ fun test1() { type Union_8_16_32_n = int8 | null | int16 | int32; -fun ans102n_8(): slice asm "b{0100001111} PUSHSLICE"; -fun ans102n_16(): slice asm "b{100000000000001111} PUSHSLICE"; -fun ans102n_32(): slice asm "b{1100000000000000000000000000001111} PUSHSLICE"; -fun ans102n_null(): slice asm "b{00} PUSHSLICE"; +fun ans102n_8(): slice asm "b{10000001111} PUSHSLICE"; +fun ans102n_16(): slice asm "b{1010000000000001111} PUSHSLICE"; +fun ans102n_32(): slice asm "b{11000000000000000000000000000001111} PUSHSLICE"; +fun ans102n_null(): slice asm "b{0} PUSHSLICE"; @method_id(102) fun test2() { @@ -65,10 +65,10 @@ fun ans104_8(): slice asm "b{0000001111} PUSHSLICE"; fun ans104_16(): slice asm "b{010000000000001111} PUSHSLICE"; fun ans104_32(): slice asm "b{1000000000000000000000000000001111} PUSHSLICE"; -fun ans104n_8(): slice asm "b{0100001111} PUSHSLICE"; -fun ans104n_16(): slice asm "b{100000000000001111} PUSHSLICE"; -fun ans104n_32(): slice asm "b{1100000000000000000000000000001111} PUSHSLICE"; -fun ans104n_null(): slice asm "b{00} PUSHSLICE"; +fun ans104n_8(): slice asm "b{10000001111} PUSHSLICE"; +fun ans104n_16(): slice asm "b{1010000000000001111} PUSHSLICE"; +fun ans104n_32(): slice asm "b{11000000000000000000000000000001111} PUSHSLICE"; +fun ans104n_null(): slice asm "b{0} PUSHSLICE"; @method_id(104) fun test4() { diff --git a/tolk-tester/tests/pack-unpack-5.tolk b/tolk-tester/tests/pack-unpack-5.tolk index 1baa24064..97b740b48 100644 --- a/tolk-tester/tests/pack-unpack-5.tolk +++ b/tolk-tester/tests/pack-unpack-5.tolk @@ -160,7 +160,7 @@ struct Test9_bits4 { f: bits4; } type Test9_f1 = int32 | int64 | int128; // auto-generated 2-bit prefix type Test9_f2 = int32 | Inner8_2; // auto-generated 1-bit prefix (Either) type Test9_f3 = bits1 | Test9_bits2 | bits3 | bits4 | bits5; // auto-generated 3-bit prefix -type Test9_f4 = bits1 | Test9_bits2 | bits3 | Test9_bits4?; // auto-generated 3-bit prefix +type Test9_f4 = bits1 | Test9_bits2 | bits3 | Test9_bits4?; // auto-generated 0 / 100/101/110/111 @method_id(109) fun test9() { @@ -213,7 +213,7 @@ fun main() { @testcase | 106 | | [ 64 64 0 0 ] [ 5 37 1 1 ] [ 5 65 0 1 ] @testcase | 107 | | [ 16 16 0 0 ] [ 16 16 0 0 ] @testcase | 108 | | [ 10 275 0 0 ] [ 34 158 0 0 ] [ 47 1143 0 1 ] -@testcase | 109 | | [ 34 130 0 0 ] [ 33 159 0 0 ] [ 4 8 0 0 ] [ 3 7 0 0 ] +@testcase | 109 | | [ 34 130 0 0 ] [ 33 159 0 0 ] [ 4 8 0 0 ] [ 1 7 0 0 ] @testcase | 110 | | [ 32 9999 0 4 ] [ 33 9999 0 8 ] @testcase | 111 | | [ 1023 1023 0 1 ] [ 0 0 2 2 ] @testcase | 120 | | 16 4128 1 1 diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index ecad40502..5f1ebcc8f 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -1146,13 +1146,16 @@ std::vector auto_generate_opcodes_for_union(TypePtr union_type, std: // okay, none of the opcodes are specified, generate a prefix tree; // examples: `int32 | int64 | int128` / `int32 | A | null` / `A | B` / `A | B | C`; - // create prefixes `0b00 0b01 0b10` / `0b01 0b10 0b00` (use 0b00 for null if exists): - // for 3/4 variants — two bits, for 5 — three - int prefix_len = static_cast(std::ceil(std::log2(t_union->size()))); - int cur_prefix = has_null ? 1 : 0; // will use 0b00 for null, so start with 0b01 + // if `null` exists, it's 0, all others are 1+tree: A|B|C|D|null => 0 | 100+A | 101+B | 110+C | 111+D; + // if no `null`, just distribute sequentially: A|B|C => 00+A | 01+B | 10+C + int n_without_null = t_union->size() - has_null; + int prefix_len = static_cast(std::ceil(std::log2(n_without_null))); + int cur_prefix = 0; for (TypePtr variant : t_union->variants) { if (variant == TypeDataNullLiteral::create()) { - result.emplace_back(0, prefix_len); + result.emplace_back(0, 1); + } else if (has_null) { + result.emplace_back((1< Date: Mon, 7 Jul 2025 04:02:48 +0300 Subject: [PATCH 348/388] [Tolk] Bump version to v1.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finally, after 7 months of development — the release --- crypto/smartcont/tolk-stdlib/common.tolk | 2 +- crypto/smartcont/tolk-stdlib/gas-payments.tolk | 2 +- crypto/smartcont/tolk-stdlib/lisp-lists.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-dicts.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk | 2 +- tolk/tolk-version.h | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index a72718246..7d21dd7d9 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -1,7 +1,7 @@ // Standard library for Tolk (LGPL licence). // It contains common functions that are available out of the box, the user doesn't have to import anything. // More specific functions are required to be imported explicitly, like "@stdlib/tvm-dicts". -tolk 0.99 +tolk 1.0 /// In Tolk v1.x there would be a type `map`. /// Currently, working with dictionaries is still low-level, with raw cells. diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk index 977fa6f2d..916cefc6c 100644 --- a/crypto/smartcont/tolk-stdlib/gas-payments.tolk +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.99 +tolk 1.0 /** Gas and payment related primitives. diff --git a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk index 2853abef0..212586b1d 100644 --- a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk +++ b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.99 +tolk 1.0 /** Lisp-style lists are nested 2-elements tuples: `[1, [2, [3, null]]]` represents list `[1, 2, 3]`. diff --git a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk index 14723682a..41e091f0a 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.99 +tolk 1.0 /** Dictionaries are represented as `cell` data type (cells can store anything, dicts in particular). diff --git a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk index 18a0e159f..d578199c7 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.99 +tolk 1.0 /// Usually `c3` has a continuation initialized by the whole code of the contract. It is used for function calls. /// The primitive returns the current value of `c3`. diff --git a/tolk/tolk-version.h b/tolk/tolk-version.h index aaf1255d2..77d745eac 100644 --- a/tolk/tolk-version.h +++ b/tolk/tolk-version.h @@ -18,6 +18,6 @@ namespace tolk { -constexpr const char* TOLK_VERSION = "0.99.0"; +constexpr const char* TOLK_VERSION = "1.0.0"; } // namespace tolk From a89cafb4e941478d34357e58e3e82a05d3b75b02 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 9 Jul 2025 11:52:38 +0300 Subject: [PATCH 349/388] Move limit for storage dict hash to config --- crypto/block/block.tlb | 2 +- crypto/block/mc-config.cpp | 1 + crypto/block/mc-config.h | 1 + crypto/block/transaction.cpp | 4 ++-- crypto/block/transaction.h | 1 + validator/impl/validate-query.cpp | 1 + 6 files changed, 7 insertions(+), 3 deletions(-) diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 83ec61ed5..7bff1ac17 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -798,7 +798,7 @@ size_limits_config#01 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells size_limits_config_v2#02 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16 max_ext_msg_size:uint32 max_ext_msg_depth:uint16 max_acc_state_cells:uint32 max_acc_state_bits:uint32 max_acc_public_libraries:uint32 defer_out_queue_size_limit:uint32 max_msg_extra_currencies:uint32 - max_acc_fixed_prefix_length:uint8 = SizeLimitsConfig; + max_acc_fixed_prefix_length:uint8 acc_state_cells_for_storage_dict:uint32 = SizeLimitsConfig; _ SizeLimitsConfig = ConfigParam 43; // key is [ wc:int32 addr:uint256 ] diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 35c3df005..bd9b6289d 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2046,6 +2046,7 @@ td::Result Config::do_get_size_limits_config(td::Refsize(); - // TODO: think about this limit (25) - if (store_storage_dict_hash && new_storage_used.cells > 25) { + if (store_storage_dict_hash && new_storage_used.cells >= cfg.size_limits.acc_state_cells_for_storage_dict) { auto r_hash = stats.get_dict_hash(); if (r_hash.is_error()) { LOG(ERROR) << "Cannot compute storage dict hash for account " << account.addr.to_hex() << ": " @@ -4184,6 +4183,7 @@ td::Status FetchConfigParams::fetch_config_params( serialize_cfg->extra_currency_v2 = config.get_global_version() >= 10; serialize_cfg->disable_anycast = config.get_global_version() >= 10; serialize_cfg->store_storage_dict_hash = config.get_global_version() >= 11; + serialize_cfg->size_limits = size_limits; } { // fetch block_grams_created diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index 1a7af2d12..9ef06e3c3 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -186,6 +186,7 @@ struct SerializeConfig { bool extra_currency_v2{false}; bool disable_anycast{false}; bool store_storage_dict_hash{false}; + SizeLimitsConfig size_limits; }; struct CreditPhase { diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index ac7f400f3..58576e027 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -1079,6 +1079,7 @@ bool ValidateQuery::fetch_config_params() { serialize_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; serialize_cfg_.disable_anycast = config_->get_global_version() >= 10; serialize_cfg_.store_storage_dict_hash = config_->get_global_version() >= 11; + serialize_cfg_.size_limits = size_limits; } { // fetch block_grams_created From a969178974d0755e9f30737e7883fba9e0b359e8 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 9 Jul 2025 11:53:08 +0300 Subject: [PATCH 350/388] BTOS opcode; decrease gas usage for HASHSU --- common/global-version.h | 2 +- crypto/fift/lib/Asm.fif | 1 + crypto/vm/cellops.cpp | 10 ++++++++++ crypto/vm/tonops.cpp | 9 ++++++--- doc/GlobalVersions.md | 12 +++++++++++- 5 files changed, 29 insertions(+), 5 deletions(-) diff --git a/common/global-version.h b/common/global-version.h index b54a3bdc5..d88a22c32 100644 --- a/common/global-version.h +++ b/common/global-version.h @@ -19,6 +19,6 @@ namespace ton { // See doc/GlobalVersions.md -constexpr int SUPPORTED_VERSION = 11; +constexpr int SUPPORTED_VERSION = 12; } diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index 941351397..3a9e3e447 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -697,6 +697,7 @@ x{CF3F} @Defop BCHKBITREFSQ x{CF40} @Defop STZEROES x{CF41} @Defop STONES x{CF42} @Defop STSAME +x{CF50} @Defop BTOS { tuck sbitrefs swap 22 + swap @havebitrefs not { swap PUSHSLICE STSLICER } { over sbitrefs 2dup 57 3 2x<= diff --git a/crypto/vm/cellops.cpp b/crypto/vm/cellops.cpp index 61ffe5c55..b76439fac 100644 --- a/crypto/vm/cellops.cpp +++ b/crypto/vm/cellops.cpp @@ -764,6 +764,15 @@ int exec_store_same(VmState* st, const char* name, int val) { return 0; } +int exec_builder_to_slice(VmState* st) { + Stack& stack = st->get_stack(); + VM_LOG(st) << "execute BTOS"; + stack.check_underflow(1); + Ref cell = stack.pop_builder().write().finalize_novm(); + stack.push_cellslice(Ref{true, NoVm(), cell}); + return 0; +} + int exec_store_const_slice(VmState* st, CellSlice& cs, unsigned args, int pfx_bits) { unsigned refs = (args >> 3) & 3; unsigned data_bits = (args & 7) * 8 + 2; @@ -866,6 +875,7 @@ void register_cell_serialize_ops(OpcodeTable& cp0) { .insert(OpcodeInstr::mksimple(0xcf40, 16, "STZEROES", std::bind(exec_store_same, _1, "STZEROES", 0))) .insert(OpcodeInstr::mksimple(0xcf41, 16, "STONES", std::bind(exec_store_same, _1, "STONES", 1))) .insert(OpcodeInstr::mksimple(0xcf42, 16, "STSAME", std::bind(exec_store_same, _1, "STSAME", -1))) + .insert(OpcodeInstr::mksimple(0xcf50, 16, "BTOS", exec_builder_to_slice)->require_version(12)) .insert(OpcodeInstr::mkext(0xcf80 >> 7, 9, 5, dump_store_const_slice, exec_store_const_slice, compute_len_store_const_slice)); } diff --git a/crypto/vm/tonops.cpp b/crypto/vm/tonops.cpp index b06fb1b77..3173f9d1c 100644 --- a/crypto/vm/tonops.cpp +++ b/crypto/vm/tonops.cpp @@ -618,10 +618,13 @@ int exec_compute_hash(VmState* st, int mode) { hash = cell->get_hash().as_array(); } else { auto cs = stack.pop_cellslice(); - vm::CellBuilder cb; + CellBuilder cb; CHECK(cb.append_cellslice_bool(std::move(cs))); - // TODO: use cb.get_hash() instead - hash = cb.finalize()->get_hash().as_array(); + if (st->get_global_version() >= 12) { + hash = cb.finalize_novm()->get_hash().as_array(); + } else { + hash = cb.finalize()->get_hash().as_array(); + } } td::RefInt256 res{true}; CHECK(res.write().import_bytes(hash.data(), hash.size(), false)); diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index ad5eba948..674fba266 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -144,6 +144,7 @@ Example: if the last masterchain block seqno is `19071` then the list contains b - Fix recursive jump to continuations with non-null control data. ## Version 10 +__Enabled in mainnet on 2025-05-07__ ### Extra currencies - Internal messages cannot carry more than 2 different extra currencies. The limit can be changed in size limits config (`ConfigParam 43`). @@ -194,6 +195,7 @@ Reserve modes `+1`, `+4` and `+8` ("reserve all except", "add original balance" - Exceeding state limits in transaction now reverts `end_lt` back to `start_lt + 1` and collects action fines. ## Version 11 +__Enabled in mainnet on 2025-07-05__ ### c7 tuple **c7** tuple extended from 17 to 18 elements: @@ -225,4 +227,12 @@ This is required to help computing storage stats in the future, after collator-v ### Other changes - Fix returning `null` as `c4` and `c5` (when VM state is not committed) in `RUNVM`. -- In new internal messages `ihr_disabled` is automatically set to `1`, `ihr_fee` is always zero. \ No newline at end of file +- In new internal messages `ihr_disabled` is automatically set to `1`, `ihr_fee` is always zero. + +## Version 12 + +### New TVM instructions +- `BTOS` (`b - s`) - same as `ENDC CTOS`, but without gas cost for cell creation and loading. Gas cost: `26`. + +### Other TVM changes +- `HASHSU` (`s - hash`) now does not spend gas for cell creation. Gas cost: `26`. \ No newline at end of file From 07f2565e8751784c8bed5c51cad5786c0d7de86e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 9 Jul 2025 14:24:49 +0300 Subject: [PATCH 351/388] HASHBU opcode --- crypto/fift/lib/Asm.fif | 1 + crypto/vm/tonops.cpp | 10 +++++++--- doc/GlobalVersions.md | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index 3a9e3e447..a4b1e144d 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -1391,6 +1391,7 @@ x{F912} @Defop ECRECOVER x{F913} @Defop SECP256K1_XONLY_PUBKEY_TWEAK_ADD x{F914} @Defop P256_CHKSIGNU x{F915} @Defop P256_CHKSIGNS +x{F916} @Defop HASHBU x{F920} @Defop RIST255_FROMHASH x{F921} @Defop RIST255_VALIDATE diff --git a/crypto/vm/tonops.cpp b/crypto/vm/tonops.cpp index 3173f9d1c..981f7c5b3 100644 --- a/crypto/vm/tonops.cpp +++ b/crypto/vm/tonops.cpp @@ -610,13 +610,13 @@ void register_prng_ops(OpcodeTable& cp0) { } int exec_compute_hash(VmState* st, int mode) { - VM_LOG(st) << "execute HASH" << (mode & 1 ? 'S' : 'C') << 'U'; + VM_LOG(st) << "execute HASH" << "CSB"[mode] << 'U'; Stack& stack = st->get_stack(); std::array hash; - if (!(mode & 1)) { + if (mode == 0) { // cell auto cell = stack.pop_cell(); hash = cell->get_hash().as_array(); - } else { + } else if (mode == 1) { // slice auto cs = stack.pop_cellslice(); CellBuilder cb; CHECK(cb.append_cellslice_bool(std::move(cs))); @@ -625,6 +625,9 @@ int exec_compute_hash(VmState* st, int mode) { } else { hash = cb.finalize()->get_hash().as_array(); } + } else { // builder + auto cb = stack.pop_builder(); + hash = cb.write().finalize_novm()->get_hash().as_array(); } td::RefInt256 res{true}; CHECK(res.write().import_bytes(hash.data(), hash.size(), false)); @@ -1368,6 +1371,7 @@ void register_ton_crypto_ops(OpcodeTable& cp0) { .insert(OpcodeInstr::mksimple(0xf913, 16, "SECP256K1_XONLY_PUBKEY_TWEAK_ADD", exec_secp256k1_xonly_pubkey_tweak_add)->require_version(9)) .insert(OpcodeInstr::mksimple(0xf914, 16, "P256_CHKSIGNU", std::bind(exec_p256_chksign, _1, false))->require_version(4)) .insert(OpcodeInstr::mksimple(0xf915, 16, "P256_CHKSIGNS", std::bind(exec_p256_chksign, _1, true))->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf916, 16, "HASHBU", std::bind(exec_compute_hash, _1, 2))->require_version(12)) .insert(OpcodeInstr::mksimple(0xf920, 16, "RIST255_FROMHASH", exec_ristretto255_from_hash)->require_version(4)) .insert(OpcodeInstr::mksimple(0xf921, 16, "RIST255_VALIDATE", std::bind(exec_ristretto255_validate, _1, false))->require_version(4)) diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index 674fba266..47683c431 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -233,6 +233,7 @@ This is required to help computing storage stats in the future, after collator-v ### New TVM instructions - `BTOS` (`b - s`) - same as `ENDC CTOS`, but without gas cost for cell creation and loading. Gas cost: `26`. +- `HASHBU` (`b - hash`) - same as `ENDC HASHCU`, but without gas cost for cell creation. Gas cost: `26`. ### Other TVM changes - `HASHSU` (`s - hash`) now does not spend gas for cell creation. Gas cost: `26`. \ No newline at end of file From 298db547f868e933e70efcef3d228fe09dca9162 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 9 Jul 2025 15:28:30 +0300 Subject: [PATCH 352/388] New account size limit for masterchain --- crypto/block/block.tlb | 2 +- crypto/block/mc-config.cpp | 2 +- crypto/block/mc-config.h | 2 +- crypto/block/transaction.cpp | 18 ++++++++++-------- crypto/block/transaction.h | 3 ++- doc/GlobalVersions.md | 6 +++++- validator/impl/validate-query.cpp | 1 + 7 files changed, 21 insertions(+), 13 deletions(-) diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 7bff1ac17..1967ca023 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -796,7 +796,7 @@ _ MisbehaviourPunishmentConfig = ConfigParam 40; size_limits_config#01 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16 max_ext_msg_size:uint32 max_ext_msg_depth:uint16 = SizeLimitsConfig; size_limits_config_v2#02 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16 - max_ext_msg_size:uint32 max_ext_msg_depth:uint16 max_acc_state_cells:uint32 max_acc_state_bits:uint32 + max_ext_msg_size:uint32 max_ext_msg_depth:uint16 max_acc_state_cells:uint32 max_mc_acc_state_cells:uint32 max_acc_public_libraries:uint32 defer_out_queue_size_limit:uint32 max_msg_extra_currencies:uint32 max_acc_fixed_prefix_length:uint8 acc_state_cells_for_storage_dict:uint32 = SizeLimitsConfig; _ SizeLimitsConfig = ConfigParam 43; diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index bd9b6289d..d7570863e 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2040,8 +2040,8 @@ td::Result Config::do_get_size_limits_config(td::Ref& old_l * This function is not called for special accounts. * * @param size_limits The size limits configuration. + * @param global_version Global version (ConfigParam 8). * @param is_account_stat Store storage stat in the Transaction's AccountStorageStat. * * @returns A `td::Status` indicating the result of the check. @@ -3182,7 +3183,8 @@ static td::uint32 get_public_libraries_diff_count(const td::Ref& old_l * - If the state limits exceed the maximum allowed range, returns an error with AccountStorageStat::errorcode_limits_exceeded code. * - If an error occurred during storage stat calculation, returns other error. */ -td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, bool is_account_stat) { +td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, int global_version, + bool is_account_stat) { auto cell_equal = [](const td::Ref& a, const td::Ref& b) -> bool { return a.is_null() || b.is_null() ? a.is_null() == b.is_null() : a->get_hash() == b->get_hash(); }; @@ -3213,13 +3215,12 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, } } - if (storage_stat.get_total_cells() > size_limits.max_acc_state_cells || - storage_stat.get_total_bits() > size_limits.max_acc_state_bits) { + td::uint32 max_cells = account.is_masterchain() && global_version >= 12 ? size_limits.max_mc_acc_state_cells + : size_limits.max_acc_state_cells; + if (storage_stat.get_total_cells() > max_cells) { return td::Status::Error(AccountStorageStat::errorcode_limits_exceeded, PSTRING() << "account state is too big: cells=" << storage_stat.get_total_cells() - << ", bits=" << storage_stat.get_total_bits() - << " (max cells=" << size_limits.max_acc_state_cells - << ", max bits=" << size_limits.max_acc_state_bits << ")"); + << " (max cells=" << max_cells << ")"); } if (account.is_masterchain() && !cell_equal(account.library, new_library)) { auto libraries_count = get_public_libraries_count(new_library); @@ -4178,6 +4179,7 @@ td::Status FetchConfigParams::fetch_config_params( action_phase_cfg->extra_currency_v2 = config.get_global_version() >= 10; action_phase_cfg->disable_anycast = config.get_global_version() >= 10; action_phase_cfg->disable_ihr_flag = config.get_global_version() >= 11; + action_phase_cfg->global_version = config.get_global_version(); } { serialize_cfg->extra_currency_v2 = config.get_global_version() >= 10; diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index 9ef06e3c3..5492ee6bf 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -177,6 +177,7 @@ struct ActionPhaseConfig { bool disable_ihr_flag{false}; td::optional mc_blackhole_addr; bool disable_anycast{false}; + int global_version = 0; const MsgPrices& fetch_msg_prices(bool is_masterchain) const { return is_masterchain ? fwd_mc : fwd_std; } @@ -413,7 +414,7 @@ struct Transaction { bool run_precompiled_contract(const ComputePhaseConfig& cfg, precompiled::PrecompiledSmartContract& precompiled); bool prepare_compute_phase(const ComputePhaseConfig& cfg); bool prepare_action_phase(const ActionPhaseConfig& cfg); - td::Status check_state_limits(const SizeLimitsConfig& size_limits, bool is_account_stat = true); + td::Status check_state_limits(const SizeLimitsConfig& size_limits, int global_version, bool is_account_stat = true); bool prepare_bounce_phase(const ActionPhaseConfig& cfg); bool compute_state(const SerializeConfig& cfg); bool serialize(const SerializeConfig& cfg); diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index 47683c431..6281735e0 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -236,4 +236,8 @@ This is required to help computing storage stats in the future, after collator-v - `HASHBU` (`b - hash`) - same as `ENDC HASHCU`, but without gas cost for cell creation. Gas cost: `26`. ### Other TVM changes -- `HASHSU` (`s - hash`) now does not spend gas for cell creation. Gas cost: `26`. \ No newline at end of file +- `HASHSU` (`s - hash`) now does not spend gas for cell creation. Gas cost: `26`. + +### Other changes +- Account size in masterchain is now limited to `2048` cells. This can be configured in size limits config (`ConfigParam 43`). + - The previous limit was the same as in basechain (`65536`). \ No newline at end of file diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 58576e027..4bef90395 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -1074,6 +1074,7 @@ bool ValidateQuery::fetch_config_params() { action_phase_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; action_phase_cfg_.disable_anycast = config_->get_global_version() >= 10; action_phase_cfg_.disable_ihr_flag = config_->get_global_version() >= 11; + action_phase_cfg_.global_version = config_->get_global_version(); } { serialize_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; From 126ac76a50d07eab5cd5df914478d8de9dec161c Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 23 Jul 2025 16:08:41 +0300 Subject: [PATCH 353/388] New bounce format --- crypto/block/block-parse.cpp | 46 ++++++-------- crypto/block/block-parse.h | 42 +++++++----- crypto/block/block.tlb | 14 +++- crypto/block/transaction.cpp | 92 +++++++++++++++++++-------- crypto/block/transaction.h | 4 +- crypto/smc-envelope/SmartContract.cpp | 4 +- crypto/vm/boc.cpp | 10 +++ crypto/vm/boc.h | 2 + crypto/vm/cells/CellBuilder.h | 3 + crypto/vm/tonops.cpp | 18 ++++-- doc/GlobalVersions.md | 33 ++++++++++ emulator/test/emulator-tests.cpp | 2 +- tonlib/tonlib/TonlibClient.cpp | 9 +-- validator/impl/collator-impl.h | 4 ++ validator/impl/collator.cpp | 8 ++- validator/impl/ihr-message.cpp | 2 +- validator/impl/liteserver.cpp | 6 +- validator/impl/validate-query.cpp | 40 +++++++++--- validator/impl/validate-query.hpp | 4 ++ 19 files changed, 242 insertions(+), 101 deletions(-) diff --git a/crypto/block/block-parse.cpp b/crypto/block/block-parse.cpp index 71ec2bcf5..9443b69f3 100644 --- a/crypto/block/block-parse.cpp +++ b/crypto/block/block-parse.cpp @@ -665,7 +665,7 @@ bool CommonMsgInfo::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const && t_MsgAddressInt.validate_skip(ops, cs, weak) // src && t_MsgAddressInt.validate_skip(ops, cs, weak) // dest && t_CurrencyCollection.validate_skip(ops, cs, weak) // value - && t_Grams.validate_skip(ops, cs, weak) // ihr_fee + && t_Grams.validate_skip(ops, cs, weak) // extra_flags && t_Grams.validate_skip(ops, cs, weak) // fwd_fee && cs.advance(64 + 32); // created_lt:uint64 created_at:uint32 case ext_in_msg_info: @@ -684,7 +684,7 @@ bool CommonMsgInfo::unpack(vm::CellSlice& cs, CommonMsgInfo::Record_int_msg_info return get_tag(cs) == int_msg_info && cs.advance(1) && cs.fetch_bool_to(data.ihr_disabled) && cs.fetch_bool_to(data.bounce) && cs.fetch_bool_to(data.bounced) && t_MsgAddressInt.fetch_to(cs, data.src) && t_MsgAddressInt.fetch_to(cs, data.dest) && t_CurrencyCollection.fetch_to(cs, data.value) && - t_Grams.fetch_to(cs, data.ihr_fee) && t_Grams.fetch_to(cs, data.fwd_fee) && + t_Grams.fetch_to(cs, data.extra_flags) && t_Grams.fetch_to(cs, data.fwd_fee) && cs.fetch_uint_to(64, data.created_lt) && cs.fetch_uint_to(32, data.created_at); } @@ -696,7 +696,7 @@ bool CommonMsgInfo::skip(vm::CellSlice& cs) const { && t_MsgAddressInt.skip(cs) // src && t_MsgAddressInt.skip(cs) // dest && t_CurrencyCollection.skip(cs) // value - && t_Grams.skip(cs) // ihr_fee + && t_Grams.skip(cs) // extra_flags && t_Grams.skip(cs) // fwd_fee && cs.advance(64 + 32); // created_lt:uint64 created_at:uint32 case ext_in_msg_info: @@ -718,7 +718,7 @@ bool CommonMsgInfo::get_created_lt(vm::CellSlice& cs, unsigned long long& create && t_MsgAddressInt.skip(cs) // src && t_MsgAddressInt.skip(cs) // dest && t_CurrencyCollection.skip(cs) // value - && t_Grams.skip(cs) // ihr_fee + && t_Grams.skip(cs) // extra_flags && t_Grams.skip(cs) // fwd_fee && cs.fetch_ulong_bool(64, created_lt) // created_lt:uint64 && cs.advance(32); // created_at:uint32 @@ -1843,26 +1843,18 @@ bool InMsg::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { return false; } -bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const { +static td::RefInt256 get_ihr_fee(const CommonMsgInfo::Record_int_msg_info &info, int global_version) { + // Legacy: extra_flags was previously ihr_fee + return global_version >= 12 ? td::zero_refint() : t_Grams.as_integer(std::move(info.extra_flags)); +} + +bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs, int global_version) const { int tag = get_tag(cs); switch (tag) { case msg_import_ext: // inbound external message return t_ImportFees.null_value(cb); // external messages have no value and no import fees case msg_import_ihr: // IHR-forwarded internal message to its final destination - if (cs.advance(3) && cs.size_refs() >= 3) { - auto msg_cs = load_cell_slice(cs.fetch_ref()); - CommonMsgInfo::Record_int_msg_info msg_info; - td::RefInt256 ihr_fee; - vm::CellBuilder aux; - // sort of Prolog-style in C++ - return t_Message.extract_info(msg_cs) && t_CommonMsgInfo.unpack(msg_cs, msg_info) && - cs.fetch_ref().not_null() && (ihr_fee = t_Grams.as_integer_skip(cs)).not_null() && - cs.fetch_ref().not_null() && !cmp(ihr_fee, t_Grams.as_integer(*msg_info.ihr_fee)) && - cb.append_cellslice_bool(msg_info.ihr_fee) // fees_collected := ihr_fee - && aux.append_cellslice_bool(msg_info.ihr_fee) && t_ExtraCurrencyCollection.null_value(aux) && - t_CurrencyCollection.add_values(cb, aux.as_cellslice_ref().write(), - msg_info.value.write()); // value_imported := ihr_fee + value - } + // IHR is not implemented return false; case msg_import_imm: // internal message re-imported from this very block if (cs.advance(3) && cs.size_refs() >= 2) { @@ -1888,7 +1880,7 @@ bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const { return t_Message.extract_info(msg_cs) && t_CommonMsgInfo.unpack(msg_cs, msg_info) && cb.append_cellslice_bool(in_msg.fwd_fee_remaining) // fees_collected := fwd_fee_remaining && t_Grams.as_integer_skip_to(msg_info.value.write(), value_grams) && - (ihr_fee = t_Grams.as_integer(std::move(msg_info.ihr_fee))).not_null() && + (ihr_fee = get_ihr_fee(msg_info, global_version)).not_null() && t_Grams.store_integer_ref(cb, value_grams + ihr_fee + fwd_fee_remaining) && cb.append_cellslice_bool( msg_info.value.write()); // value_imported = msg.value + msg.ihr_fee + fwd_fee_remaining @@ -1911,7 +1903,7 @@ bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const { return t_Message.extract_info(msg_cs) && t_CommonMsgInfo.unpack(msg_cs, msg_info) && t_Grams.store_integer_ref(cb, std::move(transit_fee)) // fees_collected := transit_fees && t_Grams.as_integer_skip_to(msg_info.value.write(), value_grams) && - (ihr_fee = t_Grams.as_integer(std::move(msg_info.ihr_fee))).not_null() && + (ihr_fee = get_ihr_fee(msg_info, global_version)).not_null() && t_Grams.store_integer_ref(cb, value_grams + ihr_fee + fwd_fee_remaining) && cb.append_cellslice_bool( msg_info.value.write()); // value_imported = msg.value + msg.ihr_fee + fwd_fee_remaining @@ -1941,8 +1933,8 @@ bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const { const InMsg t_InMsg; -const Aug_InMsgDescr aug_InMsgDescr; -const InMsgDescr t_InMsgDescr; +const Aug_InMsgDescr aug_InMsgDescrDefault(ton::SUPPORTED_VERSION); +const InMsgDescr t_InMsgDescrDefault(ton::SUPPORTED_VERSION); bool OutMsg::skip(vm::CellSlice& cs) const { switch (get_tag(cs)) { @@ -2038,7 +2030,7 @@ bool OutMsg::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { return false; } -bool OutMsg::get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs) const { +bool OutMsg::get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs, int global_version) const { auto tag = get_tag(cs); switch (tag) { case msg_export_ext: // external outbound message carries no value @@ -2071,7 +2063,7 @@ bool OutMsg::get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs) const { td::RefInt256 value_grams, ihr_fee, fwd_fee_remaining; return t_Message.extract_info(msg_cs) && t_CommonMsgInfo.unpack(msg_cs, msg_info) && (value_grams = t_Grams.as_integer_skip(msg_info.value.write())).not_null() && - (ihr_fee = t_Grams.as_integer(std::move(msg_info.ihr_fee))).not_null() && + (ihr_fee = get_ihr_fee(msg_info, global_version)).not_null() && (fwd_fee_remaining = t_Grams.as_integer(out_msg.fwd_fee_remaining)).not_null() && t_Grams.store_integer_ref(cb, value_grams + ihr_fee + fwd_fee_remaining) && cb.append_cellslice_bool(std::move(msg_info.value)); @@ -2112,8 +2104,8 @@ bool OutMsg::get_emitted_lt(vm::CellSlice& cs, unsigned long long& emitted_lt) c const OutMsg t_OutMsg; -const Aug_OutMsgDescr aug_OutMsgDescr; -const OutMsgDescr t_OutMsgDescr; +const Aug_OutMsgDescr aug_OutMsgDescrDefault(ton::SUPPORTED_VERSION); +const OutMsgDescr t_OutMsgDescrDefault(ton::SUPPORTED_VERSION); bool EnqueuedMsg::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { return cs.advance(64) && t_Ref_MsgEnvelope.validate_skip(ops, cs, weak); diff --git a/crypto/block/block-parse.h b/crypto/block/block-parse.h index fd17c6579..2eb8c5d77 100644 --- a/crypto/block/block-parse.h +++ b/crypto/block/block-parse.h @@ -405,7 +405,7 @@ struct CommonMsgInfo final : TLB_Complex { struct CommonMsgInfo::Record_int_msg_info { bool ihr_disabled, bounce, bounced; - Ref src, dest, value, ihr_fee, fwd_fee; + Ref src, dest, value, extra_flags, fwd_fee; unsigned long long created_lt; unsigned created_at; }; @@ -604,7 +604,7 @@ extern const Aug_ShardAccounts aug_ShardAccounts; struct ShardAccounts final : TLB_Complex { HashmapAugE dict_type; - ShardAccounts() : dict_type(256, aug_ShardAccounts){}; + ShardAccounts() : dict_type(256, aug_ShardAccounts) {}; bool skip(vm::CellSlice& cs) const override { return dict_type.skip(cs); } @@ -814,7 +814,7 @@ struct InMsg final : TLB_Complex { } return (int)cs.prefetch_ulong(5) - 0b00100 + 8; } - bool get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const; + bool get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs, int global_version) const; }; extern const InMsg t_InMsg; @@ -844,7 +844,7 @@ struct OutMsg final : TLB_Complex { } return t; } - bool get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs) const; + bool get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs, int global_version) const; bool get_emitted_lt(vm::CellSlice& cs, unsigned long long& emitted_lt) const; }; @@ -853,18 +853,22 @@ extern const OutMsg t_OutMsg; // next: InMsgDescr, OutMsgDescr, OutMsgQueue, and their augmentations struct Aug_InMsgDescr final : AugmentationCheckData { - Aug_InMsgDescr() : AugmentationCheckData(t_InMsg, t_ImportFees) { + explicit Aug_InMsgDescr(int global_version) + : AugmentationCheckData(t_InMsg, t_ImportFees), global_version(global_version) { } bool eval_leaf(vm::CellBuilder& cb, vm::CellSlice& cs) const override { - return t_InMsg.get_import_fees(cb, cs); + return t_InMsg.get_import_fees(cb, cs, global_version); } + int global_version; }; -extern const Aug_InMsgDescr aug_InMsgDescr; +extern const Aug_InMsgDescr aug_InMsgDescrDefault; struct InMsgDescr final : TLB_Complex { + Aug_InMsgDescr aug; HashmapAugE dict_type; - InMsgDescr() : dict_type(256, aug_InMsgDescr){}; + explicit InMsgDescr(int global_version) : aug(global_version), dict_type(256, aug) { + } bool skip(vm::CellSlice& cs) const override { return dict_type.skip(cs); } @@ -873,21 +877,25 @@ struct InMsgDescr final : TLB_Complex { } }; -extern const InMsgDescr t_InMsgDescr; +extern const InMsgDescr t_InMsgDescrDefault; struct Aug_OutMsgDescr final : AugmentationCheckData { - Aug_OutMsgDescr() : AugmentationCheckData(t_OutMsg, t_CurrencyCollection) { + explicit Aug_OutMsgDescr(int global_version) + : AugmentationCheckData(t_OutMsg, t_CurrencyCollection), global_version(global_version) { } bool eval_leaf(vm::CellBuilder& cb, vm::CellSlice& cs) const override { - return t_OutMsg.get_export_value(cb, cs); + return t_OutMsg.get_export_value(cb, cs, global_version); } + int global_version; }; -extern const Aug_OutMsgDescr aug_OutMsgDescr; +extern const Aug_OutMsgDescr aug_OutMsgDescrDefault; struct OutMsgDescr final : TLB_Complex { + Aug_OutMsgDescr aug; HashmapAugE dict_type; - OutMsgDescr() : dict_type(256, aug_OutMsgDescr){}; + explicit OutMsgDescr(int global_version) : aug(global_version), dict_type(256, aug) { + } bool skip(vm::CellSlice& cs) const override { return dict_type.skip(cs); } @@ -896,7 +904,7 @@ struct OutMsgDescr final : TLB_Complex { } }; -extern const OutMsgDescr t_OutMsgDescr; +extern const OutMsgDescr t_OutMsgDescrDefault; struct EnqueuedMsg final : TLB_Complex { int get_size(const vm::CellSlice& cs) const override { @@ -935,7 +943,7 @@ extern const Aug_DispatchQueue aug_DispatchQueue; struct OutMsgQueue final : TLB_Complex { HashmapAugE dict_type; - OutMsgQueue() : dict_type(32 + 64 + 256, aug_OutMsgQueue){}; + OutMsgQueue() : dict_type(32 + 64 + 256, aug_OutMsgQueue) {}; bool skip(vm::CellSlice& cs) const override { return dict_type.skip(cs); } @@ -1138,8 +1146,8 @@ struct Aug_ShardFees final : AugmentationCheckData { extern const Aug_ShardFees aug_ShardFees; // Validate dict of libraries in message: used when sending and receiving message -bool validate_message_libs(const td::Ref &cell); -bool validate_message_relaxed_libs(const td::Ref &cell); +bool validate_message_libs(const td::Ref& cell); +bool validate_message_relaxed_libs(const td::Ref& cell); } // namespace tlb } // namespace block diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 1967ca023..9b4af3c1e 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -125,7 +125,7 @@ currencies$_ grams:Grams other:ExtraCurrencyCollection // int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool src:MsgAddressInt dest:MsgAddressInt - value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams + value:CurrencyCollection extra_flags:(VarUInteger 16) fwd_fee:Grams created_lt:uint64 created_at:uint32 = CommonMsgInfo; ext_in_msg_info$10 src:MsgAddressExt dest:MsgAddressInt import_fee:Grams = CommonMsgInfo; @@ -134,7 +134,7 @@ ext_out_msg_info$11 src:MsgAddressInt dest:MsgAddressExt int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool src:MsgAddress dest:MsgAddressInt - value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams + value:CurrencyCollection extra_flags:(VarUInteger 16) fwd_fee:Grams created_lt:uint64 created_at:uint32 = CommonMsgInfoRelaxed; ext_out_msg_info$11 src:MsgAddress dest:MsgAddressExt created_lt:uint64 created_at:uint32 = CommonMsgInfoRelaxed; @@ -162,6 +162,16 @@ message$_ {X:Type} info:CommonMsgInfoRelaxed _ (Message Any) = MessageAny; + +_ value:CurrencyCollection created_lt:uint64 created_at:uint32 = NewBounceOriginalInfo; +_ gas_used:uint32 vm_steps:uint32 = NewBounceComputePhaseInfo; +new_bounce_body#fffffffe + original_body:(Maybe ^Cell) + original_info:^NewBounceOriginalInfo + bounced_by_phase:uint8 exit_code:int32 + compute_phase:(Maybe NewBounceComputePhaseInfo) + = NewBounceBody; + // interm_addr_regular$0 use_dest_bits:(#<= 96) = IntermediateAddress; diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index e751cbd99..b5f13b625 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -117,7 +117,7 @@ bool Account::set_address(ton::WorkchainId wc, td::ConstBitPtr new_addr) { /** * Sets the length of anycast prefix length in the account address. * - * @param new_length The new rewrite lingth. + * @param new_length The new rewrite length. * * @returns True if the length was successfully set, False otherwise. */ @@ -872,7 +872,16 @@ bool Transaction::unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* } bounce_enabled = in_msg_info.bounce; in_msg_type = 1; - td::RefInt256 ihr_fee = block::tlb::t_Grams.as_integer(in_msg_info.ihr_fee); + td::RefInt256 ihr_fee; + if (cfg->global_version >= 12) { + ihr_fee = td::zero_refint(); + td::RefInt256 extra_flags = tlb::t_Grams.as_integer(in_msg_info.extra_flags); + new_bounce_format = extra_flags->get_bit(0); + new_bounce_format_with_body = extra_flags->get_bit(1); + } else { + // Legacy: extra_flags was previously ihr_fee + ihr_fee = tlb::t_Grams.as_integer(in_msg_info.extra_flags); + } if (ihr_delivered) { in_fwd_fee = std::move(ihr_fee); } else { @@ -1277,7 +1286,7 @@ namespace transaction { * Checks if it is required to increase gas_limit (from GasLimitsPrices config) for the transaction * * In January 2024 a highload wallet of @wallet Telegram bot in mainnet was stuck because current gas limit (1M) is - * not enough to clean up old queires, thus locking funds inside. + * not enough to clean up old queries, thus locking funds inside. * See comment in crypto/smartcont/highload-wallet-v2-code.fc for details on why this happened. * Account address: EQD_v9j1rlsuHHw2FIhcsCFFSD367ldfDdCKcsNmNpIRzUlu * It was proposed to validators to increase gas limit for this account to 70M for a limited amount @@ -1507,7 +1516,7 @@ Ref Transaction::prepare_vm_c7(const ComputePhaseConfig& cfg) const { // The only context where PrevBlocksInfo (13 parameter of c7) is null is inside emulator // where it need to be set via transaction_emulator_set_prev_blocks_info (see emulator/emulator-extern.cpp) // Inside validator, collator and liteserver checking external message contexts - // prev_blocks_info is always not null, since get_prev_blocks_info() + // prev_blocks_info is always not null, since get_prev_blocks_info() // may only return tuple or raise Error (See crypto/block/mc-config.cpp#2223) tuple.push_back(vm::StackEntry::maybe(cfg.prev_blocks_info)); } @@ -2677,8 +2686,8 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, if (cfg.disable_custom_fess) { fwd_fee = ihr_fee = td::zero_refint(); } else { - fwd_fee = block::tlb::t_Grams.as_integer(info.fwd_fee); - ihr_fee = block::tlb::t_Grams.as_integer(info.ihr_fee); + fwd_fee = tlb::t_Grams.as_integer(info.fwd_fee); + ihr_fee = cfg.global_version >= 12 ? td::zero_refint() : tlb::t_Grams.as_integer(info.extra_flags); } if (cfg.disable_ihr_flag) { info.ihr_disabled = true; @@ -2916,7 +2925,9 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, // re-pack message value CHECK(req.pack_to(info.value)); CHECK(block::tlb::t_Grams.pack_integer(info.fwd_fee, fwd_fee_remain)); - CHECK(block::tlb::t_Grams.pack_integer(info.ihr_fee, ihr_fee)); + if (cfg.global_version < 12) { + CHECK(block::tlb::t_Grams.pack_integer(info.extra_flags, ihr_fee)); + } // serialize message CHECK(tlb::csr_pack(msg.info, info)); @@ -3250,8 +3261,8 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { } bounce_phase = std::make_unique(); BouncePhase& bp = *bounce_phase; - block::gen::Message::Record msg; - block::gen::CommonMsgInfo::Record_int_msg_info info; + gen::Message::Record msg; + gen::CommonMsgInfo::Record_int_msg_info info; auto cs = vm::load_cell_slice(in_msg); if (!(tlb::unpack(cs, info) && gen::t_Maybe_Either_StateInit_Ref_StateInit.skip(cs) && cs.have(1) && cs.have_refs((int)cs.prefetch_ulong(1)))) { @@ -3261,6 +3272,45 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { if (cs.fetch_ulong(1)) { cs = vm::load_cell_slice(cs.prefetch_ref()); } + + vm::CellBuilder body; + if (new_bounce_format) { + body.store_long(0xfffffffeU, 32); // new_bounce_body#fffffffe + if (new_bounce_format_with_body) { // original_body:(Maybe ^Cell) + body.store_long(1, 1); + body.store_ref(vm::CellBuilder().append_cellslice(in_msg_body).finalize_novm()); + } else { + body.store_long(0, 1); + } + body.store_ref(vm::CellBuilder() + .append_cellslice(in_msg_info.value) // value:CurrencyCollection + .store_long(in_msg_info.created_lt, 64) // created_lt:uint64 + .store_long(in_msg_info.created_at, 32) // created_at:uint32 + .finalize_novm()); // original_info:^NewBounceOriginalInfo + if (compute_phase->skip_reason != ComputePhase::sk_none) { + body.store_long(0, 8); // bounced_by_phase:uint8 + body.store_long(compute_phase->skip_reason - 1, 32); // exit_code:int32 + } else if (!compute_phase->success) { + body.store_long(1, 8); // bounced_by_phase:uint8 + body.store_long(compute_phase->exit_code, 32); // exit_code:int32 + } else { + body.store_long(2, 8); // bounced_by_phase:uint8 + body.store_long(action_phase->result_code, 32); // exit_code:int32 + } + // compute_phase:(Maybe NewBounceComputePhaseInfo) + if (compute_phase->skip_reason != ComputePhase::sk_none) { + body.store_long(0, 1); + } else { + body.store_long(1, 1); + body.store_long(compute_phase->gas_used, 32); // gas_used:uint32 + body.store_long(compute_phase->vm_steps, 32); // vm_steps:uint32 + } + } else if (cfg.bounce_msg_body) { + int body_bits = std::min((int)cs.size(), cfg.bounce_msg_body); + body.store_long_bool(-1, 32); // 0xffffffff tag + body.append_bitslice(cs.prefetch_bits(body_bits)); // truncated message body + } + info.ihr_disabled = true; info.bounce = false; info.bounced = true; @@ -3276,7 +3326,8 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { // compute size of message vm::CellStorageStat sstat; // for message size // preliminary storage estimation of the resulting message - sstat.compute_used_storage(info.value->prefetch_ref()); + sstat.add_used_storage(info.value->prefetch_ref()); + sstat.add_used_storage(body.get_refs()); bp.msg_bits = sstat.bits; bp.msg_cells = sstat.cells; // compute forwarding fees @@ -3312,26 +3363,17 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { && cb.append_cellslice_bool(info.src) // src:MsgAddressInt && cb.append_cellslice_bool(info.dest) // dest:MsgAddressInt && msg_balance.store(cb) // value:CurrencyCollection - && block::tlb::t_Grams.store_long(cb, 0) // ihr_fee:Grams + && block::tlb::t_Grams.store_long(cb, 0) // extra_flags:(VarUInteger 16) && block::tlb::t_Grams.store_long(cb, bp.fwd_fees) // fwd_fee:Grams && cb.store_long_bool(info.created_lt, 64) // created_lt:uint64 && cb.store_long_bool(info.created_at, 32) // created_at:uint32 && cb.store_bool_bool(false)); // init:(Maybe ...) - if (cfg.bounce_msg_body) { - int body_bits = std::min((int)cs.size(), cfg.bounce_msg_body); - if (cb.remaining_bits() >= body_bits + 33u) { - CHECK(cb.store_bool_bool(false) // body:(Either X ^X) -> left X - && cb.store_long_bool(-1, 32) // int = -1 ("message type") - && cb.append_bitslice(cs.prefetch_bits(body_bits))); // truncated message body - } else { - vm::CellBuilder cb2; - CHECK(cb.store_bool_bool(true) // body:(Either X ^X) -> right ^X - && cb2.store_long_bool(-1, 32) // int = -1 ("message type") - && cb2.append_bitslice(cs.prefetch_bits(body_bits)) // truncated message body - && cb.store_builder_ref_bool(std::move(cb2))); // ^X - } + if (cb.can_extend_by(1 + body.size(), body.size_refs())) { + // body:(Either X ^X) -> left X + CHECK(cb.store_bool_bool(false) && cb.append_builder_bool(body)); } else { - CHECK(cb.store_bool_bool(false)); // body:(Either ..) + // body:(Either X ^X) -> right ^X + CHECK(cb.store_bool_bool(true) && cb.store_builder_ref_bool(std::move(body))); } CHECK(cb.finalize_to(bp.out_msg)); if (verbosity > 2) { diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index 5492ee6bf..e987645db 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -196,7 +196,7 @@ struct CreditPhase { }; struct ComputePhase { - enum { sk_none, sk_no_state, sk_bad_state, sk_no_gas, sk_suspended }; + enum { sk_none = 0, sk_no_state = 1, sk_bad_state = 2, sk_no_gas = 3, sk_suspended = 4 }; int skip_reason{sk_none}; bool success{false}; bool msg_state_used{false}; @@ -358,6 +358,8 @@ struct Transaction { bool bounce_enabled{false}; bool in_msg_extern{false}; gen::CommonMsgInfo::Record_int_msg_info in_msg_info; + bool new_bounce_format{false}; + bool new_bounce_format_with_body{false}; bool use_msg_state{false}; bool is_first{false}; bool orig_addr_rewrite_set{false}; diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp index 265c602d7..a89cfa6d9 100644 --- a/crypto/smc-envelope/SmartContract.cpp +++ b/crypto/smc-envelope/SmartContract.cpp @@ -61,9 +61,9 @@ td::Ref build_internal_message(td::RefInt256 amount, td::Refbit_size(false) + 7) >> 3); b.store_long_bool(len, 4) && b.store_int256_bool(*amount, len * 8, false); // grams:Grams - b.store_zeroes(1 + 4 + 4 + 64 + 32 + 1); // extre, ihr_fee, fwd_fee, created_lt, created_at, init + b.store_zeroes(1 + 4 + 4 + 64 + 32 + 1); // extra currencies, extra_flags, fwd_fee, created_lt, created_at, init // body:(Either X ^X) - if (b.remaining_bits() >= 1 + (*body).size() && b.remaining_refs() >= (*body).size_refs()) { + if (b.remaining_bits() >= 1 + body->size() && b.remaining_refs() >= body->size_refs()) { b.store_zeroes(1); b.append_cellslice(body); } else { diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index 837624563..93fc0a89d 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -1156,6 +1156,16 @@ td::Result CellStorageStat::add_used_storage(Ref CellStorageStat::add_used_storage(td::Span> cells, bool kill_dup, + unsigned skip_count_root) { + CellInfo result; + for (const auto& cell : cells) { + TRY_RESULT(info, add_used_storage(cell, kill_dup, skip_count_root)); + result.max_merkle_depth = std::max(result.max_merkle_depth, info.max_merkle_depth); + } + return result; +} + void NewCellStorageStat::add_cell(Ref cell) { dfs(std::move(cell), true, false); } diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index fbf395538..5bf45c922 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -143,6 +143,8 @@ struct CellStorageStat { td::Result add_used_storage(const CellSlice& cs, bool kill_dup = true, unsigned skip_count_root = 0); td::Result add_used_storage(CellSlice&& cs, bool kill_dup = true, unsigned skip_count_root = 0); td::Result add_used_storage(Ref cell, bool kill_dup = true, unsigned skip_count_root = 0); + td::Result add_used_storage(td::Span> cells, bool kill_dup = true, + unsigned skip_count_root = 0); unsigned long long limit_cells = std::numeric_limits::max(); unsigned long long limit_bits = std::numeric_limits::max(); diff --git a/crypto/vm/cells/CellBuilder.h b/crypto/vm/cells/CellBuilder.h index 954a1ac08..4c701c7d2 100644 --- a/crypto/vm/cells/CellBuilder.h +++ b/crypto/vm/cells/CellBuilder.h @@ -85,6 +85,9 @@ class CellBuilder : public td::CntObject { Ref get_ref(unsigned idx) const { return idx < refs_cnt ? refs[idx] : Ref{}; } + td::Span> get_refs() const { + return {refs.data(), refs_cnt}; + } void reset(); bool reset_bool() { reset(); diff --git a/crypto/vm/tonops.cpp b/crypto/vm/tonops.cpp index 981f7c5b3..d74ecb17f 100644 --- a/crypto/vm/tonops.cpp +++ b/crypto/vm/tonops.cpp @@ -1826,6 +1826,7 @@ int exec_send_message(VmState* st) { Ref dest; td::RefInt256 value; td::RefInt256 user_fwd_fee, user_ihr_fee; + unsigned extra_flags_len = 0; bool have_extra_currencies = false; bool ext_msg = msg.info->prefetch_ulong(1); if (ext_msg) { // External message @@ -1843,17 +1844,23 @@ int exec_send_message(VmState* st) { } ihr_disabled = info.ihr_disabled || st->get_global_version() >= 11; dest = std::move(info.dest); - Ref extra; + Ref extra; if (!block::tlb::t_CurrencyCollection.unpack_special(info.value.write(), value, extra)) { throw VmError{Excno::unknown, "invalid message"}; } have_extra_currencies = !extra.is_null(); user_fwd_fee = block::tlb::t_Grams.as_integer(info.fwd_fee); - user_ihr_fee = block::tlb::t_Grams.as_integer(info.ihr_fee); + if (st->get_global_version() >= 12) { + user_ihr_fee = td::zero_refint(); + extra_flags_len = info.extra_flags->size(); + } else { + // Legacy: extra_flags was previously ihr_fee + user_ihr_fee = block::tlb::t_Grams.as_integer(info.extra_flags); + } } bool is_masterchain = parse_addr_workchain(*my_addr) == -1 || (!ext_msg && parse_addr_workchain(*dest) == -1); - td::Ref prices_cs; + Ref prices_cs; if (st->get_global_version() >= 6) { prices_cs = tuple_index(get_unpacked_config_tuple(st), is_masterchain ? 4 : 5).as_slice(); } else { @@ -1886,7 +1893,7 @@ int exec_send_message(VmState* st) { } else { max_cells = 1 << 13; } - vm::VmStorageStat stat(max_cells); + VmStorageStat stat(max_cells); CellSlice cs = load_cell_slice(msg_cell); cs.skip_first(cs.size()); if (st->get_global_version() >= 10 && have_extra_currencies) { @@ -1967,7 +1974,8 @@ int exec_send_message(VmState* st) { bits = 4 + my_addr->size() + dest->size() + stored_grams_len(value) + 1 + 32 + 64; td::RefInt256 fwd_fee_first = (fwd_fee * prices.first_frac) >> 16; bits += stored_grams_len(fwd_fee - fwd_fee_first); - bits += stored_grams_len(ihr_fee); + // Legacy: extra_flags was previously ihr_fee + bits += st->get_global_version() >= 12 ? extra_flags_len : stored_grams_len(ihr_fee); } // init bits++; diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index 6281735e0..be0fe380b 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -231,12 +231,45 @@ This is required to help computing storage stats in the future, after collator-v ## Version 12 +### Extra message flags and new bounce format +Field `ihr_fee:Grams` in internal message is now called `extra_flags:(VarUInteger 16)` (it's the same format). +This field does not represent fees. `ihr_fee` is always zero since version 11, so this field was essentially unused. + +`(extra_flags & 1) = 1` enables the new bounce format for the message. The bounced message contains information about the transaction. +If `(extra_flags & 3) = 3`, the bounced message also contains the whole body of the original message. +When the message with new bounce flag is bounced, the bounced message body has the following format (`new_bounce_body`): +``` +_ value:CurrencyCollection created_lt:uint64 created_at:uint32 = NewBounceOriginalInfo; +_ gas_used:uint32 vm_steps:uint32 = NewBounceComputePhaseInfo; + +new_bounce_body#fffffffe + original_body:(Maybe ^Cell) + original_info:^NewBounceOriginalInfo + bounced_by_phase:uint8 exit_code:int32 + compute_phase:(Maybe NewBounceComputePhaseInfo) + = NewBounceBody; +``` +- `original_body` - cell that contains the body of the original message (if `extra_flags & 2`) or nothing (if not `extra_flags & 2`). +- `original_info` - value, lt and unixtime of the original message. +- `bounced_by_phase`: + - `0` - compute phase was skipped. `exit_code` denotes the skip reason: + - `exit_code = 0` - no state (account is uninit or frozen, and no state init is present in the message). + - `exit_code = 1` - bad state (account is uninit or frozen, and state init in the message has the wrong hash). + - `exit_code = 2` - no gas. + - `exit_code = 3` - account is suspended. + - `1` - compute phase failed. `exit_code` is the value from the compute phase. + - `2` - action phase failed. `exit_code` is the value from the action phase. +- `exit_code` - 32-bit exit code, see above. +- `compute_phase` - exists if it was not skipped (`bounced_by_phase > 0`): + - `gas_used`, `vm_steps` - same as in `TrComputePhase` of the transaction. + ### New TVM instructions - `BTOS` (`b - s`) - same as `ENDC CTOS`, but without gas cost for cell creation and loading. Gas cost: `26`. - `HASHBU` (`b - hash`) - same as `ENDC HASHCU`, but without gas cost for cell creation. Gas cost: `26`. ### Other TVM changes - `HASHSU` (`s - hash`) now does not spend gas for cell creation. Gas cost: `26`. +- `SENDMSG` instruction treats `extra_flags` field accordingly (see above). ### Other changes - Account size in masterchain is now limited to `2048` cells. This can be configured in size limits config (`ConfigParam 43`). diff --git a/emulator/test/emulator-tests.cpp b/emulator/test/emulator-tests.cpp index ae273ddfd..4057f5816 100644 --- a/emulator/test/emulator-tests.cpp +++ b/emulator/test/emulator-tests.cpp @@ -221,7 +221,7 @@ TEST(Emulator, wallet_int_and_ext_msg) { { vm::CellBuilder cb; block::tlb::t_Grams.store_integer_value(cb, td::BigInt256(0)); - msg_info.ihr_fee = cb.as_cellslice_ref(); + msg_info.extra_flags = cb.as_cellslice_ref(); } msg_info.created_lt = 0; msg_info.created_at = static_cast(utime); diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 7e868480b..f95ac33bf 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -3225,15 +3225,12 @@ struct ToRawTransactions { TRY_RESULT(src, to_std_address(msg_info.src)); TRY_RESULT(dest, to_std_address(msg_info.dest)); TRY_RESULT(fwd_fee, to_balance(msg_info.fwd_fee)); - TRY_RESULT(ihr_fee, to_balance(msg_info.ihr_fee)); auto created_lt = static_cast(msg_info.created_lt); return tonlib_api::make_object( - msg_hash, - tonlib_api::make_object(src), - tonlib_api::make_object(std::move(dest)), balance, - std::move(extra_currencies), fwd_fee, ihr_fee, created_lt, std::move(body_hash), - get_data(src)); + msg_hash, tonlib_api::make_object(src), + tonlib_api::make_object(std::move(dest)), balance, std::move(extra_currencies), + fwd_fee, /* ihr_fee = */ 0, created_lt, std::move(body_hash), get_data(src)); } case block::gen::CommonMsgInfo::ext_in_msg_info: { block::gen::CommonMsgInfo::Record_ext_in_msg_info msg_info; diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index fcc95caad..47c627fd7 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -17,6 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "block-parse.h" #include "interfaces/validator-manager.h" #include "shard.hpp" #include "top-shard-descr.hpp" @@ -156,6 +157,7 @@ class Collator final : public td::actor::Actor { ton::LogicalTime shards_max_end_lt_{0}; ton::UnixTime prev_state_utime_; int global_id_{0}; + int global_version_{0}; ton::BlockSeqno min_ref_mc_seqno_{~0U}; ton::BlockSeqno vert_seqno_{~0U}, prev_vert_seqno_{~0U}; ton::BlockIdExt prev_key_block_; @@ -203,6 +205,8 @@ class Collator final : public td::actor::Actor { std::vector ext_msg_list_; std::priority_queue, std::greater> new_msgs; std::pair last_proc_int_msg_, first_unproc_int_msg_; + block::tlb::Aug_InMsgDescr aug_InMsgDescr{0}; + block::tlb::Aug_OutMsgDescr aug_OutMsgDescr{0}; std::unique_ptr in_msg_dict, out_msg_dict, old_out_msg_queue_, out_msg_queue_, sibling_out_msg_queue_; std::map unprocessed_deferred_messages_; // number of messages from dispatch queue in new_msgs diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index acf9075b4..401f721d1 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -778,6 +778,7 @@ bool Collator::unpack_last_mc_state() { CHECK(config_); config_->set_block_id_ext(mc_block_id_); global_id_ = config_->get_global_blockchain_id(); + global_version_ = config_->get_global_version(); ihr_enabled_ = config_->ihr_enabled(); create_stats_enabled_ = config_->create_stats_enabled(); report_version_ = config_->has_capability(ton::capReportVersion); @@ -2292,8 +2293,9 @@ bool Collator::do_collate() { return fatal_error("cannot fetch required configuration parameters from masterchain state"); } LOG(DEBUG) << "config parameters fetched, creating message dictionaries"; - in_msg_dict = std::make_unique(256, block::tlb::aug_InMsgDescr); - out_msg_dict = std::make_unique(256, block::tlb::aug_OutMsgDescr); + aug_InMsgDescr.global_version = aug_OutMsgDescr.global_version = global_version_; + in_msg_dict = std::make_unique(256, aug_InMsgDescr); + out_msg_dict = std::make_unique(256, aug_OutMsgDescr); LOG(DEBUG) << "message dictionaries created"; if (max_lt == start_lt) { ++max_lt; @@ -3118,7 +3120,7 @@ bool Collator::create_special_transaction(block::CurrencyCollection amount, Ref< && cb.store_long_bool(0x4ff, 11) // addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 && cb.store_bits_bool(addr) // address:bits256 => dest:MsgAddressInt && amount.store(cb) // value:CurrencyCollection - && cb.store_zeroes_bool(4 + 4) // ihr_fee:Grams fwd_fee:Grams + && cb.store_zeroes_bool(4 + 4) // extra_flags:(VarUInteger 16) fwd_fee:Grams && cb.store_long_bool(lt, 64) // created_lt:uint64 && cb.store_long_bool(now_, 32) // created_at:uint32 && cb.store_zeroes_bool(2) // init:(Maybe ...) body:(Either X ^X) = Message X diff --git a/validator/impl/ihr-message.cpp b/validator/impl/ihr-message.cpp index 4327b5dd3..75cf08bc7 100644 --- a/validator/impl/ihr-message.cpp +++ b/validator/impl/ihr-message.cpp @@ -99,7 +99,7 @@ td::Result> IhrMessageQ::create_ihr_message(td::BufferSlice dat "block header in the Merkle proof of an IHR message does not belong to the declared source block"); } vm::AugmentedDictionary out_msg_dict{vm::load_cell_slice_ref(extra.out_msg_descr), 256, - block::tlb::aug_OutMsgDescr}; + block::tlb::aug_OutMsgDescrDefault}; Bits256 key{ihr_msg->get_hash().bits()}; auto descr = out_msg_dict.lookup(key); out_msg_dict.reset(); diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index e9592f626..c369d1f53 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -2390,7 +2390,8 @@ void LiteQuery::perform_listBlockTransactions(BlockIdExt blkid, int mode, int co static td::Result> get_in_msg_metadata( const Ref& in_msg_descr_root, const Ref& trans_root) { - vm::AugmentedDictionary in_msg_descr{vm::load_cell_slice_ref(in_msg_descr_root), 256, block::tlb::aug_InMsgDescr}; + vm::AugmentedDictionary in_msg_descr{vm::load_cell_slice_ref(in_msg_descr_root), 256, + block::tlb::aug_InMsgDescrDefault}; block::gen::Transaction::Record transaction; if (!block::tlb::unpack_cell(trans_root, transaction)) { return td::Status::Error("invalid Transaction in block"); @@ -2549,7 +2550,8 @@ void LiteQuery::perform_listBlockTransactionsExt(BlockIdExt blkid, int mode, int static td::Status process_all_in_msg_metadata(const Ref& in_msg_descr_root, const std::vector>& trans_roots) { - vm::AugmentedDictionary in_msg_descr{vm::load_cell_slice_ref(in_msg_descr_root), 256, block::tlb::aug_InMsgDescr}; + vm::AugmentedDictionary in_msg_descr{vm::load_cell_slice_ref(in_msg_descr_root), 256, + block::tlb::aug_InMsgDescrDefault}; for (const Ref& trans_root : trans_roots) { block::gen::Transaction::Record transaction; if (!block::tlb::unpack_cell(trans_root, transaction)) { diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 4bef90395..fd822d2bc 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -943,6 +943,7 @@ bool ValidateQuery::try_unpack_mc_state() { return reject_query(PSTRING() << "vertical seqno mismatch: new block has " << vert_seqno_ << " while the masterchain configuration expects " << config_->get_vert_seqno()); } + global_version_ = config_->get_global_version(); prev_key_block_exists_ = config_->get_last_key_block(prev_key_block_, prev_key_block_lt_); if (prev_key_block_exists_) { prev_key_block_seqno_ = prev_key_block_.seqno(); @@ -2704,17 +2705,19 @@ bool ValidateQuery::unpack_block_data() { auto outmsg_cs = vm::load_cell_slice_ref(std::move(extra.out_msg_descr)); // run some hand-written checks from block::tlb:: // (automatic tests from block::gen:: have been already run for the entire block) - if (!block::tlb::t_InMsgDescr.validate_upto(10000000, *inmsg_cs)) { + t_InMsgDescr.aug.global_version = global_version_; + t_OutMsgDescr.aug.global_version = global_version_; + if (!t_InMsgDescr.validate_upto(10000000, *inmsg_cs)) { return reject_query("InMsgDescr of the new block failed to pass handwritten validity tests"); } - if (!block::tlb::t_OutMsgDescr.validate_upto(10000000, *outmsg_cs)) { + if (!t_OutMsgDescr.validate_upto(10000000, *outmsg_cs)) { return reject_query("OutMsgDescr of the new block failed to pass handwritten validity tests"); } if (!block::tlb::t_ShardAccountBlocks.validate_ref(10000000, extra.account_blocks)) { return reject_query("ShardAccountBlocks of the new block failed to pass handwritten validity tests"); } - in_msg_dict_ = std::make_unique(std::move(inmsg_cs), 256, block::tlb::aug_InMsgDescr); - out_msg_dict_ = std::make_unique(std::move(outmsg_cs), 256, block::tlb::aug_OutMsgDescr); + in_msg_dict_ = std::make_unique(std::move(inmsg_cs), 256, t_InMsgDescr.aug); + out_msg_dict_ = std::make_unique(std::move(outmsg_cs), 256, t_OutMsgDescr.aug); account_blocks_dict_ = std::make_unique( vm::load_cell_slice_ref(std::move(extra.account_blocks)), 256, block::tlb::aug_ShardAccountBlocks); LOG(DEBUG) << "validating InMsgDescr"; @@ -3773,7 +3776,7 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) block::tlb::MsgEnvelope::Record_std env; // int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool // src:MsgAddressInt dest:MsgAddressInt - // value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams + // value:CurrencyCollection extra_flags:(VarUInteger 16) fwd_fee:Grams // created_lt:uint64 created_at:uint32 = CommonMsgInfo; block::gen::CommonMsgInfo::Record_int_msg_info info; ton::AccountIdPrefixFull src_prefix, dest_prefix, cur_prefix, next_prefix; @@ -4335,7 +4338,7 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms block::tlb::MsgEnvelope::Record_std env; // int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool // src:MsgAddressInt dest:MsgAddressInt - // value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams + // value:CurrencyCollection extra_flags:(VarUInteger 16) fwd_fee:Grams // created_lt:uint64 created_at:uint32 = CommonMsgInfo; block::gen::CommonMsgInfo::Record_int_msg_info info; ton::AccountIdPrefixFull src_prefix, dest_prefix, cur_prefix, next_prefix; @@ -5377,6 +5380,22 @@ std::unique_ptr ValidateQuery::unpack_account(td::ConstBitPtr ad return new_acc; } +/** + * Gets IHR fee of the internal message + * + * In earlier versions (before 12) the field extra_flags was ihr_fee. + * Since version 12 ihr_fee is always zero. + * + * @param info CommonMsgInfo of the internal message + * @param global_version global version from ConfigParam 8 + * + * @returns IHR fee + */ +static td::RefInt256 get_ihr_fee(const block::gen::CommonMsgInfo::Record_int_msg_info &info, int global_version) { + // Legacy: extra_flags was previously ihr_fee + return global_version >= 12 ? td::zero_refint() : block::tlb::t_Grams.as_integer(std::move(info.extra_flags)); +} + /** * Checks the validity of a single transaction for a given account. * Performs transaction execution. @@ -5464,7 +5483,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT CHECK(money_imported.validate_unpack(info.value)); ihr_delivered = (in_msg_tag == block::gen::InMsg::msg_import_ihr); if (!ihr_delivered) { - money_imported += block::tlb::t_Grams.as_integer(info.ihr_fee); + money_imported += get_ihr_fee(info, global_version_); } CHECK(money_imported.is_valid()); } @@ -5533,7 +5552,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT // unpack exported message value (from this transaction) block::CurrencyCollection msg_export_value; CHECK(msg_export_value.unpack(info.value)); - msg_export_value += block::tlb::t_Grams.as_integer(info.ihr_fee); + msg_export_value += get_ihr_fee(info, global_version_); msg_export_value += msg_env.fwd_fee_remaining; CHECK(msg_export_value.is_valid()); money_exported += msg_export_value; @@ -6145,9 +6164,12 @@ bool ValidateQuery::check_special_message(Ref in_msg_root, const block if (block::tlb::t_Grams.as_integer(info.fwd_fee)->sgn()) { return reject_query("special message with hash "s + msg_hash.to_hex() + " has a non-zero fwd_fee"); } - if (block::tlb::t_Grams.as_integer(info.ihr_fee)->sgn()) { + if (get_ihr_fee(info, global_version_)->sgn()) { return reject_query("special message with hash "s + msg_hash.to_hex() + " has a non-zero ihr_fee"); } + if (block::tlb::t_Grams.as_integer(info.extra_flags)->sgn()) { + return reject_query("special message with hash "s + msg_hash.to_hex() + " has a non-zero extra_flags"); + } block::CurrencyCollection value; if (!value.validate_unpack(info.value)) { return reject_query("special message with hash "s + msg_hash.to_hex() + " has an invalid value"); diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 95e81cdec..f172de95d 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -18,6 +18,7 @@ */ #pragma once +#include "block-parse.h" #include "interfaces/validator-manager.h" #include "vm/cells.h" #include "vm/dict.h" @@ -190,6 +191,7 @@ class ValidateQuery : public td::actor::Actor { ton::LogicalTime max_shard_lt_{0}; int global_id_{0}; + int global_version_{0}; ton::BlockSeqno vert_seqno_{~0U}; bool ihr_enabled_{false}; bool create_stats_enabled_{false}; @@ -222,6 +224,8 @@ class ValidateQuery : public td::actor::Actor { std::map block_create_count_; unsigned block_create_total_{0}; + block::tlb::InMsgDescr t_InMsgDescr{0}; + block::tlb::OutMsgDescr t_OutMsgDescr{0}; std::unique_ptr in_msg_dict_, out_msg_dict_, account_blocks_dict_; block::ValueFlow value_flow_; block::CurrencyCollection import_created_, transaction_fees_, total_burned_{0}, fees_burned_{0}; From 8a971be7fee91636bb36d816b7ac0c4e14d57ef7 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Fri, 25 Jul 2025 12:35:28 +0300 Subject: [PATCH 354/388] Update changelog --- Changelog.md | 4 ++++ recent_changelog.md | 12 ++---------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/Changelog.md b/Changelog.md index d2f940c00..b104d10ad 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,7 @@ +## 2025.07 Accelerator Update + +Separation of validation and collation processes that allows to host them on independent machines and achieve full horizontal scaling. [More details in documentation](https://docs.ton.org/v3/documentation/infra/nodes/validation/collators) + ## 2025.06 Update 1. ADNL and candidate broadcast optimization diff --git a/recent_changelog.md b/recent_changelog.md index a178f38fa..6b65d2183 100644 --- a/recent_changelog.md +++ b/recent_changelog.md @@ -1,11 +1,3 @@ -## 2025.06 Update - -1. ADNL and candidate broadcast optimization -2. [TVM version v11](./doc/GlobalVersions.md): new opcodes, and `c7` entry to improve developer experience. It also activates storage stats and `ihr_fee` nullification. -3. Fixed `start_lt` of tick transactions [see details on 01.06.2025 incident](https://telegra.ph/Report-on-June-1-2025-Operation-Incident-06-02). -4. Introduction of persistent state sharding, as well as making serialization of large BOCs more deterministic -5. Emulator improvements: in get methods, set config from provided `c7`; allow retrieval of logs from emulator runs for get methods -6. Optimized package import for archive nodes - -Besides the work of the core team, this update is based on the efforts of the RSquad team (deterministic large BOC serialization); AArayz, wy666444, Robinlzw, Lucian-code233 from TonBit (early discovery of the TVM 11 bug); @Skydev0h (uninitialized `BLOCKLT` in get methods); and @yma-het from TONWhales (emulator improvements). +## 2025.07 Accelerator Update +Separation of validation and collation processes that allows to host them on independent machines and achieve full horizontal scaling. [More details in documentation](https://docs.ton.org/v3/documentation/infra/nodes/validation/collators) From 273663701cb202dc7fbed5fc437f26947919c26f Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 25 Jul 2025 14:16:50 +0300 Subject: [PATCH 355/388] Change exit codes for compute phase skip reason in bounce msg --- crypto/block/transaction.cpp | 4 ++-- doc/GlobalVersions.md | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index b5f13b625..8d220c7b9 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -3288,8 +3288,8 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { .store_long(in_msg_info.created_at, 32) // created_at:uint32 .finalize_novm()); // original_info:^NewBounceOriginalInfo if (compute_phase->skip_reason != ComputePhase::sk_none) { - body.store_long(0, 8); // bounced_by_phase:uint8 - body.store_long(compute_phase->skip_reason - 1, 32); // exit_code:int32 + body.store_long(0, 8); // bounced_by_phase:uint8 + body.store_long(-compute_phase->skip_reason, 32); // exit_code:int32 } else if (!compute_phase->success) { body.store_long(1, 8); // bounced_by_phase:uint8 body.store_long(compute_phase->exit_code, 32); // exit_code:int32 diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index be0fe380b..6e59955c4 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -253,10 +253,10 @@ new_bounce_body#fffffffe - `original_info` - value, lt and unixtime of the original message. - `bounced_by_phase`: - `0` - compute phase was skipped. `exit_code` denotes the skip reason: - - `exit_code = 0` - no state (account is uninit or frozen, and no state init is present in the message). - - `exit_code = 1` - bad state (account is uninit or frozen, and state init in the message has the wrong hash). - - `exit_code = 2` - no gas. - - `exit_code = 3` - account is suspended. + - `exit_code = -1` - no state (account is uninit or frozen, and no state init is present in the message). + - `exit_code = -2` - bad state (account is uninit or frozen, and state init in the message has the wrong hash). + - `exit_code = -3` - no gas. + - `exit_code = -4` - account is suspended. - `1` - compute phase failed. `exit_code` is the value from the compute phase. - `2` - action phase failed. `exit_code` is the value from the action phase. - `exit_code` - 32-bit exit code, see above. From 8c7a5a67dbc689f4c5382e952104544b33b4df24 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 31 Jul 2025 10:00:49 +0300 Subject: [PATCH 356/388] "Skip public msg send" flag for custom overlays (#1757) --- tl/generate/scheme/ton_api.tl | 2 +- tl/generate/scheme/ton_api.tlo | Bin 116068 -> 116116 bytes .../validator-engine-console-query.cpp | 3 +++ validator/full-node.cpp | 20 ++++++++++++------ validator/full-node.h | 1 + 5 files changed, 19 insertions(+), 7 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 7db6ac0e7..ebfd25def 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -691,7 +691,7 @@ engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector eng engine.validator.customOverlayNode adnl_id:int256 msg_sender:Bool msg_sender_priority:int block_sender:Bool = engine.validator.CustomOverlayNode; engine.validator.customOverlay name:string nodes:(vector engine.validator.customOverlayNode) sender_shards:(vector tonNode.shardId) - = engine.validator.CustomOverlay; + skip_public_msg_send:Bool = engine.validator.CustomOverlay; engine.validator.customOverlaysConfig overlays:(vector engine.validator.customOverlay) = engine.validator.CustomOverlaysConfig; engine.validator.collatorOptions diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index b4d84dd96804cee1ac61c758da3f1a1307cfb32f..8c13ffcb66d6296e175a89e22c09305c8f8267c7 100644 GIT binary patch delta 123 zcmaFT#XhB*eS?4m>*A7I-{mF;O2}e-cZ0L+>` Op^8yqyG%8shZg{@!ZjrT delta 92 zcmbQz&Hki|eS?4m>)D!=C32GkCFC|6OPDTT0dXhi>`(wRrhi~%)R}%jpHXG1| hdmeItg*T_{3}J+@_Uy_z08x6Nicw*^Pc@^57XScQC$Rtk diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index eaece0e87..15f126422 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1294,6 +1294,9 @@ td::Status ShowCustomOverlaysQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << " " << ton::create_shard_id(shard).to_str() << "\n"; } } + if (overlay->skip_public_msg_send_) { + td::TerminalIO::out() << "Don't send external messages to public overlays\n"; + } td::TerminalIO::out() << "\n"; } return td::Status::OK(); diff --git a/validator/full-node.cpp b/validator/full-node.cpp index df9123761..4e75ddede 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -318,21 +318,28 @@ void FullNodeImpl::send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice dat } void FullNodeImpl::send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) { - auto shard = get_shard(dst); - if (shard.empty()) { - VLOG(FULL_NODE_WARNING) << "dropping OUT ext message to unknown shard"; - return; - } + bool skip_public = false; for (auto &[_, private_overlay] : custom_overlays_) { if (private_overlay.params_.send_shard(dst.as_leaf_shard())) { for (auto &[local_id, actor] : private_overlay.actors_) { if (private_overlay.params_.msg_senders_.contains(local_id)) { td::actor::send_closure(actor, &FullNodeCustomOverlay::send_external_message, data.clone()); + if (private_overlay.params_.skip_public_msg_send_) { + skip_public = true; + } } } } } - td::actor::send_closure(shard, &FullNodeShard::send_external_message, std::move(data)); + + if (!skip_public) { + auto shard = get_shard(dst); + if (shard.empty()) { + VLOG(FULL_NODE_WARNING) << "dropping OUT ext message to unknown shard"; + return; + } + td::actor::send_closure(shard, &FullNodeShard::send_external_message, std::move(data)); + } } void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { @@ -964,6 +971,7 @@ CustomOverlayParams CustomOverlayParams::fetch(const ton_api::engine_validator_c for (const auto &shard : f.sender_shards_) { c.sender_shards_.push_back(create_shard_id(shard)); } + c.skip_public_msg_send_ = f.skip_public_msg_send_; return c; } diff --git a/validator/full-node.h b/validator/full-node.h index dc29f67ca..4c11c1fbd 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -67,6 +67,7 @@ struct CustomOverlayParams { std::map msg_senders_; std::set block_senders_; std::vector sender_shards_; + bool skip_public_msg_send_ = false; bool send_shard(const ShardIdFull& shard) const; static CustomOverlayParams fetch(const ton_api::engine_validator_customOverlay& f); From 7d1995ec0a03bac280bdf63c91f005696d3ab816 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Thu, 31 Jul 2025 10:01:13 +0300 Subject: [PATCH 357/388] Improve portable linux binaries (#1759) * add libgslcblas into portable binaries * remove commented line * fix portable linux binaries --- .github/workflows/build-ton-linux-arm64-appimage.yml | 3 +-- .github/workflows/build-ton-linux-x86-64-appimage.yml | 2 +- assembly/appimage/create-appimages.sh | 3 ++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-ton-linux-arm64-appimage.yml b/.github/workflows/build-ton-linux-arm64-appimage.yml index a0ba4a022..c8147677e 100644 --- a/.github/workflows/build-ton-linux-arm64-appimage.yml +++ b/.github/workflows/build-ton-linux-arm64-appimage.yml @@ -21,8 +21,7 @@ jobs: - name: Install system libraries run: | sudo apt update - sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache - sudo apt remove libgsl-dev + sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache libgsl-dev libblas-dev libgslcblas0 mkdir ~/.ccache 3pp - name: Install clang-16 diff --git a/.github/workflows/build-ton-linux-x86-64-appimage.yml b/.github/workflows/build-ton-linux-x86-64-appimage.yml index 7ad873d22..cb02b7bc2 100644 --- a/.github/workflows/build-ton-linux-x86-64-appimage.yml +++ b/.github/workflows/build-ton-linux-x86-64-appimage.yml @@ -21,7 +21,7 @@ jobs: - name: Install system libraries run: | sudo apt update - sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache libgsl-dev libblas-dev + sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache libgsl-dev libblas-dev libgslcblas0 mkdir ~/.ccache 3pp - name: Install clang-16 diff --git a/assembly/appimage/create-appimages.sh b/assembly/appimage/create-appimages.sh index 7bd68b6fb..f6e269072 100644 --- a/assembly/appimage/create-appimages.sh +++ b/assembly/appimage/create-appimages.sh @@ -27,7 +27,7 @@ for file in ../artifacts/*; do printf '[Desktop Entry]\nName='$appName'\nExec='$appName'\nIcon='$appName'\nType=Application\nCategories=Utility;\n' > $appName.AppDir/$appName.desktop cp ../ton.png $appName.AppDir/$appName.png cp $file $appName.AppDir/usr/bin/ - cp ../build/openssl_3/libcrypto.so.3 \ + cp ../openssl_3/libcrypto.so.3 \ /lib/$ARCH-linux-gnu/libatomic.so.1 \ /lib/$ARCH-linux-gnu/libsodium.so.23 \ /lib/$ARCH-linux-gnu/libz.so.1 \ @@ -37,6 +37,7 @@ for file in ../artifacts/*; do /lib/$ARCH-linux-gnu/libstdc++.so.6 \ /lib/$ARCH-linux-gnu/libgsl.so.27 \ /lib/$ARCH-linux-gnu/libblas.so.3 \ + /lib/$ARCH-linux-gnu/libgslcblas.so.0 \ $appName.AppDir/usr/lib/ chmod +x ./$appName.AppDir/usr/bin/$appName From 3d49e8bb57bb6fd7400e8eab213426522d658e9e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 31 Jul 2025 10:01:52 +0300 Subject: [PATCH 358/388] Fix selecting shard overlays after decreasing monitor_min_split (#1756) When node is out of sync (e.g., during initial sync), it may try to use shard overlays that do not exist anymore because monitor min split was decreased. "Download state", "download archive" and "download proof link" are the types of requests that are necessary to sync. --- validator/full-node.cpp | 16 ++++++++++------ validator/full-node.hpp | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 4e75ddede..890fd7a2e 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -444,7 +444,7 @@ void FullNodeImpl::download_zero_state(BlockIdExt id, td::uint32 priority, td::T void FullNodeImpl::download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, td::uint32 priority, td::Timestamp timeout, td::Promise promise) { - auto shard = get_shard(id.shard_full()); + auto shard = get_shard(id.shard_full(), /* historical = */ true); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping download state diff query to unknown shard"; promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); @@ -467,7 +467,7 @@ void FullNodeImpl::download_block_proof(BlockIdExt block_id, td::uint32 priority void FullNodeImpl::download_block_proof_link(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) { - auto shard = get_shard(block_id.shard_full()); + auto shard = get_shard(block_id.shard_full(), /* historical = */ true); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping download proof link query to unknown shard"; promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); @@ -490,7 +490,7 @@ void FullNodeImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeou void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) { - auto shard = get_shard(shard_prefix); + auto shard = get_shard(shard_prefix, /* historical = */ true); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping download archive query to unknown shard"; promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); @@ -519,7 +519,7 @@ void FullNodeImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std::vect timeout, std::move(promise)); } -td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { +td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard, bool historical) { if (shard.is_masterchain()) { return shards_[ShardIdFull{masterchainId}].actor.get(); } @@ -527,8 +527,12 @@ td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { return {}; } int pfx_len = shard.pfx_len(); - if (pfx_len > wc_monitor_min_split_) { - shard = shard_prefix(shard, wc_monitor_min_split_); + int min_split = wc_monitor_min_split_; + if (historical) { + min_split = td::Random::fast(0, min_split); + } + if (pfx_len > min_split) { + shard = shard_prefix(shard, min_split); } while (true) { auto it = shards_.find(shard); diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 9a70d5c0c..dfd12f054 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -131,7 +131,7 @@ class FullNodeImpl : public FullNode { FileHash zero_state_file_hash_; td::actor::ActorId get_shard(AccountIdPrefixFull dst); - td::actor::ActorId get_shard(ShardIdFull shard); + td::actor::ActorId get_shard(ShardIdFull shard, bool historical = false); std::map shards_; int wc_monitor_min_split_ = 0; From 4ec6711be40fad6a3dada187d7d87d34dc407f6a Mon Sep 17 00:00:00 2001 From: neodix42 Date: Thu, 31 Jul 2025 10:02:17 +0300 Subject: [PATCH 359/388] Bind tolk-stdlib to tolk release... (#1752) * add talk-stdlib to tolk release * add talk-stdlib to tolk release --- .github/workflows/create-tolk-release.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/create-tolk-release.yml b/.github/workflows/create-tolk-release.yml index fb8438a12..4cd30ef6f 100644 --- a/.github/workflows/create-tolk-release.yml +++ b/.github/workflows/create-tolk-release.yml @@ -9,6 +9,9 @@ on: permissions: write-all +env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + jobs: create-release: runs-on: ubuntu-22.04 @@ -152,3 +155,11 @@ jobs: asset_name: ton-wasm.zip tag: ${{ inputs.tag }} + - name: Upload stdlib + run: | + mkdir smartcont_lib + cd smartcont_lib + cp -r ../artifacts/ton-x86_64-linux/{smartcont,lib} . + zip -r smartcont_lib.zip . + gh release upload ${{ inputs.tag }} smartcont_lib.zip + From 76144d440dcaa5eff82bacf00dd0d0cf6a325375 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 31 Jul 2025 10:24:07 +0300 Subject: [PATCH 360/388] Session stats improvements (#1746) * Detailed session stats for work time and storage stat cache * Bring back the old session state file writing --- crypto/block/transaction.cpp | 27 ++-- crypto/block/transaction.h | 1 + tdutils/td/utils/Timer.h | 46 ++++++ tl/generate/scheme/ton_api.tl | 18 ++- tl/generate/scheme/ton_api.tlo | Bin 116116 -> 117588 bytes validator/impl/collator-impl.h | 5 +- validator/impl/collator.cpp | 170 ++++++++++++++--------- validator/impl/validate-query.cpp | 45 +++--- validator/impl/validate-query.hpp | 4 +- validator/interfaces/validator-manager.h | 60 +++++++- validator/manager.cpp | 48 +------ validator/manager.hpp | 5 - 12 files changed, 277 insertions(+), 152 deletions(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index ca34e14af..75ee477ba 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -1710,12 +1710,12 @@ bool Transaction::run_precompiled_contract(const ComputePhaseConfig& cfg, precom ComputePhase& cp = *compute_phase; CHECK(cp.precompiled_gas_usage); td::uint64 gas_usage = cp.precompiled_gas_usage.value(); - td::Timer timer; + td::RealCpuTimer timer; auto result = impl.run(my_addr, now, start_lt, balance, new_data, *in_msg_body, in_msg, msg_balance_remaining, in_msg_extern, compute_vm_libraries(cfg), cfg.global_version, cfg.max_vm_data_depth, new_code, cfg.unpacked_config_tuple, due_payment.not_null() ? due_payment : td::zero_refint(), gas_usage); - double elapsed = timer.elapsed(); + time_tvm = timer.elapsed_both(); cp.vm_init_state_hash = td::Bits256::zero(); cp.exit_code = result.exit_code; cp.out_of_gas = false; @@ -1726,7 +1726,7 @@ bool Transaction::run_precompiled_contract(const ComputePhaseConfig& cfg, precom cp.success = (cp.accepted && result.committed); LOG(INFO) << "Running precompiled smart contract " << impl.get_name() << ": exit_code=" << result.exit_code << " accepted=" << result.accepted << " success=" << cp.success << " gas_used=" << gas_usage - << " time=" << elapsed << "s"; + << " time=" << time_tvm.real << "s cpu_time=" << time_tvm.cpu; if (cp.accepted & use_msg_state) { was_activated = true; acc_status = Account::acc_active; @@ -1734,7 +1734,7 @@ bool Transaction::run_precompiled_contract(const ComputePhaseConfig& cfg, precom if (cfg.with_vm_log) { cp.vm_log = PSTRING() << "Running precompiled smart contract " << impl.get_name() << ": exit_code=" << result.exit_code << " accepted=" << result.accepted - << " success=" << cp.success << " gas_used=" << gas_usage << " time=" << elapsed << "s"; + << " success=" << cp.success << " gas_used=" << gas_usage << " time=" << time_tvm.real << "s"; } if (cp.success) { cp.new_data = impl.get_c4(); @@ -1943,9 +1943,9 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { LOG(DEBUG) << "starting VM"; cp.vm_init_state_hash = vm.get_state_hash(); - td::Timer timer; + td::RealCpuTimer timer; cp.exit_code = ~vm.run(); - double elapsed = timer.elapsed(); + time_tvm = timer.elapsed_both(); LOG(DEBUG) << "VM terminated with exit code " << cp.exit_code; cp.out_of_gas = (cp.exit_code == ~(int)vm::Excno::out_of_gas); cp.vm_final_state_hash = vm.get_final_state_hash(cp.exit_code); @@ -1971,7 +1971,7 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { LOG(INFO) << "steps: " << vm.get_steps_count() << " gas: used=" << gas.gas_consumed() << ", max=" << gas.gas_max << ", limit=" << gas.gas_limit << ", credit=" << gas.gas_credit; LOG(INFO) << "out_of_gas=" << cp.out_of_gas << ", accepted=" << cp.accepted << ", success=" << cp.success - << ", time=" << elapsed << "s"; + << ", time=" << time_tvm.real << "s, cpu_time=" << time_tvm.cpu; if (logger != nullptr) { cp.vm_log = logger->get_log(); } @@ -3196,7 +3196,13 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, } { TD_PERF_COUNTER(transaction_storage_stat_a); - td::Timer timer; + td::RealCpuTimer timer; + SCOPE_EXIT { + LOG_IF(INFO, timer.elapsed_real() > 0.1) << "Compute used storage (1) took " << timer.elapsed_real() << "s"; + if (is_account_stat) { + time_storage_stat += timer.elapsed_both(); + } + }; if (is_account_stat && compute_phase) { storage_stat.add_hint(compute_phase->vm_loaded_cells); } @@ -3208,9 +3214,6 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, storage_stat_updates.push_back(new_library); } TRY_STATUS(storage_stat.replace_roots({new_code, new_data, new_library}, /* check_merkle_depth = */ true)); - if (timer.elapsed() > 0.1) { - LOG(INFO) << "Compute used storage (1) took " << timer.elapsed() << "s"; - } } if (storage_stat.get_total_cells() > size_limits.max_acc_state_cells || @@ -3511,9 +3514,11 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { auto roots = new_storage_for_stat->prefetch_all_refs(); storage_stat_updates.insert(storage_stat_updates.end(), roots.begin(), roots.end()); { + td::RealCpuTimer timer; StorageStatCalculationContext context{true}; StorageStatCalculationContext::Guard guard{&context}; td::Status S = stats.replace_roots(roots); + time_storage_stat += timer.elapsed_both(); if (S.is_error()) { LOG(ERROR) << "Cannot recompute storage stats for account " << account.addr.to_hex() << ": " << S.move_as_error(); return false; diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index 1a7af2d12..83b789163 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -399,6 +399,7 @@ struct Transaction { td::optional new_storage_dict_hash; bool gas_limit_overridden{false}; std::vector> storage_stat_updates; + td::RealCpuTimer::Time time_tvm, time_storage_stat; Transaction(const Account& _account, int ttype, ton::LogicalTime req_start_lt, ton::UnixTime _now, Ref _inmsg = {}); bool unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* cfg); diff --git a/tdutils/td/utils/Timer.h b/tdutils/td/utils/Timer.h index d787f1ca2..d1ba104b0 100644 --- a/tdutils/td/utils/Timer.h +++ b/tdutils/td/utils/Timer.h @@ -81,6 +81,52 @@ class ThreadCpuTimer { bool is_paused_{false}; }; +class RealCpuTimer { + public: + RealCpuTimer() : RealCpuTimer(false) { + } + explicit RealCpuTimer(bool is_paused) : real_(is_paused), cpu_(is_paused) { + } + RealCpuTimer(const RealCpuTimer &other) = default; + RealCpuTimer &operator=(const RealCpuTimer &other) = default; + + double elapsed_real() const { + return real_.elapsed(); + } + double elapsed_cpu() const { + return cpu_.elapsed(); + } + struct Time { + double real = 0.0, cpu = 0.0; + double get(bool is_cpu) const { + return is_cpu ? cpu : real; + } + Time &operator+=(const Time &other) { + real += other.real; + cpu += other.cpu; + return *this; + } + Time operator-(const Time &other) const { + return {.real = real - other.real, .cpu = cpu - other.cpu}; + } + }; + Time elapsed_both() const { + return {.real = real_.elapsed(), .cpu = cpu_.elapsed()}; + } + void pause() { + real_.pause(); + cpu_.pause(); + } + void resume() { + real_.resume(); + cpu_.resume(); + } + + private: + Timer real_; + ThreadCpuTimer cpu_; +}; + class PerfLog; struct EmptyDeleter { template diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index ebfd25def..1b8e27365 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -955,21 +955,35 @@ validatorStats.blockStats ext_msgs:validatorStats.blockStats.extMsgsStats transactions:int shard_configuration:(vector tonNode.blockIdExt) old_out_msg_queue_size:long new_out_msg_queue_size:long msg_queue_cleaned:int neighbors:(vector validatorStats.blockStats.neighborStats) = validatorStats.BlockStats; +validatorStats.collateWorkTimeStats + total:double queue_cleanup:double prelim_storage_stat:double trx_tvm:double trx_storage_stat:double + trx_other:double final_storage_stat:double create_block:double create_collated_data:double create_block_candidate:double + = validatorStats.CollateWorkTimeStats; +validatorStats.storageStatCacheStats + small_cnt:long small_cells:long hit_cnt:long hit_cells:long miss_cnt:long miss_cells:long = validatorStats.StorageStatCacheStats; validatorStats.collatedBlock block_id:tonNode.blockIdExt collated_data_hash:int256 cc_seqno:int collated_at:double bytes:int collated_data_bytes:int attempt:int self:int256 is_validator:Bool total_time:double work_time:double cpu_work_time:double time_stats:string + work_time_real_stats:validatorStats.collateWorkTimeStats + work_time_cpu_stats:validatorStats.collateWorkTimeStats block_limits:validatorStats.blockLimitsStatus - block_stats:validatorStats.blockStats = validatorSession.stats.CollatedBlock; + block_stats:validatorStats.blockStats + storage_stat_cache:validatorStats.storageStatCacheStats = validatorSession.stats.CollatedBlock; +validatorStats.validateWorkTimeStats + total:double trx_tvm:double trx_storage_stat:double trx_other:double = validatorStats.ValidateWorkTimeStats; validatorStats.validatedBlock block_id:tonNode.blockIdExt collated_data_hash:int256 validated_at:double self:int256 valid:Bool comment:string bytes:int collated_data_bytes:int - total_time:double work_time:double cpu_work_time:double = validatorStats.ValidatedBlock; + total_time:double work_time:double cpu_work_time:double + work_time_real_stats:validatorStats.validateWorkTimeStats + work_time_cpu_stats:validatorStats.validateWorkTimeStats + storage_stat_cache:validatorStats.storageStatCacheStats = validatorStats.ValidatedBlock; validatorStats.newValidatorGroup.node id:int256 pubkey:PublicKey adnl_id:int256 weight:long = validatorStats.newValidatorGroup.Node; validatorStats.newValidatorGroup session_id:int256 shard:tonNode.shardId cc_seqno:int diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 8c13ffcb66d6296e175a89e22c09305c8f8267c7..9a5241d8084e1eb2ff75ea71d00b934fec92235b 100644 GIT binary patch delta 1122 zcma)5&ubGw6!yW|{Ag2bno8qR74%?0>s16}ZA26dh%KfS?PlCe%#zt`x;v@fq$nbI z35>W0L6H6fh6>wjFH$@!sFxBCUOafIAfB8z*$^87aSqG8H{ZPXy>GsK@jUYJWhC+K z)!Fk0!gaNk*vM-1!?U({m)$#+Z~eMx5n?AGhbMx~Y2l)CpKOf}h zVPi`1^!EyIJcf+FyiZmg)vMVOw~aos?e?K=0;9q4(SC;w_!`m4wBRyASvQ- zqPQ%KEe~>s6Xw5O27PR`K^xRC86~xb-!6fknykB&S+?Y(o?gx~3dv^?vMsn0-Rg?7AHZ22Uz*=Ba5TBW$@+&jwOtY*PBJ1 zw~IP6E*1qVnZ752QGpYr_~tPoAO7hNlNhzP3q&&RVVtfJ&FH~X msg_root, block::Account* acc, UnixTime utime, LogicalTime lt, block::StoragePhaseConfig* storage_phase_cfg, block::ComputePhaseConfig* compute_phase_cfg, block::ActionPhaseConfig* action_phase_cfg, block::SerializeConfig* serialize_cfg, bool external, - LogicalTime after_lt); + LogicalTime after_lt, CollationStats* stats = nullptr); private: void start_up() override; @@ -404,8 +404,7 @@ class Collator final : public td::actor::Actor { static td::uint32 get_skip_externals_queue_size(); private: - td::Timer work_timer_{true}; - td::ThreadCpuTimer cpu_work_timer_{true}; + td::RealCpuTimer work_timer_{true}; CollationStats stats_; void finalize_stats(); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index acf9075b4..5d0cbd2d6 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -1963,10 +1963,8 @@ bool Collator::register_shard_block_creators(std::vector creator_li */ bool Collator::try_collate() { work_timer_.resume(); - cpu_work_timer_.resume(); SCOPE_EXIT { work_timer_.pause(); - cpu_work_timer_.pause(); }; if (!preinit_complete) { LOG(WARNING) << "running do_preinit()"; @@ -2450,8 +2448,10 @@ bool Collator::dequeue_message(Ref msg_envelope, ton::LogicalTime deli * @returns True if the cleanup operation was successful, false otherwise. */ bool Collator::out_msg_queue_cleanup() { + td::RealCpuTimer timer; SCOPE_EXIT { stats_.load_fraction_queue_cleanup = block_limit_status_->load_fraction(block::ParamLimits::cl_normal); + stats_.work_time.queue_cleanup = timer.elapsed_both(); }; LOG(INFO) << "cleaning outbound queue from messages already imported by neighbors"; if (verbosity >= 2) { @@ -2677,6 +2677,10 @@ bool Collator::init_account_storage_dict(block::Account& account) { if (storage_dict_hash.is_zero()) { return true; } + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.work_time.prelim_storage_stat = timer.elapsed_both(); + }; td::Ref cached_dict_root = storage_stat_cache_ ? storage_stat_cache_(storage_dict_hash) : td::Ref{}; if (cached_dict_root.not_null()) { @@ -2684,6 +2688,14 @@ bool Collator::init_account_storage_dict(block::Account& account) { LOG(DEBUG) << "Inited storage stat from cache for account " << account.addr.to_hex() << " (" << account.storage_used.cells << " cells)"; storage_stat_cache_update_.emplace_back(cached_dict_root, account.storage_used.cells); + stats_.storage_stat_cache.hit_cnt++; + stats_.storage_stat_cache.hit_cells += account.storage_used.cells; + } else if (account.storage_used.cells >= StorageStatCache::MIN_ACCOUNT_CELLS) { + stats_.storage_stat_cache.miss_cnt++; + stats_.storage_stat_cache.miss_cells += account.storage_used.cells; + } else { + stats_.storage_stat_cache.small_cnt++; + stats_.storage_stat_cache.small_cells += account.storage_used.cells; } if (!full_collated_data_ || is_masterchain()) { if (cached_dict_root.not_null()) { @@ -2847,6 +2859,10 @@ static td::Ref clean_usage_cells(td::Ref old_root, td::Ref= StorageStatCache::MIN_ACCOUNT_CELLS; @@ -3188,6 +3204,12 @@ bool Collator::create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, t std::unique_ptr trans = std::make_unique( *acc, mask == 2 ? block::transaction::Transaction::tr_tick : block::transaction::Transaction::tr_tock, req_start_lt, now_); + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.work_time.trx_tvm += trans->time_tvm; + stats_.work_time.trx_storage_stat += trans->time_storage_stat; + stats_.work_time.trx_other += timer.elapsed_both() - trans->time_tvm - trans->time_storage_stat; + }; if (!trans->prepare_storage_phase(storage_phase_cfg_, true)) { return fatal_error(td::Status::Error( -666, std::string{"cannot create storage phase of a new transaction for smart contract "} + smc_addr.to_hex())); @@ -3292,7 +3314,7 @@ Ref Collator::create_ordinary_transaction(Ref msg_root, } set_current_tx_storage_dict(*acc); auto res = impl_create_ordinary_transaction(msg_root, acc, now_, start_lt, &storage_phase_cfg_, &compute_phase_cfg_, - &action_phase_cfg_, &serialize_cfg_, external, after_lt); + &action_phase_cfg_, &serialize_cfg_, external, after_lt, &stats_); if (res.is_error()) { auto error = res.move_as_error(); if (error.code() == -701) { @@ -3348,19 +3370,17 @@ Ref Collator::create_ordinary_transaction(Ref msg_root, * @param serialize_cfg The configuration for the serialization of the transaction. * @param external Flag indicating if the message is external. * @param after_lt The logical time after which the transaction should occur. Used only for external messages. + * @param collation_stats Stats to write real/cpu time to (optional) * * @returns A Result object containing the created transaction. * Returns error_code == 669 if the error is fatal and the block can not be produced. * Returns error_code == 701 if the transaction can not be included into block, but it's ok (external or too early internal). */ -td::Result> Collator::impl_create_ordinary_transaction(Ref msg_root, - block::Account* acc, - UnixTime utime, LogicalTime lt, - block::StoragePhaseConfig* storage_phase_cfg, - block::ComputePhaseConfig* compute_phase_cfg, - block::ActionPhaseConfig* action_phase_cfg, - block::SerializeConfig* serialize_cfg, - bool external, LogicalTime after_lt) { +td::Result> Collator::impl_create_ordinary_transaction( + Ref msg_root, block::Account* acc, UnixTime utime, LogicalTime lt, + block::StoragePhaseConfig* storage_phase_cfg, block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg, block::SerializeConfig* serialize_cfg, bool external, + LogicalTime after_lt, CollationStats* stats) { if (acc->last_trans_end_lt_ >= lt && acc->transactions.empty()) { return td::Status::Error(-669, PSTRING() << "last transaction time in the state of account " << acc->workchain << ":" << acc->addr.to_hex() << " is too large"); @@ -3372,64 +3392,74 @@ td::Result> Collator::impl_crea std::unique_ptr trans = std::make_unique( *acc, block::transaction::Transaction::tr_ord, trans_min_lt + 1, utime, msg_root); - bool ihr_delivered = false; // FIXME - if (!trans->unpack_input_msg(ihr_delivered, action_phase_cfg)) { - if (external) { - // inbound external message was not accepted - return td::Status::Error(-701, "inbound external message rejected by account "s + acc->addr.to_hex() + - " before smart-contract execution"); + { + td::RealCpuTimer timer; + SCOPE_EXIT { + if (stats) { + stats->work_time.trx_tvm += trans->time_tvm; + stats->work_time.trx_storage_stat += trans->time_storage_stat; + stats->work_time.trx_other += timer.elapsed_both() - trans->time_tvm - trans->time_storage_stat; + } + }; + bool ihr_delivered = false; // FIXME + if (!trans->unpack_input_msg(ihr_delivered, action_phase_cfg)) { + if (external) { + // inbound external message was not accepted + return td::Status::Error(-701, "inbound external message rejected by account "s + acc->addr.to_hex() + + " before smart-contract execution"); + } + return td::Status::Error(-669, "cannot unpack input message for a new transaction"); } - return td::Status::Error(-669, "cannot unpack input message for a new transaction"); - } - if (trans->bounce_enabled) { - if (!trans->prepare_storage_phase(*storage_phase_cfg, true)) { - return td::Status::Error( - -669, "cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + if (trans->bounce_enabled) { + if (!trans->prepare_storage_phase(*storage_phase_cfg, true)) { + return td::Status::Error( + -669, "cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (!external && !trans->prepare_credit_phase()) { + return td::Status::Error( + -669, "cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + } else { + if (!external && !trans->prepare_credit_phase()) { + return td::Status::Error( + -669, "cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (!trans->prepare_storage_phase(*storage_phase_cfg, true, true)) { + return td::Status::Error( + -669, "cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } } - if (!external && !trans->prepare_credit_phase()) { + if (!trans->prepare_compute_phase(*compute_phase_cfg)) { return td::Status::Error( - -669, "cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + -669, "cannot create compute phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (!trans->compute_phase->accepted) { + if (external) { + // inbound external message was not accepted + auto const& cp = *trans->compute_phase; + return td::Status::Error( + -701, PSLICE() << "inbound external message rejected by transaction " << acc->addr.to_hex() << ":\n" + << "exitcode=" << cp.exit_code << ", steps=" << cp.vm_steps << ", gas_used=" << cp.gas_used + << (cp.vm_log.empty() ? "" : "\nVM Log (truncated):\n..." + cp.vm_log)); + } else if (trans->compute_phase->skip_reason == block::ComputePhase::sk_none) { + return td::Status::Error(-669, "new ordinary transaction for smart contract "s + acc->addr.to_hex() + + " has not been accepted by the smart contract (?)"); + } } - } else { - if (!external && !trans->prepare_credit_phase()) { + if (trans->compute_phase->success && !trans->prepare_action_phase(*action_phase_cfg)) { return td::Status::Error( - -669, "cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + -669, "cannot create action phase of a new transaction for smart contract "s + acc->addr.to_hex()); } - if (!trans->prepare_storage_phase(*storage_phase_cfg, true, true)) { + if (trans->bounce_enabled && + (!trans->compute_phase->success || trans->action_phase->state_exceeds_limits || trans->action_phase->bounce) && + !trans->prepare_bounce_phase(*action_phase_cfg)) { return td::Status::Error( - -669, "cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + -669, "cannot create bounce phase of a new transaction for smart contract "s + acc->addr.to_hex()); } - } - if (!trans->prepare_compute_phase(*compute_phase_cfg)) { - return td::Status::Error( - -669, "cannot create compute phase of a new transaction for smart contract "s + acc->addr.to_hex()); - } - if (!trans->compute_phase->accepted) { - if (external) { - // inbound external message was not accepted - auto const& cp = *trans->compute_phase; - return td::Status::Error( - -701, PSLICE() << "inbound external message rejected by transaction " << acc->addr.to_hex() << ":\n" - << "exitcode=" << cp.exit_code << ", steps=" << cp.vm_steps << ", gas_used=" << cp.gas_used - << (cp.vm_log.empty() ? "" : "\nVM Log (truncated):\n..." + cp.vm_log)); - } else if (trans->compute_phase->skip_reason == block::ComputePhase::sk_none) { - return td::Status::Error(-669, "new ordinary transaction for smart contract "s + acc->addr.to_hex() + - " has not been accepted by the smart contract (?)"); + if (!trans->serialize(*serialize_cfg)) { + return td::Status::Error(-669, "cannot serialize new transaction for smart contract "s + acc->addr.to_hex()); } } - if (trans->compute_phase->success && !trans->prepare_action_phase(*action_phase_cfg)) { - return td::Status::Error( - -669, "cannot create action phase of a new transaction for smart contract "s + acc->addr.to_hex()); - } - if (trans->bounce_enabled && - (!trans->compute_phase->success || trans->action_phase->state_exceeds_limits || trans->action_phase->bounce) && - !trans->prepare_bounce_phase(*action_phase_cfg)) { - return td::Status::Error( - -669, "cannot create bounce phase of a new transaction for smart contract "s + acc->addr.to_hex()); - } - if (!trans->serialize(*serialize_cfg)) { - return td::Status::Error(-669, "cannot serialize new transaction for smart contract "s + acc->addr.to_hex()); - } return std::move(trans); } @@ -6018,6 +6048,10 @@ bool Collator::create_mc_block_extra(Ref& mc_block_extra) { * @returns True if the new block is successfully created, false otherwise. */ bool Collator::create_block() { + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.work_time.create_block += timer.elapsed_both(); + }; Ref block_info, extra; if (!create_block_info(block_info)) { return fatal_error("cannot create BlockInfo for the new block"); @@ -6095,7 +6129,7 @@ Ref Collator::collate_shard_block_descr_set() { /** * Visits certain cells in out msg queue and dispatch queue to add them to the proof * - * @returns True on success, Falise if error occurred + * @returns True on success, False if error occurred */ bool Collator::prepare_msg_queue_proof() { auto res = old_out_msg_queue_->scan_diff( @@ -6156,6 +6190,10 @@ bool Collator::prepare_msg_queue_proof() { * @returns True if the collated data was successfully created, false otherwise. */ bool Collator::create_collated_data() { + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.work_time.create_collated_data += timer.elapsed_both(); + }; // 1. store the set of used shard block descriptions if (!used_shard_block_descr_.empty()) { auto cell = collate_shard_block_descr_set(); @@ -6253,6 +6291,10 @@ bool Collator::create_collated_data() { * @returns True if the block candidate was created successfully, false otherwise. */ bool Collator::create_block_candidate() { + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.work_time.create_block_candidate += timer.elapsed_both(); + }; auto consensus_config = config_->get_consensus_config(); // 1. serialize block LOG(INFO) << "serializing new Block"; @@ -6524,9 +6566,8 @@ td::uint32 Collator::get_skip_externals_queue_size() { } void Collator::finalize_stats() { - double work_time = work_timer_.elapsed(); - double cpu_work_time = cpu_work_timer_.elapsed(); - LOG(WARNING) << "Collate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s"; + auto work_time = work_timer_.elapsed_both(); + LOG(WARNING) << "Collate query work time = " << work_time.real << "s, cpu time = " << work_time.cpu << "s"; if (block_candidate) { stats_.block_id = block_candidate->id; stats_.collated_data_hash = block_candidate->collated_file_hash; @@ -6553,8 +6594,7 @@ void Collator::finalize_stats() { block_limit_status_->limits.classify_collated_data_size(stats_.estimated_collated_data_bytes); } stats_.total_time = perf_timer_.elapsed(); - stats_.work_time = work_time; - stats_.cpu_work_time = cpu_work_time; + stats_.work_time.total = work_time; stats_.time_stats = (PSTRING() << perf_log_); if (is_masterchain() && shard_conf_) { shard_conf_->process_shard_hashes([&](const block::McShardHash& shard) { diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index ac7f400f3..fbaa06aac 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -5369,6 +5369,14 @@ std::unique_ptr ValidateQuery::unpack_account(td::ConstBitPtr ad LOG(DEBUG) << "Inited storage stat from cache for account " << addr.to_hex(256) << " (" << new_acc->storage_used.cells << " cells)"; storage_stat_cache_update_.emplace_back(dict_root, new_acc->storage_used.cells); + stats_.storage_stat_cache.hit_cnt++; + stats_.storage_stat_cache.hit_cells += new_acc->storage_used.cells; + } else if (new_acc->storage_used.cells >= StorageStatCache::MIN_ACCOUNT_CELLS) { + stats_.storage_stat_cache.miss_cnt++; + stats_.storage_stat_cache.miss_cells += new_acc->storage_used.cells; + } else { + stats_.storage_stat_cache.small_cnt++; + stats_.storage_stat_cache.small_cells += new_acc->storage_used.cells; } } } @@ -5780,6 +5788,12 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT // .... std::unique_ptr trs = std::make_unique(account, trans_type, lt, now_, in_msg_root); + td::RealCpuTimer timer; + SCOPE_EXIT { + stats_.work_time.trx_tvm += trs->time_tvm; + stats_.work_time.trx_storage_stat += trs->time_storage_stat; + stats_.work_time.trx_other += timer.elapsed_both() - trs->time_tvm - trs->time_storage_stat; + }; if (in_msg_root.not_null()) { if (!trs->unpack_input_msg(ihr_delivered, &action_phase_cfg_)) { // inbound external message was not accepted @@ -7029,10 +7043,8 @@ bool ValidateQuery::try_validate() { return true; } work_timer_.resume(); - cpu_work_timer_.resume(); SCOPE_EXIT { work_timer_.pause(); - cpu_work_timer_.pause(); }; try { if (!stage_) { @@ -7181,25 +7193,24 @@ void ValidateQuery::written_candidate() { * Sends validation work time to manager. */ void ValidateQuery::record_stats(bool valid, std::string error_message) { - ValidationStats stats; - stats.block_id = id_; - stats.collated_data_hash = block_candidate.collated_file_hash; - stats.validated_at = td::Clocks::system(); - stats.self = local_validator_id_; - stats.valid = valid; + stats_.block_id = id_; + stats_.collated_data_hash = block_candidate.collated_file_hash; + stats_.validated_at = td::Clocks::system(); + stats_.self = local_validator_id_; + stats_.valid = valid; if (valid) { - stats.comment = (PSTRING() << "OK ts=" << now_); + stats_.comment = (PSTRING() << "OK ts=" << now_); } else { - stats.comment = std::move(error_message); + stats_.comment = std::move(error_message); } - stats.actual_bytes = block_candidate.data.size(); - stats.actual_collated_data_bytes = block_candidate.collated_data.size(); - stats.total_time = perf_timer_.elapsed(); - stats.work_time = work_timer_.elapsed(); - stats.cpu_work_time = cpu_work_timer_.elapsed(); + stats_.actual_bytes = block_candidate.data.size(); + stats_.actual_collated_data_bytes = block_candidate.collated_data.size(); + stats_.total_time = perf_timer_.elapsed(); + stats_.work_time.total = work_timer_.elapsed_both(); LOG(WARNING) << "validation took " << perf_timer_.elapsed() << "s"; - LOG(WARNING) << "Validate query work time = " << stats.work_time << "s, cpu time = " << stats.cpu_work_time << "s"; - td::actor::send_closure(manager, &ValidatorManager::log_validate_query_stats, std::move(stats)); + LOG(WARNING) << "Validate query work time = " << stats_.work_time.total.real + << "s, cpu time = " << stats_.work_time.total.cpu << "s"; + td::actor::send_closure(manager, &ValidatorManager::log_validate_query_stats, std::move(stats_)); } } // namespace validator diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 95e81cdec..e43d782f5 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -413,8 +413,8 @@ class ValidateQuery : public td::actor::Actor { return true; } - td::Timer work_timer_{true}; - td::ThreadCpuTimer cpu_work_timer_{true}; + td::RealCpuTimer work_timer_{true}; + ValidationStats stats_; void record_stats(bool valid, std::string error_message = ""); }; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 6c5e8ce9e..78248610a 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -55,6 +55,17 @@ struct AsyncSerializerState { UnixTime last_written_block_ts; }; +struct StorageStatCacheStats { + td::uint64 small_cnt = 0, small_cells = 0; + td::uint64 hit_cnt = 0, hit_cells = 0; + td::uint64 miss_cnt = 0, miss_cells = 0; + + tl_object_ptr tl() const { + return create_tl_object(small_cnt, small_cells, hit_cnt, hit_cells, + miss_cnt, miss_cells); + } +}; + struct CollationStats { BlockIdExt block_id{workchainInvalid, 0, 0, RootHash::zero(), FileHash::zero()}; td::Status status = td::Status::OK(); @@ -69,7 +80,7 @@ struct CollationStats { td::uint32 estimated_bytes = 0, gas = 0, lt_delta = 0, estimated_collated_data_bytes = 0; int cat_bytes = 0, cat_gas = 0, cat_lt_delta = 0, cat_collated_data_bytes = 0; std::string limits_log; - double total_time = 0.0, work_time = 0.0, cpu_work_time = 0.0; + double total_time = 0.0; std::string time_stats; td::uint32 transactions = 0; @@ -104,6 +115,28 @@ struct CollationStats { double load_fraction_externals = -1.0; double load_fraction_new_msgs = -1.0; + struct WorkTimeStats { + td::RealCpuTimer::Time total; + td::RealCpuTimer::Time queue_cleanup; + td::RealCpuTimer::Time prelim_storage_stat; + td::RealCpuTimer::Time trx_tvm; + td::RealCpuTimer::Time trx_storage_stat; + td::RealCpuTimer::Time trx_other; + td::RealCpuTimer::Time final_storage_stat; + td::RealCpuTimer::Time create_block; + td::RealCpuTimer::Time create_collated_data; + td::RealCpuTimer::Time create_block_candidate; + + tl_object_ptr tl(bool is_cpu) const { + return create_tl_object( + total.get(is_cpu), queue_cleanup.get(is_cpu), prelim_storage_stat.get(is_cpu), trx_tvm.get(is_cpu), + trx_storage_stat.get(is_cpu), trx_other.get(is_cpu), final_storage_stat.get(is_cpu), create_block.get(is_cpu), + create_collated_data.get(is_cpu), create_block_candidate.get(is_cpu)); + } + }; + WorkTimeStats work_time; + StorageStatCacheStats storage_stat_cache; + tl_object_ptr tl() const { std::vector> shards_obj; for (const BlockIdExt& block_id : shard_configuration) { @@ -120,13 +153,13 @@ struct CollationStats { std::move(neighbors_obj)); return create_tl_object( create_tl_block_id(block_id), collated_data_hash, cc_seqno, collated_at, actual_bytes, - actual_collated_data_bytes, attempt, self.bits256_value(), is_validator, total_time, work_time, cpu_work_time, - time_stats, + actual_collated_data_bytes, attempt, self.bits256_value(), is_validator, total_time, work_time.total.real, + work_time.total.cpu, time_stats, work_time.tl(false), work_time.tl(true), create_tl_object( estimated_bytes, gas, lt_delta, estimated_collated_data_bytes, cat_bytes, cat_gas, cat_lt_delta, cat_collated_data_bytes, load_fraction_queue_cleanup, load_fraction_dispatch, load_fraction_internals, load_fraction_externals, load_fraction_new_msgs, limits_log), - std::move(block_stats)); + std::move(block_stats), storage_stat_cache.tl()); } }; @@ -138,12 +171,27 @@ struct ValidationStats { bool valid = false; std::string comment; td::uint32 actual_bytes = 0, actual_collated_data_bytes = 0; - double total_time = 0.0, work_time = 0.0, cpu_work_time = 0.0; + double total_time = 0.0; + + struct WorkTimeStats { + td::RealCpuTimer::Time total; + td::RealCpuTimer::Time trx_tvm; + td::RealCpuTimer::Time trx_storage_stat; + td::RealCpuTimer::Time trx_other; + + tl_object_ptr tl(bool is_cpu) const { + return create_tl_object( + total.get(is_cpu), trx_tvm.get(is_cpu), trx_storage_stat.get(is_cpu), trx_other.get(is_cpu)); + } + }; + WorkTimeStats work_time; + StorageStatCacheStats storage_stat_cache; tl_object_ptr tl() const { return create_tl_object( create_tl_block_id(block_id), collated_data_hash, validated_at, self.bits256_value(), valid, comment, - actual_bytes, actual_collated_data_bytes, total_time, work_time, cpu_work_time); + actual_bytes, actual_collated_data_bytes, total_time, work_time.total.real, work_time.total.cpu, + work_time.tl(false), work_time.tl(true), storage_stat_cache.tl()); } }; diff --git a/validator/manager.cpp b/validator/manager.cpp index 6e3780fef..1fcd6a520 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1992,7 +1992,6 @@ void ValidatorManagerImpl::start_up() { } validator_manager_init(opts_, actor_id(this), db_.get(), std::move(P)); - init_session_stats(); check_waiters_at_ = td::Timestamp::in(1.0); alarm_timestamp().relax(check_waiters_at_); @@ -3485,7 +3484,6 @@ void ValidatorManagerImpl::update_options(td::Ref opts) td::actor::send_closure(shard_block_verifier_, &ShardBlockVerifier::update_options, opts); } opts_ = std::move(opts); - init_session_stats(); } void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) { @@ -3739,51 +3737,19 @@ void ValidatorManagerImpl::init_validator_telemetry() { } } -void ValidatorManagerImpl::init_session_stats() { - if (opts_->get_session_logs_file() == session_stats_filename_) { - return; - } - session_stats_filename_ = opts_->get_session_logs_file(); - if (session_stats_filename_.empty()) { - session_stats_enabled_ = false; - session_stats_fd_.close(); - return; - } - auto r_fd = td::FileFd::open(session_stats_filename_, - td::FileFd::Flags::Write | td::FileFd::Flags::Append | td::FileFd::Create); - if (r_fd.is_error()) { - LOG(ERROR) << "Failed to open session stats file for writing: " << r_fd.move_as_error(); - session_stats_filename_.clear(); - session_stats_enabled_ = false; - return; - } - session_stats_fd_ = r_fd.move_as_ok(); - session_stats_enabled_ = true; -} - template void ValidatorManagerImpl::write_session_stats(const T &obj) { - if (!session_stats_enabled_) { + std::string fname = opts_->get_session_logs_file(); + if (fname.empty()) { return; } auto s = td::json_encode(td::ToJson(*obj.tl()), false); s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); - s += '\n'; - td::Slice slice{s}; - while (!slice.empty()) { - auto R = session_stats_fd_.write(slice); - if (R.is_error()) { - LOG(WARNING) << "Failed to write to session stats: " << R.move_as_error(); - } - if (R.ok() == 0) { - LOG(WARNING) << "Failed to write to session stats"; - } - slice.remove_prefix(R.ok()); - } - auto S = session_stats_fd_.sync(); - if (S.is_error()) { - LOG(WARNING) << "Failed to write to session stats: " << S; - } + + std::ofstream file; + file.open(fname, std::ios_base::app); + file << s << "\n"; + file.close(); } void ValidatorManagerImpl::init_shard_block_verifier(adnl::AdnlNodeIdShort local_id) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 9d1863c3d..952fcf49b 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -827,11 +827,6 @@ class ValidatorManagerImpl : public ValidatorManager { td::actor::ActorOwn storage_stat_cache_; - bool session_stats_enabled_ = false; - std::string session_stats_filename_; - td::FileFd session_stats_fd_; - - void init_session_stats(); template void write_session_stats(const T &obj); From 0326fc5479cec096a128a53368fa83aae6c11f8f Mon Sep 17 00:00:00 2001 From: Oleg Vallas Date: Fri, 1 Aug 2025 11:09:03 +0200 Subject: [PATCH 361/388] Improve block compression (#1710) * [improve-block-compression] Add improved block compression algorithm, saving 15% more space with almost the same performance. Add dynamic compression algorithm configuration. * [improve-block-compression] Add max_decompressed_size to decompression interface * [improve-block-compression] Fix encoding of Prunned branch cells. Add support for collated data format. * [improve-block-compression] Change compression interface to support multiple roots. Save algorithm flag in the first byte. * [improve-block-compression] Replace compression with a new one in method compress_candidate_data() * [improve-block-compression] Store decompression size at the start of the compressed data * [improve-block-compression] Remove passing max decompression size to new compression method * [improve-block-compression] Use new compression algorithm in serialize_block_full() * [improve-block-compression] Use new compression algorithm in serialize_block_candidate_broadcast() * [improve-block-compression] Small performance optimization * [improve-block-compression] Replace vector of vectors with vector of arrays for faster operations with BOC graph. Speed up by ~10%. * [improve-block-compression] Use new compression approach in serialize_block_broadcast() method * [improve-block-compression] Remove redundant debug output * [improve-block-compression] Add checks and validations for improved compression * [improve-block-compression] Remove redundant size calculation * [improve-block-compression] Add all necessary checks and validations to improved decompression method * [improve-block-compression] Add all necessary checks and validations to base compression methods * [improve-block-compression] Introduce tl objects for improved compression and integrate them into all compression usages. * [improve-block-compression] Fix support of prunned branch cells with level up to 7. * [improve-block-compression] Replace size literals with a constant. Add checks for baseline decompress. * [improve-block-compression] Add check against negative cell data length * [improve-block-compression] Catch CellWriteError from cell_builder.finalize() * [improve-block-compression] Move bits allocation and uint write into a single append function. * [improve-block-compression] Move buffer capacity check and bits reading to a separate method * [improve-block-compression] Revert original implementation of the compression methods, while keeping updated decompression methods * [improve-block-compression] Revert original implementation of the compression methods, while keeping updated decompression methods * [improve-block-compression] Store signatures directly inside tonNode_blockBroadcastCompressedV2 tl object. * [improve-block-compression] Pass max_decompression_size from config to boc_compress method. Use it to check limit on decompressed block. * [improve-block-compression] Increase max cell count to 2^32. Add node count validity check. * [improve-block-compression] Refactor types. Use unsigned types where possible. * [Compress-block-using-round-cache] Remove limitation of 2^16 root cells in BOC. * [Compress-block-using-round-cache] Regenerate ton_api.tlo after merging testnet branch. --- crypto/CMakeLists.txt | 2 + crypto/vm/boc-compression.cpp | 624 +++++++++++++++++++++ crypto/vm/boc-compression.h | 42 ++ tl/generate/scheme/ton_api.tl | 7 + tl/generate/scheme/ton_api.tlo | Bin 117588 -> 118872 bytes validator-session/candidate-serializer.cpp | 54 +- validator-session/candidate-serializer.h | 2 + validator/collator-node.cpp | 16 +- validator/full-node-fast-sync-overlays.cpp | 9 + validator/full-node-fast-sync-overlays.hpp | 2 + validator/full-node-private-overlay.cpp | 19 + validator/full-node-private-overlay.hpp | 4 + validator/full-node-serializer.cpp | 65 +++ validator/full-node-shard.cpp | 9 + validator/full-node-shard.hpp | 2 + 15 files changed, 844 insertions(+), 13 deletions(-) create mode 100644 crypto/vm/boc-compression.cpp create mode 100644 crypto/vm/boc-compression.h diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index fd3c41c09..50f890394 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -14,6 +14,7 @@ set(TON_CRYPTO_CORE_SOURCE openssl/residue.cpp openssl/rand.cpp vm/boc.cpp + vm/boc-compression.cpp vm/large-boc-serializer.cpp tl/tlblib.cpp @@ -108,6 +109,7 @@ set(TON_CRYPTO_SOURCE vm/arithops.h vm/atom.h vm/boc.h + vm/boc-compression.h vm/boc-writers.h vm/box.hpp vm/cellops.h diff --git a/crypto/vm/boc-compression.cpp b/crypto/vm/boc-compression.cpp new file mode 100644 index 000000000..63a834e08 --- /dev/null +++ b/crypto/vm/boc-compression.cpp @@ -0,0 +1,624 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2020 Telegram Systems LLP +*/ +#include "boc-compression.h" + +#include +#include "vm/boc.h" +#include "vm/boc-writers.h" +#include "vm/cells.h" +#include "vm/cellslice.h" +#include "td/utils/Slice-decl.h" +#include "td/utils/lz4.h" + +namespace vm { + +td::Result boc_compress_baseline_lz4(const std::vector>& boc_roots) { + TRY_RESULT(data, vm::std_boc_serialize_multi(std::move(boc_roots), 2)); + td::BufferSlice compressed = td::lz4_compress(data); + + // Add decompressed size at the beginning + td::BufferSlice compressed_with_size(compressed.size() + kDecompressedSizeBytes); + auto size_slice = td::BitSliceWrite(compressed_with_size.as_slice().ubegin(), kDecompressedSizeBytes * 8); + size_slice.bits().store_uint(data.size(), kDecompressedSizeBytes * 8); + memcpy(compressed_with_size.data() + kDecompressedSizeBytes, compressed.data(), compressed.size()); + + return compressed_with_size; +} + +td::Result>> boc_decompress_baseline_lz4(td::Slice compressed, int max_decompressed_size) { + // Check minimum input size for decompressed size header + if (compressed.size() < kDecompressedSizeBytes) { + return td::Status::Error("BOC decompression failed: input too small for header"); + } + + // Read decompressed size + constexpr size_t kSizeBits = kDecompressedSizeBytes * 8; + int decompressed_size = td::BitSlice(compressed.ubegin(), kSizeBits).bits().get_uint(kSizeBits); + compressed.remove_prefix(kDecompressedSizeBytes); + if (decompressed_size <= 0 || decompressed_size > max_decompressed_size) { + return td::Status::Error("BOC decompression failed: invalid decompressed size"); + } + + TRY_RESULT(decompressed, td::lz4_decompress(compressed, decompressed_size)); + TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed)); + return roots; +} + +inline void append_uint(td::BitString& bs, unsigned val, unsigned n) { + bs.reserve_bitslice(n).bits().store_uint(val, n); +} + +inline td::Result read_uint(td::BitSlice& bs, int bits) { + // Check if there enough bits available + if (bs.size() < bits) { + return td::Status::Error("BOC decompression failed: not enough bits to read"); + } + unsigned result = bs.bits().get_uint(bits); + bs.advance(bits); + return result; +} + +td::Result boc_compress_improved_structure_lz4(const std::vector>& boc_roots) { + // Input validation + if (boc_roots.empty()) { + return td::Status::Error("No root cells were provided for serialization"); + } + for (const auto& root : boc_roots) { + if (root.is_null()) { + return td::Status::Error("Cannot serialize a null cell reference into a bag of cells"); + } + } + + // Initialize data structures for graph representation + td::HashMap cell_hashes; + std::vector> boc_graph; + std::vector refs_cnt; + std::vector cell_data; + std::vector cell_type; + std::vector prunned_branch_level; + std::vector root_indexes; + size_t total_size_estimate = 0; + + // Build graph representation using recursive lambda + const auto build_graph = [&](auto&& self, td::Ref cell) -> td::Result { + if (cell.is_null()) { + return td::Status::Error("Error while importing a cell during serialization: cell is null"); + } + + auto cell_hash = cell->get_hash(); + auto it = cell_hashes.find(cell_hash); + if (it != cell_hashes.end()) { + return it->second; + } + + size_t current_cell_id = boc_graph.size(); + cell_hashes.emplace(cell_hash, current_cell_id); + + bool is_special = false; + vm::CellSlice cell_slice = vm::load_cell_slice_special(cell, is_special); + if (!cell_slice.is_valid()) { + return td::Status::Error("Invalid loaded cell data"); + } + td::BitSlice cell_bitslice = cell_slice.as_bitslice(); + + // Initialize new cell in graph + boc_graph.emplace_back(); + refs_cnt.emplace_back(cell_slice.size_refs()); + cell_type.emplace_back(size_t(cell_slice.special_type())); + prunned_branch_level.push_back(0); + + DCHECK(cell_slice.size_refs() <= 4); + + // Process special cell of type PrunnedBranch + if (cell_slice.special_type() == vm::CellTraits::SpecialType::PrunnedBranch) { + DCHECK(cell_slice.size() >= 16); + cell_data.emplace_back(cell_bitslice.subslice(16, cell_bitslice.size() - 16)); + prunned_branch_level.back() = cell_slice.data()[1]; + } else { + cell_data.emplace_back(cell_bitslice); + } + total_size_estimate += cell_bitslice.size(); + + // Process cell references + for (int i = 0; i < cell_slice.size_refs(); ++i) { + TRY_RESULT(child_id, self(self, cell_slice.prefetch_ref(i))); + boc_graph[current_cell_id][i] = child_id; + } + + return current_cell_id; + }; + + // Build the graph starting from roots + for (auto root : boc_roots) { + TRY_RESULT(root_cell_id, build_graph(build_graph, root)); + root_indexes.push_back(root_cell_id); + } + + // Check graph properties + const size_t node_count = boc_graph.size(); + std::vector> reverse_graph(node_count); + size_t edge_count = 0; + + // Build reverse graph + for (int i = 0; i < node_count; ++i) { + for (size_t child_index = 0; child_index < refs_cnt[i]; ++child_index) { + size_t child = boc_graph[i][child_index]; + ++edge_count; + reverse_graph[child].push_back(i); + } + } + + // Process cell data sizes + std::vector is_data_small(node_count, 0); + for (int i = 0; i < node_count; ++i) { + if (cell_type[i] != 1) { + is_data_small[i] = cell_data[i].size() < 128; + } + } + + // Perform topological sort + std::vector topo_order, rank(node_count); + const auto topological_sort = [&]() -> td::Status { + std::vector> queue; + queue.reserve(node_count); + std::vector in_degree(node_count); + + // Calculate in-degrees and initialize queue + for (int i = 0; i < node_count; ++i) { + in_degree[i] = refs_cnt[i]; + if (in_degree[i] == 0) { + queue.emplace_back(cell_type[i] == 0, -int(cell_data[i].size()), -i); + } + } + + if (queue.empty()) { + return td::Status::Error("Cycle detected in cell references"); + } + + std::sort(queue.begin(), queue.end()); + + // Process queue + while (!queue.empty()) { + int node = -std::get<2>(queue.back()); + queue.pop_back(); + topo_order.push_back(node); + + for (int parent : reverse_graph[node]) { + if (--in_degree[parent] == 0) { + queue.emplace_back(0, 0, -parent); + } + } + } + + if (topo_order.size() != node_count) { + return td::Status::Error("Invalid graph structure"); + } + + std::reverse(topo_order.begin(), topo_order.end()); + return td::Status::OK(); + }; + + TRY_STATUS(topological_sort()); + + // Calculate index of vertices in topsort + for (int i = 0; i < node_count; ++i) { + rank[topo_order[i]] = i; + } + + // Build compressed representation + td::BitString result; + total_size_estimate += (node_count * 10 * 8); + result.reserve_bits(total_size_estimate); + + // Store roots information + append_uint(result, root_indexes.size(), 32); + for (int root_ind : root_indexes) { + append_uint(result, rank[root_ind], 32); + } + + // Store node count + append_uint(result, node_count, 32); + + // Store cell types and sizes + for (int i = 0; i < node_count; ++i) { + size_t node = topo_order[i]; + size_t currrent_cell_type = bool(cell_type[node]) + prunned_branch_level[node]; + append_uint(result, currrent_cell_type, 4); + append_uint(result, refs_cnt[node], 4); + + if (cell_type[node] != 1) { + if (is_data_small[node]) { + append_uint(result, 1, 1); + append_uint(result, cell_data[node].size(), 7); + } else { + append_uint(result, 0, 1); + append_uint(result, 1 + cell_data[node].size() / 8, 7); + } + } + } + + // Store edge information + auto edge_bits = result.reserve_bitslice(edge_count).bits(); + for (int i = 0; i < node_count; ++i) { + size_t node = topo_order[i]; + for (size_t child_index = 0; child_index < refs_cnt[node]; ++child_index) { + size_t child = boc_graph[node][child_index]; + edge_bits.store_uint(rank[child] == i + 1, 1); + ++edge_bits; + } + } + + // Store cell data + for (size_t node : topo_order) { + if (cell_type[node] != 1 && !is_data_small[node]) { + continue; + } + result.append(cell_data[node].subslice(0, cell_data[node].size() % 8)); + } + + // Store BOC graph with optimized encoding + for (size_t i = 0; i < node_count; ++i) { + size_t node = topo_order[i]; + if (node_count <= i + 3) + continue; + + for (int j = 0; j < refs_cnt[node]; ++j) { + if (rank[boc_graph[node][j]] <= i + 1) + continue; + + int delta = rank[boc_graph[node][j]] - i - 2; // Always >= 0 because of above check + size_t required_bits = 1 + (31 ^ __builtin_clz(node_count - i - 3)); + + if (required_bits < 8 - (result.size() + 1) % 8 + 1) { + append_uint(result, delta, required_bits); + } else if (delta < (1 << (8 - (result.size() + 1) % 8))) { + size_t available_bits = 8 - (result.size() + 1) % 8; + append_uint(result, 1, 1); + append_uint(result, delta, available_bits); + } else { + append_uint(result, 0, 1); + append_uint(result, delta, required_bits); + } + } + } + + // Pad result to byte boundary + while (result.size() % 8) { + append_uint(result, 0, 1); + } + + // Store remaining cell data + for (size_t node : topo_order) { + if (cell_type[node] == 1 || is_data_small[node]) { + size_t prefix_size = cell_data[node].size() % 8; + result.append(cell_data[node].subslice(prefix_size, cell_data[node].size() - prefix_size)); + } else { + size_t data_size = cell_data[node].size() + 1; + size_t padding = (8 - data_size % 8) % 8; + + if (padding) { + append_uint(result, 0, padding); + } + append_uint(result, 1, 1); + result.append(cell_data[node]); + } + } + + // Final padding + while (result.size() % 8) { + append_uint(result, 0, 1); + } + + // Create final compressed buffer + td::BufferSlice serialized((const char*)result.bits().get_byte_ptr(), result.size() / 8); + + td::BufferSlice compressed = td::lz4_compress(serialized); + + // Add decompressed size at the beginning + td::BufferSlice compressed_with_size(compressed.size() + kDecompressedSizeBytes); + auto size_slice = td::BitSliceWrite(compressed_with_size.as_slice().ubegin(), kDecompressedSizeBytes * 8); + size_slice.bits().store_uint(serialized.size(), kDecompressedSizeBytes * 8); + memcpy(compressed_with_size.data() + kDecompressedSizeBytes, compressed.data(), compressed.size()); + + return compressed_with_size; +} + +td::Result>> boc_decompress_improved_structure_lz4(td::Slice compressed, int max_decompressed_size) { + constexpr size_t kMaxCellDataLengthBits = 1024; + + // Check minimum input size for decompressed size header + if (compressed.size() < kDecompressedSizeBytes) { + return td::Status::Error("BOC decompression failed: input too small for header"); + } + + // Read decompressed size + constexpr size_t kSizeBits = kDecompressedSizeBytes * 8; + size_t decompressed_size = td::BitSlice(compressed.ubegin(), kSizeBits).bits().get_uint(kSizeBits); + compressed.remove_prefix(kDecompressedSizeBytes); + if (decompressed_size > max_decompressed_size) { + return td::Status::Error("BOC decompression failed: invalid decompressed size"); + } + + // Decompress LZ4 data + TRY_RESULT(serialized, td::lz4_decompress(compressed, decompressed_size)); + + if (serialized.size() != decompressed_size) { + return td::Status::Error("BOC decompression failed: decompressed size mismatch"); + } + + // Initialize bit reader + td::BitSlice bit_reader(serialized.as_slice().ubegin(), serialized.as_slice().size() * 8); + size_t orig_size = bit_reader.size(); + + // Read root count + TRY_RESULT(root_count, read_uint(bit_reader, 32)); + // We assume that each cell should take at least 1 byte, even effectively serialized + // Otherwise it means that provided root_count is incorrect + if (root_count < 1 || root_count > decompressed_size) { + return td::Status::Error("BOC decompression failed: invalid root count"); + } + + std::vector root_indexes(root_count); + for (int i = 0; i < root_count; ++i) { + TRY_RESULT_ASSIGN(root_indexes[i], read_uint(bit_reader, 32)); + } + + // Read number of nodes from header + TRY_RESULT(node_count, read_uint(bit_reader, 32)); + if (node_count < 1) { + return td::Status::Error("BOC decompression failed: invalid node count"); + } + + // We assume that each cell should take at least 1 byte, even effectively serialized + // Otherwise it means that provided node_count is incorrect + if (node_count > decompressed_size) { + return td::Status::Error("BOC decompression failed: incorrect node count provided"); + } + + + // Validate root indexes + for (int i = 0; i < root_count; ++i) { + if (root_indexes[i] >= node_count) { + return td::Status::Error("BOC decompression failed: invalid root index"); + } + } + + // Initialize data structures + std::vector cell_data_length(node_count), is_data_small(node_count); + std::vector is_special(node_count), cell_refs_cnt(node_count); + std::vector prunned_branch_level(node_count, 0); + + std::vector cell_builders(node_count); + std::vector> boc_graph(node_count); + + // Read cell metadata + for (int i = 0; i < node_count; ++i) { + // Check enough bits for cell type and refs count + if (bit_reader.size() < 8) { + return td::Status::Error("BOC decompression failed: not enough bits for cell metadata"); + } + + size_t cell_type = bit_reader.bits().get_uint(4); + is_special[i] = bool(cell_type); + if (is_special[i]) { + prunned_branch_level[i] = cell_type - 1; + } + bit_reader.advance(4); + + cell_refs_cnt[i] = bit_reader.bits().get_uint(4); + bit_reader.advance(4); + if (cell_refs_cnt[i] > 4) { + return td::Status::Error("BOC decompression failed: invalid cell refs count"); + } + + if (prunned_branch_level[i]) { + size_t coef = std::bitset<4>(prunned_branch_level[i]).count(); + cell_data_length[i] = (256 + 16) * coef; + } else { + // Check enough bits for data length metadata + if (bit_reader.size() < 8) { + return td::Status::Error("BOC decompression failed: not enough bits for data length"); + } + + is_data_small[i] = bit_reader.bits().get_uint(1); + bit_reader.advance(1); + cell_data_length[i] = bit_reader.bits().get_uint(7); + bit_reader.advance(7); + + if (!is_data_small[i]) { + cell_data_length[i] *= 8; + if (!cell_data_length[i]) { + cell_data_length[i] += 1024; + } + } + } + + // Validate cell data length + if (cell_data_length[i] > kMaxCellDataLengthBits) { + return td::Status::Error("BOC decompression failed: invalid cell data length"); + } + } + + // Read direct edge connections + for (int i = 0; i < node_count; ++i) { + for (int j = 0; j < cell_refs_cnt[i]; ++j) { + TRY_RESULT(edge_connection, read_uint(bit_reader, 1)); + if (edge_connection) { + boc_graph[i][j] = i + 1; + } + } + } + + // Read initial cell data + for (int i = 0; i < node_count; ++i) { + if (prunned_branch_level[i]) { + cell_builders[i].store_long((1 << 8) + prunned_branch_level[i], 16); + } + + size_t remainder_bits = cell_data_length[i] % 8; + if (bit_reader.size() < remainder_bits) { + return td::Status::Error("BOC decompression failed: not enough bits for initial cell data"); + } + cell_builders[i].store_bits(bit_reader.subslice(0, remainder_bits)); + bit_reader.advance(remainder_bits); + cell_data_length[i] -= remainder_bits; + } + + // Decode remaining edge connections + for (size_t i = 0; i < node_count; ++i) { + if (node_count <= i + 3) { + for (int j = 0; j < cell_refs_cnt[i]; ++j) { + if (!boc_graph[i][j]) { + boc_graph[i][j] = i + 2; + } + } + continue; + } + + for (int j = 0; j < cell_refs_cnt[i]; ++j) { + if (!boc_graph[i][j]) { + size_t pref_size = (orig_size - bit_reader.size()); + size_t required_bits = 1 + (31 ^ __builtin_clz(node_count - i - 3)); + + if (required_bits < 8 - (pref_size + 1) % 8 + 1) { + TRY_RESULT_ASSIGN(boc_graph[i][j], read_uint(bit_reader, required_bits)); + boc_graph[i][j] += i + 2; + } else { + TRY_RESULT(edge_connection, read_uint(bit_reader, 1)); + if (edge_connection) { + pref_size = (orig_size - bit_reader.size()); + size_t available_bits = 8 - pref_size % 8; + TRY_RESULT_ASSIGN(boc_graph[i][j], read_uint(bit_reader, available_bits)); + boc_graph[i][j] += i + 2; + } else { + TRY_RESULT_ASSIGN(boc_graph[i][j], read_uint(bit_reader, required_bits)); + boc_graph[i][j] += i + 2; + } + } + } + } + } + + // Check if all graph connections are valid + for (int node = 0; node < node_count; ++node) { + for (int j = 0; j < cell_refs_cnt[node]; ++j) { + size_t child_node = boc_graph[node][j]; + if (child_node >= node_count) { + return td::Status::Error("BOC decompression failed: invalid graph connection"); + } + if (child_node <= node) { + return td::Status::Error("BOC decompression failed: circular reference in graph"); + } + } + } + + // Align to byte boundary + while ((orig_size - bit_reader.size()) % 8) { + TRY_RESULT(bit, read_uint(bit_reader, 1)); + } + + // Read remaining cell data + for (int i = 0; i < node_count; ++i) { + size_t padding_bits = 0; + if (!prunned_branch_level[i] && !is_data_small[i]) { + while (bit_reader.size() > 0 && bit_reader.bits()[0] == 0) { + ++padding_bits; + bit_reader.advance(1); + } + TRY_RESULT(bit, read_uint(bit_reader, 1)); + ++padding_bits; + } + if (cell_data_length[i] < padding_bits) { + return td::Status::Error("BOC decompression failed: invalid cell data length"); + } + size_t remaining_data_bits = cell_data_length[i] - padding_bits; + if (bit_reader.size() < remaining_data_bits) { + return td::Status::Error("BOC decompression failed: not enough bits for remaining cell data"); + } + + cell_builders[i].store_bits(bit_reader.subslice(0, remaining_data_bits)); + bit_reader.advance(remaining_data_bits); + } + + // Build cell tree + std::vector> nodes(node_count); + for (int i = node_count - 1; i >= 0; --i) { + try { + for (int child_index = 0; child_index < cell_refs_cnt[i]; ++child_index) { + size_t child = boc_graph[i][child_index]; + cell_builders[i].store_ref(nodes[child]); + } + try { + nodes[i] = cell_builders[i].finalize(is_special[i]); + } catch (vm::CellBuilder::CellWriteError& e) { + return td::Status::Error("BOC decompression failed: write error while finalizing cell."); + } + } catch (vm::VmError& e) { + return td::Status::Error("BOC decompression failed: VM error during cell construction"); + } + } + + std::vector> root_nodes; + root_nodes.reserve(root_count); + for (size_t index : root_indexes) { + root_nodes.push_back(nodes[index]); + } + + return root_nodes; +} + +td::Result boc_compress(const std::vector>& boc_roots, CompressionAlgorithm algo) { + // Check for empty input + if (boc_roots.empty()) { + return td::Status::Error("Cannot compress empty BOC roots"); + } + + td::BufferSlice compressed; + if (algo == CompressionAlgorithm::BaselineLZ4) { + TRY_RESULT_ASSIGN(compressed, boc_compress_baseline_lz4(boc_roots)); + } else if (algo == CompressionAlgorithm::ImprovedStructureLZ4) { + TRY_RESULT_ASSIGN(compressed, boc_compress_improved_structure_lz4(boc_roots)); + } else { + return td::Status::Error("Unknown compression algorithm"); + } + + td::BufferSlice compressed_with_algo(compressed.size() + 1); + compressed_with_algo.data()[0] = int(algo); + memcpy(compressed_with_algo.data() + 1, compressed.data(), compressed.size()); + return compressed_with_algo; +} + +td::Result>> boc_decompress(td::Slice compressed, int max_decompressed_size) { + if (compressed.size() == 0) { + return td::Status::Error("Can't decompress empty data"); + } + + int algo = int(compressed[0]); + compressed.remove_prefix(1); + + switch (algo) { + case int(CompressionAlgorithm::BaselineLZ4): + return boc_decompress_baseline_lz4(compressed, max_decompressed_size); + case int(CompressionAlgorithm::ImprovedStructureLZ4): + return boc_decompress_improved_structure_lz4(compressed, max_decompressed_size); + } + return td::Status::Error("Unknown compression algorithm"); +} + +} // namespace vm diff --git a/crypto/vm/boc-compression.h b/crypto/vm/boc-compression.h new file mode 100644 index 000000000..f143edd1f --- /dev/null +++ b/crypto/vm/boc-compression.h @@ -0,0 +1,42 @@ +/* +This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2020 Telegram Systems LLP +*/ +#pragma once + +#include "td/utils/Status.h" +#include "td/utils/buffer.h" +#include "vm/cells/CellSlice.h" +#include "vm/cells/CellBuilder.h" +#include "vm/excno.hpp" + +namespace vm { + +constexpr size_t kDecompressedSizeBytes = 4; + +enum class CompressionAlgorithm : int { BaselineLZ4 = 0, ImprovedStructureLZ4 = 1 }; + +td::Result boc_compress_baseline_lz4(const std::vector>& boc_roots); +td::Result>> boc_decompress_baseline_lz4(td::Slice compressed, int max_decompressed_size); + +td::Result boc_compress_improved_structure_lz4(const std::vector>& boc_roots); +td::Result>> boc_decompress_improved_structure_lz4(td::Slice compressed, int max_decompressed_size); + +td::Result boc_compress(const std::vector>& boc_roots, CompressionAlgorithm algo = CompressionAlgorithm::BaselineLZ4); +td::Result>> boc_decompress(td::Slice compressed, int max_decompressed_size); + +} // namespace vm diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 1b8e27365..a6359a987 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -343,6 +343,7 @@ validatorSession.candidateId src:int256 root_hash:int256 file_hash:int256 collat validatorSession.blockUpdate ts:long actions:(vector validatorSession.round.Message) state:int = validatorSession.BlockUpdate; validatorSession.candidate src:int256 round:int root_hash:int256 data:bytes collated_data:bytes = validatorSession.Candidate; validatorSession.compressedCandidate flags:# src:int256 round:int root_hash:int256 decompressed_size:int data:bytes = validatorSession.Candidate; +validatorSession.compressedCandidateV2 flags:# src:int256 round:int root_hash:int256 data:bytes = validatorSession.Candidate; validatorSession.config catchain_idle_timeout:double catchain_max_deps:int round_candidates:int next_candidate_delay:double round_attempt_duration:int max_round_attempts:int max_block_size:int max_collated_data_size:int = validatorSession.Config; @@ -430,6 +431,8 @@ tonNode.blockBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_ha proof:bytes data:bytes = tonNode.Broadcast; tonNode.blockBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int flags:# compressed:bytes = tonNode.Broadcast; +tonNode.blockBroadcastCompressedV2 id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int + signatures:(vector tonNode.blockSignature) flags:# compressed:bytes = tonNode.Broadcast; tonNode.ihrMessageBroadcast message:tonNode.ihrMessage = tonNode.Broadcast; tonNode.externalMessageBroadcast message:tonNode.externalMessage = tonNode.Broadcast; tonNode.newShardBlockBroadcast block:tonNode.newShardBlock = tonNode.Broadcast; @@ -438,6 +441,8 @@ tonNode.newBlockCandidateBroadcast id:tonNode.blockIdExt catchain_seqno:int vali collator_signature:tonNode.blockSignature data:bytes = tonNode.Broadcast; tonNode.newBlockCandidateBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int collator_signature:tonNode.blockSignature flags:# compressed:bytes = tonNode.Broadcast; +tonNode.newBlockCandidateBroadcastCompressedV2 id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int + collator_signature:tonNode.blockSignature flags:# compressed:bytes = tonNode.Broadcast; // optimistic broadcast of response to tonNode.getOutMsgQueueProof with dst_shard, block and limits arguments tonNode.outMsgQueueProofBroadcast dst_shard:tonNode.shardId block:tonNode.blockIdExt @@ -456,6 +461,7 @@ ton.blockIdApprove root_cell_hash:int256 file_hash:int256 = ton.BlockId; tonNode.dataFull id:tonNode.blockIdExt proof:bytes block:bytes is_link:Bool = tonNode.DataFull; tonNode.dataFullCompressed id:tonNode.blockIdExt flags:# compressed:bytes is_link:Bool = tonNode.DataFull; +tonNode.dataFullCompressedV2 id:tonNode.blockIdExt flags:# compressed:bytes is_link:Bool = tonNode.DataFull; tonNode.dataFullEmpty = tonNode.DataFull; tonNode.capabilities#f5bf60c0 version_major:int version_minor:int flags:# = tonNode.Capabilities; @@ -1000,6 +1006,7 @@ validatorStats.collatorNodeResponse self:int256 validator_id:int256 timestamp:do collatorNode.candidate source:PublicKey id:tonNode.blockIdExt data:bytes collated_data:bytes = collatorNode.Candidate; collatorNode.compressedCandidate flags:# source:PublicKey id:tonNode.blockIdExt decompressed_size:int data:bytes = collatorNode.Candidate; +collatorNode.compressedCandidateV2 flags:# source:PublicKey id:tonNode.blockIdExt data:bytes = collatorNode.Candidate; collatorNode.pong flags:# = collatorNode.Pong; collatorNode.error code:int message:string = collatorNode.Error; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 9a5241d8084e1eb2ff75ea71d00b934fec92235b..6475aede86c9bd31385620bc807bacdf14c86cd4 100644 GIT binary patch delta 810 zcmZWnO=uHA7|n+p^P?7H6O0;kDYc}{?z)Ys%^`uK_y?|UDHrB07u1|IQ7kCxj8ZDwY;N@B+*8gy zWP6PsjC+A21I=;4ZWk}Q4jxBJy#G<$$#nSmt{pzteaW|DY(uBA%0NYrFkt9SA6ahe zp7$r+H}8GV{Tx8|b>49;Y$HXSt$~a>dJM@+gLJO0&96M;h{f74{GkB<9fk-)ngCU7 zj6lXmI(ns>Hx{1^;6o!owY&2#u_(bMrWOK$X~-dv8B#-*uH3S?wpB&hVgU03cv){$ zuzF1o+l)(ykf6x4TCGV?`OnqZ-M)1RUllZ1OR#La~`D9^ME9Oe2oN9V8 z<0e7(s;~`fVF>z3?cd(3rMS&jB9+DlDNv=yFY%KwqCt!}K49j8nj3+b0+?eqn?J$%pR}(-)X9mVkNF zB}^H05aJ1@j5QGPHD-(&5}+Ui`GWxnik#VWIBeSvg2cAJF=I>-W&s7~eMr`I6!Ppm}dZHY|q%wcmd+d d={5&}o&eiba*$Dj4`Swm^Nc> deserialize_candi if (!compression_enabled) { return fetch_tl_object(data, true); } - TRY_RESULT(f, fetch_tl_object(data, true)); - if (f->decompressed_size_ > max_decompressed_data_size) { - return td::Status::Error("decompressed size is too big"); - } - TRY_RESULT(p, decompress_candidate_data(f->data_, f->decompressed_size_, proto_version)); - return create_tl_object(f->src_, f->round_, f->root_hash_, std::move(p.first), - std::move(p.second)); + TRY_RESULT(f, fetch_tl_object(data, true)); + td::Result> res; + ton_api::downcast_call(*f, td::overloaded( + [&](ton_api::validatorSession_candidate& c) { + res = td::Status::Error("Received decompressed tl object, while compression_enabled=true"); + }, + [&](ton_api::validatorSession_compressedCandidate& c) { + res = [&]() -> td::Result> { + if (c.decompressed_size_ > max_decompressed_data_size) { + return td::Status::Error("decompressed size is too big"); + } + TRY_RESULT(p, decompress_candidate_data(c.data_, false, c.decompressed_size_, + max_decompressed_data_size, proto_version)); + return create_tl_object(c.src_, c.round_, c.root_hash_, std::move(p.first), + std::move(p.second)); + }(); + }, + [&](ton_api::validatorSession_compressedCandidateV2& c) { + res = [&]() -> td::Result> { + if (c.data_.size() > max_decompressed_data_size) { + return td::Status::Error("Compressed data is too big"); + } + TRY_RESULT(p, decompress_candidate_data(c.data_, true, 0, + max_decompressed_data_size, proto_version)); + return create_tl_object(c.src_, c.round_, c.root_hash_, std::move(p.first), + std::move(p.second)); + }(); + })); + return res; } td::Result compress_candidate_data(td::Slice block, td::Slice collated_data, @@ -69,13 +92,20 @@ td::Result compress_candidate_data(td::Slice block, td::Slice c } td::Result> decompress_candidate_data(td::Slice compressed, + bool improved_compression, int decompressed_size, + int max_decompressed_size, int proto_version) { - TRY_RESULT(decompressed, td::lz4_decompress(compressed, decompressed_size)); - if (decompressed.size() != (size_t)decompressed_size) { - return td::Status::Error("decompressed size mismatch"); + std::vector> roots; + if (!improved_compression) { + TRY_RESULT(decompressed, td::lz4_decompress(compressed, decompressed_size)); + if (decompressed.size() != (size_t)decompressed_size) { + return td::Status::Error("decompressed size mismatch"); + } + TRY_RESULT_ASSIGN(roots, vm::std_boc_deserialize_multi(decompressed)); + } else { + TRY_RESULT_ASSIGN(roots, vm::boc_decompress(compressed, max_decompressed_size)); } - TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed)); if (roots.empty()) { return td::Status::Error("boc is empty"); } @@ -83,7 +113,7 @@ td::Result> decompress_candidate_dat roots.erase(roots.begin()); int collated_data_mode = proto_version >= 5 ? 2 : 31; TRY_RESULT(collated_data, vm::std_boc_serialize_multi(std::move(roots), collated_data_mode)); - LOG(DEBUG) << "Decompressing block candidate: " << compressed.size() << " -> " + LOG(DEBUG) << "Decompressing block candidate " << (improved_compression ? "V2:" : ":") << compressed.size() << " -> " << block_data.size() + collated_data.size(); return std::make_pair(std::move(block_data), std::move(collated_data)); } diff --git a/validator-session/candidate-serializer.h b/validator-session/candidate-serializer.h index 7cc77f0be..9ab53cc89 100644 --- a/validator-session/candidate-serializer.h +++ b/validator-session/candidate-serializer.h @@ -30,7 +30,9 @@ td::Result> deserialize_candi td::Result compress_candidate_data(td::Slice block, td::Slice collated_data, size_t& decompressed_size); td::Result> decompress_candidate_data(td::Slice compressed, + bool improved_compression, int decompressed_size, + int max_decompressed_size, int proto_version); } // namespace ton::validatorsession diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 4b0d7e140..82ca99cd2 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -664,7 +664,21 @@ td::Result CollatorNode::deserialize_candidate(tl_object_ptr td::Result { + TRY_RESULT(p, validatorsession::decompress_candidate_data( + c.data_, true, 0, max_decompressed_data_size, proto_version)); auto collated_data_hash = td::sha256_bits256(p.second); auto key = ton::PublicKey{c.source_}; if (!key.is_ed25519()) { diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index fa2e03b2e..3b4a36c6d 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -35,6 +35,10 @@ void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonN process_block_broadcast(src, query); } +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query) { + process_block_broadcast(src, query); +} + void FullNodeFastSyncOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); if (B.is_error()) { @@ -94,6 +98,11 @@ void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, process_block_candidate_broadcast(src, query); } +void FullNodeFastSyncOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query) { + process_block_candidate_broadcast(src, query); +} + void FullNodeFastSyncOverlay::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { BlockIdExt block_id; CatchainSeqno cc_seqno; diff --git a/validator/full-node-fast-sync-overlays.hpp b/validator/full-node-fast-sync-overlays.hpp index e0db87b7f..41a45cbc7 100644 --- a/validator/full-node-fast-sync-overlays.hpp +++ b/validator/full-node-fast-sync-overlays.hpp @@ -25,6 +25,7 @@ class FullNodeFastSyncOverlay : public td::actor::Actor { public: void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast& query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed& query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2& query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_outMsgQueueProofBroadcast& query); void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast& query); @@ -32,6 +33,7 @@ class FullNodeFastSyncOverlay : public td::actor::Actor { void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast& query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed& query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressedV2& query); void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast& query); void process_telemetry_broadcast(adnl::AdnlNodeIdShort src, diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp index 11c3e9d61..fd5e970ba 100644 --- a/validator/full-node-private-overlay.cpp +++ b/validator/full-node-private-overlay.cpp @@ -34,6 +34,11 @@ void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, process_block_broadcast(src, query); } +void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_blockBroadcastCompressedV2 &query) { + process_block_broadcast(src, query); +} + void FullNodePrivateBlockOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); if (B.is_error()) { @@ -63,6 +68,11 @@ void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, process_block_candidate_broadcast(src, query); } +void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query) { + process_block_candidate_broadcast(src, query); +} + void FullNodePrivateBlockOverlay::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { BlockIdExt block_id; @@ -292,6 +302,10 @@ void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNod process_block_broadcast(src, query); } +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query) { + process_block_broadcast(src, query); +} + void FullNodeCustomOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { if (!block_senders_.count(adnl::AdnlNodeIdShort(src))) { VLOG(FULL_NODE_DEBUG) << "Dropping block broadcast in private overlay \"" << name_ << "\" from unauthorized sender " @@ -330,6 +344,11 @@ void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, process_block_candidate_broadcast(src, query); } +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query) { + process_block_candidate_broadcast(src, query); +} + void FullNodeCustomOverlay::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { if (!block_senders_.count(adnl::AdnlNodeIdShort(src))) { VLOG(FULL_NODE_DEBUG) << "Dropping block candidate broadcast in private overlay \"" << name_ diff --git a/validator/full-node-private-overlay.hpp b/validator/full-node-private-overlay.hpp index 70e196ea6..355e44858 100644 --- a/validator/full-node-private-overlay.hpp +++ b/validator/full-node-private-overlay.hpp @@ -25,12 +25,14 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { public: void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query); void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query); void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_telemetry_broadcast(PublicKeyHash src, const tl_object_ptr& telemetry); @@ -106,12 +108,14 @@ class FullNodeCustomOverlay : public td::actor::Actor { public: void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query); void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query); void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); template diff --git a/validator/full-node-serializer.cpp b/validator/full-node-serializer.cpp index 94dc2155e..5665cc233 100644 --- a/validator/full-node-serializer.cpp +++ b/validator/full-node-serializer.cpp @@ -20,6 +20,7 @@ #include "auto/tl/ton_api.hpp" #include "tl-utils/tl-utils.hpp" #include "vm/boc.h" +#include "vm/boc-compression.h" #include "td/utils/lz4.h" #include "full-node.h" #include "td/utils/overloaded.h" @@ -88,6 +89,28 @@ static td::Result deserialize_block_broadcast(ton_api::tonNode_b std::move(proof)}; } +static td::Result deserialize_block_broadcast(ton_api::tonNode_blockBroadcastCompressedV2& f, + int max_decompressed_size) { + std::vector signatures; + for (auto& sig : f.signatures_) { + signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)}); + } + TRY_RESULT(roots, vm::boc_decompress(f.compressed_, max_decompressed_size)); + if (roots.size() != 2) { + return td::Status::Error("expected 2 roots in boc"); + } + TRY_RESULT(proof, vm::std_boc_serialize(roots[0], 0)); + TRY_RESULT(data, vm::std_boc_serialize(roots[1], 31)); + VLOG(FULL_NODE_DEBUG) << "Decompressing block broadcast: " << f.compressed_.size() << " -> " + << data.size() + proof.size() + signatures.size() * 96; + return BlockBroadcast{create_block_id(f.id_), + std::move(signatures), + static_cast(f.catchain_seqno_), + static_cast(f.validator_set_hash_), + std::move(data), + std::move(proof)}; +} + td::Result deserialize_block_broadcast(ton_api::tonNode_Broadcast& obj, int max_decompressed_data_size) { td::Result B; @@ -96,6 +119,9 @@ td::Result deserialize_block_broadcast(ton_api::tonNode_Broadcas [&](ton_api::tonNode_blockBroadcastCompressed& f) { B = deserialize_block_broadcast(f, max_decompressed_data_size); }, + [&](ton_api::tonNode_blockBroadcastCompressedV2& f) { + B = deserialize_block_broadcast(f, max_decompressed_data_size); + }, [&](auto&) { B = td::Status::Error("unknown broadcast type"); })); return B; } @@ -139,6 +165,20 @@ static td::Status deserialize_block_full(ton_api::tonNode_dataFullCompressed& f, return td::Status::OK(); } +static td::Status deserialize_block_full(ton_api::tonNode_dataFullCompressedV2& f, BlockIdExt& id, td::BufferSlice& proof, + td::BufferSlice& data, bool& is_proof_link, int max_decompressed_size) { + TRY_RESULT(roots, vm::boc_decompress(f.compressed_, max_decompressed_size)); + if (roots.size() != 2) { + return td::Status::Error("expected 2 roots in boc"); + } + TRY_RESULT_ASSIGN(proof, vm::std_boc_serialize(roots[0], 0)); + TRY_RESULT_ASSIGN(data, vm::std_boc_serialize(roots[1], 31)); + VLOG(FULL_NODE_DEBUG) << "Decompressing block full V2: " << f.compressed_.size() << " -> " << data.size() + proof.size(); + id = create_block_id(f.id_); + is_proof_link = f.is_link_; + return td::Status::OK(); +} + td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id, td::BufferSlice& proof, td::BufferSlice& data, bool& is_proof_link, int max_decompressed_data_size) { td::Status S; @@ -148,6 +188,9 @@ td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id [&](ton_api::tonNode_dataFullCompressed& f) { S = deserialize_block_full(f, id, proof, data, is_proof_link, max_decompressed_data_size); }, + [&](ton_api::tonNode_dataFullCompressedV2& f) { + S = deserialize_block_full(f, id, proof, data, is_proof_link, max_decompressed_data_size); + }, [&](auto&) { S = td::Status::Error("unknown data type"); })); return S; } @@ -194,6 +237,24 @@ static td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_newBloc return td::Status::OK(); } +static td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_newBlockCandidateBroadcastCompressedV2& obj, + BlockIdExt& block_id, CatchainSeqno& cc_seqno, + td::uint32& validator_set_hash, td::BufferSlice& data, + int max_decompressed_data_size) { + block_id = create_block_id(obj.id_); + cc_seqno = obj.catchain_seqno_; + validator_set_hash = obj.validator_set_hash_; + TRY_RESULT(roots, vm::boc_decompress(obj.compressed_, max_decompressed_data_size)); + if (roots.size() != 1) { + return td::Status::Error("expected 1 root in boc"); + } + auto root = std::move(roots[0]); + TRY_RESULT_ASSIGN(data, vm::std_boc_serialize(root, 31)); + VLOG(FULL_NODE_DEBUG) << "Decompressing block candidate broadcast V2: " << obj.compressed_.size() << " -> " + << data.size(); + return td::Status::OK(); +} + td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_Broadcast& obj, BlockIdExt& block_id, CatchainSeqno& cc_seqno, td::uint32& validator_set_hash, td::BufferSlice& data, int max_decompressed_data_size) { @@ -207,6 +268,10 @@ td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_Broadcast& obj S = deserialize_block_candidate_broadcast(f, block_id, cc_seqno, validator_set_hash, data, max_decompressed_data_size); }, + [&](ton_api::tonNode_newBlockCandidateBroadcastCompressedV2& f) { + S = deserialize_block_candidate_broadcast(f, block_id, cc_seqno, validator_set_hash, + data, max_decompressed_data_size); + }, [&](auto&) { S = td::Status::Error("unknown data type"); })); return S; } diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index f5a1e0b16..771b28be3 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -804,6 +804,11 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, process_block_candidate_broadcast(src, query); } +void FullNodeShardImpl::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query) { + process_block_candidate_broadcast(src, query); +} + void FullNodeShardImpl::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { BlockIdExt block_id; CatchainSeqno cc_seqno; @@ -832,6 +837,10 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_bl process_block_broadcast(src, query); } +void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query) { + process_block_broadcast(src, query); +} + void FullNodeShardImpl::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); if (B.is_error()) { diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 534693d95..a90460adb 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -152,6 +152,7 @@ class FullNodeShardImpl : public FullNodeShard { void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressedV2 &query); void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_ihrMessageBroadcast &query); @@ -163,6 +164,7 @@ class FullNodeShardImpl : public FullNodeShard { void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressedV2 &query); void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void receive_broadcast(PublicKeyHash src, td::BufferSlice query); From f4468b88f6ed3cfd6562a5d64603b899e2db6388 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 1 Aug 2025 16:48:36 +0300 Subject: [PATCH 362/388] Send broadcasts earlier in AcceptBlock --- validator/impl/accept-block.cpp | 29 +++++++++++++++-------------- validator/impl/accept-block.hpp | 1 + 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index cbb03bac1..b9aea1e7f 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -494,6 +494,17 @@ void AcceptBlockQuery::written_block_signatures() { void AcceptBlockQuery::written_block_info() { VLOG(VALIDATOR_DEBUG) << "written block info"; if (data_.not_null()) { + block_root_ = data_->root_cell(); + if (block_root_.is_null()) { + fatal_error("block data does not contain a root cell"); + return; + } + // generate proof + if (!create_new_proof()) { + fatal_error("cannot generate proof for block "s + id_.to_str()); + return; + } + send_broadcasts(); if (!apply_) { written_state({}); return; @@ -563,17 +574,6 @@ void AcceptBlockQuery::written_state(td::Ref upd_state) { CHECK(data_.not_null()); state_ = std::move(upd_state); - block_root_ = data_->root_cell(); - if (block_root_.is_null()) { - fatal_error("block data does not contain a root cell"); - return; - } - // generate proof - if (!create_new_proof()) { - fatal_error("cannot generate proof for block "s + id_.to_str()); - return; - } - if (apply_ && state_keep_old_hash_ != state_old_hash_) { fatal_error(PSTRING() << "invalid previous state hash in newly-created proof: expected " << state_->root_hash().to_hex() << ", found in update " << state_old_hash_.to_hex()); @@ -935,8 +935,11 @@ void AcceptBlockQuery::written_block_info_2() { } void AcceptBlockQuery::applied() { + finish_query(); +} + +void AcceptBlockQuery::send_broadcasts() { if (send_broadcast_mode_ == 0) { - finish_query(); return; } BlockBroadcast b; @@ -964,8 +967,6 @@ void AcceptBlockQuery::applied() { // td::actor::send_closure(manager_, &ValidatorManager::send_block_candidate_broadcast, id_, // validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), // std::move(b.data), send_broadcast_mode_); - - finish_query(); } } // namespace validator diff --git a/validator/impl/accept-block.hpp b/validator/impl/accept-block.hpp index d8b763ba0..e582650c1 100644 --- a/validator/impl/accept-block.hpp +++ b/validator/impl/accept-block.hpp @@ -89,6 +89,7 @@ class AcceptBlockQuery : public td::actor::Actor { void written_block_next(); void written_block_info_2(); void applied(); + void send_broadcasts(); private: BlockIdExt id_; From 5fa41216b969649658476694ee7bb52465e04ae8 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Mon, 4 Aug 2025 18:27:05 +0400 Subject: [PATCH 363/388] Tunnel small refactor --- adnl/adnl-network-manager.cpp | 2 +- tdnet/td/net/UdpServer.cpp | 6 ++--- validator-engine/validator-engine.cpp | 36 ++++++++++++++++----------- validator-engine/validator-engine.hpp | 1 + 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/adnl/adnl-network-manager.cpp b/adnl/adnl-network-manager.cpp index 99a74eb3b..6d748e34a 100644 --- a/adnl/adnl-network-manager.cpp +++ b/adnl/adnl-network-manager.cpp @@ -76,7 +76,7 @@ size_t AdnlNetworkManagerImpl::add_listening_udp_port(td::uint16 port) { return idx; } -#define TUNNEL_FAKE_PORT 1 +constexpr int TUNNEL_FAKE_PORT = 0; size_t AdnlNetworkManagerImpl::add_tunnel_udp_port(std::string global_config, std::string tunnel_config, td::Promise on_ready, td::actor::Scheduler *scheduler) { diff --git a/tdnet/td/net/UdpServer.cpp b/tdnet/td/net/UdpServer.cpp index 8d05a2bb8..cae5c4d51 100644 --- a/tdnet/td/net/UdpServer.cpp +++ b/tdnet/td/net/UdpServer.cpp @@ -35,9 +35,9 @@ int VERBOSITY_NAME(udp_server) = VERBOSITY_NAME(DEBUG) + 10; } namespace detail { -#define TUNNEL_BUFFER_SZ_PACKETS 100 -#define TUNNEL_MAX_PACKET_MTU 1500 -#define TUNNEL_ALARM_EVERY 0.01 +constexpr int TUNNEL_BUFFER_SZ_PACKETS = 100; +constexpr int TUNNEL_MAX_PACKET_MTU = 1500; +constexpr double TUNNEL_ALARM_EVERY = 0.01; class UdpServerTunnelImpl : public UdpServer { public: diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 8645d2280..87e454d2a 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -2094,31 +2094,24 @@ void ValidatorEngine::start_adnl() { class Handler : public ton::adnl::AdnlNetworkManager::TunnelEventsHandler { public: - Handler(ValidatorEngine *scheduler) - : validator_engine_(scheduler) { + Handler(ValidatorEngine *scheduler, const td::actor::ActorId &actor) + : validator_engine_(scheduler), validator_engine_actor_(actor) { } private: ValidatorEngine *validator_engine_; - void on_in_addr_update(td::IPAddress ip) override { - validator_engine_->scheduler_->run_in_context_external([&] { - LOG(INFO) << "[EVENT] Tunnel reinitialized, addr: " << ip; + td::actor::ActorId validator_engine_actor_; - validator_engine_->addr_lists_.clear(); - validator_engine_->add_addr(Config::Addr{}, Config::AddrCats{ - .in_addr = ip, - .is_tunnel = true, - .cats = {0, 1, 2, 3}, - }); + void on_in_addr_update(td::IPAddress ip) override { + LOG(INFO) << "[EVENT] Tunnel reinitialized, addr: " << ip; - for (auto &adnl : validator_engine_->config_.adnl_ids) { - validator_engine_->add_adnl(adnl.first, adnl.second); - } + validator_engine_->scheduler_->run_in_context_external([&] { + td::actor::send_closure(validator_engine_actor_, &ValidatorEngine::reinit_tunnel, ip); }); } }; - td::actor::send_closure(adnl_network_manager_, &ton::adnl::AdnlNetworkManager::install_tunnel_events_handler, std::make_unique(this)); + td::actor::send_closure(adnl_network_manager_, &ton::adnl::AdnlNetworkManager::install_tunnel_events_handler, std::make_unique(this, actor_id(this))); ton::adnl::AdnlCategoryMask cat_mask; for (int i = 0; i <= 3; i++) { @@ -2142,6 +2135,19 @@ void ValidatorEngine::start_adnl() { started_adnl(); } +void ValidatorEngine::reinit_tunnel(td::IPAddress ip) { + this->addr_lists_.clear(); + this->add_addr(Config::Addr{}, Config::AddrCats{ + .in_addr = ip, + .is_tunnel = true, + .cats = {0, 1, 2, 3}, + }); + + for (auto &adnl : this->config_.adnl_ids) { + this->add_adnl(adnl.first, adnl.second); + } +} + void ValidatorEngine::add_addr(const Config::Addr &addr, const Config::AddrCats &cats) { ton::adnl::AdnlCategoryMask cat_mask; for (auto cat : cats.cats) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 156806891..1ea4a372f 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -213,6 +213,7 @@ class ValidatorEngine : public td::actor::Actor { void got_state(td::Ref state); void write_config(td::Promise promise); + void reinit_tunnel(td::IPAddress ip); std::map addr_lists_; std::map prio_addr_lists_; From 5f25056c8ff59469f93e0dab396ed62603e92a4b Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Mon, 4 Aug 2025 19:09:40 +0400 Subject: [PATCH 364/388] Pass scheduler to tunnel handler instead of engine --- validator-engine/validator-engine.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 87e454d2a..cdeb5d917 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -2094,24 +2094,24 @@ void ValidatorEngine::start_adnl() { class Handler : public ton::adnl::AdnlNetworkManager::TunnelEventsHandler { public: - Handler(ValidatorEngine *scheduler, const td::actor::ActorId &actor) - : validator_engine_(scheduler), validator_engine_actor_(actor) { + Handler(td::actor::Scheduler *scheduler, const td::actor::ActorId &actor) + : scheduler_(scheduler), validator_engine_actor_(actor) { } private: - ValidatorEngine *validator_engine_; + td::actor::Scheduler* scheduler_; td::actor::ActorId validator_engine_actor_; void on_in_addr_update(td::IPAddress ip) override { LOG(INFO) << "[EVENT] Tunnel reinitialized, addr: " << ip; - validator_engine_->scheduler_->run_in_context_external([&] { + scheduler_->run_in_context_external([&] { td::actor::send_closure(validator_engine_actor_, &ValidatorEngine::reinit_tunnel, ip); }); } }; - td::actor::send_closure(adnl_network_manager_, &ton::adnl::AdnlNetworkManager::install_tunnel_events_handler, std::make_unique(this, actor_id(this))); + td::actor::send_closure(adnl_network_manager_, &ton::adnl::AdnlNetworkManager::install_tunnel_events_handler, std::make_unique(this->scheduler_, actor_id(this))); ton::adnl::AdnlCategoryMask cat_mask; for (int i = 0; i <= 3; i++) { From d2cc797857bdfafaeeb5363b9414a48e02d208c2 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 4 Aug 2025 18:28:58 +0300 Subject: [PATCH 365/388] Optimistic block generation --- tdutils/td/utils/Timer.h | 3 + tl/generate/scheme/ton_api.tl | 3 +- tl/generate/scheme/ton_api.tlo | Bin 118872 -> 119192 bytes validator-session/validator-session.cpp | 200 ++++++++++++++++---- validator-session/validator-session.h | 3 + validator-session/validator-session.hpp | 35 ++++ validator/collation-manager.cpp | 45 ++++- validator/collation-manager.hpp | 6 + validator/collator-node.cpp | 27 +-- validator/fabric.h | 28 ++- validator/impl/collator-impl.h | 22 ++- validator/impl/collator.cpp | 231 ++++++++++++++++------- validator/impl/fabric.cpp | 35 +--- validator/interfaces/validator-manager.h | 7 +- validator/manager-disk.cpp | 9 +- validator/manager-hardfork.cpp | 4 +- validator/validator-group.cpp | 120 ++++++++++-- validator/validator-group.hpp | 23 +++ 18 files changed, 606 insertions(+), 195 deletions(-) diff --git a/tdutils/td/utils/Timer.h b/tdutils/td/utils/Timer.h index d1ba104b0..de793aec0 100644 --- a/tdutils/td/utils/Timer.h +++ b/tdutils/td/utils/Timer.h @@ -160,6 +160,9 @@ class PerfLog { }; template double PerfLogAction::finish(const T &result) { + if (!perf_log_) { + return 0.0; + } if (result.is_ok()) { return perf_log_->finish_action(i_, td::Status::OK()); } else { diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index a6359a987..38dc33489 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -344,6 +344,7 @@ validatorSession.blockUpdate ts:long actions:(vector validatorSession.round.Mess validatorSession.candidate src:int256 round:int root_hash:int256 data:bytes collated_data:bytes = validatorSession.Candidate; validatorSession.compressedCandidate flags:# src:int256 round:int root_hash:int256 decompressed_size:int data:bytes = validatorSession.Candidate; validatorSession.compressedCandidateV2 flags:# src:int256 round:int root_hash:int256 data:bytes = validatorSession.Candidate; +validatorSession.optimisticCandidateBroadcast flags:# prev_candidate_id:int256 data:bytes = validatorSession.OptimisticCandidateBroadcast; validatorSession.config catchain_idle_timeout:double catchain_max_deps:int round_candidates:int next_candidate_delay:double round_attempt_duration:int max_round_attempts:int max_block_size:int max_collated_data_size:int = validatorSession.Config; @@ -962,7 +963,7 @@ validatorStats.blockStats old_out_msg_queue_size:long new_out_msg_queue_size:long msg_queue_cleaned:int neighbors:(vector validatorStats.blockStats.neighborStats) = validatorStats.BlockStats; validatorStats.collateWorkTimeStats - total:double queue_cleanup:double prelim_storage_stat:double trx_tvm:double trx_storage_stat:double + total:double optimistic_apply:double queue_cleanup:double prelim_storage_stat:double trx_tvm:double trx_storage_stat:double trx_other:double final_storage_stat:double create_block:double create_collated_data:double create_block_candidate:double = validatorStats.CollateWorkTimeStats; validatorStats.storageStatCacheStats diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 6475aede86c9bd31385620bc807bacdf14c86cd4..3b64f7517b1ad5fa9ef9bf964ec3917078c2d098 100644 GIT binary patch delta 273 zcmcbyfPKbdcHT#`^{p77z;h$-Z(D`5L+lQ^Wr;bNDTyWdMZu}X#hLkedj16^nYo$8 zC7H?76ZIH{CpQaAZsxFCz{oP|uJ4b{YwR auto to_approve = real_state_->choose_blocks_to_approve(description(), local_idx()); for (auto &block : to_approve) { auto id = SentBlock::get_block_id(block); - if (approved_.count(id) && approved_[id].first <= td::Clocks::system()) { + if (approved_.contains(id) && approved_[id].first <= td::Clocks::system()) { msgs.emplace_back(create_tl_object( cur_round_, id, approved_[id].second.clone())); cnt++; @@ -223,6 +222,19 @@ bool ValidatorSessionImpl::ensure_candidate_unique(td::uint32 src_idx, td::uint3 void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice data, td::optional expected_id, bool is_overlay_broadcast, bool is_startup) { + bool is_optimistic = false; + ValidatorSessionCandidateId optimistic_prev_candidate = ValidatorSessionCandidateId::zero(); + if (is_overlay_broadcast) { + auto R = fetch_tl_object(data, true); + if (R.is_ok()) { + if (src == local_id()) { + return; + } + is_optimistic = true; + optimistic_prev_candidate = R.ok_ref()->prev_candidate_id_; + data = std::move(R.ok_ref()->data_); + } + } // Note: src is not necessarily equal to the sender of this message: // If requested using get_broadcast_p2p, src is the creator of the block, sender possibly is some other node. auto src_idx = description().get_source_idx(src); @@ -262,13 +274,59 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice return; } + if ((td::int32)block_round < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK || + block_round >= cur_round_ + MAX_FUTURE_ROUND_BLOCK || (block_round == 0 && is_optimistic)) { + VLOG(VALIDATOR_SESSION_NOTICE) << this << "[node " << src << "][broadcast " << block_id + << "]: bad round=" << block_round << " cur_round" << cur_round_; + return; + } + + BroadcastInfo broadcast_info{.candidate_id = block_id, + .received_at = td::Clocks::system(), + .deserialize_time = deserialize_time, + .serialized_size = data.size(), + .file_hash = file_hash, + .collated_data_hash = collated_data_file_hash}; + + if (is_optimistic) { + if (optimistic_broadcasts_.contains({block_round, src_idx})) { + VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice()) + << "]: duplicate optimistic broadcast for round " << block_round; + return; + } + if (block_round > cur_round_) { + OptimisticBroadcast &optimistic_broadcast = optimistic_broadcasts_[{block_round, src_idx}]; + optimistic_broadcast.candidate = std::move(candidate); + optimistic_broadcast.prev_candidate_id = optimistic_prev_candidate; + optimistic_broadcast.broadcast_info = broadcast_info; + VLOG(VALIDATOR_SESSION_WARNING) << this << ": received optimistic broadcast " << block_id << " from " << src + << ", round " << block_round; + return; + } + if (SentBlock::get_block_id(real_state_->get_committed_block(description(), block_round - 1)) != + optimistic_prev_candidate) { + VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice()) + << "]: dropping optimistic broadcast for round " << block_round + << " - prev candidate mismatch"; + return; + } + } + process_received_block(block_round, src, src_idx, std::move(candidate), broadcast_info, is_overlay_broadcast, + is_startup); +} + +void ValidatorSessionImpl::process_received_block(td::uint32 block_round, PublicKeyHash src, td::uint32 src_idx, + tl_object_ptr candidate, + const BroadcastInfo &info, bool is_overlay_broadcast, + bool is_startup) { + ValidatorSessionCandidateId block_id = info.candidate_id; auto stat = stats_get_candidate_stat(block_round, src, block_id); if (stat) { if (stat->block_status == ValidatorSessionStats::status_none) { stat->block_status = ValidatorSessionStats::status_received; } if (stat->got_block_at <= 0.0) { - stat->got_block_at = td::Clocks::system(); + stat->got_block_at = info.received_at; if (is_overlay_broadcast) { stat->got_block_by = ValidatorSessionStats::recv_broadcast; } else if (is_startup) { @@ -277,19 +335,13 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice stat->got_block_by = ValidatorSessionStats::recv_query; } } - stat->deserialize_time = deserialize_time; - stat->serialized_size = data.size(); + stat->deserialize_time = info.deserialize_time; + stat->serialized_size = info.serialized_size; stat->block_id.root_hash = candidate->root_hash_; - stat->block_id.file_hash = file_hash; - stat->collated_data_hash = collated_data_file_hash; + stat->block_id.file_hash = info.file_hash; + stat->collated_data_hash = info.collated_data_hash; } - if ((td::int32)block_round < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK || - block_round >= cur_round_ + MAX_FUTURE_ROUND_BLOCK) { - VLOG(VALIDATOR_SESSION_NOTICE) << this << "[node " << src << "][broadcast " << block_id - << "]: bad round=" << block_round << " cur_round" << cur_round_; - return; - } auto it = blocks_.find(block_id); if (it != blocks_.end()) { it->second->round_ = std::max(it->second->round_, block_round); @@ -315,10 +367,10 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice return; } - CHECK(!pending_approve_.count(block_id)); - CHECK(!approved_.count(block_id)); - CHECK(!pending_reject_.count(block_id)); - CHECK(!rejected_.count(block_id)); + CHECK(!pending_approve_.contains(block_id)); + CHECK(!approved_.contains(block_id)); + CHECK(!pending_reject_.contains(block_id)); + CHECK(!rejected_.contains(block_id)); auto v = virtual_state_->choose_blocks_to_approve(description(), local_idx()); for (auto &b : v) { @@ -487,19 +539,15 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, GeneratedCandidate if (round != cur_round_) { return; } - td::Timer serialize_timer; - auto b = create_tl_object(local_id().tl(), round, c.candidate.id.root_hash, - std::move(c.candidate.data), - std::move(c.candidate.collated_data)); - auto B = serialize_candidate(b, compress_block_candidates_).move_as_ok(); + SentCandidateStats &send_stats = send_candidate_broadcast(round, c.candidate); if (stat) { - stat->serialize_time = serialize_timer.elapsed(); - stat->serialized_size = B.size(); + stat->serialize_time = send_stats.serialize_time; + stat->serialized_size = send_stats.serialized_size; } - td::actor::send_closure(catchain_, &catchain::CatChain::send_broadcast, std::move(B)); - - blocks_.emplace(block_id, std::move(b)); + blocks_[block_id] = create_tl_object( + local_id().tl(), round, c.candidate.id.root_hash, std::move(c.candidate.data), + std::move(c.candidate.collated_data)); pending_generate_ = false; generated_ = true; generated_block_ = block_id; @@ -507,6 +555,31 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, GeneratedCandidate request_new_block(true); } +ValidatorSessionImpl::SentCandidateStats &ValidatorSessionImpl::send_candidate_broadcast( + td::uint32 round, const BlockCandidate &candidate, + td::optional optimistic_prev_candidate) { + ValidatorSessionCandidateId candidate_id = description().candidate_id( + local_idx(), candidate.id.root_hash, candidate.id.file_hash, candidate.collated_file_hash); + SentCandidateStats &stats = sent_candidates_[candidate_id]; + if (stats.sent) { + return stats; + } + stats.sent = true; + td::Timer serialize_timer; + auto b = create_tl_object( + local_id().tl(), round, candidate.id.root_hash, candidate.data.clone(), candidate.collated_data.clone()); + auto data = serialize_candidate(b, compress_block_candidates_).move_as_ok(); + stats.serialize_time = serialize_timer.elapsed(); + stats.sent_at = td::Clocks::system(); + stats.serialized_size = data.size(); + if (optimistic_prev_candidate) { + data = create_serialize_tl_object( + 0, optimistic_prev_candidate.value(), std::move(data)); + } + td::actor::send_closure(catchain_, &catchain::CatChain::send_broadcast, std::move(data)); + return stats; +} + void ValidatorSessionImpl::signed_block(td::uint32 round, ValidatorSessionCandidateId hash, td::BufferSlice signature) { if (round != cur_round_) { return; @@ -586,7 +659,7 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { return; } } - if (pending_approve_.count(block_id) || rejected_.count(block_id)) { + if (pending_approve_.contains(block_id) || rejected_.contains(block_id)) { return; } @@ -651,7 +724,7 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { description().get_node_priority(block->get_src_idx(), cur_round_)}}, B->root_hash_, B->data_.clone(), B->collated_data_.clone(), std::move(P)); } else if (T.is_in_past()) { - if (!active_requests_.count(block_id)) { + if (!active_requests_.contains(block_id)) { auto v = virtual_state_->get_block_approvers(description(), block_id); if (v.size() > 0) { auto id = description().get_source_id(v[td::Random::fast(0, static_cast(v.size() - 1))]); @@ -796,9 +869,9 @@ void ValidatorSessionImpl::check_all() { for (auto &B : to_approve) { if (B) { auto block_id = SentBlock::get_block_id(B); - auto pending = pending_approve_.count(block_id) == 1; - auto rejected = rejected_.count(block_id) == 1; - auto accepted = approved_.count(block_id) == 1; + auto pending = pending_approve_.contains(block_id); + auto rejected = rejected_.contains(block_id); + auto accepted = approved_.contains(block_id); sb << " " << block_id << " pending: " << pending << " rejected: " << rejected << " accepted: " << accepted << "\n"; } else { @@ -960,6 +1033,18 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) { round_started_at_ = td::Timestamp::now(); round_debug_at_ = td::Timestamp::in(60.0); + for (auto it = optimistic_broadcasts_.begin(); it != optimistic_broadcasts_.end() && it->first.first <= cur_round_;) { + OptimisticBroadcast &optimistic_broadcast = it->second; + auto [block_round, src_idx] = it->first; + if (SentBlock::get_block_id(real_state_->get_committed_block(description(), block_round - 1)) == + optimistic_broadcast.prev_candidate_id) { + process_received_block(block_round, description().get_source_id(src_idx), src_idx, + std::move(optimistic_broadcast.candidate), optimistic_broadcast.broadcast_info, true, + false); + } + it = optimistic_broadcasts_.erase(it); + } + check_all(); } @@ -1010,6 +1095,7 @@ ValidatorSessionImpl::ValidatorSessionImpl(catchain::CatChainSessionId session_i , overlay_manager_(overlays) , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) { compress_block_candidates_ = opts.proto_version >= 4; + allow_optimistic_generation_ = opts.proto_version >= 5; description_ = ValidatorSessionDescription::create(std::move(opts), nodes, local_id); src_round_candidate_.resize(description_->get_total_nodes()); } @@ -1200,7 +1286,7 @@ ValidatorSessionStats::Producer *ValidatorSessionImpl::stats_get_candidate_stat( auto it2 = stats_pending_approve_.find({round, it->candidate_id}); if (it2 != stats_pending_approve_.end()) { for (td::uint32 node_id : it2->second) { - it->set_approved_by(node_id, description().get_node_weight(node_id), description().get_total_weight()); + process_approve(node_id, round, it->candidate_id); } stats_pending_approve_.erase(it2); } @@ -1249,13 +1335,7 @@ void ValidatorSessionImpl::stats_process_action(td::uint32 node_id, ton_api::val if (obj.candidate_ == skip_round_candidate_id()) { return; } - auto stat = stats_get_candidate_stat_by_id(obj.round_, obj.candidate_); - if (stat) { - stat->set_approved_by(node_id, description().get_node_weight(node_id), - description().get_total_weight()); - } else { - stats_pending_approve_[{obj.round_, obj.candidate_}].push_back(node_id); - } + process_approve(node_id, obj.round_, obj.candidate_); }, [&](const ton_api::validatorSession_message_commit &obj) { if (obj.candidate_ == skip_round_candidate_id()) { @@ -1272,6 +1352,46 @@ void ValidatorSessionImpl::stats_process_action(td::uint32 node_id, ton_api::val [](const auto &) {})); } +void ValidatorSessionImpl::process_approve(td::uint32 node_id, td::uint32 round, + ValidatorSessionCandidateId candidate_id) { + auto stat = stats_get_candidate_stat_by_id(round, candidate_id); + if (!stat) { + stats_pending_approve_[{round, candidate_id}].push_back(node_id); + return; + } + + bool was_approved_66pct = stat->approved_66pct_at > 0.0; + stat->set_approved_by(node_id, description().get_node_weight(node_id), description().get_total_weight()); + bool is_approved_66pct = stat->approved_66pct_at > 0.0; + + if (allow_optimistic_generation_ && !was_approved_66pct && is_approved_66pct && cur_round_ == round && + description().get_node_priority(local_idx(), round + 1) == 0 && blocks_.contains(candidate_id)) { + auto &block = blocks_[candidate_id]; + if (cur_round_ == first_block_round_ && + description().get_node_priority(description().get_source_idx(PublicKeyHash{block->src_}), cur_round_) == 0) { + callback_->generate_block_optimistic(BlockSourceInfo{description().get_source_public_key(local_idx()), + BlockCandidatePriority{round + 1, round + 1, 0}}, + block->data_.clone(), block->root_hash_, stat->block_id.file_hash, + [=, SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + return; + } + td::actor::send_closure( + SelfId, &ValidatorSessionImpl::generated_optimistic_candidate, + round + 1, R.move_as_ok(), candidate_id); + }); + } + } +} + +void ValidatorSessionImpl::generated_optimistic_candidate(td::uint32 round, GeneratedCandidate candidate, + ValidatorSessionCandidateId prev_candidate) { + if (cur_round_ > round) { + return; + } + send_candidate_broadcast(round, candidate.candidate, prev_candidate); +} + td::actor::ActorOwn ValidatorSession::create( catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, diff --git a/validator-session/validator-session.h b/validator-session/validator-session.h index 641fb4866..5e226f357 100644 --- a/validator-session/validator-session.h +++ b/validator-session/validator-session.h @@ -93,6 +93,9 @@ class ValidatorSession : public td::actor::Actor { ValidatorSessionFileHash file_hash, ValidatorSessionCollatedDataFileHash collated_data_file_hash, td::Promise promise) = 0; + virtual void generate_block_optimistic(BlockSourceInfo source_info, td::BufferSlice prev_block, RootHash prev_root_hash, + FileHash prev_file_hash, td::Promise promise) { + } virtual ~Callback() = default; }; diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 04fdc875c..ef0e3fba8 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -161,6 +161,7 @@ class ValidatorSessionImpl : public ValidatorSession { bool catchain_started_ = false; bool allow_unsafe_self_blocks_resync_; bool compress_block_candidates_ = false; + bool allow_optimistic_generation_ = false; ValidatorSessionStats cur_stats_; bool stats_inited_ = false; @@ -168,6 +169,30 @@ class ValidatorSessionImpl : public ValidatorSession { stats_pending_approve_; // round, candidate_id -> approvers std::map, std::vector> stats_pending_sign_; // round, candidate_id -> signers + + struct SentCandidateStats { + bool sent = false; + double sent_at = -1.0; + double serialize_time = -1.0; + size_t serialized_size = 0; + }; + std::map sent_candidates_; + + struct BroadcastInfo { + ValidatorSessionCandidateId candidate_id; + double received_at = -1.0; + double deserialize_time = -1.0; + size_t serialized_size = 0; + ValidatorSessionFileHash file_hash; + ValidatorSessionCollatedDataFileHash collated_data_hash; + }; + struct OptimisticBroadcast { + tl_object_ptr candidate; + ValidatorSessionCandidateId prev_candidate_id; + BroadcastInfo broadcast_info; + }; + std::map, OptimisticBroadcast> optimistic_broadcasts_; // round, src -> broadcast + void stats_init(); void stats_add_round(); ValidatorSessionStats::Producer *stats_get_candidate_stat( @@ -176,6 +201,10 @@ class ValidatorSessionImpl : public ValidatorSession { ValidatorSessionStats::Producer *stats_get_candidate_stat_by_id(td::uint32 round, ValidatorSessionCandidateId candidate_id); void stats_process_action(td::uint32 node_id, ton_api::validatorSession_round_Message &action); + void process_approve(td::uint32 node_id, td::uint32 round, ValidatorSessionCandidateId candidate_id); + + void generated_optimistic_candidate(td::uint32 round, GeneratedCandidate candidate, + ValidatorSessionCandidateId prev_candidate); public: ValidatorSessionImpl(catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, @@ -205,6 +234,9 @@ class ValidatorSessionImpl : public ValidatorSession { bool ensure_candidate_unique(td::uint32 src_idx, td::uint32 round, ValidatorSessionCandidateId block_id); void process_broadcast(PublicKeyHash src, td::BufferSlice data, td::optional expected_id, bool is_overlay_broadcast, bool is_startup); + void process_received_block(td::uint32 block_round, PublicKeyHash src, td::uint32 src_idx, + tl_object_ptr candidate, const BroadcastInfo &info, + bool is_overlay_broadcast, bool is_startup); void process_message(PublicKeyHash src, td::BufferSlice data); void process_query(PublicKeyHash src, td::BufferSlice data, td::Promise promise); @@ -218,6 +250,9 @@ class ValidatorSessionImpl : public ValidatorSession { td::BufferSlice signature); void generated_block(td::uint32 round, GeneratedCandidate c, double collation_time); + SentCandidateStats &send_candidate_broadcast( + td::uint32 round, const BlockCandidate &candidate, + td::optional optimistic_prev_candidate = {}); void signed_block(td::uint32 round, ValidatorSessionCandidateId hash, td::BufferSlice signature); void end_request(td::uint32 round, ValidatorSessionCandidateId block_id) { diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index 853ac37ae..9eadd3f0a 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -38,11 +38,15 @@ void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_mastercha td::Promise promise, int proto_version) { if (shard.is_masterchain()) { run_collate_query( - shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), - opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), promise.wrap([](BlockCandidate&& candidate) { + CollateParams{.shard = shard, + .min_masterchain_block_id = min_masterchain_block_id, + .prev = std::move(prev), + .creator = creator, + .validator_set = std::move(validator_set), + .collator_opts = opts_->get_collator_options()}, + manager_, td::Timestamp::in(10.0), std::move(cancellation_token), promise.wrap([](BlockCandidate&& candidate) { return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true}; - }), - adnl::AdnlNodeIdShort::zero(), std::move(cancellation_token), 0); + })); return; } collate_shard_block(shard, min_masterchain_block_id, std::move(prev), creator, priority, std::move(validator_set), @@ -50,6 +54,27 @@ void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_mastercha proto_version); } +void CollationManager::collate_next_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, + BlockIdExt prev_block_id, td::BufferSlice prev_block, + Ed25519_PublicKey creator, BlockCandidatePriority priority, + td::Ref validator_set, td::uint64 max_answer_size, + td::CancellationToken cancellation_token, + td::Promise promise, int proto_version) { + TRY_RESULT_PROMISE(promise, prev_block_data, create_block(prev_block_id, std::move(prev_block))); + run_collate_query( + CollateParams{.shard = shard, + .min_masterchain_block_id = min_masterchain_block_id, + .prev = {prev_block_id}, + .creator = creator, + .validator_set = std::move(validator_set), + .collator_opts = opts_->get_collator_options(), + .optimistic_prev_block_ = std::move(prev_block_data)}, + manager_, td::Timestamp::in(10.0), std::move(cancellation_token), promise.wrap([](BlockCandidate&& candidate) { + return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true}; + })); + // TODO: request to collator node +} + void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, Ed25519_PublicKey creator, BlockCandidatePriority priority, td::Ref validator_set, @@ -109,11 +134,15 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas if (selected_collator.is_zero() && s->self_collate) { run_collate_query( - shard, min_masterchain_block_id, std::move(prev), creator, std::move(validator_set), - opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), promise.wrap([](BlockCandidate&& candidate) { + CollateParams{.shard = shard, + .min_masterchain_block_id = min_masterchain_block_id, + .prev = std::move(prev), + .creator = creator, + .validator_set = std::move(validator_set), + .collator_opts = opts_->get_collator_options()}, + manager_, td::Timestamp::in(10.0), std::move(cancellation_token), promise.wrap([](BlockCandidate&& candidate) { return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true}; - }), - adnl::AdnlNodeIdShort::zero(), std::move(cancellation_token), 0); + })); return; } diff --git a/validator/collation-manager.hpp b/validator/collation-manager.hpp index 5b8d2b7a0..a64d43289 100644 --- a/validator/collation-manager.hpp +++ b/validator/collation-manager.hpp @@ -39,6 +39,12 @@ class CollationManager : public td::actor::Actor { td::uint64 max_answer_size, td::CancellationToken cancellation_token, td::Promise promise, int proto_version); + void collate_next_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, BlockIdExt prev_block_id, + td::BufferSlice prev_block, Ed25519_PublicKey creator, BlockCandidatePriority priority, + td::Ref validator_set, td::uint64 max_answer_size, + td::CancellationToken cancellation_token, td::Promise promise, + int proto_version); + void update_options(td::Ref opts); void validator_group_started(ShardIdFull shard); diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 82ca99cd2..ba3ff01eb 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -539,18 +539,21 @@ void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std }; cache_entry->started = true; cache_entry->block_seqno = block_seqno; - run_collate_query( - shard, last_masterchain_state_->get_block_id(), std::move(prev_blocks), Ed25519_PublicKey{td::Bits256::zero()}, - last_masterchain_state_->get_validator_set(shard), opts_->get_collator_options(), manager_, timeout, - [=, SelfId = actor_id(this), timer = td::Timer{}](td::Result R) { - FLOG(INFO) { - prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority); - sb << timer.elapsed() << ": " << (R.is_ok() ? "OK" : R.error().to_string()); - }; - td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R)); - }, - local_id_, cache_entry->cancellation_token_source.get_cancellation_token(), - CollateMode::skip_store_candidate | CollateMode::from_collator_node); + run_collate_query(CollateParams{.shard = shard, + .min_masterchain_block_id = last_masterchain_state_->get_block_id(), + .prev = std::move(prev_blocks), + .validator_set = last_masterchain_state_->get_validator_set(shard), + .collator_opts = opts_->get_collator_options(), + .collator_node_id = local_id_, + .skip_store_candidate = true}, + manager_, timeout, cache_entry->cancellation_token_source.get_cancellation_token(), + [=, SelfId = actor_id(this), timer = td::Timer{}](td::Result R) { + FLOG(INFO) { + prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority); + sb << timer.elapsed() << ": " << (R.is_ok() ? "OK" : R.error().to_string()); + }; + td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R)); + }); } void CollatorNode::process_result(std::shared_ptr cache_entry, td::Result R) { diff --git a/validator/fabric.h b/validator/fabric.h index 19319f36e..825c3b248 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -26,8 +26,23 @@ namespace ton { namespace validator { +struct CollateParams { + ShardIdFull shard; + BlockIdExt min_masterchain_block_id; + std::vector prev; + bool is_hardfork = false; + Ed25519_PublicKey creator{td::Bits256::zero()}; + td::Ref validator_set = {}; + td::Ref collator_opts = {}; + adnl::AdnlNodeIdShort collator_node_id = adnl::AdnlNodeIdShort::zero(); + bool skip_store_candidate = false; + int attempt_idx = 0; + + // Optional - used for optimistic collation + Ref optimistic_prev_block_ = {}; +}; + enum ValidateMode { fake = 1 }; -enum CollateMode { skip_store_candidate = 1, from_collator_node = 2 }; td::actor::ActorOwn create_db_actor(td::actor::ActorId manager, std::string db_root_, td::Ref opts); @@ -85,15 +100,8 @@ void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, BlockCandidate candidate, td::Ref validator_set, PublicKeyHash local_validator_id, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, unsigned mode = 0); -void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, - Ed25519_PublicKey creator, td::Ref validator_set, - td::Ref collator_opts, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise promise, - adnl::AdnlNodeIdShort collator_node_id, td::CancellationToken cancellation_token, unsigned mode, - int attempt_idx = 0); -void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise); +void run_collate_query(CollateParams params, td::actor::ActorId manager, td::Timestamp timeout, + td::CancellationToken cancellation_token, td::Promise promise); void run_liteserver_query(td::BufferSlice data, td::actor::ActorId manager, td::actor::ActorId cache, td::Promise promise); void run_fetch_account_state(WorkchainId wc, StdSmcAddress addr, td::actor::ActorId manager, diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 008001157..f9e424479 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -33,6 +33,7 @@ #include #include #include "common/global-version.h" +#include "fabric.h" namespace ton { @@ -80,7 +81,8 @@ class Collator final : public td::actor::Actor { td::Timestamp queue_cleanup_timeout_, soft_timeout_, medium_timeout_; td::Promise main_promise; adnl::AdnlNodeIdShort collator_node_id_ = adnl::AdnlNodeIdShort::zero(); - unsigned mode_ = 0; + bool skip_store_candidate_ = false; + Ref optimistic_prev_block_; int attempt_idx_; bool allow_repeat_collation_ = false; ton::BlockSeqno last_block_seqno{0}; @@ -95,11 +97,8 @@ class Collator final : public td::actor::Actor { static constexpr bool shard_splitting_enabled = true; public: - Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector prev, - Ref validator_set, Ed25519_PublicKey collator_id, Ref collator_opts, - td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, - adnl::AdnlNodeIdShort collator_node_id, td::CancellationToken cancellation_token, unsigned mode, - int attempt_idx); + Collator(CollateParams params, td::actor::ActorId manager, td::Timestamp timeout, + td::CancellationToken cancellation_token, td::Promise promise); ~Collator() override = default; bool is_busy() const { return busy_; @@ -122,6 +121,8 @@ class Collator final : public td::actor::Actor { private: void start_up() override; + void load_prev_states_blocks(); + void process_optimistic_prev_block(); void alarm() override; int verbosity{3 * 0}; int verify{1}; @@ -276,6 +277,7 @@ class Collator final : public td::actor::Actor { void after_get_shard_blocks(td::Result>> res, td::PerfLogAction token); void after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res, td::PerfLogAction token); + void after_get_shard_state_optimistic(td::Result> res, td::PerfLogAction token); bool preprocess_prev_mc_state(); bool register_mc_state(Ref other_mc_state); bool request_aux_mc_state(BlockSeqno seqno, Ref& state); @@ -326,6 +328,14 @@ class Collator final : public td::actor::Actor { bool is_masterchain() const { return shard_.is_masterchain(); } + int prev_block_idx(const BlockIdExt& id) const { + for (size_t i = 0; i < prev_blocks.size(); ++i) { + if (prev_blocks[i] == id) { + return i; + } + } + return -1; + } bool is_our_address(Ref addr_ref) const; bool is_our_address(ton::AccountIdPrefixFull addr_prefix) const; bool is_our_address(const ton::StdSmcAddress& addr) const; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 5d0cbd2d6..3ded80b6d 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -58,33 +58,21 @@ static constexpr int MAX_ATTEMPTS = 5; /** * Constructs a Collator object. * - * @param shard The shard of the new block. - * @param is_hardfork A boolean indicating whether the new block is a hardfork. - * @param min_masterchain_block_id The the minimum reference masterchain block. - * @param prev A vector of BlockIdExt representing the previous blocks. - * @param validator_set A reference to the ValidatorSet. - * @param collator_id The public key of the block creator. - * @param collator_opts A reference to CollatorOptions. + * @param params Collator parameters * @param manager The ActorId of the ValidatorManager. * @param timeout The timeout for the collator. - * @param promise The promise to return the result. - * @param collator_node_id ADNL id of the collator node that generates the block (zero if it's not a collator node) * @param cancellation_token Token to cancel collation. - * @param mode +1 - skip storing candidate to disk, +2 - called from CollatorNode. - * @param attempt_idx The index of the attempt, starting from 0. On later attempts collator decreases block limits and skips some steps. - */ -Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, - std::vector prev, td::Ref validator_set, Ed25519_PublicKey collator_id, - Ref collator_opts, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise promise, adnl::AdnlNodeIdShort collator_node_id, - td::CancellationToken cancellation_token, unsigned mode, int attempt_idx) - : shard_(shard) - , is_hardfork_(is_hardfork) - , min_mc_block_id{min_masterchain_block_id} - , prev_blocks(std::move(prev)) - , created_by_(collator_id) - , collator_opts_(collator_opts) - , validator_set_(std::move(validator_set)) + * @param promise The promise to return the result. + */ +Collator::Collator(CollateParams params, td::actor::ActorId manager, td::Timestamp timeout, + td::CancellationToken cancellation_token, td::Promise promise) + : shard_(params.shard) + , is_hardfork_(params.is_hardfork) + , min_mc_block_id{params.min_masterchain_block_id} + , prev_blocks(std::move(params.prev)) + , created_by_(params.creator) + , collator_opts_(params.collator_opts.is_null() ? td::Ref{true} : params.collator_opts) + , validator_set_(std::move(params.validator_set)) , manager(manager) , timeout(timeout) // default timeout is 10 seconds, declared in validator/validator-group.cpp:generate_block_candidate:run_collate_query @@ -92,9 +80,10 @@ Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_mastercha , soft_timeout_(td::Timestamp::at(timeout.at() - 3.0)) , medium_timeout_(td::Timestamp::at(timeout.at() - 1.5)) , main_promise(std::move(promise)) - , collator_node_id_(collator_node_id) - , mode_(mode) - , attempt_idx_(attempt_idx) + , collator_node_id_(params.collator_node_id) + , skip_store_candidate_(params.skip_store_candidate) + , optimistic_prev_block_(std::move(params.optimistic_prev_block_)) + , attempt_idx_(params.attempt_idx) , perf_timer_("collate", 0.1, [manager](double duration) { send_closure(manager, &ValidatorManager::add_perf_timer_stat, "collate", duration); @@ -204,6 +193,16 @@ void Collator::start_up() { return; } } + if (optimistic_prev_block_.not_null()) { + if (prev_blocks.size() != 1) { + fatal_error(-666, "optimistic prev block is not null, which is not allowed after merge"); + return; + } + if (prev_blocks[0] != optimistic_prev_block_->block_id()) { + fatal_error(-666, "optimistic prev block is not null, but has invalid block id"); + return; + } + } busy_ = true; step = 1; if (!is_masterchain()) { @@ -239,32 +238,10 @@ void Collator::start_up() { // 3. load previous block(s) and corresponding state(s) prev_states.resize(prev_blocks.size()); prev_block_data.resize(prev_blocks.size()); - for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { - // 3.1. load state - LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; - ++pending; - auto token = perf_log_.start_action(PSTRING() << "wait_block_state #" << i); - td::actor::send_closure_later( - manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), timeout, - [self = get_self(), i, token = std::move(token)](td::Result> res) mutable { - LOG(DEBUG) << "got answer to wait_block_state query #" << i; - td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_state, i, std::move(res), - std::move(token)); - }); - if (prev_blocks[i].seqno()) { - // 3.2. load block - // NB: we need the block itself only for extracting start_lt and end_lt to create correct prev_blk:ExtBlkRef and related Merkle proofs - LOG(DEBUG) << "sending wait_block_data() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; - ++pending; - auto token = perf_log_.start_action(PSTRING() << "wait_block_data #" << i); - td::actor::send_closure_later( - manager, &ValidatorManager::wait_block_data_short, prev_blocks[i], priority(), timeout, - [self = get_self(), i, token = std::move(token)](td::Result> res) mutable { - LOG(DEBUG) << "got answer to wait_block_data query #" << i; - td::actor::send_closure_later(std::move(self), &Collator::after_get_block_data, i, std::move(res), - std::move(token)); - }); - } + if (optimistic_prev_block_.is_null()) { + load_prev_states_blocks(); + } else { + process_optimistic_prev_block(); } if (is_hardfork_) { LOG(WARNING) << "generating a hardfork block"; @@ -312,6 +289,86 @@ void Collator::start_up() { CHECK(pending); } +/** + * Load previous states and blocks from DB + */ +void Collator::load_prev_states_blocks() { + for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { + // 3.1. load state + LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; + ++pending; + auto token = perf_log_.start_action(PSTRING() << "wait_block_state #" << i); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), timeout, + [self = get_self(), i, token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state query #" << i; + td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_state, i, std::move(res), + std::move(token)); + }); + if (prev_blocks[i].seqno()) { + // 3.2. load block + LOG(DEBUG) << "sending wait_block_data() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; + ++pending; + auto token = perf_log_.start_action(PSTRING() << "wait_block_data #" << i); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_data_short, prev_blocks[i], priority(), timeout, + [self = get_self(), i, token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_data query #" << i; + td::actor::send_closure_later(std::move(self), &Collator::after_get_block_data, i, std::move(res), + std::move(token)); + }); + } + } +} + +/** + * Write optimistic prev block as block data, load previous state to apply Merkle update to it + */ +void Collator::process_optimistic_prev_block() { + std::vector prev_prev; + BlockIdExt mc_blkid; + bool after_split; + auto S = block::unpack_block_prev_blk_try(optimistic_prev_block_->root_cell(), optimistic_prev_block_->block_id(), + prev_prev, mc_blkid, after_split); + if (S.is_error()) { + fatal_error(S.move_as_error_prefix("failed to unpack optimistic prev block: ")); + return; + } + // 3.1. load state + if (prev_prev.size() == 1) { + LOG(DEBUG) << "sending wait_block_state() query for " << prev_prev[0].to_str() << " to Manager (opt)"; + ++pending; + auto token = perf_log_.start_action("opt wait_block_state"); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_short, prev_prev[0], priority(), timeout, + [self = get_self(), token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state query (opt)"; + td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_state_optimistic, std::move(res), + std::move(token)); + }); + } else { + CHECK(prev_prev.size() == 2); + LOG(DEBUG) << "sending wait_block_state_merge() query for " << prev_prev[0].to_str() << " and " + << prev_prev[1].to_str() << " to Manager (opt)"; + ++pending; + auto token = perf_log_.start_action("opt wait_block_state_merge"); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_merge, prev_prev[0], prev_prev[1], priority(), timeout, + [self = get_self(), token = std::move(token)](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state_merge query (opt)"; + td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_state_optimistic, std::move(res), + std::move(token)); + }); + } + // 3.2. load block + LOG(DEBUG) << "use optimistic prev block " << prev_blocks[0].to_str(); + ++pending; + auto token = perf_log_.start_action(PSTRING() << "opt wait_block_data"); + td::actor::send_closure_later(actor_id(this), &Collator::after_get_block_data, 0, optimistic_prev_block_, + std::move(token)); +} + + /** * Raises an error when timeout is reached. */ @@ -377,9 +434,18 @@ bool Collator::fatal_error(td::Status error) { if (allow_repeat_collation_ && error.code() != ErrorCode::cancelled && attempt_idx_ + 1 < MAX_ATTEMPTS && !is_hardfork_ && !timeout.is_in_past()) { LOG(WARNING) << "Repeating collation (attempt #" << attempt_idx_ + 1 << ")"; - run_collate_query(shard_, min_mc_block_id, prev_blocks, created_by_, validator_set_, collator_opts_, manager, - td::Timestamp::in(10.0), std::move(main_promise), collator_node_id_, - std::move(cancellation_token_), mode_, attempt_idx_ + 1); + run_collate_query(CollateParams{.shard = shard_, + .min_masterchain_block_id = min_mc_block_id, + .prev = prev_blocks, + .is_hardfork = false, + .creator = created_by_, + .validator_set = validator_set_, + .collator_opts = collator_opts_, + .collator_node_id = collator_node_id_, + .skip_store_candidate = skip_store_candidate_, + .attempt_idx = attempt_idx_ + 1, + .optimistic_prev_block_ = optimistic_prev_block_}, + manager, td::Timestamp::in(10.0), std::move(cancellation_token_), std::move(main_promise)); } else { LOG(INFO) << "collation failed in " << perf_timer_.elapsed() << " s " << error; LOG(INFO) << perf_log_; @@ -678,7 +744,7 @@ void Collator::after_get_shard_state(int idx, td::Result> res, t * Callback function called after retrieving block data for a previous block. * * @param idx The index of the previous block (0 or 1). - * @param res The retreved block data. + * @param res The retrieved block data. */ void Collator::after_get_block_data(int idx, td::Result> res, td::PerfLogAction token) { LOG(DEBUG) << "in Collator::after_get_block_data(" << idx << ")"; @@ -757,6 +823,31 @@ void Collator::after_get_storage_stat_cache(td::Result> res, td::PerfLogAction token) { + LOG(DEBUG) << "in Collator::after_get_shard_state_optimistic()"; + token.finish(res); + if (res.is_error()) { + fatal_error(res.move_as_error()); + return; + } + td::RealCpuTimer timer; + work_timer_.resume(); + auto state = res.move_as_ok(); + auto S = state.write().apply_block(optimistic_prev_block_->block_id(), optimistic_prev_block_); + if (S.is_error()) { + fatal_error(S.move_as_error_prefix("apply error: ")); + return; + } + work_timer_.pause(); + stats_.work_time.optimistic_apply = timer.elapsed_both(); + after_get_shard_state(0, std::move(state), {}); +} + /** * Unpacks the last masterchain state and initializes the Collator object with the extracted configuration. * @@ -896,7 +987,9 @@ bool Collator::request_neighbor_msg_queues() { unsigned i = 0; for (block::McShardDescr& descr : neighbors_) { LOG(DEBUG) << "neighbor #" << i << " : " << descr.blk_.to_str(); - top_blocks.push_back(descr.blk_); + if (prev_block_idx(descr.blk_) == -1) { + top_blocks.push_back(descr.blk_); + } ++i; } ++pending; @@ -935,8 +1028,7 @@ bool Collator::request_out_msg_queue_size() { /** * Handles the result of obtaining the outbound queue for a neighbor. * - * @param i The index of the neighbor. - * @param res The obtained outbound queue. + * @param R The result of retrieving neighbor message queues (top block id -> queue). */ void Collator::got_neighbor_msg_queues(td::Result>> R, td::PerfLogAction token) { @@ -952,12 +1044,17 @@ void Collator::got_neighbor_msg_queues(td::Result= 0) { + got_neighbor_msg_queue( + i, Ref{true, descr.blk_, prev_states[prev_idx]->root_cell(), td::Ref{}, true}); + } else { + auto it = res.find(descr.blk_); + if (it == res.end()) { + fatal_error(PSTRING() << "no msg queue from neighbor #" << i); + return; + } + got_neighbor_msg_queue(i, it->second); } - got_neighbor_msg_queue(i, it->second); ++i; } check_pending(); @@ -6393,7 +6490,7 @@ bool Collator::create_block_candidate() { << consensus_config.max_collated_data_size << ")"); } // 4. save block candidate - if (mode_ & CollateMode::skip_store_candidate) { + if (skip_store_candidate_) { td::actor::send_closure_later(actor_id(this), &Collator::return_block_candidate, td::Unit()); } else { LOG(INFO) << "saving new BlockCandidate"; @@ -6579,7 +6676,7 @@ void Collator::finalize_stats() { stats_.cc_seqno = validator_set_.not_null() ? validator_set_->get_catchain_seqno() : 0; stats_.collated_at = td::Clocks::system(); stats_.attempt = attempt_idx_; - stats_.is_validator = !(mode_ & CollateMode::from_collator_node); + stats_.is_validator = collator_node_id_.is_zero(); stats_.self = stats_.is_validator ? PublicKey(pubkeys::Ed25519(created_by_)).compute_short_id() : collator_node_id_.pubkey_hash(); if (block_limit_status_) { diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index 2f6ddf1eb..f38f77320 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -212,39 +212,18 @@ void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, .release(); } -void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, - Ed25519_PublicKey creator, td::Ref validator_set, - td::Ref collator_opts, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise promise, - adnl::AdnlNodeIdShort collator_node_id, td::CancellationToken cancellation_token, unsigned mode, - int attempt_idx) { +void run_collate_query(CollateParams params, td::actor::ActorId manager, td::Timestamp timeout, + td::CancellationToken cancellation_token, td::Promise promise) { BlockSeqno seqno = 0; - for (auto& p : prev) { - if (p.seqno() > seqno) { - seqno = p.seqno(); - } - } - td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1) - << (attempt_idx ? "_" + td::to_string(attempt_idx) : ""), - shard, false, min_masterchain_block_id, std::move(prev), std::move(validator_set), - creator, std::move(collator_opts), std::move(manager), timeout, std::move(promise), - collator_node_id, std::move(cancellation_token), mode, attempt_idx) - .release(); -} - -void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise) { - BlockSeqno seqno = 0; - for (auto& p : prev) { + for (auto& p : params.prev) { if (p.seqno() > seqno) { seqno = p.seqno(); } } - td::actor::create_actor( - PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, min_masterchain_block_id, - std::move(prev), td::Ref{}, Ed25519_PublicKey{Bits256::zero()}, td::Ref{true}, - std::move(manager), timeout, std::move(promise), adnl::AdnlNodeIdShort::zero(), td::CancellationToken{}, 0, 0) + td::actor::create_actor(PSTRING() << "collate" << params.shard.to_str() << ":" << (seqno + 1) + << (params.attempt_idx ? "_" + td::to_string(params.attempt_idx) : ""), + std::move(params), std::move(manager), timeout, std::move(cancellation_token), + std::move(promise)) .release(); } diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 78248610a..90e11a35b 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -117,6 +117,7 @@ struct CollationStats { struct WorkTimeStats { td::RealCpuTimer::Time total; + td::RealCpuTimer::Time optimistic_apply; td::RealCpuTimer::Time queue_cleanup; td::RealCpuTimer::Time prelim_storage_stat; td::RealCpuTimer::Time trx_tvm; @@ -129,9 +130,9 @@ struct CollationStats { tl_object_ptr tl(bool is_cpu) const { return create_tl_object( - total.get(is_cpu), queue_cleanup.get(is_cpu), prelim_storage_stat.get(is_cpu), trx_tvm.get(is_cpu), - trx_storage_stat.get(is_cpu), trx_other.get(is_cpu), final_storage_stat.get(is_cpu), create_block.get(is_cpu), - create_collated_data.get(is_cpu), create_block_candidate.get(is_cpu)); + total.get(is_cpu), optimistic_apply.get(is_cpu), queue_cleanup.get(is_cpu), prelim_storage_stat.get(is_cpu), + trx_tvm.get(is_cpu), trx_storage_stat.get(is_cpu), trx_other.get(is_cpu), final_storage_stat.get(is_cpu), + create_block.get(is_cpu), create_collated_data.get(is_cpu), create_block_candidate.get(is_cpu)); } }; WorkTimeStats work_time; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 15271aedb..a4124904e 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -128,9 +128,12 @@ void ValidatorManagerImpl::sync_complete(td::Promise promise) { } Ed25519_PublicKey created_by{td::Bits256::zero()}; td::as(created_by.as_bits256().data() + 32 - 4) = ((unsigned)std::time(nullptr) >> 8); - run_collate_query(shard_id, last_masterchain_block_id_, prev, created_by, val_set, td::Ref{true}, - actor_id(this), td::Timestamp::in(10.0), std::move(P), adnl::AdnlNodeIdShort::zero(), - td::CancellationToken{}, 0); + run_collate_query(CollateParams{.shard = shard_id, + .min_masterchain_block_id = last_masterchain_block_id_, + .prev = prev, + .creator = created_by, + .validator_set = val_set}, + actor_id(this), td::Timestamp::in(10.0), {}, std::move(P)); } void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vector prev, BlockIdExt last, diff --git a/validator/manager-hardfork.cpp b/validator/manager-hardfork.cpp index 3798803bb..de0390ece 100644 --- a/validator/manager-hardfork.cpp +++ b/validator/manager-hardfork.cpp @@ -55,7 +55,9 @@ void ValidatorManagerImpl::sync_complete(td::Promise promise) { }); LOG(ERROR) << "running collate query"; - run_collate_hardfork(shard_id, block_id, prev, actor_id(this), td::Timestamp::in(10.0), std::move(P)); + run_collate_query( + CollateParams{.shard = shard_id, .min_masterchain_block_id = block_id, .prev = prev, .is_hardfork = true}, + actor_id(this), td::Timestamp::in(10.0), {}, std::move(P)); } void ValidatorManagerImpl::created_candidate(BlockCandidate candidate) { diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index d2db0ae99..6a6148a1b 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -23,7 +23,6 @@ #include "td/utils/overloaded.h" #include "common/delay.h" #include "ton/lite-tl.hpp" -#include "ton/ton-tl.hpp" #include "td/utils/Random.h" #include "collator-node.hpp" @@ -43,9 +42,7 @@ void ValidatorGroup::generate_block_candidate(validatorsession::BlockSourceInfo return; } td::uint32 round_id = source_info.priority.round; - if (round_id > last_known_round_id_) { - last_known_round_id_ = round_id; - } + update_round_id(round_id); if (!started_) { promise.set_error(td::Status::Error(ErrorCode::notready, "cannot collate block: group not started")); return; @@ -70,11 +67,38 @@ void ValidatorGroup::generate_block_candidate(validatorsession::BlockSourceInfo td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_candidate, source_info, std::move(cache), std::move(R)); }; + + if (optimistic_generation_ && prev_block_ids_.size() == 1 && optimistic_generation_->prev == prev_block_ids_[0] && + optimistic_generation_->round == round_id) { + if (optimistic_generation_->result) { + P.set_value(optimistic_generation_->result.value().clone()); + } else { + optimistic_generation_->promises.push_back( + [=, SelfId = actor_id(this), P = std::move(P), + cancellation_token = + cancellation_token_source_.get_cancellation_token()](td::Result R) mutable { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ValidatorGroup::generate_block_candidate_cont, source_info, std::move(P), + std::move(cancellation_token)); + } else { + P.set_value(R.move_as_ok()); + } + }); + } + return; + } + generate_block_candidate_cont(source_info, std::move(P), cancellation_token_source_.get_cancellation_token()); +} + +void ValidatorGroup::generate_block_candidate_cont(validatorsession::BlockSourceInfo source_info, + td::Promise promise, + td::CancellationToken cancellation_token) { + TRY_STATUS_PROMISE(promise, cancellation_token.check()); td::uint64 max_answer_size = config_.max_block_size + config_.max_collated_data_size + 1024; td::actor::send_closure(collation_manager_, &CollationManager::collate_block, shard_, min_masterchain_block_id_, prev_block_ids_, Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, - source_info.priority, validator_set_, max_answer_size, - cancellation_token_source_.get_cancellation_token(), std::move(P), config_.proto_version); + source_info.priority, validator_set_, max_answer_size, std::move(cancellation_token), + std::move(promise), config_.proto_version); } void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo source_info, @@ -108,9 +132,7 @@ void ValidatorGroup::validate_block_candidate(validatorsession::BlockSourceInfo return; } td::uint32 round_id = source_info.priority.round; - if (round_id > last_known_round_id_) { - last_known_round_id_ = round_id; - } + update_round_id(round_id); if (round_id < last_known_round_id_) { promise.set_error(td::Status::Error(ErrorCode::notready, "too old")); return; @@ -179,9 +201,7 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so td::Promise promise) { stats.cc_seqno = validator_set_->get_catchain_seqno(); td::uint32 round_id = source_info.priority.round; - if (round_id >= last_known_round_id_) { - last_known_round_id_ = round_id + 1; - } + update_round_id(round_id + 1); auto sig_set = create_signature_set(std::move(signatures)); validator_set_->check_signatures(root_hash, file_hash, sig_set).ensure(); auto approve_sig_set = create_signature_set(std::move(approve_signatures)); @@ -236,6 +256,10 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so cached_collated_block_ = nullptr; approved_candidates_cache_.clear(); cancellation_token_source_.cancel(); + if (optimistic_generation_ && optimistic_generation_->round == last_known_round_id_ && + optimistic_generation_->prev != next_block_id) { + optimistic_generation_ = {}; + } } void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref block, std::vector prev, @@ -262,9 +286,7 @@ void ValidatorGroup::accept_block_query(BlockIdExt block_id, td::Ref } void ValidatorGroup::skip_round(td::uint32 round_id) { - if (round_id >= last_known_round_id_) { - last_known_round_id_ = round_id + 1; - } + update_round_id(round_id + 1); } void ValidatorGroup::get_approved_candidate(PublicKey source, RootHash root_hash, FileHash file_hash, @@ -275,6 +297,66 @@ void ValidatorGroup::get_approved_candidate(PublicKey source, RootHash root_hash std::move(promise)); } +void ValidatorGroup::generate_block_optimistic(validatorsession::BlockSourceInfo source_info, + td::BufferSlice prev_block, RootHash prev_root_hash, + FileHash prev_file_hash, td::Promise promise) { + if (destroying_) { + return; + } + if (last_known_round_id_ + 1 != source_info.priority.round) { + return; + } + if (optimistic_generation_ && optimistic_generation_->round >= source_info.priority.round) { + return; + } + BlockIdExt block_id{create_next_block_id_simple(), prev_root_hash, prev_file_hash}; + optimistic_generation_ = std::make_unique(); + optimistic_generation_->round = source_info.priority.round; + optimistic_generation_->prev = BlockIdExt{create_next_block_id_simple(), prev_root_hash, prev_file_hash}; + optimistic_generation_->promises.push_back(std::move(promise)); + + td::Promise P = [=, SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_optimistic, source_info, std::move(R)); + }; + LOG(WARNING) << "Optimistically generating next block after " << block_id.to_str(); + td::uint64 max_answer_size = config_.max_block_size + config_.max_collated_data_size + 1024; + td::actor::send_closure(collation_manager_, &CollationManager::collate_next_block, shard_, min_masterchain_block_id_, + block_id, std::move(prev_block), Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, + source_info.priority, validator_set_, max_answer_size, + optimistic_generation_->cancellation_token_source.get_cancellation_token(), std::move(P), + config_.proto_version); +} + +void ValidatorGroup::generated_block_optimistic(validatorsession::BlockSourceInfo source_info, + td::Result R) { + if (!optimistic_generation_ || optimistic_generation_->round != source_info.priority.round) { + return; + } + if (R.is_error()) { + LOG(WARNING) << "Optimistic generation failed: " << R.move_as_error(); + for (auto &promise : optimistic_generation_->promises) { + promise.set_error(R.error().clone()); + } + optimistic_generation_ = {}; + return; + } + optimistic_generation_->result = R.move_as_ok(); + for (auto &promise : optimistic_generation_->promises) { + promise.set_result(optimistic_generation_->result.value().clone()); + } + optimistic_generation_->promises.clear(); +} + +void ValidatorGroup::update_round_id(td::uint32 round) { + if (last_known_round_id_ >= round) { + return; + } + last_known_round_id_ = round; + if (optimistic_generation_ && optimistic_generation_->round < round) { + optimistic_generation_ = {}; + } +} + BlockIdExt ValidatorGroup::create_next_block_id(RootHash root_hash, FileHash file_hash) const { return BlockIdExt{create_next_block_id_simple(), root_hash, file_hash}; } @@ -352,6 +434,12 @@ std::unique_ptr ValidatorGroup::ma td::actor::send_closure(id_, &ValidatorGroup::get_approved_candidate, source, root_hash, file_hash, collated_data_file_hash, std::move(promise)); } + void generate_block_optimistic(validatorsession::BlockSourceInfo source_info, td::BufferSlice prev_block, + RootHash prev_root_hash, FileHash prev_file_hash, + td::Promise promise) override { + td::actor::send_closure(id_, &ValidatorGroup::generate_block_optimistic, source_info, std::move(prev_block), + prev_root_hash, prev_file_hash, std::move(promise)); + } private: td::actor::ActorId id_; @@ -534,7 +622,7 @@ void ValidatorGroup::get_validator_group_info_for_litequery_cont( BlockIdExt id{next_block_id, candidate->id_->block_id_->root_hash_, candidate->id_->block_id_->file_hash_}; candidate->id_->block_id_ = create_tl_lite_block_id(id); candidate->available_ = - available_block_candidates_.count({candidate->id_->creator_, id, candidate->id_->collated_data_hash_}); + available_block_candidates_.contains({candidate->id_->creator_, id, candidate->id_->collated_data_hash_}); } auto result = create_tl_object(); diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index a7456cf9f..17bd23a4d 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -37,6 +37,8 @@ class ValidatorManager; class ValidatorGroup : public td::actor::Actor { public: void generate_block_candidate(validatorsession::BlockSourceInfo source_info, td::Promise promise); + void generate_block_candidate_cont(validatorsession::BlockSourceInfo source_info, + td::Promise promise, td::CancellationToken cancellation_token); void validate_block_candidate(validatorsession::BlockSourceInfo source_info, BlockCandidate block, td::Promise> promise); void accept_block_candidate(validatorsession::BlockSourceInfo source_info, td::BufferSlice block, RootHash root_hash, @@ -52,6 +54,10 @@ class ValidatorGroup : public td::actor::Actor { BlockIdExt create_next_block_id(RootHash root_hash, FileHash file_hash) const; BlockId create_next_block_id_simple() const; + void generate_block_optimistic(validatorsession::BlockSourceInfo source_info, td::BufferSlice prev_block, + RootHash prev_root_hash, FileHash prev_file_hash, td::Promise promise); + void generated_block_optimistic(validatorsession::BlockSourceInfo source_info, td::Result R); + void start(std::vector prev, BlockIdExt min_masterchain_block_id); void create_session(); void destroy(); @@ -156,6 +162,8 @@ class ValidatorGroup : public td::actor::Actor { std::shared_ptr cached_collated_block_; td::CancellationTokenSource cancellation_token_source_; + void update_round_id(td::uint32 round); + void generated_block_candidate(validatorsession::BlockSourceInfo source_info, std::shared_ptr cache, td::Result R); @@ -181,6 +189,21 @@ class ValidatorGroup : public td::actor::Actor { std::set sent_candidate_broadcasts_; void send_block_candidate_broadcast(BlockIdExt id, td::BufferSlice data); + + struct OptimisticGeneration { + td::uint32 round = 0; + BlockIdExt prev; + td::optional result; + td::CancellationTokenSource cancellation_token_source; + std::vector> promises; + + ~OptimisticGeneration() { + for (auto& promise : promises) { + promise.set_error(td::Status::Error(ErrorCode::cancelled, "Cancelled")); + } + } + }; + std::unique_ptr optimistic_generation_; }; } // namespace validator From dd130963520fb5e27691916b903d2dcf82fad29e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 4 Aug 2025 18:46:31 +0300 Subject: [PATCH 366/388] Don't wait for storing state to celldb in most cases --- validator/apply-block.cpp | 2 +- .../downloaders/wait-block-state-merge.cpp | 4 +- validator/downloaders/wait-block-state.cpp | 47 +++++--- validator/downloaders/wait-block-state.hpp | 9 +- validator/impl/accept-block.cpp | 2 +- validator/impl/check-proof.cpp | 2 +- validator/impl/collator.cpp | 6 +- validator/impl/out-msg-queue-proof.cpp | 2 +- validator/impl/validate-query.cpp | 6 +- validator/manager-disk.cpp | 17 +-- validator/manager-disk.hpp | 4 +- validator/manager-hardfork.cpp | 17 +-- validator/manager-hardfork.hpp | 4 +- validator/manager-init.cpp | 2 +- validator/manager.cpp | 104 +++++++++++------- validator/manager.hpp | 44 +++++--- validator/queue-size-counter.cpp | 6 +- validator/shard-block-retainer.cpp | 2 +- validator/shard-client.cpp | 6 +- validator/validator.h | 4 +- 20 files changed, 174 insertions(+), 116 deletions(-) diff --git a/validator/apply-block.cpp b/validator/apply-block.cpp index a35b74ce8..3c6a931cd 100644 --- a/validator/apply-block.cpp +++ b/validator/apply-block.cpp @@ -166,7 +166,7 @@ void ApplyBlock::written_block_data() { }); td::actor::send_closure(manager_, &ValidatorManager::wait_block_state, handle_, apply_block_priority(), timeout_, - std::move(P)); + true, std::move(P)); } } diff --git a/validator/downloaders/wait-block-state-merge.cpp b/validator/downloaders/wait-block-state-merge.cpp index 2b1961610..c5f1cfa7f 100644 --- a/validator/downloaders/wait-block-state-merge.cpp +++ b/validator/downloaders/wait-block-state-merge.cpp @@ -55,7 +55,7 @@ void WaitBlockStateMerge::start_up() { } }); - td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, left_, priority_, timeout_, + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, left_, priority_, timeout_, false, std::move(P_l)); auto P_r = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { @@ -66,7 +66,7 @@ void WaitBlockStateMerge::start_up() { } }); - td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, right_, priority_, timeout_, + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, right_, priority_, timeout_, false, std::move(P_r)); } diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index 92a0720bc..7ca2bbd2e 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -32,7 +32,11 @@ void WaitBlockState::alarm() { } void WaitBlockState::abort_query(td::Status reason) { - if (promise_) { + if (promise_no_store_) { + promise_no_store_.set_error( + reason.move_as_error_prefix(PSTRING() << "failed to download state " << handle_->id() << ": ")); + } + if (promise_final_) { if (priority_ > 0 || (reason.code() != ErrorCode::timeout && reason.code() != ErrorCode::notready)) { LOG(WARNING) << "aborting wait block state query for " << handle_->id() << " priority=" << priority_ << ": " << reason; @@ -40,18 +44,19 @@ void WaitBlockState::abort_query(td::Status reason) { LOG(DEBUG) << "aborting wait block state query for " << handle_->id() << " priority=" << priority_ << ": " << reason; } - promise_.set_error(reason.move_as_error_prefix(PSTRING() << "failed to download state " << handle_->id() << ": ")); + promise_final_.set_error( + reason.move_as_error_prefix(PSTRING() << "failed to download state " << handle_->id() << ": ")); } stop(); } void WaitBlockState::finish_query() { CHECK(handle_->received_state()); - /*if (handle_->id().is_masterchain() && handle_->inited_proof()) { - td::actor::send_closure(manager_, &ValidatorManager::new_block, handle_, prev_state_, [](td::Unit) {}); - }*/ - if (promise_) { - promise_.set_result(prev_state_); + if (promise_no_store_) { + promise_no_store_.set_result(prev_state_); + } + if (promise_final_) { + promise_final_.set_result(prev_state_); } stop(); } @@ -275,10 +280,16 @@ void WaitBlockState::got_block_data(td::Ref data) { } void WaitBlockState::apply() { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &WaitBlockState::abort_query, R.move_as_error_prefix("db set error: ")); + } else { + td::actor::send_closure(SelfId, &WaitBlockState::written_state, R.move_as_ok()); + } + }); + if (opts_->get_permanent_celldb()) { - td::actor::send_closure(manager_, &ValidatorManager::set_block_state_from_data, handle_, block_, - std::move(promise_)); - stop(); + td::actor::send_closure(manager_, &ValidatorManager::set_block_state_from_data, handle_, block_, std::move(P)); return; } TD_PERF_COUNTER(apply_block_to_state); @@ -289,15 +300,11 @@ void WaitBlockState::apply() { return; } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &WaitBlockState::abort_query, R.move_as_error_prefix("db set error: ")); - } else { - td::actor::send_closure(SelfId, &WaitBlockState::written_state, R.move_as_ok()); - } - }); - td::actor::send_closure(manager_, &ValidatorManager::set_block_state, handle_, prev_state_, std::move(P)); + if (promise_no_store_) { + promise_no_store_.set_result(prev_state_); + promise_no_store_ = {}; + } } void WaitBlockState::written_state(td::Ref upd_state) { @@ -317,6 +324,10 @@ void WaitBlockState::got_state_from_db(td::Ref state) { }); td::actor::send_closure(manager_, &ValidatorManager::set_block_state, handle_, prev_state_, std::move(P)); + if (promise_no_store_) { + promise_no_store_.set_result(prev_state_); + promise_no_store_ = {}; + } } else { finish_query(); } diff --git a/validator/downloaders/wait-block-state.hpp b/validator/downloaders/wait-block-state.hpp index a9317381c..cf556e33d 100644 --- a/validator/downloaders/wait-block-state.hpp +++ b/validator/downloaders/wait-block-state.hpp @@ -28,7 +28,8 @@ class WaitBlockState : public td::actor::Actor { public: WaitBlockState(BlockHandle handle, td::uint32 priority, td::Ref opts, td::Ref last_masterchain_state, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise> promise, + td::Timestamp timeout, td::Promise> promise_no_store, + td::Promise> promise_final, td::Ref persistent_state_desc = {}) : handle_(std::move(handle)) , priority_(priority) @@ -36,7 +37,8 @@ class WaitBlockState : public td::actor::Actor { , last_masterchain_state_(last_masterchain_state) , manager_(manager) , timeout_(timeout) - , promise_(std::move(promise)) + , promise_no_store_(std::move(promise_no_store)) + , promise_final_(std::move(promise_final)) , persistent_state_desc_(std::move(persistent_state_desc)) , perf_timer_("waitstate", 1.0, [manager](double duration) { send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitstate", duration); @@ -96,7 +98,8 @@ class WaitBlockState : public td::actor::Actor { td::Ref last_masterchain_state_; td::actor::ActorId manager_; td::Timestamp timeout_; - td::Promise> promise_; + td::Promise> promise_no_store_; + td::Promise> promise_final_; td::Ref persistent_state_desc_; td::Ref prev_state_; diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index b9aea1e7f..883994089 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -626,7 +626,7 @@ void AcceptBlockQuery::got_last_mc_block(std::pair, Bl VLOG(VALIDATOR_DEBUG) << "shardchain block refers to newer masterchain block " << mc_blkid_.to_str() << ", trying to obtain it"; td::actor::send_closure_later(manager_, &ValidatorManager::wait_block_state_short, mc_blkid_, priority(), timeout_, - [SelfId = actor_id(this)](td::Result> R) { + false, [SelfId = actor_id(this)](td::Result> R) { check_send_error(SelfId, R) || td::actor::send_closure_bool(SelfId, &AcceptBlockQuery::got_mc_state, R.move_as_ok()); diff --git a/validator/impl/check-proof.cpp b/validator/impl/check-proof.cpp index 30a13c089..12d336ad1 100644 --- a/validator/impl/check-proof.cpp +++ b/validator/impl/check-proof.cpp @@ -344,7 +344,7 @@ void CheckProof::got_block_handle(BlockHandle handle) { process_masterchain_state(); return; } - td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, prev_[0], priority(), timeout_, + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, prev_[0], priority(), timeout_, false, [SelfId = actor_id(this)](td::Result> R) { check_send_error(SelfId, R) || td::actor::send_closure_bool(SelfId, &CheckProof::got_masterchain_state, diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 3ded80b6d..c2ca14f99 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -299,7 +299,7 @@ void Collator::load_prev_states_blocks() { ++pending; auto token = perf_log_.start_action(PSTRING() << "wait_block_state #" << i); td::actor::send_closure_later( - manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), timeout, + manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), timeout, false, [self = get_self(), i, token = std::move(token)](td::Result> res) mutable { LOG(DEBUG) << "got answer to wait_block_state query #" << i; td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_state, i, std::move(res), @@ -340,7 +340,7 @@ void Collator::process_optimistic_prev_block() { ++pending; auto token = perf_log_.start_action("opt wait_block_state"); td::actor::send_closure_later( - manager, &ValidatorManager::wait_block_state_short, prev_prev[0], priority(), timeout, + manager, &ValidatorManager::wait_block_state_short, prev_prev[0], priority(), timeout, false, [self = get_self(), token = std::move(token)](td::Result> res) mutable { LOG(DEBUG) << "got answer to wait_block_state query (opt)"; td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_state_optimistic, std::move(res), @@ -574,7 +574,7 @@ bool Collator::request_aux_mc_state(BlockSeqno seqno, Ref& st ++pending; auto token = perf_log_.start_action(PSTRING() << "auxiliary wait_block_state " << blkid.to_str()); td::actor::send_closure_later( - manager, &ValidatorManager::wait_block_state_short, blkid, priority(), timeout, + manager, &ValidatorManager::wait_block_state_short, blkid, priority(), timeout, false, [self = get_self(), blkid, token = std::move(token)](td::Result> res) mutable { LOG(DEBUG) << "got answer to wait_block_state query for " << blkid.to_str(); td::actor::send_closure_later(std::move(self), &Collator::after_get_aux_shard_state, blkid, std::move(res), diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index e3330ea42..dc56997a1 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -334,7 +334,7 @@ void OutMsgQueueImporter::get_proof_local(std::shared_ptr entry, Blo return; } td::actor::send_closure( - manager_, &ValidatorManager::wait_block_state_short, block, 0, entry->timeout, + manager_, &ValidatorManager::wait_block_state_short, block, 0, entry->timeout, false, [=, SelfId = actor_id(this), manager = manager_, timeout = entry->timeout, retry_after = td::Timestamp::in(0.1)](td::Result> R) mutable { if (R.is_error()) { diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index fbaa06aac..01ded4dfa 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -372,7 +372,7 @@ void ValidateQuery::start_up() { LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; ++pending; td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), - timeout, [self = get_self(), i](td::Result> res) -> void { + timeout, false, [self = get_self(), i](td::Result> res) -> void { LOG(DEBUG) << "got answer to wait_block_state_short query #" << i; td::actor::send_closure_later( std::move(self), &ValidateQuery::after_get_shard_state, i, std::move(res)); @@ -784,7 +784,7 @@ void ValidateQuery::got_mc_handle(td::Result res) { } auto mc_handle = res.move_as_ok(); td::actor::send_closure_later( - manager, &ValidatorManager::wait_block_state, mc_handle, priority(), timeout, + manager, &ValidatorManager::wait_block_state, mc_handle, priority(), timeout, false, [self = get_self(), id = id_, mc_handle](td::Result> res) { LOG(DEBUG) << "got answer to wait_block_state() query for masterchain block"; if (res.is_ok() && mc_handle->id().seqno() > 0 && !mc_handle->inited_proof()) { @@ -1764,7 +1764,7 @@ bool ValidateQuery::request_aux_mc_state(BlockSeqno seqno, Ref> res) { LOG(DEBUG) << "got answer to wait_block_state query for " << blkid.to_str(); td::actor::send_closure_later(std::move(self), diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index a4124904e..be57a632d 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -321,14 +321,15 @@ void ValidatorManagerImpl::dec_pending_new_blocks() { } void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) { + bool wait_store, td::Promise> promise) { auto it = wait_state_.find(handle->id()); if (it == wait_state_.end()) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle->id(), std::move(R)); }); auto id = td::actor::create_actor("waitstate", handle, 0, opts_, last_masterchain_state_, - actor_id(this), td::Timestamp::in(10.0), std::move(P)) + actor_id(this), td::Timestamp::in(10.0), + td::Promise>{}, std::move(P)) .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); @@ -340,14 +341,14 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior } void ValidatorManagerImpl::wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) { + bool wait_store, td::Promise> promise) { auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), timeout, promise = std::move(promise)](td::Result R) mutable { + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); return; } - td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_state, R.move_as_ok(), 0, timeout, + td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_state, R.move_as_ok(), 0, timeout, wait_store, std::move(promise)); }); get_block_handle(block_id, true, std::move(P)); @@ -401,7 +402,7 @@ void ValidatorManagerImpl::wait_prev_block_state(BlockHandle handle, td::uint32 auto shard = handle->id().shard_full(); auto prev_shard = handle->one_prev(true).shard_full(); if (shard == prev_shard) { - wait_block_state_short(handle->one_prev(true), 0, timeout, std::move(promise)); + wait_block_state_short(handle->one_prev(true), 0, timeout, true, std::move(promise)); } else { CHECK(shard_parent(shard) == prev_shard); bool left = shard_child(prev_shard, true) == shard; @@ -420,7 +421,7 @@ void ValidatorManagerImpl::wait_prev_block_state(BlockHandle handle, td::uint32 } } }); - wait_block_state_short(handle->one_prev(true), 0, timeout, std::move(P)); + wait_block_state_short(handle->one_prev(true), 0, timeout, true, std::move(P)); } } else { wait_block_state_merge(handle->one_prev(true), handle->one_prev(false), 0, timeout, std::move(promise)); @@ -495,7 +496,7 @@ void ValidatorManagerImpl::wait_block_message_queue(BlockHandle handle, td::uint } }); - wait_block_state(handle, 0, timeout, std::move(P)); + wait_block_state(handle, 0, timeout, true, std::move(P)); } void ValidatorManagerImpl::wait_block_message_queue_short(BlockIdExt block_id, td::uint32 priority, diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 6c8aafd72..bdde74b2d 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -163,9 +163,9 @@ class ValidatorManagerImpl : public ValidatorManager { std::function write_data, td::Promise promise) override; void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; - void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) override; - void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, + void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) override; void wait_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, td::Promise>> promise) override { diff --git a/validator/manager-hardfork.cpp b/validator/manager-hardfork.cpp index de0390ece..717ab3aa5 100644 --- a/validator/manager-hardfork.cpp +++ b/validator/manager-hardfork.cpp @@ -167,14 +167,15 @@ void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { } void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) { + bool wait_store, td::Promise> promise) { auto it = wait_state_.find(handle->id()); if (it == wait_state_.end()) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle->id(), std::move(R)); }); auto id = td::actor::create_actor("waitstate", handle, 0, opts_, last_masterchain_state_, - actor_id(this), td::Timestamp::in(10.0), std::move(P)) + actor_id(this), td::Timestamp::in(10.0), + td::Promise>{}, std::move(P)) .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); @@ -186,14 +187,14 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior } void ValidatorManagerImpl::wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) { + bool wait_store, td::Promise> promise) { auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), timeout, promise = std::move(promise)](td::Result R) mutable { + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); return; } - td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_state, R.move_as_ok(), 0, timeout, + td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_state, R.move_as_ok(), 0, timeout, wait_store, std::move(promise)); }); get_block_handle(block_id, true, std::move(P)); @@ -247,7 +248,7 @@ void ValidatorManagerImpl::wait_prev_block_state(BlockHandle handle, td::uint32 auto shard = handle->id().shard_full(); auto prev_shard = handle->one_prev(true).shard_full(); if (shard == prev_shard) { - wait_block_state_short(handle->one_prev(true), 0, timeout, std::move(promise)); + wait_block_state_short(handle->one_prev(true), 0, timeout, false, std::move(promise)); } else { CHECK(shard_parent(shard) == prev_shard); bool left = shard_child(prev_shard, true) == shard; @@ -266,7 +267,7 @@ void ValidatorManagerImpl::wait_prev_block_state(BlockHandle handle, td::uint32 } } }); - wait_block_state_short(handle->one_prev(true), 0, timeout, std::move(P)); + wait_block_state_short(handle->one_prev(true), 0, timeout, false, std::move(P)); } } else { wait_block_state_merge(handle->one_prev(true), handle->one_prev(false), 0, timeout, std::move(promise)); @@ -341,7 +342,7 @@ void ValidatorManagerImpl::wait_block_message_queue(BlockHandle handle, td::uint } }); - wait_block_state(handle, 0, timeout, std::move(P)); + wait_block_state(handle, 0, timeout, true, std::move(P)); } void ValidatorManagerImpl::wait_block_message_queue_short(BlockIdExt block_id, td::uint32 priority, diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index b3a6c0b2f..70876084a 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -201,9 +201,9 @@ class ValidatorManagerImpl : public ValidatorManager { void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override { UNREACHABLE(); } - void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) override; - void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, + void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) override; void wait_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, td::Promise>> promise) override { diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index f9a55c6ae..886a71544 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -348,7 +348,7 @@ void ValidatorManagerMasterchainStarter::got_init_block_handle(BlockHandle handl handle_ = std::move(handle); if (!handle_->received_state()) { LOG(ERROR) << "db inconsistent: last state ( " << handle_->id() << " ) not received"; - td::actor::send_closure(manager_, &ValidatorManager::wait_block_state, handle_, 1, td::Timestamp::in(600.0), + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state, handle_, 1, td::Timestamp::in(600.0), true, [SelfId = actor_id(this), handle = handle_](td::Result> R) { td::actor::send_closure( SelfId, &ValidatorManagerMasterchainStarter::got_init_block_handle, handle); diff --git a/validator/manager.cpp b/validator/manager.cpp index 1fcd6a520..6a11b6bfa 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -579,7 +579,7 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refblock_id(), 0, td::Timestamp::in(60.0), std::move(P)); + wait_block_state_short(desc->block_id(), 0, td::Timestamp::in(60.0), true, std::move(P)); } if (validating_masterchain()) { td::MultiPromise mp; @@ -796,7 +796,7 @@ void ValidatorManagerImpl::run_ext_query(td::BufferSlice data, td::Promise> promise) { + bool wait_store, td::Promise> promise) { auto it0 = block_state_cache_.find(handle->id()); if (it0 != block_state_cache_.end()) { it0->second.ttl_ = td::Timestamp::in(30.0); @@ -805,33 +805,43 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior } auto it = wait_state_.find(handle->id()); if (it == wait_state_.end()) { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { - td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); + auto P1 = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R), true); + }); + auto P2 = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R), false); }); auto id = td::actor::create_actor("waitstate", handle, priority, opts_, last_masterchain_state_, - actor_id(this), td::Timestamp::at(timeout.at() + 10.0), std::move(P), - get_block_persistent_state_to_download(handle->id())) + actor_id(this), td::Timestamp::at(timeout.at() + 10.0), std::move(P1), + std::move(P2), get_block_persistent_state_to_download(handle->id())) .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); } - it->second.waiting_.emplace_back(timeout, priority, std::move(promise)); + if (wait_store) { + it->second.waiting_.emplace_back(timeout, priority, std::move(promise)); + } else if (it->second.preliminary_done_) { + promise.set_result(it->second.preliminary_result_); + return; + } else { + it->second.waiting_preliminary_.emplace_back(timeout, priority, std::move(promise)); + } auto X = it->second.get_timeout(); td::actor::send_closure(it->second.actor_, &WaitBlockState::update_timeout, X.first, X.second); } void ValidatorManagerImpl::wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) { + bool wait_store, td::Promise> promise) { auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), priority, timeout, promise = std::move(promise)](td::Result R) mutable { + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); return; } td::actor::send_closure(SelfId, &ValidatorManagerImpl::wait_block_state, R.move_as_ok(), priority, timeout, - std::move(promise)); + wait_store, std::move(promise)); }); get_block_handle(block_id, true, std::move(P)); } @@ -954,7 +964,7 @@ void ValidatorManagerImpl::wait_prev_block_state(BlockHandle handle, td::uint32 auto shard = handle->id().shard_full(); auto prev_shard = handle->one_prev(true).shard_full(); if (shard == prev_shard) { - wait_block_state_short(handle->one_prev(true), priority, timeout, std::move(promise)); + wait_block_state_short(handle->one_prev(true), priority, timeout, false, std::move(promise)); } else { CHECK(shard_parent(shard) == prev_shard); bool left = shard_child(prev_shard, true) == shard; @@ -973,7 +983,7 @@ void ValidatorManagerImpl::wait_prev_block_state(BlockHandle handle, td::uint32 } } }); - wait_block_state_short(handle->one_prev(true), priority, timeout, std::move(P)); + wait_block_state_short(handle->one_prev(true), priority, timeout, false, std::move(P)); } } else { wait_block_state_merge(handle->one_prev(true), handle->one_prev(false), priority, timeout, std::move(promise)); @@ -1048,7 +1058,7 @@ void ValidatorManagerImpl::wait_block_message_queue(BlockHandle handle, td::uint } }); - wait_block_state(handle, priority, timeout, std::move(P)); + wait_block_state(handle, priority, timeout, false, std::move(P)); } void ValidatorManagerImpl::wait_block_message_queue_short(BlockIdExt block_id, td::uint32 priority, @@ -1309,37 +1319,51 @@ void ValidatorManagerImpl::get_block_by_seqno_from_db(AccountIdPrefixFull accoun td::actor::send_closure(db_, &Db::get_block_by_seqno, account, seqno, std::move(promise)); } -void ValidatorManagerImpl::finished_wait_state(BlockHandle handle, td::Result> R) { - if (R.is_ok()) { - block_state_cache_[handle->id()] = {R.ok(), td::Timestamp::in(30.0)}; - } +void ValidatorManagerImpl::finished_wait_state(BlockHandle handle, td::Result> R, + bool preliminary) { auto it = wait_state_.find(handle->id()); - if (it != wait_state_.end()) { - if (R.is_error()) { - auto S = R.move_as_error(); - if (S.code() != ErrorCode::timeout) { - for (auto &X : it->second.waiting_) { - X.promise.set_error(S.clone()); - } - } else if (it->second.waiting_.size() != 0) { - auto X = it->second.get_timeout(); - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { - td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); - }); - auto id = td::actor::create_actor("waitstate", handle, X.second, opts_, last_masterchain_state_, - actor_id(this), X.first, std::move(P), - get_block_persistent_state_to_download(handle->id())) - .release(); - it->second.actor_ = id; - return; - } - } else { - auto r = R.move_as_ok(); + if (it == wait_state_.end()) { + return; + } + if (R.is_ok()) { + auto r = R.move_as_ok(); + for (auto &X : it->second.waiting_preliminary_) { + X.promise.set_result(r); + } + it->second.preliminary_done_ = true; + it->second.preliminary_result_ = r; + it->second.waiting_preliminary_.clear(); + if (!preliminary) { + block_state_cache_[handle->id()] = {r, td::Timestamp::in(30.0)}; for (auto &X : it->second.waiting_) { X.promise.set_result(r); } + wait_state_.erase(it); + } + } else if (!preliminary) { + auto S = R.move_as_error(); + if (S.code() != ErrorCode::timeout) { + for (auto &X : it->second.waiting_) { + X.promise.set_error(S.clone()); + } + for (auto &X : it->second.waiting_preliminary_) { + X.promise.set_error(S.clone()); + } + wait_state_.erase(it); + } else if (!it->second.waiting_.empty() || !it->second.waiting_preliminary_.empty()) { + auto X = it->second.get_timeout(); + auto P1 = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R), true); + }); + auto P2 = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R), false); + }); + auto id = td::actor::create_actor("waitstate", handle, X.second, opts_, last_masterchain_state_, + actor_id(this), X.first, std::move(P1), std::move(P2), + get_block_persistent_state_to_download(handle->id())) + .release(); + it->second.actor_ = id; } - wait_state_.erase(it); } } @@ -2756,7 +2780,7 @@ void ValidatorManagerImpl::got_next_gc_masterchain_handle(BlockHandle handle) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::got_next_gc_masterchain_state, std::move(handle), td::Ref{R.move_as_ok()}); }); - wait_block_state(handle, 0, td::Timestamp::in(60.0), std::move(P)); + wait_block_state(handle, 0, td::Timestamp::in(60.0), true, std::move(P)); } void ValidatorManagerImpl::got_next_gc_masterchain_state(BlockHandle handle, td::Ref state) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 952fcf49b..6f7b73447 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -166,9 +166,17 @@ class ValidatorManagerImpl : public ValidatorManager { WaitList() = default; std::pair get_timeout() const { + return get_timeout_impl(waiting_); + } + void check_timers() { + check_timers_impl(waiting_); + } + + protected: + static std::pair get_timeout_impl(const std::vector> &waiting) { td::Timestamp t = td::Timestamp::now(); td::uint32 prio = 0; - for (auto &v : waiting_) { + for (auto &v : waiting) { if (v.timeout.at() > t.at()) { t = v.timeout; } @@ -178,10 +186,10 @@ class ValidatorManagerImpl : public ValidatorManager { } return {td::Timestamp::at(t.at() + 10.0), prio}; } - void check_timers() { + static void check_timers_impl(std::vector> &waiting) { td::uint32 j = 0; - auto f = waiting_.begin(); - auto t = waiting_.end(); + auto f = waiting.begin(); + auto t = waiting.end(); while (f < t) { if (f->timeout.is_in_past()) { f->promise.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); @@ -192,16 +200,26 @@ class ValidatorManagerImpl : public ValidatorManager { j++; } } - waiting_.resize(j); + waiting.resize(j); } }; template - struct WaitListCaching : public WaitList { - bool done_ = false; - ResType result_; - td::Timestamp remove_at_; + struct WaitListPreliminary : WaitList { + std::vector> waiting_preliminary_; + bool preliminary_done_ = false; + ResType preliminary_result_; + + std::pair get_timeout() const { + auto t1 = WaitList::get_timeout_impl(this->waiting_); + auto t2 = WaitList::get_timeout_impl(waiting_preliminary_); + return {std::max(t1.first, t2.first), std::max(t1.second, t2.second)}; + } + void check_timers() { + WaitList::check_timers_impl(this->waiting_); + WaitList::check_timers_impl(waiting_preliminary_); + } }; - std::map>> wait_state_; + std::map>> wait_state_; std::map>> wait_block_data_; struct CachedBlockState { @@ -441,9 +459,9 @@ class ValidatorManagerImpl : public ValidatorManager { std::function write_data, td::Promise promise) override; void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; - void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) override; - void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, + void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) override; void wait_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, td::Timestamp timeout, td::Promise>> promise) override; @@ -585,7 +603,7 @@ class ValidatorManagerImpl : public ValidatorManager { void register_block_handle(BlockHandle handle); - void finished_wait_state(BlockHandle handle, td::Result> R); + void finished_wait_state(BlockHandle handle, td::Result> R, bool preliminary); void finished_wait_data(BlockHandle handle, td::Result> R); void start_up() override; diff --git a/validator/queue-size-counter.cpp b/validator/queue-size-counter.cpp index 4fe55ae31..487bdefc8 100644 --- a/validator/queue-size-counter.cpp +++ b/validator/queue-size-counter.cpp @@ -112,7 +112,7 @@ void QueueSizeCounter::get_queue_size_ex(ton::BlockIdExt block_id, bool calc_who } BlockHandle handle = R.move_as_ok(); td::actor::send_closure( - manager, &ValidatorManager::wait_block_state, handle, 0, td::Timestamp::in(10.0), + manager, &ValidatorManager::wait_block_state, handle, 0, td::Timestamp::in(10.0), false, [SelfId, handle](td::Result> R) mutable { if (R.is_error()) { td::actor::send_closure(SelfId, &QueueSizeCounter::on_error, handle->id(), @@ -159,7 +159,7 @@ void QueueSizeCounter::get_queue_size_cont(BlockHandle handle, td::Ref> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &QueueSizeCounter::on_error, state->get_block_id(), R.move_as_error()); @@ -213,7 +213,7 @@ void QueueSizeCounter::process_top_shard_blocks() { return; } td::actor::send_closure( - manager, &ValidatorManager::wait_block_state_short, R.ok()->id(), 0, td::Timestamp::in(10.0), + manager, &ValidatorManager::wait_block_state_short, R.ok()->id(), 0, td::Timestamp::in(10.0), false, [=](td::Result> R) { if (R.is_error()) { LOG(WARNING) << "Failed to get masterchain state: " << R.move_as_error(); diff --git a/validator/shard-block-retainer.cpp b/validator/shard-block-retainer.cpp index 528f6bcf7..8ef4d8f21 100644 --- a/validator/shard-block-retainer.cpp +++ b/validator/shard-block-retainer.cpp @@ -109,7 +109,7 @@ void ShardBlockRetainer::new_shard_block_description(td::Refblock_id(), 0, td::Timestamp::in(30.0), + manager_, &ValidatorManager::wait_block_state_short, desc->block_id(), 0, td::Timestamp::in(30.0), true, [SelfId = actor_id(this), desc](td::Result> R) { if (R.is_error()) { LOG(WARNING) << "Wait block state for " << desc->block_id().to_str() << " : " << R.move_as_error(); diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index e5fd0a2ac..49ee7b6e8 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -161,7 +161,7 @@ void ShardClient::download_masterchain_state() { } }); td::actor::send_closure(manager_, &ValidatorManager::wait_block_state, masterchain_block_handle_, - shard_client_priority(), td::Timestamp::in(600), std::move(P)); + shard_client_priority(), td::Timestamp::in(600), true, std::move(P)); } void ShardClient::got_masterchain_block_state(td::Ref state) { @@ -201,7 +201,7 @@ void ShardClient::apply_all_shards() { } }); td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, shard->top_block_id(), - shard_client_priority(), td::Timestamp::in(1500), std::move(Q)); + shard_client_priority(), td::Timestamp::in(1500), true, std::move(Q)); } } for (const auto &[wc, desc] : masterchain_state_->get_workchain_list()) { @@ -216,7 +216,7 @@ void ShardClient::apply_all_shards() { }); td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, BlockIdExt{wc, shardIdAll, 0, desc->zerostate_root_hash, desc->zerostate_file_hash}, - shard_client_priority(), td::Timestamp::in(1500), std::move(Q)); + shard_client_priority(), td::Timestamp::in(1500), true, std::move(Q)); } } } diff --git a/validator/validator.h b/validator/validator.h index c04f835a7..e73614950 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -325,9 +325,9 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_block_by_seqno_from_db(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise promise) = 0; - virtual void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + virtual void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) = 0; - virtual void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, + virtual void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, bool wait_store, td::Promise> promise) = 0; virtual void wait_neighbor_msg_queue_proofs(ShardIdFull dst_shard, std::vector blocks, From 6cbae0344edc6d2a206a45d6aa7fe8220e9e7b1b Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 5 Aug 2025 23:08:10 +0300 Subject: [PATCH 367/388] Refactor CollatorNode --- validator/CMakeLists.txt | 6 +- validator/collation-manager.cpp | 2 +- .../collator-node/collator-node-session.cpp | 237 ++++++++++ .../collator-node/collator-node-session.hpp | 79 ++++ .../{ => collator-node}/collator-node.cpp | 419 ++++++------------ .../{ => collator-node}/collator-node.hpp | 40 +- validator/manager.cpp | 7 +- validator/manager.hpp | 2 +- validator/validator-group.cpp | 4 +- 9 files changed, 471 insertions(+), 325 deletions(-) create mode 100644 validator/collator-node/collator-node-session.cpp create mode 100644 validator/collator-node/collator-node-session.hpp rename validator/{ => collator-node}/collator-node.cpp (53%) rename validator/{ => collator-node}/collator-node.hpp (74%) diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index 6f68f55fd..98987c78a 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -64,7 +64,8 @@ set(VALIDATOR_HEADERS shard-block-retainer.hpp collation-manager.hpp - collator-node.hpp + collator-node/collator-node.hpp + collator-node/collator-node-session.hpp manager-disk.h manager-disk.hpp manager-init.h @@ -81,7 +82,8 @@ set(VALIDATOR_SOURCE apply-block.cpp block-handle.cpp collation-manager.cpp - collator-node.cpp + collator-node/collator-node.cpp + collator-node/collator-node-session.cpp get-next-key-blocks.cpp import-db-slice.cpp import-db-slice-local.cpp diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index 9eadd3f0a..9260dfb50 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -16,7 +16,7 @@ */ #include "collation-manager.hpp" -#include "collator-node.hpp" +#include "collator-node/collator-node.hpp" #include "fabric.h" #include "td/utils/Random.h" diff --git a/validator/collator-node/collator-node-session.cpp b/validator/collator-node/collator-node-session.cpp new file mode 100644 index 000000000..76c6f9974 --- /dev/null +++ b/validator/collator-node/collator-node-session.cpp @@ -0,0 +1,237 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ + +#include "collator-node-session.hpp" + +#include "fabric.h" + +namespace ton::validator { + +static BlockSeqno get_next_block_seqno(const std::vector& prev) { + if (prev.size() == 1) { + return prev[0].seqno() + 1; + } + CHECK(prev.size() == 2); + return std::max(prev[0].seqno(), prev[1].seqno()) + 1; +} + +CollatorNodeSession::CollatorNodeSession(ShardIdFull shard, std::vector prev, + td::Ref validator_set, BlockIdExt min_masterchain_block_id, + bool can_generate, adnl::AdnlNodeIdShort local_id, + td::Ref opts, + td::actor::ActorId manager, + td::actor::ActorId adnl, td::actor::ActorId rldp) + : shard_(shard) + , prev_(std::move(prev)) + , validator_set_(validator_set) + , min_masterchain_block_id_(min_masterchain_block_id) + , can_generate_(can_generate) + , local_id_(local_id) + , opts_(opts) + , manager_(manager) + , adnl_(adnl) + , rldp_(rldp) + , next_block_seqno_(get_next_block_seqno(prev_)) { +} + +void CollatorNodeSession::start_up() { + LOG(INFO) << "Starting collator node session, shard " << shard_.to_str() << ", cc_seqno " + << validator_set_->get_catchain_seqno() << ", next block seqno " << next_block_seqno_; + + if (can_generate_) { + generate_block(prev_, {}, td::Timestamp::in(10.0), [](td::Result) {}); + } +} + +void CollatorNodeSession::tear_down() { + LOG(INFO) << "Finishing collator node session, shard " << shard_.to_str() << ", cc_seqno " + << validator_set_->get_catchain_seqno(); + for (auto& [_, entry] : cache_) { + entry->cancel(td::Status::Error("validator session finished")); + } +} + +void CollatorNodeSession::new_shard_block_accepted(BlockIdExt block_id, bool can_generate) { + CHECK(block_id.shard_full() == shard_); + can_generate_ = can_generate; + if (next_block_seqno_ > block_id.seqno()) { + return; + } + LOG(DEBUG) << "New shard block " << block_id.to_str(); + next_block_seqno_ = block_id.seqno() + 1; + prev_ = {block_id}; + + while (!cache_.empty()) { + auto& [cache_prev, entry] = *cache_.begin(); + if (entry->block_seqno < next_block_seqno_) { + entry->cancel(td::Status::Error(PSTRING() << "next block seqno " << entry->block_seqno << " is too old, expected " + << next_block_seqno_)); + } else if (entry->block_seqno == next_block_seqno_ && prev_ != cache_prev) { + entry->cancel(td::Status::Error(PSTRING() << "invalid prev blocks for seqno " << entry->block_seqno)); + } else { + break; + } + if (!entry->has_external_query_at && entry->has_internal_query_at) { + LOG(INFO) << "generate block query" + << ": shard=" << shard_.to_str() << ", cc_seqno=" << validator_set_->get_catchain_seqno() + << ", next_block_seqno=" << entry->block_seqno + << ": nobody asked for block, but we tried to generate it"; + } + if (entry->has_external_query_at && !entry->has_internal_query_at) { + LOG(INFO) << "generate block query" + << ": shard=" << shard_.to_str() << ", cc_seqno=" << validator_set_->get_catchain_seqno() + << ", next_block_seqno=" << entry->block_seqno + << ": somebody asked for block we didn't even try to generate"; + } + cache_.erase(cache_.begin()); + } + + if (can_generate_) { + generate_block(prev_, {}, td::Timestamp::in(10.0), [](td::Result) {}); + } +} + +void CollatorNodeSession::generate_block(std::vector prev_blocks, + td::optional o_priority, td::Timestamp timeout, + td::Promise promise) { + bool is_external = !o_priority; + BlockSeqno block_seqno = get_next_block_seqno(prev_blocks); + if (next_block_seqno_ > block_seqno) { + promise.set_error(td::Status::Error(PSTRING() << "next block seqno " << block_seqno << " is too old, expected " + << next_block_seqno_)); + return; + } + if (next_block_seqno_ == block_seqno && prev_ != prev_blocks) { + promise.set_error(td::Status::Error("invalid prev_blocks")); + return; + } + if (next_block_seqno_ + 10 < block_seqno) { + promise.set_error(td::Status::Error(PSTRING() << "next block seqno " << block_seqno << " is too new, current is " + << next_block_seqno_)); + return; + } + + static auto prefix_inner = [](td::StringBuilder& sb, const ShardIdFull& shard, CatchainSeqno cc_seqno, + BlockSeqno block_seqno, const td::optional& o_priority) { + sb << "generate block query" + << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno; + if (o_priority) { + sb << " external{"; + sb << "round_offset=" << o_priority.value().round - o_priority.value().first_block_round + << ",priority=" << o_priority.value().priority; + sb << ",first_block_round=" << o_priority.value().first_block_round; + sb << "}"; + } else { + sb << " internal"; + } + }; + auto prefix = [&](td::StringBuilder& sb) { + prefix_inner(sb, shard_, validator_set_->get_catchain_seqno(), block_seqno, o_priority); + }; + + auto cache_entry = cache_[prev_blocks]; + if (cache_entry == nullptr) { + cache_entry = cache_[prev_blocks] = std::make_shared(); + } + if (is_external && !cache_entry->has_external_query_at) { + cache_entry->has_external_query_at = td::Timestamp::now(); + if (cache_entry->has_internal_query_at && cache_entry->has_external_query_at) { + FLOG(INFO) { + prefix(sb); + sb << ": got external query " << cache_entry->has_external_query_at - cache_entry->has_internal_query_at + << "s after internal query [WON]"; + }; + } + } + if (!is_external && !cache_entry->has_internal_query_at) { + cache_entry->has_internal_query_at = td::Timestamp::now(); + if (cache_entry->has_internal_query_at && cache_entry->has_external_query_at) { + FLOG(INFO) { + prefix(sb); + sb << ": got internal query " << cache_entry->has_internal_query_at - cache_entry->has_external_query_at + << "s after external query [LOST]"; + }; + } + } + if (cache_entry->result) { + auto has_result_ago = td::Timestamp::now() - cache_entry->has_result_at; + FLOG(INFO) { + prefix(sb); + sb << ": using cached result " << " generated " << has_result_ago << "s ago"; + sb << (is_external ? " for external query [WON]" : " for internal query "); + }; + promise.set_result(cache_entry->result.value().clone()); + return; + } + cache_entry->promises.push_back(std::move(promise)); + + if (cache_entry->started) { + FLOG(INFO) { + prefix(sb); + sb << ": collation in progress, waiting"; + }; + return; + } + FLOG(INFO) { + prefix(sb); + sb << ": starting collation"; + }; + cache_entry->started = true; + cache_entry->block_seqno = block_seqno; + run_collate_query(CollateParams{.shard = shard_, + .min_masterchain_block_id = min_masterchain_block_id_, + .prev = std::move(prev_blocks), + .validator_set = validator_set_, + .collator_opts = opts_->get_collator_options(), + .collator_node_id = local_id_, + .skip_store_candidate = true}, + manager_, timeout, cache_entry->cancellation_token_source.get_cancellation_token(), + [=, shard = shard_, cc_seqno = validator_set_->get_catchain_seqno(), SelfId = actor_id(this), + timer = td::Timer{}](td::Result R) mutable { + FLOG(INFO) { + prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority); + sb << timer.elapsed() << ": " << (R.is_ok() ? "OK" : R.error().to_string()); + }; + td::actor::send_closure(SelfId, &CollatorNodeSession::process_result, cache_entry, std::move(R)); + }); +} + +void CollatorNodeSession::process_result(std::shared_ptr cache_entry, td::Result R) { + if (R.is_error()) { + cache_entry->started = false; + for (auto& p : cache_entry->promises) { + p.set_error(R.error().clone()); + } + } else { + cache_entry->result = R.move_as_ok(); + cache_entry->has_result_at = td::Timestamp::now(); + for (auto& p : cache_entry->promises) { + p.set_result(cache_entry->result.value().clone()); + } + } + cache_entry->promises.clear(); +} + +void CollatorNodeSession::CacheEntry::cancel(td::Status reason) { + for (auto& promise : promises) { + promise.set_error(reason.clone()); + } + promises.clear(); + cancellation_token_source.cancel(); +} + +} // namespace ton::validator diff --git a/validator/collator-node/collator-node-session.hpp b/validator/collator-node/collator-node-session.hpp new file mode 100644 index 000000000..51e1b7cf6 --- /dev/null +++ b/validator/collator-node/collator-node-session.hpp @@ -0,0 +1,79 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "interfaces/validator-manager.h" +#include "rldp/rldp.h" +#include "rldp2/rldp.h" +#include +#include + +namespace ton::validator { + +class ValidatorManager; + +class CollatorNodeSession : public td::actor::Actor { + public: + CollatorNodeSession(ShardIdFull shard, std::vector prev, td::Ref validator_set, + BlockIdExt min_masterchain_block_id, bool can_generate, adnl::AdnlNodeIdShort local_id, + td::Ref opts, td::actor::ActorId manager, + td::actor::ActorId adnl, td::actor::ActorId rldp); + + void start_up() override; + void tear_down() override; + + void update_options(td::Ref opts) { + opts_ = std::move(opts); + } + + void new_shard_block_accepted(BlockIdExt block_id, bool can_generate); + + void generate_block(std::vector prev_blocks, td::optional o_priority, + td::Timestamp timeout, td::Promise promise); + + private: + ShardIdFull shard_; + std::vector prev_; + td::Ref validator_set_; + BlockIdExt min_masterchain_block_id_; + bool can_generate_; + adnl::AdnlNodeIdShort local_id_; + td::Ref opts_; + td::actor::ActorId manager_; + td::actor::ActorId adnl_; + td::actor::ActorId rldp_; + + struct CacheEntry { + bool started = false; + td::Timestamp has_internal_query_at; + td::Timestamp has_external_query_at; + td::Timestamp has_result_at; + BlockSeqno block_seqno = 0; + td::optional result; + td::CancellationTokenSource cancellation_token_source; + std::vector> promises; + + void cancel(td::Status reason); + }; + + BlockSeqno next_block_seqno_; + std::map, std::shared_ptr> cache_; + + void process_result(std::shared_ptr cache_entry, td::Result R); +}; + +} // namespace ton::validator diff --git a/validator/collator-node.cpp b/validator/collator-node/collator-node.cpp similarity index 53% rename from validator/collator-node.cpp rename to validator/collator-node/collator-node.cpp index ba3ff01eb..6f0cae419 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node/collator-node.cpp @@ -75,6 +75,17 @@ void CollatorNode::add_shard(ShardIdFull shard) { } LOG(INFO) << "Collator node: local_id=" << local_id_ << " , shard=" << shard.to_str(); collating_shards_.push_back(shard); + if (last_masterchain_state_.is_null()) { + return; + } + for (auto& [group_shard, validator_group] : validator_groups_) { + if (validator_group.actor.empty() && shard_intersects(shard, group_shard)) { + validator_group.actor = td::actor::create_actor( + PSTRING() << "collatornode" << shard.to_str(), shard, validator_group.prev, + last_masterchain_state_->get_validator_set(group_shard), last_masterchain_state_->get_block_id(), + can_generate(), local_id_, opts_, manager_, adnl_, rldp_); + } + } } void CollatorNode::del_shard(ShardIdFull shard) { @@ -82,6 +93,20 @@ void CollatorNode::del_shard(ShardIdFull shard) { if (it != collating_shards_.end()) { collating_shards_.erase(it); } + for (auto& [group_shard, validator_group] : validator_groups_) { + if (!validator_group.actor.empty() && shard_intersects(shard, group_shard) && !can_collate_shard(group_shard)) { + validator_group.actor = {}; + } + } +} + +void CollatorNode::update_options(td::Ref opts) { + for (auto& [_, shard] : validator_groups_) { + if (!shard.actor.empty()) { + td::actor::send_closure(shard.actor, &CollatorNodeSession::update_options, opts); + } + } + opts_ = std::move(opts); } void CollatorNode::new_masterchain_block_notification(td::Ref state) { @@ -91,7 +116,7 @@ void CollatorNode::new_masterchain_block_notification(td::Ref last_key_block_seqno_ = state->last_key_block_id().seqno(); mc_config_status_ = check_mc_config(); if (mc_config_status_.is_error()) { - LOG(ERROR) << "Cannot validate masterchain config (possibly outdated software):" << mc_config_status_; + LOG(ERROR) << "Cannot validate masterchain config (possibly outdated software): " << mc_config_status_; } } @@ -131,36 +156,42 @@ void CollatorNode::new_masterchain_block_notification(td::Ref new_shards.emplace(shard, std::vector{v->top_block_id()}); } } - - for (auto& [shard, prev] : new_shards) { - CatchainSeqno cc_seqno = state->get_validator_set(shard)->get_catchain_seqno(); - auto it = validator_groups_.emplace(shard, ValidatorGroupInfo{}); - ValidatorGroupInfo& info = it.first->second; - if (it.second || info.cc_seqno != cc_seqno) { - info.cleanup(); - info.cc_seqno = cc_seqno; - } - } for (auto it = validator_groups_.begin(); it != validator_groups_.end();) { if (new_shards.contains(it->first)) { ++it; } else { - it->second.cleanup(); it = validator_groups_.erase(it); } } for (auto& [shard, prev] : new_shards) { - ValidatorGroupInfo& info = validator_groups_[shard]; - update_validator_group_info(shard, std::move(prev), info.cc_seqno); - auto it = future_validator_groups_.find({shard, info.cc_seqno}); - if (it != future_validator_groups_.end()) { - for (auto& new_prev : it->second.pending_blocks) { - update_validator_group_info(shard, std::move(new_prev), info.cc_seqno); + auto validator_set = state->get_validator_set(shard); + CatchainSeqno cc_seqno = validator_set->get_catchain_seqno(); + auto [it, created] = validator_groups_.emplace(shard, ValidatorGroupInfo{}); + it->second.prev = std::move(prev); + if (created || it->second.cc_seqno != cc_seqno) { + it->second.cc_seqno = cc_seqno; + if (can_collate_shard(shard)) { + it->second.actor = td::actor::create_actor( + PSTRING() << "collatornode" << shard.to_str(), shard, it->second.prev, validator_set, + last_masterchain_state_->get_block_id(), can_generate(), local_id_, opts_, manager_, adnl_, rldp_); } - for (auto& promise : it->second.promises) { + } else if (!it->second.actor.empty() && prev.size() == 1) { + td::actor::send_closure(it->second.actor, &CollatorNodeSession::new_shard_block_accepted, prev[0], + can_generate()); + } + auto it2 = future_validator_groups_.find({shard, cc_seqno}); + if (it2 != future_validator_groups_.end()) { + FutureValidatorGroup& future_group = it2->second; + if (!it->second.actor.empty()) { + for (const BlockIdExt& block_id : future_group.pending_blocks) { + td::actor::send_closure(it->second.actor, &CollatorNodeSession::new_shard_block_accepted, block_id, + can_generate()); + } + } + for (auto& promise : future_group.promises) { promise.set_value(td::Unit()); } - future_validator_groups_.erase(it); + future_validator_groups_.erase(it2); } } @@ -181,82 +212,24 @@ void CollatorNode::update_shard_client_handle(BlockHandle shard_client_handle) { shard_client_handle_ = shard_client_handle; } -void CollatorNode::update_validator_group_info(ShardIdFull shard, std::vector prev, - CatchainSeqno cc_seqno) { - if (!can_collate_shard(shard)) { +void CollatorNode::new_shard_block_accepted(BlockIdExt block_id, CatchainSeqno cc_seqno) { + if (!can_collate_shard(block_id.shard_full())) { return; } - CHECK(prev.size() == 1 || prev.size() == 2); - BlockSeqno next_block_seqno = prev[0].seqno() + 1; - if (prev.size() == 2) { - next_block_seqno = std::max(next_block_seqno, prev[1].seqno() + 1); - } - auto it = validator_groups_.find(shard); - if (it != validator_groups_.end()) { - ValidatorGroupInfo& info = it->second; - if (info.cc_seqno == cc_seqno) { // block from currently known validator group - if (info.next_block_seqno < next_block_seqno) { - LOG(DEBUG) << "updated validator group info: shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno - << ", next_block_seqno=" << next_block_seqno; - info.next_block_seqno = next_block_seqno; - info.prev = std::move(prev); - for (auto cache_it = info.cache.begin(); cache_it != info.cache.end();) { - auto& [cached_prev, cache_entry] = *cache_it; - if (cache_entry->block_seqno < info.next_block_seqno) { - cache_entry->cancel(td::Status::Error(PSTRING() << "next block seqno " << cache_entry->block_seqno - << " is too small, expected " << info.next_block_seqno)); - if (!cache_entry->has_external_query_at && cache_entry->has_internal_query_at) { - LOG(INFO) << "generate block query" - << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno - << ", next_block_seqno=" << cache_entry->block_seqno - << ": nobody asked for block, but we tried to generate it"; - } - if (cache_entry->has_external_query_at && !cache_entry->has_internal_query_at) { - LOG(INFO) << "generate block query" - << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno - << ", next_block_seqno=" << cache_entry->block_seqno - << ": somebody asked for block we didn't even tried to generate"; - } - cache_it = info.cache.erase(cache_it); - continue; - } - if (cache_entry->block_seqno == info.next_block_seqno && cached_prev != info.prev) { - cache_entry->cancel(td::Status::Error("invalid prev blocks")); - if (!cache_entry->has_external_query_at && cache_entry->has_internal_query_at) { - LOG(INFO) << "generate block query" - << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno - << ", next_block_seqno=" << cache_entry->block_seqno - << ": nobody asked for block, but we tried to generate it"; - } - if (cache_entry->has_external_query_at && !cache_entry->has_internal_query_at) { - LOG(INFO) << "generate block query" - << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno - << ", next_block_seqno=" << cache_entry->block_seqno - << ": somebody asked for block we didn't even tried to generate"; - } - cache_it = info.cache.erase(cache_it); - continue; - } - ++cache_it; - } - auto S = check_out_of_sync(); - if (S.is_error()) { - LOG(DEBUG) << "not generating block automatically: " << S; - return; - } - if (mc_config_status_.is_error()) { - LOG(DEBUG) << "not generating block automatically: unsupported mc config: " << mc_config_status_; - return; - } - generate_block(shard, cc_seqno, info.prev, {}, td::Timestamp::in(10.0), [](td::Result) {}); - } - return; + auto it = validator_groups_.find(block_id.shard_full()); + if (it == validator_groups_.end() || it->second.cc_seqno != cc_seqno) { + auto future_group = get_future_validator_group(block_id.shard_full(), cc_seqno); + if (future_group.is_error()) { + LOG(DEBUG) << "Dropping new shard block " << block_id.to_str() << " cc_seqno=" << cc_seqno << " : " + << future_group.error(); + } else { + LOG(DEBUG) << "New shard block in future validator group " << block_id.to_str() << " cc_seqno=" << cc_seqno; + future_group.ok()->pending_blocks.push_back(block_id); } + return; } - auto future_validator_group = get_future_validator_group(shard, cc_seqno); - if (future_validator_group.is_ok()) { - // future validator group, remember for later - future_validator_group.ok()->pending_blocks.push_back(std::move(prev)); + if (!it->second.actor.empty()) { + td::actor::send_closure(it->second.actor, &CollatorNodeSession::new_shard_block_accepted, block_id, can_generate()); } } @@ -276,34 +249,17 @@ td::Result CollatorNode::get_future_validat return td::Status::Error("no such shard"); } if (cc_seqno < it->second.cc_seqno) { // past validator group - return td::Status::Error(PSTRING() << "cc_seqno " << cc_seqno << " is outdated (current is" << it->second.cc_seqno - << ")"); + return td::Status::Error(PSTRING() << "cc_seqno " << cc_seqno << " for shard " << shard.to_str() + << " is outdated (current is" << it->second.cc_seqno << ")"); } if (cc_seqno - it->second.cc_seqno > 1) { // future validator group, cc_seqno too big - return td::Status::Error(PSTRING() << "cc_seqno " << cc_seqno << " is too big (currently known is" - << it->second.cc_seqno << ")"); + return td::Status::Error(PSTRING() << "cc_seqno " << cc_seqno << " for shard " << shard.to_str() + << " is too big (currently known is" << it->second.cc_seqno << ")"); } // future validator group return &future_validator_groups_[{shard, cc_seqno}]; } -void CollatorNode::ValidatorGroupInfo::cleanup() { - prev.clear(); - next_block_seqno = 0; - for (auto& [_, cache_entry] : cache) { - cache_entry->cancel(td::Status::Error("validator group is outdated")); - } - cache.clear(); -} - -void CollatorNode::CacheEntry::cancel(td::Status reason) { - for (auto& promise : promises) { - promise.set_error(reason.clone()); - } - promises.clear(); - cancellation_token_source.cancel(); -} - static td::BufferSlice serialize_error(td::Status error) { return create_serialize_tl_object(error.code(), error.message().c_str()); } @@ -423,19 +379,19 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data return; } LOG(INFO) << "got adnl query from " << src << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno; - generate_block(shard, cc_seqno, std::move(prev_blocks), priority, td::Timestamp::in(10.0), std::move(new_promise)); + process_generate_block_query(shard, cc_seqno, std::move(prev_blocks), priority, td::Timestamp::in(10.0), + std::move(new_promise)); } -void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std::vector prev_blocks, - std::optional o_priority, td::Timestamp timeout, - td::Promise promise) { - bool is_external = !o_priority; +void CollatorNode::process_generate_block_query(ShardIdFull shard, CatchainSeqno cc_seqno, + std::vector prev_blocks, BlockCandidatePriority priority, + td::Timestamp timeout, td::Promise promise) { if (last_masterchain_state_.is_null()) { promise.set_error(td::Status::Error(ErrorCode::notready, "not ready")); return; } - if (!can_collate_shard(shard)) { - promise.set_error(td::Status::Error(PSTRING() << "this node can't collate shard " << shard.to_str())); + if (timeout.is_in_past()) { + promise.set_error(td::Status::Error(ErrorCode::timeout)); return; } auto it = validator_groups_.find(shard); @@ -443,133 +399,19 @@ void CollatorNode::generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std TRY_RESULT_PROMISE(promise, future_validator_group, get_future_validator_group(shard, cc_seqno)); future_validator_group->promises.push_back([=, SelfId = actor_id(this), prev_blocks = std::move(prev_blocks), promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - return; - } - if (timeout.is_in_past()) { - promise.set_error(td::Status::Error(ErrorCode::timeout)); - return; - } - td::actor::send_closure(SelfId, &CollatorNode::generate_block, shard, cc_seqno, std::move(prev_blocks), - std::move(o_priority), timeout, std::move(promise)); + TRY_STATUS_PROMISE(promise, R.move_as_status()); + td::actor::send_closure(SelfId, &CollatorNode::process_generate_block_query, shard, cc_seqno, + std::move(prev_blocks), std::move(priority), timeout, std::move(promise)); }); return; } ValidatorGroupInfo& validator_group_info = it->second; - BlockSeqno block_seqno = prev_blocks.at(0).seqno() + 1; - if (prev_blocks.size() == 2) { - block_seqno = std::max(block_seqno, prev_blocks.at(1).seqno() + 1); - } - if (validator_group_info.next_block_seqno > block_seqno) { - promise.set_error(td::Status::Error(PSTRING() << "next block seqno " << block_seqno << " is too small, expected " - << validator_group_info.next_block_seqno)); - return; - } - if (validator_group_info.next_block_seqno == block_seqno && validator_group_info.prev != prev_blocks) { - promise.set_error(td::Status::Error("invalid prev_blocks")); + if (validator_group_info.actor.empty()) { + promise.set_error(td::Status::Error(PSTRING() << "cannot collate shard " << shard.to_str())); return; } - - static auto prefix_inner = [](auto& sb, auto& shard, auto cc_seqno, auto block_seqno, - const std::optional& o_priority) { - sb << "generate block query" - << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno; - if (o_priority) { - sb << " external{"; - sb << "round_offset=" << o_priority->round - o_priority->first_block_round - << ",priority=" << o_priority->priority; - sb << ",first_block_round=" << o_priority->first_block_round; - sb << "}"; - } else { - sb << " internal"; - } - }; - auto prefix = [&](auto& sb) { prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority); }; - - auto cache_entry = validator_group_info.cache[prev_blocks]; - if (cache_entry == nullptr) { - cache_entry = validator_group_info.cache[prev_blocks] = std::make_shared(); - } - if (is_external && !cache_entry->has_external_query_at) { - cache_entry->has_external_query_at = td::Timestamp::now(); - if (cache_entry->has_internal_query_at && cache_entry->has_external_query_at) { - FLOG(INFO) { - prefix(sb); - sb << ": got external query " - << cache_entry->has_external_query_at.at() - cache_entry->has_internal_query_at.at() - << "s after internal query [WON]"; - }; - } - } - if (!is_external && !cache_entry->has_internal_query_at) { - cache_entry->has_internal_query_at = td::Timestamp::now(); - if (cache_entry->has_internal_query_at && cache_entry->has_external_query_at) { - FLOG(INFO) { - prefix(sb); - sb << ": got internal query " - << cache_entry->has_internal_query_at.at() - cache_entry->has_external_query_at.at() - << "s after external query [LOST]"; - }; - } - } - if (cache_entry->result) { - auto has_result_ago = td::Timestamp::now().at() - cache_entry->has_result_at.at(); - FLOG(INFO) { - prefix(sb); - sb << ": using cached result " << " generated " << has_result_ago << "s ago"; - sb << (is_external ? " for external query [WON]" : " for internal query "); - }; - - promise.set_result(cache_entry->result.value().clone()); - return; - } - cache_entry->promises.push_back(std::move(promise)); - - if (cache_entry->started) { - FLOG(INFO) { - prefix(sb); - sb << ": collation in progress, waiting"; - }; - return; - } - FLOG(INFO) { - prefix(sb); - sb << ": starting collation"; - }; - cache_entry->started = true; - cache_entry->block_seqno = block_seqno; - run_collate_query(CollateParams{.shard = shard, - .min_masterchain_block_id = last_masterchain_state_->get_block_id(), - .prev = std::move(prev_blocks), - .validator_set = last_masterchain_state_->get_validator_set(shard), - .collator_opts = opts_->get_collator_options(), - .collator_node_id = local_id_, - .skip_store_candidate = true}, - manager_, timeout, cache_entry->cancellation_token_source.get_cancellation_token(), - [=, SelfId = actor_id(this), timer = td::Timer{}](td::Result R) { - FLOG(INFO) { - prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority); - sb << timer.elapsed() << ": " << (R.is_ok() ? "OK" : R.error().to_string()); - }; - td::actor::send_closure(SelfId, &CollatorNode::process_result, cache_entry, std::move(R)); - }); -} - -void CollatorNode::process_result(std::shared_ptr cache_entry, td::Result R) { - if (R.is_error()) { - cache_entry->started = false; - for (auto& p : cache_entry->promises) { - p.set_error(R.error().clone()); - } - } else { - cache_entry->result = R.move_as_ok(); - cache_entry->has_result_at = td::Timestamp::now(); - for (auto& p : cache_entry->promises) { - p.set_result(cache_entry->result.value().clone()); - } - } - cache_entry->promises.clear(); + td::actor::send_closure(validator_group_info.actor, &CollatorNodeSession::generate_block, std::move(prev_blocks), + priority, timeout, std::move(promise)); } td::Status CollatorNode::check_out_of_sync() { @@ -645,53 +487,54 @@ tl_object_ptr CollatorNode::serialize_candidate td::Result CollatorNode::deserialize_candidate(tl_object_ptr f, int max_decompressed_data_size, int proto_version) { td::Result res; - ton_api::downcast_call(*f, td::overloaded( - [&](ton_api::collatorNode_candidate& c) { - res = [&]() -> td::Result { - auto hash = td::sha256_bits256(c.collated_data_); - auto key = ton::PublicKey{c.source_}; - if (!key.is_ed25519()) { - return td::Status::Error("invalid pubkey"); - } - auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; - return BlockCandidate{e_key, create_block_id(c.id_), hash, std::move(c.data_), - std::move(c.collated_data_)}; - }(); - }, - [&](ton_api::collatorNode_compressedCandidate& c) { - res = [&]() -> td::Result { - if (c.decompressed_size_ <= 0) { - return td::Status::Error("invalid decompressed size"); - } - if (c.decompressed_size_ > max_decompressed_data_size) { - return td::Status::Error("decompressed size is too big"); - } - TRY_RESULT(p, validatorsession::decompress_candidate_data( - c.data_, false, c.decompressed_size_, max_decompressed_data_size, proto_version)); - auto collated_data_hash = td::sha256_bits256(p.second); - auto key = ton::PublicKey{c.source_}; - if (!key.is_ed25519()) { - return td::Status::Error("invalid pubkey"); - } - auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; - return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, - std::move(p.first), std::move(p.second)}; - }(); - }, - [&](ton_api::collatorNode_compressedCandidateV2& c) { - res = [&]() -> td::Result { - TRY_RESULT(p, validatorsession::decompress_candidate_data( - c.data_, true, 0, max_decompressed_data_size, proto_version)); - auto collated_data_hash = td::sha256_bits256(p.second); - auto key = ton::PublicKey{c.source_}; - if (!key.is_ed25519()) { - return td::Status::Error("invalid pubkey"); - } - auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; - return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, - std::move(p.first), std::move(p.second)}; - }(); - })); + ton_api::downcast_call( + *f, td::overloaded( + [&](ton_api::collatorNode_candidate& c) { + res = [&]() -> td::Result { + auto hash = td::sha256_bits256(c.collated_data_); + auto key = PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), hash, std::move(c.data_), + std::move(c.collated_data_)}; + }(); + }, + [&](ton_api::collatorNode_compressedCandidate& c) { + res = [&]() -> td::Result { + if (c.decompressed_size_ <= 0) { + return td::Status::Error("invalid decompressed size"); + } + if (c.decompressed_size_ > max_decompressed_data_size) { + return td::Status::Error("decompressed size is too big"); + } + TRY_RESULT(p, validatorsession::decompress_candidate_data(c.data_, false, c.decompressed_size_, + max_decompressed_data_size, proto_version)); + auto collated_data_hash = td::sha256_bits256(p.second); + auto key = PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, std::move(p.first), + std::move(p.second)}; + }(); + }, + [&](ton_api::collatorNode_compressedCandidateV2& c) { + res = [&]() -> td::Result { + TRY_RESULT(p, validatorsession::decompress_candidate_data(c.data_, true, 0, + max_decompressed_data_size, proto_version)); + auto collated_data_hash = td::sha256_bits256(p.second); + auto key = PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, std::move(p.first), + std::move(p.second)}; + }(); + })); return res; } diff --git a/validator/collator-node.hpp b/validator/collator-node/collator-node.hpp similarity index 74% rename from validator/collator-node.hpp rename to validator/collator-node/collator-node.hpp index a4df03681..d07116979 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node/collator-node.hpp @@ -16,6 +16,7 @@ */ #pragma once +#include "collator-node-session.hpp" #include "interfaces/validator-manager.h" #include "rldp/rldp.h" #include "rldp2/rldp.h" @@ -36,16 +37,17 @@ class CollatorNode : public td::actor::Actor { void add_shard(ShardIdFull shard); void del_shard(ShardIdFull shard); + void update_options(td::Ref opts); + void new_masterchain_block_notification(td::Ref state); void update_shard_client_handle(BlockHandle shard_client_handle); - void update_validator_group_info(ShardIdFull shard, std::vector prev, CatchainSeqno cc_seqno); - - void update_options(td::Ref opts) { - opts_ = std::move(opts); - } + void new_shard_block_accepted(BlockIdExt block_id, CatchainSeqno cc_seqno); private: void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); + void process_generate_block_query(ShardIdFull shard, CatchainSeqno cc_seqno, std::vector prev_blocks, + BlockCandidatePriority priority, td::Timestamp timeout, + td::Promise promise); void process_ping(adnl::AdnlNodeIdShort src, ton_api::collatorNode_ping& ping, td::Promise promise); bool can_collate_shard(ShardIdFull shard) const; @@ -58,28 +60,13 @@ class CollatorNode : public td::actor::Actor { std::vector collating_shards_; std::set validator_adnl_ids_; - struct CacheEntry { - bool started = false; - td::Timestamp has_internal_query_at; - td::Timestamp has_external_query_at; - td::Timestamp has_result_at; - BlockSeqno block_seqno = 0; - td::optional result; - td::CancellationTokenSource cancellation_token_source; - std::vector> promises; - - void cancel(td::Status reason); - }; struct ValidatorGroupInfo { CatchainSeqno cc_seqno{0}; std::vector prev; - BlockSeqno next_block_seqno{0}; - std::map, std::shared_ptr> cache; - - void cleanup(); + td::actor::ActorOwn actor; }; struct FutureValidatorGroup { - std::vector> pending_blocks; + std::vector pending_blocks; std::vector> promises; }; std::map validator_groups_; @@ -93,14 +80,13 @@ class CollatorNode : public td::actor::Actor { td::Result get_future_validator_group(ShardIdFull shard, CatchainSeqno cc_seqno); - void generate_block(ShardIdFull shard, CatchainSeqno cc_seqno, std::vector prev_blocks, - std::optional o_priority, td::Timestamp timeout, - td::Promise promise); - void process_result(std::shared_ptr cache_entry, td::Result R); - td::Status check_out_of_sync(); td::Status check_mc_config(); + bool can_generate() { + return check_out_of_sync().is_ok() && mc_config_status_.is_ok(); + } + public: static tl_object_ptr serialize_candidate(const BlockCandidate& block, bool compress); static td::Result deserialize_candidate(tl_object_ptr f, diff --git a/validator/manager.cpp b/validator/manager.cpp index 6a11b6bfa..4862c4727 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -238,8 +238,7 @@ void ValidatorManagerImpl::new_block_broadcast(BlockBroadcast broadcast, td::Pro void ValidatorManagerImpl::validated_block_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno) { for (auto &[_, collator_node] : collator_nodes_) { if (collator_node.can_collate_shard(block_id.shard_full())) { - td::actor::send_closure(collator_node.actor, &CollatorNode::update_validator_group_info, block_id.shard_full(), - std::vector{block_id}, cc_seqno); + td::actor::send_closure(collator_node.actor, &CollatorNode::new_shard_block_accepted, block_id, cc_seqno); } } } @@ -599,8 +598,8 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refshard())) { - td::actor::send_closure(collator_node.actor, &CollatorNode::update_validator_group_info, desc->shard(), - std::vector{desc->block_id()}, desc->catchain_seqno()); + td::actor::send_closure(collator_node.actor, &CollatorNode::new_shard_block_accepted, desc->block_id(), + desc->catchain_seqno()); } } } diff --git a/validator/manager.hpp b/validator/manager.hpp index 6f7b73447..68799c372 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -38,7 +38,7 @@ #include "storage-stat-cache.hpp" #include "validator-telemetry.hpp" #include "impl/candidates-buffer.hpp" -#include "collator-node.hpp" +#include "collator-node/collator-node.hpp" #include "shard-block-verifier.hpp" #include "shard-block-retainer.hpp" #include "td/utils/LRUCache.h" diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 6a6148a1b..163eb90bf 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -24,7 +24,7 @@ #include "common/delay.h" #include "ton/lite-tl.hpp" #include "td/utils/Random.h" -#include "collator-node.hpp" +#include "collator-node/collator-node.hpp" namespace ton { @@ -213,7 +213,7 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so return; } auto next_block_id = create_next_block_id(root_hash, file_hash); - LOG(WARNING) << "Accepted block " << next_block_id; + LOG(WARNING) << "Accepted block " << next_block_id.to_str(); stats.block_id = next_block_id; td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, std::move(stats)); auto block = From badc4fd4ff96ab80f690d9800d13982e14ab444e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 6 Aug 2025 15:10:34 +0300 Subject: [PATCH 368/388] Enable optimistic collation in proto version 6 --- validator-session/validator-session.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index f0dbbfc8d..40ecf1b2e 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -1095,7 +1095,7 @@ ValidatorSessionImpl::ValidatorSessionImpl(catchain::CatChainSessionId session_i , overlay_manager_(overlays) , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) { compress_block_candidates_ = opts.proto_version >= 4; - allow_optimistic_generation_ = opts.proto_version >= 5; + allow_optimistic_generation_ = opts.proto_version >= 6; description_ = ValidatorSessionDescription::create(std::move(opts), nodes, local_id); src_round_candidate_.resize(description_->get_total_nodes()); } From 8a49651b611c3b0ca18a8f92c8598a6fcca2c50f Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 7 Aug 2025 12:42:03 +0300 Subject: [PATCH 369/388] Allow hostnames instead of IPs for liteservers (#1771) --- adnl/adnl-ext-client.cpp | 14 ++++++++++++++ adnl/adnl-ext-client.h | 2 ++ adnl/adnl-ext-client.hpp | 4 ++++ .../blockchain-explorer-query.cpp | 6 +++--- blockchain-explorer/blockchain-explorer.cpp | 8 ++++---- blockchain-explorer/blockchain-explorer.hpp | 2 +- lite-client/ext-client.cpp | 11 +++++------ lite-client/lite-client.cpp | 2 +- lite-client/query-utils.cpp | 13 +++++++++++-- lite-client/query-utils.hpp | 9 ++++++--- tl/generate/scheme/ton_api.tl | 4 ++-- tl/generate/scheme/ton_api.tlo | Bin 119192 -> 119264 bytes utils/proxy-liteserver.cpp | 7 +++---- 13 files changed, 56 insertions(+), 26 deletions(-) diff --git a/adnl/adnl-ext-client.cpp b/adnl/adnl-ext-client.cpp index 9602b521e..873d34b11 100644 --- a/adnl/adnl-ext-client.cpp +++ b/adnl/adnl-ext-client.cpp @@ -31,6 +31,14 @@ void AdnlExtClientImpl::alarm() { next_create_at_ = td::Timestamp::in(10.0); alarm_timestamp() = next_create_at_; + if (!dst_host_.empty()) { + auto S = dst_addr_.init_host_port(dst_host_); + if (S.is_error()) { + LOG(INFO) << "failed to connect to " << dst_host_ << ": " << S; + return; + } + LOG(DEBUG) << "resolved " << dst_host_ << " -> " << dst_addr_; + } auto fd = td::SocketFd::open(dst_addr_); if (fd.is_error()) { LOG(INFO) << "failed to connect to " << dst_addr_ << ": " << fd.move_as_error(); @@ -166,6 +174,12 @@ td::actor::ActorOwn AdnlExtClient::create(AdnlNodeIdFull dst, td: return td::actor::create_actor("extclient", std::move(dst), dst_addr, std::move(callback)); } +td::actor::ActorOwn AdnlExtClient::create(AdnlNodeIdFull dst, std::string dst_host, + std::unique_ptr callback) { + return td::actor::create_actor("extclient", std::move(dst), std::move(dst_host), + std::move(callback)); +} + td::actor::ActorOwn AdnlExtClient::create(AdnlNodeIdFull dst, PrivateKey local_id, td::IPAddress dst_addr, std::unique_ptr callback) { diff --git a/adnl/adnl-ext-client.h b/adnl/adnl-ext-client.h index b9a5d570a..9e537a5a5 100644 --- a/adnl/adnl-ext-client.h +++ b/adnl/adnl-ext-client.h @@ -39,6 +39,8 @@ class AdnlExtClient : public td::actor::Actor { td::Promise promise) = 0; static td::actor::ActorOwn create(AdnlNodeIdFull dst, td::IPAddress dst_addr, std::unique_ptr callback); + static td::actor::ActorOwn create(AdnlNodeIdFull dst, std::string dst_host, + std::unique_ptr callback); static td::actor::ActorOwn create(AdnlNodeIdFull dst, PrivateKey local_id, td::IPAddress dst_addr, std::unique_ptr callback); }; diff --git a/adnl/adnl-ext-client.hpp b/adnl/adnl-ext-client.hpp index 1dd7d2ba3..76de83ab4 100644 --- a/adnl/adnl-ext-client.hpp +++ b/adnl/adnl-ext-client.hpp @@ -71,6 +71,9 @@ class AdnlExtClientImpl : public AdnlExtClient { AdnlExtClientImpl(AdnlNodeIdFull dst_id, td::IPAddress dst_addr, std::unique_ptr callback) : dst_(std::move(dst_id)), dst_addr_(dst_addr), callback_(std::move(callback)) { } + AdnlExtClientImpl(AdnlNodeIdFull dst_id, std::string dst_host, std::unique_ptr callback) + : dst_(std::move(dst_id)), dst_host_(std::move(dst_host)), callback_(std::move(callback)) { + } AdnlExtClientImpl(AdnlNodeIdFull dst_id, PrivateKey local_id, td::IPAddress dst_addr, std::unique_ptr callback) : dst_(std::move(dst_id)), local_id_(local_id), dst_addr_(dst_addr), callback_(std::move(callback)) { @@ -133,6 +136,7 @@ class AdnlExtClientImpl : public AdnlExtClient { AdnlNodeIdFull dst_; PrivateKey local_id_; td::IPAddress dst_addr_; + std::string dst_host_; std::unique_ptr callback_; diff --git a/blockchain-explorer/blockchain-explorer-query.cpp b/blockchain-explorer/blockchain-explorer-query.cpp index 26a6787e1..d540d4878 100644 --- a/blockchain-explorer/blockchain-explorer-query.cpp +++ b/blockchain-explorer/blockchain-explorer-query.cpp @@ -1429,10 +1429,10 @@ void HttpQueryStatus::finish_query() { A << "" << static_cast(x->ts_.at_unix()) << ""; } A << "\n"; - for (td::uint32 i = 0; i < results_.ips.size(); i++) { + for (td::uint32 i = 0; i < results_.addrs.size(); i++) { A << ""; - if (results_.ips[i].is_valid()) { - A << "" << results_.ips[i].get_ip_str() << ":" << results_.ips[i].get_port() << ""; + if (!results_.addrs[i].empty()) { + A << "" << results_.addrs[i] << ""; } else { A << "hidden"; } diff --git a/blockchain-explorer/blockchain-explorer.cpp b/blockchain-explorer/blockchain-explorer.cpp index ca50d5266..d21bc465c 100644 --- a/blockchain-explorer/blockchain-explorer.cpp +++ b/blockchain-explorer/blockchain-explorer.cpp @@ -187,7 +187,7 @@ class CoreActor : public CoreActorInterface { std::mutex queue_mutex_; std::mutex res_mutex_; std::map> results_; - std::vector addrs_; + std::vector addrs_; static CoreActor* instance_; td::actor::ActorId self_id_; @@ -220,7 +220,7 @@ class CoreActor : public CoreActorInterface { } void get_results(td::uint32 max, td::Promise promise) override { RemoteNodeStatusList r; - r.ips = hide_ips_ ? std::vector{addrs_.size()} : addrs_; + r.addrs = hide_ips_ ? std::vector{addrs_.size()} : addrs_; auto it = results_.rbegin(); while (it != results_.rend() && r.results.size() < max) { r.results.push_back(it->second); @@ -445,14 +445,14 @@ class CoreActor : public CoreActorInterface { r_servers.ensure(); servers = r_servers.move_as_ok(); for (const auto& serv : servers) { - addrs_.push_back(serv.addr); + addrs_.push_back(serv.hostname); } } else { if (!remote_addr_.is_valid()) { LOG(FATAL) << "remote addr not set"; } - addrs_.push_back(remote_addr_); servers.push_back(liteclient::LiteServerConfig{ton::adnl::AdnlNodeIdFull{remote_public_key_}, remote_addr_}); + addrs_.push_back(servers.back().hostname); } n_servers_ = servers.size(); client_ = liteclient::ExtClient::create(std::move(servers), make_callback(), true); diff --git a/blockchain-explorer/blockchain-explorer.hpp b/blockchain-explorer/blockchain-explorer.hpp index 1bae362de..abbbde9e9 100644 --- a/blockchain-explorer/blockchain-explorer.hpp +++ b/blockchain-explorer/blockchain-explorer.hpp @@ -59,7 +59,7 @@ class CoreActorInterface : public td::actor::Actor { }; struct RemoteNodeStatusList { - std::vector ips; + std::vector addrs; std::vector> results; }; virtual ~CoreActorInterface() = default; diff --git a/lite-client/ext-client.cpp b/lite-client/ext-client.cpp index fd624ff8c..d6b798222 100644 --- a/lite-client/ext-client.cpp +++ b/lite-client/ext-client.cpp @@ -104,7 +104,7 @@ class ExtClientImpl : public ExtClient { promise.set_result(std::move(R)); }; LOG(DEBUG) << "Sending query " << query_info.to_str() << " to server #" << server.idx << " (" - << server.config.addr.get_ip_str() << ":" << server.config.addr.get_port() << ")"; + << server.config.hostname << ")"; send_closure(server.client, &ton::adnl::AdnlExtClient::send_query, std::move(name), std::move(data), timeout, std::move(P)); } @@ -173,9 +173,9 @@ class ExtClientImpl : public ExtClient { td::actor::ActorId parent_; size_t idx_; }; - LOG(INFO) << "Connecting to liteserver #" << server.idx << " (" << server.config.addr.get_ip_str() << ":" - << server.config.addr.get_port() << ") for query " << (query_info ? query_info->to_str() : "[none]"); - server.client = ton::adnl::AdnlExtClient::create(server.config.adnl_id, server.config.addr, + LOG(INFO) << "Connecting to liteserver #" << server.idx << " (" << server.config.hostname << ") for query " + << (query_info ? query_info->to_str() : "[none]"); + server.client = ton::adnl::AdnlExtClient::create(server.config.adnl_id, server.config.hostname, std::make_unique(actor_id(this), server_idx)); } @@ -204,8 +204,7 @@ class ExtClientImpl : public ExtClient { continue; } if (server.timeout.is_in_past()) { - LOG(INFO) << "Closing connection to liteserver #" << server.idx << " (" << server.config.addr.get_ip_str() - << ":" << server.config.addr.get_port() << ")"; + LOG(INFO) << "Closing connection to liteserver #" << server.idx << " (" << server.config.hostname << ")"; server.client.reset(); server.alive = false; server.ignore_until = {}; diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index b4cc2709d..9e260dea0 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -108,7 +108,7 @@ void TestNode::run() { if (single_liteserver_idx_ != -1) { // Use single liteserver from config CHECK(single_liteserver_idx_ >= 0 && (size_t)single_liteserver_idx_ < servers.size()); td::TerminalIO::out() << "using liteserver #" << single_liteserver_idx_ << " with addr " - << servers[single_liteserver_idx_].addr << "\n"; + << servers[single_liteserver_idx_].hostname << "\n"; servers = {servers[single_liteserver_idx_]}; } } diff --git a/lite-client/query-utils.cpp b/lite-client/query-utils.cpp index b46d46a5c..7c9e7cfc7 100644 --- a/lite-client/query-utils.cpp +++ b/lite-client/query-utils.cpp @@ -318,16 +318,25 @@ bool LiteServerConfig::Slice::accepts_query(const QueryInfo& query_info) const { td::Result> LiteServerConfig::parse_global_config( const ton_api::liteclient_config_global& config) { std::vector servers; + auto get_hostname = [](const auto& f) -> std::string { + if (f->hostname_.empty()) { + return PSTRING() << td::IPAddress::ipv4_to_str(f->ip_) << ":" << f->port_; + } + if (f->port_ == 0) { + return f->hostname_; + } + return PSTRING() << f->hostname_ << ":" << f->port_; + }; for (const auto& f : config.liteservers_) { LiteServerConfig server; - TRY_STATUS(server.addr.init_host_port(td::IPAddress::ipv4_to_str(f->ip_), f->port_)); + server.hostname = get_hostname(f); server.adnl_id = adnl::AdnlNodeIdFull{PublicKey{f->id_}}; server.is_full = true; servers.push_back(std::move(server)); } for (const auto& f : config.liteservers_v2_) { LiteServerConfig server; - TRY_STATUS(server.addr.init_host_port(td::IPAddress::ipv4_to_str(f->ip_), f->port_)); + server.hostname = get_hostname(f); server.adnl_id = adnl::AdnlNodeIdFull{PublicKey{f->id_}}; server.is_full = false; for (const auto& slice_obj : f->slices_) { diff --git a/lite-client/query-utils.hpp b/lite-client/query-utils.hpp index 28500e266..d21424695 100644 --- a/lite-client/query-utils.hpp +++ b/lite-client/query-utils.hpp @@ -73,11 +73,14 @@ struct LiteServerConfig { public: ton::adnl::AdnlNodeIdFull adnl_id; - td::IPAddress addr; + std::string hostname; LiteServerConfig() = default; - LiteServerConfig(ton::adnl::AdnlNodeIdFull adnl_id, td::IPAddress addr) - : is_full(true), adnl_id(adnl_id), addr(addr) { + LiteServerConfig(ton::adnl::AdnlNodeIdFull adnl_id, std::string hostname) + : is_full(true), adnl_id(adnl_id), hostname(std::move(hostname)) { + } + LiteServerConfig(ton::adnl::AdnlNodeIdFull adnl_id, td::IPAddress ip) + : is_full(true), adnl_id(adnl_id), hostname(PSTRING() << ip.get_ip_str() << ":" << ip.get_port()) { } bool accepts_query(const QueryInfo& query_info) const; diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 38dc33489..d89000b2d 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -657,8 +657,8 @@ liteserver.descV2.sliceSimple shards:(vector tonNode.shardId) = liteserver.descV liteserver.descV2.shardInfo shard_id:tonNode.shardId seqno:int utime:int lt:long = liteserver.descV2.ShardInfo; liteserver.descV2.sliceTimed shards_from:(vector liteserver.descV2.shardInfo) shards_to:(vector liteserver.descV2.shardInfo) = liteserver.descV2.Slice; -liteserver.desc id:PublicKey ip:int port:int = liteserver.Desc; -liteserver.descV2 id:PublicKey ip:int port:int slices:(vector liteserver.descV2.Slice) = liteserver.DescV2; +liteserver.desc id:PublicKey ip:int port:int hostname:string = liteserver.Desc; +liteserver.descV2 id:PublicKey ip:int port:int hostname:string slices:(vector liteserver.descV2.Slice) = liteserver.DescV2; liteclient.config.global liteservers:(vector liteserver.desc) liteservers_v2:(vector liteserver.descV2) validator:validator.config.global = liteclient.config.Global; engine.adnl id:int256 category:int = engine.Adnl; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 3b64f7517b1ad5fa9ef9bf964ec3917078c2d098..80ba81372a715addfc20adb7639f91fea625f8b4 100644 GIT binary patch delta 177 zcmbQSnEkADhdgxo004nAN09&k delta 130 zcmaE`n0>}#_6-UuEG0`kk4#ok(O~J8;rHGgpklHcBGhr;0xUH7$bA7H1k(=i{ Hbol@P)c!H} diff --git a/utils/proxy-liteserver.cpp b/utils/proxy-liteserver.cpp index a9baa7595..e89cb1bbf 100644 --- a/utils/proxy-liteserver.cpp +++ b/utils/proxy-liteserver.cpp @@ -184,7 +184,7 @@ class ProxyLiteserver : public td::actor::Actor { for (size_t i = 0; i < servers_.size(); ++i) { Server& server = servers_[i]; - server.client = adnl::AdnlExtClient::create(server.config.adnl_id, server.config.addr, + server.client = adnl::AdnlExtClient::create(server.config.adnl_id, server.config.hostname, std::make_unique(actor_id(this), i)); server.alive = false; } @@ -197,7 +197,7 @@ class ProxyLiteserver : public td::actor::Actor { } server.alive = ready; LOG(WARNING) << (ready ? "Connected to" : "Disconnected from") << " server #" << idx << " (" - << server.config.addr.get_ip_str() << ":" << server.config.addr.get_port() << ")"; + << server.config.hostname << ")"; } void create_ext_server() { @@ -295,8 +295,7 @@ class ProxyLiteserver : public td::actor::Actor { Server& server = servers_[server_idx]; LOG(INFO) << "Sending query " << query_info.to_str() << (wait_mc_seqno_obj ? PSTRING() << " (wait seqno " << wait_mc_seqno_obj->seqno_ << ")" : "") - << ", size=" << data.size() << ", to server #" << server_idx << " (" << server.config.addr.get_ip_str() - << ":" << server.config.addr.get_port() << ")"; + << ", size=" << data.size() << ", to server #" << server_idx << " (" << server.config.hostname << ")"; BlockSeqno wait_mc_seqno = wait_mc_seqno_obj ? wait_mc_seqno_obj->seqno_ : 0; wait_mc_seqno = std::max(wait_mc_seqno, last_known_masterchain_seqno_); From 501790f3518887fff86991dbf893f82151a1fc3d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 7 Aug 2025 15:37:21 +0300 Subject: [PATCH 370/388] Optimistic collation on collator nodes --- tl/generate/scheme/ton_api.tl | 5 +- tl/generate/scheme/ton_api.tlo | Bin 119264 -> 119800 bytes ton/ton-types.h | 2 + validator-session/validator-session-types.h | 4 +- validator-session/validator-session.cpp | 10 +- validator/CMakeLists.txt | 2 + validator/collation-manager.cpp | 162 ++++++++++++++---- validator/collation-manager.hpp | 28 ++- .../collator-node/collator-node-session.cpp | 95 +++++++++- .../collator-node/collator-node-session.hpp | 26 ++- validator/collator-node/collator-node.cpp | 159 +++++++---------- validator/collator-node/collator-node.hpp | 12 +- validator/collator-node/utils.cpp | 93 ++++++++++ validator/collator-node/utils.hpp | 27 +++ validator/fabric.h | 2 +- validator/impl/collator-impl.h | 2 +- validator/impl/collator.cpp | 17 +- validator/impl/validate-query.cpp | 4 +- validator/manager-hardfork.hpp | 5 +- validator/manager.cpp | 2 +- validator/shard-block-verifier.cpp | 2 +- validator/validator-group.cpp | 2 +- 22 files changed, 480 insertions(+), 181 deletions(-) create mode 100644 validator/collator-node/utils.cpp create mode 100644 validator/collator-node/utils.hpp diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index d89000b2d..3f0757993 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -1008,7 +1008,7 @@ validatorStats.collatorNodeResponse self:int256 validator_id:int256 timestamp:do collatorNode.candidate source:PublicKey id:tonNode.blockIdExt data:bytes collated_data:bytes = collatorNode.Candidate; collatorNode.compressedCandidate flags:# source:PublicKey id:tonNode.blockIdExt decompressed_size:int data:bytes = collatorNode.Candidate; collatorNode.compressedCandidateV2 flags:# source:PublicKey id:tonNode.blockIdExt data:bytes = collatorNode.Candidate; -collatorNode.pong flags:# = collatorNode.Pong; +collatorNode.pong#5bbf0521 flags:# version:flags.0?int = collatorNode.Pong; collatorNode.error code:int message:string = collatorNode.Error; shardBlockVerifier.subscribed flags:# = shardBlockVerifier.Subscribed; @@ -1017,6 +1017,9 @@ shardBlockVerifier.confirmBlocks blocks:(vector tonNode.blockIdExt) = shardBlock ---functions--- collatorNode.generateBlock shard:tonNode.shardId cc_seqno:int prev_blocks:(vector tonNode.blockIdExt) creator:int256 round:int first_block_round:int priority:int = collatorNode.Candidate; +collatorNode.generateBlockOptimistic shard:tonNode.shardId cc_seqno:int prev_blocks:(vector tonNode.blockIdExt) + creator:int256 round:int first_block_round:int priority:int = collatorNode.Candidate; +collatorNode.requestBlockCallback flags:# block_id:tonNode.BlockIdExt = collatorNode.Candidate; collatorNode.ping flags:# = collatorNode.Pong; shardBlockVerifier.subscribe shard:tonNode.shardId flags:# = shardBlockVerifier.Subscribed; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 80ba81372a715addfc20adb7639f91fea625f8b4..ce74dbb37275ffa2745966d24b2e1a77a6d32679 100644 GIT binary patch delta 166 zcmaE`nEl6c_6-I#j7*amtF$M-`N1(=MUYWovX7TYb3)Dbgc`;djEsx6d)6_QGifVK z(fp{AoS&1ESdw4lm!FcVm!6uJT9jCl>Xeh8ob6vwl9`)XT#}j0z_7i)o^c|h;iCN= zR}@in6r~oHrWTigH8>~c prev; td::uint32 self_idx = 0; PublicKeyHash self = PublicKeyHash::zero(); - std::vector nodes; + std::vector nodes{}; tl_object_ptr tl() const { std::vector> prev_arr; @@ -257,7 +257,7 @@ struct EndValidatorGroupStats { ValidatorSessionId session_id = ValidatorSessionId::zero(); double timestamp = -1.0; PublicKeyHash self = PublicKeyHash::zero(); - std::vector nodes; + std::vector nodes{}; tl_object_ptr tl() const { std::vector> nodes_arr; diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 40ecf1b2e..4a0631407 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -336,7 +336,7 @@ void ValidatorSessionImpl::process_received_block(td::uint32 block_round, Public } } stat->deserialize_time = info.deserialize_time; - stat->serialized_size = info.serialized_size; + stat->serialized_size = (int)info.serialized_size; stat->block_id.root_hash = candidate->root_hash_; stat->block_id.file_hash = info.file_hash; stat->collated_data_hash = info.collated_data_hash; @@ -473,7 +473,7 @@ void ValidatorSessionImpl::candidate_decision_ok(td::uint32 round, ValidatorSess stat->block_status = ValidatorSessionStats::status_approved; stat->comment = PSTRING() << "ts=" << ok_from; stat->validation_time = validation_time; - stat->gen_utime = (double)ok_from; + stat->gen_utime = (int)ok_from; stat->validated_at = td::Clocks::system(); stat->validation_cached = validation_cached; } @@ -528,7 +528,11 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, GeneratedCandidate stat->block_status = ValidatorSessionStats::status_received; stat->collation_time = collation_time; stat->collated_at = td::Clocks::system(); - stat->got_block_at = td::Clocks::system(); + if (auto it = sent_candidates_.find(block_id); it != sent_candidates_.end()) { + stat->got_block_at = it->second.sent_at; + } else { + stat->got_block_at = td::Clocks::system(); + } stat->got_block_by = ValidatorSessionStats::recv_collated; stat->collation_cached = c.is_cached; stat->self_collated = c.self_collated; diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index 98987c78a..e33010fe1 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -66,6 +66,7 @@ set(VALIDATOR_HEADERS collation-manager.hpp collator-node/collator-node.hpp collator-node/collator-node-session.hpp + collator-node/utils.hpp manager-disk.h manager-disk.hpp manager-init.h @@ -84,6 +85,7 @@ set(VALIDATOR_SOURCE collation-manager.cpp collator-node/collator-node.cpp collator-node/collator-node-session.cpp + collator-node/utils.cpp get-next-key-blocks.cpp import-db-slice.cpp import-db-slice-local.cpp diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index 9260dfb50..e842b76d4 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -17,6 +17,7 @@ #include "collation-manager.hpp" #include "collator-node/collator-node.hpp" +#include "collator-node/utils.hpp" #include "fabric.h" #include "td/utils/Random.h" @@ -29,6 +30,29 @@ namespace ton::validator { void CollationManager::start_up() { td::actor::send_closure(rldp_, &rldp2::Rldp::add_id, local_id_); update_collators_list(*opts_->get_collators_list()); + + class Cb : public adnl::Adnl::Callback { + public: + explicit Cb(td::actor::ActorId id) : id_(std::move(id)) { + } + void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { + } + void receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + td::actor::send_closure(id_, &CollationManager::receive_query, src, std::move(data), std::move(promise)); + } + + private: + td::actor::ActorId id_; + }; + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_requestBlockCallback::ID), + std::make_unique(actor_id(this))); +} + +void CollationManager::tear_down() { + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_requestBlockCallback::ID)); } void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, @@ -54,25 +78,47 @@ void CollationManager::collate_block(ShardIdFull shard, BlockIdExt min_mastercha proto_version); } -void CollationManager::collate_next_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, - BlockIdExt prev_block_id, td::BufferSlice prev_block, - Ed25519_PublicKey creator, BlockCandidatePriority priority, - td::Ref validator_set, td::uint64 max_answer_size, - td::CancellationToken cancellation_token, - td::Promise promise, int proto_version) { - TRY_RESULT_PROMISE(promise, prev_block_data, create_block(prev_block_id, std::move(prev_block))); - run_collate_query( - CollateParams{.shard = shard, - .min_masterchain_block_id = min_masterchain_block_id, - .prev = {prev_block_id}, - .creator = creator, - .validator_set = std::move(validator_set), - .collator_opts = opts_->get_collator_options(), - .optimistic_prev_block_ = std::move(prev_block_data)}, - manager_, td::Timestamp::in(10.0), std::move(cancellation_token), promise.wrap([](BlockCandidate&& candidate) { - return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true}; - })); - // TODO: request to collator node +void CollationManager::collate_block_optimistic(ShardIdFull shard, BlockIdExt min_masterchain_block_id, + BlockIdExt prev_block_id, td::BufferSlice prev_block, + Ed25519_PublicKey creator, BlockCandidatePriority priority, + td::Ref validator_set, td::uint64 max_answer_size, + td::CancellationToken cancellation_token, + td::Promise promise, int proto_version) { + if (shard.is_masterchain()) { + TRY_RESULT_PROMISE(promise, prev_block_data, create_block(prev_block_id, std::move(prev_block))); + run_collate_query( + CollateParams{.shard = shard, + .min_masterchain_block_id = min_masterchain_block_id, + .prev = {prev_block_id}, + .creator = creator, + .validator_set = std::move(validator_set), + .collator_opts = opts_->get_collator_options(), + .optimistic_prev_block = std::move(prev_block_data)}, + manager_, td::Timestamp::in(10.0), std::move(cancellation_token), promise.wrap([](BlockCandidate&& candidate) { + return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true}; + })); + return; + } + + auto& entry = optimistic_prev_cache_[prev_block_id]; + entry.block_data = std::move(prev_block); + ++entry.refcnt; + promise = [this, SelfId = actor_id(this), prev_block_id, + promise = std::move(promise)](td::Result R) mutable { + promise.set_result(std::move(R)); + td::actor::send_lambda_later(SelfId, [=, this]() { + auto it = optimistic_prev_cache_.find(prev_block_id); + CHECK(it != optimistic_prev_cache_.end()); + CHECK(it->second.refcnt > 0); + if (--it->second.refcnt == 0) { + optimistic_prev_cache_.erase(it); + } + }); + }; + + collate_shard_block(shard, min_masterchain_block_id, {prev_block_id}, creator, priority, std::move(validator_set), + max_answer_size, std::move(cancellation_token), std::move(promise), td::Timestamp::in(10.0), + proto_version, true); } void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, @@ -80,7 +126,7 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, td::CancellationToken cancellation_token, td::Promise promise, td::Timestamp timeout, - int proto_version) { + int proto_version, bool is_optimistic) { TRY_STATUS_PROMISE(promise, cancellation_token.check()); ShardInfo* s = select_shard_info(shard); if (s == nullptr) { @@ -91,12 +137,22 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas adnl::AdnlNodeIdShort selected_collator = adnl::AdnlNodeIdShort::zero(); size_t selected_idx = 0; + auto check_collator = [&](const adnl::AdnlNodeIdShort& id) -> bool { + auto& collator = collators_[id]; + if (!collator.alive) { + return false; + } + if (is_optimistic && collator.version < CollatorNode::VERSION_OPTIMISTIC_COLLATE) { + return false; + } + return true; + }; switch (s->select_mode) { case CollatorsList::mode_random: { int cnt = 0; for (size_t i = 0; i < s->collators.size(); ++i) { adnl::AdnlNodeIdShort collator = s->collators[i]; - if (collators_[collator].alive) { + if (check_collator(collator)) { ++cnt; if (td::Random::fast(1, cnt) == 1) { selected_collator = collator; @@ -109,7 +165,7 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas case CollatorsList::mode_ordered: { for (size_t i = 0; i < s->collators.size(); ++i) { adnl::AdnlNodeIdShort collator = s->collators[i]; - if (collators_[collator].alive) { + if (check_collator(collator)) { selected_collator = collator; selected_idx = i; break; @@ -121,7 +177,7 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas size_t iters = 0; for (size_t i = s->cur_idx; iters < s->collators.size(); (++i) %= s->collators.size(), ++iters) { adnl::AdnlNodeIdShort& collator = s->collators[i]; - if (collators_[collator].alive) { + if (check_collator(collator)) { selected_collator = collator; selected_idx = i; s->cur_idx = (i + 1) % s->collators.size(); @@ -133,13 +189,20 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas } if (selected_collator.is_zero() && s->self_collate) { + td::Ref optimistic_prev_block; + if (is_optimistic) { + CHECK(prev.size() == 1); + TRY_RESULT_PROMISE_ASSIGN(promise, optimistic_prev_block, + create_block(prev[0], optimistic_prev_cache_.at(prev[0]).block_data.clone())); + } run_collate_query( CollateParams{.shard = shard, .min_masterchain_block_id = min_masterchain_block_id, .prev = std::move(prev), .creator = creator, .validator_set = std::move(validator_set), - .collator_opts = opts_->get_collator_options()}, + .collator_opts = opts_->get_collator_options(), + .optimistic_prev_block = std::move(optimistic_prev_block)}, manager_, td::Timestamp::in(10.0), std::move(cancellation_token), promise.wrap([](BlockCandidate&& candidate) { return GeneratedCandidate{.candidate = std::move(candidate), .is_cached = false, .self_collated = true}; })); @@ -175,19 +238,26 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas [=, promise = std::move(promise)]() mutable { td::actor::send_closure(SelfId, &CollationManager::collate_shard_block, shard, min_masterchain_block_id, prev, creator, priority, validator_set, max_answer_size, cancellation_token, - std::move(promise), timeout, proto_version); + std::move(promise), timeout, proto_version, is_optimistic); }, retry_at); }; if (selected_collator.is_zero()) { - P.set_error(td::Status::Error(PSTRING() << "shard " << shard.to_str() << " has no alive collator node")); + P.set_error(td::Status::Error(PSTRING() << "shard " << shard.to_str() << " has no suitable collator node")); return; } - td::BufferSlice query = create_serialize_tl_object( - create_tl_shard_id(shard), validator_set->get_catchain_seqno(), std::move(prev_blocks), creator.as_bits256(), - priority.round, priority.first_block_round, priority.priority); + td::BufferSlice query; + if (is_optimistic) { + query = create_serialize_tl_object( + create_tl_shard_id(shard), validator_set->get_catchain_seqno(), std::move(prev_blocks), creator.as_bits256(), + priority.round, priority.first_block_round, priority.priority); + } else { + query = create_serialize_tl_object( + create_tl_shard_id(shard), validator_set->get_catchain_seqno(), std::move(prev_blocks), creator.as_bits256(), + priority.round, priority.first_block_round, priority.priority); + } LOG(INFO) << "sending collate query for " << next_block_id.to_str() << ": send to #" << selected_idx << "(" << selected_collator << ")"; @@ -201,9 +271,8 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas return; } TRY_RESULT_PROMISE(P, f, fetch_tl_object(data, true)); - TRY_RESULT_PROMISE( - P, candidate, - CollatorNode::deserialize_candidate(std::move(f), td::narrow_cast(max_answer_size), proto_version)); + TRY_RESULT_PROMISE(P, candidate, + deserialize_candidate(std::move(f), td::narrow_cast(max_answer_size), proto_version)); if (candidate.pubkey.as_bits256() != creator.as_bits256()) { P.set_error(td::Status::Error("collate query: block candidate source mismatch")); return; @@ -363,7 +432,8 @@ void CollationManager::alarm() { } if (collator.ping_at.is_in_past()) { collator.sent_ping = true; - td::BufferSlice query = create_serialize_tl_object(0); + td::BufferSlice query = + create_serialize_tl_object(ton_api::collatorNode_pong::VERSION_MASK); td::Promise P = [=, id = id, SelfId = actor_id(this)](td::Result R) mutable { td::actor::send_closure(SelfId, &CollationManager::got_pong, id, std::move(R)); }; @@ -399,9 +469,11 @@ void CollationManager::got_pong(adnl::AdnlNodeIdShort id, td::Resultflags_ & ton_api::collatorNode_pong::VERSION_MASK ? pong->version_ : 0; + LOG(DEBUG) << "pong from " << id << " : OK, version=" << collator.version; } collator.ping_at = td::Timestamp::in(td::Random::fast(10.0, 20.0)); if (collator.active_cnt && !collator.sent_ping) { @@ -421,4 +493,26 @@ void CollationManager::on_collate_query_error(adnl::AdnlNodeIdShort id) { } } +void CollationManager::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, + td::Promise promise) { + if (!collators_.contains(src)) { + promise.set_error(td::Status::Error("got request from unknown collator")); + return; + } + TRY_RESULT_PROMISE(promise, query, fetch_tl_object(data, true)); + BlockIdExt block_id = create_block_id(query->block_id_); + auto it = optimistic_prev_cache_.find(block_id); + if (it == optimistic_prev_cache_.end()) { + LOG(INFO) << "collatorNode.requestBlockCallback from " << src << " block " << block_id.to_str() << " : not found"; + promise.set_error(td::Status::Error("block not found")); + return; + } + LOG(INFO) << "collatorNode.requestBlockCallback from " << src << " block " << block_id.to_str() << " : OK"; + promise.set_value( + serialize_tl_object(serialize_candidate(BlockCandidate(Ed25519_PublicKey{td::Bits256::zero()}, block_id, + td::Bits256::zero(), it->second.block_data.clone(), {}), + true), + true)); +} + } // namespace ton::validator diff --git a/validator/collation-manager.hpp b/validator/collation-manager.hpp index a64d43289..37a45fb38 100644 --- a/validator/collation-manager.hpp +++ b/validator/collation-manager.hpp @@ -27,11 +27,13 @@ class ValidatorManager; class CollationManager : public td::actor::Actor { public: CollationManager(adnl::AdnlNodeIdShort local_id, td::Ref opts, - td::actor::ActorId manager, td::actor::ActorId rldp) - : local_id_(local_id), opts_(opts), manager_(manager), rldp_(rldp) { + td::actor::ActorId manager, td::actor::ActorId adnl, + td::actor::ActorId rldp) + : local_id_(local_id), opts_(opts), manager_(manager), adnl_(adnl), rldp_(rldp) { } void start_up() override; + void tear_down() override; void alarm() override; void collate_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, @@ -39,11 +41,11 @@ class CollationManager : public td::actor::Actor { td::uint64 max_answer_size, td::CancellationToken cancellation_token, td::Promise promise, int proto_version); - void collate_next_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, BlockIdExt prev_block_id, - td::BufferSlice prev_block, Ed25519_PublicKey creator, BlockCandidatePriority priority, - td::Ref validator_set, td::uint64 max_answer_size, - td::CancellationToken cancellation_token, td::Promise promise, - int proto_version); + void collate_block_optimistic(ShardIdFull shard, BlockIdExt min_masterchain_block_id, BlockIdExt prev_block_id, + td::BufferSlice prev_block, Ed25519_PublicKey creator, BlockCandidatePriority priority, + td::Ref validator_set, td::uint64 max_answer_size, + td::CancellationToken cancellation_token, td::Promise promise, + int proto_version); void update_options(td::Ref opts); @@ -56,13 +58,14 @@ class CollationManager : public td::actor::Actor { adnl::AdnlNodeIdShort local_id_; td::Ref opts_; td::actor::ActorId manager_; + td::actor::ActorId adnl_; td::actor::ActorId rldp_; void collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, Ed25519_PublicKey creator, BlockCandidatePriority priority, td::Ref validator_set, td::uint64 max_answer_size, td::CancellationToken cancellation_token, td::Promise promise, - td::Timestamp timeout, int proto_version); + td::Timestamp timeout, int proto_version, bool is_optimistic = false); void update_collators_list(const CollatorsList& collators_list); @@ -73,6 +76,7 @@ class CollationManager : public td::actor::Actor { size_t active_cnt = 0; td::Timestamp last_ping_at = td::Timestamp::never(); td::Status last_ping_status = td::Status::Error("not pinged"); + int version = -1; }; std::map collators_; @@ -92,6 +96,14 @@ class CollationManager : public td::actor::Actor { ShardInfo* select_shard_info(ShardIdFull shard); void got_pong(adnl::AdnlNodeIdShort id, td::Result R); void on_collate_query_error(adnl::AdnlNodeIdShort id); + + void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); + + struct OptimisticPrevCache { + td::BufferSlice block_data; + size_t refcnt = 0; + }; + std::map optimistic_prev_cache_; }; } // namespace ton::validator diff --git a/validator/collator-node/collator-node-session.cpp b/validator/collator-node/collator-node-session.cpp index 76c6f9974..8e131f3c0 100644 --- a/validator/collator-node/collator-node-session.cpp +++ b/validator/collator-node/collator-node-session.cpp @@ -17,7 +17,9 @@ #include "collator-node-session.hpp" +#include "collator-node.hpp" #include "fabric.h" +#include "utils.hpp" namespace ton::validator { @@ -31,7 +33,7 @@ static BlockSeqno get_next_block_seqno(const std::vector& prev) { CollatorNodeSession::CollatorNodeSession(ShardIdFull shard, std::vector prev, td::Ref validator_set, BlockIdExt min_masterchain_block_id, - bool can_generate, adnl::AdnlNodeIdShort local_id, + bool can_generate, Ref state, adnl::AdnlNodeIdShort local_id, td::Ref opts, td::actor::ActorId manager, td::actor::ActorId adnl, td::actor::ActorId rldp) @@ -46,6 +48,7 @@ CollatorNodeSession::CollatorNodeSession(ShardIdFull shard, std::vectorget_catchain_seqno() << ", next block seqno " << next_block_seqno_; if (can_generate_) { - generate_block(prev_, {}, td::Timestamp::in(10.0), [](td::Result) {}); + generate_block(prev_, {}, {}, td::Timestamp::in(10.0), [](td::Result) {}); } } @@ -101,14 +104,22 @@ void CollatorNodeSession::new_shard_block_accepted(BlockIdExt block_id, bool can } if (can_generate_) { - generate_block(prev_, {}, td::Timestamp::in(10.0), [](td::Result) {}); + generate_block(prev_, {}, {}, td::Timestamp::in(10.0), [](td::Result) {}); } } +void CollatorNodeSession::update_masterchain_config(td::Ref state) { + ValidatorSessionConfig config = state->get_consensus_config(); + proto_version_ = config.proto_version; + max_candidate_size_ = config.max_block_size + config.max_collated_data_size + 1024; +} + void CollatorNodeSession::generate_block(std::vector prev_blocks, - td::optional o_priority, td::Timestamp timeout, + td::optional o_priority, + td::Ref o_optimistic_prev_block, td::Timestamp timeout, td::Promise promise) { bool is_external = !o_priority; + bool is_optimistic = o_optimistic_prev_block.not_null(); BlockSeqno block_seqno = get_next_block_seqno(prev_blocks); if (next_block_seqno_ > block_seqno) { promise.set_error(td::Status::Error(PSTRING() << "next block seqno " << block_seqno << " is too old, expected " @@ -126,7 +137,8 @@ void CollatorNodeSession::generate_block(std::vector prev_blocks, } static auto prefix_inner = [](td::StringBuilder& sb, const ShardIdFull& shard, CatchainSeqno cc_seqno, - BlockSeqno block_seqno, const td::optional& o_priority) { + BlockSeqno block_seqno, const td::optional& o_priority, + bool is_optimistic) { sb << "generate block query" << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno << ", next_block_seqno=" << block_seqno; if (o_priority) { @@ -138,9 +150,12 @@ void CollatorNodeSession::generate_block(std::vector prev_blocks, } else { sb << " internal"; } + if (is_optimistic) { + sb << " opt"; + } }; auto prefix = [&](td::StringBuilder& sb) { - prefix_inner(sb, shard_, validator_set_->get_catchain_seqno(), block_seqno, o_priority); + prefix_inner(sb, shard_, validator_set_->get_catchain_seqno(), block_seqno, o_priority, is_optimistic); }; auto cache_entry = cache_[prev_blocks]; @@ -198,13 +213,14 @@ void CollatorNodeSession::generate_block(std::vector prev_blocks, .validator_set = validator_set_, .collator_opts = opts_->get_collator_options(), .collator_node_id = local_id_, - .skip_store_candidate = true}, + .skip_store_candidate = true, + .optimistic_prev_block = o_optimistic_prev_block}, manager_, timeout, cache_entry->cancellation_token_source.get_cancellation_token(), [=, shard = shard_, cc_seqno = validator_set_->get_catchain_seqno(), SelfId = actor_id(this), timer = td::Timer{}](td::Result R) mutable { FLOG(INFO) { - prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority); - sb << timer.elapsed() << ": " << (R.is_ok() ? "OK" : R.error().to_string()); + prefix_inner(sb, shard, cc_seqno, block_seqno, o_priority, is_optimistic); + sb << ": " << (R.is_ok() ? "OK" : R.error().to_string()) << " time=" << timer.elapsed(); }; td::actor::send_closure(SelfId, &CollatorNodeSession::process_result, cache_entry, std::move(R)); }); @@ -226,6 +242,67 @@ void CollatorNodeSession::process_result(std::shared_ptr cache_entry cache_entry->promises.clear(); } +void CollatorNodeSession::process_request(adnl::AdnlNodeIdShort src, std::vector prev_blocks, + BlockCandidatePriority priority, bool is_optimistic, td::Timestamp timeout, + td::Promise promise) { + if (is_optimistic) { + if (prev_blocks.size() != 1) { + promise.set_error(td::Status::Error("optimistic collation, expected 1 prev block")); + return; + } + auto it = cache_.find(prev_blocks); + if (it == cache_.end() || it->second->started) { + BlockIdExt prev_block = prev_blocks[0]; + td::actor::send_closure( + manager_, &ValidatorManager::get_candidate_data_by_block_id_from_db, prev_block, + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + td::actor::send_closure(SelfId, &CollatorNodeSession::process_request_optimistic_cont, src, prev_block, + priority, timeout, std::move(promise), std::move(R)); + }); + return; + } + } + generate_block(std::move(prev_blocks), priority, {}, timeout, std::move(promise)); +} + +void CollatorNodeSession::process_request_optimistic_cont(adnl::AdnlNodeIdShort src, BlockIdExt prev_block_id, + BlockCandidatePriority priority, td::Timestamp timeout, + td::Promise promise, + td::Result prev_block_data) { + if (prev_block_data.is_ok()) { + TRY_RESULT_PROMISE_PREFIX(promise, prev_block, create_block(prev_block_id, prev_block_data.move_as_ok()), + "invalid prev block data in db: "); + LOG(INFO) << "got prev block from db for optimistic collation: " << prev_block_id.to_str(); + generate_block({prev_block_id}, priority, prev_block, timeout, std::move(promise)); + return; + } + td::actor::send_closure( + rldp_, &rldp2::Rldp::send_query_ex, local_id_, src, "getprevblock", + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + td::actor::send_closure(SelfId, &CollatorNodeSession::process_request_optimistic_cont2, prev_block_id, priority, + timeout, std::move(promise), std::move(R)); + }, + timeout, + create_serialize_tl_object(0, create_tl_block_id(prev_block_id)), + max_candidate_size_); +} + +void CollatorNodeSession::process_request_optimistic_cont2(BlockIdExt prev_block_id, BlockCandidatePriority priority, + td::Timestamp timeout, td::Promise promise, + td::Result R) { + TRY_RESULT_PROMISE_PREFIX(promise, response, std::move(R), + "failed to download prev block data for optimistic collation: "); + TRY_RESULT_PROMISE_PREFIX(promise, f, fetch_tl_object(response, true), + "failed to download prev block data for optimistic collation: "); + TRY_RESULT_PROMISE_PREFIX(promise, candidate, + deserialize_candidate(std::move(f), max_candidate_size_, proto_version_), + "failed to download prev block data for optimistic collation: "); + TRY_RESULT_PROMISE_PREFIX(promise, prev_block, create_block(prev_block_id, std::move(candidate.data)), + "invalid prev block data from validator: "); + LOG(INFO) << "got prev block from validator for optimistic collation: " << prev_block_id.to_str(); + generate_block({prev_block_id}, priority, prev_block, timeout, std::move(promise)); +} + void CollatorNodeSession::CacheEntry::cancel(td::Status reason) { for (auto& promise : promises) { promise.set_error(reason.clone()); diff --git a/validator/collator-node/collator-node-session.hpp b/validator/collator-node/collator-node-session.hpp index 51e1b7cf6..0f9f03d30 100644 --- a/validator/collator-node/collator-node-session.hpp +++ b/validator/collator-node/collator-node-session.hpp @@ -29,9 +29,10 @@ class ValidatorManager; class CollatorNodeSession : public td::actor::Actor { public: CollatorNodeSession(ShardIdFull shard, std::vector prev, td::Ref validator_set, - BlockIdExt min_masterchain_block_id, bool can_generate, adnl::AdnlNodeIdShort local_id, - td::Ref opts, td::actor::ActorId manager, - td::actor::ActorId adnl, td::actor::ActorId rldp); + BlockIdExt min_masterchain_block_id, bool can_generate, td::Ref state, + adnl::AdnlNodeIdShort local_id, td::Ref opts, + td::actor::ActorId manager, td::actor::ActorId adnl, + td::actor::ActorId rldp); void start_up() override; void tear_down() override; @@ -42,8 +43,9 @@ class CollatorNodeSession : public td::actor::Actor { void new_shard_block_accepted(BlockIdExt block_id, bool can_generate); - void generate_block(std::vector prev_blocks, td::optional o_priority, - td::Timestamp timeout, td::Promise promise); + void process_request(adnl::AdnlNodeIdShort src, std::vector prev_blocks, BlockCandidatePriority priority, + bool is_optimistic, td::Timestamp timeout, td::Promise promise); + void update_masterchain_config(td::Ref state); private: ShardIdFull shard_; @@ -73,7 +75,21 @@ class CollatorNodeSession : public td::actor::Actor { BlockSeqno next_block_seqno_; std::map, std::shared_ptr> cache_; + td::uint32 proto_version_ = 0; + td::uint32 max_candidate_size_ = 0; + + void generate_block(std::vector prev_blocks, td::optional o_priority, + td::Ref o_optimistic_prev_block, td::Timestamp timeout, + td::Promise promise); void process_result(std::shared_ptr cache_entry, td::Result R); + + void process_request_optimistic_cont(adnl::AdnlNodeIdShort src, BlockIdExt prev_block_id, + BlockCandidatePriority priority, td::Timestamp timeout, + td::Promise promise, + td::Result prev_block_data); + void process_request_optimistic_cont2(BlockIdExt prev_block_id, BlockCandidatePriority priority, + td::Timestamp timeout, td::Promise promise, + td::Result R); }; } // namespace ton::validator diff --git a/validator/collator-node/collator-node.cpp b/validator/collator-node/collator-node.cpp index 6f0cae419..dd0dfb792 100644 --- a/validator/collator-node/collator-node.cpp +++ b/validator/collator-node/collator-node.cpp @@ -23,7 +23,7 @@ #include "checksum.h" #include "impl/collator-impl.h" #include "impl/shard.hpp" -#include "validator-session/candidate-serializer.h" +#include "utils.hpp" namespace ton::validator { @@ -55,6 +55,9 @@ void CollatorNode::start_up() { td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlock::ID), std::make_unique(actor_id(this))); + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlockOptimistic::ID), + std::make_unique(actor_id(this))); td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, adnl::Adnl::int_to_bytestring(ton_api::collatorNode_ping::ID), std::make_unique(actor_id(this))); @@ -64,6 +67,8 @@ void CollatorNode::start_up() { void CollatorNode::tear_down() { td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlock::ID)); + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, + adnl::Adnl::int_to_bytestring(ton_api::collatorNode_generateBlockOptimistic::ID)); td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id_, adnl::Adnl::int_to_bytestring(ton_api::collatorNode_ping::ID)); } @@ -83,7 +88,7 @@ void CollatorNode::add_shard(ShardIdFull shard) { validator_group.actor = td::actor::create_actor( PSTRING() << "collatornode" << shard.to_str(), shard, validator_group.prev, last_masterchain_state_->get_validator_set(group_shard), last_masterchain_state_->get_block_id(), - can_generate(), local_id_, opts_, manager_, adnl_, rldp_); + can_generate(), last_masterchain_state_, local_id_, opts_, manager_, adnl_, rldp_); } } } @@ -118,9 +123,7 @@ void CollatorNode::new_masterchain_block_notification(td::Ref if (mc_config_status_.is_error()) { LOG(ERROR) << "Cannot validate masterchain config (possibly outdated software): " << mc_config_status_; } - } - if (validator_adnl_ids_.empty() || state->is_key_state()) { validator_adnl_ids_.clear(); for (int next : {-1, 0, 1}) { td::Ref vals = state->get_total_validator_set(next); @@ -135,6 +138,11 @@ void CollatorNode::new_masterchain_block_notification(td::Ref } } } + for (auto& [_, group] : validator_groups_) { + if (!group.actor.empty()) { + td::actor::send_closure(group.actor, &CollatorNodeSession::update_masterchain_config, state); + } + } } std::map> new_shards; @@ -173,7 +181,8 @@ void CollatorNode::new_masterchain_block_notification(td::Ref if (can_collate_shard(shard)) { it->second.actor = td::actor::create_actor( PSTRING() << "collatornode" << shard.to_str(), shard, it->second.prev, validator_set, - last_masterchain_state_->get_block_id(), can_generate(), local_id_, opts_, manager_, adnl_, rldp_); + last_masterchain_state_->get_block_id(), can_generate(), last_masterchain_state_, local_id_, opts_, + manager_, adnl_, rldp_); } } else if (!it->second.actor.empty() && prev.size() == 1) { td::actor::send_closure(it->second.actor, &CollatorNodeSession::new_shard_block_accepted, prev[0], @@ -250,11 +259,11 @@ td::Result CollatorNode::get_future_validat } if (cc_seqno < it->second.cc_seqno) { // past validator group return td::Status::Error(PSTRING() << "cc_seqno " << cc_seqno << " for shard " << shard.to_str() - << " is outdated (current is" << it->second.cc_seqno << ")"); + << " is outdated (current is " << it->second.cc_seqno << ")"); } if (cc_seqno - it->second.cc_seqno > 1) { // future validator group, cc_seqno too big return td::Status::Error(PSTRING() << "cc_seqno " << cc_seqno << " for shard " << shard.to_str() - << " is too big (currently known is" << it->second.cc_seqno << ")"); + << " is too big (currently known is " << it->second.cc_seqno << ")"); } // future validator group return &future_validator_groups_[{shard, cc_seqno}]; @@ -326,17 +335,39 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data return; } - TRY_RESULT_PROMISE(promise, f, fetch_tl_object(data, true)); - ShardIdFull shard = create_shard_id(f->shard_); - CatchainSeqno cc_seqno = f->cc_seqno_; + bool is_optimistic = false; + ShardIdFull shard; + CatchainSeqno cc_seqno; std::vector prev_blocks; - for (const auto& b : f->prev_blocks_) { - prev_blocks.push_back(create_block_id(b)); + BlockCandidatePriority priority; + Ed25519_PublicKey creator; + if (auto R = fetch_tl_object(data, true); R.is_ok()) { + auto f = R.move_as_ok(); + shard = create_shard_id(f->shard_); + cc_seqno = f->cc_seqno_; + for (const auto& b : f->prev_blocks_) { + prev_blocks.push_back(create_block_id(b)); + } + priority = BlockCandidatePriority{.round = static_cast(f->round_), + .first_block_round = static_cast(f->first_block_round_), + .priority = f->priority_}; + creator = Ed25519_PublicKey(f->creator_); + } else if (auto R = fetch_tl_object(data, true); R.is_ok()) { + is_optimistic = true; + auto f = R.move_as_ok(); + shard = create_shard_id(f->shard_); + cc_seqno = f->cc_seqno_; + for (const auto& b : f->prev_blocks_) { + prev_blocks.push_back(create_block_id(b)); + } + priority = BlockCandidatePriority{.round = static_cast(f->round_), + .first_block_round = static_cast(f->first_block_round_), + .priority = f->priority_}; + creator = Ed25519_PublicKey(f->creator_); + } else { + promise.set_error(td::Status::Error("cannot parse request")); + return; } - auto priority = BlockCandidatePriority{.round = static_cast(f->round_), - .first_block_round = static_cast(f->first_block_round_), - .priority = f->priority_}; - Ed25519_PublicKey creator(f->creator_); td::Promise new_promise = [promise = std::move(promise), src, shard](td::Result R) mutable { if (R.is_error()) { @@ -378,14 +409,16 @@ void CollatorNode::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data new_promise.set_error(td::Status::Error(PSTRING() << "invalid size of prev_blocks: " << prev_blocks.size())); return; } - LOG(INFO) << "got adnl query from " << src << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno; - process_generate_block_query(shard, cc_seqno, std::move(prev_blocks), priority, td::Timestamp::in(10.0), - std::move(new_promise)); + LOG(INFO) << "got adnl query from " << src << ": shard=" << shard.to_str() << ", cc_seqno=" << cc_seqno + << (is_optimistic ? ", optimistic" : ""); + process_generate_block_query(src, shard, cc_seqno, std::move(prev_blocks), priority, is_optimistic, + td::Timestamp::in(10.0), std::move(new_promise)); } -void CollatorNode::process_generate_block_query(ShardIdFull shard, CatchainSeqno cc_seqno, +void CollatorNode::process_generate_block_query(adnl::AdnlNodeIdShort src, ShardIdFull shard, CatchainSeqno cc_seqno, std::vector prev_blocks, BlockCandidatePriority priority, - td::Timestamp timeout, td::Promise promise) { + bool is_optimistic, td::Timestamp timeout, + td::Promise promise) { if (last_masterchain_state_.is_null()) { promise.set_error(td::Status::Error(ErrorCode::notready, "not ready")); return; @@ -400,8 +433,8 @@ void CollatorNode::process_generate_block_query(ShardIdFull shard, CatchainSeqno future_validator_group->promises.push_back([=, SelfId = actor_id(this), prev_blocks = std::move(prev_blocks), promise = std::move(promise)](td::Result R) mutable { TRY_STATUS_PROMISE(promise, R.move_as_status()); - td::actor::send_closure(SelfId, &CollatorNode::process_generate_block_query, shard, cc_seqno, - std::move(prev_blocks), std::move(priority), timeout, std::move(promise)); + td::actor::send_closure(SelfId, &CollatorNode::process_generate_block_query, src, shard, cc_seqno, + std::move(prev_blocks), std::move(priority), is_optimistic, timeout, std::move(promise)); }); return; } @@ -410,8 +443,8 @@ void CollatorNode::process_generate_block_query(ShardIdFull shard, CatchainSeqno promise.set_error(td::Status::Error(PSTRING() << "cannot collate shard " << shard.to_str())); return; } - td::actor::send_closure(validator_group_info.actor, &CollatorNodeSession::generate_block, std::move(prev_blocks), - priority, timeout, std::move(promise)); + td::actor::send_closure(validator_group_info.actor, &CollatorNodeSession::process_request, src, + std::move(prev_blocks), priority, is_optimistic, timeout, std::move(promise)); } td::Status CollatorNode::check_out_of_sync() { @@ -461,7 +494,12 @@ void CollatorNode::process_ping(adnl::AdnlNodeIdShort src, ton_api::collatorNode LOG(DEBUG) << "got ping from " << src; TRY_STATUS_PROMISE(promise, check_out_of_sync()); TRY_STATUS_PROMISE_PREFIX(promise, mc_config_status_.clone(), "unsupported mc config: "); - promise.set_result(create_serialize_tl_object(0)); + auto pong = create_tl_object(); + if (ping.flags_ & ton_api::collatorNode_pong::VERSION_MASK) { + pong->flags_ |= ton_api::collatorNode_pong::VERSION_MASK; + pong->version_ = COLLATOR_NODE_VERSION; + } + promise.set_result(serialize_tl_object(pong, true)); } bool CollatorNode::can_collate_shard(ShardIdFull shard) const { @@ -469,73 +507,4 @@ bool CollatorNode::can_collate_shard(ShardIdFull shard) const { [&](const ShardIdFull& our_shard) { return shard_intersects(shard, our_shard); }); } -tl_object_ptr CollatorNode::serialize_candidate(const BlockCandidate& block, - bool compress) { - if (!compress) { - return create_tl_object( - PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), block.data.clone(), - block.collated_data.clone()); - } - size_t decompressed_size; - td::BufferSlice compressed = - validatorsession::compress_candidate_data(block.data, block.collated_data, decompressed_size).move_as_ok(); - return create_tl_object( - 0, PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), - (int)decompressed_size, std::move(compressed)); -} - -td::Result CollatorNode::deserialize_candidate(tl_object_ptr f, - int max_decompressed_data_size, int proto_version) { - td::Result res; - ton_api::downcast_call( - *f, td::overloaded( - [&](ton_api::collatorNode_candidate& c) { - res = [&]() -> td::Result { - auto hash = td::sha256_bits256(c.collated_data_); - auto key = PublicKey{c.source_}; - if (!key.is_ed25519()) { - return td::Status::Error("invalid pubkey"); - } - auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; - return BlockCandidate{e_key, create_block_id(c.id_), hash, std::move(c.data_), - std::move(c.collated_data_)}; - }(); - }, - [&](ton_api::collatorNode_compressedCandidate& c) { - res = [&]() -> td::Result { - if (c.decompressed_size_ <= 0) { - return td::Status::Error("invalid decompressed size"); - } - if (c.decompressed_size_ > max_decompressed_data_size) { - return td::Status::Error("decompressed size is too big"); - } - TRY_RESULT(p, validatorsession::decompress_candidate_data(c.data_, false, c.decompressed_size_, - max_decompressed_data_size, proto_version)); - auto collated_data_hash = td::sha256_bits256(p.second); - auto key = PublicKey{c.source_}; - if (!key.is_ed25519()) { - return td::Status::Error("invalid pubkey"); - } - auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; - return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, std::move(p.first), - std::move(p.second)}; - }(); - }, - [&](ton_api::collatorNode_compressedCandidateV2& c) { - res = [&]() -> td::Result { - TRY_RESULT(p, validatorsession::decompress_candidate_data(c.data_, true, 0, - max_decompressed_data_size, proto_version)); - auto collated_data_hash = td::sha256_bits256(p.second); - auto key = PublicKey{c.source_}; - if (!key.is_ed25519()) { - return td::Status::Error("invalid pubkey"); - } - auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; - return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, std::move(p.first), - std::move(p.second)}; - }(); - })); - return res; -} - } // namespace ton::validator diff --git a/validator/collator-node/collator-node.hpp b/validator/collator-node/collator-node.hpp index d07116979..cb9cec47c 100644 --- a/validator/collator-node/collator-node.hpp +++ b/validator/collator-node/collator-node.hpp @@ -45,9 +45,9 @@ class CollatorNode : public td::actor::Actor { private: void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); - void process_generate_block_query(ShardIdFull shard, CatchainSeqno cc_seqno, std::vector prev_blocks, - BlockCandidatePriority priority, td::Timestamp timeout, - td::Promise promise); + void process_generate_block_query(adnl::AdnlNodeIdShort src, ShardIdFull shard, CatchainSeqno cc_seqno, + std::vector prev_blocks, BlockCandidatePriority priority, + bool is_optimistic, td::Timestamp timeout, td::Promise promise); void process_ping(adnl::AdnlNodeIdShort src, ton_api::collatorNode_ping& ping, td::Promise promise); bool can_collate_shard(ShardIdFull shard) const; @@ -87,10 +87,10 @@ class CollatorNode : public td::actor::Actor { return check_out_of_sync().is_ok() && mc_config_status_.is_ok(); } + static constexpr int COLLATOR_NODE_VERSION = 1; + public: - static tl_object_ptr serialize_candidate(const BlockCandidate& block, bool compress); - static td::Result deserialize_candidate(tl_object_ptr f, - int max_decompressed_data_size, int proto_version); + static constexpr int VERSION_OPTIMISTIC_COLLATE = 1; }; } // namespace ton::validator diff --git a/validator/collator-node/utils.cpp b/validator/collator-node/utils.cpp new file mode 100644 index 000000000..23e2ec40f --- /dev/null +++ b/validator/collator-node/utils.cpp @@ -0,0 +1,93 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "utils.hpp" +#include "checksum.h" +#include "keys/keys.hpp" +#include "ton/ton-tl.hpp" +#include "validator-session/candidate-serializer.h" + +namespace ton::validator { + +tl_object_ptr serialize_candidate(const BlockCandidate& block, bool compress) { + if (!compress) { + return create_tl_object( + PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), block.data.clone(), + block.collated_data.clone()); + } + size_t decompressed_size; + td::BufferSlice compressed = + validatorsession::compress_candidate_data(block.data, block.collated_data, decompressed_size).move_as_ok(); + return create_tl_object( + 0, PublicKey{pubkeys::Ed25519{block.pubkey.as_bits256()}}.tl(), create_tl_block_id(block.id), + (int)decompressed_size, std::move(compressed)); +} + +td::Result deserialize_candidate(tl_object_ptr f, + int max_decompressed_data_size, int proto_version) { + td::Result res; + ton_api::downcast_call( + *f, td::overloaded( + [&](ton_api::collatorNode_candidate& c) { + res = [&]() -> td::Result { + auto hash = td::sha256_bits256(c.collated_data_); + auto key = PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), hash, std::move(c.data_), + std::move(c.collated_data_)}; + }(); + }, + [&](ton_api::collatorNode_compressedCandidate& c) { + res = [&]() -> td::Result { + if (c.decompressed_size_ <= 0) { + return td::Status::Error("invalid decompressed size"); + } + if (c.decompressed_size_ > max_decompressed_data_size) { + return td::Status::Error("decompressed size is too big"); + } + TRY_RESULT(p, validatorsession::decompress_candidate_data(c.data_, false, c.decompressed_size_, + max_decompressed_data_size, proto_version)); + auto collated_data_hash = td::sha256_bits256(p.second); + auto key = PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, std::move(p.first), + std::move(p.second)}; + }(); + }, + [&](ton_api::collatorNode_compressedCandidateV2& c) { + res = [&]() -> td::Result { + TRY_RESULT(p, validatorsession::decompress_candidate_data(c.data_, true, 0, + max_decompressed_data_size, proto_version)); + auto collated_data_hash = td::sha256_bits256(p.second); + auto key = PublicKey{c.source_}; + if (!key.is_ed25519()) { + return td::Status::Error("invalid pubkey"); + } + auto e_key = Ed25519_PublicKey{key.ed25519_value().raw()}; + return BlockCandidate{e_key, create_block_id(c.id_), collated_data_hash, std::move(p.first), + std::move(p.second)}; + }(); + })); + return res; +} + +} // namespace ton::validator diff --git a/validator/collator-node/utils.hpp b/validator/collator-node/utils.hpp new file mode 100644 index 000000000..26b67d5a9 --- /dev/null +++ b/validator/collator-node/utils.hpp @@ -0,0 +1,27 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "ton/ton-types.h" +#include "tl/generate/auto/tl/ton_api.h" + +namespace ton::validator { + +tl_object_ptr serialize_candidate(const BlockCandidate& block, bool compress); +td::Result deserialize_candidate(tl_object_ptr f, + int max_decompressed_data_size, int proto_version); +} // namespace ton::validator diff --git a/validator/fabric.h b/validator/fabric.h index 825c3b248..e0088a045 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -39,7 +39,7 @@ struct CollateParams { int attempt_idx = 0; // Optional - used for optimistic collation - Ref optimistic_prev_block_ = {}; + Ref optimistic_prev_block = {}; }; enum ValidateMode { fake = 1 }; diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index f9e424479..0aa0e26ad 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -331,7 +331,7 @@ class Collator final : public td::actor::Actor { int prev_block_idx(const BlockIdExt& id) const { for (size_t i = 0; i < prev_blocks.size(); ++i) { if (prev_blocks[i] == id) { - return i; + return (int)i; } } return -1; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index c2ca14f99..b2e5c725b 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -82,7 +82,7 @@ Collator::Collator(CollateParams params, td::actor::ActorId ma , main_promise(std::move(promise)) , collator_node_id_(params.collator_node_id) , skip_store_candidate_(params.skip_store_candidate) - , optimistic_prev_block_(std::move(params.optimistic_prev_block_)) + , optimistic_prev_block_(std::move(params.optimistic_prev_block)) , attempt_idx_(params.attempt_idx) , perf_timer_("collate", 0.1, [manager](double duration) { @@ -368,7 +368,6 @@ void Collator::process_optimistic_prev_block() { std::move(token)); } - /** * Raises an error when timeout is reached. */ @@ -444,7 +443,7 @@ bool Collator::fatal_error(td::Status error) { .collator_node_id = collator_node_id_, .skip_store_candidate = skip_store_candidate_, .attempt_idx = attempt_idx_ + 1, - .optimistic_prev_block_ = optimistic_prev_block_}, + .optimistic_prev_block = optimistic_prev_block_}, manager, td::Timestamp::in(10.0), std::move(cancellation_token_), std::move(main_promise)); } else { LOG(INFO) << "collation failed in " << perf_timer_.elapsed() << " s " << error; @@ -6668,8 +6667,8 @@ void Collator::finalize_stats() { if (block_candidate) { stats_.block_id = block_candidate->id; stats_.collated_data_hash = block_candidate->collated_file_hash; - stats_.actual_bytes = block_candidate->data.size(); - stats_.actual_collated_data_bytes = block_candidate->collated_data.size(); + stats_.actual_bytes = (td::uint32)block_candidate->data.size(); + stats_.actual_collated_data_bytes = (td::uint32)block_candidate->collated_data.size(); } else { stats_.block_id.id = new_id; } @@ -6680,10 +6679,10 @@ void Collator::finalize_stats() { stats_.self = stats_.is_validator ? PublicKey(pubkeys::Ed25519(created_by_)).compute_short_id() : collator_node_id_.pubkey_hash(); if (block_limit_status_) { - stats_.estimated_bytes = block_limit_status_->estimate_block_size(); - stats_.gas = block_limit_status_->gas_used; - stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt; - stats_.estimated_collated_data_bytes = block_limit_status_->collated_data_size_estimate; + stats_.estimated_bytes = (td::uint32)block_limit_status_->estimate_block_size(); + stats_.gas = (td::uint32)block_limit_status_->gas_used; + stats_.lt_delta = (td::uint32)(block_limit_status_->cur_lt - block_limit_status_->limits.start_lt); + stats_.estimated_collated_data_bytes = (td::uint32)block_limit_status_->collated_data_size_estimate; stats_.cat_bytes = block_limit_status_->limits.classify_size(stats_.estimated_bytes); stats_.cat_gas = block_limit_status_->limits.classify_gas(stats_.gas); stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 01ded4dfa..00337f3c7 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -7203,8 +7203,8 @@ void ValidateQuery::record_stats(bool valid, std::string error_message) { } else { stats_.comment = std::move(error_message); } - stats_.actual_bytes = block_candidate.data.size(); - stats_.actual_collated_data_bytes = block_candidate.collated_data.size(); + stats_.actual_bytes = (td::uint32)block_candidate.data.size(); + stats_.actual_collated_data_bytes = (td::uint32)block_candidate.collated_data.size(); stats_.total_time = perf_timer_.elapsed(); stats_.work_time.total = work_timer_.elapsed_both(); LOG(WARNING) << "validation took " << perf_timer_.elapsed() << "s"; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 70876084a..5757c2bd4 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -185,7 +185,8 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { UNREACHABLE(); } - void set_block_state_from_data_preliminary(std::vector> blocks, td::Promise promise) { + void set_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) override { UNREACHABLE(); } void get_cell_db_reader(td::Promise> promise) override; @@ -246,7 +247,7 @@ class ValidatorManagerImpl : public ValidatorManager { promise.set_value(td::Unit()); } void send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data, int mode) { + td::BufferSlice data, int mode) override { callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data), mode); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 4862c4727..f61bd5f0b 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2675,7 +2675,7 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group td::actor::ActorId ValidatorManagerImpl::get_collation_manager(adnl::AdnlNodeIdShort adnl_id) { auto &actor = collation_managers_[adnl_id]; if (actor.empty()) { - actor = td::actor::create_actor("collation", adnl_id, opts_, actor_id(this), rldp2_); + actor = td::actor::create_actor("collation", adnl_id, opts_, actor_id(this), adnl_, rldp2_); } return actor.get(); } diff --git a/validator/shard-block-verifier.cpp b/validator/shard-block-verifier.cpp index ebd611144..0403ff22a 100644 --- a/validator/shard-block-verifier.cpp +++ b/validator/shard-block-verifier.cpp @@ -131,7 +131,7 @@ void ShardBlockVerifier::process_message(adnl::AdnlNodeIdShort src, td::BufferSl int ShardBlockVerifier::get_config_shard_idx(const ShardIdFull& shard_id) const { for (size_t i = 0; i < config_->shards.size(); i++) { if (shard_intersects(shard_id, config_->shards[i].shard_id)) { - return i; + return (int)i; } } return -1; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 163eb90bf..7cc980c00 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -320,7 +320,7 @@ void ValidatorGroup::generate_block_optimistic(validatorsession::BlockSourceInfo }; LOG(WARNING) << "Optimistically generating next block after " << block_id.to_str(); td::uint64 max_answer_size = config_.max_block_size + config_.max_collated_data_size + 1024; - td::actor::send_closure(collation_manager_, &CollationManager::collate_next_block, shard_, min_masterchain_block_id_, + td::actor::send_closure(collation_manager_, &CollationManager::collate_block_optimistic, shard_, min_masterchain_block_id_, block_id, std::move(prev_block), Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, source_info.priority, validator_set_, max_answer_size, optimistic_generation_->cancellation_token_source.get_cancellation_token(), std::move(P), From e173f11332bcea97835f7e3bbc3a030f490abf4b Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 7 Aug 2025 16:13:58 +0300 Subject: [PATCH 371/388] Wait for block in validator-session.cpp --- validator-session/validator-session.cpp | 66 +++++++++++++++++++------ validator-session/validator-session.hpp | 3 +- 2 files changed, 52 insertions(+), 17 deletions(-) diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 4a0631407..964f097e6 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -361,6 +361,12 @@ void ValidatorSessionImpl::process_received_block(td::uint32 block_round, Public } blocks_[block_id] = std::move(candidate); + if (auto it = block_waiters_.find(block_id); it != block_waiters_.end()) { + for (auto &promise : it->second) { + promise.set_result(td::Unit()); + } + block_waiters_.erase(it); + } VLOG(VALIDATOR_SESSION_WARNING) << this << ": received broadcast " << block_id; if (block_round != cur_round_) { @@ -552,6 +558,12 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, GeneratedCandidate blocks_[block_id] = create_tl_object( local_id().tl(), round, c.candidate.id.root_hash, std::move(c.candidate.data), std::move(c.candidate.collated_data)); + if (auto it = block_waiters_.find(block_id); it != block_waiters_.end()) { + for (auto &promise : it->second) { + promise.set_result(td::Unit()); + } + block_waiters_.erase(it); + } pending_generate_ = false; generated_ = true; generated_block_ = block_id; @@ -742,7 +754,6 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { VLOG(VALIDATOR_SESSION_WARNING) << print_id << ": failed to get candidate " << hash << " from " << id << ": " << R.move_as_error(); } else { - LOG(ERROR) << "QQQQQ Got block " << R.ok().size(); td::actor::send_closure(SelfId, &ValidatorSessionImpl::process_broadcast, src_id, R.move_as_ok(), candidate_id, false, false); } @@ -1369,25 +1380,48 @@ void ValidatorSessionImpl::process_approve(td::uint32 node_id, td::uint32 round, bool is_approved_66pct = stat->approved_66pct_at > 0.0; if (allow_optimistic_generation_ && !was_approved_66pct && is_approved_66pct && cur_round_ == round && - description().get_node_priority(local_idx(), round + 1) == 0 && blocks_.contains(candidate_id)) { - auto &block = blocks_[candidate_id]; - if (cur_round_ == first_block_round_ && - description().get_node_priority(description().get_source_idx(PublicKeyHash{block->src_}), cur_round_) == 0) { - callback_->generate_block_optimistic(BlockSourceInfo{description().get_source_public_key(local_idx()), - BlockCandidatePriority{round + 1, round + 1, 0}}, - block->data_.clone(), block->root_hash_, stat->block_id.file_hash, - [=, SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - return; - } - td::actor::send_closure( - SelfId, &ValidatorSessionImpl::generated_optimistic_candidate, - round + 1, R.move_as_ok(), candidate_id); - }); + cur_round_ == first_block_round_ && description().get_node_priority(local_idx(), round + 1) == 0 && + blocks_.contains(candidate_id) && + description().get_node_priority(description().get_source_idx(stat->validator_id), cur_round_) == 0) { + if (blocks_.contains(candidate_id)) { + generate_block_optimistic(round, candidate_id); + } else { + block_waiters_[candidate_id].push_back([=, SelfId = actor_id(this)](td::Result R) { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &ValidatorSessionImpl::generate_block_optimistic, round, candidate_id); + } + }); } } } +void ValidatorSessionImpl::generate_block_optimistic(td::uint32 cur_round, + ValidatorSessionCandidateId prev_candidate_id) { + if (cur_round != cur_round_) { + return; + } + auto it = blocks_.find(prev_candidate_id); + if (it == blocks_.end()) { + return; + } + auto &block = it->second; + auto stat = stats_get_candidate_stat_by_id(cur_round, prev_candidate_id); + if (!stat) { + return; + } + callback_->generate_block_optimistic(BlockSourceInfo{description().get_source_public_key(local_idx()), + BlockCandidatePriority{cur_round + 1, cur_round + 1, 0}}, + block->data_.clone(), block->root_hash_, stat->block_id.file_hash, + [=, SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + return; + } + td::actor::send_closure(SelfId, + &ValidatorSessionImpl::generated_optimistic_candidate, + cur_round + 1, R.move_as_ok(), prev_candidate_id); + }); +} + void ValidatorSessionImpl::generated_optimistic_candidate(td::uint32 round, GeneratedCandidate candidate, ValidatorSessionCandidateId prev_candidate) { if (cur_round_ > round) { diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index ef0e3fba8..83189868a 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -76,6 +76,7 @@ class ValidatorSessionImpl : public ValidatorSession { std::map> blocks_; // src_round_candidate_[src_id][round] -> candidate id std::vector> src_round_candidate_; + std::map>> block_waiters_; catchain::CatChainSessionId unique_hash_; @@ -202,7 +203,7 @@ class ValidatorSessionImpl : public ValidatorSession { ValidatorSessionCandidateId candidate_id); void stats_process_action(td::uint32 node_id, ton_api::validatorSession_round_Message &action); void process_approve(td::uint32 node_id, td::uint32 round, ValidatorSessionCandidateId candidate_id); - + void generate_block_optimistic(td::uint32 cur_round, ValidatorSessionCandidateId prev_candidate_id); void generated_optimistic_candidate(td::uint32 round, GeneratedCandidate candidate, ValidatorSessionCandidateId prev_candidate); From 90616d8cb75bd359dacab0be6c8e9fff13033b25 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 7 Aug 2025 16:18:14 +0300 Subject: [PATCH 372/388] Fix compilation error --- crypto/vm/boc-compression.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/vm/boc-compression.cpp b/crypto/vm/boc-compression.cpp index 63a834e08..db04f162f 100644 --- a/crypto/vm/boc-compression.cpp +++ b/crypto/vm/boc-compression.cpp @@ -283,7 +283,7 @@ td::Result boc_compress_improved_structure_lz4(const std::vecto continue; int delta = rank[boc_graph[node][j]] - i - 2; // Always >= 0 because of above check - size_t required_bits = 1 + (31 ^ __builtin_clz(node_count - i - 3)); + size_t required_bits = 1 + (31 ^ td::count_leading_zeroes32(node_count - i - 3)); if (required_bits < 8 - (result.size() + 1) % 8 + 1) { append_uint(result, delta, required_bits); @@ -494,7 +494,7 @@ td::Result>> boc_decompress_improved_structure_lz4 for (int j = 0; j < cell_refs_cnt[i]; ++j) { if (!boc_graph[i][j]) { size_t pref_size = (orig_size - bit_reader.size()); - size_t required_bits = 1 + (31 ^ __builtin_clz(node_count - i - 3)); + size_t required_bits = 1 + (31 ^ td::count_leading_zeroes32(node_count - i - 3)); if (required_bits < 8 - (pref_size + 1) % 8 + 1) { TRY_RESULT_ASSIGN(boc_graph[i][j], read_uint(bit_reader, required_bits)); From 038db1b223ccd3efe8d0b1ace3411d6a40c49afb Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 11 Aug 2025 11:18:55 +0300 Subject: [PATCH 373/388] Bugfixes in storage daemon --- storage/NodeActor.cpp | 30 ++++++++++++++--------- storage/storage-daemon/storage-daemon.cpp | 3 +++ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/storage/NodeActor.cpp b/storage/NodeActor.cpp index e24fffff5..92a83968f 100644 --- a/storage/NodeActor.cpp +++ b/storage/NodeActor.cpp @@ -27,6 +27,7 @@ #include "td/utils/overloaded.h" #include "tl-utils/tl-utils.hpp" #include "auto/tl/ton_api.hpp" +#include "common/delay.h" #include "td/actor/MultiPromise.h" namespace ton { @@ -545,8 +546,9 @@ void NodeActor::loop_queries() { auto it = peers_.find(part.peer_id); CHECK(it != peers_.end()); auto &state = it->second.state; - CHECK(state->peer_state_ready_); - CHECK(state->peer_state_.load().will_upload); + if (!state->peer_state_ready_ || !state->peer_state_.load().will_upload) { + continue; + } CHECK(state->node_queries_active_.size() < MAX_PEER_TOTAL_QUERIES); auto part_id = part.part_id; if (state->node_queries_active_.insert(static_cast(part_id)).second) { @@ -787,15 +789,21 @@ void NodeActor::db_store_torrent_meta() { return; } next_db_store_meta_at_ = td::Timestamp::never(); - auto meta = torrent_.get_meta_str(); - db_->set(create_hash_tl_object(torrent_.get_hash()), td::BufferSlice(meta), - [new_count = (td::int64)torrent_.get_ready_parts_count(), SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &NodeActor::after_db_store_torrent_meta, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &NodeActor::after_db_store_torrent_meta, new_count); - } - }); + auto meta = torrent_.get_meta(); + delay_action( + [SelfId = actor_id(this), meta = std::move(meta), db = db_, hash = torrent_.get_hash(), + new_count = (td::int64)torrent_.get_ready_parts_count()]() { + auto meta_str = meta.serialize(); + db->set(create_hash_tl_object(hash), td::BufferSlice(meta_str), + [=](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &NodeActor::after_db_store_torrent_meta, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &NodeActor::after_db_store_torrent_meta, new_count); + } + }); + }, + td::Timestamp::now()); } void NodeActor::after_db_store_torrent_meta(td::Result R) { diff --git a/storage/storage-daemon/storage-daemon.cpp b/storage/storage-daemon/storage-daemon.cpp index 98388428d..d1cbf37b5 100644 --- a/storage/storage-daemon/storage-daemon.cpp +++ b/storage/storage-daemon/storage-daemon.cpp @@ -42,6 +42,8 @@ #if TD_DARWIN || TD_LINUX #include #endif +#include "td/utils/port/rlimit.h" + #include namespace ton { @@ -950,6 +952,7 @@ int main(int argc, char *argv[]) { SCOPE_EXIT { td::log_interface = td::default_log_interface; }; + LOG_STATUS(td::change_maximize_rlimit(td::RlimitType::nofile, 786432)); td::IPAddress ip_addr; bool client_mode = false; From 37f10de155f9fb51037c048a8e3d3d94ef91b6cc Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 12 Aug 2025 13:54:32 +0300 Subject: [PATCH 374/388] Change new bounce format: include full or partial original body --- crypto/block/block.tlb | 2 +- crypto/block/transaction.cpp | 7 +++---- crypto/block/transaction.h | 2 +- doc/GlobalVersions.md | 6 +++--- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 9b4af3c1e..03b92cd2b 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -166,7 +166,7 @@ _ (Message Any) = MessageAny; _ value:CurrencyCollection created_lt:uint64 created_at:uint32 = NewBounceOriginalInfo; _ gas_used:uint32 vm_steps:uint32 = NewBounceComputePhaseInfo; new_bounce_body#fffffffe - original_body:(Maybe ^Cell) + original_body:^Cell original_info:^NewBounceOriginalInfo bounced_by_phase:uint8 exit_code:int32 compute_phase:(Maybe NewBounceComputePhaseInfo) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 8d220c7b9..3d83a93da 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -877,7 +877,7 @@ bool Transaction::unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* ihr_fee = td::zero_refint(); td::RefInt256 extra_flags = tlb::t_Grams.as_integer(in_msg_info.extra_flags); new_bounce_format = extra_flags->get_bit(0); - new_bounce_format_with_body = extra_flags->get_bit(1); + new_bounce_format_full_body = extra_flags->get_bit(1); } else { // Legacy: extra_flags was previously ihr_fee ihr_fee = tlb::t_Grams.as_integer(in_msg_info.extra_flags); @@ -3276,11 +3276,10 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { vm::CellBuilder body; if (new_bounce_format) { body.store_long(0xfffffffeU, 32); // new_bounce_body#fffffffe - if (new_bounce_format_with_body) { // original_body:(Maybe ^Cell) - body.store_long(1, 1); + if (new_bounce_format_full_body) { // original_body:^Cell body.store_ref(vm::CellBuilder().append_cellslice(in_msg_body).finalize_novm()); } else { - body.store_long(0, 1); + body.store_ref(vm::CellBuilder().store_bits(in_msg_body->as_bitslice()).finalize_novm()); } body.store_ref(vm::CellBuilder() .append_cellslice(in_msg_info.value) // value:CurrencyCollection diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index e987645db..d6ab6c870 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -359,7 +359,7 @@ struct Transaction { bool in_msg_extern{false}; gen::CommonMsgInfo::Record_int_msg_info in_msg_info; bool new_bounce_format{false}; - bool new_bounce_format_with_body{false}; + bool new_bounce_format_full_body{false}; bool use_msg_state{false}; bool is_first{false}; bool orig_addr_rewrite_set{false}; diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index 6e59955c4..e2336e694 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -236,20 +236,20 @@ Field `ihr_fee:Grams` in internal message is now called `extra_flags:(VarUIntege This field does not represent fees. `ihr_fee` is always zero since version 11, so this field was essentially unused. `(extra_flags & 1) = 1` enables the new bounce format for the message. The bounced message contains information about the transaction. -If `(extra_flags & 3) = 3`, the bounced message also contains the whole body of the original message. +If `(extra_flags & 3) = 3`, the bounced message contains the whole body of the original message. Otherwise, only the bits from the root of the original body are returned. When the message with new bounce flag is bounced, the bounced message body has the following format (`new_bounce_body`): ``` _ value:CurrencyCollection created_lt:uint64 created_at:uint32 = NewBounceOriginalInfo; _ gas_used:uint32 vm_steps:uint32 = NewBounceComputePhaseInfo; new_bounce_body#fffffffe - original_body:(Maybe ^Cell) + original_body:^Cell original_info:^NewBounceOriginalInfo bounced_by_phase:uint8 exit_code:int32 compute_phase:(Maybe NewBounceComputePhaseInfo) = NewBounceBody; ``` -- `original_body` - cell that contains the body of the original message (if `extra_flags & 2`) or nothing (if not `extra_flags & 2`). +- `original_body` - cell that contains the body of the original message. If `extra_flags & 2` then the whole body is returned, otherwise it is only the root without refs. - `original_info` - value, lt and unixtime of the original message. - `bounced_by_phase`: - `0` - compute phase was skipped. `exit_code` denotes the skip reason: From c7b6c2599b1612bc7589199d9424e6d3bffed7f1 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 12 Aug 2025 14:47:02 +0300 Subject: [PATCH 375/388] Use rldp2 for downloading candidates in validator session --- validator-session/CMakeLists.txt | 2 +- validator-session/validator-session.cpp | 5 ++--- validator-session/validator-session.h | 4 ++-- validator-session/validator-session.hpp | 4 ++-- validator/validator-group.cpp | 10 +++++----- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/validator-session/CMakeLists.txt b/validator-session/CMakeLists.txt index 94363c79c..6d4b00bd6 100644 --- a/validator-session/CMakeLists.txt +++ b/validator-session/CMakeLists.txt @@ -26,4 +26,4 @@ target_include_directories(validatorsession PUBLIC $/.. ${OPENSSL_INCLUDE_DIR} ) -target_link_libraries(validatorsession PRIVATE tdutils tdactor adnl rldp tl_api dht tdfec overlay catchain) +target_link_libraries(validatorsession PRIVATE tdutils tdactor adnl rldp2 tl_api dht tdfec overlay catchain) diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 964f097e6..9720ab34f 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -1097,7 +1097,7 @@ ValidatorSessionImpl::ValidatorSessionImpl(catchain::CatChainSessionId session_i PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, td::actor::ActorId keyring, - td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync) : unique_hash_(session_id) @@ -1221,7 +1221,6 @@ void ValidatorSessionImpl::start_up() { virtual_state_ = real_state_; check_all(); - td::actor::send_closure(rldp_, &rldp::Rldp::add_id, description().get_source_adnl_id(local_idx())); } void ValidatorSessionImpl::stats_init() { @@ -1434,7 +1433,7 @@ td::actor::ActorOwn ValidatorSession::create( catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, + td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync) { return td::actor::create_actor("session", session_id, std::move(opts), local_id, std::move(nodes), std::move(callback), keyring, adnl, rldp, diff --git a/validator-session/validator-session.h b/validator-session/validator-session.h index 5e226f357..dddd3092e 100644 --- a/validator-session/validator-session.h +++ b/validator-session/validator-session.h @@ -20,7 +20,7 @@ #include "adnl/adnl.h" #include "adnl/utils.hpp" -#include "rldp/rldp.h" +#include "rldp2/rldp.h" #include "ton/ton-types.h" @@ -112,7 +112,7 @@ class ValidatorSession : public td::actor::Actor { catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, + td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync); virtual ~ValidatorSession() = default; }; diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 83189868a..8af0fd0ec 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -86,7 +86,7 @@ class ValidatorSessionImpl : public ValidatorSession { td::actor::ActorId keyring_; td::actor::ActorId adnl_; - td::actor::ActorId rldp_; + td::actor::ActorId rldp_; td::actor::ActorId overlay_manager_; td::actor::ActorOwn catchain_; std::unique_ptr description_; @@ -211,7 +211,7 @@ class ValidatorSessionImpl : public ValidatorSession { ValidatorSessionImpl(catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, std::vector nodes, std::unique_ptr callback, td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays, + td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, std::string db_suffix, bool allow_unsafe_self_blocks_resync); void start_up() override; void alarm() override; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 7cc980c00..4a8310862 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -473,14 +473,17 @@ void ValidatorGroup::create_session() { } CHECK(found); + td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_adnl_id_); + td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_adnl_id_); + config_.catchain_opts.broadcast_speed_multiplier = opts_->get_catchain_broadcast_speed_multiplier(); if (!config_.new_catchain_ids) { session_ = validatorsession::ValidatorSession::create(session_id_, config_, local_id_, std::move(vec), - make_validator_session_callback(), keyring_, adnl_, rldp_, + make_validator_session_callback(), keyring_, adnl_, rldp2_, overlays_, db_root_, "-", allow_unsafe_self_blocks_resync_); } else { session_ = validatorsession::ValidatorSession::create( - session_id_, config_, local_id_, std::move(vec), make_validator_session_callback(), keyring_, adnl_, rldp_, + session_id_, config_, local_id_, std::move(vec), make_validator_session_callback(), keyring_, adnl_, rldp2_, overlays_, db_root_ + "/catchains/", PSTRING() << "." << shard_.workchain << "." << shard_.shard << "." << validator_set_->get_catchain_seqno() << ".", @@ -495,9 +498,6 @@ void ValidatorGroup::create_session() { if (started_) { td::actor::send_closure(session_, &validatorsession::ValidatorSession::start); } - - td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_adnl_id_); - td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_adnl_id_); } void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterchain_block_id) { From a70bdfb53f188752b41add4ce7e13ae1472b7f84 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 12 Aug 2025 16:29:40 +0300 Subject: [PATCH 376/388] Disable optimistic generation in masterchain --- validator-session/validator-session.cpp | 1 + validator/validator-group.cpp | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 9720ab34f..bddf56004 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -1413,6 +1413,7 @@ void ValidatorSessionImpl::generate_block_optimistic(td::uint32 cur_round, block->data_.clone(), block->root_hash_, stat->block_id.file_hash, [=, SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { + LOG(DEBUG) << "Optimistic generation error: " << R.move_as_error(); return; } td::actor::send_closure(SelfId, diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 4a8310862..450453788 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -301,12 +301,19 @@ void ValidatorGroup::generate_block_optimistic(validatorsession::BlockSourceInfo td::BufferSlice prev_block, RootHash prev_root_hash, FileHash prev_file_hash, td::Promise promise) { if (destroying_) { + promise.set_error(td::Status::Error("validator session finished")); + return; + } + if (shard_.is_masterchain()) { + promise.set_error(td::Status::Error("no optimistic generation in masterchain")); return; } if (last_known_round_id_ + 1 != source_info.priority.round) { + promise.set_error(td::Status::Error("too old round")); return; } if (optimistic_generation_ && optimistic_generation_->round >= source_info.priority.round) { + promise.set_error(td::Status::Error("optimistic generation already in progress")); return; } BlockIdExt block_id{create_next_block_id_simple(), prev_root_hash, prev_file_hash}; From 3f216b6a83377d27d6df3fa27e3e4005ae4f7535 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 14 Aug 2025 17:14:12 +0300 Subject: [PATCH 377/388] Fix returning error in wait block state --- validator/downloaders/wait-block-state.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index 7ca2bbd2e..a69e2b130 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -34,7 +34,7 @@ void WaitBlockState::alarm() { void WaitBlockState::abort_query(td::Status reason) { if (promise_no_store_) { promise_no_store_.set_error( - reason.move_as_error_prefix(PSTRING() << "failed to download state " << handle_->id() << ": ")); + reason.clone().move_as_error_prefix(PSTRING() << "failed to download state " << handle_->id() << ": ")); } if (promise_final_) { if (priority_ > 0 || (reason.code() != ErrorCode::timeout && reason.code() != ErrorCode::notready)) { From c48413a2e8f38f9c6b4b726141049a1471653c74 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Sat, 16 Aug 2025 17:54:35 +0300 Subject: [PATCH 378/388] Fix error processing in Collator::process_optimistic_prev_block --- validator/impl/collator-impl.h | 2 +- validator/impl/collator.cpp | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 0aa0e26ad..03eedb841 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -122,7 +122,7 @@ class Collator final : public td::actor::Actor { private: void start_up() override; void load_prev_states_blocks(); - void process_optimistic_prev_block(); + bool process_optimistic_prev_block(); void alarm() override; int verbosity{3 * 0}; int verify{1}; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index b2e5c725b..e38d30e1f 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -241,7 +241,9 @@ void Collator::start_up() { if (optimistic_prev_block_.is_null()) { load_prev_states_blocks(); } else { - process_optimistic_prev_block(); + if (!process_optimistic_prev_block()) { + return; + } } if (is_hardfork_) { LOG(WARNING) << "generating a hardfork block"; @@ -324,15 +326,14 @@ void Collator::load_prev_states_blocks() { /** * Write optimistic prev block as block data, load previous state to apply Merkle update to it */ -void Collator::process_optimistic_prev_block() { +bool Collator::process_optimistic_prev_block() { std::vector prev_prev; BlockIdExt mc_blkid; bool after_split; auto S = block::unpack_block_prev_blk_try(optimistic_prev_block_->root_cell(), optimistic_prev_block_->block_id(), prev_prev, mc_blkid, after_split); if (S.is_error()) { - fatal_error(S.move_as_error_prefix("failed to unpack optimistic prev block: ")); - return; + return fatal_error(S.move_as_error_prefix("failed to unpack optimistic prev block: ")); } // 3.1. load state if (prev_prev.size() == 1) { @@ -366,6 +367,7 @@ void Collator::process_optimistic_prev_block() { auto token = perf_log_.start_action(PSTRING() << "opt wait_block_data"); td::actor::send_closure_later(actor_id(this), &Collator::after_get_block_data, 0, optimistic_prev_block_, std::move(token)); + return true; } /** From e169659d81d9f3962028c15fcd844155e0f73884 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 18 Aug 2025 11:08:56 +0300 Subject: [PATCH 379/388] Blockchain explorer: fix search by utime --- blockchain-explorer/blockchain-explorer-query.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/blockchain-explorer/blockchain-explorer-query.cpp b/blockchain-explorer/blockchain-explorer-query.cpp index d540d4878..6919c5c14 100644 --- a/blockchain-explorer/blockchain-explorer-query.cpp +++ b/blockchain-explorer/blockchain-explorer-query.cpp @@ -525,8 +525,8 @@ HttpQueryBlockSearch::HttpQueryBlockSearch(std::map op } if (opts.count("utime") == 1) { try { - seqno_ = static_cast(std::stoull(opts["utime"])); - mode_ = 1; + utime_ = static_cast(std::stoull(opts["utime"])); + mode_ = 4; } catch (...) { error_ = td::Status::Error("cannot parse utime"); return; From f416812d3f94aa5b4d0d120f37e6a0027be8e9fd Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 19 Aug 2025 18:47:43 +0300 Subject: [PATCH 380/388] Optimistic validation --- validator-session/validator-session.cpp | 42 +++++++ validator-session/validator-session.h | 10 +- validator-session/validator-session.hpp | 3 + validator/fabric.h | 18 ++- validator/impl/fabric.cpp | 18 ++- validator/impl/validate-query.cpp | 152 +++++++++++++++++++----- validator/impl/validate-query.hpp | 24 +++- validator/manager-disk.cpp | 9 +- validator/validator-group.cpp | 141 ++++++++++++++++------ validator/validator-group.hpp | 3 +- 10 files changed, 324 insertions(+), 96 deletions(-) diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index bddf56004..27c105929 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -294,6 +294,12 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice << "]: duplicate optimistic broadcast for round " << block_round; return; } + int priority = description().get_node_priority(src_idx, block_round); + if (priority < 0) { + VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice()) + << "]: node is not a producer in round " << block_round; + return; + } if (block_round > cur_round_) { OptimisticBroadcast &optimistic_broadcast = optimistic_broadcasts_[{block_round, src_idx}]; optimistic_broadcast.candidate = std::move(candidate); @@ -301,6 +307,11 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice optimistic_broadcast.broadcast_info = broadcast_info; VLOG(VALIDATOR_SESSION_WARNING) << this << ": received optimistic broadcast " << block_id << " from " << src << ", round " << block_round; + validate_optimistic_broadcast( + BlockSourceInfo{description().get_source_public_key(src_idx), + BlockCandidatePriority{block_round, block_round, priority}}, + optimistic_broadcast.candidate->root_hash_, optimistic_broadcast.candidate->data_.clone(), + optimistic_broadcast.candidate->collated_data_.clone(), optimistic_broadcast.prev_candidate_id); return; } if (SentBlock::get_block_id(real_state_->get_committed_block(description(), block_round - 1)) != @@ -387,6 +398,37 @@ void ValidatorSessionImpl::process_received_block(td::uint32 block_round, Public } } +void ValidatorSessionImpl::validate_optimistic_broadcast(BlockSourceInfo source_info, + ValidatorSessionRootHash root_hash, td::BufferSlice data, + td::BufferSlice collated_data, + ValidatorSessionCandidateId prev_candidate_id) { + if (source_info.priority.round <= cur_round_) { + VLOG(VALIDATOR_SESSION_DEBUG) << this << ": validate optimistic broadcast from " + << source_info.source.compute_short_id() << " : too old"; + return; + } + auto it = blocks_.find(prev_candidate_id); + if (it == blocks_.end()) { + VLOG(VALIDATOR_SESSION_DEBUG) << this << ": validate optimistic broadcast from " + << source_info.source.compute_short_id() << " : wait for prev block"; + block_waiters_[prev_candidate_id].push_back( + [=, SelfId = actor_id(this), data = std::move(data), + collated_data = std::move(collated_data)](td::Result R) mutable { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &ValidatorSessionImpl::validate_optimistic_broadcast, source_info, + root_hash, std::move(data), std::move(collated_data), prev_candidate_id); + } + }); + return; + } + VLOG(VALIDATOR_SESSION_DEBUG) << this << ": validate optimistic broadcast from " + << source_info.source.compute_short_id(); + callback_->on_optimistic_candidate( + source_info, root_hash, std::move(data), std::move(collated_data), + description().get_source_public_key(description().get_source_idx(PublicKeyHash{it->second->src_})), + it->second->root_hash_, it->second->data_.clone(), it->second->collated_data_.clone()); +} + void ValidatorSessionImpl::process_message(PublicKeyHash src, td::BufferSlice data) { } diff --git a/validator-session/validator-session.h b/validator-session/validator-session.h index dddd3092e..f8e2d62e7 100644 --- a/validator-session/validator-session.h +++ b/validator-session/validator-session.h @@ -93,8 +93,14 @@ class ValidatorSession : public td::actor::Actor { ValidatorSessionFileHash file_hash, ValidatorSessionCollatedDataFileHash collated_data_file_hash, td::Promise promise) = 0; - virtual void generate_block_optimistic(BlockSourceInfo source_info, td::BufferSlice prev_block, RootHash prev_root_hash, - FileHash prev_file_hash, td::Promise promise) { + virtual void generate_block_optimistic(BlockSourceInfo source_info, td::BufferSlice prev_block, + RootHash prev_root_hash, FileHash prev_file_hash, + td::Promise promise) { + } + virtual void on_optimistic_candidate(BlockSourceInfo source_info, ValidatorSessionRootHash root_hash, + td::BufferSlice data, td::BufferSlice collated_data, PublicKey prev_source, + ValidatorSessionRootHash prev_root_hash, td::BufferSlice prev_data, + td::BufferSlice prev_collated_data) { } virtual ~Callback() = default; }; diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 8af0fd0ec..4861c52e1 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -238,6 +238,9 @@ class ValidatorSessionImpl : public ValidatorSession { void process_received_block(td::uint32 block_round, PublicKeyHash src, td::uint32 src_idx, tl_object_ptr candidate, const BroadcastInfo &info, bool is_overlay_broadcast, bool is_startup); + void validate_optimistic_broadcast(BlockSourceInfo source_info, ValidatorSessionRootHash root_hash, + td::BufferSlice data, td::BufferSlice collated_data, + ValidatorSessionCandidateId prev_candidate_id); void process_message(PublicKeyHash src, td::BufferSlice data); void process_query(PublicKeyHash src, td::BufferSlice data, td::Promise promise); diff --git a/validator/fabric.h b/validator/fabric.h index e0088a045..8a3f6193f 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -42,7 +42,17 @@ struct CollateParams { Ref optimistic_prev_block = {}; }; -enum ValidateMode { fake = 1 }; +struct ValidateParams { + ShardIdFull shard; + BlockIdExt min_masterchain_block_id; + std::vector prev; + td::Ref validator_set = {}; + PublicKeyHash local_validator_id = PublicKeyHash::zero();; + bool is_fake = false; + + // Optional - used for validation of optimistic candidates + Ref optimistic_prev_block = {}; +}; td::actor::ActorOwn create_db_actor(td::actor::ActorId manager, std::string db_root_, td::Ref opts); @@ -96,10 +106,8 @@ void run_check_proof_query(BlockIdExt id, td::Ref proof, td::actor::Actor td::Ref rel_key_block_proof, bool skip_check_signatures = false); void run_check_proof_link_query(BlockIdExt id, td::Ref proof, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); -void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, - BlockCandidate candidate, td::Ref validator_set, PublicKeyHash local_validator_id, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, unsigned mode = 0); +void run_validate_query(BlockCandidate candidate, ValidateParams params, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise promise); void run_collate_query(CollateParams params, td::actor::ActorId manager, td::Timestamp timeout, td::CancellationToken cancellation_token, td::Promise promise); void run_liteserver_query(td::BufferSlice data, td::actor::ActorId manager, diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index f38f77320..4cf334928 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -192,23 +192,19 @@ void run_check_proof_link_query(BlockIdExt id, td::Ref proof, td::act .release(); } -void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, - BlockCandidate candidate, td::Ref validator_set, PublicKeyHash local_validator_id, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, unsigned mode) { +void run_validate_query(BlockCandidate candidate, ValidateParams params, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise promise) { BlockSeqno seqno = 0; - for (auto& p : prev) { + for (auto& p : params.prev) { if (p.seqno() > seqno) { seqno = p.seqno(); } } - bool is_fake = mode & ValidateMode::fake; static std::atomic idx; - td::actor::create_actor(PSTRING() << (is_fake ? "fakevalidate" : "validateblock") << shard.to_str() - << ":" << (seqno + 1) << "#" << idx.fetch_add(1), - shard, min_masterchain_block_id, std::move(prev), std::move(candidate), - std::move(validator_set), local_validator_id, std::move(manager), timeout, - std::move(promise), mode) + td::actor::create_actor( + PSTRING() << (params.is_fake ? "fakevalidate" : "validateblock") << params.shard.to_str() << ":" << (seqno + 1) + << "#" << idx.fetch_add(1), + std::move(candidate), std::move(params), std::move(manager), timeout, std::move(promise)) .release(); } diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 00337f3c7..adcf72eb2 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -59,33 +59,29 @@ std::string ErrorCtx::as_string() const { /** * Constructs a ValidateQuery object. * - * @param shard The shard of the block being validated. - * @param min_masterchain_block_id The minimum allowed masterchain block reference for the block. - * @param prev A vector of BlockIdExt representing the previous blocks. * @param candidate The BlockCandidate to be validated. - * @param validator_set A reference to the ValidatorSet. + * @param params Validation parameters * @param manager The ActorId of the ValidatorManager. * @param timeout The timeout for the validation. * @param promise The Promise to return the ValidateCandidateResult to. - * @param mode +1 - fake mode */ -ValidateQuery::ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, - BlockCandidate candidate, Ref validator_set, - PublicKeyHash local_validator_id, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise promise, unsigned mode) - : shard_(shard) +ValidateQuery::ValidateQuery(BlockCandidate candidate, ValidateParams params, + td::actor::ActorId manager, td::Timestamp timeout, + td::Promise promise) + : shard_(params.shard) , id_(candidate.id) - , min_mc_block_id(min_masterchain_block_id) - , prev_blocks(std::move(prev)) + , min_mc_block_id(params.min_masterchain_block_id) + , prev_blocks(std::move(params.prev)) , block_candidate(std::move(candidate)) - , validator_set_(std::move(validator_set)) - , local_validator_id_(local_validator_id) + , validator_set_(std::move(params.validator_set)) + , local_validator_id_(params.local_validator_id) , manager(manager) , timeout(timeout) , main_promise(std::move(promise)) - , is_fake_(mode & ValidateMode::fake) + , is_fake_(params.is_fake) , shard_pfx_(shard_.shard) , shard_pfx_len_(ton::shard_prefix_length(shard_)) + , optimistic_prev_block_(std::move(params.optimistic_prev_block)) , perf_timer_("validateblock", 0.1, [manager](double duration) { send_closure(manager, &ValidatorManager::add_perf_timer_stat, "validateblock", duration); }) { @@ -350,6 +346,21 @@ void ValidateQuery::start_up() { // return; } } + if (optimistic_prev_block_.not_null()) { + if (is_masterchain()) { + fatal_error("optimistic validation in masterchain is not supported"); + return; + } + if (prev_blocks.size() != 1) { + fatal_error("optimistic prev block is not null, which is not allowed after merge"); + return; + } + if (prev_blocks[0] != optimistic_prev_block_->block_id()) { + fatal_error("optimistic prev block is not null, but has invalid block id"); + return; + } + LOG(WARNING) << "Optimistic prev block id = " << optimistic_prev_block_->block_id().to_str(); + } // 2. learn latest masterchain state and block id LOG(DEBUG) << "sending get_top_masterchain_state_block() to Manager"; ++pending; @@ -366,17 +377,13 @@ void ValidateQuery::start_up() { } // 4. load state(s) corresponding to previous block(s) (not full-collated-data or masterchain) prev_states.resize(prev_blocks.size()); - if (is_masterchain() || !full_collated_data_) { - for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { - // 4.1. load state - LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; - ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), - timeout, false, [self = get_self(), i](td::Result> res) -> void { - LOG(DEBUG) << "got answer to wait_block_state_short query #" << i; - td::actor::send_closure_later( - std::move(self), &ValidateQuery::after_get_shard_state, i, std::move(res)); - }); + if (is_masterchain() || !full_collated_data_) { + if (optimistic_prev_block_.is_null()) { + load_prev_states(); + } else { + if (!process_optimistic_prev_block()) { + return; + } } } // 4. request masterchain handle and state referred to in the block @@ -409,6 +416,84 @@ void ValidateQuery::start_up() { CHECK(pending); } +/** + * Load previous states from DB + */ +void ValidateQuery::load_prev_states() { + for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { + // 4.1. load state + LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; + ++pending; + td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), + timeout, false, [self = get_self(), i](td::Result> res) -> void { + LOG(DEBUG) << "got answer to wait_block_state_short query #" << i; + td::actor::send_closure_later( + std::move(self), &ValidateQuery::after_get_shard_state, i, std::move(res)); + }); + } +} + +/** + * Load previous state for optimistic prev block to apply Merkle update to it + */ +bool ValidateQuery::process_optimistic_prev_block() { + std::vector prev_prev; + BlockIdExt mc_blkid; + bool after_split; + auto S = block::unpack_block_prev_blk_try(optimistic_prev_block_->root_cell(), optimistic_prev_block_->block_id(), + prev_prev, mc_blkid, after_split); + if (S.is_error()) { + return fatal_error(S.move_as_error_prefix("failed to unpack optimistic prev block: ")); + } + // 4.1. load state + if (prev_prev.size() == 1) { + LOG(DEBUG) << "sending wait_block_state() query for " << prev_prev[0].to_str() << " to Manager (opt)"; + ++pending; + td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_prev[0], priority(), timeout, + false, [self = get_self()](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state query (opt)"; + td::actor::send_closure_later(std::move(self), + &ValidateQuery::after_get_shard_state_optimistic, + std::move(res)); + }); + } else { + CHECK(prev_prev.size() == 2); + LOG(DEBUG) << "sending wait_block_state_merge() query for " << prev_prev[0].to_str() << " and " + << prev_prev[1].to_str() << " to Manager (opt)"; + ++pending; + td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_merge, prev_prev[0], prev_prev[1], + priority(), timeout, [self = get_self()](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state_merge query (opt)"; + td::actor::send_closure_later(std::move(self), + &ValidateQuery::after_get_shard_state_optimistic, + std::move(res)); + }); + } + return true; +} + +/** + * Callback function called after retrieving previous state for optimistic prev block + * + * @param res The retrieved state. + */ +void ValidateQuery::after_get_shard_state_optimistic(td::Result> res) { + LOG(DEBUG) << "in ValidateQuery::after_get_shard_state_optimistic()"; + if (res.is_error()) { + fatal_error(res.move_as_error()); + return; + } + work_timer_.resume(); + auto state = res.move_as_ok(); + auto S = state.write().apply_block(optimistic_prev_block_->block_id(), optimistic_prev_block_); + if (S.is_error()) { + fatal_error(S.move_as_error_prefix("apply error: ")); + return; + } + work_timer_.pause(); + after_get_shard_state(0, std::move(state)); +} + /** * Unpacks and validates a block candidate. * @@ -1619,11 +1704,16 @@ bool ValidateQuery::request_neighbor_queues() { for (block::McShardDescr& descr : neighbors_) { LOG(DEBUG) << "requesting outbound queue of neighbor #" << i << " : " << descr.blk_.to_str(); ++pending; - send_closure_later(manager, &ValidatorManager::wait_block_message_queue_short, descr.blk_, priority(), timeout, - [self = get_self(), i](td::Result> res) { - td::actor::send_closure(std::move(self), &ValidateQuery::got_neighbor_out_queue, i, - std::move(res)); - }); + if (int prev_idx = prev_block_idx(descr.blk_); prev_idx >= 0) { + td::actor::send_closure(actor_id(this), &ValidateQuery::got_neighbor_out_queue, i, + prev_states.at(prev_idx)->message_queue()); + } else { + send_closure_later(manager, &ValidatorManager::wait_block_message_queue_short, descr.blk_, priority(), timeout, + [self = get_self(), i](td::Result> res) { + td::actor::send_closure(std::move(self), &ValidateQuery::got_neighbor_out_queue, i, + std::move(res)); + }); + } ++i; } } diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index e43d782f5..29cdbd8d2 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -18,6 +18,7 @@ */ #pragma once +#include "fabric.h" #include "interfaces/validator-manager.h" #include "vm/cells.h" #include "vm/dict.h" @@ -112,15 +113,13 @@ class ValidateQuery : public td::actor::Actor { return SUPPORTED_VERSION; } static constexpr long long supported_capabilities() { - return ton::capCreateStatsEnabled | ton::capBounceMsgBody | ton::capReportVersion | ton::capShortDequeue | - ton::capStoreOutMsgQueueSize | ton::capMsgMetadata | ton::capDeferMessages | ton::capFullCollatedData; + return capCreateStatsEnabled | capBounceMsgBody | capReportVersion | capShortDequeue | capStoreOutMsgQueueSize | + capMsgMetadata | capDeferMessages | capFullCollatedData; } public: - ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, - BlockCandidate candidate, td::Ref validator_set, PublicKeyHash local_validator_id, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise, unsigned mode = 0); + ValidateQuery(BlockCandidate candidate, ValidateParams params, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise promise); private: int verbosity{3 * 1}; @@ -153,6 +152,7 @@ class ValidateQuery : public td::actor::Actor { td::BitArray<64> shard_pfx_; int shard_pfx_len_; td::Bits256 created_by_; + Ref optimistic_prev_block_; Ref prev_state_root_; Ref state_root_; @@ -269,6 +269,10 @@ class ValidateQuery : public td::actor::Actor { void alarm() override; void start_up() override; + void load_prev_states(); + bool process_optimistic_prev_block(); + void after_get_shard_state_optimistic(td::Result> res); + bool save_candidate(); void written_candidate(); @@ -290,6 +294,14 @@ class ValidateQuery : public td::actor::Actor { bool is_masterchain() const { return shard_.is_masterchain(); } + int prev_block_idx(const BlockIdExt& id) const { + for (size_t i = 0; i < prev_blocks.size(); ++i) { + if (prev_blocks[i] == id) { + return (int)i; + } + } + return -1; + } td::actor::ActorId get_self() { return actor_id(this); } diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index be57a632d..eb06dd304 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -156,8 +156,13 @@ void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vector prev, BlockIdExt last, diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 450453788..5261560e8 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -126,20 +126,54 @@ void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo } void ValidatorGroup::validate_block_candidate(validatorsession::BlockSourceInfo source_info, BlockCandidate block, - td::Promise> promise) { + td::Promise> promise, + td::optional optimistic_prev_block) { if (destroying_) { promise.set_error(td::Status::Error("validator session finished")); return; } + bool is_optimistic = (bool)optimistic_prev_block; + if (is_optimistic && shard_.is_masterchain()) { + promise.set_error(td::Status::Error("no optimistic validation in masterchain")); + return; + } td::uint32 round_id = source_info.priority.round; - update_round_id(round_id); + if (!is_optimistic) { + update_round_id(round_id); + } if (round_id < last_known_round_id_) { promise.set_error(td::Status::Error(ErrorCode::notready, "too old")); return; } + if (is_optimistic && round_id > last_known_round_id_ + 1) { + promise.set_error(td::Status::Error(ErrorCode::notready, "too new")); + return; + } + if (is_optimistic && shard_.is_masterchain()) { + promise.set_error(td::Status::Error("optimistic validation in masterchain is not supported")); + return; + } auto next_block_id = create_next_block_id(block.id.root_hash, block.id.file_hash); block.id = next_block_id; + auto prev = prev_block_ids_; + if (is_optimistic) { + if (round_id > last_known_round_id_) { + ++block.id.id.seqno; + } + optimistic_prev_block.value().id.id = block.id.id; + --optimistic_prev_block.value().id.id.seqno; + if (round_id == last_known_round_id_) { + if (prev_block_ids_ != std::vector{optimistic_prev_block.value().id}) { + promise.set_error(td::Status::Error("wrong prev block for optimistic validation")); + return; + } + optimistic_prev_block = {}; + is_optimistic = false; + } else { + prev = {optimistic_prev_block.value().id}; + } + } CacheKey cache_key = block_to_cache_key(block); auto it = approved_candidates_cache_.find(cache_key); @@ -148,44 +182,60 @@ void ValidatorGroup::validate_block_candidate(validatorsession::BlockSourceInfo return; } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), source_info, block = block.clone(), - promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - auto S = R.move_as_error(); - if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) { - LOG(ERROR) << "failed to validate candidate: " << S; - } - delay_action( - [SelfId, source_info, block = std::move(block), promise = std::move(promise)]() mutable { - td::actor::send_closure(SelfId, &ValidatorGroup::validate_block_candidate, std::move(source_info), - std::move(block), std::move(promise)); - }, - td::Timestamp::in(0.1)); - } else { - auto v = R.move_as_ok(); - v.visit(td::overloaded( - [&](UnixTime ts) { - td::actor::send_closure(SelfId, &ValidatorGroup::update_approve_cache, block_to_cache_key(block), ts); - td::actor::send_closure(SelfId, &ValidatorGroup::add_available_block_candidate, block.pubkey.as_bits256(), - block.id, block.collated_file_hash); - if (need_send_candidate_broadcast(source_info, block.id.is_masterchain())) { - td::actor::send_closure(SelfId, &ValidatorGroup::send_block_candidate_broadcast, block.id, - block.data.clone()); - } - promise.set_value({ts, false}); - }, - [&](CandidateReject reject) { - promise.set_error( - td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad candidate: " << reject.reason)); - })); - } - }); + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), source_info, block = block.clone(), + optimistic_prev_block = is_optimistic ? optimistic_prev_block.value().clone() : td::optional{}, + promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + auto S = R.move_as_error(); + if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) { + LOG(ERROR) << "failed to validate candidate: " << S; + } + delay_action( + [SelfId, source_info, block = std::move(block), promise = std::move(promise), + optimistic_prev_block = std::move(optimistic_prev_block)]() mutable { + td::actor::send_closure(SelfId, &ValidatorGroup::validate_block_candidate, std::move(source_info), + std::move(block), std::move(promise), std::move(optimistic_prev_block)); + }, + td::Timestamp::in(0.1)); + } else { + auto v = R.move_as_ok(); + v.visit(td::overloaded( + [&](UnixTime ts) { + td::actor::send_closure(SelfId, &ValidatorGroup::update_approve_cache, block_to_cache_key(block), ts); + td::actor::send_closure(SelfId, &ValidatorGroup::add_available_block_candidate, + block.pubkey.as_bits256(), block.id, block.collated_file_hash); + if (need_send_candidate_broadcast(source_info, block.id.is_masterchain())) { + td::actor::send_closure(SelfId, &ValidatorGroup::send_block_candidate_broadcast, block.id, + block.data.clone()); + } + promise.set_value({ts, false}); + }, + [&](CandidateReject reject) { + promise.set_error( + td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad candidate: " << reject.reason)); + })); + } + }); if (!started_) { P.set_error(td::Status::Error(ErrorCode::notready, "validator group not started")); return; } VLOG(VALIDATOR_DEBUG) << "validating block candidate " << next_block_id; - run_validate_query(shard_, min_masterchain_block_id_, prev_block_ids_, std::move(block), validator_set_, local_id_, + td::Ref optimistic_prev_block_data; + if (is_optimistic) { + TRY_RESULT_PROMISE_PREFIX_ASSIGN( + P, optimistic_prev_block_data, + create_block(optimistic_prev_block.value().id, std::move(optimistic_prev_block.value().data)), + "failed to parse optimistic prev block: "); + } + run_validate_query(std::move(block), + ValidateParams{.shard = shard_, + .min_masterchain_block_id = min_masterchain_block_id_, + .prev = std::move(prev), + .validator_set = validator_set_, + .local_validator_id = local_id_, + .optimistic_prev_block = optimistic_prev_block_data}, manager_, td::Timestamp::in(15.0), std::move(P)); } @@ -254,7 +304,6 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so std::move(approve_sig_set), send_broadcast_mode, std::move(promise)); prev_block_ids_ = std::vector{next_block_id}; cached_collated_block_ = nullptr; - approved_candidates_cache_.clear(); cancellation_token_source_.cancel(); if (optimistic_generation_ && optimistic_generation_->round == last_known_round_id_ && optimistic_generation_->prev != next_block_id) { @@ -405,7 +454,7 @@ std::unique_ptr ValidatorGroup::ma sha256_bits256(collated_data.as_slice()), data.clone(), collated_data.clone()}; td::actor::send_closure(id_, &ValidatorGroup::validate_block_candidate, std::move(source_info), - std::move(candidate), std::move(P)); + std::move(candidate), std::move(P), td::optional{}); } void on_generate_slot(validatorsession::BlockSourceInfo source_info, td::Promise promise) override { @@ -447,6 +496,23 @@ std::unique_ptr ValidatorGroup::ma td::actor::send_closure(id_, &ValidatorGroup::generate_block_optimistic, source_info, std::move(prev_block), prev_root_hash, prev_file_hash, std::move(promise)); } + void on_optimistic_candidate(validatorsession::BlockSourceInfo source_info, + validatorsession::ValidatorSessionRootHash root_hash, td::BufferSlice data, + td::BufferSlice collated_data, PublicKey prev_source, + validatorsession::ValidatorSessionRootHash prev_root_hash, td::BufferSlice prev_data, + td::BufferSlice prev_collated_data) override { + BlockCandidate candidate{Ed25519_PublicKey{source_info.source.ed25519_value().raw()}, + BlockIdExt{0, 0, 0, root_hash, sha256_bits256(data.as_slice())}, + sha256_bits256(collated_data.as_slice()), data.clone(), collated_data.clone()}; + BlockCandidate prev_candidate{Ed25519_PublicKey{prev_source.ed25519_value().raw()}, + BlockIdExt{0, 0, 0, prev_root_hash, sha256_bits256(prev_data.as_slice())}, + sha256_bits256(prev_collated_data.as_slice()), prev_data.clone(), + prev_collated_data.clone()}; + + td::actor::send_closure( + id_, &ValidatorGroup::validate_block_candidate, std::move(source_info), std::move(candidate), + [](td::Result>) mutable {}, std::move(prev_candidate)); + } private: td::actor::ActorId id_; @@ -511,7 +577,6 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch prev_block_ids_ = prev; min_masterchain_block_id_ = min_masterchain_block_id; cached_collated_block_ = nullptr; - approved_candidates_cache_.clear(); started_ = true; if (init_) { diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 17bd23a4d..56d89dda4 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -40,7 +40,8 @@ class ValidatorGroup : public td::actor::Actor { void generate_block_candidate_cont(validatorsession::BlockSourceInfo source_info, td::Promise promise, td::CancellationToken cancellation_token); void validate_block_candidate(validatorsession::BlockSourceInfo source_info, BlockCandidate block, - td::Promise> promise); + td::Promise> promise, + td::optional optimistic_prev_block); void accept_block_candidate(validatorsession::BlockSourceInfo source_info, td::BufferSlice block, RootHash root_hash, FileHash file_hash, std::vector signatures, std::vector approve_signatures, From ff010ef78ad7cc568363494e4f4abade60545033 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 19 Aug 2025 19:29:03 +0300 Subject: [PATCH 381/388] More session stats for collation/validation time --- tl/generate/scheme/ton_api.tl | 16 +- tl/generate/scheme/ton_api.tlo | Bin 119800 -> 118952 bytes validator/impl/collator-impl.h | 2 +- validator/impl/collator.cpp | 23 +-- validator/impl/validate-query.cpp | 209 ++++++++++++++--------- validator/impl/validate-query.hpp | 24 +-- validator/interfaces/validator-manager.h | 28 +-- 7 files changed, 175 insertions(+), 127 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 3f0757993..6e1a0fc4f 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -962,10 +962,6 @@ validatorStats.blockStats ext_msgs:validatorStats.blockStats.extMsgsStats transactions:int shard_configuration:(vector tonNode.blockIdExt) old_out_msg_queue_size:long new_out_msg_queue_size:long msg_queue_cleaned:int neighbors:(vector validatorStats.blockStats.neighborStats) = validatorStats.BlockStats; -validatorStats.collateWorkTimeStats - total:double optimistic_apply:double queue_cleanup:double prelim_storage_stat:double trx_tvm:double trx_storage_stat:double - trx_other:double final_storage_stat:double create_block:double create_collated_data:double create_block_candidate:double - = validatorStats.CollateWorkTimeStats; validatorStats.storageStatCacheStats small_cnt:long small_cells:long hit_cnt:long hit_cells:long miss_cnt:long miss_cells:long = validatorStats.StorageStatCacheStats; @@ -974,22 +970,20 @@ validatorStats.collatedBlock bytes:int collated_data_bytes:int attempt:int self:int256 is_validator:Bool total_time:double work_time:double cpu_work_time:double time_stats:string - work_time_real_stats:validatorStats.collateWorkTimeStats - work_time_cpu_stats:validatorStats.collateWorkTimeStats + work_time_real_stats:string + work_time_cpu_stats:string block_limits:validatorStats.blockLimitsStatus block_stats:validatorStats.blockStats storage_stat_cache:validatorStats.storageStatCacheStats = validatorSession.stats.CollatedBlock; -validatorStats.validateWorkTimeStats - total:double trx_tvm:double trx_storage_stat:double trx_other:double = validatorStats.ValidateWorkTimeStats; validatorStats.validatedBlock block_id:tonNode.blockIdExt collated_data_hash:int256 validated_at:double self:int256 valid:Bool comment:string bytes:int collated_data_bytes:int - total_time:double work_time:double cpu_work_time:double - work_time_real_stats:validatorStats.validateWorkTimeStats - work_time_cpu_stats:validatorStats.validateWorkTimeStats + total_time:double work_time:double cpu_work_time:double time_stats:string + work_time_real_stats:string + work_time_cpu_stats:string storage_stat_cache:validatorStats.storageStatCacheStats = validatorStats.ValidatedBlock; validatorStats.newValidatorGroup.node id:int256 pubkey:PublicKey adnl_id:int256 weight:long = validatorStats.newValidatorGroup.Node; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index ce74dbb37275ffa2745966d24b2e1a77a6d32679..ee5fb5ce817f1079c69a32c1625e810f0248b3a2 100644 GIT binary patch delta 200 zcmeydoPEVYcHT#`^{p77zOBfmNH77c6 zPjqIyFAY{P-6ft;ffJEjg>m{BCPsnjGfEg`CjU4n zFnvQJV+~l_c7r6w9W2{_WH6rK02@AiNfFQ-kToE4_>s&}C}#9voNkcF=z?b0pF~Cj Lu-@fAN*Ln-c%w~@ delta 712 zcmZ3nkp0JUcHT#`^{p77z;h$-dpnlYQymW{*?&=aHwmj>HA{XiU}0w>4?H;)PV2u*(2qCGvQlu=}QPCBE=Gr0r`G8F#R-f;}<)U>u{EGR)CnXMn^( zR&WEoF8%DjCTmH4Nn+0Aclpu+`2{7JxtYZ!naS~q1qC^k(;sp(O7rm+mZp}b#wX{b zCgznEOn#RwB_do?QAQkDaeQ)OUJ5AcQW+*I9FqZu&i05j#w#4G;JBJ@n9nG#h7u;gpa4mMf`tVw zShA&v@DvZiQw(5BML@ saved); + void return_block_candidate(td::Result saved, td::PerfLogAction token); bool update_last_proc_int_msg(const std::pair& new_lt_hash); td::CancellationToken cancellation_token_; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index e38d30e1f..608c8f67a 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -573,7 +573,7 @@ bool Collator::request_aux_mc_state(BlockSeqno seqno, Ref& st CHECK(blkid.is_valid_ext() && blkid.is_masterchain()); LOG(DEBUG) << "sending auxiliary wait_block_state() query for " << blkid.to_str() << " to Manager"; ++pending; - auto token = perf_log_.start_action(PSTRING() << "auxiliary wait_block_state " << blkid.to_str()); + auto token = perf_log_.start_action(PSTRING() << "auxiliary wait_block_state " << blkid.seqno()); td::actor::send_closure_later( manager, &ValidatorManager::wait_block_state_short, blkid, priority(), timeout, false, [self = get_self(), blkid, token = std::move(token)](td::Result> res) mutable { @@ -6492,16 +6492,18 @@ bool Collator::create_block_candidate() { } // 4. save block candidate if (skip_store_candidate_) { - td::actor::send_closure_later(actor_id(this), &Collator::return_block_candidate, td::Unit()); + td::actor::send_closure_later(actor_id(this), &Collator::return_block_candidate, td::Unit(), td::PerfLogAction{}); } else { LOG(INFO) << "saving new BlockCandidate"; - td::actor::send_closure_later( - manager, &ValidatorManager::set_block_candidate, block_candidate->id, block_candidate->clone(), - validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), - [self = get_self()](td::Result saved) -> void { - LOG(DEBUG) << "got answer to set_block_candidate"; - td::actor::send_closure_later(std::move(self), &Collator::return_block_candidate, std::move(saved)); - }); + auto token = perf_log_.start_action("set_block_candidate"); + td::actor::send_closure_later(manager, &ValidatorManager::set_block_candidate, block_candidate->id, + block_candidate->clone(), validator_set_->get_catchain_seqno(), + validator_set_->get_validator_set_hash(), + [self = get_self(), token = std::move(token)](td::Result saved) mutable { + LOG(DEBUG) << "got answer to set_block_candidate"; + td::actor::send_closure_later(std::move(self), &Collator::return_block_candidate, + std::move(saved), std::move(token)); + }); } // 5. communicate about bad and delayed external messages if (!bad_ext_msgs_.empty() || !delay_ext_msgs_.empty()) { @@ -6521,8 +6523,9 @@ bool Collator::create_block_candidate() { * * @param saved The result of saving the block candidate to the disk. */ -void Collator::return_block_candidate(td::Result saved) { +void Collator::return_block_candidate(td::Result saved, td::PerfLogAction token) { // 6. return data to the original "caller" + token.finish(saved); if (saved.is_error()) { auto err = saved.move_as_error(); LOG(ERROR) << "cannot save block candidate: " << err.to_string(); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index adcf72eb2..4285a385c 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -365,10 +365,12 @@ void ValidateQuery::start_up() { LOG(DEBUG) << "sending get_top_masterchain_state_block() to Manager"; ++pending; td::actor::send_closure_later(manager, &ValidatorManager::get_top_masterchain_state_block, - [self = get_self()](td::Result, BlockIdExt>> res) { + [self = get_self(), token = perf_log_.start_action("get_top_masterchain_state_block")]( + td::Result, BlockIdExt>> res) mutable { LOG(DEBUG) << "got answer to get_top_masterchain_state_block"; - td::actor::send_closure_later( - std::move(self), &ValidateQuery::after_get_latest_mc_state, std::move(res)); + td::actor::send_closure_later(std::move(self), + &ValidateQuery::after_get_latest_mc_state, + std::move(res), std::move(token)); }); // 3. unpack block candidate (while necessary data is being loaded) if (!unpack_block_candidate()) { @@ -389,12 +391,13 @@ void ValidateQuery::start_up() { // 4. request masterchain handle and state referred to in the block if (!is_masterchain()) { ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::get_block_handle, mc_blkid_, true, - [self = get_self()](td::Result res) { - LOG(DEBUG) << "got answer to get_block_handle() query for masterchain block"; - td::actor::send_closure_later(std::move(self), &ValidateQuery::got_mc_handle, - std::move(res)); - }); + td::actor::send_closure_later( + manager, &ValidatorManager::get_block_handle, mc_blkid_, true, + [self = get_self(), token = perf_log_.start_action("get_block_handle")](td::Result res) mutable { + LOG(DEBUG) << "got answer to get_block_handle() query for masterchain block"; + td::actor::send_closure_later(std::move(self), &ValidateQuery::got_mc_handle, std::move(res), + std::move(token)); + }); } else { if (prev_blocks[0] != mc_blkid_) { soft_reject_query("cannot validate masterchain block "s + id_.to_str() + @@ -406,12 +409,14 @@ void ValidateQuery::start_up() { // 5. get storage stat cache ++pending; LOG(DEBUG) << "sending get_storage_stat_cache() query to Manager"; - td::actor::send_closure_later( - manager, &ValidatorManager::get_storage_stat_cache, - [self = get_self()](td::Result(const td::Bits256&)>> res) { - LOG(DEBUG) << "got answer to get_storage_stat_cache() query"; - td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_storage_stat_cache, std::move(res)); - }); + td::actor::send_closure_later(manager, &ValidatorManager::get_storage_stat_cache, + [self = get_self(), token = perf_log_.start_action("get_storage_stat_cache")]( + td::Result(const td::Bits256&)>> res) mutable { + LOG(DEBUG) << "got answer to get_storage_stat_cache() query"; + td::actor::send_closure_later(std::move(self), + &ValidateQuery::after_get_storage_stat_cache, + std::move(res), std::move(token)); + }); // ... CHECK(pending); } @@ -424,12 +429,14 @@ void ValidateQuery::load_prev_states() { // 4.1. load state LOG(DEBUG) << "sending wait_block_state() query #" << i << " for " << prev_blocks[i].to_str() << " to Manager"; ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), - timeout, false, [self = get_self(), i](td::Result> res) -> void { - LOG(DEBUG) << "got answer to wait_block_state_short query #" << i; - td::actor::send_closure_later( - std::move(self), &ValidateQuery::after_get_shard_state, i, std::move(res)); - }); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_short, prev_blocks[i], priority(), timeout, false, + [self = get_self(), i, token = perf_log_.start_action(PSTRING() << "wait_block_state #" << i)]( + td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state_short query #" << i; + td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_shard_state, i, std::move(res), + std::move(token)); + }); } } @@ -449,25 +456,27 @@ bool ValidateQuery::process_optimistic_prev_block() { if (prev_prev.size() == 1) { LOG(DEBUG) << "sending wait_block_state() query for " << prev_prev[0].to_str() << " to Manager (opt)"; ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, prev_prev[0], priority(), timeout, - false, [self = get_self()](td::Result> res) mutable { - LOG(DEBUG) << "got answer to wait_block_state query (opt)"; - td::actor::send_closure_later(std::move(self), - &ValidateQuery::after_get_shard_state_optimistic, - std::move(res)); - }); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_short, prev_prev[0], priority(), timeout, false, + [self = get_self(), + token = perf_log_.start_action("opt wait_block_state")](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state query (opt)"; + td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_shard_state_optimistic, + std::move(res), std::move(token)); + }); } else { CHECK(prev_prev.size() == 2); LOG(DEBUG) << "sending wait_block_state_merge() query for " << prev_prev[0].to_str() << " and " << prev_prev[1].to_str() << " to Manager (opt)"; ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_merge, prev_prev[0], prev_prev[1], - priority(), timeout, [self = get_self()](td::Result> res) mutable { - LOG(DEBUG) << "got answer to wait_block_state_merge query (opt)"; - td::actor::send_closure_later(std::move(self), - &ValidateQuery::after_get_shard_state_optimistic, - std::move(res)); - }); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_merge, prev_prev[0], prev_prev[1], priority(), timeout, + [self = get_self(), + token = perf_log_.start_action("opt wait_block_state_merge")](td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state_merge query (opt)"; + td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_shard_state_optimistic, + std::move(res), std::move(token)); + }); } return true; } @@ -477,12 +486,14 @@ bool ValidateQuery::process_optimistic_prev_block() { * * @param res The retrieved state. */ -void ValidateQuery::after_get_shard_state_optimistic(td::Result> res) { +void ValidateQuery::after_get_shard_state_optimistic(td::Result> res, td::PerfLogAction token) { + token.finish(res); LOG(DEBUG) << "in ValidateQuery::after_get_shard_state_optimistic()"; if (res.is_error()) { fatal_error(res.move_as_error()); return; } + td::RealCpuTimer timer; work_timer_.resume(); auto state = res.move_as_ok(); auto S = state.write().apply_block(optimistic_prev_block_->block_id(), optimistic_prev_block_); @@ -491,7 +502,8 @@ void ValidateQuery::after_get_shard_state_optimistic(td::Result> return; } work_timer_.pause(); - after_get_shard_state(0, std::move(state)); + stats_.work_time.optimistic_apply = timer.elapsed_both(); + after_get_shard_state(0, std::move(state), {}); } /** @@ -783,10 +795,12 @@ bool ValidateQuery::extract_collated_data() { void ValidateQuery::request_latest_mc_state() { ++pending; td::actor::send_closure_later(manager, &ValidatorManager::get_top_masterchain_state_block, - [self = get_self()](td::Result, BlockIdExt>> res) { + [self = get_self(), token = perf_log_.start_action("get_top_masterchain_state_block")]( + td::Result, BlockIdExt>> res) mutable { LOG(DEBUG) << "got answer to get_top_masterchain_state_block"; - td::actor::send_closure_later( - std::move(self), &ValidateQuery::after_get_latest_mc_state, std::move(res)); + td::actor::send_closure_later(std::move(self), + &ValidateQuery::after_get_latest_mc_state, + std::move(res), std::move(token)); }); } @@ -795,7 +809,9 @@ void ValidateQuery::request_latest_mc_state() { * * @param res The result of the retrieval of the latest masterchain state. */ -void ValidateQuery::after_get_latest_mc_state(td::Result, BlockIdExt>> res) { +void ValidateQuery::after_get_latest_mc_state(td::Result, BlockIdExt>> res, + td::PerfLogAction token) { + token.finish(res); LOG(WARNING) << "in ValidateQuery::after_get_latest_mc_state()"; --pending; if (res.is_error()) { @@ -836,7 +852,8 @@ void ValidateQuery::after_get_latest_mc_state(td::Result> res) { +void ValidateQuery::after_get_mc_state(td::Result> res, td::PerfLogAction token) { + token.finish(res); CHECK(!is_masterchain()); LOG(WARNING) << "in ValidateQuery::after_get_mc_state() for " << mc_blkid_.to_str(); --pending; @@ -861,7 +878,8 @@ void ValidateQuery::after_get_mc_state(td::Result> res) { * * @param res The result of retrieving the masterchain block handle. */ -void ValidateQuery::got_mc_handle(td::Result res) { +void ValidateQuery::got_mc_handle(td::Result res, td::PerfLogAction token) { + token.finish(res); LOG(DEBUG) << "in ValidateQuery::got_mc_handle() for " << mc_blkid_.to_str(); if (res.is_error()) { fatal_error(res.move_as_error()); @@ -870,13 +888,15 @@ void ValidateQuery::got_mc_handle(td::Result res) { auto mc_handle = res.move_as_ok(); td::actor::send_closure_later( manager, &ValidatorManager::wait_block_state, mc_handle, priority(), timeout, false, - [self = get_self(), id = id_, mc_handle](td::Result> res) { + [self = get_self(), id = id_, mc_handle, + token = perf_log_.start_action("mc wait_block_state")](td::Result> res) mutable { LOG(DEBUG) << "got answer to wait_block_state() query for masterchain block"; if (res.is_ok() && mc_handle->id().seqno() > 0 && !mc_handle->inited_proof()) { res = td::Status::Error(-666, "reference masterchain block "s + mc_handle->id().to_str() + " for block " + id.to_str() + " does not have a valid proof"); } - td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_mc_state, std::move(res)); + td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_mc_state, std::move(res), + std::move(token)); }); } @@ -885,7 +905,9 @@ void ValidateQuery::got_mc_handle(td::Result res) { * * @param res The retrieved storage stat cache. */ -void ValidateQuery::after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res) { +void ValidateQuery::after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res, + td::PerfLogAction token) { + token.finish(res); --pending; if (res.is_error()) { LOG(INFO) << "after_get_storage_stat_cache : " << res.error(); @@ -906,7 +928,8 @@ void ValidateQuery::after_get_storage_stat_cache(td::Result> res) { +void ValidateQuery::after_get_shard_state(int idx, td::Result> res, td::PerfLogAction token) { + token.finish(res); LOG(WARNING) << "in ValidateQuery::after_get_shard_state(" << idx << ")"; --pending; if (res.is_error()) { @@ -1684,7 +1707,8 @@ bool ValidateQuery::request_neighbor_queues() { return fatal_error("neighbor from masterchain is not the last mc block"); } ++pending; - send_closure_later(get_self(), &ValidateQuery::got_neighbor_out_queue, i, mc_state_->message_queue()); + send_closure_later(get_self(), &ValidateQuery::got_neighbor_out_queue, i, mc_state_->message_queue(), + td::PerfLogAction{}); ++i; continue; } @@ -1697,7 +1721,8 @@ bool ValidateQuery::request_neighbor_queues() { return reject_query("cannot fetch shard state from collated data", state.move_as_error()); } ++pending; - send_closure_later(get_self(), &ValidateQuery::got_neighbor_out_queue, i, state.move_as_ok()->message_queue()); + send_closure_later(get_self(), &ValidateQuery::got_neighbor_out_queue, i, state.move_as_ok()->message_queue(), + td::PerfLogAction{}); ++i; } } else { @@ -1706,13 +1731,15 @@ bool ValidateQuery::request_neighbor_queues() { ++pending; if (int prev_idx = prev_block_idx(descr.blk_); prev_idx >= 0) { td::actor::send_closure(actor_id(this), &ValidateQuery::got_neighbor_out_queue, i, - prev_states.at(prev_idx)->message_queue()); + prev_states.at(prev_idx)->message_queue(), td::PerfLogAction{}); } else { - send_closure_later(manager, &ValidatorManager::wait_block_message_queue_short, descr.blk_, priority(), timeout, - [self = get_self(), i](td::Result> res) { - td::actor::send_closure(std::move(self), &ValidateQuery::got_neighbor_out_queue, i, - std::move(res)); - }); + send_closure_later( + manager, &ValidatorManager::wait_block_message_queue_short, descr.blk_, priority(), timeout, + [self = get_self(), i, token = perf_log_.start_action(PSTRING() << "wait_block_message_queue #" << i)]( + td::Result> res) mutable { + td::actor::send_closure(std::move(self), &ValidateQuery::got_neighbor_out_queue, i, std::move(res), + std::move(token)); + }); } ++i; } @@ -1727,7 +1754,8 @@ bool ValidateQuery::request_neighbor_queues() { * @param i The index of the neighbor. * @param res The obtained outbound queue. */ -void ValidateQuery::got_neighbor_out_queue(int i, td::Result> res) { +void ValidateQuery::got_neighbor_out_queue(int i, td::Result> res, td::PerfLogAction token) { + token.finish(res); --pending; if (res.is_error()) { fatal_error(res.move_as_error()); @@ -1854,13 +1882,15 @@ bool ValidateQuery::request_aux_mc_state(BlockSeqno seqno, Ref> res) { - LOG(DEBUG) << "got answer to wait_block_state query for " << blkid.to_str(); - td::actor::send_closure_later(std::move(self), - &ValidateQuery::after_get_aux_shard_state, blkid, - std::move(res)); - }); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state_short, blkid, priority(), timeout, false, + [self = get_self(), blkid, + token = perf_log_.start_action(PSTRING() << "auxiliary wait_block_state " << blkid.seqno())]( + td::Result> res) mutable { + LOG(DEBUG) << "got answer to wait_block_state query for " << blkid.to_str(); + td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_aux_shard_state, blkid, std::move(res), + std::move(token)); + }); state.clear(); return true; } @@ -1890,7 +1920,9 @@ Ref ValidateQuery::get_aux_mc_state(BlockSeqno seqno) const { * @param blkid The BlockIdExt of the shard state. * @param res The result of retrieving the shard state. */ -void ValidateQuery::after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res) { +void ValidateQuery::after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res, + td::PerfLogAction token) { + token.finish(res); LOG(DEBUG) << "in ValidateQuery::after_get_aux_shard_state(" << blkid.to_str() << ")"; --pending; if (res.is_error()) { @@ -2286,11 +2318,12 @@ bool ValidateQuery::check_shard_layout() { } if (!new_top_shard_blocks.empty()) { ++pending; - td::actor::send_closure(manager, &ValidatorManager::wait_verify_shard_blocks, std::move(new_top_shard_blocks), - [SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &ValidateQuery::verified_shard_blocks, - R.move_as_status()); - }); + td::actor::send_closure( + manager, &ValidatorManager::wait_verify_shard_blocks, std::move(new_top_shard_blocks), + [SelfId = actor_id(this), + token = perf_log_.start_action("wait_verify_shard_blocks")](td::Result R) mutable { + td::actor::send_closure(SelfId, &ValidateQuery::verified_shard_blocks, R.move_as_status(), std::move(token)); + }); } return check_mc_validator_info(is_key_block_ || (now_ / ccvc.mc_cc_lifetime > prev_now_ / ccvc.mc_cc_lifetime)); } @@ -2465,11 +2498,13 @@ bool ValidateQuery::prepare_out_msg_queue_size() { out_msg_queue_size_known_ = true; for (size_t i = 0; i < prev_blocks.size(); ++i) { ++pending; - send_closure_later(manager, &ValidatorManager::get_out_msg_queue_size, prev_blocks[i], - [self = get_self(), i](td::Result res) { - td::actor::send_closure(std::move(self), &ValidateQuery::got_out_queue_size, i, - std::move(res)); - }); + send_closure_later( + manager, &ValidatorManager::get_out_msg_queue_size, prev_blocks[i], + [self = get_self(), i, token = perf_log_.start_action(PSTRING() << "get_out_msg_queue_size #" << i)]( + td::Result res) mutable { + td::actor::send_closure(std::move(self), &ValidateQuery::got_out_queue_size, i, std::move(res), + std::move(token)); + }); } return true; } @@ -2482,7 +2517,8 @@ bool ValidateQuery::prepare_out_msg_queue_size() { * @param i The index of the previous block (0 or 1). * @param res The result object containing the size of the queue. */ -void ValidateQuery::got_out_queue_size(size_t i, td::Result res) { +void ValidateQuery::got_out_queue_size(size_t i, td::Result res, td::PerfLogAction token) { + token.finish(res); --pending; if (res.is_error()) { fatal_error( @@ -2502,7 +2538,8 @@ void ValidateQuery::got_out_queue_size(size_t i, td::Result res) { * * @param S The status of the operation (OK on success). */ -void ValidateQuery::verified_shard_blocks(td::Status S) { +void ValidateQuery::verified_shard_blocks(td::Status S, td::PerfLogAction token) { + token.finish(S); --pending; if (S.is_error()) { fatal_error(S.move_as_error_prefix("failed to verify shard blocks: ")); @@ -7254,13 +7291,14 @@ bool ValidateQuery::try_validate() { * @returns True. */ bool ValidateQuery::save_candidate() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &ValidateQuery::abort_query, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &ValidateQuery::written_candidate); - } - }); + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), token = perf_log_.start_action("set_block_candidate")](td::Result R) mutable { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ValidateQuery::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ValidateQuery::written_candidate, std::move(token)); + } + }); td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, id_, block_candidate.clone(), validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), std::move(P)); @@ -7275,7 +7313,8 @@ bool ValidateQuery::save_candidate() { * Callback function called after saving block candidate. * Finishes validation. */ -void ValidateQuery::written_candidate() { +void ValidateQuery::written_candidate(td::PerfLogAction token) { + token.finish(td::Status::OK()); finish_query(); } @@ -7297,9 +7336,11 @@ void ValidateQuery::record_stats(bool valid, std::string error_message) { stats_.actual_collated_data_bytes = (td::uint32)block_candidate.collated_data.size(); stats_.total_time = perf_timer_.elapsed(); stats_.work_time.total = work_timer_.elapsed_both(); + stats_.time_stats = (PSTRING() << perf_log_); LOG(WARNING) << "validation took " << perf_timer_.elapsed() << "s"; LOG(WARNING) << "Validate query work time = " << stats_.work_time.total.real << "s, cpu time = " << stats_.work_time.total.cpu << "s"; + LOG(WARNING) << perf_log_; td::actor::send_closure(manager, &ValidatorManager::log_validate_query_stats, std::move(stats_)); } diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 29cdbd8d2..df19b24c1 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -253,6 +253,7 @@ class ValidateQuery : public td::actor::Actor { bool have_unprocessed_account_dispatch_queue_ = false; td::PerfWarningTimer perf_timer_; + td::PerfLog perf_log_; static constexpr td::uint32 priority() { return 2; @@ -271,10 +272,10 @@ class ValidateQuery : public td::actor::Actor { void load_prev_states(); bool process_optimistic_prev_block(); - void after_get_shard_state_optimistic(td::Result> res); + void after_get_shard_state_optimistic(td::Result> res, td::PerfLogAction token); bool save_candidate(); - void written_candidate(); + void written_candidate(td::PerfLogAction token); bool fatal_error(td::Status error); bool fatal_error(int err_code, std::string err_msg); @@ -307,11 +308,12 @@ class ValidateQuery : public td::actor::Actor { } void request_latest_mc_state(); - void after_get_latest_mc_state(td::Result, BlockIdExt>> res); - void after_get_mc_state(td::Result> res); - void got_mc_handle(td::Result res); - void after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res); - void after_get_shard_state(int idx, td::Result> res); + void after_get_latest_mc_state(td::Result, BlockIdExt>> res, td::PerfLogAction token); + void after_get_mc_state(td::Result> res, td::PerfLogAction token); + void got_mc_handle(td::Result res, td::PerfLogAction token); + void after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res, + td::PerfLogAction token); + void after_get_shard_state(int idx, td::Result> res, td::PerfLogAction token); bool process_mc_state(Ref mc_state); bool try_unpack_mc_state(); bool fetch_config_params(); @@ -331,12 +333,12 @@ class ValidateQuery : public td::actor::Actor { bool unpack_one_prev_state(block::ShardState& ss, BlockIdExt blkid, Ref prev_state_root); bool split_prev_state(block::ShardState& ss); bool request_neighbor_queues(); - void got_neighbor_out_queue(int i, td::Result> res); + void got_neighbor_out_queue(int i, td::Result> res, td::PerfLogAction token); bool register_mc_state(Ref other_mc_state); bool request_aux_mc_state(BlockSeqno seqno, Ref& state); Ref get_aux_mc_state(BlockSeqno seqno) const; - void after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res); + void after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res, td::PerfLogAction token); bool check_one_shard(const block::McShardHash& info, const block::McShardHash* sibling, const block::WorkchainInfo* wc_info, const block::CatchainValidatorsConfig& ccvc, bool& is_new); @@ -346,8 +348,8 @@ class ValidateQuery : public td::actor::Actor { bool check_mc_validator_info(bool update_mc_cc); bool check_utime_lt(); bool prepare_out_msg_queue_size(); - void got_out_queue_size(size_t i, td::Result res); - void verified_shard_blocks(td::Status S); + void got_out_queue_size(size_t i, td::Result res, td::PerfLogAction token); + void verified_shard_blocks(td::Status S, td::PerfLogAction token); bool fix_one_processed_upto(block::MsgProcessedUpto& proc, ton::ShardIdFull owner, bool allow_cur = false); bool fix_processed_upto(block::MsgProcessedUptoCollection& upto, bool allow_cur = false); diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 90e11a35b..ade27c4ee 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -128,11 +128,16 @@ struct CollationStats { td::RealCpuTimer::Time create_collated_data; td::RealCpuTimer::Time create_block_candidate; - tl_object_ptr tl(bool is_cpu) const { - return create_tl_object( - total.get(is_cpu), optimistic_apply.get(is_cpu), queue_cleanup.get(is_cpu), prelim_storage_stat.get(is_cpu), - trx_tvm.get(is_cpu), trx_storage_stat.get(is_cpu), trx_other.get(is_cpu), final_storage_stat.get(is_cpu), - create_block.get(is_cpu), create_collated_data.get(is_cpu), create_block_candidate.get(is_cpu)); + std::string to_str(bool is_cpu) const { + return PSTRING() << "total=" << total.get(is_cpu) << " optimistic_apply=" << optimistic_apply.get(is_cpu) + << " queue_cleanup=" << queue_cleanup.get(is_cpu) + << " prelim_storage_stat=" << prelim_storage_stat.get(is_cpu) + << " trx_tvm=" << trx_tvm.get(is_cpu) << " trx_storage_stat=" << trx_storage_stat.get(is_cpu) + << " trx_other=" << trx_other.get(is_cpu) + << " final_storage_stat=" << final_storage_stat.get(is_cpu) + << " create_block=" << create_block.get(is_cpu) + << " create_collated_data=" << create_collated_data.get(is_cpu) + << " create_block_candidate=" << create_block_candidate.get(is_cpu); } }; WorkTimeStats work_time; @@ -155,7 +160,7 @@ struct CollationStats { return create_tl_object( create_tl_block_id(block_id), collated_data_hash, cc_seqno, collated_at, actual_bytes, actual_collated_data_bytes, attempt, self.bits256_value(), is_validator, total_time, work_time.total.real, - work_time.total.cpu, time_stats, work_time.tl(false), work_time.tl(true), + work_time.total.cpu, time_stats, work_time.to_str(false), work_time.to_str(true), create_tl_object( estimated_bytes, gas, lt_delta, estimated_collated_data_bytes, cat_bytes, cat_gas, cat_lt_delta, cat_collated_data_bytes, load_fraction_queue_cleanup, load_fraction_dispatch, load_fraction_internals, @@ -173,16 +178,19 @@ struct ValidationStats { std::string comment; td::uint32 actual_bytes = 0, actual_collated_data_bytes = 0; double total_time = 0.0; + std::string time_stats; struct WorkTimeStats { td::RealCpuTimer::Time total; + td::RealCpuTimer::Time optimistic_apply; td::RealCpuTimer::Time trx_tvm; td::RealCpuTimer::Time trx_storage_stat; td::RealCpuTimer::Time trx_other; - tl_object_ptr tl(bool is_cpu) const { - return create_tl_object( - total.get(is_cpu), trx_tvm.get(is_cpu), trx_storage_stat.get(is_cpu), trx_other.get(is_cpu)); + std::string to_str(bool is_cpu) const { + return PSTRING() << "total=" << total.get(is_cpu) << " optimistic_apply=" << optimistic_apply.get(is_cpu) + << " trx_tvm=" << trx_tvm.get(is_cpu) << " trx_storage_stat=" << trx_storage_stat.get(is_cpu) + << " trx_other=" << trx_other.get(is_cpu); } }; WorkTimeStats work_time; @@ -192,7 +200,7 @@ struct ValidationStats { return create_tl_object( create_tl_block_id(block_id), collated_data_hash, validated_at, self.bits256_value(), valid, comment, actual_bytes, actual_collated_data_bytes, total_time, work_time.total.real, work_time.total.cpu, - work_time.tl(false), work_time.tl(true), storage_stat_cache.tl()); + time_stats, work_time.to_str(false), work_time.to_str(true), storage_stat_cache.tl()); } }; From d268ac0cb441c3d6e70aae9b6b7655c7e04bb0ff Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 10 Sep 2025 10:14:57 +0300 Subject: [PATCH 382/388] Set block id in ConfigInfo::extract_config (#1780) This fixes calculating prev_blocks_info in tonlib and similar places --- crypto/block/check-proof.cpp | 2 +- crypto/block/mc-config.cpp | 16 +++++----------- crypto/block/mc-config.h | 5 ++--- tonlib/tonlib/LastConfig.cpp | 5 +++-- tonlib/tonlib/TonlibClient.cpp | 2 +- validator/collator-node/collator-node.cpp | 3 ++- validator/impl/collator.cpp | 3 +-- validator/impl/liteserver.cpp | 19 ++++++++++--------- validator/impl/liteserver.hpp | 2 +- validator/impl/shard.cpp | 7 +++---- validator/impl/validate-query.cpp | 13 ++++++------- 11 files changed, 35 insertions(+), 42 deletions(-) diff --git a/crypto/block/check-proof.cpp b/crypto/block/check-proof.cpp index 431a03fec..c3c9085da 100644 --- a/crypto/block/check-proof.cpp +++ b/crypto/block/check-proof.cpp @@ -492,7 +492,7 @@ td::Status BlockProofLink::validate(td::uint32* save_utime) const { return td::Status::Error("BlockProofLink contains a state proof for "s + from.to_str() + " with incorrect root hash"); } - TRY_RESULT(config, block::ConfigInfo::extract_config(vstate_root, block::ConfigInfo::needPrevBlocks)); + TRY_RESULT(config, block::ConfigInfo::extract_config(vstate_root, from, block::ConfigInfo::needPrevBlocks)); if (!config->check_old_mc_block_id(to, true)) { return td::Status::Error("cannot check that "s + to.to_str() + " is indeed a previous masterchain block of " + from.to_str() + " using the presented Merkle proof of masterchain state"); diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 35c3df005..ce11bcaec 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -92,22 +92,16 @@ td::Result> Config::extract_from_state(Ref mc_ return unpack_config(std::move(extra.config), mode); } -td::Result> ConfigInfo::extract_config(std::shared_ptr static_boc, - int mode) { - TRY_RESULT(rc, static_boc->get_root_count()); - if (rc != 1) { - return td::Status::Error(-668, "Masterchain state BoC is invalid"); - } - TRY_RESULT(root, static_boc->get_root_cell(0)); - return extract_config(std::move(root), mode); -} - -td::Result> ConfigInfo::extract_config(Ref mc_state_root, int mode) { +td::Result> ConfigInfo::extract_config(Ref mc_state_root, + ton::BlockIdExt mc_block_id, int mode) { if (mc_state_root.is_null()) { return td::Status::Error("configuration state root cell is null"); } auto config = std::unique_ptr{new ConfigInfo(std::move(mc_state_root), mode)}; TRY_STATUS(config->unpack_wrapped()); + if (!config->set_block_id_ext(mc_block_id)) { + return td::Status::Error("failed to set mc block id"); + } return std::move(config); } diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index c5a6d7722..03dda0b70 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -775,9 +775,8 @@ class ConfigInfo : public Config, public ShardConfig { std::vector compute_validator_set_cc(ton::ShardIdFull shard, ton::UnixTime time, ton::CatchainSeqno* cc_seqno_delta = nullptr) const; td::Result> get_prev_blocks_info() const; - static td::Result> extract_config(std::shared_ptr static_boc, - int mode = 0); - static td::Result> extract_config(Ref mc_state_root, int mode = 0); + static td::Result> extract_config(Ref mc_state_root, + ton::BlockIdExt mc_block_id, int mode = 0); private: ConfigInfo(Ref mc_state_root, int _mode = 0); diff --git a/tonlib/tonlib/LastConfig.cpp b/tonlib/tonlib/LastConfig.cpp index e972d84e0..45d6c9ebb 100644 --- a/tonlib/tonlib/LastConfig.cpp +++ b/tonlib/tonlib/LastConfig.cpp @@ -93,8 +93,9 @@ td::Status LastConfig::process_config_proof(ton::ton_api::object_ptrstate_proof_.as_slice(), raw_config->config_proof_.as_slice())); - TRY_RESULT(config, block::ConfigInfo::extract_config( - std::move(state), block::ConfigInfo::needPrevBlocks | block::ConfigInfo::needCapabilities)); + TRY_RESULT(config, + block::ConfigInfo::extract_config( + std::move(state), blkid, block::ConfigInfo::needPrevBlocks | block::ConfigInfo::needCapabilities)); for (auto i : params_) { VLOG(last_config) << "ConfigParam(" << i << ") = "; diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 7e868480b..3132bf766 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -2168,7 +2168,7 @@ class RunEmulator : public TonlibQueryActor { } try { - auto r_config = block::ConfigInfo::extract_config(mc_state_root_, 0b11'11111111); + auto r_config = block::ConfigInfo::extract_config(mc_state_root_, block_id_.mc, 0b11'11111111); if (r_config.is_error()) { check(r_config.move_as_error()); return; diff --git a/validator/collator-node/collator-node.cpp b/validator/collator-node/collator-node.cpp index dd0dfb792..9b03d9dc0 100644 --- a/validator/collator-node/collator-node.cpp +++ b/validator/collator-node/collator-node.cpp @@ -465,7 +465,8 @@ td::Status CollatorNode::check_mc_config() { } TRY_RESULT_PREFIX( config, - block::ConfigInfo::extract_config(last_masterchain_state_->root_cell(), block::ConfigInfo::needCapabilities), + block::ConfigInfo::extract_config(last_masterchain_state_->root_cell(), last_masterchain_state_->get_block_id(), + block::ConfigInfo::needCapabilities), "cannot unpack masterchain config"); if (config->get_global_version() > Collator::supported_version()) { return td::Status::Error(PSTRING() << "unsupported global version " << config->get_global_version() diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 608c8f67a..ead5ec038 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -856,7 +856,7 @@ void Collator::after_get_shard_state_optimistic(td::Result> res, */ bool Collator::unpack_last_mc_state() { auto res = block::ConfigInfo::extract_config( - mc_state_root, + mc_state_root, mc_block_id_, block::ConfigInfo::needShardHashes | block::ConfigInfo::needLibraries | block::ConfigInfo::needValidatorSet | block::ConfigInfo::needWorkchainInfo | block::ConfigInfo::needCapabilities | block::ConfigInfo::needPrevBlocks | @@ -868,7 +868,6 @@ bool Collator::unpack_last_mc_state() { } config_ = res.move_as_ok(); CHECK(config_); - config_->set_block_id_ext(mc_block_id_); global_id_ = config_->get_global_blockchain_id(); ihr_enabled_ = config_->ihr_enabled(); create_stats_enabled_ = config_->create_stats_enabled(); diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index e9592f626..2fabe8cb0 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -917,7 +917,8 @@ void LiteQuery::continue_getLibraries(Ref mc_s mc_state_ = Ref(std::move(mc_state)); CHECK(mc_state_.not_null()); - auto rconfig = block::ConfigInfo::extract_config(mc_state_->root_cell(), block::ConfigInfo::needLibraries); + auto rconfig = block::ConfigInfo::extract_config(mc_state_->root_cell(), mc_state_->get_block_id(), + block::ConfigInfo::needLibraries); if (rconfig.is_error()) { fatal_error("cannot extract library list block configuration from masterchain state"); return; @@ -1251,9 +1252,10 @@ bool LiteQuery::make_shard_info_proof(Ref& proof, BlockIdExt& blkid, A return true; } -bool LiteQuery::make_ancestor_block_proof(Ref& proof, Ref state_root, const BlockIdExt& old_blkid) { - vm::MerkleProofBuilder mpb{std::move(state_root)}; - auto rconfig = block::ConfigInfo::extract_config(mpb.root(), block::ConfigInfo::needPrevBlocks); +bool LiteQuery::make_ancestor_block_proof(Ref& proof, Ref mc_state, const BlockIdExt& old_blkid) { + vm::MerkleProofBuilder mpb{mc_state->root_cell()}; + auto rconfig = + block::ConfigInfo::extract_config(mpb.root(), mc_state->get_block_id(), block::ConfigInfo::needPrevBlocks); if (rconfig.is_error()) { return fatal_error( "cannot extract previous block configuration from masterchain state while constructing Merkle proof for "s + @@ -1319,13 +1321,12 @@ void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) { vm::AugmentedDictionary accounts_dict{vm::load_cell_slice_ref(sstate.accounts), 256, block::tlb::aug_ShardAccounts}; auto acc_csr = accounts_dict.lookup(acc_addr_); if (mode_ & 0x80000000) { - auto config = block::ConfigInfo::extract_config(mc_state_->root_cell(), 0xFFFF); + auto config = block::ConfigInfo::extract_config(mc_state_->root_cell(), mc_state_->get_block_id(), 0xFFFF); if (config.is_error()) { fatal_error(config.move_as_error()); return; } auto rconfig = config.move_as_ok(); - rconfig->set_block_id_ext(mc_state_->get_block_id()); acc_state_promise_.set_value(std::make_tuple( std::move(acc_csr), sstate.gen_utime, sstate.gen_lt, std::move(rconfig) )); @@ -1502,7 +1503,7 @@ void LiteQuery::finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice LOG(DEBUG) << "creating VM with gas limit " << gas_limit; // **** INIT VM **** auto r_config = block::ConfigInfo::extract_config( - mc_state_->root_cell(), + mc_state_->root_cell(), mc_state_->get_block_id(), block::ConfigInfo::needLibraries | block::ConfigInfo::needCapabilities | block::ConfigInfo::needPrevBlocks); if (r_config.is_error()) { fatal_error(r_config.move_as_error()); @@ -1905,7 +1906,7 @@ void LiteQuery::continue_getConfigParams(int mode, std::vector param_list) if (mode & block::ConfigInfo::needPrevBlocks) { mode |= block::ConfigInfo::needCapabilities; } - auto res = block::ConfigInfo::extract_config(mpb.root(), mode); + auto res = block::ConfigInfo::extract_config(mpb.root(), keyblk ? base_blk_id_ : mc_state_->get_block_id(), mode); if (res.is_error()) { fatal_error(res.move_as_error()); return; @@ -3045,7 +3046,7 @@ bool LiteQuery::construct_proof_link_back_cont(ton::BlockIdExt cur, ton::BlockId return fatal_error("cannot construct proof for state of masterchain block "s + cur.to_str()); } // construct proof that `next` is listed in OldMcBlocksInfo of `mc_state_` - if (!make_ancestor_block_proof(state_proof, mc_state_->root_cell(), next)) { + if (!make_ancestor_block_proof(state_proof, mc_state_, next)) { return fatal_error("cannot prove that "s + next.to_str() + " is in the previous block set of the masterchain state of " + cur.to_str()); } diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index fc8735332..afeabb9c0 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -221,7 +221,7 @@ class LiteQuery : public td::actor::Actor { bool make_shard_info_proof(Ref& proof, Ref& info, ShardIdFull shard, bool exact = true); bool make_shard_info_proof(Ref& proof, Ref& info, AccountIdPrefixFull prefix); bool make_shard_info_proof(Ref& proof, BlockIdExt& blkid, AccountIdPrefixFull prefix); - bool make_ancestor_block_proof(Ref& proof, Ref state_root, const BlockIdExt& old_blkid); + bool make_ancestor_block_proof(Ref& proof, Ref mc_state, const BlockIdExt& old_blkid); }; } // namespace validator diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index 69bbfdd62..5803eac89 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -376,9 +376,9 @@ td::Status MasterchainStateQ::mc_init() { td::Status MasterchainStateQ::mc_reinit() { auto res = block::ConfigInfo::extract_config( - root_cell(), block::ConfigInfo::needStateRoot | block::ConfigInfo::needValidatorSet | - block::ConfigInfo::needShardHashes | block::ConfigInfo::needPrevBlocks | - block::ConfigInfo::needWorkchainInfo); + root_cell(), blkid, + block::ConfigInfo::needStateRoot | block::ConfigInfo::needValidatorSet | block::ConfigInfo::needShardHashes | + block::ConfigInfo::needPrevBlocks | block::ConfigInfo::needWorkchainInfo); cur_validators_.reset(); next_validators_.reset(); if (res.is_error()) { @@ -386,7 +386,6 @@ td::Status MasterchainStateQ::mc_reinit() { } config_ = res.move_as_ok(); CHECK(config_); - CHECK(config_->set_block_id_ext(get_block_id())); cur_validators_ = config_->get_cur_validator_set(); diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 4285a385c..da5d7bf73 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -1009,7 +1009,7 @@ bool ValidateQuery::try_unpack_mc_state() { return fatal_error(-666, "latest masterchain state does not have a root cell"); } auto res = block::ConfigInfo::extract_config( - mc_state_root_, + mc_state_root_, mc_blkid_, block::ConfigInfo::needShardHashes | block::ConfigInfo::needLibraries | block::ConfigInfo::needValidatorSet | block::ConfigInfo::needWorkchainInfo | block::ConfigInfo::needStateExtraRoot | block::ConfigInfo::needCapabilities | block::ConfigInfo::needPrevBlocks | @@ -1020,7 +1020,6 @@ bool ValidateQuery::try_unpack_mc_state() { } config_ = res.move_as_ok(); CHECK(config_); - config_->set_block_id_ext(mc_blkid_); ihr_enabled_ = config_->ihr_enabled(); create_stats_enabled_ = config_->create_stats_enabled(); if (config_->has_capabilities() && (config_->get_capabilities() & ~supported_capabilities())) { @@ -1514,17 +1513,17 @@ bool ValidateQuery::compute_next_state() { } } auto r_config_info = block::ConfigInfo::extract_config( - state_root_, block::ConfigInfo::needShardHashes | block::ConfigInfo::needLibraries | - block::ConfigInfo::needValidatorSet | block::ConfigInfo::needWorkchainInfo | - block::ConfigInfo::needStateExtraRoot | block::ConfigInfo::needAccountsRoot | - block::ConfigInfo::needSpecialSmc | block::ConfigInfo::needCapabilities); + state_root_, id_, + block::ConfigInfo::needShardHashes | block::ConfigInfo::needLibraries | block::ConfigInfo::needValidatorSet | + block::ConfigInfo::needWorkchainInfo | block::ConfigInfo::needStateExtraRoot | + block::ConfigInfo::needAccountsRoot | block::ConfigInfo::needSpecialSmc | + block::ConfigInfo::needCapabilities); if (r_config_info.is_error()) { return reject_query("cannot extract configuration from new masterchain state "s + mc_blkid_.to_str() + " : " + r_config_info.error().to_string()); } new_config_ = r_config_info.move_as_ok(); CHECK(new_config_); - new_config_->set_block_id_ext(id_); } return true; } From 3c10903dc59140e7dd13673f13d813d31ea88006 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 15 Sep 2025 16:15:09 +0300 Subject: [PATCH 383/388] Send validator telemetry from all nodes in fast-sync overlay (#1798) --- create-hardfork/create-hardfork.cpp | 3 -- test/test-ton-collator.cpp | 3 -- validator/full-node-fast-sync-overlays.cpp | 15 ++++++++ validator/full-node-fast-sync-overlays.hpp | 3 ++ validator/full-node-private-overlay.cpp | 24 +++++++++---- validator/full-node-private-overlay.hpp | 3 ++ validator/full-node.cpp | 21 ----------- validator/full-node.hpp | 1 - validator/interfaces/validator-manager.h | 1 - validator/manager-disk.hpp | 2 -- validator/manager-hardfork.hpp | 2 -- validator/manager.cpp | 42 ---------------------- validator/manager.hpp | 8 ----- validator/validator-telemetry.cpp | 4 +-- validator/validator-telemetry.hpp | 25 +++++++------ validator/validator.h | 1 - 16 files changed, 52 insertions(+), 106 deletions(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index e2f77dd86..aadf877ca 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -280,9 +280,6 @@ class HardforkCreator : public td::actor::Actor { void new_key_block(ton::validator::BlockHandle handle) override { } - void send_validator_telemetry(ton::PublicKeyHash key, - ton::tl_object_ptr telemetry) override { - } }; td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::install_callback, diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index 89be586e6..3ee723ec8 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -381,9 +381,6 @@ class TestNode : public td::actor::Actor { void new_key_block(ton::validator::BlockHandle handle) override { } - void send_validator_telemetry(ton::PublicKeyHash key, - ton::tl_object_ptr telemetry) override { - } }; td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::install_callback, diff --git a/validator/full-node-fast-sync-overlays.cpp b/validator/full-node-fast-sync-overlays.cpp index 3b4a36c6d..e9927518a 100644 --- a/validator/full-node-fast-sync-overlays.cpp +++ b/validator/full-node-fast-sync-overlays.cpp @@ -325,6 +325,21 @@ void FullNodeFastSyncOverlay::init() { std::make_unique(actor_id(this)), rules, std::move(scope), options); inited_ = true; + if (shard_.is_masterchain()) { + class TelemetryCallback : public ValidatorTelemetry::Callback { + public: + explicit TelemetryCallback(td::actor::ActorId id) : id_(id) { + } + void send_telemetry(tl_object_ptr telemetry) override { + td::actor::send_closure(id_, &FullNodeFastSyncOverlay::send_validator_telemetry, std::move(telemetry)); + } + + private: + td::actor::ActorId id_; + }; + telemetry_sender_ = td::actor::create_actor( + "telemetry", local_id_, std::make_unique(actor_id(this))); + } } void FullNodeFastSyncOverlay::tear_down() { diff --git a/validator/full-node-fast-sync-overlays.hpp b/validator/full-node-fast-sync-overlays.hpp index 41a45cbc7..3505ae5eb 100644 --- a/validator/full-node-fast-sync-overlays.hpp +++ b/validator/full-node-fast-sync-overlays.hpp @@ -17,6 +17,8 @@ #pragma once #include "full-node.h" +#include "validator-telemetry.hpp" + #include namespace ton::validator::fullnode { @@ -108,6 +110,7 @@ class FullNodeFastSyncOverlay : public td::actor::Actor { void init(); void get_stats_extra(td::Promise promise); + td::actor::ActorOwn telemetry_sender_; bool collect_telemetry_ = false; std::ofstream telemetry_file_; }; diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp index fd5e970ba..7ea813a46 100644 --- a/validator/full-node-private-overlay.cpp +++ b/validator/full-node-private-overlay.cpp @@ -99,7 +99,7 @@ void FullNodePrivateBlockOverlay::process_block_candidate_broadcast(PublicKeyHas } void FullNodePrivateBlockOverlay::process_telemetry_broadcast( - PublicKeyHash src, const tl_object_ptr& telemetry) { + PublicKeyHash src, const tl_object_ptr &telemetry) { if (telemetry->adnl_id_ != src.bits256_value()) { VLOG(FULL_NODE_WARNING) << "Invalid telemetry broadcast from " << src << ": adnl_id mismatch"; return; @@ -117,9 +117,7 @@ void FullNodePrivateBlockOverlay::process_telemetry_broadcast( } VLOG(FULL_NODE_DEBUG) << "Got telemetry broadcast from " << src; auto s = td::json_encode(td::ToJson(*telemetry), false); - std::erase_if(s, [](char c) { - return c == '\n' || c == '\r'; - }); + std::erase_if(s, [](char c) { return c == '\n' || c == '\r'; }); telemetry_file_ << s << "\n"; telemetry_file_.flush(); if (telemetry_file_.fail()) { @@ -141,9 +139,7 @@ void FullNodePrivateBlockOverlay::receive_broadcast(PublicKeyHash src, td::Buffe } return; } - ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto& obj) { - Self->process_broadcast(src, obj); - }); + ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto &obj) { Self->process_broadcast(src, obj); }); } void FullNodePrivateBlockOverlay::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, @@ -286,6 +282,20 @@ void FullNodePrivateBlockOverlay::init() { td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_id_); inited_ = true; + + class TelemetryCallback : public ValidatorTelemetry::Callback { + public: + explicit TelemetryCallback(td::actor::ActorId id) : id_(id) { + } + void send_telemetry(tl_object_ptr telemetry) override { + td::actor::send_closure(id_, &FullNodePrivateBlockOverlay::send_validator_telemetry, std::move(telemetry)); + } + + private: + td::actor::ActorId id_; + }; + telemetry_sender_ = td::actor::create_actor("telemetry", local_id_, + std::make_unique(actor_id(this))); } void FullNodePrivateBlockOverlay::tear_down() { diff --git a/validator/full-node-private-overlay.hpp b/validator/full-node-private-overlay.hpp index 355e44858..2cbe25f68 100644 --- a/validator/full-node-private-overlay.hpp +++ b/validator/full-node-private-overlay.hpp @@ -17,6 +17,8 @@ #pragma once #include "full-node.h" +#include "validator-telemetry.hpp" + #include namespace ton::validator::fullnode { @@ -100,6 +102,7 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { void try_init(); void init(); + td::actor::ActorOwn telemetry_sender_; bool collect_telemetry_ = false; std::ofstream telemetry_file_; }; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 890fd7a2e..d648fc821 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -625,24 +625,6 @@ void FullNodeImpl::new_key_block(BlockHandle handle) { } } -void FullNodeImpl::send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) { - if (use_old_private_overlays_) { - auto it = private_block_overlays_.find(key); - if (it == private_block_overlays_.end()) { - VLOG(FULL_NODE_INFO) << "Cannot send validator telemetry for " << key << " : no private block overlay"; - return; - } - td::actor::send_closure(it->second, &FullNodePrivateBlockOverlay::send_validator_telemetry, std::move(telemetry)); - } else { - auto overlay = fast_sync_overlays_.get_masterchain_overlay_for(adnl::AdnlNodeIdShort{telemetry->adnl_id_}); - if (overlay.empty()) { - VLOG(FULL_NODE_INFO) << "Cannot send validator telemetry for adnl id " << key << " : no fast sync overlay"; - return; - } - td::actor::send_closure(overlay, &FullNodeFastSyncOverlay::send_validator_telemetry, std::move(telemetry)); - } -} - void FullNodeImpl::process_block_broadcast(BlockBroadcast broadcast) { send_block_broadcast_to_custom_overlays(broadcast); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_block_broadcast, std::move(broadcast), @@ -788,9 +770,6 @@ void FullNodeImpl::start_up() { void new_key_block(BlockHandle handle) override { td::actor::send_closure(id_, &FullNodeImpl::new_key_block, std::move(handle)); } - void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override { - td::actor::send_closure(id_, &FullNodeImpl::send_validator_telemetry, key, std::move(telemetry)); - } explicit Callback(td::actor::ActorId id) : id_(id) { } diff --git a/validator/full-node.hpp b/validator/full-node.hpp index dfd12f054..482a916bb 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -90,7 +90,6 @@ class FullNodeImpl : public FullNode { void got_key_block_config(td::Ref config); void new_key_block(BlockHandle handle); - void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry); void process_block_broadcast(BlockBroadcast broadcast) override; void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index ade27c4ee..90e524756 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -311,7 +311,6 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void send_ihr_message(td::Ref message) = 0; virtual void send_top_shard_block_description(td::Ref desc) = 0; virtual void send_block_broadcast(BlockBroadcast broadcast, int mode) = 0; - virtual void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) = 0; virtual void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index bdde74b2d..1c4d00d72 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -275,8 +275,6 @@ class ValidatorManagerImpl : public ValidatorManager { void send_top_shard_block_description(td::Ref desc) override; void send_block_broadcast(BlockBroadcast broadcast, int mode) override { } - void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override { - } void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) override { diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 5757c2bd4..37b5d6dd3 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -352,8 +352,6 @@ class ValidatorManagerImpl : public ValidatorManager { } void send_block_broadcast(BlockBroadcast broadcast, int mode) override { } - void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override { - } void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) override { diff --git a/validator/manager.cpp b/validator/manager.cpp index f61bd5f0b..a5315788d 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1867,11 +1867,6 @@ void ValidatorManagerImpl::send_block_broadcast(BlockBroadcast broadcast, int mo callback_->send_broadcast(std::move(broadcast), mode); } -void ValidatorManagerImpl::send_validator_telemetry(PublicKeyHash key, - tl_object_ptr telemetry) { - callback_->send_validator_telemetry(key, std::move(telemetry)); -} - void ValidatorManagerImpl::send_get_out_msg_queue_proof_request( ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) { @@ -2065,7 +2060,6 @@ void ValidatorManagerImpl::started(ValidatorManagerInitResult R) { if (opts_->nonfinal_ls_queries_enabled()) { candidates_buffer_ = td::actor::create_actor("candidates-buffer", actor_id(this)); } - init_validator_telemetry(); auto Q = td::PromiseCreator::lambda( [SelfId = actor_id(this)](td::Result>> R) { @@ -2263,7 +2257,6 @@ void ValidatorManagerImpl::new_masterchain_block() { td::actor::send_closure(serializer_, &AsyncStateSerializer::update_last_known_key_block_ts, last_key_block_handle_->unix_time()); } - init_validator_telemetry(); } update_shard_overlays(); @@ -3725,41 +3718,6 @@ void ValidatorManagerImpl::CheckedExtMsgCounter::before_query() { } } -void ValidatorManagerImpl::init_validator_telemetry() { - if (last_masterchain_state_.is_null()) { - return; - } - td::Ref validator_set = last_masterchain_state_->get_total_validator_set(0); - if (validator_set.is_null()) { - validator_telemetry_.clear(); - return; - } - std::set processed; - for (auto &key : temp_keys_) { - if (const ValidatorDescr *desc = validator_set->get_validator(key.bits256_value())) { - processed.insert(key); - adnl::AdnlNodeIdShort adnl_id; - if (desc->addr.is_zero()) { - adnl_id = adnl::AdnlNodeIdShort{ValidatorFullId{desc->key}.compute_short_id()}; - } else { - adnl_id = adnl::AdnlNodeIdShort{desc->addr}; - } - auto &telemetry = validator_telemetry_[key]; - if (telemetry.empty()) { - telemetry = td::actor::create_actor("telemetry", key, adnl_id, - opts_->zero_block_id().file_hash, actor_id(this)); - } - } - } - for (auto it = validator_telemetry_.begin(); it != validator_telemetry_.end();) { - if (processed.contains(it->first)) { - ++it; - } else { - it = validator_telemetry_.erase(it); - } - } -} - template void ValidatorManagerImpl::write_session_stats(const T &obj) { std::string fname = opts_->get_session_logs_file(); diff --git a/validator/manager.hpp b/validator/manager.hpp index 68799c372..b3fdbc344 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -36,7 +36,6 @@ #include "token-manager.h" #include "queue-size-counter.hpp" #include "storage-stat-cache.hpp" -#include "validator-telemetry.hpp" #include "impl/candidates-buffer.hpp" #include "collator-node/collator-node.hpp" #include "shard-block-verifier.hpp" @@ -384,7 +383,6 @@ class ValidatorManagerImpl : public ValidatorManager { } void add_temp_key(PublicKeyHash key, td::Promise promise) override { temp_keys_.insert(key); - init_validator_telemetry(); promise.set_value(td::Unit()); } void del_permanent_key(PublicKeyHash key, td::Promise promise) override { @@ -393,7 +391,6 @@ class ValidatorManagerImpl : public ValidatorManager { } void del_temp_key(PublicKeyHash key, td::Promise promise) override { temp_keys_.erase(key); - init_validator_telemetry(); promise.set_value(td::Unit()); } @@ -558,7 +555,6 @@ class ValidatorManagerImpl : public ValidatorManager { void send_ihr_message(td::Ref message) override; void send_top_shard_block_description(td::Ref desc) override; void send_block_broadcast(BlockBroadcast broadcast, int mode) override; - void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override; void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, td::Promise>> promise) override; @@ -824,10 +820,6 @@ class ValidatorManagerImpl : public ValidatorManager { void iterate_temp_block_handles(std::function f) override; - std::map> validator_telemetry_; - - void init_validator_telemetry(); - struct Collator { td::actor::ActorOwn actor; std::set shards; diff --git a/validator/validator-telemetry.cpp b/validator/validator-telemetry.cpp index 403dd6f9f..b870e8b84 100644 --- a/validator/validator-telemetry.cpp +++ b/validator/validator-telemetry.cpp @@ -51,7 +51,7 @@ void ValidatorTelemetry::start_up() { cpu_cores_ = r_cpu_cores.move_as_ok(); } - LOG(DEBUG) << "Initializing validator telemetry, key = " << key_ << ", adnl_id = " << local_id_; + LOG(DEBUG) << "Initializing validator telemetry, adnl_id = " << local_id_; alarm_timestamp().relax(send_telemetry_at_ = td::Timestamp::in(td::Random::fast(30.0, 60.0))); } @@ -81,7 +81,7 @@ void ValidatorTelemetry::send_telemetry() { .cpu_threads_count; LOG(DEBUG) << "Sending validator telemetry for adnl id " << local_id_; - td::actor::send_closure(manager_, &ValidatorManager::send_validator_telemetry, key_, std::move(telemetry)); + callback_->send_telemetry(std::move(telemetry)); } } // namespace ton::validator diff --git a/validator/validator-telemetry.hpp b/validator/validator-telemetry.hpp index 73908bdd1..d83641dfe 100644 --- a/validator/validator-telemetry.hpp +++ b/validator/validator-telemetry.hpp @@ -33,23 +33,23 @@ namespace ton::validator { class ValidatorManager; class ValidatorTelemetry : public td::actor::Actor { -public: - ValidatorTelemetry(PublicKeyHash key, adnl::AdnlNodeIdShort local_id, td::Bits256 zero_state_file_hash, - td::actor::ActorId manager) - : key_(key) - , local_id_(local_id) - , zero_state_file_hash_(zero_state_file_hash) - , manager_(std::move(manager)) { + public: + class Callback { + public: + virtual ~Callback() = default; + virtual void send_telemetry(tl_object_ptr telemetry) = 0; + }; + + ValidatorTelemetry(adnl::AdnlNodeIdShort local_id, std::unique_ptr callback) + : local_id_(local_id), callback_(std::move(callback)) { } void start_up() override; void alarm() override; -private: - PublicKeyHash key_; + private: adnl::AdnlNodeIdShort local_id_; - td::Bits256 zero_state_file_hash_; - td::actor::ActorId manager_; + std::unique_ptr callback_; std::string node_version_; std::string os_version_; @@ -61,6 +61,5 @@ class ValidatorTelemetry : public td::actor::Actor { void send_telemetry(); static constexpr double PERIOD = 600.0; - static constexpr td::uint32 MAX_SIZE = 8192; }; -} // namespace ton::validator \ No newline at end of file +} // namespace ton::validator diff --git a/validator/validator.h b/validator/validator.h index e73614950..826fc8f8f 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -242,7 +242,6 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Promise>> promise) = 0; virtual void new_key_block(BlockHandle handle) = 0; - virtual void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) = 0; }; virtual ~ValidatorManagerInterface() = default; From a7c001036139326a83fe7dbcc346f196c6430024 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 15 Sep 2025 16:20:00 +0300 Subject: [PATCH 384/388] Ban collator node when it returns invalid block (#1797) --- tl/generate/scheme/ton_api.tl | 2 +- tl/generate/scheme/ton_api.tlo | Bin 118952 -> 118988 bytes .../validator-engine-console-query.cpp | 3 + validator/collation-manager.cpp | 104 +++++++++++------- validator/collation-manager.hpp | 6 + validator/validator-group.cpp | 22 +++- validator/validator-group.hpp | 1 + 7 files changed, 95 insertions(+), 43 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 6e1a0fc4f..d737dea5a 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -786,7 +786,7 @@ engine.validator.shardOutQueueSize size:long = engine.validator.ShardOutQueueSiz engine.validator.exportedPrivateKeys encrypted_data:bytes = engine.validator.ExportedPrivateKeys; engine.validator.collationManagerStats.shard shard_id:tonNode.shardId self_collate:Bool select_mode:string active:Bool collators:(vector int256) = engine.validator.collationManagerStats.Shard; -engine.validator.collationManagerStats.collator adnl_id:int256 active:Bool alive:Bool ping_in:double last_ping_ago:double last_ping_status:string = engine.validator.collationManagerStats.Collator; +engine.validator.collationManagerStats.collator adnl_id:int256 active:Bool alive:Bool ping_in:double last_ping_ago:double last_ping_status:string banned_for:double = engine.validator.collationManagerStats.Collator; engine.validator.collationManagerStats.localId adnl_id:int256 shards:(vector engine.validator.collationManagerStats.shard) collators:(vector engine.validator.collationManagerStats.collator) = engine.validator.collationManagerStats.LocalId; engine.validator.collationManagerStats local_ids:(vector engine.validator.collationManagerStats.localId) = engine.validator.CollationManagerStats; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index ee5fb5ce817f1079c69a32c1625e810f0248b3a2..bf531b7af9deb192d7628c25692399ab8c7691e8 100644 GIT binary patch delta 92 zcmZ3nkp0X;_6;rytj=m&`TCO|DyVNRQ24R|!JRzeA_pf(^5!uiANI)$FJ7PAaFB(E jD=9HAFEu4TEx(9ivcfSL7Lbz3Etk%K8Jj~chx`QqORytm delta 76 zcmX@JkbT8M_6;rytd5*Y;`);xDyVNRQ24R|!JRzeA_pf(^5!uiAGXO0FJ7O%fsK)a S1tc_i#icV~#^#F4A%6h~5*wEQ diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 15f126422..1a9ca052e 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1856,6 +1856,9 @@ td::Status GetCollationManagerStatsQuery::receive(td::BufferSlice data) { } sb << td::StringBuilder::FixedDouble(collator->last_ping_ago_, 3) << ": " << status; } + if (collator->banned_for_ > 0.0) { + sb << " banned_for=" << td::StringBuilder::FixedDouble(std::max(collator->banned_for_, 0.0), 3); + } td::TerminalIO::out() << sb.as_cslice() << "\n"; } } diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index e842b76d4..7d694af98 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -137,53 +137,61 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas adnl::AdnlNodeIdShort selected_collator = adnl::AdnlNodeIdShort::zero(); size_t selected_idx = 0; - auto check_collator = [&](const adnl::AdnlNodeIdShort& id) -> bool { - auto& collator = collators_[id]; - if (!collator.alive) { - return false; - } - if (is_optimistic && collator.version < CollatorNode::VERSION_OPTIMISTIC_COLLATE) { - return false; - } - return true; - }; - switch (s->select_mode) { - case CollatorsList::mode_random: { - int cnt = 0; - for (size_t i = 0; i < s->collators.size(); ++i) { - adnl::AdnlNodeIdShort collator = s->collators[i]; - if (check_collator(collator)) { - ++cnt; - if (td::Random::fast(1, cnt) == 1) { + for (int allow_banned = 0; allow_banned < 2; ++allow_banned) { + auto check_collator = [&](const adnl::AdnlNodeIdShort& id) -> bool { + auto& collator = collators_[id]; + if (!collator.alive) { + return false; + } + if (collator.banned_until && !allow_banned) { + return false; + } + if (is_optimistic && collator.version < CollatorNode::VERSION_OPTIMISTIC_COLLATE) { + return false; + } + return true; + }; + switch (s->select_mode) { + case CollatorsList::mode_random: { + int cnt = 0; + for (size_t i = 0; i < s->collators.size(); ++i) { + adnl::AdnlNodeIdShort collator = s->collators[i]; + if (check_collator(collator)) { + ++cnt; + if (td::Random::fast(1, cnt) == 1) { + selected_collator = collator; + selected_idx = i; + } + } + } + break; + } + case CollatorsList::mode_ordered: { + for (size_t i = 0; i < s->collators.size(); ++i) { + adnl::AdnlNodeIdShort collator = s->collators[i]; + if (check_collator(collator)) { selected_collator = collator; selected_idx = i; + break; } } + break; } - break; - } - case CollatorsList::mode_ordered: { - for (size_t i = 0; i < s->collators.size(); ++i) { - adnl::AdnlNodeIdShort collator = s->collators[i]; - if (check_collator(collator)) { - selected_collator = collator; - selected_idx = i; - break; + case CollatorsList::mode_round_robin: { + size_t iters = 0; + for (size_t i = s->cur_idx; iters < s->collators.size(); (++i) %= s->collators.size(), ++iters) { + adnl::AdnlNodeIdShort& collator = s->collators[i]; + if (check_collator(collator)) { + selected_collator = collator; + selected_idx = i; + s->cur_idx = (i + 1) % s->collators.size(); + break; + } } + break; } - break; } - case CollatorsList::mode_round_robin: { - size_t iters = 0; - for (size_t i = s->cur_idx; iters < s->collators.size(); (++i) %= s->collators.size(), ++iters) { - adnl::AdnlNodeIdShort& collator = s->collators[i]; - if (check_collator(collator)) { - selected_collator = collator; - selected_idx = i; - s->cur_idx = (i + 1) % s->collators.size(); - break; - } - } + if (!selected_collator.is_zero() || s->self_collate) { break; } } @@ -372,11 +380,21 @@ void CollationManager::get_stats( } obj->last_ping_ago_ = collator.last_ping_at ? td::Time::now() - collator.last_ping_at.at() : -1.0; obj->last_ping_status_ = collator.last_ping_status.is_ok() ? "OK" : collator.last_ping_status.message().str(); + obj->banned_for_ = collator.banned_until ? collator.banned_until.in() : -1.0; stats->collators_.push_back(std::move(obj)); } promise.set_value(std::move(stats)); } +void CollationManager::ban_collator(adnl::AdnlNodeIdShort collator_id, std::string reason) { + auto it = collators_.find(collator_id); + if (it == collators_.end()) { + return; + } + alarm_timestamp().relax(it->second.banned_until = td::Timestamp::in(BAN_DURATION)); + LOG(ERROR) << "Ban collator " << collator_id << " for " << BAN_DURATION << "s: " << reason; +} + void CollationManager::update_collators_list(const CollatorsList& collators_list) { shards_.clear(); for (auto& [_, collator] : collators_) { @@ -427,6 +445,14 @@ CollationManager::ShardInfo* CollationManager::select_shard_info(ShardIdFull sha void CollationManager::alarm() { alarm_timestamp() = td::Timestamp::never(); for (auto& [id, collator] : collators_) { + if (collator.banned_until) { + if (collator.banned_until.is_in_past()) { + collator.banned_until = {}; + LOG(ERROR) << "Unban collator " << id; + } else { + alarm_timestamp().relax(collator.banned_until); + } + } if (collator.active_cnt == 0 || collator.sent_ping) { continue; } diff --git a/validator/collation-manager.hpp b/validator/collation-manager.hpp index 37a45fb38..6691a387e 100644 --- a/validator/collation-manager.hpp +++ b/validator/collation-manager.hpp @@ -54,6 +54,8 @@ class CollationManager : public td::actor::Actor { void get_stats(td::Promise> promise); + void ban_collator(adnl::AdnlNodeIdShort collator_id, std::string reason); + private: adnl::AdnlNodeIdShort local_id_; td::Ref opts_; @@ -77,6 +79,8 @@ class CollationManager : public td::actor::Actor { td::Timestamp last_ping_at = td::Timestamp::never(); td::Status last_ping_status = td::Status::Error("not pinged"); int version = -1; + // Collator is banned when in returns invalid block + td::Timestamp banned_until = td::Timestamp::never(); }; std::map collators_; @@ -104,6 +108,8 @@ class CollationManager : public td::actor::Actor { size_t refcnt = 0; }; std::map optimistic_prev_cache_; + + static constexpr double BAN_DURATION = 300.0; }; } // namespace ton::validator diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 5261560e8..c79dbda4a 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -117,6 +117,9 @@ void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo if (need_send_candidate_broadcast(source_info, shard_.is_masterchain())) { send_block_candidate_broadcast(c.candidate.id, c.candidate.data.clone()); } + if (!c.self_collated) { + block_collator_node_id_[c.candidate.id] = adnl::AdnlNodeIdShort{c.collator_node_id}; + } cache->result = std::move(c); for (auto &p : cache->promises) { p.set_value(cache->result.value().clone()); @@ -182,10 +185,15 @@ void ValidatorGroup::validate_block_candidate(validatorsession::BlockSourceInfo return; } + auto it2 = block_collator_node_id_.find(block.id); + adnl::AdnlNodeIdShort collator_node_id = + it2 == block_collator_node_id_.end() ? adnl::AdnlNodeIdShort::zero() : it2->second; + auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), source_info, block = block.clone(), + [=, SelfId = actor_id(this), block = block.clone(), optimistic_prev_block = is_optimistic ? optimistic_prev_block.value().clone() : td::optional{}, - promise = std::move(promise)](td::Result R) mutable { + promise = std::move(promise), + collation_manager = collation_manager_](td::Result R) mutable { if (R.is_error()) { auto S = R.move_as_error(); if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) { @@ -212,6 +220,10 @@ void ValidatorGroup::validate_block_candidate(validatorsession::BlockSourceInfo promise.set_value({ts, false}); }, [&](CandidateReject reject) { + if (!collator_node_id.is_zero()) { + td::actor::send_closure(collation_manager, &CollationManager::ban_collator, collator_node_id, + PSTRING() << "bad candidate " << block.id.to_str() << " : " << reject.reason); + } promise.set_error( td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad candidate: " << reject.reason)); })); @@ -396,7 +408,11 @@ void ValidatorGroup::generated_block_optimistic(validatorsession::BlockSourceInf optimistic_generation_ = {}; return; } - optimistic_generation_->result = R.move_as_ok(); + GeneratedCandidate c = R.move_as_ok(); + if (!c.self_collated) { + block_collator_node_id_[c.candidate.id] = adnl::AdnlNodeIdShort{c.collator_node_id}; + } + optimistic_generation_->result = std::move(c); for (auto &promise : optimistic_generation_->promises) { promise.set_result(optimistic_generation_->result.value().clone()); } diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 56d89dda4..3df456efa 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -188,6 +188,7 @@ class ValidatorGroup : public td::actor::Actor { } std::set sent_candidate_broadcasts_; + std::map block_collator_node_id_; void send_block_candidate_broadcast(BlockIdExt id, td::BufferSlice data); From 41849db9133dff529e6316015e9512035c05e0c8 Mon Sep 17 00:00:00 2001 From: mkiesel Date: Mon, 15 Sep 2025 20:46:12 +0200 Subject: [PATCH 385/388] avoid touching packfiles on startup (#1793) ArchiveFile::start_up() truncates packfiles unconditionally, causing unneeded filesystem writes (mtime) and complicating auditing on file level. Co-authored-by: mkiesel --- validator/db/package.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/validator/db/package.cpp b/validator/db/package.cpp index cc699baa8..be9616422 100644 --- a/validator/db/package.cpp +++ b/validator/db/package.cpp @@ -48,8 +48,16 @@ Package::Package(td::FileFd fd) : fd_(std::move(fd)) { } td::Status Package::truncate(td::uint64 size) { - TRY_STATUS(fd_.seek(size + header_size())); - return fd_.truncate_to_current_position(size + header_size()); + auto target_size = size + header_size(); + TRY_RESULT(current_size, fd_.get_size()); + + // Only truncate if the size actually differs to avoid updating mtime unnecessarily + if (current_size != target_size) { + TRY_STATUS(fd_.seek(target_size)); + return fd_.truncate_to_current_position(target_size); + } + + return td::Status::OK(); } td::uint64 Package::append(std::string filename, td::Slice data, bool sync) { From b8c878074521f0da8f65c4e2a815e593163c7a69 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 16 Sep 2025 16:00:50 +0300 Subject: [PATCH 386/388] Fix compilation error (#1801) --- crypto/vm/boc-compression.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/crypto/vm/boc-compression.cpp b/crypto/vm/boc-compression.cpp index db04f162f..d37e0380a 100644 --- a/crypto/vm/boc-compression.cpp +++ b/crypto/vm/boc-compression.cpp @@ -19,6 +19,7 @@ #include "boc-compression.h" #include +#include #include "vm/boc.h" #include "vm/boc-writers.h" #include "vm/cells.h" From 743deba3f465cc90b970ea9f0413f2762c6aec26 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Tue, 16 Sep 2025 17:01:20 +0400 Subject: [PATCH 387/388] Fix failing Github MacOS actions due to updated XCode (#1800) * add libgslcblas into portable binaries * remove commented line * update target OSX for macos 15 * ensure system headers come from the right macOS SDK. --- .github/workflows/build-ton-macos-15-arm64-shared.yml | 2 +- assembly/native/build-macos-portable.sh | 2 ++ assembly/native/build-macos-shared.sh | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-ton-macos-15-arm64-shared.yml b/.github/workflows/build-ton-macos-15-arm64-shared.yml index 1423b4eb0..a1e0782b7 100644 --- a/.github/workflows/build-ton-macos-15-arm64-shared.yml +++ b/.github/workflows/build-ton-macos-15-arm64-shared.yml @@ -36,7 +36,7 @@ jobs: git submodule update cp assembly/native/build-macos-shared.sh . chmod +x build-macos-shared.sh - ./build-macos-shared.sh -t -c -o 15.0 + ./build-macos-shared.sh -t -c -o 16.0 ccache -sp - name: Run Tests diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index feb5bf72f..302effb71 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -127,6 +127,8 @@ fi cmake -GNinja .. \ -DPORTABLE=1 \ -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=$OSX_TARGET \ +-DCMAKE_CXX_FLAGS="-stdlib=libc++" \ +-DCMAKE_SYSROOT=$(xcrun --show-sdk-path) \ -DCMAKE_BUILD_TYPE=Release \ -DOPENSSL_FOUND=1 \ -DOPENSSL_INCLUDE_DIR=$opensslPath/include \ diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh index f5aa15b4a..c2aa09ced 100644 --- a/assembly/native/build-macos-shared.sh +++ b/assembly/native/build-macos-shared.sh @@ -79,6 +79,8 @@ brew install openssl@3 brew unlink openssl@3 && brew link --overwrite openssl@3 cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \ +-DCMAKE_CXX_FLAGS="-stdlib=libc++" \ +-DCMAKE_SYSROOT=$(xcrun --show-sdk-path) \ -DLZ4_FOUND=1 \ -DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a \ -DLZ4_INCLUDE_DIRS=$lz4Path/lib From bb0af68fb61eed2df855df289cd5137e542c56f3 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Thu, 25 Sep 2025 12:44:49 +0300 Subject: [PATCH 388/388] Set default global version to SUPPORTED_VERSION for getmethods in emulator (#1814) Co-authored-by: SpyCheese --- crypto/smc-envelope/SmartContract.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp index a89cfa6d9..c067b88ff 100644 --- a/crypto/smc-envelope/SmartContract.cpp +++ b/crypto/smc-envelope/SmartContract.cpp @@ -162,7 +162,9 @@ td::Ref prepare_vm_c7(SmartContract::Args args, td::Ref cod vm::load_cell_slice_ref(address), // myself:MsgAddressInt vm::StackEntry::maybe(config) // vm::StackEntry::maybe(td::Ref()) }; - if (args.config && args.config.value()->get_global_version() >= 4) { + + int global_version = args.config ? args.config.value()->get_global_version() : SUPPORTED_VERSION; + if (global_version >= 4) { tuple.push_back(vm::StackEntry::maybe(code)); // code:Cell tuple.push_back(block::CurrencyCollection::zero().as_vm_tuple()); // in_msg_value:[Integer (Maybe Cell)] tuple.push_back(td::zero_refint()); // storage_fees:Integer @@ -173,17 +175,18 @@ td::Ref prepare_vm_c7(SmartContract::Args args, td::Ref cod // prev_key_block:BlockId ] : PrevBlocksInfo tuple.push_back(args.prev_blocks_info ? args.prev_blocks_info.value() : vm::StackEntry{}); // prev_block_info } - if (args.config && args.config.value()->get_global_version() >= 6) { - tuple.push_back(args.config.value()->get_unpacked_config_tuple(now)); // unpacked_config_tuple + if (global_version >= 6) { + tuple.push_back(args.config ? args.config.value()->get_unpacked_config_tuple(now) + : vm::StackEntry{}); // unpacked_config_tuple tuple.push_back(td::zero_refint()); // due_payment // precompiled_gas_usage:(Maybe Integer) td::optional precompiled; - if (code.not_null()) { + if (code.not_null() && args.config) { precompiled = args.config.value()->get_precompiled_contracts_config().get_contract(code->get_hash().bits()); } tuple.push_back(precompiled ? td::make_refint(precompiled.value().gas_usage) : vm::StackEntry()); } - if (args.config && args.config.value()->get_global_version() >= 11) { + if (global_version >= 11) { tuple.push_back(block::transaction::Transaction::prepare_in_msg_params_tuple(nullptr, {}, {})); } auto tuple_ref = td::make_cnt_ref>(std::move(tuple)); @@ -263,7 +266,7 @@ SmartContract::Answer run_smartcont(SmartContract::State state, td::Refdump(os, 2); LOG(DEBUG) << "VM stack:\n" << os.str(); } - int global_version = config ? config->get_global_version() : 0; + int global_version = config ? config->get_global_version() : SUPPORTED_VERSION; vm::VmState vm{state.code, global_version, std::move(stack), gas, 1, state.data, log}; vm.set_c7(std::move(c7)); vm.set_chksig_always_succeed(ignore_chksig);

(point: P) { return point.x + point.y; } + +@method_id(118) +fun test18() { + var p = `Point` { x: 8, `y`: 9 } as `PointAlias`?; + if (p != null) { + return sumXY(p); + } + return sumXY(p = getStorage1(80).lastPoint); +} + +global gPoint: Point; +global gPointN: Point?; + +@method_id(119) +fun test19() { + gPoint = { x: 1, y: 2 }; + assignCoords(mutate gPoint); + gPointN = getStorage1(9).lastPoint; + return (gPoint, gPointN, gPointN!.x); +} + +@method_id(120) +fun test20(setNull: bool) { + gPoint = generatePoint(8, 9); + gPointN = setNull ? null : gPoint; + if (!setNull) { + gPoint.x = gPointN!.y = 80; + } else { + (gPoint as Point?)!.y += 5; + } + return (gPoint, gPointN); +} + +global gJustInt: JustInt; +global gJustIntN: JustInt?; + +@method_id(121) +fun test21(setNull: bool) { + var value = 8; + gJustInt = { value }; + gJustIntN = setNull ? null : { value: value += 9 }; + (gJustInt.value, value) = (value, gJustInt.value); + return (gJustInt, gJustIntN, value, value * (gJustIntN == null ? 1 : gJustIntN!.value)); +} + +fun asm_func(x1: JustInt, x2: int, x3: JustIntAlias, x4: JustIntAlias, x5: int, x6: JustInt, x7: int): + (int, JustInt, int, JustInt, int, int, JustIntAlias2) + asm (x4 x5 x6 x7 x1 x2 x3->0 1 2 3 4 5 6) "NOP"; + +fun someOp(mutate self: JustInt, y: int): int { + val (newX, newY) = (self.value + y, y * 10); + self = { value: newX }; + return newY; +} + +@method_id(122) +fun test22(x: JustInt) { + return asm_func(x, x.value += 1, x, x, x.someOp(x.value / 20), x, x.value = x.value * 2); +} + +@method_id(123) +fun test23(p: Point) { + var complex = p as Point | (JustInt | Storage); + return match (complex) { + JustInt => 10 * complex.value, + PointAlias => 20, + Storage => 30 + complex.lastPoint.x, + }; +} + +@method_id(124) +fun test24(x: JustInt | JustIntAlias2 | JustIntAlias): JustInt { + return match (x) { + JustIntAlias => x + }; +} + +fun getPointOrStorage(getPoint: bool): Point | Storage { + return getPoint + ? Point { x: 10, y: 20 } + : Storage { owner: { id: 10, name: "" }, lastPoint: { x: 30, y: 40 } }; +} + +@method_id(125) +fun test25() { + var r1 = getPointOrStorage(true); + if (r1 !is Storage && r1 !is builder && r1 != null) { + match (val d = getPointOrStorage(false)) { + Point => { + r1.x += d.x; + r1.y += d.y; + } + Storage => { + r1.x += d.lastPoint.x; + r1.y += d.lastPoint.y; + } + } + } + return r1; +} + +fun acceptSomeUnion(r: Point | int | builder) { + return (r is Point) ? r.x + r.y : (r is int) ? r : null; +} + +@method_id(126) +fun test26() { + __expect_type(acceptSomeUnion, "(Point | int | builder) -> int?"); + return acceptSomeUnion({ x: 8, y: 10 })! + acceptSomeUnion(100)!; +} + +struct Has2EmptyNullable { + f1: Empty?; + f2: Empty? +} + +fun get2EmptyNullable(null1: bool, null2: bool): Has2EmptyNullable { + return { f1: null1 ? null : {}, f2: null2 ? null : {} }; +} + +@method_id(127) +fun test27() { + var (t1, t2) = (get2EmptyNullable(true, true), get2EmptyNullable(false, false)); + var (t3, t4) = (t1 as Has2EmptyNullable?, null as Has2EmptyNullable?); + return (t1, t2, 777, t1.f1 == null, t1.f2 == null, t2.f1 == null, t2.f2 == null, 777, t3, 777, t3!, 777, t4); +} + +fun getEmptyOrIntInt(getEmpty: bool): Empty | (int, int) { + return getEmpty ? {} : (1, 2); +} + +@method_id(128) +fun test28(getEmpty: bool) { + var t1: Empty | (int, int) = getEmptyOrIntInt(getEmpty); + var t2 = (getEmpty ? Empty{} : Has2EmptyNullable{ f1: null, f2: {} }) as EmptyAlias | Has2EmptyNullable | null; + return (t1, t1 is Empty, t1 is (int, int), t1 is builder, 777, t2, t2 is Empty, t2 is Has2EmptyNullable, t2 == null); +} + +struct Empty1 {} +struct Empty2 {} + +@method_id(129) +fun test29() { + var c = (null as Empty1 | Empty2?); + if (c == null) { + return c; + } + return null; +} + +@method_id(130) +fun test30(is1: bool) { + var e: Empty1 | Empty2 = is1 ? Empty1{} : Empty2 {}; + var n: Empty1 | int | Empty2 = 4; + var c: Empty1 | Empty2? = null; + if (is1) { + n = e; + } else { + c = e; + } + __expect_type(test29(), "null"); + return (e, 777, n, 777, c, 777, (test29() as Empty1? | Empty2), (test29() as Empty1 | Empty2 | null)!); +} + +struct IntOrNull { + e: Empty; + x: int?; +} + +@method_id(131) +fun test31() { + var s1 = { x: 5, e: {} } as IntOrNull?; + var s2 = { e: {}, x: null } as IntOrNull?; + var s3 = null as IntOrNull?; + return (s1, s2, s3, 777, s1 == null, s2 == null, s3 == null, 777, s1!.x == null, s2!.x == null); +} + + + + + +fun main(x: int8, y: MInt) { + __expect_type(PointAlias{x,y}, "Point"); + __expect_type(Point{x,y} as Point, "Point"); + __expect_type(test3(), "Point"); + __expect_type(maxCoord, "(Point) -> int8"); +} + +type PointAlias = Point; + +/** +@testcase | 101 | | 1 2 3 [ 1 5 ] +@testcase | 102 | | 1 2 3 [ 1 5 ] +@testcase | 103 | | 5 6 +@testcase | 104 | | 0 10 20 30 40 -1 +@testcase | 105 | | 35 30 45 +@testcase | 106 | | 10 20 +@testcase | 107 | | 5 5 10 20 15 +@testcase | 108 | | 777 131 777 0 777 777 777 131 777 132 777 777 +@testcase | 109 | | 70 30 20 20 -80 +@testcase | 110 | 0 | (null) (null) 0 +@testcase | 110 | -1 | 10 20 133 +@testcase | 111 | 0 | 0 2 10 20 30 +@testcase | 111 | -1 | 0 0 3 4 7 +@testcase | 112 | 0 | (null) (null) 0 +@testcase | 112 | -1 | 1 2 133 +@testcase | 113 | | (null) (null) 0 +@testcase | 114 | | 1 2 +@testcase | 115 | | (null) (null) (null) (null) 0 +@testcase | 116 | | -1 -4 [ 8 4 ] 7 (null) 0 [ 10 11 ] 135 +@testcase | 117 | 5 | 5 5 5 5 5 +@testcase | 117 | null | (null) (null) (null) -1 (null) +@testcase | 118 | | 17 +@testcase | 119 | | 10 20 9 10 133 9 +@testcase | 120 | 0 | 80 9 8 80 133 +@testcase | 120 | -1 | 8 14 (null) (null) 0 +@testcase | 121 | 0 | 17 17 8 136 +@testcase | 121 | -1 | 8 (null) 8 8 +@testcase | 122 | 100 | 101 50 106 212 100 101 101 +@testcase | 124 | 66 | 66 +@testcase | 125 | | (null) (null) 40 60 133 +@testcase | 126 | | 118 +@testcase | 127 | | 0 0 131 131 777 -1 -1 0 0 777 0 0 138 777 0 0 777 (null) (null) 0 +@testcase | 128 | -1 | (null) (null) 131 -1 0 0 777 (null) (null) 131 -1 0 0 +@testcase | 128 | 0 | 1 2 129 0 -1 0 777 0 131 138 0 -1 0 +@testcase | 129 | | (null) +@testcase | 130 | -1 | 140 777 (null) 140 777 0 777 0 0 +@testcase | 130 | 0 | 139 777 4 1 777 139 777 0 0 +@testcase | 131 | | 5 141 (null) 141 (null) 0 777 0 0 -1 777 0 -1 + +@fif_codegen +""" + test6 PROC:<{ + // + 10 PUSHINT // p.x + 20 PUSHINT // p.x p.y + }> +""" + +@fif_codegen +""" + sumXY PROC:<{ + // point.x point.y + ADD // '2 + }> +""" +@fif_codegen_avoid sumXY + +@fif_codegen +""" + test23 PROC:<{ + // p.x p.y + 2DROP + 20 PUSHINT // '10=20 + }> +""" + +@fif_codegen +""" + test29 PROC:<{ + // + PUSHNULL // '5 + }> +""" + + */ diff --git a/tolk-tester/tests/type-aliases-tests.tolk b/tolk-tester/tests/type-aliases-tests.tolk index aa67bcad7..68935733c 100644 --- a/tolk-tester/tests/type-aliases-tests.tolk +++ b/tolk-tester/tests/type-aliases-tests.tolk @@ -7,6 +7,9 @@ type Pair2_v2 = (MInt, MInt); type MBool = bool; type Tuple2Int = [int, int]; +struct Point { x: int; y: int } +type PointAlias = Point; +type PointAlias2 = PointAlias; fun test1(x: MInt): MVoid { var y = x; @@ -36,6 +39,10 @@ fun test1(x: MInt): MVoid { __expect_type(x as MInt?, "MInt?"); __expect_type(x as MIntN, "MIntN"); + __expect_type(PointAlias{x:0,y:0}, "Point"); + __expect_type(PointAlias2{x:0,y:0}, "Point"); + __expect_type(Point{x:0,y:0} as PointAlias, "PointAlias"); + if (x) { return; } } diff --git a/tolk-tester/tests/var-apply-tests.tolk b/tolk-tester/tests/var-apply-tests.tolk index 3411f4a06..e4318dc15 100644 --- a/tolk-tester/tests/var-apply-tests.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -222,6 +222,26 @@ fun testCallbackUnion(call1st: bool): (int, int) | int { return cb(10) as int } +struct WithCallbacks { + f1: (int -> int)?, + beginCell: builder -> cell, +} + +fun doEndCellActually(w: WithCallbacks, b: builder) { return w.beginCell(b); } + +@method_id(115) +fun testCallableFields() { + var w: WithCallbacks = { f1: null, beginCell: endCell }; + if (10 > 3) { + w.f1 = justAdd2; + } + return ( + w.f1!(5), + w.doEndCellActually(beginCell().storeInt(6, 32)).beginParse().loadInt(32), + beginParse(w.beginCell(beginCell().storeInt(7, 32))).loadInt(32) + ); +} + fun main() {} /** @@ -246,4 +266,5 @@ fun main() {} @testcase | 113 | -1 | 12 -1 @testcase | 114 | -1 | 2 3 130 @testcase | 114 | 0 | (null) 12 1 +@testcase | 115 | | 7 6 7 */ diff --git a/tolk/abscode.cpp b/tolk/abscode.cpp index 90c7fbcf1..984d6ffc8 100644 --- a/tolk/abscode.cpp +++ b/tolk/abscode.cpp @@ -401,7 +401,14 @@ std::vector CodeBlob::create_var(TypePtr var_type, SrcLocation loc, s std::vector ir_idx; int stack_w = var_type->get_width_on_stack(); ir_idx.reserve(stack_w); - if (const TypeDataTensor* t_tensor = var_type->try_as()) { + if (const TypeDataStruct* t_struct = var_type->try_as()) { + for (int i = 0; i < t_struct->struct_ref->get_num_fields(); ++i) { + StructFieldPtr field_ref = t_struct->struct_ref->get_field(i); + std::string sub_name = name.empty() || t_struct->struct_ref->get_num_fields() == 1 ? name : name + "." + field_ref->name; + std::vector nested = create_var(field_ref->declared_type, loc, std::move(sub_name)); + ir_idx.insert(ir_idx.end(), nested.begin(), nested.end()); + } + } else if (const TypeDataTensor* t_tensor = var_type->try_as()) { for (int i = 0; i < t_tensor->size(); ++i) { std::string sub_name = name.empty() ? name : name + "." + std::to_string(i); std::vector nested = create_var(t_tensor->items[i], loc, std::move(sub_name)); diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index ec9d689d4..6fc23457b 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -385,6 +385,40 @@ static V parse_maybe_instantiationTs_after_identifier(L } } +static V parse_object_field(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.check(tok_identifier, "field name"); + auto v_ident = createV(lex.cur_location(), lex.cur_str()); + lex.next(); + + if (lex.tok() == tok_comma || lex.tok() == tok_clbrace) { + auto v_same_ident = createV(v_ident->loc, v_ident->name); + auto v_same_expr = createV(v_ident->loc, v_same_ident, nullptr); + return createV(loc, v_ident, v_same_expr); + } + + lex.expect(tok_colon, "`:`"); + return createV(loc, v_ident, parse_expr(lex)); +} + +static V parse_object_body(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_opbrace, "`{`"); + + std::vector fields; + while (lex.tok() != tok_clbrace) { + fields.push_back(parse_object_field(lex)); + if (lex.tok() == tok_comma) { + lex.next(); + } else if (lex.tok() != tok_clbrace) { + lex.unexpected("`,`"); + } + } + lex.expect(tok_clbrace, "`}`"); + + return createV(loc, std::move(fields)); +} + // `throw code` / `throw (code)` / `throw (code, arg)` // it's technically a statement (can't occur "in any place of expression"), // but inside `match` arm it can appear without semicolon: `pattern => throw 123` @@ -587,8 +621,14 @@ static AnyExprV parse_expr100(Lexer& lex) { if (lex.tok() == tok_lt) { v_instantiationTs = parse_maybe_instantiationTs_after_identifier(lex); } + if (lex.tok() == tok_opbrace) { + auto explicit_ref = createV(loc, v_ident, v_instantiationTs); + return createV(loc, explicit_ref, parse_object_body(lex)); + } return createV(loc, v_ident, v_instantiationTs); } + case tok_opbrace: + return createV(loc, createV(loc), parse_object_body(lex)); case tok_match: return parse_match_expression(lex); default: @@ -1227,6 +1267,52 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector(loc, v_ident, v_param_list, v_body, ret_type, genericsT_list, std::move(method_id), flags); } +static AnyV parse_struct_field(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.check(tok_identifier, "field name"); + auto v_ident = createV(lex.cur_location(), lex.cur_str()); + lex.next(); + lex.expect(tok_colon, "`: `"); + TypePtr declared_type = parse_type_from_tokens(lex); + + if (lex.tok() == tok_assign) { // `id: int = 3` + lex.error("default values for fields not supported yet"); + } + + return createV(loc, v_ident, declared_type); +} + +static V parse_struct_body(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_opbrace, "`{`"); + + std::vector fields; + while (lex.tok() != tok_clbrace) { + fields.push_back(parse_struct_field(lex)); + if (lex.tok() == tok_comma || lex.tok() == tok_semicolon) { + lex.next(); + } else if (lex.tok() != tok_clbrace) { + lex.unexpected("`;` or `,`"); + } + } + lex.expect(tok_clbrace, "`}`"); + + return createV(loc, std::move(fields)); +} + +static AnyV parse_struct_declaration(Lexer& lex, const std::vector>& annotations) { + SrcLocation loc = lex.cur_location(); + if (!annotations.empty()) { + lex.error("@annotations are not applicable to type alias declaration"); + } + lex.expect(tok_struct, "`struct`"); + lex.check(tok_identifier, "identifier"); + auto v_ident = createV(lex.cur_location(), lex.cur_str()); + lex.next(); + + return createV(loc, v_ident, parse_struct_body(lex)); +} + static AnyV parse_tolk_required_version(Lexer& lex) { SrcLocation loc = lex.cur_location(); lex.next_special(tok_semver, "semver"); // syntax: "tolk 0.6" @@ -1301,9 +1387,12 @@ AnyV parse_src_file_to_ast(const SrcFile* file) { toplevel_declarations.push_back(parse_function_declaration(lex, annotations)); annotations.clear(); break; + case tok_struct: + toplevel_declarations.push_back(parse_struct_declaration(lex, annotations)); + annotations.clear(); + break; case tok_export: - case tok_struct: case tok_enum: case tok_operator: case tok_infix: diff --git a/tolk/ast-replacer.h b/tolk/ast-replacer.h index 417ff411c..7108b95b0 100644 --- a/tolk/ast-replacer.h +++ b/tolk/ast-replacer.h @@ -119,6 +119,9 @@ class ASTReplacerInFunctionBody : public ASTReplacer { virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } // statements virtual AnyV replace(V v) { return replace_children(v); } virtual AnyV replace(V v) { return replace_children(v); } @@ -160,6 +163,9 @@ class ASTReplacerInFunctionBody : public ASTReplacer { case ast_not_null_operator: return replace(v->as()); case ast_match_expression: return replace(v->as()); case ast_match_arm: return replace(v->as()); + case ast_object_field: return replace(v->as()); + case ast_object_body: return replace(v->as()); + case ast_object_literal: return replace(v->as()); default: throw UnexpectedASTNodeType(v, "ASTReplacerInFunctionBody::replace"); } diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index 66781f1f0..1217fa45b 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -136,6 +136,15 @@ class ASTReplicatorFunction : public ASTReplicator { virtual V clone(V v) { return createV(v->loc, v->pattern_kind, clone(v->exact_type), clone(v->get_pattern_expr()), clone(v->get_body())); } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_field_identifier()), clone(v->get_init_val())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_all_fields())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_maybe_reference()), clone(v->get_body())); + } // statements @@ -220,6 +229,9 @@ class ASTReplicatorFunction : public ASTReplicator { case ast_not_null_operator: return clone(v->as()); case ast_match_expression: return clone(v->as()); case ast_match_arm: return clone(v->as()); + case ast_object_field: return clone(v->as()); + case ast_object_body: return clone(v->as()); + case ast_object_literal: return clone(v->as()); default: throw UnexpectedASTNodeType(v, "ASTReplicatorFunction::clone"); } diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index a232888f2..7782c506d 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -61,6 +61,9 @@ class ASTStringifier final : public ASTVisitor { {ast_not_null_operator, "ast_not_null_operator"}, {ast_match_expression, "ast_match_expression"}, {ast_match_arm, "ast_match_arm"}, + {ast_object_field, "ast_object_field"}, + {ast_object_body, "ast_object_body"}, + {ast_object_literal, "ast_object_literal"}, // statements {ast_empty_statement, "ast_empty_statement"}, {ast_block_statement, "ast_block_statement"}, @@ -85,6 +88,9 @@ class ASTStringifier final : public ASTVisitor { {ast_global_var_declaration, "ast_global_var_declaration"}, {ast_constant_declaration, "ast_constant_declaration"}, {ast_type_alias_declaration, "ast_type_alias_declaration"}, + {ast_struct_field, "ast_struct_field"}, + {ast_struct_body, "ast_struct_body"}, + {ast_struct_declaration, "ast_struct_declaration"}, {ast_tolk_required_version, "ast_tolk_required_version"}, {ast_import_directive, "ast_import_directive"}, {ast_tolk_file, "ast_tolk_file"}, @@ -163,6 +169,10 @@ class ASTStringifier final : public ASTVisitor { return static_cast(v->as()->get_identifier()->name); case ast_type_alias_declaration: return "type " + static_cast(v->as()->get_identifier()->name); + case ast_struct_field: + return static_cast(v->as()->get_identifier()->name) + ": " + v->as()->declared_type->as_human_readable(); + case ast_struct_declaration: + return "struct " + static_cast(v->as()->get_identifier()->name); case ast_assign: return "="; case ast_set_assign: @@ -224,6 +234,10 @@ class ASTStringifier final : public ASTVisitor { return "(expression)"; } return "(else)"; + case ast_object_field: + return static_cast(v->as()->get_field_name()); + case ast_object_literal: + return "↓" + std::to_string(v->as()->get_body()->get_num_fields()); case ast_tolk_required_version: return static_cast(v->as()->semver); case ast_import_directive: @@ -289,6 +303,9 @@ class ASTStringifier final : public ASTVisitor { case ast_not_null_operator: return handle_vertex(v->as()); case ast_match_expression: return handle_vertex(v->as()); case ast_match_arm: return handle_vertex(v->as()); + case ast_object_field: return handle_vertex(v->as()); + case ast_object_body: return handle_vertex(v->as()); + case ast_object_literal: return handle_vertex(v->as()); // statements case ast_empty_statement: return handle_vertex(v->as()); case ast_block_statement: return handle_vertex(v->as()); @@ -313,6 +330,9 @@ class ASTStringifier final : public ASTVisitor { case ast_global_var_declaration: return handle_vertex(v->as()); case ast_constant_declaration: return handle_vertex(v->as()); case ast_type_alias_declaration: return handle_vertex(v->as()); + case ast_struct_field: return handle_vertex(v->as()); + case ast_struct_body: return handle_vertex(v->as()); + case ast_struct_declaration: return handle_vertex(v->as()); case ast_tolk_required_version: return handle_vertex(v->as()); case ast_import_directive: return handle_vertex(v->as()); case ast_tolk_file: return handle_vertex(v->as()); diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h index eac7e98ef..256f8d886 100644 --- a/tolk/ast-visitor.h +++ b/tolk/ast-visitor.h @@ -118,6 +118,9 @@ class ASTVisitorFunctionBody : public ASTVisitor { virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } // statements virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } @@ -160,6 +163,9 @@ class ASTVisitorFunctionBody : public ASTVisitor { case ast_not_null_operator: return visit(v->as()); case ast_match_expression: return visit(v->as()); case ast_match_arm: return visit(v->as()); + case ast_object_field: return visit(v->as()); + case ast_object_body: return visit(v->as()); + case ast_object_literal: return visit(v->as()); // statements case ast_empty_statement: return visit(v->as()); case ast_block_statement: return visit(v->as()); diff --git a/tolk/ast.cpp b/tolk/ast.cpp index 9fadd5f6c..083abca0a 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -176,6 +176,18 @@ void Vertex::assign_resolved_type(TypePtr underlying this->underlying_type = underlying_type; } +void Vertex::assign_field_ref(StructFieldPtr field_ref) { + this->field_ref = field_ref; +} + +void Vertex::assign_resolved_type(TypePtr declared_type) { + this->declared_type = declared_type; +} + +void Vertex::assign_struct_ref(StructPtr struct_ref) { + this->struct_ref = struct_ref; +} + void Vertex::assign_resolved_type(TypePtr substituted_type) { this->substituted_type = substituted_type; } @@ -208,6 +220,14 @@ void Vertex::assign_target(const DotTarget& target) { this->target = target; } +void Vertex::assign_field_ref(StructFieldPtr field_ref) { + this->field_ref = field_ref; +} + +void Vertex::assign_struct_ref(StructPtr struct_ref) { + this->struct_ref = struct_ref; +} + void Vertex::assign_fun_ref(FunctionPtr fun_ref) { this->fun_ref = fun_ref; } diff --git a/tolk/ast.h b/tolk/ast.h index 9dce458ec..6531d50a8 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -93,6 +93,9 @@ enum ASTNodeType { ast_not_null_operator, ast_match_expression, ast_match_arm, + ast_object_field, + ast_object_body, + ast_object_literal, // statements ast_empty_statement, ast_block_statement, @@ -117,6 +120,9 @@ enum ASTNodeType { ast_global_var_declaration, ast_constant_declaration, ast_type_alias_declaration, + ast_struct_field, + ast_struct_body, + ast_struct_declaration, ast_tolk_required_version, ast_import_directive, ast_tolk_file, @@ -562,11 +568,13 @@ struct Vertex final : ASTExprUnary { typedef std::variant< FunctionPtr, // for `t.tupleAt` target is `tupleAt` global function + StructFieldPtr, // for `user.id` target is field `id` of struct `User` int // for `t.0` target is "indexed access" 0 > DotTarget; - DotTarget target = static_cast(nullptr); // filled at type inferring + DotTarget target = static_cast(nullptr); // filled at type inferring bool is_target_fun_ref() const { return std::holds_alternative(target); } + bool is_target_struct_field() const { return std::holds_alternative(target); } bool is_target_indexed_access() const { return std::holds_alternative(target); } AnyExprV get_obj() const { return child; } @@ -780,6 +788,59 @@ struct Vertex final : ASTExprBinary { , pattern_kind(pattern_kind), exact_type(exact_type) {} }; +template<> +// ast_object_field is one field at object creation +// example: `Point { x: 2, y: 3 }` is object creation, its body contains 2 fields +struct Vertex final : ASTExprUnary { +private: + V identifier; + +public: + StructFieldPtr field_ref = nullptr; // assigned at type inferring + + AnyExprV get_init_val() const { return child; } + std::string_view get_field_name() const { return identifier->name; } + V get_field_identifier() const { return identifier; } + + Vertex* mutate() const { return const_cast(this); } + void assign_field_ref(StructFieldPtr field_ref); + + Vertex(SrcLocation loc, V name_identifier, AnyExprV init_val) + : ASTExprUnary(ast_object_field, loc, init_val) + , identifier(name_identifier) {} +}; + +template<> +// ast_object_body is `{ ... }` inside object initialization, it contains fields +// examples: see below +struct Vertex final : ASTExprVararg { + int get_num_fields() const { return size(); } + auto get_field(int i) const { return child(i)->as(); } + std::vector get_all_fields() const { return children; } + + Vertex(SrcLocation loc, std::vector&& fields) + : ASTExprVararg(ast_object_body, loc, std::move(fields)) {} +}; + +template<> +// ast_object_literal is creating an instance of a struct with initial values of fields, like objects in TypeScript +// example: `Point { ... }` (object creation has explicit ref and body) +// example: `var v: Point = { ... }` (object creation has only body, struct_ref is determined from the left) +struct Vertex final : ASTExprBinary { + StructPtr struct_ref = nullptr; // assigned at type inferring + + bool has_explicit_ref() const { return lhs->type != ast_empty_expression; } + auto get_explicit_ref() const { return lhs->as(); } + AnyExprV get_maybe_reference() const { return lhs; } // ast_empty_expression or ast_reference + auto get_body() const { return rhs->as(); } + + Vertex* mutate() const { return const_cast(this); } + void assign_struct_ref(StructPtr struct_ref); + + Vertex(SrcLocation loc, AnyExprV maybe_reference, V body) + : ASTExprBinary(ast_object_literal, loc, maybe_reference, body) {} +}; + // // --------------------------------------------------------- @@ -1127,6 +1188,52 @@ struct Vertex final : ASTOtherVararg { , underlying_type(underlying_type) {} }; +template<> +// ast_struct_field is one field at struct declaration +// example: `struct Point { x: int, y: int }` is struct declaration, its body contains 2 fields +struct Vertex final : ASTOtherVararg { + StructFieldPtr field_ref = nullptr; // filled after register + TypePtr declared_type; + + auto get_identifier() const { return children.at(0)->as(); } + + Vertex* mutate() const { return const_cast(this); } + void assign_field_ref(StructFieldPtr field_ref); + void assign_resolved_type(TypePtr declared_type); + + Vertex(SrcLocation loc, V name_identifier, TypePtr declared_type) + : ASTOtherVararg(ast_struct_field, loc, {name_identifier}) + , declared_type(declared_type) {} +}; + +template<> +// ast_struct_body is `{ ... }` inside struct declaration, it contains fields +// example: `struct Storage { owner: User; validUntil: int }` its body contains 2 fields +struct Vertex final : ASTOtherVararg { + int get_num_fields() const { return size(); } + auto get_field(int i) const { return children.at(i)->as(); } + + Vertex(SrcLocation loc, std::vector&& fields) + : ASTOtherVararg(ast_struct_body, loc, std::move(fields)) {} +}; + +template<> +// ast_struct_declaration is declaring a struct with fields (each having declared_type), like interfaces in TypeScript +// example: `struct Storage { owner: User; validUntil: int }` +// currently, Tolk doesn't have "implements" or whatever, so struct declaration contains only body +struct Vertex final : ASTOtherVararg { + StructPtr struct_ref = nullptr; // filled after register + + auto get_identifier() const { return children.at(0)->as(); } + auto get_struct_body() const { return children.at(1)->as(); } + + Vertex* mutate() const { return const_cast(this); } + void assign_struct_ref(StructPtr struct_ref); + + Vertex(SrcLocation loc, V name_identifier, V struct_body) + : ASTOtherVararg(ast_struct_declaration, loc, {name_identifier, struct_body}) {} +}; + template<> // ast_tolk_required_version is a preamble fixating compiler's version at the top of the file // example: `tolk 0.6` diff --git a/tolk/fwd-declarations.h b/tolk/fwd-declarations.h index 7255a8588..ff2e32a36 100644 --- a/tolk/fwd-declarations.h +++ b/tolk/fwd-declarations.h @@ -32,12 +32,16 @@ struct FunctionData; struct GlobalVarData; struct GlobalConstData; struct AliasDefData; +struct StructFieldData; +struct StructData; using LocalVarPtr = const LocalVarData*; using FunctionPtr = const FunctionData*; using GlobalVarPtr = const GlobalVarData*; using GlobalConstPtr = const GlobalConstData*; using AliasDefPtr = const AliasDefData*; +using StructFieldPtr = const StructFieldData*; +using StructPtr = const StructData*; class TypeData; using TypePtr = const TypeData*; diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 3778b8cab..8f4a8ddab 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -100,6 +100,23 @@ class VarsModificationWatcher { static VarsModificationWatcher vars_modification_watcher; +static int calc_offset_on_stack(const TypeDataTensor* t_tensor, int index_at) { + int stack_offset = 0; + for (int i = 0; i < index_at; ++i) { + stack_offset += t_tensor->items[i]->get_width_on_stack(); + } + return stack_offset; +} + +static int calc_offset_on_stack(const TypeDataStruct* t_struct, int field_idx) { + int stack_offset = 0; + for (int i = 0; i < field_idx; ++i) { + stack_offset += t_struct->struct_ref->fields[i]->declared_type->get_width_on_stack(); + } + return stack_offset; +} + + // Main goal of LValContext is to handle non-primitive lvalues. At IR level, a usual local variable // exists, but on its change, something non-trivial should happen. // Example: `globalVar = 9` actually does `Const $5 = 9` + `Let $6 = $5` + `SetGlob "globVar" = $6` @@ -143,20 +160,25 @@ class LValContext { // example: `global v: (int, int); v.1 = 5`, implicit var is created `$tmp = 5`, and when it's modified, // we need to partially update w; essentially, apply_partially_rewrite() above will be called struct ModifiedFieldOfGlobal { - AnyExprV tensor_obj; - int index_at; + AnyExprV tensor_obj; // it's a tensor or struct + int index_at; // for tensors, it's index_at; for structs, it's field_idx std::vector lval_ir_idx; void apply(CodeBlob& code, SrcLocation loc) const { LValContext local_lval; local_lval.enter_rval_inside_lval(); std::vector obj_ir_idx = pre_compile_expr(tensor_obj, code, nullptr, &local_lval); - const TypeDataTensor* t_tensor = tensor_obj->inferred_type->unwrap_alias()->try_as(); - tolk_assert(t_tensor); - int stack_width = t_tensor->items[index_at]->get_width_on_stack(); - int stack_offset = 0; - for (int i = 0; i < index_at; ++i) { - stack_offset += t_tensor->items[i]->get_width_on_stack(); + + int stack_width, stack_offset; + TypePtr obj_type = tensor_obj->inferred_type->unwrap_alias(); + if (const TypeDataTensor* t_tensor = obj_type->try_as()) { + stack_width = t_tensor->items[index_at]->get_width_on_stack(); + stack_offset = calc_offset_on_stack(t_tensor, index_at); + } else if (const TypeDataStruct* t_struct = obj_type->try_as()) { + stack_width = t_struct->struct_ref->get_field(index_at)->declared_type->get_width_on_stack(); + stack_offset = calc_offset_on_stack(t_struct, index_at); + } else { + tolk_assert(false); } std::vector field_ir_idx = {obj_ir_idx.begin() + stack_offset, obj_ir_idx.begin() + stack_offset + stack_width}; tolk_assert(field_ir_idx.size() == lval_ir_idx.size()); @@ -270,7 +292,7 @@ static AnyExprV unwrap_not_null_operator(AnyExprV v) { static V calc_sink_leftmost_obj(V v) { AnyExprV leftmost_obj = unwrap_not_null_operator(v->get_obj()); while (auto v_dot = leftmost_obj->try_as()) { - if (!v_dot->is_target_indexed_access()) { + if (!v_dot->is_target_indexed_access() && !v_dot->is_target_struct_field()) { break; } leftmost_obj = unwrap_not_null_operator(v_dot->get_obj()); @@ -350,13 +372,15 @@ static std::vector> pre_compile_tensor_inner(CodeBlob& co } static std::vector pre_compile_tensor(CodeBlob& code, const std::vector& args, - LValContext* lval_ctx = nullptr) { - std::vector types_list; - types_list.reserve(args.size()); - for (AnyExprV item : args) { - types_list.push_back(item->inferred_type); + LValContext* lval_ctx = nullptr, const TypeDataTensor* tensor_target_type = nullptr) { + if (tensor_target_type == nullptr) { + std::vector types_list; + types_list.reserve(args.size()); + for (AnyExprV item : args) { + types_list.push_back(item->inferred_type); + } + tensor_target_type = TypeDataTensor::create(std::move(types_list))->try_as(); } - const TypeDataTensor* tensor_target_type = TypeDataTensor::create(std::move(types_list))->try_as(); std::vector> res_lists = pre_compile_tensor_inner(code, args, tensor_target_type, lval_ctx); std::vector res; for (const std::vector& list : res_lists) { @@ -489,10 +513,8 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectoror_null->get_width_on_stack() == 1) { - // a generalized union type "T | ..." takes at least 2 slots, but "T | null" can be 1 - tolk_assert(t_union->or_null); + // in general, pass `T1` to `T2?` when `T2?` still occupies 1 stack slot (value or TVM NULL) + if (t_union && t_union->is_primitive_nullable() && orig_w == 1) { // rvect has 1 slot, either value or TVM NULL return rvect; } @@ -502,8 +524,8 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectoror_null->get_width_on_stack() == 1) { - tolk_assert(o_union->or_null); + if (o_union && o_union->is_primitive_nullable() && target_w == 1) { + // rvect has 1 slot, but its contents is compile-time guaranteed to match target_type return rvect; } @@ -518,7 +540,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vector 0 && t_union->or_null->get_width_on_stack() == 1) { + if (t_union && t_union->is_primitive_nullable() && orig_w > 0) { // nothing except "T1 | T2 | ... null" can be cast to 1-slot nullable `T1?` tolk_assert(o_union && o_union->has_null() && o_union->has_variant_with_type_id(t_union->or_null)); // here we exploit rvect shape, how union types and multi-slot nullables are stored on a stack @@ -527,7 +549,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vector transition_expr_to_runtime_type_impl(std::vectoror_null && t_union->or_null->get_width_on_stack() == 0); + tolk_assert(t_union->has_null() && target_w == 1); std::vector new_rvect = code.create_tmp_var(TypeDataInt::create(), loc, "(UTag)"); code.emplace_back(loc, Op::_IntConst, new_rvect, td::make_refint(0)); return new_rvect; @@ -569,10 +592,11 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectoror_null); + tolk_assert(orig_w == 1 && o_union->has_null()); FunctionPtr null_sym = lookup_global_symbol("__null")->try_as(); std::vector new_rvect = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); code.emplace_back(loc, Op::_Call, new_rvect, std::vector{}, null_sym); @@ -583,7 +607,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectoror_null->get_width_on_stack() == 1) { + if (o_union && o_union->is_primitive_nullable() && t_union) { tolk_assert(t_union->has_null() && t_union->has_variant_with_type_id(o_union->or_null) && target_w > 1); // the transformation is tricky: // when value is null, we need to achieve "... (null) 0" (value is already null, so "... value 0") @@ -1089,10 +1113,31 @@ static std::vector process_dot_access(V v, CodeBlob& // it's `t.0`, `getUser().id`, and `t.tupleSize` (as a reference, not as a call) if (!v->is_target_fun_ref()) { TypePtr obj_type = v->get_obj()->inferred_type->unwrap_alias(); - int index_at = std::get(v->target); + // `user.id`; internally, a struct (an object) is a tensor + if (const auto* t_struct = obj_type->try_as()) { + StructFieldPtr field_ref = std::get(v->target); + // handle `globalObj.field = rhs`, special case, then the global will be read on demand + if (lval_ctx && !lval_ctx->is_rval_inside_lval()) { + if (auto sink = calc_sink_leftmost_obj(v); sink && sink->sym->try_as()) { + std::vector lval_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(lval-global-field)"); + lval_ctx->capture_field_of_global_modification(v->get_obj(), field_ref->field_idx, lval_ir_idx); + return lval_ir_idx; + } + } + std::vector lhs_vars = pre_compile_expr(v->get_obj(), code, nullptr, lval_ctx); + int stack_width = field_ref->declared_type->get_width_on_stack(); + int stack_offset = calc_offset_on_stack(t_struct, field_ref->field_idx); + std::vector rvect{lhs_vars.begin() + stack_offset, lhs_vars.begin() + stack_offset + stack_width}; + // an object field might be smart cast at this point, for example we're in `if (user.t != null)` + // it means that we must drop the null flag (if `user.t` is a tensor), or maybe perform other stack transformations + // (from original rvect = (vars of user.t) to fit smart cast) + rvect = transition_to_target_type(std::move(rvect), code, field_ref->declared_type, v->inferred_type, v->loc); + return transition_to_target_type(std::move(rvect), code, target_type, v); + } // `tensorVar.0` if (const auto* t_tensor = obj_type->try_as()) { - // handle `tensorVar.0 = rhs` if tensors is a global, special case, then the global will be read on demand + int index_at = std::get(v->target); + // handle `globalTensorVar.0 = rhs`, special case, then the global will be read on demand if (lval_ctx && !lval_ctx->is_rval_inside_lval()) { if (auto sink = calc_sink_leftmost_obj(v); sink && sink->sym->try_as()) { std::vector lval_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(lval-global-tensor)"); @@ -1103,10 +1148,7 @@ static std::vector process_dot_access(V v, CodeBlob& // since a tensor of N elems are N vars on a stack actually, calculate offset std::vector lhs_vars = pre_compile_expr(v->get_obj(), code, nullptr, lval_ctx); int stack_width = t_tensor->items[index_at]->get_width_on_stack(); - int stack_offset = 0; - for (int i = 0; i < index_at; ++i) { - stack_offset += t_tensor->items[i]->get_width_on_stack(); - } + int stack_offset = calc_offset_on_stack(t_tensor, index_at); std::vector rvect{lhs_vars.begin() + stack_offset, lhs_vars.begin() + stack_offset + stack_width}; // a tensor index might be smart cast at this point, for example we're in `if (t.1 != null)` // it means that we must drop the null flag (if `t.1` is a tensor), or maybe perform other stack transformations @@ -1116,6 +1158,7 @@ static std::vector process_dot_access(V v, CodeBlob& } // `tupleVar.0` if (obj_type->try_as() || obj_type->try_as()) { + int index_at = std::get(v->target); // handle `tupleVar.0 = rhs`, "0 SETINDEX" will be called when this was is modified if (lval_ctx && !lval_ctx->is_rval_inside_lval() && calc_sink_leftmost_obj(v)) { std::vector lval_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(lval-tuple-field)"); @@ -1281,6 +1324,32 @@ static std::vector process_typed_tuple(V v, CodeBlob return transition_to_target_type(std::move(left), code, target_type, v); } +static std::vector process_object_literal(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + // an object (an instance of a struct) is actually a tensor at low-level + // for example, `struct User { id: int; name: slice; }` occupies 2 slots + // fields of a tensor are placed in order of declaration (in a literal they might be shuffled) + std::vector tensor_items; + std::vector target_types; + tensor_items.reserve(v->struct_ref->get_num_fields()); + target_types.reserve(v->struct_ref->get_num_fields()); + for (StructFieldPtr field_ref : v->struct_ref->fields) { + AnyExprV v_init_val = nullptr; + for (int i = 0; i < v->get_body()->get_num_fields(); ++i) { + auto v_field = v->get_body()->get_field(i); + if (v_field->get_field_name() == field_ref->name) { + v_init_val = v_field->get_init_val(); + break; + } + } + tolk_assert(v_init_val); + tensor_items.push_back(v_init_val); + target_types.push_back(field_ref->declared_type); + } + const auto* tensor_target_type = TypeDataTensor::create(std::move(target_types))->try_as(); + std::vector rvect = pre_compile_tensor(code, tensor_items, lval_ctx, tensor_target_type); + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + static std::vector process_int_const(V v, CodeBlob& code, TypePtr target_type) { std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(int-const)"); code.emplace_back(v->loc, Op::_IntConst, rvect, v->intval); @@ -1370,6 +1439,8 @@ std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr targ return process_tensor(v->as(), code, target_type, lval_ctx); case ast_typed_tuple: return process_typed_tuple(v->as(), code, target_type, lval_ctx); + case ast_object_literal: + return process_object_literal(v->as(), code, target_type, lval_ctx); case ast_int_const: return process_int_const(v->as(), code, target_type); case ast_string_const: diff --git a/tolk/pipe-calc-rvalue-lvalue.cpp b/tolk/pipe-calc-rvalue-lvalue.cpp index b910eface..b00a76696 100644 --- a/tolk/pipe-calc-rvalue-lvalue.cpp +++ b/tolk/pipe-calc-rvalue-lvalue.cpp @@ -226,6 +226,25 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { parent::visit(v); } + void visit(V v) override { + tolk_assert(cur_state == MarkingState::RValue); + mark_vertex(v); + parent::visit(v); + } + + void visit(V v) override { + tolk_assert(cur_state == MarkingState::RValue); + mark_vertex(v); + parent::visit(v); + } + + void visit(V v) override { + mark_vertex(v); + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + void visit(V v) override { tolk_assert(cur_state == MarkingState::LValue); mark_vertex(v); diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 4b0f8d53f..46de607c8 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -128,6 +128,9 @@ static void handle_possible_compiler_internal_call(FunctionPtr cur_f, Vtry_as()) { return TypeDataAlias::create(alias_ref); } + if (StructPtr struct_ref = sym->try_as()) { + return struct_ref->struct_type; + } tolk_assert(false); } return child; @@ -468,6 +471,8 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (!lhs->inferred_type->can_rhs_be_assigned(rhs_type)) { if (lhs->try_as()) { fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to variable of type " + to_string(lhs)); + } else if (lhs->try_as()) { + fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to field of type " + to_string(lhs)); } else { fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to " + to_string(lhs)); } @@ -616,6 +621,14 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } } + void visit(V v) override { + parent::visit(v->get_init_val()); + + if (!v->field_ref->declared_type->can_rhs_be_assigned(v->get_init_val()->inferred_type)) { + fire(cur_f, v->get_init_val()->loc, "can not assign " + to_string(v->get_init_val()) + " to field of type " + to_string(v->field_ref->declared_type)); + } + } + void visit(V v) override { parent::visit(v); diff --git a/tolk/pipe-check-rvalue-lvalue.cpp b/tolk/pipe-check-rvalue-lvalue.cpp index 886091127..fed6de13d 100644 --- a/tolk/pipe-check-rvalue-lvalue.cpp +++ b/tolk/pipe-check-rvalue-lvalue.cpp @@ -149,6 +149,31 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { } void visit(V v) override { + // check for `immutableVal.field = rhs` or any other mutation of an immutable tensor/tuple/object + // don't allow cheating like `((immutableVal!)).field = rhs` + if (v->is_lvalue) { + AnyExprV leftmost_obj = v->get_obj(); + while (true) { + if (auto as_dot = leftmost_obj->try_as()) { + leftmost_obj = as_dot->get_obj(); + } else if (auto as_par = leftmost_obj->try_as()) { + leftmost_obj = as_par->get_expr(); + } else if (auto as_cast = leftmost_obj->try_as()) { + leftmost_obj = as_cast->get_expr(); + } else if (auto as_nn = leftmost_obj->try_as()) { + leftmost_obj = as_nn->get_expr(); + } else { + break; + } + } + + if (auto as_ref = leftmost_obj->try_as()) { + if (LocalVarPtr var_ref = as_ref->sym->try_as(); var_ref && var_ref->is_immutable()) { + fire_error_modifying_immutable_variable(cur_f, leftmost_obj, var_ref); + } + } + } + // a reference to a method used as rvalue, like `var v = t.tupleAt` if (v->is_rvalue && v->is_target_fun_ref()) { validate_function_used_as_noncall(cur_f, v, std::get(v->target)); diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 0e1c5af98..9cca8a292 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -120,6 +120,11 @@ static std::string to_string(FunctionPtr fun_ref) { return "`" + fun_ref->as_human_readable() + "`"; } +GNU_ATTRIBUTE_NOINLINE +static std::string to_string(std::string_view string_view) { + return static_cast(string_view); +} + // fire a general error, just a wrapper over `throw` GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire(FunctionPtr cur_f, SrcLocation loc, const std::string& message) { @@ -247,7 +252,7 @@ class InferTypesAndCallsAndFieldsVisitor final { case ast_braced_expression: return infer_braced_expression(v->as(), std::move(flow), used_as_condition); case ast_reference: - return infer_reference(v->as(), std::move(flow), used_as_condition); + return infer_reference(v->as(), std::move(flow), used_as_condition, hint); case ast_dot_access: return infer_dot_access(v->as(), std::move(flow), used_as_condition, hint); case ast_function_call: @@ -260,6 +265,8 @@ class InferTypesAndCallsAndFieldsVisitor final { return infer_null_keyword(v->as(), std::move(flow), used_as_condition); case ast_match_expression: return infer_match_expression(v->as(), std::move(flow), used_as_condition, hint); + case ast_object_literal: + return infer_object_literal(v->as(), std::move(flow), used_as_condition, hint); case ast_underscore: return infer_underscore(v->as(), std::move(flow), used_as_condition, hint); case ast_empty_expression: @@ -685,7 +692,7 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(flow), used_as_condition); } - ExprFlow infer_reference(V v, FlowContext&& flow, bool used_as_condition) { + ExprFlow infer_reference(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) const { if (LocalVarPtr var_ref = v->sym->try_as()) { TypePtr declared_or_smart_casted = flow.smart_cast_if_exists(SinkExpression(var_ref)); tolk_assert(declared_or_smart_casted != nullptr); // all local vars are presented in flow @@ -728,6 +735,9 @@ class InferTypesAndCallsAndFieldsVisitor final { } else if (AliasDefPtr alias_ref = v->sym->try_as()) { fire(cur_f, v->loc, "type `" + alias_ref->name + "` can not be used as a value"); + } else if (StructPtr struct_ref = v->sym->try_as()) { + fire(cur_f, v->loc, "struct `" + struct_ref->name + "` can not be used as a value"); + } else { tolk_assert(false); } @@ -790,8 +800,23 @@ class InferTypesAndCallsAndFieldsVisitor final { V v_instantiationTs = v->get_instantiationTs(); std::string_view field_name = v_ident->name; - // it can be indexed access (`tensorVar.0`, `tupleVar.1`) or a method (`t.tupleSize`) - // at first, check for indexed access + // check for field access (`user.id`), when obj is a struct + if (const TypeDataStruct* obj_struct = unwrapped_obj_type->try_as()) { + if (StructFieldPtr field_ref = obj_struct->struct_ref->find_field(field_name)) { + v->mutate()->assign_target(field_ref); + TypePtr inferred_type = field_ref->declared_type; + if (SinkExpression s_expr = extract_sink_expression_from_vertex(v)) { + if (TypePtr smart_casted = flow.smart_cast_if_exists(s_expr)) { + inferred_type = smart_casted; + } + } + assign_inferred_type(v, inferred_type); + return ExprFlow(std::move(flow), used_as_condition); + } + // if field_name doesn't exist, don't fire an error now — maybe, it's `user.globalFunction()` + } + + // check for indexed access (`tensorVar.0` / `tupleVar.1`) if (field_name[0] >= '0' && field_name[0] <= '9') { int index_at = std::stoi(std::string(field_name)); if (const auto* t_tensor = unwrapped_obj_type->try_as()) { @@ -836,14 +861,28 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, item_type); return ExprFlow(std::move(flow), used_as_condition); } - fire(cur_f, v_ident->loc, "type " + to_string(obj_type) + " is not indexable"); + // numeric field not found, fire an error + if (unwrapped_obj_type->try_as()) { // `user.100500` + fire(cur_f, v_ident->loc, "field `" + static_cast(field_name) + "` doesn't exist in type " + to_string(obj_type)); + } else { + fire(cur_f, v_ident->loc, "type " + to_string(obj_type) + " is not indexable"); + } } - // for now, Tolk doesn't have fields and object-scoped methods; `t.tupleSize` is a global function `tupleSize` + // check for method (`t.tupleSize` / `user.globalFunction`) + // for now, Tolk doesn't have object-scoped methods; `t.tupleSize` is a global function `tupleSize` const Symbol* sym = lookup_global_symbol(field_name); FunctionPtr fun_ref = sym ? sym->try_as() : nullptr; + + // not a field, not a method — fire an error if (!fun_ref) { - fire(cur_f, v_ident->loc, "non-existing field `" + static_cast(field_name) + "` of type " + to_string(obj_type)); + // as a special case, handle accessing fields of nullable objects, to show a more precise error + if (const TypeDataUnion* as_union = unwrapped_obj_type->try_as(); as_union && as_union->or_null) { + if (const TypeDataStruct* n_struct = as_union->or_null->try_as(); n_struct && n_struct->struct_ref->find_field(field_name)) { + fire(cur_f, v_ident->loc, "can not access field `" + static_cast(field_name) + "` of a possibly nullable object " + to_string(obj_type) + "; check it via `obj != null` or use non-null assertion `obj!` operator"); + } + } + fire(cur_f, v_ident->loc, "field `" + static_cast(field_name) + "` doesn't exist in type " + to_string(obj_type)); } // `t.tupleSize` is ok, `cs.tupleSize` not @@ -896,7 +935,9 @@ class InferTypesAndCallsAndFieldsVisitor final { // it can be indexed access (`tensorVar.0()`, `tupleVar.1()`) or a method (`t.tupleSize()`) std::string_view field_name = v_dot->get_field_name(); - if (field_name[0] >= '0' && field_name[0] <= '9') { + if (const TypeDataStruct* t_struct = dot_obj->inferred_type->unwrap_alias()->try_as(); t_struct && t_struct->struct_ref->find_field(field_name)) { + // `t.someField`, it's a field, probably a callable, fun_ref remains nullptr + } else if (field_name[0] >= '0' && field_name[0] <= '9') { // indexed access `ab.2()`, then treat `ab.2` just like an expression, fun_ref remains nullptr // infer_dot_access() will be called for a callee, it will check index correctness } else { @@ -904,7 +945,7 @@ class InferTypesAndCallsAndFieldsVisitor final { const Symbol* sym = lookup_global_symbol(field_name); fun_ref = sym ? sym->try_as() : nullptr; if (!fun_ref) { - fire(cur_f, v_dot->get_identifier()->loc, "non-existing method `" + static_cast(field_name) + "` of type " + to_string(dot_obj)); + fire(cur_f, v_dot->get_identifier()->loc, "non-existing method `" + to_string(field_name) + "` of type " + to_string(dot_obj)); } } @@ -1112,6 +1153,76 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(match_out_flow), used_as_condition); } + ExprFlow infer_object_literal(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + StructPtr struct_ref = nullptr; + if (v->has_explicit_ref()) { + V v_ref = v->get_explicit_ref(); + struct_ref = v_ref->sym->try_as(); + if (!struct_ref) { + if (AliasDefPtr as_alias = v_ref->sym->try_as()) { + if (const TypeDataStruct* as_struct = as_alias->underlying_type->unwrap_alias()->try_as()) { + struct_ref = as_struct->struct_ref; + } + } + } + if (!struct_ref) { + fire(cur_f, v_ref->loc, "`" + to_string(v_ref->get_name()) + "` does not name a struct"); + } + if (UNLIKELY(v_ref->has_instantiationTs())) { + fire(cur_f, v_ref->get_instantiationTs()->loc, "generic T not expected here"); + } + } + if (!struct_ref && hint) { + if (const TypeDataStruct* hint_struct = hint->unwrap_alias()->try_as()) { + struct_ref = hint_struct->struct_ref; + } + if (const TypeDataUnion* hint_union = hint->unwrap_alias()->try_as()) { + StructPtr last_struct = nullptr; + int n_structs = 0; + for (TypePtr variant : hint_union->variants) { + if (const TypeDataStruct* variant_struct = variant->unwrap_alias()->try_as()) { + last_struct = variant_struct->struct_ref; + n_structs++; + } + } + if (n_structs == 1) { + struct_ref = last_struct; + } + } + } + if (!struct_ref) { + fire(cur_f, v->loc, "can not detect struct name; use either `var v: StructName = { ... }` or `var v = StructName { ... }`"); + } + + uint64_t occurred_mask = 0; + for (int i = 0; i < v->get_body()->get_num_fields(); ++i) { + auto v_field = v->get_body()->get_field(i); + std::string_view field_name = v_field->get_field_name(); + StructFieldPtr field_ref = struct_ref->find_field(field_name); + if (!field_ref) { + fire(cur_f, v_field->loc, "field `" + to_string(field_name) + "` not found in struct `" + struct_ref->name + "`"); + } + v_field->mutate()->assign_field_ref(field_ref); + + if (occurred_mask & (1ULL << field_ref->field_idx)) { + fire(cur_f, v_field->loc, "duplicate field initialization"); + } + occurred_mask |= 1ULL << field_ref->field_idx; + flow = infer_any_expr(v_field->get_init_val(), std::move(flow), false, field_ref->declared_type).out_flow; + assign_inferred_type(v_field, v_field->get_init_val()); + } + for (StructFieldPtr field_ref : struct_ref->fields) { + if (!(occurred_mask & (1ULL << field_ref->field_idx))) { + fire(cur_f, v->get_body()->loc, "field `" + field_ref->name + "` missed in initialization of struct `" + struct_ref->name + "`"); + } + } + + v->mutate()->assign_struct_ref(struct_ref); + assign_inferred_type(v, struct_ref->struct_type); + assign_inferred_type(v->get_body(), v); + return ExprFlow(std::move(flow), used_as_condition); + } + static ExprFlow infer_underscore(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { // if execution is here, underscore is either used as lhs of assignment, or incorrectly, like `f(_)` // more precise is to always set unknown here, but for incorrect usages, instead of an error diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index 363952cbd..718547198 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -140,6 +140,24 @@ static void register_type_alias(V v) { v->mutate()->assign_alias_ref(a_sym); } +static void register_struct(V v) { + auto v_body = v->get_struct_body(); + + std::vector fields; + fields.reserve(v_body->get_num_fields()); + for (int i = 0; i < v_body->get_num_fields(); ++i) { + auto v_field = v_body->get_field(i); + StructFieldData* field_ref = new StructFieldData(static_cast(v_field->get_identifier()->name), v_field->loc, i, v_field->declared_type); + fields.emplace_back(field_ref); + v_field->mutate()->assign_field_ref(field_ref); + } + + StructData* s_sym = new StructData(static_cast(v->get_identifier()->name), v->loc, std::move(fields)); + + G.symtable.add_struct(s_sym); + v->mutate()->assign_struct_ref(s_sym); +} + static LocalVarData register_parameter(V v, int idx) { if (v->is_underscore()) { return {"", v->loc, v->declared_type, 0, idx}; @@ -248,6 +266,9 @@ static void iterate_through_file_symbols(const SrcFile* file) { case ast_type_alias_declaration: register_type_alias(v->as()); break; + case ast_struct_declaration: + register_struct(v->as()); + break; case ast_function_declaration: register_function(v->as()); break; diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 797795afd..304a3497d 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -160,6 +160,12 @@ struct TypeDataResolver { } return TypeDataAlias::create(alias_ref); } + if (StructPtr struct_ref = sym->try_as()) { + if (!struct_ref->struct_type) { + resolve_and_mutate_struct_and_fields(struct_ref); + } + return struct_ref->struct_type; + } } if (un->text == "auto") { throw ParseError(cur_f, un->loc, "`auto` type does not exist; just omit a type for local variable (will be inferred from assignment); parameters should always be typed"); @@ -187,6 +193,27 @@ struct TypeDataResolver { alias_ref->mutate()->assign_resolved_type(underlying_type); called_stack.pop_back(); } + + static void resolve_and_mutate_struct_and_fields(StructPtr struct_ref) { + static std::vector called_stack; + + // prevent recursion like `struct A { field: A }` + // currently, a struct is a tensor, and recursion always leads to infinite size (`A?` also, it's also on a stack) + // if there would be an annotation to store a struct in a tuple, then it has to be reconsidered + bool contains = std::find(called_stack.begin(), called_stack.end(), struct_ref) != called_stack.end(); + if (contains) { + throw ParseError(struct_ref->loc, "struct `" + struct_ref->name + "` size is infinity due to recursive fields"); + } + + called_stack.push_back(struct_ref); + for (int i = 0; i < struct_ref->get_num_fields(); ++i) { + StructFieldPtr field_ref = struct_ref->get_field(i); + TypePtr declared_type = finalize_type_data(nullptr, field_ref->declared_type, nullptr); + field_ref->mutate()->assign_resolved_type(declared_type); + } + struct_ref->mutate()->assign_resolved_type(TypeDataStruct::create(struct_ref)); + called_stack.pop_back(); + } }; static TypePtr finalize_type_data(FunctionPtr cur_f, TypePtr type_data, const GenericsDeclaration* genericTs) { @@ -280,7 +307,7 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { switch (v->pattern_kind) { case MatchArmKind::exact_type: { if (const TypeDataUnresolved* maybe_ident = v->exact_type->try_as()) { - if (const Symbol* sym = current_scope.lookup_symbol(maybe_ident->text); sym && !sym->try_as()) { + if (const Symbol* sym = current_scope.lookup_symbol(maybe_ident->text); sym && !sym->try_as() && !sym->try_as()) { auto v_ident = createV(v->loc, sym->name); AnyExprV pattern_expr = createV(v->loc, v_ident, nullptr); parent::visit(pattern_expr); @@ -425,6 +452,14 @@ void pipeline_resolve_identifiers_and_assign_symbols() { } else if (auto v_alias = v->try_as()) { TypeDataResolver::resolve_and_mutate_type_alias(v_alias->alias_ref); v_alias->mutate()->assign_resolved_type(v_alias->alias_ref->underlying_type); + + } else if (auto v_struct = v->try_as()) { + TypeDataResolver::resolve_and_mutate_struct_and_fields(v_struct->struct_ref); + auto v_body = v_struct->get_struct_body(); + for (int i = 0; i < v_body->get_num_fields(); ++i) { + auto v_field = v_body->get_field(i); + v_field->mutate()->assign_resolved_type(v_struct->struct_ref->get_field(i)->declared_type); + } } } } diff --git a/tolk/smart-casts-cfg.cpp b/tolk/smart-casts-cfg.cpp index 65ab7fb80..b4974ef48 100644 --- a/tolk/smart-casts-cfg.cpp +++ b/tolk/smart-casts-cfg.cpp @@ -98,9 +98,16 @@ namespace tolk { std::string SinkExpression::to_string() const { std::string result = var_ref->name; uint64_t cur_path = index_path; + TypePtr cur_type = var_ref->declared_type; while (cur_path != 0) { result += "."; - result += std::to_string((cur_path & 0xFF) - 1); + if (const TypeDataStruct* t_struct = cur_type->try_as()) { + StructFieldPtr field_ref = t_struct->struct_ref->get_field((cur_path & 0xFF) - 1); + result += field_ref->name; + cur_type = field_ref->declared_type; + } else { + result += std::to_string((cur_path & 0xFF) - 1); + } cur_path >>= 8; } return result; @@ -400,11 +407,13 @@ SinkExpression extract_sink_expression_from_vertex(AnyExprV v) { } } - if (auto as_dot = v->try_as(); as_dot && as_dot->is_target_indexed_access()) { + if (auto as_dot = v->try_as()) { V cur_dot = as_dot; uint64_t index_path = 0; - while (cur_dot->is_target_indexed_access()) { - int index_at = std::get(cur_dot->target); + while (cur_dot->is_target_indexed_access() || cur_dot->is_target_struct_field()) { + int index_at = cur_dot->is_target_indexed_access() + ? std::get(cur_dot->target) + : std::get(cur_dot->target)->field_idx; index_path = (index_path << 8) + index_at + 1; if (auto parent_dot = unwrap_not_null_operator(cur_dot->get_obj())->try_as()) { cur_dot = parent_dot; @@ -413,7 +422,7 @@ SinkExpression extract_sink_expression_from_vertex(AnyExprV v) { } } if (auto as_ref = unwrap_not_null_operator(cur_dot->get_obj())->try_as()) { - if (LocalVarPtr var_ref = as_ref->sym->try_as()) { + if (LocalVarPtr var_ref = as_ref->sym->try_as(); var_ref && index_path) { return SinkExpression(var_ref, index_path); } } @@ -448,14 +457,20 @@ TypePtr calc_declared_type_before_smart_cast(AnyExprV v) { } } - if (auto as_dot = v->try_as(); as_dot && as_dot->is_target_indexed_access()) { + if (auto as_dot = v->try_as()) { TypePtr obj_type = as_dot->get_obj()->inferred_type->unwrap_alias(); // v already inferred; hence, index_at is correct - int index_at = std::get(as_dot->target); - if (const auto* t_tensor = obj_type->try_as()) { - return t_tensor->items[index_at]; + if (as_dot->is_target_struct_field()) { + StructFieldPtr field_ref = std::get(as_dot->target); + return field_ref->declared_type; } - if (const auto* t_tuple = obj_type->try_as()) { - return t_tuple->items[index_at]; + if (as_dot->is_target_indexed_access()) { + int index_at = std::get(as_dot->target); + if (const auto* t_tensor = obj_type->try_as()) { + return t_tensor->items[index_at]; + } + if (const auto* t_tuple = obj_type->try_as()) { + return t_tuple->items[index_at]; + } } } diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 4942fe870..ccd56a011 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -117,6 +117,23 @@ void AliasDefData::assign_resolved_type(TypePtr underlying_type) { this->underlying_type = underlying_type; } +void StructFieldData::assign_resolved_type(TypePtr declared_type) { + this->declared_type = declared_type; +} + +void StructData::assign_resolved_type(TypePtr struct_type) { + this->struct_type = struct_type; +} + +StructFieldPtr StructData::find_field(std::string_view field_name) const { + for (StructFieldPtr field : fields) { + if (field->name == field_name) { + return field; + } + } + return nullptr; +} + GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_redefinition_of_symbol(SrcLocation loc, const Symbol* previous) { SrcLocation prev_loc = previous->loc; @@ -161,6 +178,14 @@ void GlobalSymbolTable::add_type_alias(AliasDefPtr a_sym) { } } +void GlobalSymbolTable::add_struct(StructPtr s_sym) { + auto key = key_hash(s_sym->name); + auto [it, inserted] = entries.emplace(key, s_sym); + if (!inserted) { + fire_error_redefinition_of_symbol(s_sym->loc, it->second); + } +} + const Symbol* lookup_global_symbol(std::string_view name) { return G.symtable.lookup(name); } diff --git a/tolk/symtable.h b/tolk/symtable.h index a27aecd44..269aee90f 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -228,6 +228,37 @@ struct AliasDefData final : Symbol { void assign_resolved_type(TypePtr underlying_type); }; +struct StructFieldData final : Symbol { + int field_idx; + TypePtr declared_type; + + StructFieldData* mutate() const { return const_cast(this); } + void assign_resolved_type(TypePtr declared_type); + + StructFieldData(std::string name, SrcLocation loc, int field_idx, TypePtr declared_type) + : Symbol(std::move(name), loc) + , field_idx(field_idx) + , declared_type(declared_type) { + } +}; + +struct StructData final : Symbol { + std::vector fields; + const TypeData* struct_type = nullptr; // it's TypeDataStruct, assigned at resolve identifiers + + int get_num_fields() const { return static_cast(fields.size()); } + StructFieldPtr get_field(int i) const { return fields.at(i); } + StructFieldPtr find_field(std::string_view field_name) const; + + StructData* mutate() const { return const_cast(this); } + void assign_resolved_type(TypePtr struct_type); + + StructData(std::string name, SrcLocation loc, std::vector&& fields) + : Symbol(std::move(name), loc) + , fields(std::move(fields)) { + } +}; + class GlobalSymbolTable { std::unordered_map entries; @@ -240,6 +271,7 @@ class GlobalSymbolTable { void add_global_var(GlobalVarPtr g_sym); void add_global_const(GlobalConstPtr c_sym); void add_type_alias(AliasDefPtr a_sym); + void add_struct(StructPtr s_sym); const Symbol* lookup(std::string_view name) const { const auto it = entries.find(key_hash(name)); diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index a63d555c1..402b79b38 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -218,6 +218,20 @@ TypePtr TypeDataGenericT::create(std::string&& nameT) { return hash.register_unique(new TypeDataGenericT(std::move(nameT))); } +TypePtr TypeDataStruct::create(StructPtr struct_ref) { + TypeDataHasherForUnique hash(8315986401043319583ULL); + hash.feed_string(struct_ref->name); + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + int width_on_stack = 0; + for (StructFieldPtr field_ref : struct_ref->fields) { + width_on_stack += field_ref->declared_type->get_width_on_stack(); + } + return hash.register_unique(new TypeDataStruct(width_on_stack, struct_ref)); +} + TypePtr TypeDataTensor::create(std::vector&& items) { TypeDataHasherForUnique hash(3159238551239480381ULL); for (TypePtr item : items) { @@ -385,6 +399,10 @@ int TypeDataGenericT::get_type_id() const { return TypeIdCalculation::assign_type_id(this); } +int TypeDataStruct::get_type_id() const { + return TypeIdCalculation::assign_type_id(this); +} + int TypeDataTensor::get_type_id() const { assert(!has_genericT_inside()); return TypeIdCalculation::assign_type_id(this); @@ -451,6 +469,10 @@ std::string TypeDataFunCallable::as_human_readable() const { return result; } +std::string TypeDataStruct::as_human_readable() const { + return struct_ref->name; +} + std::string TypeDataTensor::as_human_readable() const { std::string result = "("; for (TypePtr item : items) { @@ -731,6 +753,16 @@ bool TypeDataGenericT::can_rhs_be_assigned(TypePtr rhs) const { return false; } +bool TypeDataStruct::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } + return rhs == TypeDataNever::create(); +} + bool TypeDataTensor::can_rhs_be_assigned(TypePtr rhs) const { if (const auto* as_tensor = rhs->try_as(); as_tensor && as_tensor->size() == size()) { for (int i = 0; i < size(); ++i) { @@ -986,6 +1018,16 @@ bool TypeDataGenericT::can_be_casted_with_as_operator(TypePtr cast_to) const { return true; } +bool TypeDataStruct::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } + return cast_to == this; +} + bool TypeDataTensor::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const auto* to_tensor = cast_to->try_as(); to_tensor && to_tensor->size() == size()) { for (int i = 0; i < size(); ++i) { @@ -1108,6 +1150,18 @@ bool TypeDataAlias::can_hold_tvm_null_instead() const { return underlying_type->can_hold_tvm_null_instead(); } +bool TypeDataStruct::can_hold_tvm_null_instead() const { + if (get_width_on_stack() != 1) { // example that can hold null: `{ field: int }` + return false; // another example: `{ e: Empty, field: ((), int) }` + } // examples can NOT: `{ field1: int, field2: int }`, `{ field1: int? }` + for (StructFieldPtr field : struct_ref->fields) { + if (field->declared_type->get_width_on_stack() == 1 && !field->declared_type->can_hold_tvm_null_instead()) { + return false; + } + } + return true; +} + bool TypeDataTensor::can_hold_tvm_null_instead() const { if (get_width_on_stack() != 1) { // `(int, int)` / `()` can not hold null instead, since null is 1 slot return false; // only `((), int)` and similar can: diff --git a/tolk/type-system.h b/tolk/type-system.h index 56fd02e5c..6a6b1625a 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -17,6 +17,7 @@ #pragma once #include "src-file.h" +#include "fwd-declarations.h" #include #include #include @@ -337,6 +338,28 @@ class TypeDataGenericT final : public TypeData { bool can_be_casted_with_as_operator(TypePtr cast_to) const override; }; +/* + * `A`, `User`, `SomeStruct` is TypeDataStruct. At TVM level, structs are tensors. + * In the code, creating a struct is either `var v: A = { ... }` (by hint) or `var v = A { ... }`. + * Fields of a struct have their own types (accessed by struct_ref). + */ +class TypeDataStruct final : public TypeData { + TypeDataStruct(int width_on_stack, StructPtr struct_ref) + : TypeData(0, width_on_stack) + , struct_ref(struct_ref) {} + +public: + StructPtr struct_ref; + + static TypePtr create(StructPtr struct_ref); + + int get_type_id() const override; + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + bool can_hold_tvm_null_instead() const override; +}; + /* * `(int, slice)` is TypeDataTensor of 2 elements. Tensor of N elements occupies N stack slots. * Of course, there may be nested tensors, like `(int, (int, slice), cell)`. From a01053789ae92d990892b20a057523eea556b1bb Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 17 Apr 2025 08:22:45 +0300 Subject: [PATCH 229/388] [Tolk] Support default values for fields, simplify constants Now fields can have default values, even complex ones: > tensor: (int, coins) = (2, ton("0.05")) Initial value must be a constant expression. Global constants can also now be of any type, including tensors and nullables. Constant evaluator was almost dropped off, evaluation is now done at IR (Ops) level. --- tolk-tester/tests/constants-tests.tolk | 21 +- .../tests/invalid-semantics/err-4190.tolk | 10 + .../tests/invalid-semantics/err-4270.tolk | 8 + .../tests/invalid-symbol/err-2098.tolk | 9 + .../tests/invalid-syntax/err-3150.tolk | 9 - .../tests/invalid-syntax/err-3650.tolk | 8 + .../tests/invalid-typing/err-6740.tolk | 9 + tolk-tester/tests/match-by-expr-tests.tolk | 4 +- tolk-tester/tests/struct-tests.tolk | 33 +++ tolk-tester/tests/use-before-declare.tolk | 2 +- tolk/ast-from-tokens.cpp | 8 +- tolk/ast-replacer.h | 1 + tolk/ast-visitor.h | 1 + tolk/ast.cpp | 2 +- tolk/ast.h | 10 +- tolk/compiler-state.cpp | 4 + tolk/compiler-state.h | 1 + tolk/constant-evaluator.cpp | 197 +++++------------- tolk/constant-evaluator.h | 34 +-- tolk/pipe-ast-to-legacy.cpp | 20 +- tolk/pipe-check-inferred-types.cpp | 17 ++ tolk/pipe-constant-folding.cpp | 49 +++-- tolk/pipe-infer-types-and-calls.cpp | 18 +- tolk/pipe-register-symbols.cpp | 8 +- tolk/pipe-resolve-identifiers.cpp | 6 + tolk/symtable.cpp | 8 +- tolk/symtable.h | 13 +- 27 files changed, 266 insertions(+), 244 deletions(-) create mode 100644 tolk-tester/tests/invalid-semantics/err-4190.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4270.tolk create mode 100644 tolk-tester/tests/invalid-symbol/err-2098.tolk delete mode 100644 tolk-tester/tests/invalid-syntax/err-3150.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3650.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6740.tolk diff --git a/tolk-tester/tests/constants-tests.tolk b/tolk-tester/tests/constants-tests.tolk index d5519b0ef..128a5f7ea 100644 --- a/tolk-tester/tests/constants-tests.tolk +++ b/tolk-tester/tests/constants-tests.tolk @@ -29,6 +29,12 @@ const true3 = true1 && true2; const false1 = !true; const false2 = false1 || false; +const tens1 = (1, 2); +const tens2: (int, int, (int8, int16)) = (tens1.1, tens1.0 << 2, tens1); + +const intOrN: int? = null; +const int32Or64: int32 | int64 = 7 as int64; + fun iget1(): int { return int1; } fun iget2(): int { return int2; } fun iget3(): int { return int1+int2; } @@ -71,6 +77,17 @@ fun test3() { return (false1, false2); } +@method_id(104) +fun test4() { + __expect_type(tens1, "(int, int)"); + return (tens1.0, tens2.2); +} + +@method_id(105) +fun test5() { + return (intOrN == null, int32Or64 is int32, int32Or64); +} + fun main() { var i1: int = iget1(); var i2: int = iget2(); @@ -98,6 +115,8 @@ fun main() { @testcase | 101 | | 0 -1 @testcase | 102 | | -1 -1 -1 @testcase | 103 | | 0 0 +@testcase | 104 | | 1 1 2 +@testcase | 105 | | -1 0 7 48 -@code_hash 28102194299745406750019953961984060488024870092664444642078578246708959881688 +@code_hash 80040709432962217077682091261201772251141677197885524779745956896218368868623 */ diff --git a/tolk-tester/tests/invalid-semantics/err-4190.tolk b/tolk-tester/tests/invalid-semantics/err-4190.tolk new file mode 100644 index 000000000..70355bf98 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4190.tolk @@ -0,0 +1,10 @@ +struct A { + ok: coins = ton("0.05"), + err: int = now(), +} + +/** +@compilation_should_fail +@stderr not a constant expression +@stderr err: int + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4270.tolk b/tolk-tester/tests/invalid-semantics/err-4270.tolk new file mode 100644 index 000000000..14aef95ca --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4270.tolk @@ -0,0 +1,8 @@ +struct A { + tens: (int, int) = (1, 2 + now()), +} + +/** +@compilation_should_fail +@stderr not a constant expression + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2098.tolk b/tolk-tester/tests/invalid-symbol/err-2098.tolk new file mode 100644 index 000000000..5c2b952bb --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2098.tolk @@ -0,0 +1,9 @@ +struct A { + a: int = 1 << CCC; +} + +/** +@compilation_should_fail +@stderr undefined symbol `CCC` +@stderr 1 << CCC + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3150.tolk b/tolk-tester/tests/invalid-syntax/err-3150.tolk deleted file mode 100644 index 8828fcadf..000000000 --- a/tolk-tester/tests/invalid-syntax/err-3150.tolk +++ /dev/null @@ -1,9 +0,0 @@ -struct A { - id: int = 3; -} - -/** -@compilation_should_fail -@stderr default values for fields not supported yet -@stderr id: int = 3 -*/ diff --git a/tolk-tester/tests/invalid-syntax/err-3650.tolk b/tolk-tester/tests/invalid-syntax/err-3650.tolk new file mode 100644 index 000000000..5482e7b9c --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3650.tolk @@ -0,0 +1,8 @@ +struct A { + v1 = 0, +} + +/** +@compilation_should_fail +@stderr expected `: `, got `=` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6740.tolk b/tolk-tester/tests/invalid-typing/err-6740.tolk new file mode 100644 index 000000000..e3dd3a470 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6740.tolk @@ -0,0 +1,9 @@ +struct A { + v1: int = "asdf"; +} + +/** +@compilation_should_fail +@stderr can not assign `slice` to `int` +@stderr v1: int = "asdf" + */ diff --git a/tolk-tester/tests/match-by-expr-tests.tolk b/tolk-tester/tests/match-by-expr-tests.tolk index d9ecb3cfe..80287c9bb 100644 --- a/tolk-tester/tests/match-by-expr-tests.tolk +++ b/tolk-tester/tests/match-by-expr-tests.tolk @@ -155,12 +155,12 @@ fun main() {} 100 PUSHINT // '1=100 }>ELSE<{ // x DUP // x x - 2 EQINT // x '6 + 2 EQINT // x '8 IF:<{ // x DROP // 200 PUSHINT // '1=200 }>ELSE<{ // x - 3 EQINT // '9 + 3 EQINT // '13 IF:<{ // 300 PUSHINT // '1=300 }>ELSE<{ // diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index 91dc00543..4a676a3db 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -428,7 +428,38 @@ fun test31() { return (s1, s2, s3, 777, s1 == null, s2 == null, s3 == null, 777, s1!.x == null, s2!.x == null); } +struct PointDef0 { + x: int = 0, + y: int = 0, +} + +@method_id(132) +fun test32() { + var p1: PointDef0 = { x: 10, y: 20 }; + var p2: PointDef0 = { x: 10 }; + var p3: PointDef0 = { y: 20 }; + var p4: PointDef0 = { }; + return (p1, p2, p3, p4, PointDef0{}); +} +struct WithDefaults { + f1: (bool, int) = (true, 0), + f2: int, + f3: slice? = stringHexToSlice("010203"), + f4: PointDef0? = null, + f5: int32 | int64 = 0 as int32, +} + +@method_id(133) +fun test33(): WithDefaults { + var w1: WithDefaults = { f2: 0 }; + assert(w1.f1.0 && w1.f1.1 == 0 && w1.f3!.getRemainingBitsCount() == 24 && w1.f4 == null && w1.f5 is int32) throw 100; + var w2: WithDefaults? = { f1: (false, 55), f2: 10, f5: 8 as int64 }; + assert(w2.f1.0 != true && w2.f3!.getRemainingBitsCount() == 24 && w2.f4 == null && w2.f5 is int64 && w2.f5 == 8) throw 100; + var w3: (int, WithDefaults) = (0, { f2: 7, f4: {y: 20} }); + assert(w3.1.f4 != null && w3.1.f4.x == 0 && w3.1.f4.y == 20) throw 100; + return { f2: 5, f3: null }; +} @@ -480,6 +511,8 @@ type PointAlias = Point; @testcase | 130 | -1 | 140 777 (null) 140 777 0 777 0 0 @testcase | 130 | 0 | 139 777 4 1 777 139 777 0 0 @testcase | 131 | | 5 141 (null) 141 (null) 0 777 0 0 -1 777 0 -1 +@testcase | 132 | | 10 20 10 0 0 20 0 0 0 0 +@testcase | 133 | | -1 0 5 (null) (null) (null) 0 0 46 @fif_codegen """ diff --git a/tolk-tester/tests/use-before-declare.tolk b/tolk-tester/tests/use-before-declare.tolk index 9c5f06349..f444c48ef 100644 --- a/tolk-tester/tests/use-before-declare.tolk +++ b/tolk-tester/tests/use-before-declare.tolk @@ -58,7 +58,7 @@ const first = 1; """ test2 PROC:<{ // - 2 PUSHINT // '0=2 + 2 PUSHINT // '2 }> """ */ diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 6fc23457b..939dd799c 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -1275,11 +1275,15 @@ static AnyV parse_struct_field(Lexer& lex) { lex.expect(tok_colon, "`: `"); TypePtr declared_type = parse_type_from_tokens(lex); + AnyExprV default_value = nullptr; if (lex.tok() == tok_assign) { // `id: int = 3` - lex.error("default values for fields not supported yet"); + lex.next(); + default_value = parse_expr(lex); + } else { + default_value = createV(lex.cur_location()); } - return createV(loc, v_ident, declared_type); + return createV(loc, v_ident, default_value, declared_type); } static V parse_struct_body(Lexer& lex) { diff --git a/tolk/ast-replacer.h b/tolk/ast-replacer.h index 7108b95b0..40cd8e489 100644 --- a/tolk/ast-replacer.h +++ b/tolk/ast-replacer.h @@ -206,6 +206,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { const std::vector& get_all_not_builtin_functions(); const std::vector& get_all_declared_constants(); +const std::vector& get_all_declared_structs(); template void replace_ast_of_all_functions() { diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h index 256f8d886..12e65e270 100644 --- a/tolk/ast-visitor.h +++ b/tolk/ast-visitor.h @@ -197,6 +197,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { const std::vector& get_all_not_builtin_functions(); const std::vector& get_all_declared_constants(); +const std::vector& get_all_declared_structs(); template void visit_ast_of_all_functions() { diff --git a/tolk/ast.cpp b/tolk/ast.cpp index 083abca0a..399947446 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -126,7 +126,7 @@ void Vertex::assign_sym(const Symbol* sym) { this->sym = sym; } -void Vertex::assign_literal_value(ConstantValue&& literal_value) { +void Vertex::assign_literal_value(std::string&& literal_value) { this->literal_value = std::move(literal_value); } diff --git a/tolk/ast.h b/tolk/ast.h index 6531d50a8..71bc9c40b 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -501,10 +501,10 @@ template<> // note, that TVM doesn't have strings, it has only slices, so "hello" has type slice struct Vertex final : ASTExprLeaf { std::string_view str_val; - ConstantValue literal_value; // value of type `slice`, calculated after type inferring, at constants evaluation + std::string literal_value; // when "some_str" is a standalone string, value of type `slice`, for x{...} Fift output Vertex* mutate() const { return const_cast(this); } - void assign_literal_value(ConstantValue&& literal_value); + void assign_literal_value(std::string&& literal_value); Vertex(SrcLocation loc, std::string_view str_val) : ASTExprLeaf(ast_string_const, loc) @@ -1196,13 +1196,15 @@ struct Vertex final : ASTOtherVararg { TypePtr declared_type; auto get_identifier() const { return children.at(0)->as(); } + bool has_default_value() const { return children.at(1)->type != ast_empty_expression; } + auto get_default_value() const { return child_as_expr(1); } Vertex* mutate() const { return const_cast(this); } void assign_field_ref(StructFieldPtr field_ref); void assign_resolved_type(TypePtr declared_type); - Vertex(SrcLocation loc, V name_identifier, TypePtr declared_type) - : ASTOtherVararg(ast_struct_field, loc, {name_identifier}) + Vertex(SrcLocation loc, V name_identifier, AnyExprV default_value, TypePtr declared_type) + : ASTOtherVararg(ast_struct_field, loc, {name_identifier, default_value}) , declared_type(declared_type) {} }; diff --git a/tolk/compiler-state.cpp b/tolk/compiler-state.cpp index e02d0220a..f5bda29e6 100644 --- a/tolk/compiler-state.cpp +++ b/tolk/compiler-state.cpp @@ -74,4 +74,8 @@ const std::vector& get_all_declared_constants() { return G.all_constants; } +const std::vector& get_all_declared_structs() { + return G.all_structs; +} + } // namespace tolk diff --git a/tolk/compiler-state.h b/tolk/compiler-state.h index 1d166a3ab..b39e62fb3 100644 --- a/tolk/compiler-state.h +++ b/tolk/compiler-state.h @@ -99,6 +99,7 @@ struct CompilerState { std::vector all_get_methods; std::vector all_global_vars; std::vector all_constants; + std::vector all_structs; AllRegisteredSrcFiles all_src_files; bool is_verbosity(int gt_eq) const { return settings.verbosity >= gt_eq; } diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp index ff8dbbce1..a281cfbe3 100644 --- a/tolk/constant-evaluator.cpp +++ b/tolk/constant-evaluator.cpp @@ -137,16 +137,11 @@ static td::RefInt256 parse_nanotons_as_floating_string(SrcLocation loc, std::str return td::make_refint(is_negative ? -result : result); } -static ConstantValue parse_vertex_string_const(V v) { - td::Slice str_slice = td::Slice(v->str_val.data(), v->str_val.size()); - return ConstantValue(td::hex_encode(str_slice)); -} - // given `ton("0.05")` evaluate it to 50000000 // given `stringCrc32("some_str")` evaluate it // etc. // currently, all compile-time functions accept 1 argument, a literal string -static ConstantValue parse_vertex_call_to_compile_time_function(V v, std::string_view f_name) { +static CompileTimeFunctionResult parse_vertex_call_to_compile_time_function(V v, std::string_view f_name) { tolk_assert(v->get_num_args() == 1); // checked by type inferring AnyExprV v_arg = v->get_arg(0)->get_expr(); @@ -163,27 +158,27 @@ static ConstantValue parse_vertex_call_to_compile_time_function(Vloc, str)); + return parse_nanotons_as_floating_string(v_arg->loc, str); } if (f_name == "stringCrc32") { // previously, postfix "..."c - return ConstantValue(td::make_refint(td::crc32(td::Slice{str.data(), str.size()}))); + return td::make_refint(td::crc32(td::Slice{str.data(), str.size()})); } if (f_name == "stringCrc16") { // previously, there was no postfix in FunC, no way to calc at compile-time - return ConstantValue(td::make_refint(td::crc16(td::Slice{str.data(), str.size()}))); + return td::make_refint(td::crc16(td::Slice{str.data(), str.size()})); } if (f_name == "stringSha256") { // previously, postfix "..."H unsigned char hash[32]; digest::hash_str(hash, str.data(), str.size()); - return ConstantValue(td::bits_to_refint(hash, 256, false)); + return td::bits_to_refint(hash, 256, false); } if (f_name == "stringSha256_32") { // previously, postfix "..."h unsigned char hash[32]; digest::hash_str(hash, str.data(), str.size()); - return ConstantValue(td::bits_to_refint(hash, 32, false)); + return td::bits_to_refint(hash, 32, false); } if (f_name == "stringAddressToSlice") { // previously, postfix "..."a @@ -202,7 +197,7 @@ static ConstantValue parse_vertex_call_to_compile_time_function(V(4) << (64 - 3), 3); td::bitstring::bits_store_long_top(data, 3, static_cast(workchain) << (64 - 8), 8); td::bitstring::bits_memcpy(data, 3 + 8, addr.bits().ptr, 0, ton::StdSmcAddress::size()); - return ConstantValue(td::BitSlice{data, sizeof(data)}.to_hex()); + return td::BitSlice{data, sizeof(data)}.to_hex(); } if (f_name == "stringHexToSlice") { // previously, postfix "..."s @@ -211,7 +206,7 @@ static ConstantValue parse_vertex_call_to_compile_time_function(Verror("invalid hex bitstring constant"); } - return ConstantValue(static_cast(str)); + return static_cast(str); } if (f_name == "stringToBase256") { // previously, postfix "..."u @@ -222,149 +217,56 @@ static ConstantValue parse_vertex_call_to_compile_time_function(Verror("too long integer ascii-constant"); } - return ConstantValue(std::move(intval)); + return std::move(intval); } tolk_assert(false); - return ConstantValue(); } -struct ConstantEvaluator { - static bool is_overflow(const td::RefInt256& intval) { - return intval.is_null() || !intval->signed_fits_bits(257); +struct ConstantExpressionChecker { + static void handle_unary_operator(V v) { + visit(v->get_rhs()); } - static ConstantValue handle_unary_operator(V v, const ConstantValue& rhs) { - tolk_assert(rhs.is_int()); // type inferring has passed before, so it's int/bool - td::RefInt256 intval = rhs.as_int(); - - switch (v->tok) { - case tok_minus: - intval = -intval; - break; - case tok_plus: - break; - case tok_bitwise_not: - intval = ~intval; - break; - case tok_logical_not: - intval = td::make_refint(intval == 0 ? -1 : 0); - break; - default: - v->error("not a constant expression"); - } - - if (is_overflow(intval)) { - v->error("integer overflow"); - } - return ConstantValue(std::move(intval)); - } - - static ConstantValue handle_binary_operator(V v, const ConstantValue& lhs, const ConstantValue& rhs) { - tolk_assert(lhs.is_int() && rhs.is_int()); // type inferring has passed before, so they are int/bool - td::RefInt256 lhs_intval = lhs.as_int(); - td::RefInt256 rhs_intval = rhs.as_int(); - td::RefInt256 intval; - - switch (v->tok) { - case tok_minus: - intval = lhs_intval - rhs_intval; - break; - case tok_plus: - intval = lhs_intval + rhs_intval; - break; - case tok_mul: - intval = lhs_intval * rhs_intval; - break; - case tok_div: - intval = lhs_intval / rhs_intval; - break; - case tok_mod: - intval = lhs_intval % rhs_intval; - break; - case tok_lshift: - intval = lhs_intval << static_cast(rhs_intval->to_long()); - break; - case tok_rshift: - intval = lhs_intval >> static_cast(rhs_intval->to_long()); - break; - case tok_bitwise_and: - intval = lhs_intval & rhs_intval; - break; - case tok_bitwise_or: - intval = lhs_intval | rhs_intval; - break; - case tok_bitwise_xor: - intval = lhs_intval ^ rhs_intval; - break; - case tok_eq: - intval = td::make_refint(lhs_intval == rhs_intval ? -1 : 0); - break; - case tok_lt: - intval = td::make_refint(lhs_intval < rhs_intval ? -1 : 0); - break; - case tok_gt: - intval = td::make_refint(lhs_intval > rhs_intval ? -1 : 0); - break; - case tok_leq: - intval = td::make_refint(lhs_intval <= rhs_intval ? -1 : 0); - break; - case tok_geq: - intval = td::make_refint(lhs_intval >= rhs_intval ? -1 : 0); - break; - case tok_neq: - intval = td::make_refint(lhs_intval != rhs_intval ? -1 : 0); - break; - case tok_logical_and: - intval = td::make_refint(lhs_intval != 0 && rhs_intval != 0 ? -1 : 0); - break; - case tok_logical_or: - intval = td::make_refint(lhs_intval != 0 || rhs_intval != 0 ? -1 : 0); - break; - default: - v->error("unsupported binary operator in constant expression"); - } - - if (is_overflow(intval)) { - v->error("integer overflow"); - } - return ConstantValue(std::move(intval)); + static void handle_binary_operator(V v) { + visit(v->get_lhs()); + visit(v->get_rhs()); } // `const a = 1 + b`, we met `b` - static ConstantValue handle_reference(V v) { + static void handle_reference(V v) { GlobalConstPtr const_ref = v->sym->try_as(); if (!const_ref) { v->error("symbol `" + static_cast(v->get_name()) + "` is not a constant"); } - - if (!const_ref->value.initialized()) { // maybe, `b` was already calculated - eval_and_assign_const_init_value(const_ref); // if not, dig recursively into `b` - } - return const_ref->value; } // `const a = ton("0.05")`, we met `ton("0.05")` - static ConstantValue handle_function_call(V v) { + static void handle_function_call(V v) { if (v->fun_maybe && v->fun_maybe->is_compile_time_only()) { - return parse_vertex_call_to_compile_time_function(v, v->fun_maybe->name); + // `ton(local_var)` is denied; it's validated not here, but when replacing its value with a calculated one + return; } v->error("not a constant expression"); } - static ConstantValue visit(AnyExprV v) { - if (auto v_int = v->try_as()) { - return ConstantValue(v_int->intval); + // `const a = (1, 2)`, why not; or it's a default value of a field + static void handle_tensor(V v) { + for (int i = 0; i < v->size(); ++i) { + visit(v->get_item(i)); } - if (auto v_bool = v->try_as()) { - return ConstantValue(v_bool->bool_val ? -1 : 0); + } + + static void visit(AnyExprV v) { + if (v->try_as() || v->try_as() || v->try_as() || v->try_as()) { + return; } if (auto v_unop = v->try_as()) { - return handle_unary_operator(v_unop, visit(v_unop->get_rhs())); + return handle_unary_operator(v_unop); } if (auto v_binop = v->try_as()) { - return handle_binary_operator(v_binop, visit(v_binop->get_lhs()), visit(v_binop->get_rhs())); + return handle_binary_operator(v_binop); } if (auto v_ref = v->try_as()) { return handle_reference(v_ref); @@ -372,50 +274,45 @@ struct ConstantEvaluator { if (auto v_call = v->try_as()) { return handle_function_call(v_call); } + if (auto v_tensor = v->try_as()) { + return handle_tensor(v_tensor); + } if (auto v_par = v->try_as()) { return visit(v_par->get_expr()); } if (auto v_cast_to = v->try_as()) { return visit(v_cast_to->get_expr()); } - if (auto v_string = v->try_as()) { - return parse_vertex_string_const(v_string); + if (auto v_dot = v->try_as(); v_dot && (v_dot->is_target_indexed_access() || v_dot->is_target_fun_ref())) { + return visit(v_dot->get_obj()); } v->error("not a constant expression"); } - // evaluate `2 + 3` into 5 - // type inferring has already passed, to types are correct, `1 + ""` can not occur + // check that `2 + 3` is constant + // type inferring has already passed, so types are correct, `1 + ""` can not occur // if v is not a constant expression like `foo()`, an exception is thrown - static ConstantValue eval_expression_expected_to_be_constant(AnyExprV v) { - return visit(v); + static void check_expression_expected_to_be_constant(AnyExprV v) { + visit(v); } }; -ConstantValue eval_string_const_standalone(AnyExprV v_string) { +void check_expression_is_constant(AnyExprV v_expr) { + ConstantExpressionChecker::check_expression_expected_to_be_constant(v_expr); +} + +std::string eval_string_const_standalone(AnyExprV v_string) { auto v = v_string->try_as(); tolk_assert(v); - return parse_vertex_string_const(v); + td::Slice str_slice = td::Slice(v->str_val.data(), v->str_val.size()); + return td::hex_encode(str_slice); } -ConstantValue eval_call_to_compile_time_function(AnyExprV v_call) { +CompileTimeFunctionResult eval_call_to_compile_time_function(AnyExprV v_call) { auto v = v_call->try_as(); tolk_assert(v && v->fun_maybe->is_compile_time_only()); return parse_vertex_call_to_compile_time_function(v, v->fun_maybe->name); } -// evaluate `2 + 3` into `5` -// if v is not a constant expression like `gVar`, an exception is thrown -ConstantValue eval_expression_expected_to_be_constant(AnyExprV v) { - return ConstantEvaluator::eval_expression_expected_to_be_constant(v); -} - -// evaluate `const a = 2 + 3` into `const a = 5` -// recursive initializers `const a = b; const b = a` don't exist, checked on type inferring -// if init_value is not a constant expression like `const a = foo()`, an exception is thrown -void eval_and_assign_const_init_value(GlobalConstPtr const_ref) { - ConstantValue init_value = ConstantEvaluator::eval_expression_expected_to_be_constant(const_ref->init_value); - const_ref->mutate()->assign_const_value(std::move(init_value)); -} } // namespace tolk diff --git a/tolk/constant-evaluator.h b/tolk/constant-evaluator.h index 1a7f364fb..b7a9fc7fe 100644 --- a/tolk/constant-evaluator.h +++ b/tolk/constant-evaluator.h @@ -22,34 +22,10 @@ namespace tolk { -class ConstantValue { - std::variant< - td::RefInt256, // is set for int, intN, coins, bool - std::string // is set for slice, bytesN - > value; - -public: - ConstantValue() = default; // by default, not initialized - - explicit ConstantValue(int value) - : value(td::make_refint(value)) {} - explicit ConstantValue(td::RefInt256 value) - : value(std::move(value)) {} - explicit ConstantValue(std::string value) - : value(std::move(value)) {} - - bool initialized() const { return is_slice() || std::get(value).not_null(); } - - bool is_int() const { return std::holds_alternative(value); } - bool is_slice() const { return std::holds_alternative(value); } - - td::RefInt256 as_int() const { return std::get(value); } - const std::string& as_slice() const { return std::get(value); } -}; - -ConstantValue eval_string_const_standalone(AnyExprV v_string); -ConstantValue eval_call_to_compile_time_function(AnyExprV v_call); -ConstantValue eval_expression_expected_to_be_constant(AnyExprV v); -void eval_and_assign_const_init_value(GlobalConstPtr const_ref); +typedef std::variant CompileTimeFunctionResult; + +void check_expression_is_constant(AnyExprV v_expr); +std::string eval_string_const_standalone(AnyExprV v_string); +CompileTimeFunctionResult eval_call_to_compile_time_function(AnyExprV v_call); } // namespace tolk diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 8f4a8ddab..cd6e38f0c 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -20,7 +20,6 @@ #include "ast-visitor.h" #include "type-system.h" #include "common/refint.h" -#include "constant-evaluator.h" #include "smart-casts-cfg.h" #include @@ -876,15 +875,8 @@ std::vector pre_compile_symbol(SrcLocation loc, const Symbol* sym, Co return local_ir_idx; } if (GlobalConstPtr const_ref = sym->try_as()) { - if (const_ref->value.is_int()) { - std::vector rvect = code.create_tmp_var(TypeDataInt::create(), loc, "(glob-const)"); - code.emplace_back(loc, Op::_IntConst, rvect, const_ref->value.as_int()); - return rvect; - } else { - std::vector rvect = code.create_tmp_var(TypeDataSlice::create(), loc, "(glob-const)"); - code.emplace_back(loc, Op::_SliceConst, rvect, const_ref->value.as_slice()); - return rvect; - } + tolk_assert(lval_ctx == nullptr); + return pre_compile_expr(const_ref->init_value, code, const_ref->declared_type); } if (FunctionPtr fun_ref = sym->try_as()) { std::vector rvect = code.create_tmp_var(fun_ref->inferred_full_type, loc, "(glob-var-fun)"); @@ -1341,7 +1333,10 @@ static std::vector process_object_literal(V v, Co break; } } - tolk_assert(v_init_val); + if (!v_init_val) { + tolk_assert(field_ref->has_default_value()); + v_init_val = field_ref->default_value; // it may be a complex expression, but it's constant, checked earlier + } tensor_items.push_back(v_init_val); target_types.push_back(field_ref->declared_type); } @@ -1359,9 +1354,8 @@ static std::vector process_int_const(V v, CodeBlob& co } static std::vector process_string_const(V v, CodeBlob& code, TypePtr target_type) { - tolk_assert(v->literal_value.is_slice()); std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(str-const)"); - code.emplace_back(v->loc, Op::_SliceConst, rvect, v->literal_value.as_slice()); + code.emplace_back(v->loc, Op::_SliceConst, rvect, v->literal_value); return transition_to_target_type(std::move(rvect), code, target_type, v); } diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 46de607c8..2b87d77cb 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -747,6 +747,16 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } } } + + // given struct field `a: int = 2 + 3` check types within its default_value + void start_visiting_field_default(StructFieldPtr field_ref) { + parent::visit(field_ref->default_value); + + TypePtr inferred_type = field_ref->default_value->inferred_type; + if (!field_ref->declared_type->can_rhs_be_assigned(inferred_type)) { + throw ParseError(field_ref->loc, "can not assign " + to_string(inferred_type) + " to " + to_string(field_ref->declared_type)); + } + } }; void pipeline_check_inferred_types() { @@ -756,6 +766,13 @@ void pipeline_check_inferred_types() { for (GlobalConstPtr const_ref : get_all_declared_constants()) { visitor.start_visiting_constant(const_ref); } + for (StructPtr struct_ref : get_all_declared_structs()) { + for (StructFieldPtr field_ref : struct_ref->fields) { + if (field_ref->has_default_value()) { + visitor.start_visiting_field_default(field_ref); + } + } + } } } // namespace tolk diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp index c64f7bff5..a5e68c6bf 100644 --- a/tolk/pipe-constant-folding.cpp +++ b/tolk/pipe-constant-folding.cpp @@ -18,6 +18,7 @@ #include "ast.h" #include "ast-replacer.h" #include "type-system.h" +#include "constant-evaluator.h" /* * This pipe is supposed to do constant folding, like replacing `2 + 3` with `5`. @@ -57,8 +58,8 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { return inner; } - static V create_string_const(SrcLocation loc, ConstantValue&& literal_value) { - auto v_string = createV(loc, literal_value.as_slice()); + static V create_string_const(SrcLocation loc, std::string&& literal_value) { + auto v_string = createV(loc, literal_value); v_string->assign_inferred_type(TypeDataSlice::create()); v_string->assign_literal_value(std::move(literal_value)); v_string->assign_rvalue_true(); @@ -112,14 +113,12 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { // replace `ton("0.05")` with 50000000 / `stringCrc32("some_str")` with calculated value / etc. if (v->fun_maybe && v->fun_maybe->is_compile_time_only()) { - ConstantValue value = eval_call_to_compile_time_function(v); - if (value.is_int()) { - return create_int_const(v->loc, value.as_int()); + CompileTimeFunctionResult value = eval_call_to_compile_time_function(v); + if (std::holds_alternative(value)) { + return create_int_const(v->loc, std::move(std::get(value))); + } else { + return create_string_const(v->loc, std::move(std::get(value))); } - if (value.is_slice()) { - return create_string_const(v->loc, std::move(value)); - } - tolk_assert(false); } return v; @@ -128,7 +127,7 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { AnyExprV replace(V v) override { // when "some_str" occurs as a standalone constant (not inside `stringCrc32("some_str")`, // it's actually a slice - ConstantValue literal_value = eval_string_const_standalone(v); + std::string literal_value = eval_string_const_standalone(v); v->mutate()->assign_literal_value(std::move(literal_value)); return v; @@ -140,10 +139,7 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { // replace `2 + 3 => ...` with `5 => ...` // non-constant expressions like `foo() => ...` fire an error here if (v->pattern_kind == MatchArmKind::const_expression && v->get_pattern_expr()->type != ast_int_const) { - ConstantValue value = eval_expression_expected_to_be_constant(v->get_pattern_expr()); - tolk_assert(value.is_int()); - AnyExprV v_new_pattern = create_int_const(v->get_pattern_expr()->loc, value.as_int()); - v->mutate()->assign_resolved_pattern(MatchArmKind::const_expression, nullptr, v_new_pattern); + check_expression_is_constant(v->get_pattern_expr()); } return v; @@ -153,15 +149,32 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function(); } + + // used to replace `ton("0.05")` and other compile-time functions inside fields defaults, etc. + AnyExprV replace_in_expression(AnyExprV init_value) { + return parent::replace(init_value); + } }; void pipeline_constant_folding() { - // here (after type inferring) evaluate `const a = 2 + 3` into `5` + ConstantFoldingReplacer replacer; + + // here (after type inferring) check that `const a = 2 + 3` is a valid constant expression // non-constant expressions like `const a = foo()` fire an error here + // also, replace `const a = ton("0.05")` with `const a = 50000000` for (GlobalConstPtr const_ref : get_all_declared_constants()) { - // for `const a = b`, `b` could be already calculated while calculating `a` - if (!const_ref->value.initialized()) { - eval_and_assign_const_init_value(const_ref); + check_expression_is_constant(const_ref->init_value); + AnyExprV replaced = replacer.replace_in_expression(const_ref->init_value); + const_ref->mutate()->assign_init_value(replaced); + } + // do the same for default values of struct fields, they must be constant expressions + for (StructPtr struct_ref : get_all_declared_structs()) { + for (StructFieldPtr field_ref : struct_ref->fields) { + if (field_ref->has_default_value()) { + check_expression_is_constant(field_ref->default_value); + AnyExprV replaced = replacer.replace_in_expression(field_ref->default_value); + field_ref->mutate()->assign_default_value(replaced); + } } } diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 9cca8a292..ee2e0f27b 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -1212,7 +1212,7 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v_field, v_field->get_init_val()); } for (StructFieldPtr field_ref : struct_ref->fields) { - if (!(occurred_mask & (1ULL << field_ref->field_idx))) { + if (!(occurred_mask & (1ULL << field_ref->field_idx)) && !field_ref->has_default_value()) { fire(cur_f, v->get_body()->loc, "field `" + field_ref->name + "` missed in initialization of struct `" + struct_ref->name + "`"); } } @@ -1439,6 +1439,12 @@ class InferTypesAndCallsAndFieldsVisitor final { infer_any_expr(const_ref->init_value, std::move(const_flow), false, const_ref->declared_type); const_ref->mutate()->assign_inferred_type(const_ref->declared_type == nullptr ? const_ref->init_value->inferred_type : const_ref->declared_type); } + + // given struct field `a: int = 2 + 3` infer that default value is int, assign inferred_type to all nodes + void start_visiting_field_default(StructFieldPtr field_ref) { + FlowContext field_flow; + infer_any_expr(field_ref->default_value, std::move(field_flow), false, field_ref->declared_type); + } }; class LaunchInferTypesAndMethodsOnce final { @@ -1510,7 +1516,15 @@ void pipeline_infer_types_and_calls_and_fields() { visitor.start_visiting_constant(const_ref); } } - // (later, at constant folding, `const a = 2 + 3` will be evaluated to 5) + + // infer types for default values in structs + for (StructPtr struct_ref : get_all_declared_structs()) { + for (StructFieldPtr field_ref : struct_ref->fields) { + if (field_ref->has_default_value()) { + visitor.start_visiting_field_default(field_ref); + } + } + } } void pipeline_infer_types_and_calls_and_fields(FunctionPtr fun_ref) { diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index 718547198..6bb93d55a 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -19,7 +19,6 @@ #include "src-file.h" #include "ast.h" #include "compiler-state.h" -#include "constant-evaluator.h" #include "generics-helpers.h" #include "td/utils/crypto.h" #include "type-system.h" @@ -116,9 +115,6 @@ static const GenericsDeclaration* construct_genericTs(V v_li static void register_constant(V v) { GlobalConstData* c_sym = new GlobalConstData(static_cast(v->get_identifier()->name), v->loc, v->declared_type, v->get_init_value()); - // init value of constant is not evaluated here - // at first, it will be type checked (in type inference pipe) - // then, at constant folding pipe, `const a = 2 + 3` will be evaluated to 5 G.symtable.add_global_const(c_sym); G.all_constants.push_back(c_sym); @@ -147,7 +143,8 @@ static void register_struct(V v) { fields.reserve(v_body->get_num_fields()); for (int i = 0; i < v_body->get_num_fields(); ++i) { auto v_field = v_body->get_field(i); - StructFieldData* field_ref = new StructFieldData(static_cast(v_field->get_identifier()->name), v_field->loc, i, v_field->declared_type); + AnyExprV default_value = v_field->has_default_value() ? v_field->get_default_value() : nullptr; + StructFieldData* field_ref = new StructFieldData(static_cast(v_field->get_identifier()->name), v_field->loc, i, v_field->declared_type, default_value); fields.emplace_back(field_ref); v_field->mutate()->assign_field_ref(field_ref); } @@ -155,6 +152,7 @@ static void register_struct(V v) { StructData* s_sym = new StructData(static_cast(v->get_identifier()->name), v->loc, std::move(fields)); G.symtable.add_struct(s_sym); + G.all_structs.push_back(s_sym); v->mutate()->assign_struct_ref(s_sym); } diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 304a3497d..545cf50fc 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -422,6 +422,11 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { // `const a = b`, resolve `b` parent::visit(v->get_init_value()); } + + void start_visiting_struct_field(V v) { + // field `a: int = C`, resolve `C` + parent::visit(v->get_default_value()); + } }; NameAndScopeResolver AssignSymInsideFunctionVisitor::current_scope; @@ -458,6 +463,7 @@ void pipeline_resolve_identifiers_and_assign_symbols() { auto v_body = v_struct->get_struct_body(); for (int i = 0; i < v_body->get_num_fields(); ++i) { auto v_field = v_body->get_field(i); + visitor.start_visiting_struct_field(v_field); v_field->mutate()->assign_resolved_type(v_struct->struct_ref->get_field(i)->declared_type); } } diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index ccd56a011..97e48f7fc 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -97,8 +97,8 @@ void GlobalConstData::assign_inferred_type(TypePtr inferred_type) { this->inferred_type = inferred_type; } -void GlobalConstData::assign_const_value(ConstantValue&& value) { - this->value = std::move(value); +void GlobalConstData::assign_init_value(AnyExprV init_value) { + this->init_value = init_value; } void LocalVarData::assign_ir_idx(std::vector&& ir_idx) { @@ -121,6 +121,10 @@ void StructFieldData::assign_resolved_type(TypePtr declared_type) { this->declared_type = declared_type; } +void StructFieldData::assign_default_value(AnyExprV default_value) { + this->default_value = default_value; +} + void StructData::assign_resolved_type(TypePtr struct_type) { this->struct_type = struct_type; } diff --git a/tolk/symtable.h b/tolk/symtable.h index 269aee90f..9ab7f7b96 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -18,7 +18,6 @@ #include "src-file.h" #include "fwd-declarations.h" -#include "constant-evaluator.h" #include "crypto/common/refint.h" #include #include @@ -200,7 +199,6 @@ struct GlobalVarData final : Symbol { struct GlobalConstData final : Symbol { AnyExprV init_value; - ConstantValue value; TypePtr declared_type; // `const a: int = ...`; nullptr for `const a = ...` TypePtr inferred_type = nullptr; // filled at type inferring pass @@ -213,7 +211,7 @@ struct GlobalConstData final : Symbol { GlobalConstData* mutate() const { return const_cast(this); } void assign_resolved_type(TypePtr declared_type); void assign_inferred_type(TypePtr inferred_type); - void assign_const_value(ConstantValue&& value); + void assign_init_value(AnyExprV init_value); }; struct AliasDefData final : Symbol { @@ -231,14 +229,19 @@ struct AliasDefData final : Symbol { struct StructFieldData final : Symbol { int field_idx; TypePtr declared_type; + AnyExprV default_value; // nullptr if no default + + bool has_default_value() const { return default_value != nullptr; } StructFieldData* mutate() const { return const_cast(this); } void assign_resolved_type(TypePtr declared_type); + void assign_default_value(AnyExprV default_value); - StructFieldData(std::string name, SrcLocation loc, int field_idx, TypePtr declared_type) + StructFieldData(std::string name, SrcLocation loc, int field_idx, TypePtr declared_type, AnyExprV default_value) : Symbol(std::move(name), loc) , field_idx(field_idx) - , declared_type(declared_type) { + , declared_type(declared_type) + , default_value(default_value) { } }; From 09019fea5d924d8dbb424867d3f453864ca4bd29 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 20 Feb 2025 19:10:08 +0400 Subject: [PATCH 230/388] [Tolk] Refactor parsing types, introduce AST type nodes Until this MR, parsing any type from tokens immediately resulted in TypePtr. This MR introduces an intermediate AST representation for types, which are resolved at a later step. It's a necessary refactoring towards generics. --- .../tests/invalid-semantics/err-4517.tolk | 2 +- .../tests/invalid-syntax/err-3457.tolk | 16 +- tolk-tester/tests/match-by-expr-tests.tolk | 13 +- tolk-tester/tests/nullable-tensors.tolk | 6 +- tolk-tester/tests/smart-cast-tests.tolk | 8 +- tolk-tester/tests/struct-tests.tolk | 2 +- tolk-tester/tests/union-types-tests.tolk | 38 +- tolk-tester/tests/var-apply-tests.tolk | 2 +- tolk/CMakeLists.txt | 1 + tolk/ast-from-tokens.cpp | 183 +++++-- tolk/ast-replacer.h | 12 +- tolk/ast-replicator.h | 105 ++-- tolk/ast-stringifier.h | 85 ++-- tolk/ast-visitor.h | 22 +- tolk/ast.cpp | 66 +-- tolk/ast.h | 322 ++++++++----- tolk/fwd-declarations.h | 2 + tolk/generics-helpers.cpp | 135 ++---- tolk/generics-helpers.h | 4 +- tolk/lexer.cpp | 9 - tolk/lexer.h | 1 - tolk/pipe-ast-to-legacy.cpp | 42 +- tolk/pipe-calc-rvalue-lvalue.cpp | 2 +- tolk/pipe-check-inferred-types.cpp | 66 +-- tolk/pipe-check-rvalue-lvalue.cpp | 6 - tolk/pipe-constant-folding.cpp | 12 +- tolk/pipe-infer-types-and-calls.cpp | 83 ++-- tolk/pipe-optimize-boolean-expr.cpp | 2 +- tolk/pipe-refine-lvalue-for-mutate.cpp | 4 +- tolk/pipe-register-symbols.cpp | 61 +-- tolk/pipe-resolve-identifiers.cpp | 231 ++------- tolk/pipe-resolve-types.cpp | 455 ++++++++++++++++++ tolk/pipeline.h | 3 + tolk/smart-casts-cfg.cpp | 8 +- tolk/symtable.cpp | 10 +- tolk/symtable.h | 80 ++- tolk/tolk.cpp | 3 +- tolk/tolk.h | 6 + tolk/type-system.cpp | 278 +---------- tolk/type-system.h | 68 +-- 40 files changed, 1328 insertions(+), 1126 deletions(-) create mode 100644 tolk/pipe-resolve-types.cpp diff --git a/tolk-tester/tests/invalid-semantics/err-4517.tolk b/tolk-tester/tests/invalid-semantics/err-4517.tolk index f84e3ac1b..5497ebb41 100644 --- a/tolk-tester/tests/invalid-semantics/err-4517.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4517.tolk @@ -10,5 +10,5 @@ fun f(x: int): int { /** @compilation_should_fail -@stderr symbol `g` is not a constant +@stderr unknown type name `g` */ diff --git a/tolk-tester/tests/invalid-syntax/err-3457.tolk b/tolk-tester/tests/invalid-syntax/err-3457.tolk index 8cfcf8e6c..44c1b8954 100644 --- a/tolk-tester/tests/invalid-syntax/err-3457.tolk +++ b/tolk-tester/tests/invalid-syntax/err-3457.tolk @@ -1,4 +1,5 @@ type asdf = int; +const gcon = 10; fun main() { match (10) { @@ -7,12 +8,21 @@ fun main() { var asdf = 5; match (10) { - asdf => 2, // it's match by expression now + asdf => 2, // also match by type }; + + match (10) { + gcon => 2, // match by expression (it's constant) + }; + + var ten = 10; + match (10) { + ten => 10, // not a constant, tries to lookup a type, fails + } } /** @compilation_should_fail -@stderr symbol `asdf` is not a constant -@stderr asdf => 2 +@stderr unknown type name `ten` +@stderr ten => 10 */ diff --git a/tolk-tester/tests/match-by-expr-tests.tolk b/tolk-tester/tests/match-by-expr-tests.tolk index 80287c9bb..7941b4600 100644 --- a/tolk-tester/tests/match-by-expr-tests.tolk +++ b/tolk-tester/tests/match-by-expr-tests.tolk @@ -117,7 +117,18 @@ fun test11(x: bool) { } -fun main() {} +type asdf = int; + +fun main() { + match (10) { + asdf => 1, // it's match by type + }; + + var asdf = 5; + return match (10) { + asdf => 2, // also match by type, regardless of local var + }; +} /** @testcase | 101 | 1 | 100 diff --git a/tolk-tester/tests/nullable-tensors.tolk b/tolk-tester/tests/nullable-tensors.tolk index c1de8c0b7..41b3d62b8 100644 --- a/tolk-tester/tests/nullable-tensors.tolk +++ b/tolk-tester/tests/nullable-tensors.tolk @@ -519,14 +519,14 @@ fun main(){} @testcase | 132 | | -1 0 -1 0 777 (null) (null) -1 0 0 @testcase | 133 | | 60 @testcase | 134 | | 11 21 129 -@testcase | 135 | | [ 10 ] [ (null) ] (null) 777 10 146 (null) 146 (null) 0 777 10 143 (null) 143 (null) 0 777 0 0 -1 0 0 -1 0 0 -1 777 0 -1 0 0 -1 0 +@testcase | 135 | | [ 10 ] [ (null) ] (null) 777 10 143 (null) 143 (null) 0 777 10 144 (null) 144 (null) 0 777 0 0 -1 0 0 -1 0 0 -1 777 0 -1 0 0 -1 0 @testcase | 136 | 9 | 9 0 @testcase | 136 | null | (null) -1 -@testcase | 140 | 8 9 | 8 9 144 (null) (null) 0 +@testcase | 140 | 8 9 | 8 9 145 (null) (null) 0 @testcase | 141 | | (null) 10 @testcase | 142 | | 3 3 1 2 @testcase | 143 | -1 0 | 1 2 1 3 777 (null) (null) (null) (null) 0 -@testcase | 143 | 0 -1 | 1 (null) 0 3 777 1 2 1 3 145 +@testcase | 143 | 0 -1 | 1 (null) 0 3 777 1 2 1 3 146 @fif_codegen """ diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index 77374bdce..aac1028f3 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -727,16 +727,16 @@ fun main(x: int?): int { @testcase | 139 | | 16 @testcase | 140 | 5 | 25 @testcase | 141 | | 1 2 -@testcase | 142 | | 5 3 (null) (null) 0 132 3 (null) (null) 0 +@testcase | 142 | | 5 3 (null) (null) 0 133 3 (null) (null) 0 @testcase | 143 | | 10 11 (null) 10 11 (null) (null) 0 @testcase | 144 | | 10 11 (null) 10 11 (null) (null) 0 -@testcase | 145 | | 5 3 (null) (null) 0 132 3 (null) (null) (null) (null) 0 3 (null) (null) 0 -@testcase | 146 | | 3 4 5 3 4 5 131 +@testcase | 145 | | 5 3 (null) (null) 0 133 3 (null) (null) (null) (null) 0 3 (null) (null) 0 +@testcase | 146 | | 3 4 5 3 4 5 132 @testcase | 147 | | (null) (null) 100 (null) 100 (null) (null) 0 @testcase | 158 | | 123 10 123 5 @testcase | 160 | | 101 109 @testcase | 161 | 9 9 | (null) (null) 0 (null) (null) -@testcase | 161 | 19 0 | 19 0 133 19 0 +@testcase | 161 | 19 0 | 19 0 129 19 0 @stderr warning: expression of type `int` can never be `null`, this condition is always true @stderr warning: unreachable code diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index 4a676a3db..8dd2f3455 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -506,7 +506,7 @@ type PointAlias = Point; @testcase | 126 | | 118 @testcase | 127 | | 0 0 131 131 777 -1 -1 0 0 777 0 0 138 777 0 0 777 (null) (null) 0 @testcase | 128 | -1 | (null) (null) 131 -1 0 0 777 (null) (null) 131 -1 0 0 -@testcase | 128 | 0 | 1 2 129 0 -1 0 777 0 131 138 0 -1 0 +@testcase | 128 | 0 | 1 2 130 0 -1 0 777 0 131 138 0 -1 0 @testcase | 129 | | (null) @testcase | 130 | -1 | 140 777 (null) 140 777 0 777 0 0 @testcase | 130 | 0 | 139 777 4 1 777 139 777 0 0 diff --git a/tolk-tester/tests/union-types-tests.tolk b/tolk-tester/tests/union-types-tests.tolk index 8cd97f941..0bc88c366 100644 --- a/tolk-tester/tests/union-types-tests.tolk +++ b/tolk-tester/tests/union-types-tests.tolk @@ -791,11 +791,11 @@ fun main() { @testcase | 105 | | 5 5 1 5 1 5 @testcase | 106 | 1 | (null) 0 @testcase | 106 | 2 | 2 1 -@testcase | 107 | 1 | (null) 2 3 131 (null) 2 3 131 -@testcase | 107 | 3 | 6 7 8 132 6 7 8 132 -@testcase | 108 | | (null) 2 3 131 (null) (null) (null) 0 (null) 2 3 131 (null) 2 3 131 (null) (null) (null) 0 -@testcase | 109 | | 6 7 8 132 0 (null) -1 (null) (null) (null) 0 -1 (null) (null) (null) 0 -@testcase | 110 | | 1 (null) (null) 0 (null) (null) 0 134 +@testcase | 107 | 1 | (null) 2 3 129 (null) 2 3 129 +@testcase | 107 | 3 | 6 7 8 130 6 7 8 130 +@testcase | 108 | | (null) 2 3 129 (null) (null) (null) 0 (null) 2 3 129 (null) 2 3 129 (null) (null) (null) 0 +@testcase | 109 | | 6 7 8 130 0 (null) -1 (null) (null) (null) 0 -1 (null) (null) (null) 0 +@testcase | 110 | | 1 (null) (null) 0 (null) (null) 0 135 @testcase | 120 | | 5 @testcase | 121 | | -1 0 0 -1 -1 0 @testcase | 122 | 0 0 1 | 36 @@ -807,21 +807,21 @@ fun main() { @testcase | 125 | 1 1 | 42 @testcase | 126 | | 42 @testcase | 127 | | 1 2 5 1 -@testcase | 128 | | (null) (null) 0 777 (null) 2 1 777 3 4 131 +@testcase | 128 | | (null) (null) 0 777 (null) 2 1 777 3 4 129 @testcase | 129 | | (null) (null) 0 (null) 5 1 -@testcase | 130 | 0 | (null) 5 1 777 (null) (null) 0 777 1 2 131 777 (null) [ 6 ] 135 777 (null) (null) 0 -@testcase | 130 | -1 | (null) (null) 0 777 (null) (null) 0 777 1 2 131 777 (null) [ 6 ] 135 777 (null) (null) 0 -@testcase | 131 | | (null) 777 5 777 1 2 777 (null) 777 5 777 (null) (null) 0 777 1 2 131 -@testcase | 132 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 131 777 (null) (null) (null) 0 -@testcase | 133 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 131 777 (null) (null) (null) 0 -@testcase | 134 | | (null) 5 1 777 (null) 5 1 777 1 2 131 777 1 2 131 777 (null) 5 42 777 5 42 777 5 42 -@testcase | 135 | | (null) 5 1 777 (null) (null) 1 2 131 777 (null) (null) 1 2 131 777 (null) (null) 0 -@testcase | 136 | | 1 2 131 777 1 2 140 777 1 2 141 777 (null) 1 2 138 +@testcase | 130 | 0 | (null) 5 1 777 (null) (null) 0 777 1 2 129 777 (null) [ 6 ] 136 777 (null) (null) 0 +@testcase | 130 | -1 | (null) (null) 0 777 (null) (null) 0 777 1 2 129 777 (null) [ 6 ] 136 777 (null) (null) 0 +@testcase | 131 | | (null) 777 5 777 1 2 777 (null) 777 5 777 (null) (null) 0 777 1 2 129 +@testcase | 132 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 129 777 (null) (null) (null) 0 +@testcase | 133 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 129 777 (null) (null) (null) 0 +@testcase | 134 | | (null) 5 1 777 (null) 5 1 777 1 2 129 777 1 2 129 777 (null) 5 42 777 5 42 777 5 42 +@testcase | 135 | | (null) 5 1 777 (null) (null) 1 2 129 777 (null) (null) 1 2 129 777 (null) (null) 0 +@testcase | 136 | | 1 2 129 777 1 2 140 777 1 2 141 777 (null) 1 2 142 @testcase | 137 | 1 2 | 1 2 777 1 2 777 1 2 777 [ 1 2 ] -@testcase | 138 | | 1 (null) (null) (null) (null) 0 2 149 777 1 (null) 0 2 143 777 1 5 1 2 143 777 1 (null) 5 1 2 144 +@testcase | 138 | | 1 (null) (null) (null) (null) 0 2 146 777 1 (null) 0 2 147 777 1 5 1 2 147 777 1 (null) 5 1 2 148 @testcase | 139 | | 1 (null) (null) (null) (null) 0 2 777 1 (null) 0 2 777 1 5 1 2 777 1 (null) 5 1 2 @testcase | 140 | 5 | 5 42 -@testcase | 140 | 15 | 15 129 +@testcase | 140 | 15 | 15 131 @testcase | 140 | 90 | 90 49 @testcase | 141 | 7 8 | 7 7 1 8 8 1 @testcase | 141 | null null | (null) (null) 0 (null) (null) 0 @@ -842,9 +842,9 @@ fun main() { @testcase | 154 | | 100 1 @testcase | 155 | | 5 1 5 1 @testcase | 156 | 1 2 -1 | 2 44 2 2 44 2 2 44 2 44 2 2 2 -@testcase | 157 | 1 | (null) 0 (null) 145 777 1 2 131 777 145 145 777 0 0 -1 0 -1 0 0 777 -1 0 -@testcase | 157 | 0 | (null) 0 (null) 145 777 (null) (null) 145 777 0 0 777 0 0 -1 0 0 -1 0 777 0 -1 -@testcase | 158 | | (null) (null) 145 (null) (null) 145 (null) 0 (null) +@testcase | 157 | 1 | (null) 0 (null) 149 777 1 2 129 777 149 149 777 0 0 -1 0 -1 0 0 777 -1 0 +@testcase | 157 | 0 | (null) 0 (null) 149 777 (null) (null) 149 777 0 0 777 0 0 -1 0 0 -1 0 777 0 -1 +@testcase | 158 | | (null) (null) 149 (null) (null) 149 (null) 0 (null) @testcase | 159 | 0 4 | 456 @testcase | 159 | null 0 | 123 @testcase | 160 | | 10 0 diff --git a/tolk-tester/tests/var-apply-tests.tolk b/tolk-tester/tests/var-apply-tests.tolk index e4318dc15..aec77391f 100644 --- a/tolk-tester/tests/var-apply-tests.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -264,7 +264,7 @@ fun main() {} @testcase | 112 | -1 -10 -20 | -1 @testcase | 113 | 0 | 12 12 @testcase | 113 | -1 | 12 -1 -@testcase | 114 | -1 | 2 3 130 +@testcase | 114 | -1 | 2 3 131 @testcase | 114 | 0 | (null) 12 1 @testcase | 115 | | 7 6 7 */ diff --git a/tolk/CMakeLists.txt b/tolk/CMakeLists.txt index c4abfec04..c6e992c27 100644 --- a/tolk/CMakeLists.txt +++ b/tolk/CMakeLists.txt @@ -9,6 +9,7 @@ set(TOLK_SOURCE pipe-discover-parse-sources.cpp pipe-register-symbols.cpp pipe-resolve-identifiers.cpp + pipe-resolve-types.cpp pipe-calc-rvalue-lvalue.cpp pipe-infer-types-and-calls.cpp pipe-check-inferred-types.cpp diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 939dd799c..36c657c26 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -76,12 +76,12 @@ static void fire_error_mix_and_or_no_parenthesis(SrcLocation loc, std::string_vi // that's why if rhs->type == ast_binary_operator, it's not surrounded by parenthesis) static void diagnose_bitwise_precedence(SrcLocation loc, std::string_view operator_name, AnyExprV lhs, AnyExprV rhs) { // handle "flags & 0xFF != 0" (rhs = "0xFF != 0") - if (rhs->type == ast_binary_operator && is_comparison_binary_op(rhs->as()->tok)) { + if (rhs->kind == ast_binary_operator && is_comparison_binary_op(rhs->as()->tok)) { fire_error_lower_precedence(loc, operator_name, rhs->as()->operator_name); } // handle "0 != flags & 0xFF" (lhs = "0 != flags") - if (lhs->type == ast_binary_operator && is_comparison_binary_op(lhs->as()->tok)) { + if (lhs->kind == ast_binary_operator && is_comparison_binary_op(lhs->as()->tok)) { fire_error_lower_precedence(loc, operator_name, lhs->as()->operator_name); } } @@ -105,21 +105,22 @@ static void diagnose_and_or_precedence(SrcLocation loc, AnyExprV lhs, TokenType // diagnose "a << 8 + 1" (equivalent to "a << 9", probably unexpected) static void diagnose_addition_in_bitshift(SrcLocation loc, std::string_view bitshift_operator_name, AnyExprV rhs) { - if (rhs->type == ast_binary_operator && is_add_or_sub_binary_op(rhs->as()->tok)) { + if (rhs->kind == ast_binary_operator && is_add_or_sub_binary_op(rhs->as()->tok)) { fire_error_lower_precedence(loc, bitshift_operator_name, rhs->as()->operator_name); } } // replace (a == null) and similar to ast_is_type_operator(a, null) (as if `a is null` was written) static AnyExprV maybe_replace_eq_null_with_isNull_check(V v) { - bool has_null = v->get_lhs()->type == ast_null_keyword || v->get_rhs()->type == ast_null_keyword; + bool has_null = v->get_lhs()->kind == ast_null_keyword || v->get_rhs()->kind == ast_null_keyword; bool replace = has_null && (v->tok == tok_eq || v->tok == tok_neq); if (!replace) { return v; } - AnyExprV v_nullable = v->get_lhs()->type == ast_null_keyword ? v->get_rhs() : v->get_lhs(); - return createV(v->loc, v_nullable, TypeDataNullLiteral::create(), v->tok == tok_neq); + AnyExprV v_nullable = v->get_lhs()->kind == ast_null_keyword ? v->get_rhs() : v->get_lhs(); + AnyTypeV rhs_null_type = createV(v->loc, "null"); + return createV(v->loc, v_nullable, rhs_null_type, v->tok == tok_neq); } // parse `123` / `0xFF` / `0b10001` to td::RefInt256 @@ -142,11 +143,116 @@ static td::RefInt256 parse_tok_int_const(std::string_view text) { -/* - * - * PARSE SOURCE - * - */ +// -------------------------------------------- +// parsing type from tokens +// +// here we implement parsing types (mostly after colon) to AnyTypeV +// example: `var v: int` is leaf "int" +// example: `var v: (User?, [cell])` is tensor(nullable(leaf "User"), brackets(leaf "cell")) +// +// later, after all symbols are registered, types are resolved to TypePtr, see pipe-resolve-types.cpp +// + +static AnyTypeV parse_type_expression(Lexer& lex); + +static std::vector parse_nested_type_list(Lexer& lex, TokenType tok_op, const char* s_op, TokenType tok_cl, const char* s_cl) { + lex.expect(tok_op, s_op); + std::vector sub_types; + while (true) { + if (lex.tok() == tok_cl) { // empty lists allowed + lex.next(); + break; + } + + sub_types.emplace_back(parse_type_expression(lex)); + if (lex.tok() == tok_comma) { + lex.next(); + } else if (lex.tok() != tok_cl) { + lex.unexpected(s_cl); + } + } + return sub_types; +} + +static std::vector parse_nested_type_list_in_parenthesis(Lexer& lex) { + return parse_nested_type_list(lex, tok_oppar, "`(`", tok_clpar, "`)` or `,`"); +} + +static AnyTypeV parse_simple_type(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + switch (lex.tok()) { + case tok_self: + case tok_identifier: + case tok_null: { + std::string_view text = lex.cur_str(); + lex.next(); + return createV(loc, text); + } + case tok_oppar: { + return createV(loc, parse_nested_type_list_in_parenthesis(lex)); + } + case tok_opbracket: { + return createV(loc, parse_nested_type_list(lex, tok_opbracket, "`[`", tok_clbracket, "`]` or `,`")); + } + default: + lex.unexpected(""); + } +} + +static AnyTypeV parse_type_nullable(Lexer& lex) { + AnyTypeV result = parse_simple_type(lex); + + if (lex.tok() == tok_question) { + lex.next(); + result = createV(result->loc, result); + } + + return result; +} + +static AnyTypeV parse_type_expression(Lexer& lex) { + if (lex.tok() == tok_bitwise_or) { // `type T = | T1 | T2 | ...` (each per line) (like in TypeScript) + lex.next(); + } + + AnyTypeV result = parse_type_nullable(lex); + + if (lex.tok() == tok_bitwise_or) { // `int | slice`, `Pair2 | (Pair3 | null)` + std::vector items; + items.emplace_back(result); + while (lex.tok() == tok_bitwise_or) { + lex.next(); + items.emplace_back(parse_type_nullable(lex)); + } + result = createV(result->loc, std::move(items)); + } + + if (lex.tok() == tok_arrow) { // `int -> int`, `(cell, slice) -> void`, `int -> int -> int`, `int | cell -> void` + lex.next(); + std::vector params_and_return; + if (auto p_tensor = result->try_as()) { + params_and_return.reserve(p_tensor->get_items().size()); + params_and_return.insert(params_and_return.begin(), p_tensor->get_items().begin(), p_tensor->get_items().end()); + } else { + params_and_return.reserve(2); + params_and_return.push_back(result); + } + params_and_return.push_back(parse_type_expression(lex)); + result = createV(result->loc, std::move(params_and_return)); + } + + return result; +} + +static AnyTypeV parse_type_from_tokens(Lexer& lex) { + return parse_type_expression(lex); +} + + + +// -------------------------------------------- +// parsing expressions and statements +// AnyExprV parse_expr(Lexer& lex); @@ -178,7 +284,7 @@ static AnyV parse_parameter(Lexer& lex, bool is_first) { // parameter type after colon are mandatory lex.expect(tok_colon, "`: `"); - TypePtr param_type = parse_type_from_tokens(lex); + AnyTypeV param_type = parse_type_from_tokens(lex); return createV(loc, param_name, param_type, declared_as_mutate); } @@ -193,7 +299,7 @@ static AnyV parse_global_var_declaration(Lexer& lex, const std::vector(lex.cur_location(), lex.cur_str()); lex.next(); lex.expect(tok_colon, "`:`"); - TypePtr declared_type = parse_type_from_tokens(lex); + AnyTypeV declared_type = parse_type_from_tokens(lex); if (lex.tok() == tok_comma) { lex.error("multiple declarations are not allowed, split globals on separate lines"); } @@ -213,7 +319,7 @@ static AnyV parse_constant_declaration(Lexer& lex, const std::vector(lex.cur_location(), lex.cur_str()); lex.next(); - TypePtr declared_type = nullptr; + AnyTypeV declared_type = nullptr; if (lex.tok() == tok_colon) { lex.next(); declared_type = parse_type_from_tokens(lex); @@ -237,7 +343,7 @@ static AnyV parse_type_alias_declaration(Lexer& lex, const std::vector(lex.cur_location(), lex.cur_str()); lex.next(); lex.expect(tok_assign, "`=`"); - TypePtr underlying_type = parse_type_from_tokens(lex); + AnyTypeV underlying_type = parse_type_from_tokens(lex); lex.expect(tok_semicolon, "`;`"); return createV(loc, v_ident, underlying_type); } @@ -267,11 +373,11 @@ static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable) { args.push_back(parse_var_declaration_lhs(lex, is_immutable)); } lex.expect(tok_clbracket, "`]`"); - return createV(loc, std::move(args)); + return createV(loc, std::move(args)); } if (lex.tok() == tok_identifier) { auto v_ident = createV(loc, lex.cur_str()); - TypePtr declared_type = nullptr; + AnyTypeV declared_type = nullptr; bool marked_as_redef = false; lex.next(); if (lex.tok() == tok_colon) { @@ -284,7 +390,7 @@ static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable) { return createV(loc, v_ident, declared_type, is_immutable, marked_as_redef); } if (lex.tok() == tok_underscore) { - TypePtr declared_type = nullptr; + AnyTypeV declared_type = nullptr; lex.next(); if (lex.tok() == tok_colon) { lex.next(); @@ -449,7 +555,7 @@ static AnyV parse_throw_expression(Lexer& lex) { static V parse_match_arm(Lexer& lex) { SrcLocation loc = lex.cur_location(); MatchArmKind pattern_kind = static_cast(-1); - TypePtr exact_type = nullptr; + AnyTypeV exact_type = nullptr; AnyExprV pattern_expr = nullptr; Lexer::SavedPositionForLookahead backup = lex.save_parsing_position(); @@ -525,7 +631,7 @@ static V parse_match_expression(Lexer& lex) { if (lex.tok() == tok_clbrace) { break; } - if (!was_comma && v_arm->get_body()->type != ast_braced_expression) { + if (!was_comma && v_arm->get_body()->kind != ast_braced_expression) { lex.unexpected("`,`"); } } @@ -566,7 +672,7 @@ static AnyExprV parse_expr100(Lexer& lex) { lex.next(); if (lex.tok() == tok_clbracket) { lex.next(); - return createV(loc, {}); + return createV(loc, {}); } std::vector items(1, parse_expr(lex)); while (lex.tok() == tok_comma) { @@ -577,7 +683,7 @@ static AnyExprV parse_expr100(Lexer& lex) { items.emplace_back(parse_expr(lex)); } lex.expect(tok_clbracket, "`]`"); - return createV(loc, std::move(items)); + return createV(loc, std::move(items)); } case tok_int_const: { std::string_view orig_str = lex.cur_str(); @@ -707,13 +813,13 @@ static AnyExprV parse_expr40(Lexer& lex) { if (lex.tok() == tok_as) { SrcLocation loc = lex.cur_location(); lex.next(); - TypePtr cast_to_type = parse_type_from_tokens(lex); + AnyTypeV cast_to_type = parse_type_from_tokens(lex); lhs = createV(loc, lhs, cast_to_type); } else if (lex.tok() == tok_is) { SrcLocation loc = lex.cur_location(); lex.next(); - TypePtr rhs_type = parse_type_from_tokens(lex); - bool is_negated = lhs->type == ast_not_null_operator; // `a !is ...`, now lhs = `a!` + AnyTypeV rhs_type = parse_type_from_tokens(lex); + bool is_negated = lhs->kind == ast_not_null_operator; // `a !is ...`, now lhs = `a!` if (is_negated) { lhs = lhs->as()->get_expr(); } @@ -862,7 +968,9 @@ static V parse_block_statement(Lexer& lex) { if (lex.tok() == tok_clbrace) { break; } - bool does_end_with_brace = v->type == ast_if_statement || v->type == ast_while_statement || v->type == ast_match_expression || v->type == ast_try_catch_statement || v->type == ast_repeat_statement || v->type == ast_block_statement; + bool does_end_with_brace = v->kind == ast_if_statement || v->kind == ast_while_statement + || v->kind == ast_match_expression + || v->kind == ast_try_catch_statement || v->kind == ast_repeat_statement || v->kind == ast_block_statement; if (!does_end_with_brace) { lex.expect(tok_semicolon, "`;`"); } @@ -1038,6 +1146,12 @@ AnyV parse_statement(Lexer& lex) { } } + +// -------------------------------------------- +// parsing top-level declarations +// + + static AnyV parse_func_body(Lexer& lex) { return parse_block_statement(lex); } @@ -1136,7 +1250,7 @@ static V parse_annotation(Lexer& lex) { v_arg = createV(loc, {}); break; case AnnotationKind::method_id: - if (!v_arg || v_arg->size() != 1 || v_arg->get_item(0)->type != ast_int_const) { + if (!v_arg || v_arg->size() != 1 || v_arg->get_item(0)->kind != ast_int_const) { throw ParseError(loc, "expecting `(number)` after " + static_cast(name)); } break; @@ -1178,7 +1292,7 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vectorempty() && v_param_list->get_param(0)->param_name == "self"; int n_mutate_params = v_param_list->get_mutate_params_count(); - TypePtr ret_type = nullptr; + AnyTypeV ret_type = nullptr; bool returns_self = false; if (lex.tok() == tok_colon) { // : (if absent, it means "auto infer", not void) lex.next(); @@ -1186,9 +1300,9 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector(lex.cur_location(), "void"); + lex.next(); } else { ret_type = parse_type_from_tokens(lex); } @@ -1273,7 +1387,7 @@ static AnyV parse_struct_field(Lexer& lex) { auto v_ident = createV(lex.cur_location(), lex.cur_str()); lex.next(); lex.expect(tok_colon, "`: `"); - TypePtr declared_type = parse_type_from_tokens(lex); + AnyTypeV declared_type = parse_type_from_tokens(lex); AnyExprV default_value = nullptr; if (lex.tok() == tok_assign) { // `id: int = 3` @@ -1344,7 +1458,12 @@ static AnyV parse_import_directive(Lexer& lex) { return createV(loc, v_str); // semicolon is not necessary } -// the main (exported) function + +// -------------------------------------------- +// parse .tolk source file to AST +// (the main, exported, function) +// + AnyV parse_src_file_to_ast(const SrcFile* file) { std::vector toplevel_declarations; std::vector> annotations; diff --git a/tolk/ast-replacer.h b/tolk/ast-replacer.h index 40cd8e489..54d1f8e26 100644 --- a/tolk/ast-replacer.h +++ b/tolk/ast-replacer.h @@ -96,7 +96,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } - virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } @@ -135,12 +135,12 @@ class ASTReplacerInFunctionBody : public ASTReplacer { virtual AnyV replace(V v) { return replace_children(v); } AnyExprV replace(AnyExprV v) final { - switch (v->type) { + switch (v->kind) { case ast_empty_expression: return replace(v->as()); case ast_parenthesized_expression: return replace(v->as()); case ast_braced_expression: return replace(v->as()); case ast_tensor: return replace(v->as()); - case ast_typed_tuple: return replace(v->as()); + case ast_bracket_tuple: return replace(v->as()); case ast_reference: return replace(v->as()); case ast_local_var_lhs: return replace(v->as()); case ast_local_vars_declaration: return replace(v->as()); @@ -167,12 +167,12 @@ class ASTReplacerInFunctionBody : public ASTReplacer { case ast_object_body: return replace(v->as()); case ast_object_literal: return replace(v->as()); default: - throw UnexpectedASTNodeType(v, "ASTReplacerInFunctionBody::replace"); + throw UnexpectedASTNodeKind(v, "ASTReplacerInFunctionBody::replace"); } } AnyV replace(AnyV v) final { - switch (v->type) { + switch (v->kind) { case ast_empty_statement: return replace(v->as()); case ast_block_statement: return replace(v->as()); case ast_return_statement: return replace(v->as()); @@ -185,7 +185,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { case ast_try_catch_statement: return replace(v->as()); #ifdef TOLK_DEBUG case ast_asm_body: - throw UnexpectedASTNodeType(v, "ASTReplacer::replace"); + throw UnexpectedASTNodeKind(v, "ASTReplacer::replace"); #endif default: { // be very careful, don't forget to handle all statements (not expressions) above! diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index 1217fa45b..3b0b458e4 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -25,7 +25,7 @@ class ASTReplicator { protected: virtual AnyV clone(AnyV v) = 0; virtual AnyExprV clone(AnyExprV v) = 0; - virtual TypePtr clone(TypePtr) = 0; + virtual AnyTypeV clone(AnyTypeV v) = 0; public: virtual ~ASTReplicator() = default; @@ -53,6 +53,36 @@ class ASTReplicatorFunction : public ASTReplicator { return result; } + std::vector clone(const std::vector& items) { + std::vector result; + result.reserve(items.size()); + for (AnyTypeV item : items) { + result.push_back(clone(item)); + } + return result; + } + + // types + + virtual V clone(V v) { + return createV(v->loc, v->text); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_inner())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_items())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_items())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_params_and_return())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_variants())); + } + // expressions virtual V clone(V v) { @@ -67,14 +97,14 @@ class ASTReplicatorFunction : public ASTReplicator { virtual V clone(V v) { return createV(v->loc, clone(v->get_items())); } - virtual V clone(V v) { - return createV(v->loc, clone(v->get_items())); + virtual V clone(V v) { + return createV(v->loc, clone(v->get_items())); } virtual V clone(V v) { return createV(v->loc, clone(v->get_identifier()), v->has_instantiationTs() ? clone(v->get_instantiationTs()) : nullptr); } virtual V clone(V v) { - return createV(v->loc, clone(v->get_identifier()), clone(v->declared_type), v->is_immutable, v->marked_as_redef); + return createV(v->loc, clone(v->get_identifier()), clone(v->type_node), v->is_immutable, v->marked_as_redef); } virtual V clone(V v) { return createV(v->loc, clone(v->get_expr())); @@ -122,10 +152,10 @@ class ASTReplicatorFunction : public ASTReplicator { return createV(v->loc, clone(v->get_cond()), clone(v->get_when_true()), clone(v->get_when_false())); } virtual V clone(V v) { - return createV(v->loc, clone(v->get_expr()), clone(v->cast_to_type)); + return createV(v->loc, clone(v->get_expr()), clone(v->type_node)); } virtual V clone(V v) { - return createV(v->loc, clone(v->get_expr()), clone(v->rhs_type), v->is_negated); + return createV(v->loc, clone(v->get_expr()), clone(v->type_node), v->is_negated); } virtual V clone(V v) { return createV(v->loc, clone(v->get_expr())); @@ -134,7 +164,7 @@ class ASTReplicatorFunction : public ASTReplicator { return createV(v->loc, clone(v->get_all_children())); } virtual V clone(V v) { - return createV(v->loc, v->pattern_kind, clone(v->exact_type), clone(v->get_pattern_expr()), clone(v->get_body())); + return createV(v->loc, v->pattern_kind, clone(v->pattern_type_node), clone(v->get_pattern_expr()), clone(v->get_body())); } virtual V clone(V v) { return createV(v->loc, clone(v->get_field_identifier()), clone(v->get_init_val())); @@ -187,26 +217,48 @@ class ASTReplicatorFunction : public ASTReplicator { virtual V clone(V v) { return createV(v->loc, v->name); } + virtual V clone(V v) { + return createV(v->loc, v->nameT); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_items())); + } virtual V clone(V v) { - return createV(v->loc, clone(v->substituted_type)); + return createV(v->loc, clone(v->type_node)); } virtual V clone(V v) { return createV(v->loc, clone(v->get_items())); } virtual V clone(V v) { - return createV(v->loc, v->param_name, clone(v->declared_type), v->declared_as_mutate); + return createV(v->loc, v->param_name, clone(v->type_node), v->declared_as_mutate); } virtual V clone(V v) { return createV(v->loc, clone(v->get_params())); } + AnyTypeV clone(AnyTypeV v) final { + if (v == nullptr) { + return nullptr; + } + switch (v->kind) { + case ast_type_leaf_text: return clone(v->as()); + case ast_type_question_nullable: return clone(v->as()); + case ast_type_parenthesis_tensor: return clone(v->as()); + case ast_type_bracket_tuple: return clone(v->as()); + case ast_type_arrow_callable: return clone(v->as()); + case ast_type_vertical_bar_union: return clone(v->as()); + default: + throw UnexpectedASTNodeKind(v, "ASTReplicatorFunction::clone"); + } + } + AnyExprV clone(AnyExprV v) final { - switch (v->type) { + switch (v->kind) { case ast_empty_expression: return clone(v->as()); case ast_parenthesized_expression: return clone(v->as()); case ast_braced_expression: return clone(v->as()); case ast_tensor: return clone(v->as()); - case ast_typed_tuple: return clone(v->as()); + case ast_bracket_tuple: return clone(v->as()); case ast_reference: return clone(v->as()); case ast_local_var_lhs: return clone(v->as()); case ast_local_vars_declaration: return clone(v->as()); @@ -233,12 +285,12 @@ class ASTReplicatorFunction : public ASTReplicator { case ast_object_body: return clone(v->as()); case ast_object_literal: return clone(v->as()); default: - throw UnexpectedASTNodeType(v, "ASTReplicatorFunction::clone"); + throw UnexpectedASTNodeKind(v, "ASTReplicatorFunction::clone"); } } AnyV clone(AnyV v) final { - switch (v->type) { + switch (v->kind) { case ast_empty_statement: return clone(v->as()); case ast_block_statement: return clone(v->as()); case ast_return_statement: return clone(v->as()); @@ -252,6 +304,8 @@ class ASTReplicatorFunction : public ASTReplicator { case ast_asm_body: return clone(v->as()); // other AST nodes that can be children of ast nodes of function body case ast_identifier: return clone(v->as()); + case ast_genericsT_item: return clone(v->as()); + case ast_genericsT_list: return clone(v->as()); case ast_instantiationT_item: return clone(v->as()); case ast_instantiationT_list: return clone(v->as()); case ast_parameter: return clone(v->as()); @@ -265,21 +319,18 @@ class ASTReplicatorFunction : public ASTReplicator { } } - TypePtr clone(TypePtr t) override { - return t; - } - - public: - virtual V clone_function_body(V v_function) { +public: + // the cloned function becomes a deep copy, all AST nodes are copied, no previous pointers left + V clone_function_ast(V v_orig, V new_name_ident) { return createV( - v_function->loc, - clone(v_function->get_identifier()), - clone(v_function->get_param_list()), - clone(v_function->get_body()->as()), - clone(v_function->declared_return_type), - v_function->genericsT_list, - v_function->method_id, - v_function->flags + v_orig->loc, + new_name_ident, + clone(v_orig->get_param_list()), + clone(v_orig->get_body()), + clone(v_orig->return_type_node), + clone(v_orig->genericsT_list), + v_orig->method_id, + v_orig->flags ); } }; diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index 7782c506d..ea991da20 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -31,14 +31,21 @@ namespace tolk { class ASTStringifier final : public ASTVisitor { - constexpr static std::pair name_pairs[] = { + constexpr static std::pair name_pairs[] = { {ast_identifier, "ast_identifier"}, + // types + {ast_type_leaf_text, "ast_type_leaf_text"}, + {ast_type_question_nullable, "ast_type_question_nullable"}, + {ast_type_parenthesis_tensor, "ast_type_parenthesis_tensor"}, + {ast_type_bracket_tuple, "ast_type_bracket_tuple"}, + {ast_type_arrow_callable, "ast_type_arrow_callable"}, + {ast_type_vertical_bar_union, "ast_type_vertical_bar_union"}, // expressions {ast_empty_expression, "ast_empty_expression"}, {ast_parenthesized_expression, "ast_parenthesized_expression"}, {ast_braced_expression, "ast_braced_expression"}, {ast_tensor, "ast_tensor"}, - {ast_typed_tuple, "ast_typed_tuple"}, + {ast_bracket_tuple, "ast_bracket_tuple"}, {ast_reference, "ast_reference"}, {ast_local_var_lhs, "ast_local_var_lhs"}, {ast_local_vars_declaration, "ast_local_vars_declaration"}, @@ -108,19 +115,19 @@ class ASTStringifier final : public ASTVisitor { static_assert(std::size(annotation_kinds) == static_cast(AnnotationKind::unknown), "annotation_kinds needs to be updated"); - template - constexpr static const char* ast_node_type_to_string() { - return name_pairs[node_type].second; + template + constexpr static const char* ast_node_kind_to_string() { + return name_pairs[node_kind].second; } int depth = 0; std::string out; bool colored = false; - template - void handle_vertex(V v) { + template + void handle_vertex(V v) { out += std::string(depth * 2, ' '); - out += ast_node_type_to_string(); + out += ast_node_kind_to_string(); if (std::string postfix = specific_str(v); !postfix.empty()) { out += colored ? " \x1b[34m" : " // "; out += postfix; @@ -133,7 +140,9 @@ class ASTStringifier final : public ASTVisitor { } static std::string specific_str(AnyV v) { - switch (v->type) { + switch (v->kind) { + case ast_type_leaf_text: + return static_cast(v->as()->text); case ast_identifier: return static_cast(v->as()->name); case ast_reference: { @@ -170,7 +179,7 @@ class ASTStringifier final : public ASTVisitor { case ast_type_alias_declaration: return "type " + static_cast(v->as()->get_identifier()->name); case ast_struct_field: - return static_cast(v->as()->get_identifier()->name) + ": " + v->as()->declared_type->as_human_readable(); + return static_cast(v->as()->get_identifier()->name) + ": " + ast_type_node_to_string(v->as()->type_node); case ast_struct_declaration: return "struct " + static_cast(v->as()->get_identifier()->name); case ast_assign: @@ -182,24 +191,21 @@ class ASTStringifier final : public ASTVisitor { case ast_binary_operator: return static_cast(v->as()->operator_name); case ast_cast_as_operator: - return v->as()->cast_to_type->as_human_readable(); + return ast_type_node_to_string(v->as()->type_node); case ast_is_type_operator: { std::string prefix = v->as()->is_negated ? "!is " : "is "; - return prefix + v->as()->rhs_type->as_human_readable(); + return prefix + ast_type_node_to_string(v->as()->type_node); } case ast_block_statement: return "↓" + std::to_string(v->as()->get_items().size()); case ast_instantiationT_item: - return v->as()->substituted_type->as_human_readable(); + return ast_type_node_to_string(v->as()->type_node); case ast_if_statement: return v->as()->is_ifnot ? "ifnot" : ""; case ast_annotation: return annotation_kinds[static_cast(v->as()->kind)].second; - case ast_parameter: { - std::ostringstream os; - os << v->as()->declared_type; - return static_cast(v->as()->param_name) + ": " + os.str(); - } + case ast_parameter: + return static_cast(v->as()->param_name) + ": " + ast_type_node_to_string(v->as()->type_node); case ast_function_declaration: { std::string param_names; for (int i = 0; i < v->as()->get_num_params(); i++) { @@ -210,25 +216,24 @@ class ASTStringifier final : public ASTVisitor { return "fun " + static_cast(v->as()->get_identifier()->name) + "(" + param_names + ")"; } case ast_local_var_lhs: { - std::ostringstream os; - os << (v->as()->inferred_type ? v->as()->inferred_type->as_human_readable() : v->as()->declared_type->as_human_readable()); + std::string str_type = v->as()->inferred_type ? v->as()->inferred_type->as_human_readable() : ast_type_node_to_string(v->as()->type_node); if (v->as()->get_name().empty()) { - return "_: " + os.str(); + return "_: " + str_type; } - return static_cast(v->as()->get_name()) + ":" + os.str(); + return static_cast(v->as()->get_name()) + ": " + str_type; } case ast_instantiationT_list: { std::string result = "<"; for (AnyV item : v->as()->get_items()) { if (result.size() > 1) result += ","; - result += item->as()->substituted_type->as_human_readable(); + result += ast_type_node_to_string(item->as()->type_node); } return result + ">"; } case ast_match_arm: if (v->as()->pattern_kind == MatchArmKind::exact_type) { - return v->as()->exact_type->as_human_readable(); + return ast_type_node_to_string(v->as()->pattern_type_node); } if (v->as()->pattern_kind == MatchArmKind::const_expression) { return "(expression)"; @@ -260,7 +265,7 @@ class ASTStringifier final : public ASTVisitor { } static std::string to_string_without_children(AnyV v) { - std::string result = ast_node_type_to_string(v->type); + std::string result = ast_node_kind_to_string(v->kind); if (std::string postfix = specific_str(v); !postfix.empty()) { result += ' '; result += specific_str(v); @@ -268,19 +273,39 @@ class ASTStringifier final : public ASTVisitor { return result; } - static const char* ast_node_type_to_string(ASTNodeType node_type) { - return name_pairs[node_type].second; + static std::string ast_type_node_to_string(AnyTypeV type_node) { + if (type_node == nullptr) { + return ""; + } + if (auto v_leaf = type_node->try_as()) { + return static_cast(v_leaf->text); + } + if (auto v_nullable = type_node->try_as()) { + return ast_type_node_to_string(v_nullable->get_inner()) + "?"; + } + return ast_node_kind_to_string(type_node->kind); + } + + static const char* ast_node_kind_to_string(ASTNodeKind node_kind) { + return name_pairs[node_kind].second; } void visit(AnyV v) override { - switch (v->type) { + switch (v->kind) { case ast_identifier: return handle_vertex(v->as()); + // types + case ast_type_leaf_text: return handle_vertex(v->as()); + case ast_type_question_nullable: return handle_vertex(v->as()); + case ast_type_parenthesis_tensor: return handle_vertex(v->as()); + case ast_type_bracket_tuple: return handle_vertex(v->as()); + case ast_type_arrow_callable: return handle_vertex(v->as()); + case ast_type_vertical_bar_union: return handle_vertex(v->as()); // expressions case ast_empty_expression: return handle_vertex(v->as()); case ast_parenthesized_expression: return handle_vertex(v->as()); case ast_braced_expression: return handle_vertex(v->as()); case ast_tensor: return handle_vertex(v->as()); - case ast_typed_tuple: return handle_vertex(v->as()); + case ast_bracket_tuple: return handle_vertex(v->as()); case ast_reference: return handle_vertex(v->as()); case ast_local_var_lhs: return handle_vertex(v->as()); case ast_local_vars_declaration: return handle_vertex(v->as()); @@ -337,7 +362,7 @@ class ASTStringifier final : public ASTVisitor { case ast_import_directive: return handle_vertex(v->as()); case ast_tolk_file: return handle_vertex(v->as()); default: - throw UnexpectedASTNodeType(v, "ASTStringifier::visit"); + throw UnexpectedASTNodeKind(v, "ASTStringifier::visit"); } } }; diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h index 12e65e270..11ecbe80f 100644 --- a/tolk/ast-visitor.h +++ b/tolk/ast-visitor.h @@ -21,7 +21,7 @@ /* * A module implementing base functionality of read-only traversing a vertex tree. - * Since a vertex in general doesn't store a vector of children, iterating is possible only for concrete node_type. + * Since a vertex in general doesn't store a vector of children, iterating is possible only for concrete node_kind. * E.g., for ast_if_statement, visit nodes cond, if-body and else-body. For ast_string_const, nothing. And so on. * Visitors below are helpers to inherit from and handle specific vertex types. * @@ -37,6 +37,16 @@ namespace tolk { class ASTVisitor { protected: + GNU_ATTRIBUTE_ALWAYS_INLINE static void visit_children(const ASTTypeLeaf* v) { + static_cast(v); + } + + GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTTypeVararg* v) { + for (AnyTypeV child : v->children) { + visit(child); + } + } + GNU_ATTRIBUTE_ALWAYS_INLINE static void visit_children(const ASTExprLeaf* v) { static_cast(v); } @@ -95,7 +105,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } - virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } @@ -134,13 +144,13 @@ class ASTVisitorFunctionBody : public ASTVisitor { virtual void visit(V v) { return visit_children(v); } void visit(AnyV v) final { - switch (v->type) { + switch (v->kind) { // expressions case ast_empty_expression: return visit(v->as()); case ast_parenthesized_expression: return visit(v->as()); case ast_braced_expression: return visit(v->as()); case ast_tensor: return visit(v->as()); - case ast_typed_tuple: return visit(v->as()); + case ast_bracket_tuple: return visit(v->as()); case ast_reference: return visit(v->as()); case ast_local_var_lhs: return visit(v->as()); case ast_local_vars_declaration: return visit(v->as()); @@ -179,10 +189,10 @@ class ASTVisitorFunctionBody : public ASTVisitor { case ast_try_catch_statement: return visit(v->as()); #ifdef TOLK_DEBUG case ast_asm_body: - throw UnexpectedASTNodeType(v, "ASTVisitor; forgot to filter out asm functions in should_visit_function()?"); + throw UnexpectedASTNodeKind(v, "ASTVisitor; forgot to filter out asm functions in should_visit_function()?"); #endif default: - throw UnexpectedASTNodeType(v, "ASTVisitorFunctionBody::visit"); + throw UnexpectedASTNodeKind(v, "ASTVisitorFunctionBody::visit"); } } diff --git a/tolk/ast.cpp b/tolk/ast.cpp index 399947446..24c797bd2 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -36,10 +36,10 @@ void ASTNodeBase::debug_print() const { #endif // TOLK_DEBUG -UnexpectedASTNodeType::UnexpectedASTNodeType(AnyV v_unexpected, const char* place_where): v_unexpected(v_unexpected) { - message = "Unexpected ASTNodeType "; +UnexpectedASTNodeKind::UnexpectedASTNodeKind(AnyV v_unexpected, const char* place_where): v_unexpected(v_unexpected) { + message = "Unexpected ASTNodeKind "; #ifdef TOLK_DEBUG - message += ASTStringifier::ast_node_type_to_string(v_unexpected->type); + message += ASTStringifier::ast_node_kind_to_string(v_unexpected->kind); message += " "; #endif message += "in "; @@ -105,6 +105,10 @@ int Vertex::get_mutate_params_count() const { // Therefore, there is a guarantee, that all AST mutations are done via these methods, // easily searched by usages, and there is no another way to modify any other field. +void ASTNodeDeclaredTypeBase::assign_resolved_type(TypePtr resolved_type) { + this->resolved_type = resolved_type; +} + void ASTNodeExpressionBase::assign_inferred_type(TypePtr type) { this->inferred_type = type; } @@ -134,72 +138,32 @@ void Vertex::assign_fun_ref(FunctionPtr fun_ref) { this->fun_maybe = fun_ref; } -void Vertex::assign_resolved_type(TypePtr cast_to_type) { - this->cast_to_type = cast_to_type; -} - -void Vertex::assign_resolved_type(TypePtr rhs_type) { - this->rhs_type = rhs_type; -} - void Vertex::assign_is_negated(bool is_negated) { this->is_negated = is_negated; } -void Vertex::assign_resolved_pattern(MatchArmKind pattern_kind, TypePtr exact_type, AnyExprV pattern_expr) { +void Vertex::assign_resolved_pattern(MatchArmKind pattern_kind, AnyExprV pattern_expr) { + this->pattern_type_node = nullptr; this->pattern_kind = pattern_kind; - this->exact_type = exact_type; this->lhs = pattern_expr; } -void Vertex::assign_var_ref(GlobalVarPtr var_ref) { - this->var_ref = var_ref; -} - -void Vertex::assign_resolved_type(TypePtr declared_type) { - this->declared_type = declared_type; +void Vertex::assign_glob_ref(GlobalVarPtr glob_ref) { + this->glob_ref = glob_ref; } void Vertex::assign_const_ref(GlobalConstPtr const_ref) { this->const_ref = const_ref; } -void Vertex::assign_resolved_type(TypePtr declared_type) { - this->declared_type = declared_type; -} - void Vertex::assign_alias_ref(AliasDefPtr alias_ref) { this->alias_ref = alias_ref; } -void Vertex::assign_resolved_type(TypePtr underlying_type) { - this->underlying_type = underlying_type; -} - -void Vertex::assign_field_ref(StructFieldPtr field_ref) { - this->field_ref = field_ref; -} - -void Vertex::assign_resolved_type(TypePtr declared_type) { - this->declared_type = declared_type; -} - void Vertex::assign_struct_ref(StructPtr struct_ref) { this->struct_ref = struct_ref; } -void Vertex::assign_resolved_type(TypePtr substituted_type) { - this->substituted_type = substituted_type; -} - -void Vertex::assign_param_ref(LocalVarPtr param_ref) { - this->param_ref = param_ref; -} - -void Vertex::assign_resolved_type(TypePtr declared_type) { - this->declared_type = declared_type; -} - void Vertex::assign_fun_ref(FunctionPtr fun_ref) { this->fun_ref = fun_ref; } @@ -232,18 +196,10 @@ void Vertex::assign_fun_ref(FunctionPtr fun_ref) { this->fun_ref = fun_ref; } -void Vertex::assign_resolved_type(TypePtr declared_return_type) { - this->declared_return_type = declared_return_type; -} - void Vertex::assign_var_ref(LocalVarPtr var_ref) { this->var_ref = var_ref; } -void Vertex::assign_resolved_type(TypePtr declared_type) { - this->declared_type = declared_type; -} - void Vertex::assign_src_file(const SrcFile* file) { this->file = file; } diff --git a/tolk/ast.h b/tolk/ast.h index 71bc9c40b..dbe1e02b0 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -27,35 +27,33 @@ * Here we introduce AST representation of Tolk source code. * Historically, in FunC, there was no AST: while lexing, symbols were registered, types were inferred, and so on. * There was no way to perform any more or less semantic analysis. - * In Tolk, I've implemented parsing .tolk files into AST at first, and then converting this AST - * into legacy representation (see pipe-ast-to-legacy.cpp). - * In the future, more and more code analysis will be moved out of legacy to AST-level. + * In Tolk, all files are parsed into AST, and all semantic analysis is done at the AST level. * * From the user's point of view, all AST vertices are constant. All API is based on constancy. * Even though fields of vertex structs are public, they can't be modified, since vertices are accepted by const ref. * Generally, there are three ways of accepting a vertex: * * AnyV (= const ASTNodeBase*) - * the only you can do with this vertex is to see v->type (ASTNodeType) and to cast via v->as() + * the only you can do with this vertex is to see v->kind (ASTNodeKind) and to cast via v->as() * * AnyExprV (= const ASTNodeExpressionBase*) * in contains expression-specific properties (lvalue/rvalue, inferred type) - * * V (= const Vertex*) + * * V (= const Vertex*) * a specific type of vertex, you can use its fields and methods * There is one way of creating a vertex: - * * createV(...constructor_args) (= new Vertex(...)) + * * createV(...constructor_args) (= new Vertex(...)) * vertices are currently created on a heap, without any custom memory arena, just allocated and never deleted * The only way to modify a field is to use "mutate()" method (drops constancy, the only point of mutation) - * and then to call "assign_*" method, like "assign_sym", "assign_src_file", etc. + * and then to call "assign_*" method, like "assign_sym", "assign_src_file", etc. * - * Having AnyV and knowing its node_type, a call - * v->as() + * Having AnyV and knowing its node_kind, a call + * v->as() * will return a typed vertex. - * There is also a shorthand v->try_as() which returns V or nullptr if types don't match: + * There is also a shorthand v->try_as() which returns V or nullptr if types don't match: * if (auto v_int = v->try_as()) * Note, that there casts are NOT DYNAMIC. ASTNode is not a virtual base, it has no vtable. * So, as<...>() is just a compile-time casting, without any runtime overhead. * * Note, that ASTNodeBase doesn't store any vector of children. That's why there is no way to loop over - * a random (unknown) vertex. Only a concrete Vertex stores its children (if any). + * a random (unknown) vertex. Only a concrete Vertex stores its children (if any). * Hence, to iterate over a custom vertex (e.g., a function body), one should inherit some kind of ASTVisitor. * Besides read-only visiting, there is a "visit and replace" pattern. * See ast-visitor.h and ast-replacer.h. @@ -63,14 +61,21 @@ namespace tolk { -enum ASTNodeType { +enum ASTNodeKind { ast_identifier, + // types + ast_type_leaf_text, + ast_type_question_nullable, + ast_type_parenthesis_tensor, + ast_type_bracket_tuple, + ast_type_arrow_callable, + ast_type_vertical_bar_union, // expressions ast_empty_expression, ast_parenthesized_expression, ast_braced_expression, ast_tensor, - ast_typed_tuple, + ast_bracket_tuple, ast_reference, ast_local_var_lhs, ast_local_vars_declaration, @@ -143,19 +148,19 @@ enum class MatchArmKind { // for `match` expression, each of arms `pattern => else_branch, // `else => body` }; -template +template struct Vertex; -template -using V = const Vertex*; +template +using V = const Vertex*; #define createV new Vertex -struct UnexpectedASTNodeType final : std::exception { +struct UnexpectedASTNodeKind final : std::exception { AnyV v_unexpected; std::string message; - explicit UnexpectedASTNodeType(AnyV v_unexpected, const char* place_where); + explicit UnexpectedASTNodeKind(AnyV v_unexpected, const char* place_where); const char* what() const noexcept override { return message.c_str(); @@ -165,25 +170,25 @@ struct UnexpectedASTNodeType final : std::exception { // --------------------------------------------------------- struct ASTNodeBase { - const ASTNodeType type; + const ASTNodeKind kind; const SrcLocation loc; - ASTNodeBase(ASTNodeType type, SrcLocation loc) : type(type), loc(loc) {} + ASTNodeBase(ASTNodeKind kind, SrcLocation loc) : kind(kind), loc(loc) {} ASTNodeBase(const ASTNodeBase&) = delete; - template - V as() const { + template + V as() const { #ifdef TOLK_DEBUG - if (type != node_type) { - throw Fatal("v->as<...> to wrong node_type"); + if (kind != node_kind) { + throw Fatal("v->as<...> to wrong node_kind"); } #endif - return static_cast>(this); + return static_cast>(this); } - template - V try_as() const { - return type == node_type ? static_cast>(this) : nullptr; + template + V try_as() const { + return kind == node_kind ? static_cast>(this) : nullptr; } #ifdef TOLK_DEBUG @@ -196,9 +201,16 @@ struct ASTNodeBase { void error(const std::string& err_msg) const; }; -struct ASTNodeExpressionBase : ASTNodeBase { - friend class ASTDuplicatorFunction; +struct ASTNodeDeclaredTypeBase : ASTNodeBase { + TypePtr resolved_type = nullptr; + + ASTNodeDeclaredTypeBase* mutate() const { return const_cast(this); } + void assign_resolved_type(TypePtr resolved_type); + + ASTNodeDeclaredTypeBase(ASTNodeKind kind, SrcLocation loc) : ASTNodeBase(kind, loc) {} +}; +struct ASTNodeExpressionBase : ASTNodeBase { TypePtr inferred_type = nullptr; bool is_rvalue: 1 = false; bool is_lvalue: 1 = false; @@ -211,11 +223,27 @@ struct ASTNodeExpressionBase : ASTNodeBase { void assign_lvalue_true(); void assign_always_true_or_false(int flow_true_false_state); - ASTNodeExpressionBase(ASTNodeType type, SrcLocation loc) : ASTNodeBase(type, loc) {} + ASTNodeExpressionBase(ASTNodeKind kind, SrcLocation loc) : ASTNodeBase(kind, loc) {} }; struct ASTNodeStatementBase : ASTNodeBase { - ASTNodeStatementBase(ASTNodeType type, SrcLocation loc) : ASTNodeBase(type, loc) {} + ASTNodeStatementBase(ASTNodeKind kind, SrcLocation loc) : ASTNodeBase(kind, loc) {} +}; + +struct ASTTypeLeaf : ASTNodeDeclaredTypeBase { +protected: + ASTTypeLeaf(ASTNodeKind kind, SrcLocation loc) + : ASTNodeDeclaredTypeBase(kind, loc) {} +}; + +struct ASTTypeVararg : ASTNodeDeclaredTypeBase { + friend class ASTVisitor; + +protected: + std::vector children; + + ASTTypeVararg(ASTNodeKind kind, SrcLocation loc, std::vector&& children) + : ASTNodeDeclaredTypeBase(kind, loc), children(std::move(children)) {} }; struct ASTExprLeaf : ASTNodeExpressionBase { @@ -223,8 +251,8 @@ struct ASTExprLeaf : ASTNodeExpressionBase { friend class ASTReplacer; protected: - ASTExprLeaf(ASTNodeType type, SrcLocation loc) - : ASTNodeExpressionBase(type, loc) {} + ASTExprLeaf(ASTNodeKind kind, SrcLocation loc) + : ASTNodeExpressionBase(kind, loc) {} }; struct ASTExprUnary : ASTNodeExpressionBase { @@ -234,8 +262,8 @@ struct ASTExprUnary : ASTNodeExpressionBase { protected: AnyExprV child; - ASTExprUnary(ASTNodeType type, SrcLocation loc, AnyExprV child) - : ASTNodeExpressionBase(type, loc), child(child) {} + ASTExprUnary(ASTNodeKind kind, SrcLocation loc, AnyExprV child) + : ASTNodeExpressionBase(kind, loc), child(child) {} }; struct ASTExprBinary : ASTNodeExpressionBase { @@ -246,8 +274,8 @@ struct ASTExprBinary : ASTNodeExpressionBase { AnyExprV lhs; AnyExprV rhs; - ASTExprBinary(ASTNodeType type, SrcLocation loc, AnyExprV lhs, AnyExprV rhs) - : ASTNodeExpressionBase(type, loc), lhs(lhs), rhs(rhs) {} + ASTExprBinary(ASTNodeKind kind, SrcLocation loc, AnyExprV lhs, AnyExprV rhs) + : ASTNodeExpressionBase(kind, loc), lhs(lhs), rhs(rhs) {} }; struct ASTExprVararg : ASTNodeExpressionBase { @@ -259,8 +287,8 @@ struct ASTExprVararg : ASTNodeExpressionBase { AnyExprV child(int i) const { return children.at(i); } - ASTExprVararg(ASTNodeType type, SrcLocation loc, std::vector&& children) - : ASTNodeExpressionBase(type, loc), children(std::move(children)) {} + ASTExprVararg(ASTNodeKind kind, SrcLocation loc, std::vector&& children) + : ASTNodeExpressionBase(kind, loc), children(std::move(children)) {} public: int size() const { return static_cast(children.size()); } @@ -274,8 +302,8 @@ struct ASTExprBlockOfStatements : ASTNodeExpressionBase { protected: AnyV child_block_statement; - ASTExprBlockOfStatements(ASTNodeType type, SrcLocation loc, AnyV child_block_statement) - : ASTNodeExpressionBase(type, loc), child_block_statement(child_block_statement) {} + ASTExprBlockOfStatements(ASTNodeKind kind, SrcLocation loc, AnyV child_block_statement) + : ASTNodeExpressionBase(kind, loc), child_block_statement(child_block_statement) {} }; struct ASTStatementUnary : ASTNodeStatementBase { @@ -287,8 +315,8 @@ struct ASTStatementUnary : ASTNodeStatementBase { AnyExprV child_as_expr() const { return reinterpret_cast(child); } - ASTStatementUnary(ASTNodeType type, SrcLocation loc, AnyV child) - : ASTNodeStatementBase(type, loc), child(child) {} + ASTStatementUnary(ASTNodeKind kind, SrcLocation loc, AnyV child) + : ASTNodeStatementBase(kind, loc), child(child) {} }; struct ASTStatementVararg : ASTNodeStatementBase { @@ -300,8 +328,8 @@ struct ASTStatementVararg : ASTNodeStatementBase { AnyExprV child_as_expr(int i) const { return reinterpret_cast(children.at(i)); } - ASTStatementVararg(ASTNodeType type, SrcLocation loc, std::vector children) - : ASTNodeStatementBase(type, loc), children(std::move(children)) {} + ASTStatementVararg(ASTNodeKind kind, SrcLocation loc, std::vector&& children) + : ASTNodeStatementBase(kind, loc), children(std::move(children)) {} public: int size() const { return static_cast(children.size()); } @@ -313,8 +341,8 @@ struct ASTOtherLeaf : ASTNodeBase { friend class ASTReplacer; protected: - ASTOtherLeaf(ASTNodeType type, SrcLocation loc) - : ASTNodeBase(type, loc) {} + ASTOtherLeaf(ASTNodeKind kind, SrcLocation loc) + : ASTNodeBase(kind, loc) {} }; struct ASTOtherVararg : ASTNodeBase { @@ -326,8 +354,8 @@ struct ASTOtherVararg : ASTNodeBase { AnyExprV child_as_expr(int i) const { return reinterpret_cast(children.at(i)); } - ASTOtherVararg(ASTNodeType type, SrcLocation loc, std::vector children) - : ASTNodeBase(type, loc), children(std::move(children)) {} + ASTOtherVararg(ASTNodeKind kind, SrcLocation loc, std::vector&& children) + : ASTNodeBase(kind, loc), children(std::move(children)) {} public: int size() const { return static_cast(children.size()); } @@ -352,6 +380,74 @@ struct Vertex final : ASTOtherLeaf { }; +// +// --------------------------------------------------------- +// types +// + + +template<> +// ast_type_leaf_text is a type node without children: "int", "User", "T", etc. +// after resolving, it will become TypeDataInt / TypeDataStruct / etc. +struct Vertex final : ASTTypeLeaf { + std::string_view text; + + Vertex(SrcLocation loc, std::string_view text) + : ASTTypeLeaf(ast_type_leaf_text, loc) + , text(text) {} +}; + +template<> +// ast_type_question_nullable is "T?" +// after resolving, it will become a union "T | null", but at AST level, it's a separate node +struct Vertex final : ASTTypeVararg { + AnyTypeV get_inner() const { return children.at(0); } + + Vertex(SrcLocation loc, AnyTypeV inner) + : ASTTypeVararg(ast_type_question_nullable, loc, {inner}) {} +}; + +template<> +// ast_type_parenthesis_tensor is "(T1, T2, ...)" +// after resolving, it will become TypeDataTensor +struct Vertex final : ASTTypeVararg { + const std::vector& get_items() const { return children; } + + Vertex(SrcLocation loc, std::vector&& items) + : ASTTypeVararg(ast_type_parenthesis_tensor, loc, std::move(items)) {} +}; + +template<> +// ast_type_bracket_tuple is "[T1, T2, ...]" +// after resolving, it will become TypeDataBrackets +struct Vertex final : ASTTypeVararg { + const std::vector& get_items() const { return children; } + + Vertex(SrcLocation loc, std::vector&& items) + : ASTTypeVararg(ast_type_bracket_tuple, loc, std::move(items)) {} +}; + +template<> +// ast_type_arrow_callable is "(T1, T2, ...) -> TResult" +// after resolving, it will become TypeDataFunCallable +struct Vertex final : ASTTypeVararg { + const std::vector& get_params_and_return() const { return children; } + + Vertex(SrcLocation loc, std::vector&& params_and_return) + : ASTTypeVararg(ast_type_arrow_callable, loc, std::move(params_and_return)) {} +}; + +template<> +// ast_type_vertical_bar_union is "T1 | T2 | ..." +// after resolving, it will become TypeDataUnion +struct Vertex final : ASTTypeVararg { + const std::vector& get_variants() const { return children; } + + Vertex(SrcLocation loc, std::vector&& variants) + : ASTTypeVararg(ast_type_vertical_bar_union, loc, std::move(variants)) {} +}; + + // // --------------------------------------------------------- // expressions @@ -404,16 +500,16 @@ struct Vertex final : ASTExprVararg { }; template<> -// ast_typed_tuple is a set of expressions in [square brackets] +// ast_bracket_tuple is a set of expressions in [square brackets] // in TVM, it's a TVM tuple, that occupies 1 slot, but the compiler knows its "typed structure" // example: `[1, x]`, `[[0]]` (nested) // typed tuples can be assigned to N variables, like `[one, _, three] = [1,2,3]` -struct Vertex final : ASTExprVararg { +struct Vertex final : ASTExprVararg { const std::vector& get_items() const { return children; } AnyExprV get_item(int i) const { return child(i); } Vertex(SrcLocation loc, std::vector items) - : ASTExprVararg(ast_typed_tuple, loc, std::move(items)) {} + : ASTExprVararg(ast_bracket_tuple, loc, std::move(items)) {} }; template<> @@ -453,7 +549,7 @@ struct Vertex final : ASTExprLeaf { public: LocalVarPtr var_ref = nullptr; // filled on resolve identifiers; for `redef` points to declared above; for underscore, name is empty - TypePtr declared_type; // not null for `var x: int = rhs`, otherwise nullptr + AnyTypeV type_node; // exists for `var x: int = rhs`, otherwise nullptr bool is_immutable; // declared via 'val', not 'var' bool marked_as_redef; // var (existing_var redef, new_var: int) = ... @@ -462,11 +558,10 @@ struct Vertex final : ASTExprLeaf { Vertex* mutate() const { return const_cast(this); } void assign_var_ref(LocalVarPtr var_ref); - void assign_resolved_type(TypePtr declared_type); - Vertex(SrcLocation loc, V identifier, TypePtr declared_type, bool is_immutable, bool marked_as_redef) + Vertex(SrcLocation loc, V identifier, AnyTypeV type_node, bool is_immutable, bool marked_as_redef) : ASTExprLeaf(ast_local_var_lhs, loc) - , identifier(identifier), declared_type(declared_type), is_immutable(is_immutable), marked_as_redef(marked_as_redef) {} + , identifier(identifier), type_node(type_node), is_immutable(is_immutable), marked_as_redef(marked_as_redef) {} }; template<> @@ -475,7 +570,7 @@ template<> // for `var (x, [y])` its expr is "tensor (local var, typed tuple (local var))" // for assignment `var x = 5`, this node is `var x`, lhs of assignment struct Vertex final : ASTExprUnary { - AnyExprV get_expr() const { return child; } // ast_local_var_lhs / ast_tensor / ast_typed_tuple + AnyExprV get_expr() const { return child; } // ast_local_var_lhs / ast_tensor / ast_bracket_tuple Vertex(SrcLocation loc, AnyExprV expr) : ASTExprUnary(ast_local_vars_declaration, loc, expr) {} @@ -602,7 +697,7 @@ struct Vertex final : ASTExprBinary { FunctionPtr fun_maybe = nullptr; // filled while type inferring for `globalF()` / `obj.f()`; remains nullptr for `local_var()` / `getF()()` AnyExprV get_callee() const { return lhs; } - bool is_dot_call() const { return lhs->type == ast_dot_access; } + bool is_dot_call() const { return lhs->kind == ast_dot_access; } AnyExprV get_dot_obj() const { return lhs->as()->get_obj(); } auto get_arg_list() const { return rhs->as(); } int get_num_args() const { return rhs->as()->size(); } @@ -711,16 +806,13 @@ template<> // ast_cast_as_operator is explicit casting with "as" keyword // examples: `arg as int` / `null as cell` / `t.tupleAt(2) as slice` struct Vertex final : ASTExprUnary { - AnyExprV get_expr() const { return child; } - - TypePtr cast_to_type; + AnyTypeV type_node; - Vertex* mutate() const { return const_cast(this); } - void assign_resolved_type(TypePtr cast_to_type); + AnyExprV get_expr() const { return child; } - Vertex(SrcLocation loc, AnyExprV expr, TypePtr cast_to_type) + Vertex(SrcLocation loc, AnyExprV expr, AnyTypeV type_node) : ASTExprUnary(ast_cast_as_operator, loc, expr) - , cast_to_type(cast_to_type) {} + , type_node(type_node) {} }; template<> @@ -729,16 +821,15 @@ template<> struct Vertex final : ASTExprUnary { AnyExprV get_expr() const { return child; } - TypePtr rhs_type; - bool is_negated; // `!is type`, `!= null` + AnyTypeV type_node; + bool is_negated; // `!is type`, `!= null` Vertex* mutate() const { return const_cast(this); } - void assign_resolved_type(TypePtr rhs_type); void assign_is_negated(bool is_negated); - Vertex(SrcLocation loc, AnyExprV expr, TypePtr rhs_type, bool is_negated) + Vertex(SrcLocation loc, AnyExprV expr, AnyTypeV type_node, bool is_negated) : ASTExprUnary(ast_is_type_operator, loc, expr) - , rhs_type(rhs_type), is_negated(is_negated) {} + , type_node(type_node), is_negated(is_negated) {} }; template<> @@ -775,17 +866,17 @@ template<> // example: `a+b => { ...; return 0; }` (match by expression, inferred_type of body is "never" (unreachable end)) struct Vertex final : ASTExprBinary { MatchArmKind pattern_kind; - TypePtr exact_type; // for MatchArmKind::exact_type; otherwise, nullptr + AnyTypeV pattern_type_node; // for MatchArmKind::exact_type; otherwise nullptr AnyExprV get_pattern_expr() const { return lhs; } AnyExprV get_body() const { return rhs; } // remember, it may be V Vertex* mutate() const { return const_cast(this); } - void assign_resolved_pattern(MatchArmKind pattern_kind, TypePtr exact_type, AnyExprV pattern_expr); + void assign_resolved_pattern(MatchArmKind pattern_kind, AnyExprV pattern_expr); - Vertex(SrcLocation loc, MatchArmKind pattern_kind, TypePtr exact_type, AnyExprV pattern_expr, AnyExprV body) + Vertex(SrcLocation loc, MatchArmKind pattern_kind, AnyTypeV pattern_type_node, AnyExprV pattern_expr, AnyExprV body) : ASTExprBinary(ast_match_arm, loc, pattern_expr, body) - , pattern_kind(pattern_kind), exact_type(exact_type) {} + , pattern_kind(pattern_kind), pattern_type_node(pattern_type_node) {} }; template<> @@ -829,7 +920,7 @@ template<> struct Vertex final : ASTExprBinary { StructPtr struct_ref = nullptr; // assigned at type inferring - bool has_explicit_ref() const { return lhs->type != ast_empty_expression; } + bool has_explicit_ref() const { return lhs->kind != ast_empty_expression; } auto get_explicit_ref() const { return lhs->as(); } AnyExprV get_maybe_reference() const { return lhs; } // ast_empty_expression or ast_reference auto get_body() const { return rhs->as(); } @@ -882,7 +973,7 @@ template<> // note, that for `return;` (without a value, meaning "void"), in AST, it's stored as empty expression struct Vertex : ASTStatementUnary { AnyExprV get_return_value() const { return child_as_expr(); } - bool has_return_value() const { return child->type != ast_empty_expression; } + bool has_return_value() const { return child->kind != ast_empty_expression; } Vertex(SrcLocation loc, AnyExprV child) : ASTStatementUnary(ast_return_statement, loc, child) {} @@ -944,7 +1035,7 @@ template<> // when thrown arg is missing, it's stored as empty expression struct Vertex final : ASTStatementVararg { AnyExprV get_thrown_code() const { return child_as_expr(0); } - bool has_thrown_arg() const { return child_as_expr(1)->type != ast_empty_expression; } + bool has_thrown_arg() const { return child_as_expr(1)->kind != ast_empty_expression; } AnyExprV get_thrown_arg() const { return child_as_expr(1); } Vertex(SrcLocation loc, AnyExprV thrown_code, AnyExprV thrown_arg) @@ -1028,14 +1119,11 @@ template<> // ast_instantiationT_item is manual substitution of generic T used in code, mostly for func calls // examples: `g()` / `t.tupleFirst()` / `f<(int, slice), builder>()` struct Vertex final : ASTOtherLeaf { - TypePtr substituted_type; + AnyTypeV type_node; - Vertex* mutate() const { return const_cast(this); } - void assign_resolved_type(TypePtr substituted_type); - - Vertex(SrcLocation loc, TypePtr substituted_type) + Vertex(SrcLocation loc, AnyTypeV type_node) : ASTOtherLeaf(ast_instantiationT_item, loc) - , substituted_type(substituted_type) {} + , type_node(type_node) {} }; template<> @@ -1053,20 +1141,15 @@ template<> // ast_parameter is a parameter of a function in its declaration // example: `fun f(a: int, mutate b: slice)` has 2 parameters struct Vertex final : ASTOtherLeaf { - LocalVarPtr param_ref = nullptr; // filled on resolve identifiers std::string_view param_name; - TypePtr declared_type; + AnyTypeV type_node; // always exists, typing parameters is mandatory bool declared_as_mutate; // declared as `mutate param_name` bool is_underscore() const { return param_name.empty(); } - Vertex* mutate() const { return const_cast(this); } - void assign_param_ref(LocalVarPtr param_ref); - void assign_resolved_type(TypePtr declared_type); - - Vertex(SrcLocation loc, std::string_view param_name, TypePtr declared_type, bool declared_as_mutate) + Vertex(SrcLocation loc, std::string_view param_name, AnyTypeV type_node, bool declared_as_mutate) : ASTOtherLeaf(ast_parameter, loc) - , param_name(param_name), declared_type(declared_type), declared_as_mutate(declared_as_mutate) {} + , param_name(param_name), type_node(type_node), declared_as_mutate(declared_as_mutate) {} }; template<> @@ -1107,28 +1190,27 @@ template<> // their body is either sequence (regular code function), or `asm`, or `builtin` struct Vertex final : ASTOtherVararg { auto get_identifier() const { return children.at(0)->as(); } - int get_num_params() const { return children.at(1)->as()->size(); } + int get_num_params() const { return children.at(1)->as()->size(); } auto get_param_list() const { return children.at(1)->as(); } auto get_param(int i) const { return children.at(1)->as()->get_param(i); } AnyV get_body() const { return children.at(2); } // ast_block_statement / ast_asm_body FunctionPtr fun_ref = nullptr; // filled after register - TypePtr declared_return_type; // filled at ast parsing; if unspecified (nullptr), means "auto infer" + AnyTypeV return_type_node; // if unspecified (nullptr), means "auto infer" V genericsT_list; // for non-generics it's nullptr td::RefInt256 method_id; // specified via @method_id annotation int flags; // from enum in FunctionData - bool is_asm_function() const { return children.at(2)->type == ast_asm_body; } - bool is_code_function() const { return children.at(2)->type == ast_block_statement; } - bool is_builtin_function() const { return children.at(2)->type == ast_empty_statement; } + bool is_asm_function() const { return children.at(2)->kind == ast_asm_body; } + bool is_code_function() const { return children.at(2)->kind == ast_block_statement; } + bool is_builtin_function() const { return children.at(2)->kind == ast_empty_statement; } Vertex* mutate() const { return const_cast(this); } void assign_fun_ref(FunctionPtr fun_ref); - void assign_resolved_type(TypePtr declared_return_type); - Vertex(SrcLocation loc, V name_identifier, V parameters, AnyV body, TypePtr declared_return_type, V genericsT_list, td::RefInt256 method_id, int flags) + Vertex(SrcLocation loc, V name_identifier, V parameters, AnyV body, AnyTypeV return_type_node, V genericsT_list, td::RefInt256 method_id, int flags) : ASTOtherVararg(ast_function_declaration, loc, {name_identifier, parameters, body}) - , declared_return_type(declared_return_type), genericsT_list(genericsT_list), method_id(std::move(method_id)), flags(flags) {} + , return_type_node(return_type_node), genericsT_list(genericsT_list), method_id(std::move(method_id)), flags(flags) {} }; template<> @@ -1136,18 +1218,17 @@ template<> // example: `global g: int;` // note, that globals don't have default values, since there is no single "entrypoint" for a contract struct Vertex final : ASTOtherVararg { - GlobalVarPtr var_ref = nullptr; // filled after register - TypePtr declared_type; // filled always, typing globals is mandatory + GlobalVarPtr glob_ref = nullptr; // filled after register + AnyTypeV type_node; // always exists, typing globals is mandatory auto get_identifier() const { return children.at(0)->as(); } Vertex* mutate() const { return const_cast(this); } - void assign_var_ref(GlobalVarPtr var_ref); - void assign_resolved_type(TypePtr declared_type); + void assign_glob_ref(GlobalVarPtr glob_ref); - Vertex(SrcLocation loc, V name_identifier, TypePtr declared_type) + Vertex(SrcLocation loc, V name_identifier, AnyTypeV type_node) : ASTOtherVararg(ast_global_var_declaration, loc, {name_identifier}) - , declared_type(declared_type) {} + , type_node(type_node) {} }; template<> @@ -1155,18 +1236,17 @@ template<> // example: `const op = 0x123;` struct Vertex final : ASTOtherVararg { GlobalConstPtr const_ref = nullptr; // filled after register - TypePtr declared_type; // not null for `const op: int = ...` + AnyTypeV type_node; // exists for `const op: int = rhs`, otherwise nullptr auto get_identifier() const { return children.at(0)->as(); } AnyExprV get_init_value() const { return child_as_expr(1); } Vertex* mutate() const { return const_cast(this); } void assign_const_ref(GlobalConstPtr const_ref); - void assign_resolved_type(TypePtr declared_type); - Vertex(SrcLocation loc, V name_identifier, TypePtr declared_type, AnyExprV init_value) + Vertex(SrcLocation loc, V name_identifier, AnyTypeV type_node, AnyExprV init_value) : ASTOtherVararg(ast_constant_declaration, loc, {name_identifier, init_value}) - , declared_type(declared_type) {} + , type_node(type_node) {} }; template<> @@ -1174,38 +1254,32 @@ template<> // example: `type UserId = int;` // see TypeDataAlias in type-system.h struct Vertex final : ASTOtherVararg { - AliasDefPtr alias_ref = nullptr; // filled after register, contains TypeDataAlias - TypePtr underlying_type; // at the right of `=` + AliasDefPtr alias_ref = nullptr; // filled after register + AnyTypeV underlying_type_node; // at the right of `=` auto get_identifier() const { return children.at(0)->as(); } Vertex* mutate() const { return const_cast(this); } void assign_alias_ref(AliasDefPtr alias_ref); - void assign_resolved_type(TypePtr underlying_type); - Vertex(SrcLocation loc, V name_identifier, TypePtr underlying_type) + Vertex(SrcLocation loc, V name_identifier, AnyTypeV underlying_type_node) : ASTOtherVararg(ast_type_alias_declaration, loc, {name_identifier}) - , underlying_type(underlying_type) {} + , underlying_type_node(underlying_type_node) {} }; template<> // ast_struct_field is one field at struct declaration // example: `struct Point { x: int, y: int }` is struct declaration, its body contains 2 fields struct Vertex final : ASTOtherVararg { - StructFieldPtr field_ref = nullptr; // filled after register - TypePtr declared_type; + AnyTypeV type_node; // always exists, typing struct fields is mandatory auto get_identifier() const { return children.at(0)->as(); } - bool has_default_value() const { return children.at(1)->type != ast_empty_expression; } + bool has_default_value() const { return children.at(1)->kind != ast_empty_expression; } auto get_default_value() const { return child_as_expr(1); } - Vertex* mutate() const { return const_cast(this); } - void assign_field_ref(StructFieldPtr field_ref); - void assign_resolved_type(TypePtr declared_type); - - Vertex(SrcLocation loc, V name_identifier, AnyExprV default_value, TypePtr declared_type) + Vertex(SrcLocation loc, V name_identifier, AnyExprV default_value, AnyTypeV type_node) : ASTOtherVararg(ast_struct_field, loc, {name_identifier, default_value}) - , declared_type(declared_type) {} + , type_node(type_node) {} }; template<> diff --git a/tolk/fwd-declarations.h b/tolk/fwd-declarations.h index ff2e32a36..cda17c424 100644 --- a/tolk/fwd-declarations.h +++ b/tolk/fwd-declarations.h @@ -19,10 +19,12 @@ namespace tolk { struct ASTNodeBase; +struct ASTNodeDeclaredTypeBase; struct ASTNodeExpressionBase; struct ASTNodeStatementBase; using AnyV = const ASTNodeBase*; +using AnyTypeV = const ASTNodeDeclaredTypeBase*; using AnyExprV = const ASTNodeExpressionBase*; using AnyStatementV = const ASTNodeStatementBase*; diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index 94a43e9fc..e73639b86 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -97,9 +97,9 @@ void GenericSubstitutionsDeduceForCall::consider_next_condition(TypePtr param_ty consider_next_condition(p_tensor->items[i], a_tensor->items[i]); } } - } else if (const auto* p_tuple = param_type->try_as()) { + } else if (const auto* p_tuple = param_type->try_as()) { // `arg: [int, T]` called as `f([5, cs])` => T is slice - if (const auto* a_tuple = arg_type->unwrap_alias()->try_as(); a_tuple && a_tuple->size() == p_tuple->size()) { + if (const auto* a_tuple = arg_type->unwrap_alias()->try_as(); a_tuple && a_tuple->size() == p_tuple->size()) { for (int i = 0; i < a_tuple->size(); ++i) { consider_next_condition(p_tuple->items[i], a_tuple->items[i]); } @@ -146,41 +146,6 @@ int GenericSubstitutionsDeduceForCall::get_first_not_deduced_idx() const { return -1; } -// clone the body of `f` replacing T everywhere with a substitution -// before: `fun f(v: T) { var cp: [T] = [v]; }` -// after: `fun f(v: int) { var cp: [int] = [v]; }` -// an instantiated function becomes a deep copy, all AST nodes are copied, no previous pointers left -class GenericFunctionReplicator final : public ASTReplicatorFunction { - const GenericsDeclaration* genericTs; - const std::vector& substitutionTs; - -protected: - using ASTReplicatorFunction::clone; - - TypePtr clone(TypePtr t) override { - return replace_genericT_with_deduced(t, genericTs, substitutionTs); - } - -public: - GenericFunctionReplicator(const GenericsDeclaration* genericTs, const std::vector& substitutionTs) - : genericTs(genericTs) - , substitutionTs(substitutionTs) { - } - - V clone_function_body(V v_function) override { - return createV( - v_function->loc, - clone(v_function->get_identifier()), - clone(v_function->get_param_list()), - clone(v_function->get_body()), - clone(v_function->declared_return_type), - nullptr, // a newly-created function is not generic - v_function->method_id, - v_function->flags - ); - } -}; - std::string GenericsDeclaration::as_human_readable() const { std::string result = "<"; for (const GenericsItem& item : itemsT) { @@ -202,16 +167,9 @@ int GenericsDeclaration::find_nameT(std::string_view nameT) const { return -1; } -// after creating a deep copy of `f` like `f`, its new and fresh body needs the previous pipeline to run -// for example, all local vars need to be registered as symbols, etc. -static void run_pipeline_for_instantiated_function(FunctionPtr inst_fun_ref) { - // these pipes are exactly the same as in tolk.cpp — all preceding (and including) type inferring - pipeline_resolve_identifiers_and_assign_symbols(inst_fun_ref); - pipeline_calculate_rvalue_lvalue(inst_fun_ref); - pipeline_infer_types_and_calls_and_fields(inst_fun_ref); -} - -std::string generate_instantiated_name(const std::string& orig_name, const std::vector& substitutions) { +// when cloning `f`, original name is "f", we need a new name for symtable and output +// name of an instantiated function will be "f" and similar (yes, with "<" symbol, it's okay to Fift) +static std::string generate_instantiated_name(const std::string& orig_name, const std::vector& substitutions) { // an instantiated function name will be "{orig_name}<{T1,T2,...}>" std::string name = orig_name; name += "<"; @@ -226,53 +184,56 @@ std::string generate_instantiated_name(const std::string& orig_name, const std:: return name; } -FunctionPtr instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, const std::string& inst_name, std::vector&& substitutionTs) { - tolk_assert(fun_ref->genericTs); +FunctionPtr instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, std::vector&& substitutionTs) { + tolk_assert(fun_ref->is_generic_function() && !fun_ref->is_method_id_not_empty()); - // if `f` was earlier instantiated, return it - if (const auto* existing = lookup_global_symbol(inst_name)) { - FunctionPtr inst_ref = existing->try_as(); - tolk_assert(inst_ref); - return inst_ref; + // fun_ref->name = "f", inst_name will be "f" and similar + std::string new_name = generate_instantiated_name(fun_ref->name, substitutionTs); + if (const Symbol* existing_sym = lookup_global_symbol(new_name)) { + FunctionPtr existing_ref = existing_sym->try_as(); + tolk_assert(existing_ref); + return existing_ref; } - std::vector parameters; - parameters.reserve(fun_ref->get_num_params()); - for (const LocalVarData& orig_p : fun_ref->parameters) { - parameters.emplace_back(orig_p.name, orig_p.loc, replace_genericT_with_deduced(orig_p.declared_type, fun_ref->genericTs, substitutionTs), orig_p.flags, orig_p.param_idx); - } - TypePtr declared_return_type = replace_genericT_with_deduced(fun_ref->declared_return_type, fun_ref->genericTs, substitutionTs); const GenericsInstantiation* instantiationTs = new GenericsInstantiation(loc, std::move(substitutionTs)); + ASTReplicatorFunction replicator; - if (fun_ref->is_asm_function()) { - FunctionData* inst_ref = new FunctionData(inst_name, fun_ref->loc, declared_return_type, std::move(parameters), fun_ref->flags, nullptr, instantiationTs, new FunctionBodyAsm, fun_ref->ast_root); - inst_ref->arg_order = fun_ref->arg_order; - inst_ref->ret_order = fun_ref->ret_order; - G.symtable.add_function(inst_ref); - G.all_functions.push_back(inst_ref); - run_pipeline_for_instantiated_function(inst_ref); - return inst_ref; - } - + // built-in functions don't have AST to clone, types of parameters don't exist in AST, etc. + // nevertheless, for outer code to follow a single algorithm, + // when calling `debugPrint(x)`, we clone it as "debugPrint", replace types, and insert into symtable if (fun_ref->is_builtin_function()) { - FunctionData* inst_ref = new FunctionData(inst_name, fun_ref->loc, declared_return_type, std::move(parameters), fun_ref->flags, nullptr, instantiationTs, fun_ref->body, fun_ref->ast_root); - inst_ref->arg_order = fun_ref->arg_order; - inst_ref->ret_order = fun_ref->ret_order; - G.symtable.add_function(inst_ref); - return inst_ref; - } - - GenericFunctionReplicator replicator(fun_ref->genericTs, instantiationTs->substitutions); - V inst_root = replicator.clone_function_body(fun_ref->ast_root->as()); + std::vector new_parameters; + new_parameters.reserve(fun_ref->get_num_params()); + for (const LocalVarData& orig_p : fun_ref->parameters) { + TypePtr new_param_type = replace_genericT_with_deduced(orig_p.declared_type, fun_ref->genericTs, instantiationTs->substitutions); + new_parameters.emplace_back(orig_p.name, orig_p.loc, new_param_type, orig_p.flags, orig_p.param_idx); + } + TypePtr new_return_type = replace_genericT_with_deduced(fun_ref->declared_return_type, fun_ref->genericTs, instantiationTs->substitutions); + FunctionData* new_fun_ref = new FunctionData(new_name, fun_ref->loc, new_return_type, std::move(new_parameters), fun_ref->flags, nullptr, instantiationTs, fun_ref->body, fun_ref->ast_root); + new_fun_ref->arg_order = fun_ref->arg_order; + new_fun_ref->ret_order = fun_ref->ret_order; + G.symtable.add_function(new_fun_ref); + return new_fun_ref; + } + + // for `f` (both asm and regular), create "f" with AST fully cloned + // it means, that types still contain T: `f(v: T): T`, but since type resolving knows + // it's instantiation, when resolving types, it substitutes T=int + V orig_root = fun_ref->ast_root->as(); + V new_name_ident = createV(orig_root->get_identifier()->loc, new_name); + V new_root = replicator.clone_function_ast(orig_root, new_name_ident); + + FunctionPtr new_fun_ref = pipeline_register_instantiated_generic_function(new_root, instantiationTs); + tolk_assert(new_fun_ref); + // body of a cloned function (it's cloned at type inferring step) needs the previous pipeline to run + // for example, all local vars need to be registered as symbols, etc. + // these pipes are exactly the same as in tolk.cpp — all preceding (and including) type inferring + pipeline_resolve_identifiers_and_assign_symbols(new_fun_ref); + pipeline_resolve_types_and_aliases(new_fun_ref); + pipeline_calculate_rvalue_lvalue(new_fun_ref); + pipeline_infer_types_and_calls_and_fields(new_fun_ref); - FunctionData* inst_ref = new FunctionData(inst_name, fun_ref->loc, declared_return_type, std::move(parameters), fun_ref->flags, nullptr, instantiationTs, new FunctionBodyCode, inst_root); - inst_ref->arg_order = fun_ref->arg_order; - inst_ref->ret_order = fun_ref->ret_order; - inst_root->mutate()->assign_fun_ref(inst_ref); - G.symtable.add_function(inst_ref); - G.all_functions.push_back(inst_ref); - run_pipeline_for_instantiated_function(inst_ref); - return inst_ref; + return new_fun_ref; } } // namespace tolk diff --git a/tolk/generics-helpers.h b/tolk/generics-helpers.h index 5ed245aff..140d27815 100644 --- a/tolk/generics-helpers.h +++ b/tolk/generics-helpers.h @@ -41,7 +41,6 @@ struct GenericsDeclaration { std::string as_human_readable() const; size_t size() const { return itemsT.size(); } - bool has_nameT(std::string_view nameT) const { return find_nameT(nameT) != -1; } int find_nameT(std::string_view nameT) const; std::string get_nameT(int idx) const { return static_cast(itemsT[idx].nameT); } }; @@ -96,7 +95,6 @@ struct GenericDeduceError final : std::exception { } }; -std::string generate_instantiated_name(const std::string& orig_name, const std::vector& substitutions); -FunctionPtr instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, const std::string& inst_name, std::vector&& substitutionTs); +FunctionPtr instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, std::vector&& substitutionTs); } // namespace tolk diff --git a/tolk/lexer.cpp b/tolk/lexer.cpp index 822abe931..32989b3fa 100644 --- a/tolk/lexer.cpp +++ b/tolk/lexer.cpp @@ -557,15 +557,6 @@ Lexer::Lexer(const SrcFile* file) next(); } -Lexer::Lexer(std::string_view text) - : file(nullptr) - , p_start(text.data()) - , p_end(p_start + text.size()) - , p_next(p_start) - , location() { - next(); -} - void Lexer::next() { while (cur_token_idx == last_token_idx && !is_eof()) { update_location(); diff --git a/tolk/lexer.h b/tolk/lexer.h index 6e9ab6a41..9c8545bf3 100644 --- a/tolk/lexer.h +++ b/tolk/lexer.h @@ -168,7 +168,6 @@ class Lexer { }; explicit Lexer(const SrcFile* file); - explicit Lexer(std::string_view text); Lexer(const Lexer&) = delete; Lexer &operator=(const Lexer&) = delete; diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index cd6e38f0c..2139ae714 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -296,7 +296,7 @@ static V calc_sink_leftmost_obj(V v) { } leftmost_obj = unwrap_not_null_operator(v_dot->get_obj()); } - return leftmost_obj->type == ast_reference ? leftmost_obj->as() : nullptr; + return leftmost_obj->kind == ast_reference ? leftmost_obj->as() : nullptr; } @@ -390,12 +390,12 @@ static std::vector pre_compile_tensor(CodeBlob& code, const std::vect static std::vector pre_compile_let(CodeBlob& code, AnyExprV lhs, AnyExprV rhs, SrcLocation loc) { // [lhs] = [rhs]; since type checking is ok, it's the same as "lhs = rhs" - if (lhs->type == ast_typed_tuple && rhs->type == ast_typed_tuple) { + if (lhs->kind == ast_bracket_tuple && rhs->kind == ast_bracket_tuple) { // note: there are no type transitions (adding nullability flag, etc.), since only 1-slot elements allowed in tuples LValContext local_lval; - std::vector left = pre_compile_tensor(code, lhs->as()->get_items(), &local_lval); + std::vector left = pre_compile_tensor(code, lhs->as()->get_items(), &local_lval); vars_modification_watcher.trigger_callbacks(left, loc); - std::vector rvect = pre_compile_tensor(code, rhs->as()->get_items()); + std::vector rvect = pre_compile_tensor(code, rhs->as()->get_items()); code.emplace_back(loc, Op::_Let, left, rvect); local_lval.after_let(std::move(left), code, loc); std::vector right = code.create_tmp_var(TypeDataTuple::create(), loc, "(tuple)"); @@ -403,12 +403,12 @@ static std::vector pre_compile_let(CodeBlob& code, AnyExprV lhs, AnyE return right; } // [lhs] = rhs; it's un-tuple to N left vars - if (lhs->type == ast_typed_tuple) { + if (lhs->kind == ast_bracket_tuple) { LValContext local_lval; - std::vector left = pre_compile_tensor(code, lhs->as()->get_items(), &local_lval); + std::vector left = pre_compile_tensor(code, lhs->as()->get_items(), &local_lval); vars_modification_watcher.trigger_callbacks(left, loc); std::vector right = pre_compile_expr(rhs, code, nullptr); - const TypeDataTypedTuple* inferred_tuple = rhs->inferred_type->unwrap_alias()->try_as(); + const TypeDataBrackets* inferred_tuple = rhs->inferred_type->unwrap_alias()->try_as(); std::vector types_list = inferred_tuple->items; std::vector rvect = code.create_tmp_var(TypeDataTensor::create(std::move(types_list)), rhs->loc, "(unpack-tuple)"); code.emplace_back(lhs->loc, Op::_UnTuple, rvect, std::move(right)); @@ -417,7 +417,7 @@ static std::vector pre_compile_let(CodeBlob& code, AnyExprV lhs, AnyE return right; } // small optimization: `var x = rhs` or `local_var = rhs` (90% cases), LValContext not needed actually - if (lhs->type == ast_local_var_lhs || (lhs->type == ast_reference && lhs->as()->sym->try_as())) { + if (lhs->kind == ast_local_var_lhs || (lhs->kind == ast_reference && lhs->as()->sym->try_as())) { std::vector left = pre_compile_expr(lhs, code, nullptr); // effectively, local_var->ir_idx vars_modification_watcher.trigger_callbacks(left, loc); std::vector right = pre_compile_expr(rhs, code, lhs->inferred_type); @@ -818,7 +818,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as() && original_type->try_as()) { + if (target_type->try_as() && original_type->try_as()) { tolk_assert(target_w == 1 && orig_w == 1); return rvect; } @@ -969,7 +969,7 @@ static std::vector process_binary_operator(V v, return transition_to_target_type(std::move(rvect), code, target_type, v); } - throw UnexpectedASTNodeType(v, "process_binary_operator"); + throw UnexpectedASTNodeKind(v, "process_binary_operator"); } static std::vector process_unary_operator(V v, CodeBlob& code, TypePtr target_type) { @@ -1001,14 +1001,14 @@ static std::vector process_ternary_operator(V v } static std::vector process_cast_as_operator(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { - TypePtr child_target_type = v->cast_to_type; + TypePtr child_target_type = v->type_node->resolved_type; std::vector rvect = pre_compile_expr(v->get_expr(), code, child_target_type, lval_ctx); return transition_to_target_type(std::move(rvect), code, target_type, v); } static std::vector process_is_type_operator(V v, CodeBlob& code, TypePtr target_type) { TypePtr lhs_type = v->get_expr()->inferred_type->unwrap_alias(); - TypePtr cmp_type = v->rhs_type->unwrap_alias(); + TypePtr cmp_type = v->type_node->resolved_type->unwrap_alias(); bool is_null_check = cmp_type == TypeDataNullLiteral::create(); // `v == null`, not `v is T` tolk_assert(!cmp_type->try_as()); // `v is int|slice` is a type checker error @@ -1061,7 +1061,7 @@ static std::vector process_match_expression(V v auto v_ith_arm = v->get_arm(i); std::vector eq_ith_ir_idx; if (is_match_by_type) { - TypePtr cmp_type = v_ith_arm->exact_type->unwrap_alias(); + TypePtr cmp_type = v_ith_arm->pattern_type_node->resolved_type->unwrap_alias(); tolk_assert(!cmp_type->try_as()); // `match` over `int|slice` is a type checker error eq_ith_ir_idx = pre_compile_is_type(code, lhs_type, cmp_type, subj_ir_idx, v_ith_arm->loc, "(arm-cond-eq)"); } else { @@ -1149,7 +1149,7 @@ static std::vector process_dot_access(V v, CodeBlob& return transition_to_target_type(std::move(rvect), code, target_type, v); } // `tupleVar.0` - if (obj_type->try_as() || obj_type->try_as()) { + if (obj_type->try_as() || obj_type->try_as()) { int index_at = std::get(v->target); // handle `tupleVar.0 = rhs`, "0 SETINDEX" will be called when this was is modified if (lval_ctx && !lval_ctx->is_rval_inside_lval() && calc_sink_leftmost_obj(v)) { @@ -1218,7 +1218,7 @@ static std::vector process_function_call(V v, Code if (delta_self) { args.push_back(v->get_dot_obj()); obj_leftmost = v->get_dot_obj(); - while (obj_leftmost->type == ast_function_call && obj_leftmost->as()->is_dot_call() && obj_leftmost->as()->fun_maybe && obj_leftmost->as()->fun_maybe->does_return_self()) { + while (obj_leftmost->kind == ast_function_call && obj_leftmost->as()->is_dot_call() && obj_leftmost->as()->fun_maybe && obj_leftmost->as()->fun_maybe->does_return_self()) { obj_leftmost = obj_leftmost->as()->get_dot_obj(); } } @@ -1306,7 +1306,7 @@ static std::vector process_tensor(V v, CodeBlob& code, Ty return transition_to_target_type(std::move(rvect), code, target_type, v); } -static std::vector process_typed_tuple(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { +static std::vector process_typed_tuple(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { if (lval_ctx) { // todo some time, make "var (a, [b,c]) = (1, [2,3])" work v->error("[...] can not be used as lvalue here"); } @@ -1400,7 +1400,7 @@ static std::vector process_empty_expression(V v } std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { - switch (v->type) { + switch (v->kind) { case ast_reference: return process_reference(v->as(), code, target_type, lval_ctx); case ast_assign: @@ -1431,8 +1431,8 @@ std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr targ return process_braced_expression(v->as(), code, target_type); case ast_tensor: return process_tensor(v->as(), code, target_type, lval_ctx); - case ast_typed_tuple: - return process_typed_tuple(v->as(), code, target_type, lval_ctx); + case ast_bracket_tuple: + return process_typed_tuple(v->as(), code, target_type, lval_ctx); case ast_object_literal: return process_object_literal(v->as(), code, target_type, lval_ctx); case ast_int_const: @@ -1452,7 +1452,7 @@ std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr targ case ast_empty_expression: return process_empty_expression(v->as(), code, target_type); default: - throw UnexpectedASTNodeType(v, "pre_compile_expr"); + throw UnexpectedASTNodeKind(v, "pre_compile_expr"); } } @@ -1646,7 +1646,7 @@ static void append_implicit_return_statement(SrcLocation loc_end, CodeBlob& code void process_any_statement(AnyV v, CodeBlob& code) { - switch (v->type) { + switch (v->kind) { case ast_block_statement: return process_block_statement(v->as(), code); case ast_return_statement: diff --git a/tolk/pipe-calc-rvalue-lvalue.cpp b/tolk/pipe-calc-rvalue-lvalue.cpp index b00a76696..b6a239116 100644 --- a/tolk/pipe-calc-rvalue-lvalue.cpp +++ b/tolk/pipe-calc-rvalue-lvalue.cpp @@ -95,7 +95,7 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { restore_state(saved); } - void visit(V v) override { + void visit(V v) override { mark_vertex(v); MarkingState saved = enter_rvalue_if_none(); parent::visit(v); diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 2b87d77cb..8c42b3f54 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -31,6 +31,11 @@ static std::string to_string(AnyExprV v_with_type) { return "`" + v_with_type->inferred_type->as_human_readable() + "`"; } +GNU_ATTRIBUTE_NOINLINE +static std::string to_string(std::string_view string_view) { + return static_cast(string_view); +} + GNU_ATTRIBUTE_NOINLINE static std::string expression_as_string(AnyExprV v) { if (auto v_ref = v->try_as()) { @@ -44,12 +49,6 @@ static std::string expression_as_string(AnyExprV v) { return "expression"; } -// fire a general "type mismatch" error, just a wrapper over `throw` -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire(FunctionPtr cur_f, SrcLocation loc, const std::string& message) { - throw ParseError(cur_f, loc, message); -} - // fire an error on `!cell` / `+slice` GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_cannot_apply_operator(FunctionPtr cur_f, SrcLocation loc, std::string_view operator_name, AnyExprV unary_expr) { @@ -117,28 +116,10 @@ static void handle_possible_compiler_internal_call(FunctionPtr cur_f, Vname == "__expect_type") { tolk_assert(v->get_num_args() == 2); - TypePtr expected_type = parse_type_from_string(v->get_arg(1)->get_expr()->as()->str_val); - if (expected_type->has_unresolved_inside()) { // only aliases can be inside - expected_type = expected_type->replace_children_custom([cur_f, v](TypePtr child) -> TypePtr { - if (const auto* as_unresolved = child->try_as()) { - const Symbol* sym = lookup_global_symbol(as_unresolved->text); - if (!sym) { - throw ParseError(cur_f, v->loc, "invalid __expect_type"); - } - if (AliasDefPtr alias_ref = sym->try_as()) { - return TypeDataAlias::create(alias_ref); - } - if (StructPtr struct_ref = sym->try_as()) { - return struct_ref->struct_type; - } - tolk_assert(false); - } - return child; - }); - } + std::string_view expected_type_str = v->get_arg(1)->get_expr()->as()->str_val; TypePtr expr_type = v->get_arg(0)->inferred_type; - if (expected_type->as_human_readable() != expr_type->as_human_readable()) { - fire(cur_f, v->loc, "__expect_type failed: expected " + to_string(expected_type) + ", got " + to_string(expr_type)); + if (expected_type_str != expr_type->as_human_readable()) { + fire(cur_f, v->loc, "__expect_type failed: expected `" + to_string(expected_type_str) + "`, got " + to_string(expr_type)); } } } @@ -282,23 +263,24 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { parent::visit(v->get_expr()); - if (!v->get_expr()->inferred_type->can_be_casted_with_as_operator(v->cast_to_type)) { - fire(cur_f, v->loc, "type " + to_string(v->get_expr()) + " can not be cast to " + to_string(v->cast_to_type)); + if (!v->get_expr()->inferred_type->can_be_casted_with_as_operator(v->type_node->resolved_type)) { + fire(cur_f, v->loc, "type " + to_string(v->get_expr()) + " can not be cast to " + to_string(v->type_node->resolved_type)); } } void visit(V v) override { parent::visit(v->get_expr()); + TypePtr rhs_type = v->type_node->resolved_type; - if (v->rhs_type->unwrap_alias()->try_as()) { // `v is T1 | T2` / `v is T?` is disallowed + if (rhs_type->unwrap_alias()->try_as()) { // `v is T1 | T2` / `v is T?` is disallowed fire(cur_f, v->loc, "union types are not allowed, use concrete types in `is`"); } if ((v->is_always_true && !v->is_negated) || (v->is_always_false && v->is_negated)) { - v->loc.show_warning(expression_as_string(v->get_expr()) + " is always " + to_string(v->rhs_type) + ", this condition is always " + (v->is_always_true ? "true" : "false")); + v->loc.show_warning(expression_as_string(v->get_expr()) + " is always " + to_string(rhs_type) + ", this condition is always " + (v->is_always_true ? "true" : "false")); } if ((v->is_always_false && !v->is_negated) || (v->is_always_true && v->is_negated)) { - v->loc.show_warning(expression_as_string(v->get_expr()) + " of type " + to_string(v->get_expr()) + " can never be " + to_string(v->rhs_type) + ", this condition is always " + (v->is_always_true ? "true" : "false")); + v->loc.show_warning(expression_as_string(v->get_expr()) + " of type " + to_string(v->get_expr()) + " can never be " + to_string(rhs_type) + ", this condition is always " + (v->is_always_true ? "true" : "false")); } } @@ -312,7 +294,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // if operator `!` used for non-nullable, probably a warning should be printed } - void visit(V v) override { + void visit(V v) override { parent::visit(v); for (int i = 0; i < v->size(); ++i) { @@ -328,7 +310,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { if (v->is_target_indexed_access()) { TypePtr obj_type = v->get_obj()->inferred_type->unwrap_alias(); - if (v->inferred_type->get_width_on_stack() != 1 && (obj_type->try_as() || obj_type->try_as())) { + if (v->inferred_type->get_width_on_stack() != 1 && (obj_type->try_as() || obj_type->try_as())) { fire_error_cannot_put_non1_stack_width_arg_to_tuple(cur_f, v->loc, v->inferred_type); } } @@ -405,7 +387,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // inside `var v: int = rhs` / `var _ = rhs` / `var v redef = rhs` (lhs is "v" / "_" / "v") if (auto lhs_var = lhs->try_as()) { - TypePtr declared_type = lhs_var->declared_type; // `var v: int = rhs` (otherwise, nullptr) + TypePtr declared_type = lhs_var->type_node ? lhs_var->type_node->resolved_type : nullptr; if (lhs_var->marked_as_redef) { tolk_assert(lhs_var->var_ref && lhs_var->var_ref->declared_type); declared_type = lhs_var->var_ref->declared_type; @@ -416,7 +398,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } } else { if (rhs_type == TypeDataNullLiteral::create()) { - fire_error_assign_always_null_to_variable(cur_f, err_loc->loc, lhs_var->var_ref->try_as(), corresponding_maybe_rhs && corresponding_maybe_rhs->type == ast_null_keyword); + fire_error_assign_always_null_to_variable(cur_f, err_loc->loc, lhs_var->var_ref->try_as(), corresponding_maybe_rhs && corresponding_maybe_rhs->kind == ast_null_keyword); } } return; @@ -441,15 +423,15 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { // `[v1, v2] = rhs` / `var [v1, v2] = rhs` (rhs may be `[1,2]` or `tupleVar` or `someF()`, doesn't matter) // dig recursively into v1 and v2 with corresponding rhs i-th item of a tuple - if (auto lhs_tuple = lhs->try_as()) { - const TypeDataTypedTuple* rhs_type_tuple = rhs_type->unwrap_alias()->try_as(); + if (auto lhs_tuple = lhs->try_as()) { + const TypeDataBrackets* rhs_type_tuple = rhs_type->unwrap_alias()->try_as(); if (!rhs_type_tuple) { fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to a tuple"); } if (lhs_tuple->size() != rhs_type_tuple->size()) { fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + ", sizes mismatch"); } - V rhs_tuple_maybe = corresponding_maybe_rhs ? corresponding_maybe_rhs->try_as() : nullptr; + V rhs_tuple_maybe = corresponding_maybe_rhs ? corresponding_maybe_rhs->try_as() : nullptr; for (int i = 0; i < lhs_tuple->size(); ++i) { process_assignment_lhs(lhs_tuple->get_item(i), rhs_type_tuple->items[i], rhs_tuple_maybe ? rhs_tuple_maybe->get_item(i) : nullptr); } @@ -497,7 +479,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { static bool is_expr_valid_as_return_self(AnyExprV return_expr) { // `return self` - if (return_expr->type == ast_reference && return_expr->as()->get_name() == "self") { + if (return_expr->kind == ast_reference && return_expr->as()->get_name() == "self") { return true; } // `return self.someMethod()` @@ -548,7 +530,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { } has_type_arm = true; - TypePtr lhs_type = v_arm->exact_type->unwrap_alias(); // `lhs_type => ...` + TypePtr lhs_type = v_arm->pattern_type_node->resolved_type->unwrap_alias(); // `lhs_type => ...` if (lhs_type->try_as()) { fire(cur_f, v_arm->loc, "wrong pattern matching: union types are not allowed, use concrete types in `match`"); } @@ -605,7 +587,7 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { missing += to_string(variant); } } - throw ParseError(cur_f, v->loc, "`match` does not cover all possible types; missing types are: " + missing); + fire(cur_f, v->loc, "`match` does not cover all possible types; missing types are: " + missing); } // `match` by expression, if it's not statement, should have `else` (unless it's match over bool with const true/false) if (has_expr_arm && !has_else_arm && !v->is_statement()) { diff --git a/tolk/pipe-check-rvalue-lvalue.cpp b/tolk/pipe-check-rvalue-lvalue.cpp index fed6de13d..aa274a436 100644 --- a/tolk/pipe-check-rvalue-lvalue.cpp +++ b/tolk/pipe-check-rvalue-lvalue.cpp @@ -29,12 +29,6 @@ namespace tolk { -// fire a general error, just a wrapper over `throw` -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire(FunctionPtr cur_f, SrcLocation loc, const std::string& message) { - throw ParseError(cur_f, loc, message); -} - GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_cannot_be_used_as_lvalue(FunctionPtr cur_f, AnyV v, const std::string& details) { // example: `f() = 32` diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp index a5e68c6bf..30d0ff562 100644 --- a/tolk/pipe-constant-folding.cpp +++ b/tolk/pipe-constant-folding.cpp @@ -71,7 +71,7 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { TokenType t = v->tok; // convert "-1" (tok_minus tok_int_const) to a const -1 - if (t == tok_minus && v->get_rhs()->type == ast_int_const) { + if (t == tok_minus && v->get_rhs()->kind == ast_int_const) { td::RefInt256 intval = v->get_rhs()->as()->intval; tolk_assert(!intval.is_null()); intval = -intval; @@ -81,16 +81,16 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { return create_int_const(v->loc, std::move(intval)); } // same for "+1" - if (t == tok_plus && v->get_rhs()->type == ast_int_const) { + if (t == tok_plus && v->get_rhs()->kind == ast_int_const) { return v->get_rhs(); } // `!true` / `!false` - if (t == tok_logical_not && v->get_rhs()->type == ast_bool_const) { + if (t == tok_logical_not && v->get_rhs()->kind == ast_bool_const) { return create_bool_const(v->loc, !v->get_rhs()->as()->bool_val); } // `!0` - if (t == tok_logical_not && v->get_rhs()->type == ast_int_const) { + if (t == tok_logical_not && v->get_rhs()->kind == ast_int_const) { return create_bool_const(v->loc, v->get_rhs()->as()->intval == 0); } @@ -101,7 +101,7 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { parent::replace(v); // `null == null` / `null != null` - if (v->get_expr()->type == ast_null_keyword && v->rhs_type == TypeDataNullLiteral::create()) { + if (v->get_expr()->kind == ast_null_keyword && v->type_node->resolved_type == TypeDataNullLiteral::create()) { return create_bool_const(v->loc, !v->is_negated); } @@ -138,7 +138,7 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { // replace `2 + 3 => ...` with `5 => ...` // non-constant expressions like `foo() => ...` fire an error here - if (v->pattern_kind == MatchArmKind::const_expression && v->get_pattern_expr()->type != ast_int_const) { + if (v->pattern_kind == MatchArmKind::const_expression && v->get_pattern_expr()->kind != ast_int_const) { check_expression_is_constant(v->get_pattern_expr()); } diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index ee2e0f27b..f1e5b4c25 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -125,12 +125,6 @@ static std::string to_string(std::string_view string_view) { return static_cast(string_view); } -// fire a general error, just a wrapper over `throw` -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire(FunctionPtr cur_f, SrcLocation loc, const std::string& message) { - throw ParseError(cur_f, loc, message); -} - // fire an error when `fun f(...) asm ...` is called with T=(int,int) or other non-1 width on stack // asm functions generally can't handle it, they expect T to be a TVM primitive // (in FunC, `forall` type just couldn't be unified with non-primitives; in Tolk, generic T is expectedly inferred) @@ -161,7 +155,7 @@ class InferTypesAndCallsAndFieldsVisitor final { GNU_ATTRIBUTE_ALWAYS_INLINE static void assign_inferred_type(AnyExprV dst, AnyExprV src) { #ifdef TOLK_DEBUG - tolk_assert(src->inferred_type != nullptr && !src->inferred_type->has_unresolved_inside() && !src->inferred_type->has_genericT_inside()); + tolk_assert(src->inferred_type != nullptr && !src->inferred_type->has_genericT_inside()); #endif dst->mutate()->assign_inferred_type(src->inferred_type); } @@ -169,28 +163,28 @@ class InferTypesAndCallsAndFieldsVisitor final { GNU_ATTRIBUTE_ALWAYS_INLINE static void assign_inferred_type(AnyExprV dst, TypePtr inferred_type) { #ifdef TOLK_DEBUG - tolk_assert(inferred_type != nullptr && !inferred_type->has_unresolved_inside() && !inferred_type->has_genericT_inside()); + tolk_assert(inferred_type != nullptr && !inferred_type->has_genericT_inside()); #endif dst->mutate()->assign_inferred_type(inferred_type); } static void assign_inferred_type(LocalVarPtr local_var_or_param, TypePtr inferred_type) { #ifdef TOLK_DEBUG - tolk_assert(inferred_type != nullptr && !inferred_type->has_unresolved_inside() && !inferred_type->has_genericT_inside()); + tolk_assert(inferred_type != nullptr && !inferred_type->has_genericT_inside()); #endif local_var_or_param->mutate()->assign_inferred_type(inferred_type); } static void assign_inferred_type(FunctionPtr fun_ref, TypePtr inferred_return_type, TypePtr inferred_full_type) { #ifdef TOLK_DEBUG - tolk_assert(inferred_return_type != nullptr && !inferred_return_type->has_unresolved_inside() && !inferred_return_type->has_genericT_inside()); + tolk_assert(inferred_return_type != nullptr && !inferred_return_type->has_genericT_inside()); #endif fun_ref->mutate()->assign_inferred_type(inferred_return_type, inferred_full_type); } // traverse children in any statement FlowContext process_any_statement(AnyV v, FlowContext&& flow) { - switch (v->type) { + switch (v->kind) { case ast_block_statement: return process_block_statement(v->as(), std::move(flow)); case ast_return_statement: @@ -220,7 +214,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // returns ExprFlow: out_facts that are "definitely known" after evaluating the whole expression // if used_as_condition, true_facts/false_facts are also calculated (don't calculate them always for optimization) ExprFlow infer_any_expr(AnyExprV v, FlowContext&& flow, bool used_as_condition, TypePtr hint = nullptr) { - switch (v->type) { + switch (v->kind) { case ast_int_const: return infer_int_const(v->as(), std::move(flow), used_as_condition); case ast_string_const: @@ -259,8 +253,8 @@ class InferTypesAndCallsAndFieldsVisitor final { return infer_function_call(v->as(), std::move(flow), used_as_condition, hint); case ast_tensor: return infer_tensor(v->as(), std::move(flow), used_as_condition, hint); - case ast_typed_tuple: - return infer_typed_tuple(v->as(), std::move(flow), used_as_condition, hint); + case ast_bracket_tuple: + return infer_typed_tuple(v->as(), std::move(flow), used_as_condition, hint); case ast_null_keyword: return infer_null_keyword(v->as(), std::move(flow), used_as_condition); case ast_match_expression: @@ -272,7 +266,7 @@ class InferTypesAndCallsAndFieldsVisitor final { case ast_empty_expression: return infer_empty_expression(v->as(), std::move(flow), used_as_condition); default: - throw UnexpectedASTNodeType(v, "infer_any_expr"); + throw UnexpectedASTNodeKind(v, "infer_any_expr"); } } @@ -323,7 +317,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (v->marked_as_redef) { assign_inferred_type(v, v->var_ref->declared_type); } else { - assign_inferred_type(v, v->declared_type ? v->declared_type : TypeDataUnknown::create()); + assign_inferred_type(v, v->type_node ? v->type_node->resolved_type : TypeDataUnknown::create()); } return ExprFlow(std::move(flow), used_as_condition); } @@ -357,14 +351,14 @@ class InferTypesAndCallsAndFieldsVisitor final { } assign_inferred_type(lhs, TypeDataTensor::create(std::move(types_list))); - } else if (auto lhs_tuple = lhs->try_as()) { + } else if (auto lhs_tuple = lhs->try_as()) { std::vector types_list; types_list.reserve(lhs_tuple->size()); for (int i = 0; i < lhs_tuple->size(); ++i) { flow = infer_left_side_of_assignment(lhs_tuple->get_item(i), std::move(flow)); types_list.push_back(lhs_tuple->get_item(i)->inferred_type); } - assign_inferred_type(lhs, TypeDataTypedTuple::create(std::move(types_list))); + assign_inferred_type(lhs, TypeDataBrackets::create(std::move(types_list))); } else if (auto lhs_par = lhs->try_as()) { flow = infer_left_side_of_assignment(lhs_par->get_expr(), std::move(flow)); @@ -399,7 +393,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // inside `var v: int = rhs` / `var _ = rhs` / `var v redef = rhs` (lhs is "v" / "_" / "v") if (auto lhs_var = lhs->try_as()) { - TypePtr declared_type = lhs_var->marked_as_redef ? lhs_var->var_ref->declared_type : lhs_var->declared_type; + TypePtr declared_type = lhs_var->marked_as_redef ? lhs_var->var_ref->declared_type : lhs_var->type_node ? lhs_var->type_node->resolved_type : nullptr; if (lhs_var->inferred_type == TypeDataUnknown::create()) { assign_inferred_type(lhs_var, rhs_type); assign_inferred_type(lhs_var->var_ref, rhs_type); @@ -426,8 +420,8 @@ class InferTypesAndCallsAndFieldsVisitor final { // `[v1, v2] = rhs` / `var [v1, v2] = rhs` (rhs may be `[1,2]` or `tupleVar` or `someF()`, doesn't matter) // dig recursively into v1 and v2 with corresponding rhs i-th item of a tuple - if (auto lhs_tuple = lhs->try_as()) { - const TypeDataTypedTuple* rhs_type_tuple = rhs_type->unwrap_alias()->try_as(); + if (auto lhs_tuple = lhs->try_as()) { + const TypeDataBrackets* rhs_type_tuple = rhs_type->unwrap_alias()->try_as(); std::vector types_list; types_list.reserve(lhs_tuple->size()); for (int i = 0; i < lhs_tuple->size(); ++i) { @@ -435,7 +429,7 @@ class InferTypesAndCallsAndFieldsVisitor final { process_assignment_lhs_after_infer_rhs(lhs_tuple->get_item(i), ith_rhs_type, out_flow); types_list.push_back(lhs_tuple->get_item(i)->inferred_type); } - assign_inferred_type(lhs, TypeDataTypedTuple::create(std::move(types_list))); + assign_inferred_type(lhs, TypeDataBrackets::create(std::move(types_list))); return; } @@ -619,8 +613,8 @@ class InferTypesAndCallsAndFieldsVisitor final { ExprFlow infer_cast_as_operator(V v, FlowContext&& flow, bool used_as_condition) { // for `expr as `, use this type for hint, so that `t.tupleAt(0) as int` is ok - ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false, v->cast_to_type); - assign_inferred_type(v, v->cast_to_type); + ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false, v->type_node->resolved_type); + assign_inferred_type(v, v->type_node->resolved_type); if (!used_as_condition) { return after_expr; @@ -632,10 +626,10 @@ class InferTypesAndCallsAndFieldsVisitor final { ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false); assign_inferred_type(v, TypeDataBool::create()); - TypePtr rhs_type = v->rhs_type->unwrap_alias(); + TypePtr rhs_type = v->type_node->resolved_type->unwrap_alias(); TypePtr expr_type = v->get_expr()->inferred_type->unwrap_alias(); TypePtr non_rhs_type = calculate_type_subtract_rhs_type(expr_type, rhs_type); - if (expr_type->equal_to(v->rhs_type)) { // `expr is ` is always true + if (expr_type->equal_to(rhs_type)) { // `expr is ` is always true v->mutate()->assign_always_true_or_false(v->is_negated ? 2 : 1); } else if (non_rhs_type == TypeDataNever::create()) { // `expr is ` is always false v->mutate()->assign_always_true_or_false(v->is_negated ? 1 : 2); @@ -760,7 +754,7 @@ class InferTypesAndCallsAndFieldsVisitor final { std::vector substitutions; substitutions.reserve(instantiationT_list->size()); for (int i = 0; i < instantiationT_list->size(); ++i) { - substitutions.push_back(instantiationT_list->get_item(i)->substituted_type); + substitutions.push_back(instantiationT_list->get_item(i)->type_node->resolved_type); } return substitutions; @@ -782,10 +776,9 @@ class InferTypesAndCallsAndFieldsVisitor final { } } - std::string inst_name = generate_instantiated_name(fun_ref->name, substitutionTs); // make deep clone of `f` with substitutionTs // (if `f` was already instantiated, it will be immediately returned from a symbol table) - return instantiate_generic_function(loc, fun_ref, inst_name, std::move(substitutionTs)); + return instantiate_generic_function(loc, fun_ref, std::move(substitutionTs)); } ExprFlow infer_dot_access(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { @@ -833,7 +826,7 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, inferred_type); return ExprFlow(std::move(flow), used_as_condition); } - if (const auto* t_tuple = unwrapped_obj_type->try_as()) { + if (const auto* t_tuple = unwrapped_obj_type->try_as()) { if (index_at >= t_tuple->size()) { fire(cur_f, v_ident->loc, "invalid tuple index, expected 0.." + std::to_string(t_tuple->items.size() - 1)); } @@ -1095,8 +1088,8 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(flow), used_as_condition); } - ExprFlow infer_typed_tuple(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { - const TypeDataTypedTuple* tuple_hint = hint ? hint->unwrap_alias()->try_as() : nullptr; + ExprFlow infer_typed_tuple(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + const TypeDataBrackets* tuple_hint = hint ? hint->unwrap_alias()->try_as() : nullptr; std::vector types_list; types_list.reserve(v->get_items().size()); for (int i = 0; i < v->size(); ++i) { @@ -1104,7 +1097,7 @@ class InferTypesAndCallsAndFieldsVisitor final { flow = infer_any_expr(item, std::move(flow), false, tuple_hint && i < tuple_hint->size() ? tuple_hint->items[i] : nullptr).out_flow; types_list.emplace_back(item->inferred_type); } - assign_inferred_type(v, TypeDataTypedTuple::create(std::move(types_list))); + assign_inferred_type(v, TypeDataBrackets::create(std::move(types_list))); return ExprFlow(std::move(flow), used_as_condition); } @@ -1125,7 +1118,7 @@ class InferTypesAndCallsAndFieldsVisitor final { auto v_arm = v->get_arm(i); FlowContext arm_flow = infer_any_expr(v_arm->get_pattern_expr(), arms_entry_facts.clone(), false, nullptr).out_flow; if (s_expr && v_arm->pattern_kind == MatchArmKind::exact_type) { - arm_flow.register_known_type(s_expr, v_arm->exact_type); + arm_flow.register_known_type(s_expr, v_arm->pattern_type_node->resolved_type); } arm_flow = infer_any_expr(v_arm->get_body(), std::move(arm_flow), false, hint).out_flow; match_out_flow = i == 0 ? std::move(arm_flow) : FlowContext::merge_flow(std::move(match_out_flow), std::move(arm_flow)); @@ -1218,7 +1211,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } v->mutate()->assign_struct_ref(struct_ref); - assign_inferred_type(v, struct_ref->struct_type); + assign_inferred_type(v, TypeDataStruct::create(struct_ref)); assign_inferred_type(v->get_body(), v); return ExprFlow(std::move(flow), used_as_condition); } @@ -1241,7 +1234,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // (but don't print a warning if it's already unreachable, for example we're inside always-false if) bool initially_unreachable = flow.is_unreachable(); for (AnyV item : v->get_items()) { - if (flow.is_unreachable() && !initially_unreachable && !v->first_unreachable && item->type != ast_empty_statement) { + if (flow.is_unreachable() && !initially_unreachable && !v->first_unreachable && item->kind != ast_empty_statement) { v->mutate()->assign_first_unreachable(item); // a warning will be printed later, after type checking } flow = process_any_statement(item, std::move(flow)); @@ -1435,15 +1428,17 @@ class InferTypesAndCallsAndFieldsVisitor final { // given `const a = 2 + 3` infer that it's int // for `const a: int = ...` still infer all sub expressions (to be checked in a later pipe) void start_visiting_constant(GlobalConstPtr const_ref) { - FlowContext const_flow; - infer_any_expr(const_ref->init_value, std::move(const_flow), false, const_ref->declared_type); + infer_any_expr(const_ref->init_value, FlowContext(), false, const_ref->declared_type); const_ref->mutate()->assign_inferred_type(const_ref->declared_type == nullptr ? const_ref->init_value->inferred_type : const_ref->declared_type); } // given struct field `a: int = 2 + 3` infer that default value is int, assign inferred_type to all nodes - void start_visiting_field_default(StructFieldPtr field_ref) { - FlowContext field_flow; - infer_any_expr(field_ref->default_value, std::move(field_flow), false, field_ref->declared_type); + void start_visiting_struct_fields(StructPtr struct_ref) { + for (StructFieldPtr field_ref : struct_ref->fields) { + if (field_ref->has_default_value()) { + infer_any_expr(field_ref->default_value, FlowContext(), false, field_ref->declared_type); + } + } } }; @@ -1519,11 +1514,7 @@ void pipeline_infer_types_and_calls_and_fields() { // infer types for default values in structs for (StructPtr struct_ref : get_all_declared_structs()) { - for (StructFieldPtr field_ref : struct_ref->fields) { - if (field_ref->has_default_value()) { - visitor.start_visiting_field_default(field_ref); - } - } + visitor.start_visiting_struct_fields(struct_ref); } } diff --git a/tolk/pipe-optimize-boolean-expr.cpp b/tolk/pipe-optimize-boolean-expr.cpp index 09282cfd9..4b05923c2 100644 --- a/tolk/pipe-optimize-boolean-expr.cpp +++ b/tolk/pipe-optimize-boolean-expr.cpp @@ -117,7 +117,7 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { if (v->tok == tok_eq || v->tok == tok_neq) { AnyExprV lhs = v->get_lhs(); AnyExprV rhs = v->get_rhs(); - if (expect_boolean(lhs->inferred_type) && rhs->type == ast_bool_const) { + if (expect_boolean(lhs->inferred_type) && rhs->kind == ast_bool_const) { // `boolVar == true` / `boolVar != false` if (rhs->as()->bool_val ^ (v->tok == tok_neq)) { return lhs; diff --git a/tolk/pipe-refine-lvalue-for-mutate.cpp b/tolk/pipe-refine-lvalue-for-mutate.cpp index a8b4f1ae5..5187457b2 100644 --- a/tolk/pipe-refine-lvalue-for-mutate.cpp +++ b/tolk/pipe-refine-lvalue-for-mutate.cpp @@ -35,7 +35,7 @@ namespace tolk { GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_invalid_mutate_arg_passed(AnyExprV v, FunctionPtr fun_ref, const LocalVarData& p_sym, bool called_as_method, bool arg_passed_as_mutate, AnyV arg_expr) { - std::string arg_str(arg_expr->type == ast_reference ? arg_expr->as()->get_name() : "obj"); + std::string arg_str(arg_expr->kind == ast_reference ? arg_expr->as()->get_name() : "obj"); // case: `loadInt(cs, 32)`; suggest: `cs.loadInt(32)` if (p_sym.is_mutate_parameter() && !arg_passed_as_mutate && !called_as_method && p_sym.param_idx == 0 && fun_ref->does_accept_self()) { @@ -92,7 +92,7 @@ class RefineLvalueForMutateArgumentsVisitor final : public ASTVisitorFunctionBod break; } } - bool will_be_extracted_as_tmp_var = leftmost_obj->type == ast_function_call; + bool will_be_extracted_as_tmp_var = leftmost_obj->kind == ast_function_call; if (!will_be_extracted_as_tmp_var) { leftmost_obj->mutate()->assign_lvalue_true(); v->get_dot_obj()->mutate()->assign_lvalue_true(); diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index 6bb93d55a..7a09defae 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -21,7 +21,6 @@ #include "compiler-state.h" #include "generics-helpers.h" #include "td/utils/crypto.h" -#include "type-system.h" #include /* @@ -60,10 +59,7 @@ static int calculate_method_id_by_func_name(std::string_view func_name) { return static_cast(crc & 0xffff) | 0x10000; } -static void validate_arg_ret_order_of_asm_function(V v_body, int n_params, TypePtr ret_type) { - if (!ret_type) { - v_body->error("asm function must declare return type (before asm instructions)"); - } +static void validate_arg_ret_order_of_asm_function(V v_body, int n_params) { if (n_params > 16) { v_body->error("asm function can have at most 16 parameters"); } @@ -114,7 +110,7 @@ static const GenericsDeclaration* construct_genericTs(V v_li } static void register_constant(V v) { - GlobalConstData* c_sym = new GlobalConstData(static_cast(v->get_identifier()->name), v->loc, v->declared_type, v->get_init_value()); + GlobalConstData* c_sym = new GlobalConstData(static_cast(v->get_identifier()->name), v->loc, v->type_node, v->get_init_value()); G.symtable.add_global_const(c_sym); G.all_constants.push_back(c_sym); @@ -122,15 +118,15 @@ static void register_constant(V v) { } static void register_global_var(V v) { - GlobalVarData* g_sym = new GlobalVarData(static_cast(v->get_identifier()->name), v->loc, v->declared_type); + GlobalVarData* g_sym = new GlobalVarData(static_cast(v->get_identifier()->name), v->loc, v->type_node); G.symtable.add_global_var(g_sym); G.all_global_vars.push_back(g_sym); - v->mutate()->assign_var_ref(g_sym); + v->mutate()->assign_glob_ref(g_sym); } static void register_type_alias(V v) { - AliasDefData* a_sym = new AliasDefData(static_cast(v->get_identifier()->name), v->loc, v->underlying_type); + AliasDefData* a_sym = new AliasDefData(static_cast(v->get_identifier()->name), v->loc, v->underlying_type_node); G.symtable.add_type_alias(a_sym); v->mutate()->assign_alias_ref(a_sym); @@ -144,9 +140,7 @@ static void register_struct(V v) { for (int i = 0; i < v_body->get_num_fields(); ++i) { auto v_field = v_body->get_field(i); AnyExprV default_value = v_field->has_default_value() ? v_field->get_default_value() : nullptr; - StructFieldData* field_ref = new StructFieldData(static_cast(v_field->get_identifier()->name), v_field->loc, i, v_field->declared_type, default_value); - fields.emplace_back(field_ref); - v_field->mutate()->assign_field_ref(field_ref); + fields.emplace_back(new StructFieldData(static_cast(v_field->get_identifier()->name), v_field->loc, i, v_field->type_node, default_value)); } StructData* s_sym = new StructData(static_cast(v->get_identifier()->name), v->loc, std::move(fields)); @@ -158,7 +152,7 @@ static void register_struct(V v) { static LocalVarData register_parameter(V v, int idx) { if (v->is_underscore()) { - return {"", v->loc, v->declared_type, 0, idx}; + return LocalVarData{"", v->loc, v->type_node, 0, idx}; } int flags = 0; @@ -168,30 +162,21 @@ static LocalVarData register_parameter(V v, int idx) { if (!v->declared_as_mutate && idx == 0 && v->param_name == "self") { flags |= LocalVarData::flagImmutable; } - return LocalVarData(static_cast(v->param_name), v->loc, v->declared_type, flags, idx); + return LocalVarData(static_cast(v->param_name), v->loc, v->type_node, flags, idx); } -static void register_function(V v) { +static void register_function(V v, const GenericsInstantiation* instantiationTs = nullptr) { std::string_view func_name = v->get_identifier()->name; - // calculate TypeData of a function - std::vector arg_types; std::vector parameters; - int n_params = v->get_num_params(); int n_mutate_params = 0; - arg_types.reserve(n_params); - parameters.reserve(n_params); - for (int i = 0; i < n_params; ++i) { + parameters.reserve(v->get_num_params()); + for (int i = 0; i < v->get_num_params(); ++i) { auto v_param = v->get_param(i); - arg_types.emplace_back(v_param->declared_type); parameters.emplace_back(register_parameter(v_param, i)); n_mutate_params += static_cast(v_param->declared_as_mutate); } - const GenericsDeclaration* genericTs = nullptr; - if (v->genericsT_list) { - genericTs = construct_genericTs(v->genericsT_list); - } if (v->is_builtin_function()) { const Symbol* sym = lookup_global_symbol(func_name); FunctionPtr fun_ref = sym ? sym->try_as() : nullptr; @@ -202,15 +187,15 @@ static void register_function(V v) { return; } - if (G.is_verbosity(1) && v->is_code_function()) { - std::cerr << "fun " << func_name << " : " << v->declared_return_type << std::endl; - } - - FunctionBody f_body = v->get_body()->type == ast_block_statement ? static_cast(new FunctionBodyCode) : static_cast(new FunctionBodyAsm); - FunctionData* f_sym = new FunctionData(static_cast(func_name), v->loc, v->declared_return_type, std::move(parameters), 0, genericTs, nullptr, f_body, v); + const GenericsDeclaration* genericTs = v->genericsT_list ? construct_genericTs(v->genericsT_list) : nullptr; + FunctionBody f_body = v->get_body()->kind == ast_block_statement ? static_cast(new FunctionBodyCode) : static_cast(new FunctionBodyAsm); + FunctionData* f_sym = new FunctionData(static_cast(func_name), v->loc, v->return_type_node, std::move(parameters), 0, genericTs, instantiationTs, f_body, v); - if (const auto* v_asm = v->get_body()->try_as()) { - validate_arg_ret_order_of_asm_function(v_asm, v->get_num_params(), v->declared_return_type); + if (auto v_asm = v->get_body()->try_as()) { + if (!v->return_type_node) { + v_asm->error("asm function must declare return type (before asm instructions)"); + } + validate_arg_ret_order_of_asm_function(v_asm, v->get_num_params()); f_sym->arg_order = v_asm->arg_order; f_sym->ret_order = v_asm->ret_order; } @@ -248,7 +233,7 @@ static void iterate_through_file_symbols(const SrcFile* file) { tolk_assert(file && file->ast); for (AnyV v : file->ast->as()->get_toplevel_declarations()) { - switch (v->type) { + switch (v->kind) { case ast_import_directive: // on `import "another-file.tolk"`, register symbols from that file at first // (for instance, it can calculate constants, which are used in init_val of constants in current file below import) @@ -282,4 +267,10 @@ void pipeline_register_global_symbols() { } } +FunctionPtr pipeline_register_instantiated_generic_function(AnyV cloned_v, const GenericsInstantiation* instantiationTs) { + auto v = cloned_v->as(); + register_function(v, instantiationTs); + return lookup_global_symbol(v->get_identifier()->name)->try_as(); +} + } // namespace tolk diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 545cf50fc..9f089b064 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -18,32 +18,21 @@ #include "platform-utils.h" #include "compiler-state.h" #include "src-file.h" -#include "generics-helpers.h" #include "ast.h" #include "ast-visitor.h" -#include "type-system.h" #include /* - * This pipe resolves identifiers (local variables and types) in all functions bodies. + * This pipe resolves identifiers (local variables, globals, constants, etc.) in all functions bodies. * It happens before type inferring, but after all global symbols are registered. * It means, that for any symbol `x` we can look up whether it's a global name or not. * - * About resolving variables. * Example: `var x = 10; x = 20;` both `x` point to one LocalVarData. * Example: `x = 20` undefined symbol `x` is also here (unless it's a global) * Variables scoping and redeclaration are also here. * Note, that `x` is stored as `ast_reference (ast_identifier "x")`. More formally, "references" are resolved. * "Reference" in AST, besides the identifier, stores optional generics instantiation. `x` is grammar-valid. * - * About resolving types. At the moment of parsing, `int`, `cell` and other predefined are parsed as TypeDataInt, etc. - * All the others are stored as TypeDataUnresolved, to be resolved here, after global symtable is filled. - * Example: `var x: T = 0` unresolved "T" is replaced by TypeDataGenericT inside `f`. - * Example: `f()` unresolved "MyAlias" is replaced by TypeDataAlias inside the reference. - * Example: `fun f(): KKK` unresolved "KKK" fires an error "unknown type name". - * When structures and type aliases are implemented, their resolving will also be done here. - * See finalize_type_data(). - * * Note, that functions/methods binding is NOT here. * In other words, for ast_function_call `beginCell()` and `t.tupleAt(0)`, their fun_ref is NOT filled here. * Functions/methods binding is done later, simultaneously with type inferring and generics instantiation. @@ -53,7 +42,6 @@ * As a result of this step, * * every V::sym is filled, pointing either to a local var/parameter, or to a global symbol * (exceptional for function calls and methods, their references are bound later) - * * all TypeData in all symbols is ready for analyzing, TypeDataUnresolved won't occur later in pipeline */ namespace tolk { @@ -67,11 +55,6 @@ static void fire_error_undefined_symbol(FunctionPtr cur_f, V v) } } -GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_unknown_type_name(FunctionPtr cur_f, SrcLocation loc, const std::string &text) { - throw ParseError(cur_f, loc, "unknown type name `" + text + "`"); -} - static void check_import_exists_when_using_sym(FunctionPtr cur_f, AnyV v_usage, const Symbol* used_sym) { SrcLocation sym_loc = used_sym->loc; if (!v_usage->loc.is_symbol_from_same_or_builtin_file(sym_loc)) { @@ -135,100 +118,13 @@ struct NameAndScopeResolver { } }; -struct TypeDataResolver { - static TypePtr finalize_type_data(FunctionPtr cur_f, TypePtr type_data, const GenericsDeclaration* genericTs) { - if (type_data) { - if (type_data->has_unresolved_inside()) { - type_data = resolve_identifiers_in_type_data(cur_f, type_data, genericTs); - } - } - return type_data; - } - - GNU_ATTRIBUTE_NOINLINE - static TypePtr resolve_identifiers_in_type_data(FunctionPtr cur_f, TypePtr type_data, const GenericsDeclaration* genericTs) { - return type_data->replace_children_custom([cur_f, genericTs](TypePtr child) { - if (const TypeDataUnresolved* un = child->try_as()) { - if (genericTs && genericTs->has_nameT(un->text)) { - std::string nameT = un->text; - return TypeDataGenericT::create(std::move(nameT)); - } - if (const Symbol* sym = lookup_global_symbol(un->text)) { - if (AliasDefPtr alias_ref = sym->try_as()) { - if (alias_ref->underlying_type->has_unresolved_inside()) { - resolve_and_mutate_type_alias(alias_ref); - } - return TypeDataAlias::create(alias_ref); - } - if (StructPtr struct_ref = sym->try_as()) { - if (!struct_ref->struct_type) { - resolve_and_mutate_struct_and_fields(struct_ref); - } - return struct_ref->struct_type; - } - } - if (un->text == "auto") { - throw ParseError(cur_f, un->loc, "`auto` type does not exist; just omit a type for local variable (will be inferred from assignment); parameters should always be typed"); - } - if (un->text == "self") { - throw ParseError(cur_f, un->loc, "`self` type can be used only as a return type of a function (enforcing it to be chainable)"); - } - fire_error_unknown_type_name(cur_f, un->loc, un->text); - } - return child; - }); - } - - static void resolve_and_mutate_type_alias(AliasDefPtr alias_ref) { - static std::vector called_stack; - - // prevent recursion like `type A = B; type B = A` - bool contains = std::find(called_stack.begin(), called_stack.end(), alias_ref) != called_stack.end(); - if (contains) { - throw ParseError(alias_ref->loc, "type `" + alias_ref->name + "` circularly references itself"); - } - - called_stack.push_back(alias_ref); - TypePtr underlying_type = finalize_type_data(nullptr, alias_ref->underlying_type, nullptr); - alias_ref->mutate()->assign_resolved_type(underlying_type); - called_stack.pop_back(); - } - - static void resolve_and_mutate_struct_and_fields(StructPtr struct_ref) { - static std::vector called_stack; - - // prevent recursion like `struct A { field: A }` - // currently, a struct is a tensor, and recursion always leads to infinite size (`A?` also, it's also on a stack) - // if there would be an annotation to store a struct in a tuple, then it has to be reconsidered - bool contains = std::find(called_stack.begin(), called_stack.end(), struct_ref) != called_stack.end(); - if (contains) { - throw ParseError(struct_ref->loc, "struct `" + struct_ref->name + "` size is infinity due to recursive fields"); - } - - called_stack.push_back(struct_ref); - for (int i = 0; i < struct_ref->get_num_fields(); ++i) { - StructFieldPtr field_ref = struct_ref->get_field(i); - TypePtr declared_type = finalize_type_data(nullptr, field_ref->declared_type, nullptr); - field_ref->mutate()->assign_resolved_type(declared_type); - } - struct_ref->mutate()->assign_resolved_type(TypeDataStruct::create(struct_ref)); - called_stack.pop_back(); - } -}; - -static TypePtr finalize_type_data(FunctionPtr cur_f, TypePtr type_data, const GenericsDeclaration* genericTs) { - return TypeDataResolver::finalize_type_data(cur_f, type_data, genericTs); -} - - class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { // more correctly this field shouldn't be static, but currently there is no need to make it a part of state static NameAndScopeResolver current_scope; static FunctionPtr cur_f; - static const GenericsDeclaration* current_genericTs; - static LocalVarPtr create_local_var_sym(std::string_view name, SrcLocation loc, TypePtr declared_type, bool immutable) { - LocalVarData* v_sym = new LocalVarData(static_cast(name), loc, declared_type, immutable * LocalVarData::flagImmutable, -1); + static LocalVarPtr create_local_var_sym(std::string_view name, SrcLocation loc, AnyTypeV declared_type_node, bool immutable) { + LocalVarData* v_sym = new LocalVarData(static_cast(name), loc, declared_type_node, immutable * LocalVarData::flagImmutable, -1); current_scope.add_local_var(v_sym); return v_sym; } @@ -253,9 +149,7 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { } v->mutate()->assign_var_ref(var_ref); } else { - TypePtr declared_type = finalize_type_data(cur_f, v->declared_type, current_genericTs); - LocalVarPtr var_ref = create_local_var_sym(v->get_name(), v->loc, declared_type, v->is_immutable); - v->mutate()->assign_resolved_type(declared_type); + LocalVarPtr var_ref = create_local_var_sym(v->get_name(), v->loc, v->type_node, v->is_immutable); v->mutate()->assign_var_ref(var_ref); } } @@ -276,15 +170,6 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { if (!sym->try_as()) { check_import_exists_when_using_sym(cur_f, v, sym); } - - // for `f` / `f`, resolve "MyAlias" and "T" - // (for function call `f()`, this v (ast_reference `f`) is callee) - if (auto v_instantiationTs = v->get_instantiationTs()) { - for (int i = 0; i < v_instantiationTs->size(); ++i) { - TypePtr substituted_type = finalize_type_data(cur_f, v_instantiationTs->get_item(i)->substituted_type, current_genericTs); - v_instantiationTs->get_item(i)->mutate()->assign_resolved_type(substituted_type); - } - } } void visit(V v) override { @@ -306,17 +191,14 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { switch (v->pattern_kind) { case MatchArmKind::exact_type: { - if (const TypeDataUnresolved* maybe_ident = v->exact_type->try_as()) { - if (const Symbol* sym = current_scope.lookup_symbol(maybe_ident->text); sym && !sym->try_as() && !sym->try_as()) { + if (auto maybe_ident = v->pattern_type_node->try_as()) { + if (const Symbol* sym = current_scope.lookup_symbol(maybe_ident->text); sym && sym->try_as()) { auto v_ident = createV(v->loc, sym->name); AnyExprV pattern_expr = createV(v->loc, v_ident, nullptr); parent::visit(pattern_expr); - v->mutate()->assign_resolved_pattern(MatchArmKind::const_expression, nullptr, pattern_expr); - return; + v->mutate()->assign_resolved_pattern(MatchArmKind::const_expression, pattern_expr); } } - TypePtr resolved_exact_type = finalize_type_data(cur_f, v->exact_type, current_genericTs); - v->mutate()->assign_resolved_pattern(MatchArmKind::exact_type, resolved_exact_type, v->get_pattern_expr()); break; } case MatchArmKind::const_expression: { @@ -329,30 +211,6 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { } } - void visit(V v) override { - // for `t.tupleAt` / `obj.method`, resolve "MyAlias" and "T" - // (for function call `t.tupleAt()`, this v (ast_dot_access `t.tupleAt`) is callee) - if (auto v_instantiationTs = v->get_instantiationTs()) { - for (int i = 0; i < v_instantiationTs->size(); ++i) { - TypePtr substituted_type = finalize_type_data(cur_f, v_instantiationTs->get_item(i)->substituted_type, current_genericTs); - v_instantiationTs->get_item(i)->mutate()->assign_resolved_type(substituted_type); - } - } - parent::visit(v->get_obj()); - } - - void visit(V v) override { - TypePtr cast_to_type = finalize_type_data(cur_f, v->cast_to_type, current_genericTs); - v->mutate()->assign_resolved_type(cast_to_type); - parent::visit(v->get_expr()); - } - - void visit(V v) override { - TypePtr rhs_type = finalize_type_data(cur_f, v->rhs_type, current_genericTs); - v->mutate()->assign_resolved_type(rhs_type); - parent::visit(v->get_expr()); - } - void visit(V v) override { if (v->empty()) { return; @@ -382,56 +240,41 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { public: bool should_visit_function(FunctionPtr fun_ref) override { - // this pipe is done just after parsing - // visit both asm and code functions, resolve identifiers in parameter/return types everywhere - // for generic functions, unresolved "T" will be replaced by TypeDataGenericT - return true; + return fun_ref->is_code_function(); } void start_visiting_function(FunctionPtr fun_ref, V v) override { cur_f = fun_ref; - current_genericTs = fun_ref->genericTs; + auto v_block = v->get_body()->as(); + current_scope.open_scope(v->loc); for (int i = 0; i < v->get_num_params(); ++i) { - const LocalVarData& param_var = fun_ref->parameters[i]; - TypePtr declared_type = finalize_type_data(cur_f, param_var.declared_type, fun_ref->genericTs); - v->get_param(i)->mutate()->assign_param_ref(¶m_var); - v->get_param(i)->mutate()->assign_resolved_type(declared_type); - param_var.mutate()->assign_resolved_type(declared_type); - } - TypePtr return_type = finalize_type_data(cur_f, fun_ref->declared_return_type, fun_ref->genericTs); - v->mutate()->assign_resolved_type(return_type); - fun_ref->mutate()->assign_resolved_type(return_type); - - if (fun_ref->is_code_function()) { - auto v_block = v->get_body()->as(); - current_scope.open_scope(v->loc); - for (int i = 0; i < v->get_num_params(); ++i) { - current_scope.add_local_var(&fun_ref->parameters[i]); - } - parent::visit(v_block); - current_scope.close_scope(v_block->loc_end); - tolk_assert(current_scope.scopes.empty()); + current_scope.add_local_var(&fun_ref->parameters[i]); } + parent::visit(v_block); + current_scope.close_scope(v_block->loc_end); + tolk_assert(current_scope.scopes.empty()); - current_genericTs = nullptr; cur_f = nullptr; } - void start_visiting_constant(V v) { + void start_visiting_constant(GlobalConstPtr const_ref) { // `const a = b`, resolve `b` - parent::visit(v->get_init_value()); + parent::visit(const_ref->init_value); } - void start_visiting_struct_field(V v) { + void start_visiting_struct_fields(StructPtr struct_ref) { // field `a: int = C`, resolve `C` - parent::visit(v->get_default_value()); + for (StructFieldPtr field_ref : struct_ref->fields) { + if (field_ref->has_default_value()) { + parent::visit(field_ref->default_value); + } + } } }; NameAndScopeResolver AssignSymInsideFunctionVisitor::current_scope; FunctionPtr AssignSymInsideFunctionVisitor::cur_f = nullptr; -const GenericsDeclaration* AssignSymInsideFunctionVisitor::current_genericTs = nullptr; void pipeline_resolve_identifiers_and_assign_symbols() { AssignSymInsideFunctionVisitor visitor; @@ -439,33 +282,17 @@ void pipeline_resolve_identifiers_and_assign_symbols() { for (AnyV v : file->ast->as()->get_toplevel_declarations()) { if (auto v_func = v->try_as()) { tolk_assert(v_func->fun_ref); - visitor.start_visiting_function(v_func->fun_ref, v_func); - - } else if (auto v_global = v->try_as()) { - TypePtr declared_type = finalize_type_data(nullptr, v_global->var_ref->declared_type, nullptr); - v_global->mutate()->assign_resolved_type(declared_type); - v_global->var_ref->mutate()->assign_resolved_type(declared_type); - - } else if (auto v_const = v->try_as()) { - visitor.start_visiting_constant(v_const); - if (v_const->declared_type) { - TypePtr declared_type = finalize_type_data(nullptr, v_const->const_ref->declared_type, nullptr); - v_const->mutate()->assign_resolved_type(declared_type); - v_const->const_ref->mutate()->assign_resolved_type(declared_type); + if (visitor.should_visit_function(v_func->fun_ref)) { + visitor.start_visiting_function(v_func->fun_ref, v_func); } - } else if (auto v_alias = v->try_as()) { - TypeDataResolver::resolve_and_mutate_type_alias(v_alias->alias_ref); - v_alias->mutate()->assign_resolved_type(v_alias->alias_ref->underlying_type); + } else if (auto v_const = v->try_as()) { + tolk_assert(v_const->const_ref); + visitor.start_visiting_constant(v_const->const_ref); } else if (auto v_struct = v->try_as()) { - TypeDataResolver::resolve_and_mutate_struct_and_fields(v_struct->struct_ref); - auto v_body = v_struct->get_struct_body(); - for (int i = 0; i < v_body->get_num_fields(); ++i) { - auto v_field = v_body->get_field(i); - visitor.start_visiting_struct_field(v_field); - v_field->mutate()->assign_resolved_type(v_struct->struct_ref->get_field(i)->declared_type); - } + tolk_assert(v_struct->struct_ref); + visitor.start_visiting_struct_fields(v_struct->struct_ref); } } } diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp new file mode 100644 index 000000000..3af32fa96 --- /dev/null +++ b/tolk/pipe-resolve-types.cpp @@ -0,0 +1,455 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "tolk.h" +#include "platform-utils.h" +#include "compiler-state.h" +#include "ast.h" +#include "ast-visitor.h" +#include "generics-helpers.h" +#include "type-system.h" +#include + +namespace tolk { + +/* + * This pipe transforms AST of types into TypePtr. + * It happens after all global symbols were registered, and all local references were bound. + * + * At the moment of parsing, `int`, `cell` and other were parsed as AnyTypeV (ast_type_leaf_text and others). + * Example: `var x: int = ...` to TypeDataInt + * Example: `fun f(a: cell): (int, User)` param to TypeDataCell, return type to TypeDataTensor(TypeDataInt, TypeDataStruct) + * Example: `var x: T = 0` to TypeDataGenericT inside `f` + * Example: `f()` to TypeDataAlias inside instantiation list + * Example: `fun f(): KKK` fires an error "unknown type name" + * + * Types resolving is done everywhere: inside functions bodies, in struct fields, inside globals declaration, etc. + * See finalize_type_node(). + * + * Note, that resolving T to TypeDataGenericT (and replacing T with substitution when instantiating a generic type) + * is also done here, see genericTs and instantiationTs. + */ + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_unknown_type_name(FunctionPtr cur_f, SrcLocation loc, std::string_view text) { + if (text == "auto") { + fire(cur_f, loc, "`auto` type does not exist; just omit a type for local variable (will be inferred from assignment); parameters should always be typed"); + } + if (text == "self") { + fire(cur_f, loc, "`self` type can be used only as a return type of a function (enforcing it to be chainable)"); + } + fire(cur_f, loc, "unknown type name `" + static_cast(text) + "`"); +} + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_void_type_not_allowed_inside_union(FunctionPtr cur_f, SrcLocation loc, TypePtr disallowed_variant) { + fire(cur_f, loc, "type `" + disallowed_variant->as_human_readable() + "` is not allowed inside a union"); +} + +static TypePtr parse_intN(std::string_view strN, bool is_unsigned) { + int n; + auto result = std::from_chars(strN.data() + 3 + static_cast(is_unsigned), strN.data() + strN.size(), n); + bool parsed = result.ec == std::errc() && result.ptr == strN.data() + strN.size(); + if (!parsed || n <= 0 || n > 256 + static_cast(is_unsigned)) { + return nullptr; // `int1000`, maybe it's user-defined alias, let it be unresolved + } + return TypeDataIntN::create(is_unsigned, false, n); +} + +static TypePtr parse_bytesN(std::string_view strN, bool is_bits) { + int n; + auto result = std::from_chars(strN.data() + 5 - static_cast(is_bits), strN.data() + strN.size(), n); + bool parsed = result.ec == std::errc() && result.ptr == strN.data() + strN.size(); + if (!parsed || n <= 0 || n > 1024) { + return nullptr; // `bytes9999`, maybe it's user-defined alias, let it be unresolved + } + return TypeDataBytesN::create(is_bits, n); +} + +static TypePtr try_parse_predefined_type(std::string_view str) { + switch (str.size()) { + case 3: + if (str == "int") return TypeDataInt::create(); + break; + case 4: + if (str == "cell") return TypeDataCell::create(); + if (str == "void") return TypeDataVoid::create(); + if (str == "bool") return TypeDataBool::create(); + if (str == "null") return TypeDataNullLiteral::create(); + break; + case 5: + if (str == "slice") return TypeDataSlice::create(); + if (str == "tuple") return TypeDataTuple::create(); + if (str == "coins") return TypeDataCoins::create(); + if (str == "never") return TypeDataNever::create(); + break; + case 7: + if (str == "builder") return TypeDataBuilder::create(); + break; + case 8: + if (str == "varint16") return TypeDataIntN::create(false, true, 16); + if (str == "varint32") return TypeDataIntN::create(false, true, 32); + break; + case 12: + if (str == "continuation") return TypeDataContinuation::create(); + break; + default: + break; + } + + if (str.starts_with("int")) { + if (TypePtr intN = parse_intN(str, false)) { + return intN; + } + } + if (str.size() > 4 && str.starts_with("uint")) { + if (TypePtr uintN = parse_intN(str, true)) { + return uintN; + } + } + if (str.size() > 4 && str.starts_with("bits")) { + if (TypePtr bitsN = parse_bytesN(str, true)) { + return bitsN; + } + } + if (str.size() > 5 && str.starts_with("bytes")) { + if (TypePtr bytesN = parse_bytesN(str, false)) { + return bytesN; + } + } + + return nullptr; +} + +class TypeNodesVisitorResolver { + FunctionPtr cur_f; // exists if we're inside its body + const GenericsDeclaration* genericTs; // `` if we're inside `f` or `f` + const GenericsInstantiation* instantiationTs; // `` if we're inside `f` + + TypePtr parse_ast_type_node(AnyTypeV v) { + switch (v->kind) { + case ast_type_leaf_text: { + std::string_view text = v->as()->text; + if (TypePtr predefined_type = try_parse_predefined_type(text)) { + return predefined_type; + } + if (genericTs) { + // if we're inside `f`, replace "T" with TypeDataGenericT + // if we're inside `f`, replace "T" with TypeDataInt (substitution) + if (int idx = genericTs->find_nameT(text); idx != -1) { + if (instantiationTs) { + return instantiationTs->substitutions[idx]; + } + return TypeDataGenericT::create(static_cast(text)); + } + } + if (const Symbol* sym = lookup_global_symbol(text)) { + if (TypePtr struct_or_other = try_resolve_user_defined_type(sym)) { + return struct_or_other; + } + } + fire_error_unknown_type_name(cur_f, v->as()->loc, text); + } + + case ast_type_question_nullable: { + TypePtr inner = finalize_type_node(v->as()->get_inner()); + TypePtr result = TypeDataUnion::create_nullable(inner); + if (const TypeDataUnion* t_union = result->try_as()) { + validate_resulting_union_type(t_union, cur_f, v->loc); + } + return result; + } + + case ast_type_parenthesis_tensor: { + std::vector items = finalize_type_node(v->as()->get_items()); + if (items.size() == 1) { + return items.front(); + } + return TypeDataTensor::create(std::move(items)); + } + + case ast_type_bracket_tuple: { + std::vector items = finalize_type_node(v->as()->get_items()); + return TypeDataBrackets::create(std::move(items)); + } + + case ast_type_arrow_callable: { + std::vector params_and_return = finalize_type_node(v->as()->get_params_and_return()); + TypePtr return_type = params_and_return.back(); + params_and_return.pop_back(); + return TypeDataFunCallable::create(std::move(params_and_return), return_type); + } + + case ast_type_vertical_bar_union: { + std::vector variants = finalize_type_node(v->as()->get_variants()); + TypePtr result = TypeDataUnion::create(std::move(variants)); + if (const TypeDataUnion* t_union = result->try_as()) { + validate_resulting_union_type(t_union, cur_f, v->loc); + } + return result; + } + + default: + throw UnexpectedASTNodeKind(v, "resolve_ast_type_node"); + } + } + + static TypePtr try_resolve_user_defined_type(const Symbol* sym) { + if (AliasDefPtr alias_ref = sym->try_as()) { + if (!alias_ref->was_visited_by_resolver()) { + visit_symbol(alias_ref); + } + return TypeDataAlias::create(alias_ref); + } + if (StructPtr struct_ref = sym->try_as()) { + if (!struct_ref->was_visited_by_resolver()) { + visit_symbol(struct_ref); + } + return TypeDataStruct::create(struct_ref); + } + return nullptr; + } + + static void validate_resulting_union_type(const TypeDataUnion* t_union, FunctionPtr cur_f, SrcLocation loc) { + for (TypePtr variant : t_union->variants) { + if (variant == TypeDataVoid::create() || variant == TypeDataNever::create()) { + fire_void_type_not_allowed_inside_union(cur_f, loc, variant); + } + } + } + +public: + + TypeNodesVisitorResolver(FunctionPtr cur_f, const GenericsDeclaration* genericTs, const GenericsInstantiation* instantiationTs) + : cur_f(cur_f) + , genericTs(genericTs) + , instantiationTs(instantiationTs) {} + + TypePtr finalize_type_node(AnyTypeV type_node) { +#ifdef TOLK_DEBUG + tolk_assert(type_node != nullptr); +#endif + TypePtr resolved_type = parse_ast_type_node(type_node); + type_node->mutate()->assign_resolved_type(resolved_type); + return resolved_type; + } + + std::vector finalize_type_node(const std::vector& type_node_array) { + std::vector result; + result.reserve(type_node_array.size()); + for (AnyTypeV v : type_node_array) { + result.push_back(finalize_type_node(v)); + } + return result; + } + + static void visit_symbol(GlobalVarPtr glob_ref) { + TypeNodesVisitorResolver visitor(nullptr, nullptr, nullptr); + TypePtr declared_type = visitor.finalize_type_node(glob_ref->type_node); + glob_ref->mutate()->assign_resolved_type(declared_type); + } + + static void visit_symbol(GlobalConstPtr const_ref) { + if (!const_ref->type_node) { + return; + } + + TypeNodesVisitorResolver visitor(nullptr, nullptr, nullptr); + TypePtr declared_type = visitor.finalize_type_node(const_ref->type_node); + const_ref->mutate()->assign_resolved_type(declared_type); + } + + static void visit_symbol(AliasDefPtr alias_ref) { + static std::vector called_stack; + + // prevent recursion like `type A = B; type B = A` + bool contains = std::find(called_stack.begin(), called_stack.end(), alias_ref) != called_stack.end(); + if (contains) { + throw ParseError(alias_ref->loc, "type `" + alias_ref->name + "` circularly references itself"); + } + + called_stack.push_back(alias_ref); + TypeNodesVisitorResolver visitor(nullptr, nullptr, nullptr); + TypePtr underlying_type = visitor.finalize_type_node(alias_ref->underlying_type_node); + alias_ref->mutate()->assign_resolved_type(underlying_type); + alias_ref->mutate()->assign_visited_by_resolver(); + called_stack.pop_back(); + } + + static void visit_symbol(StructPtr struct_ref) { + static std::vector called_stack; + + // prevent recursion like `struct A { field: A }` + // currently, a struct is a tensor, and recursion always leads to infinite size (`A?` also, it's also on a stack) + // if there would be an annotation to store a struct in a tuple, then it has to be reconsidered + bool contains = std::find(called_stack.begin(), called_stack.end(), struct_ref) != called_stack.end(); + if (contains) { + throw ParseError(struct_ref->loc, "struct `" + struct_ref->name + "` size is infinity due to recursive fields"); + } + + called_stack.push_back(struct_ref); + TypeNodesVisitorResolver visitor(nullptr, nullptr, nullptr); + for (int i = 0; i < struct_ref->get_num_fields(); ++i) { + StructFieldPtr field_ref = struct_ref->get_field(i); + TypePtr declared_type = visitor.finalize_type_node(field_ref->type_node); + field_ref->mutate()->assign_resolved_type(declared_type); + } + struct_ref->mutate()->assign_visited_by_resolver(); + called_stack.pop_back(); + } +}; + +class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { + static TypeNodesVisitorResolver type_nodes_visitor; + + static TypePtr finalize_type_node(AnyTypeV type_node) { + return type_nodes_visitor.finalize_type_node(type_node); + } + +protected: + + void visit(V v) override { + if (v->type_node) { + TypePtr declared_type = finalize_type_node(v->type_node); + v->var_ref->mutate()->assign_resolved_type(declared_type); + } + } + + void visit(V v) override { + // for `f` / `f`, resolve "MyAlias" and "T" + // (for function call `f()`, this v (ast_reference `f`) is callee) + if (auto v_instantiationTs = v->get_instantiationTs()) { + for (int i = 0; i < v_instantiationTs->size(); ++i) { + finalize_type_node(v_instantiationTs->get_item(i)->type_node); + } + } + } + + void visit(V v) override { + if (v->pattern_type_node) { + finalize_type_node(v->pattern_type_node); + } + parent::visit(v->get_pattern_expr()); + parent::visit(v->get_body()); + } + + void visit(V v) override { + // for `t.tupleAt` / `obj.method`, resolve "MyAlias" and "T" + // (for function call `t.tupleAt()`, this v (ast_dot_access `t.tupleAt`) is callee) + if (auto v_instantiationTs = v->get_instantiationTs()) { + for (int i = 0; i < v_instantiationTs->size(); ++i) { + finalize_type_node(v_instantiationTs->get_item(i)->type_node); + } + } + parent::visit(v->get_obj()); + } + + void visit(V v) override { + finalize_type_node(v->type_node); + parent::visit(v->get_expr()); + } + + void visit(V v) override { + finalize_type_node(v->type_node); + parent::visit(v->get_expr()); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return !fun_ref->is_builtin_function(); + } + + void start_visiting_function(FunctionPtr fun_ref, V v) override { + type_nodes_visitor = TypeNodesVisitorResolver(fun_ref, fun_ref->genericTs, fun_ref->instantiationTs); + + for (int i = 0; i < v->get_num_params(); ++i) { + const LocalVarData& param_var = fun_ref->parameters[i]; + TypePtr declared_type = finalize_type_node(param_var.type_node); + param_var.mutate()->assign_resolved_type(declared_type); + } + if (fun_ref->return_type_node) { + TypePtr declared_return_type = finalize_type_node(fun_ref->return_type_node); + fun_ref->mutate()->assign_resolved_type(declared_return_type); + } + + if (fun_ref->is_code_function()) { + parent::visit(v->get_body()->as()); + } + + type_nodes_visitor = TypeNodesVisitorResolver(nullptr, nullptr, nullptr); + } + + void start_visiting_constant(GlobalConstPtr const_ref) { + // `const a = 0 as int8`, resolve types there + // same for struct field `v: int8 = 0 as int8` + parent::visit(const_ref->init_value); + } + + void start_visiting_struct_fields(StructPtr struct_ref) { + // same for struct field `v: int8 = 0 as int8` + for (StructFieldPtr field_ref : struct_ref->fields) { + if (field_ref->has_default_value()) { + parent::visit(field_ref->default_value); + } + } + } +}; + +TypeNodesVisitorResolver ResolveTypesInsideFunctionVisitor::type_nodes_visitor(nullptr, nullptr, nullptr); + +void pipeline_resolve_types_and_aliases() { + ResolveTypesInsideFunctionVisitor visitor; + + for (const SrcFile* file : G.all_src_files) { + for (AnyV v : file->ast->as()->get_toplevel_declarations()) { + if (auto v_func = v->try_as()) { + tolk_assert(v_func->fun_ref); + if (visitor.should_visit_function(v_func->fun_ref)) { + visitor.start_visiting_function(v_func->fun_ref, v_func); + } + + } else if (auto v_global = v->try_as()) { + TypeNodesVisitorResolver::visit_symbol(v_global->glob_ref); + + } else if (auto v_const = v->try_as()) { + if (v_const->type_node) { + TypeNodesVisitorResolver::visit_symbol(v_const->const_ref); + } + visitor.start_visiting_constant(v_const->const_ref); + + } else if (auto v_alias = v->try_as()) { + if (!v_alias->alias_ref->was_visited_by_resolver()) { + TypeNodesVisitorResolver::visit_symbol(v_alias->alias_ref); + } + + } else if (auto v_struct = v->try_as()) { + if (!v_struct->struct_ref->was_visited_by_resolver()) { + TypeNodesVisitorResolver::visit_symbol(v_struct->struct_ref); + } + visitor.start_visiting_struct_fields(v_struct->struct_ref); + } + } + } +} + +void pipeline_resolve_types_and_aliases(FunctionPtr fun_ref) { + ResolveTypesInsideFunctionVisitor visitor; + if (visitor.should_visit_function(fun_ref)) { + visitor.start_visiting_function(fun_ref, fun_ref->ast_root->as()); + } +} + +} // namespace tolk diff --git a/tolk/pipeline.h b/tolk/pipeline.h index 0a71d751d..acd0fd87a 100644 --- a/tolk/pipeline.h +++ b/tolk/pipeline.h @@ -34,6 +34,7 @@ void pipeline_discover_and_parse_sources(const std::string& stdlib_filename, con void pipeline_register_global_symbols(); void pipeline_resolve_identifiers_and_assign_symbols(); +void pipeline_resolve_types_and_aliases(); void pipeline_calculate_rvalue_lvalue(); void pipeline_infer_types_and_calls_and_fields(); void pipeline_check_inferred_types(); @@ -49,7 +50,9 @@ void pipeline_generate_fif_output_to_std_cout(); // these pipes also can be called per-function individually // they are called for instantiated generics functions, when `f` is deeply cloned as `f` +FunctionPtr pipeline_register_instantiated_generic_function(AnyV cloned_v, const GenericsInstantiation* instantiationTs); void pipeline_resolve_identifiers_and_assign_symbols(FunctionPtr); +void pipeline_resolve_types_and_aliases(FunctionPtr); void pipeline_calculate_rvalue_lvalue(FunctionPtr); void pipeline_detect_unreachable_statements(FunctionPtr); void pipeline_infer_types_and_calls_and_fields(FunctionPtr); diff --git a/tolk/smart-casts-cfg.cpp b/tolk/smart-casts-cfg.cpp index b4974ef48..dd6a5abbf 100644 --- a/tolk/smart-casts-cfg.cpp +++ b/tolk/smart-casts-cfg.cpp @@ -178,8 +178,8 @@ static TypePtr calculate_type_lca(TypePtr a, TypePtr b, bool* became_union = nul return TypeDataTensor::create(std::move(types_lca)); } - const auto* tuple1 = a->try_as(); - const auto* tuple2 = b->try_as(); + const auto* tuple1 = a->try_as(); + const auto* tuple2 = b->try_as(); if (tuple1 && tuple2 && tuple1->size() == tuple2->size()) { std::vector types_lca; types_lca.reserve(tuple1->size()); @@ -190,7 +190,7 @@ static TypePtr calculate_type_lca(TypePtr a, TypePtr b, bool* became_union = nul } types_lca.push_back(next); } - return TypeDataTypedTuple::create(std::move(types_lca)); + return TypeDataBrackets::create(std::move(types_lca)); } if (const auto* a_alias = a->try_as()) { @@ -468,7 +468,7 @@ TypePtr calc_declared_type_before_smart_cast(AnyExprV v) { if (const auto* t_tensor = obj_type->try_as()) { return t_tensor->items[index_at]; } - if (const auto* t_tuple = obj_type->try_as()) { + if (const auto* t_tuple = obj_type->try_as()) { return t_tuple->items[index_at]; } } diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 97e48f7fc..e2a0a1836 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -22,7 +22,7 @@ namespace tolk { std::string FunctionData::as_human_readable() const { - if (!genericTs) { + if (!is_generic_function()) { return name; // if it's generic instantiation like `f`, its name is "f", not "f" } return name + genericTs->as_human_readable(); @@ -113,6 +113,10 @@ void LocalVarData::assign_inferred_type(TypePtr inferred_type) { this->declared_type = inferred_type; } +void AliasDefData::assign_visited_by_resolver() { + this->flags |= flagVisitedByResolver; +} + void AliasDefData::assign_resolved_type(TypePtr underlying_type) { this->underlying_type = underlying_type; } @@ -125,8 +129,8 @@ void StructFieldData::assign_default_value(AnyExprV default_value) { this->default_value = default_value; } -void StructData::assign_resolved_type(TypePtr struct_type) { - this->struct_type = struct_type; +void StructData::assign_visited_by_resolver() { + this->flags |= flagVisitedByResolver; } StructFieldPtr StructData::find_field(std::string_view field_name) const { diff --git a/tolk/symtable.h b/tolk/symtable.h index 9ab7f7b96..4252386c0 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -51,13 +51,21 @@ struct LocalVarData final : Symbol { flagImmutable = 2, // variable was declared via `val` (not `var`) }; - TypePtr declared_type; // either at declaration `var x:int`, or if omitted, from assigned value `var x=2` + AnyTypeV type_node; // either at declaration `var x:int`, or if omitted, from assigned value `var x=2` + TypePtr declared_type = nullptr; // = resolved type_node int flags; int param_idx; // 0...N for function parameters, -1 for local vars std::vector ir_idx; + LocalVarData(std::string name, SrcLocation loc, AnyTypeV type_node, int flags, int param_idx) + : Symbol(std::move(name), loc) + , type_node(type_node) + , flags(flags) + , param_idx(param_idx) { + } LocalVarData(std::string name, SrcLocation loc, TypePtr declared_type, int flags, int param_idx) : Symbol(std::move(name), loc) + , type_node(nullptr) // for built-in functions (their parameters) , declared_type(declared_type) , flags(flags) , param_idx(param_idx) { @@ -110,19 +118,31 @@ struct FunctionData final : Symbol { std::vector parameters; std::vector arg_order, ret_order; - TypePtr declared_return_type; // may be nullptr, meaning "auto infer" + AnyTypeV return_type_node; // may be nullptr, meaning "auto infer" + TypePtr declared_return_type = nullptr; // = resolved return_type_node TypePtr inferred_return_type = nullptr; // assigned on type inferring TypePtr inferred_full_type = nullptr; // assigned on type inferring, it's TypeDataFunCallable(params -> return) const GenericsDeclaration* genericTs; const GenericsInstantiation* instantiationTs; FunctionBody body; - AnyV ast_root; // V for user-defined (not builtin) + AnyV ast_root; // V for user-defined (not builtin) + FunctionData(std::string name, SrcLocation loc, AnyTypeV return_type_node, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsInstantiation* instantiationTs, FunctionBody body, AnyV ast_root) + : Symbol(std::move(name), loc) + , flags(initial_flags) + , parameters(std::move(parameters)) + , return_type_node(return_type_node) + , genericTs(genericTs) + , instantiationTs(instantiationTs) + , body(body) + , ast_root(ast_root) { + } FunctionData(std::string name, SrcLocation loc, TypePtr declared_return_type, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsInstantiation* instantiationTs, FunctionBody body, AnyV ast_root) : Symbol(std::move(name), loc) , flags(initial_flags) , parameters(std::move(parameters)) + , return_type_node(nullptr) // for built-in functions, defined in sources , declared_return_type(declared_return_type) , genericTs(genericTs) , instantiationTs(instantiationTs) @@ -146,7 +166,7 @@ struct FunctionData final : Symbol { bool is_asm_function() const { return std::holds_alternative(body); } bool is_builtin_function() const { return ast_root == nullptr; } - bool is_generic_function() const { return genericTs != nullptr; } + bool is_generic_function() const { return genericTs != nullptr && instantiationTs == nullptr; } bool is_instantiation_of_generic_function() const { return instantiationTs != nullptr; } bool is_inline() const { return flags & flagInline; } @@ -182,12 +202,13 @@ struct GlobalVarData final : Symbol { flagReallyUsed = 1, // calculated via dfs from used functions; unused globals are not codegenerated }; - TypePtr declared_type; // always exists, declaring globals without type is prohibited + AnyTypeV type_node; // `global a: int;` always exists, declaring globals without type is prohibited + TypePtr declared_type = nullptr; // = resolved type_node int flags = 0; - GlobalVarData(std::string name, SrcLocation loc, TypePtr declared_type) + GlobalVarData(std::string name, SrcLocation loc, AnyTypeV type_node) : Symbol(std::move(name), loc) - , declared_type(declared_type) { + , type_node(type_node) { } bool is_really_used() const { return flags & flagReallyUsed; } @@ -198,14 +219,15 @@ struct GlobalVarData final : Symbol { }; struct GlobalConstData final : Symbol { + AnyTypeV type_node; // exists for `const op: int = rhs`, otherwise nullptr + TypePtr declared_type = nullptr; // = resolved type_node + TypePtr inferred_type = nullptr; AnyExprV init_value; - TypePtr declared_type; // `const a: int = ...`; nullptr for `const a = ...` - TypePtr inferred_type = nullptr; // filled at type inferring pass - GlobalConstData(std::string name, SrcLocation loc, TypePtr declared_type, AnyExprV init_value) + GlobalConstData(std::string name, SrcLocation loc, AnyTypeV type_node, AnyExprV init_value) : Symbol(std::move(name), loc) - , init_value(init_value) - , declared_type(declared_type) { + , type_node(type_node) + , init_value(init_value) { } GlobalConstData* mutate() const { return const_cast(this); } @@ -215,21 +237,31 @@ struct GlobalConstData final : Symbol { }; struct AliasDefData final : Symbol { - TypePtr underlying_type; + enum { + flagVisitedByResolver = 1, + }; + + AnyTypeV underlying_type_node; + TypePtr underlying_type = nullptr; // = resolved underlying_type_node + int flags = 0; - AliasDefData(std::string name, SrcLocation loc, TypePtr underlying_type) + AliasDefData(std::string name, SrcLocation loc, AnyTypeV underlying_type_node) : Symbol(std::move(name), loc) - , underlying_type(underlying_type) { + , underlying_type_node(underlying_type_node) { } + bool was_visited_by_resolver() const { return flags & flagVisitedByResolver; } + AliasDefData* mutate() const { return const_cast(this); } + void assign_visited_by_resolver(); void assign_resolved_type(TypePtr underlying_type); }; struct StructFieldData final : Symbol { int field_idx; - TypePtr declared_type; - AnyExprV default_value; // nullptr if no default + AnyTypeV type_node; + TypePtr declared_type = nullptr; // = resolved type_node + AnyExprV default_value; // nullptr if no default bool has_default_value() const { return default_value != nullptr; } @@ -237,24 +269,30 @@ struct StructFieldData final : Symbol { void assign_resolved_type(TypePtr declared_type); void assign_default_value(AnyExprV default_value); - StructFieldData(std::string name, SrcLocation loc, int field_idx, TypePtr declared_type, AnyExprV default_value) + StructFieldData(std::string name, SrcLocation loc, int field_idx, AnyTypeV type_node, AnyExprV default_value) : Symbol(std::move(name), loc) , field_idx(field_idx) - , declared_type(declared_type) + , type_node(type_node) , default_value(default_value) { } }; struct StructData final : Symbol { + enum { + flagVisitedByResolver = 1, + }; + std::vector fields; - const TypeData* struct_type = nullptr; // it's TypeDataStruct, assigned at resolve identifiers + int flags = 0; int get_num_fields() const { return static_cast(fields.size()); } StructFieldPtr get_field(int i) const { return fields.at(i); } StructFieldPtr find_field(std::string_view field_name) const; + bool was_visited_by_resolver() const { return flags & flagVisitedByResolver; } + StructData* mutate() const { return const_cast(this); } - void assign_resolved_type(TypePtr struct_type); + void assign_visited_by_resolver(); StructData(std::string name, SrcLocation loc, std::vector&& fields) : Symbol(std::move(name), loc) diff --git a/tolk/tolk.cpp b/tolk/tolk.cpp index 71d1969de..fc38571bc 100644 --- a/tolk/tolk.cpp +++ b/tolk/tolk.cpp @@ -57,6 +57,7 @@ int tolk_proceed(const std::string &entrypoint_filename) { pipeline_register_global_symbols(); pipeline_resolve_identifiers_and_assign_symbols(); + pipeline_resolve_types_and_aliases(); pipeline_calculate_rvalue_lvalue(); pipeline_infer_types_and_calls_and_fields(); pipeline_check_inferred_types(); @@ -77,7 +78,7 @@ int tolk_proceed(const std::string &entrypoint_filename) { } catch (ParseError& error) { std::cerr << error << std::endl; return 2; - } catch (UnexpectedASTNodeType& error) { + } catch (UnexpectedASTNodeKind& error) { std::cerr << "fatal: " << error.what() << std::endl; std::cerr << "It's a compiler bug, please report to developers" << std::endl; return 2; diff --git a/tolk/tolk.h b/tolk/tolk.h index 74d8fdd9f..12bc6b694 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -34,6 +34,12 @@ namespace tolk { GNU_ATTRIBUTE_COLD GNU_ATTRIBUTE_NORETURN void on_assertion_failed(const char *description, const char *file_name, int line_number); +// fire a general error, just a wrapper over `throw` +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +inline void fire(FunctionPtr cur_f, SrcLocation loc, const std::string& message) { + throw ParseError(cur_f, loc, message); +} + /* * * ABSTRACT CODE diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index 402b79b38..7f2101ca4 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -249,7 +249,7 @@ TypePtr TypeDataTensor::create(std::vector&& items) { return hash.register_unique(new TypeDataTensor(hash.children_flags(), width_on_stack, std::move(items))); } -TypePtr TypeDataTypedTuple::create(std::vector&& items) { +TypePtr TypeDataBrackets::create(std::vector&& items) { TypeDataHasherForUnique hash(9189266157349499320ULL); for (TypePtr item : items) { hash.feed_child(item); @@ -259,7 +259,7 @@ TypePtr TypeDataTypedTuple::create(std::vector&& items) { if (TypePtr existing = hash.get_existing()) { return existing; } - return hash.register_unique(new TypeDataTypedTuple(hash.children_flags(), std::move(items))); + return hash.register_unique(new TypeDataBrackets(hash.children_flags(), std::move(items))); } TypePtr TypeDataIntN::create(bool is_unsigned, bool is_variadic, int n_bits) { @@ -296,12 +296,12 @@ TypePtr TypeDataUnion::create(std::vector&& variants) { return existing; } - // at the moment of parsing, union type can contain unresolved symbols + // for generics, like `f(v: T?)`, no type_id exists // in this case, don't try to flatten: we have no info - // after symbols resolving, a new union type (with resolved variants) will be created + // after instantiation, a new union type (with resolved variants) will be created bool not_ready_yet = false; for (TypePtr variant : variants) { - not_ready_yet |= variant->has_unresolved_inside() || variant->has_genericT_inside(); + not_ready_yet |= variant->has_genericT_inside(); } if (not_ready_yet) { TypePtr or_null = nullptr; @@ -368,17 +368,6 @@ TypePtr TypeDataUnion::create_nullable(TypePtr nullable) { return create({nullable, TypeDataNullLiteral::create()}); } -TypePtr TypeDataUnresolved::create(std::string&& text, SrcLocation loc) { - TypeDataHasherForUnique hash(3680147223540048162ULL); - hash.feed_string(text); - // hash.feed_hash(*reinterpret_cast(&loc)); - - if (TypePtr existing = hash.get_existing()) { - return existing; - } - return hash.register_unique(new TypeDataUnresolved(std::move(text), loc)); -} - // -------------------------------------------- // get_type_id() @@ -396,7 +385,8 @@ int TypeDataFunCallable::get_type_id() const { } int TypeDataGenericT::get_type_id() const { - return TypeIdCalculation::assign_type_id(this); + assert(false); // generics must have been instantiated in advance + throw Fatal("unexpected get_type_id() call"); } int TypeDataStruct::get_type_id() const { @@ -408,7 +398,7 @@ int TypeDataTensor::get_type_id() const { return TypeIdCalculation::assign_type_id(this); } -int TypeDataTypedTuple::get_type_id() const { +int TypeDataBrackets::get_type_id() const { assert(!has_genericT_inside()); return TypeIdCalculation::assign_type_id(this); } @@ -439,11 +429,6 @@ int TypeDataUnknown::get_type_id() const { throw Fatal("unexpected get_type_id() call"); } -int TypeDataUnresolved::get_type_id() const { - assert(false); // unresolved can be inside a union at parsing, but is resolved is advance - throw Fatal("unexpected get_type_id() call"); -} - // -------------------------------------------- // as_human_readable() @@ -485,7 +470,7 @@ std::string TypeDataTensor::as_human_readable() const { return result; } -std::string TypeDataTypedTuple::as_human_readable() const { +std::string TypeDataBrackets::as_human_readable() const { std::string result = "["; for (TypePtr item : items) { if (result.size() > 1) { @@ -534,48 +519,6 @@ std::string TypeDataUnion::as_human_readable() const { } -// -------------------------------------------- -// traverse() -// -// invokes a callback for TypeData itself and all its children -// only non-trivial implementations are here; by default (no children), `callback(this)` is executed -// - -void TypeDataAlias::traverse(const TraverserCallbackT& callback) const { - callback(this); - underlying_type->traverse(callback); -} - -void TypeDataFunCallable::traverse(const TraverserCallbackT& callback) const { - callback(this); - for (TypePtr param : params_types) { - param->traverse(callback); - } - return_type->traverse(callback); -} - -void TypeDataTensor::traverse(const TraverserCallbackT& callback) const { - callback(this); - for (TypePtr item : items) { - item->traverse(callback); - } -} - -void TypeDataTypedTuple::traverse(const TraverserCallbackT& callback) const { - callback(this); - for (TypePtr item : items) { - item->traverse(callback); - } -} - -void TypeDataUnion::traverse(const TraverserCallbackT& callback) const { - callback(this); - for (TypePtr variant : variants) { - variant->traverse(callback); - } -} - - // -------------------------------------------- // replace_children_custom() // @@ -602,7 +545,7 @@ TypePtr TypeDataTensor::replace_children_custom(const ReplacerCallbackT& callbac return callback(create(std::move(mapped))); } -TypePtr TypeDataTypedTuple::replace_children_custom(const ReplacerCallbackT& callback) const { +TypePtr TypeDataBrackets::replace_children_custom(const ReplacerCallbackT& callback) const { std::vector mapped; mapped.reserve(items.size()); for (TypePtr item : items) { @@ -778,8 +721,8 @@ bool TypeDataTensor::can_rhs_be_assigned(TypePtr rhs) const { return rhs == TypeDataNever::create(); } -bool TypeDataTypedTuple::can_rhs_be_assigned(TypePtr rhs) const { - if (const auto* as_tuple = rhs->try_as(); as_tuple && as_tuple->size() == size()) { +bool TypeDataBrackets::can_rhs_be_assigned(TypePtr rhs) const { + if (const auto* as_tuple = rhs->try_as(); as_tuple && as_tuple->size() == size()) { for (int i = 0; i < size(); ++i) { if (!items[i]->can_rhs_be_assigned(as_tuple->items[i])) { return false; @@ -852,11 +795,6 @@ bool TypeDataUnknown::can_rhs_be_assigned(TypePtr rhs) const { return true; } -bool TypeDataUnresolved::can_rhs_be_assigned(TypePtr rhs) const { - assert(false); - return false; -} - bool TypeDataNever::can_rhs_be_assigned(TypePtr rhs) const { return true; } @@ -1046,8 +984,8 @@ bool TypeDataTensor::can_be_casted_with_as_operator(TypePtr cast_to) const { return false; } -bool TypeDataTypedTuple::can_be_casted_with_as_operator(TypePtr cast_to) const { - if (const auto* to_tuple = cast_to->try_as(); to_tuple && to_tuple->size() == size()) { +bool TypeDataBrackets::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const auto* to_tuple = cast_to->try_as(); to_tuple && to_tuple->size() == size()) { for (int i = 0; i < size(); ++i) { if (!items[i]->can_be_casted_with_as_operator(to_tuple->items[i])) { return false; @@ -1124,10 +1062,6 @@ bool TypeDataUnknown::can_be_casted_with_as_operator(TypePtr cast_to) const { return cast_to->get_width_on_stack() == 1; } -bool TypeDataUnresolved::can_be_casted_with_as_operator(TypePtr cast_to) const { - return false; -} - bool TypeDataNever::can_be_casted_with_as_operator(TypePtr cast_to) const { return true; } @@ -1251,190 +1185,6 @@ TypePtr TypeDataUnion::calculate_exact_variant_to_fit_rhs(TypePtr rhs_type) cons } -// -------------------------------------------- -// parsing type from tokens -// -// here we implement parsing types (mostly after colon) to TypeData -// example: `var v: int` is TypeDataInt -// example: `var v: (builder?, [cell])` is TypeDataTensor(TypeDataUnion(TypeDataBuilder,TypeDataNullLiteral), TypeDataTypedTuple(TypeDataCell)) -// example: `fun f(): ()` is TypeDataTensor() (an empty one) -// -// note, that unrecognized type names (MyEnum, MyStruct, T) are parsed as TypeDataUnresolved, -// and later, when all files are parsed and all symbols registered, such identifiers are resolved -// example: `fun f(v: T)` at first v is TypeDataUnresolved("T"), later becomes TypeDataGenericT -// see finalize_type_data() -// -// note, that `self` does not name a type, it can appear only as a return value of a function (parsed specially) -// when `self` appears as a type, it's parsed as TypeDataUnresolved, and later an error is emitted -// - -static TypePtr parse_type_expression(Lexer& lex); - -std::vector parse_nested_type_list(Lexer& lex, TokenType tok_op, const char* s_op, TokenType tok_cl, const char* s_cl) { - lex.expect(tok_op, s_op); - std::vector sub_types; - while (true) { - if (lex.tok() == tok_cl) { // empty lists allowed - lex.next(); - break; - } - - sub_types.emplace_back(parse_type_expression(lex)); - if (lex.tok() == tok_comma) { - lex.next(); - } else if (lex.tok() != tok_cl) { - lex.unexpected(s_cl); - } - } - return sub_types; -} - -std::vector parse_nested_type_list_in_parenthesis(Lexer& lex) { - return parse_nested_type_list(lex, tok_oppar, "`(`", tok_clpar, "`)` or `,`"); -} - -static TypePtr parse_intN(std::string_view strN, bool is_unsigned) { - int n; - auto result = std::from_chars(strN.data() + 3 + static_cast(is_unsigned), strN.data() + strN.size(), n); - bool parsed = result.ec == std::errc() && result.ptr == strN.data() + strN.size(); - if (!parsed || n <= 0 || n > 256 + static_cast(is_unsigned)) { - return nullptr; // `int1000`, maybe it's user-defined alias, let it be unresolved - } - return TypeDataIntN::create(is_unsigned, false, n); -} - -static TypePtr parse_bytesN(std::string_view strN, bool is_bits) { - int n; - auto result = std::from_chars(strN.data() + 5 - static_cast(is_bits), strN.data() + strN.size(), n); - bool parsed = result.ec == std::errc() && result.ptr == strN.data() + strN.size(); - if (!parsed || n <= 0 || n > 1024) { - return nullptr; // `bytes9999`, maybe it's user-defined alias, let it be unresolved - } - return TypeDataBytesN::create(is_bits, n); -} - -static TypePtr parse_simple_type(Lexer& lex) { - switch (lex.tok()) { - case tok_self: - case tok_identifier: { - SrcLocation loc = lex.cur_location(); - std::string_view str = lex.cur_str(); - lex.next(); - switch (str.size()) { - case 3: - if (str == "int") return TypeDataInt::create(); - break; - case 4: - if (str == "cell") return TypeDataCell::create(); - if (str == "void") return TypeDataVoid::create(); - if (str == "bool") return TypeDataBool::create(); - break; - case 5: - if (str == "slice") return TypeDataSlice::create(); - if (str == "tuple") return TypeDataTuple::create(); - if (str == "coins") return TypeDataCoins::create(); - if (str == "never") return TypeDataNever::create(); - break; - case 7: - if (str == "builder") return TypeDataBuilder::create(); - break; - case 8: - if (str == "varint16") return TypeDataIntN::create(false, true, 16); - if (str == "varint32") return TypeDataIntN::create(false, true, 32); - break; - case 12: - if (str == "continuation") return TypeDataContinuation::create(); - break; - default: - break; - } - if (str.starts_with("int")) { - if (TypePtr intN = parse_intN(str, false)) { - return intN; - } - } - if (str.size() > 4 && str.starts_with("uint")) { - if (TypePtr uintN = parse_intN(str, true)) { - return uintN; - } - } - if (str.size() > 4 && str.starts_with("bits")) { - if (TypePtr bitsN = parse_bytesN(str, true)) { - return bitsN; - } - } - if (str.size() > 5 && str.starts_with("bytes")) { - if (TypePtr bytesN = parse_bytesN(str, false)) { - return bytesN; - } - } - return TypeDataUnresolved::create(std::string(str), loc); - } - case tok_null: - lex.next(); - return TypeDataNullLiteral::create(); - case tok_oppar: { - std::vector items = parse_nested_type_list_in_parenthesis(lex); - if (items.size() == 1) { - return items.front(); - } - return TypeDataTensor::create(std::move(items)); - } - case tok_opbracket: { - std::vector items = parse_nested_type_list(lex, tok_opbracket, "`[`", tok_clbracket, "`]` or `,`"); - return TypeDataTypedTuple::create(std::move(items)); - } - default: - lex.unexpected(""); - } -} - -static TypePtr parse_type_nullable(Lexer& lex) { - TypePtr result = parse_simple_type(lex); - - if (lex.tok() == tok_question) { - lex.next(); - result = TypeDataUnion::create_nullable(result); - } - - return result; -} - -static TypePtr parse_type_expression(Lexer& lex) { - TypePtr result = parse_type_nullable(lex); - - if (lex.tok() == tok_bitwise_or) { // `int | slice`, `Pair2 | (Pair3 | null)` - std::vector items; - items.emplace_back(result); - while (lex.tok() == tok_bitwise_or) { - lex.next(); - items.emplace_back(parse_type_nullable(lex)); - } - result = TypeDataUnion::create(std::move(items)); - } - - if (lex.tok() == tok_arrow) { // `int -> int`, `(cell, slice) -> void`, `int -> int -> int`, `int | cell -> void` - lex.next(); - TypePtr return_type = parse_type_expression(lex); - std::vector params_types = {result}; - if (const auto* as_tensor = result->try_as()) { - params_types = as_tensor->items; - } - result = TypeDataFunCallable::create(std::move(params_types), return_type); - } - - return result; -} - -TypePtr parse_type_from_tokens(Lexer& lex) { - return parse_type_expression(lex); -} - -// for internal usage only -TypePtr parse_type_from_string(std::string_view text) { - Lexer lex(text); - return parse_type_expression(lex); -} std::ostream& operator<<(std::ostream& os, TypePtr type_data) { return os << (type_data ? type_data->as_human_readable() : "(nullptr-type)"); diff --git a/tolk/type-system.h b/tolk/type-system.h index 6a6b1625a..cd0c34581 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -16,11 +16,10 @@ */ #pragma once -#include "src-file.h" #include "fwd-declarations.h" -#include -#include #include +#include +#include namespace tolk { @@ -30,21 +29,14 @@ namespace tolk { * Every unique TypeData is created only once, so for example TypeDataTensor::create(int, int) * returns one and the same pointer always. * - * In Tolk code, types after colon `var v: (int, T)` are parsed to TypeData. - * See parse_type_from_tokens(). - * So, AST nodes which can have declared types (local/global variables and others) store a pointer to TypeData. - * - * Type inferring also creates TypeData for inferred expressions. All AST expression nodes have inferred_type. + * Type inferring creates TypeData for inferred expressions. All AST expression nodes have inferred_type. * For example, `1 + 2`, both operands are TypeDataInt, its result is also TypeDataInt. * Type checking also uses TypeData. For example, `var i: slice = 1 + 2`, at first rhs (TypeDataInt) is inferred, * then lhs (TypeDataSlice from declaration) is checked whether rhs can be assigned. * See can_rhs_be_assigned(). * - * Note, that while initial parsing Tolk files to AST, known types (`int`, `cell`, etc.) are created as-is, - * but user-defined types (`T`, `MyStruct`, `MyAlias`) are saved as TypeDataUnresolved. - * After all symbols have been registered, resolving identifiers step is executed, where particularly - * all TypeDataUnresolved instances are converted to a resolved type. At inferring, no unresolved remain. - * For instance, `fun f(v: T)`, at first "T" of `v` is unresolved, and then converted to TypeDataGenericT. + * At the moment of parsing, types after colon `var v: (int, T)` are parsed to AST (AnyTypeV), + * and all symbols have been registered, AST representation resolved to TypeData, see pipe-resolve-types.cpp. */ class TypeData { // bits of flag_mask, to store often-used properties and return them without tree traversing @@ -58,8 +50,7 @@ class TypeData { enum flag_mask { flag_contains_unknown_inside = 1 << 1, flag_contains_genericT_inside = 1 << 2, - flag_contains_unresolved_inside = 1 << 3, - flag_contains_type_alias_inside = 1 << 4, + flag_contains_type_alias_inside = 1 << 3, }; explicit TypeData(int flags_with_children, int width_on_stack) @@ -89,10 +80,8 @@ class TypeData { bool has_unknown_inside() const { return flags & flag_contains_unknown_inside; } bool has_genericT_inside() const { return flags & flag_contains_genericT_inside; } - bool has_unresolved_inside() const { return flags & flag_contains_unresolved_inside; } bool has_type_alias_inside() const { return flags & flag_contains_type_alias_inside; } - using TraverserCallbackT = std::function; using ReplacerCallbackT = std::function; virtual int get_type_id() const = 0; @@ -104,10 +93,6 @@ class TypeData { return true; } - virtual void traverse(const TraverserCallbackT& callback) const { - callback(this); - } - virtual TypePtr replace_children_custom(const ReplacerCallbackT& callback) const { return callback(this); } @@ -135,7 +120,6 @@ class TypeDataAlias final : public TypeData { std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; - void traverse(const TraverserCallbackT& callback) const override; bool can_hold_tvm_null_instead() const override; }; @@ -313,7 +297,6 @@ class TypeDataFunCallable final : public TypeData { std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; - void traverse(const TraverserCallbackT& callback) const override; TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; }; @@ -382,18 +365,17 @@ class TypeDataTensor final : public TypeData { std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; - void traverse(const TraverserCallbackT& callback) const override; TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; bool can_hold_tvm_null_instead() const override; }; /* - * `[int, slice]` is TypeDataTypedTuple, a TVM 'tuple' under the hood, contained in 1 stack slot. + * `[int, slice]` is TypeDataBrackets, a TVM 'tuple' under the hood, contained in 1 stack slot. * Unlike TypeDataTuple (untyped tuples), it has a predefined inner structure and can be assigned as - * `var [i, cs] = [0, ""]` (where a and b become two separate variables on a stack, int and slice). + * `var [i, cs] = [0, ""]` (where i and cs become two separate variables on a stack, int and slice). */ -class TypeDataTypedTuple final : public TypeData { - TypeDataTypedTuple(int children_flags, std::vector&& items) +class TypeDataBrackets final : public TypeData { + TypeDataBrackets(int children_flags, std::vector&& items) : TypeData(children_flags, 1) , items(std::move(items)) {} @@ -408,7 +390,6 @@ class TypeDataTypedTuple final : public TypeData { std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; - void traverse(const TraverserCallbackT& callback) const override; TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; }; @@ -533,7 +514,6 @@ class TypeDataUnion final : public TypeData { std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; bool can_be_casted_with_as_operator(TypePtr cast_to) const override; - void traverse(const TraverserCallbackT& callback) const override; TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; bool can_hold_tvm_null_instead() const override; }; @@ -559,30 +539,6 @@ class TypeDataUnknown final : public TypeData { bool can_be_casted_with_as_operator(TypePtr cast_to) const override; }; -/* - * "Unresolved" is not actually a type — it's an intermediate state between parsing and resolving. - * At parsing to AST, unrecognized type names (MyEnum, MyStruct, T) are parsed as TypeDataUnresolved, - * and after all source files parsed and global symbols registered, they are replaced by actual ones. - * Example: `fun f(v: T)` at first v is TypeDataUnresolved("T"), later becomes TypeDataGenericT. - */ -class TypeDataUnresolved final : public TypeData { - TypeDataUnresolved(std::string&& text, SrcLocation loc) - : TypeData(flag_contains_unresolved_inside, -999999) - , text(std::move(text)) - , loc(loc) {} - -public: - const std::string text; - const SrcLocation loc; - - static TypePtr create(std::string&& text, SrcLocation loc); - - int get_type_id() const override; - std::string as_human_readable() const override { return text + "*"; } - bool can_rhs_be_assigned(TypePtr rhs) const override; - bool can_be_casted_with_as_operator(TypePtr cast_to) const override; -}; - /* * `never` is a special type meaning "no value can be hold". * Is may appear due to smart casts, for example `if (x == null && x != null)` makes x "never". @@ -629,10 +585,6 @@ class TypeDataVoid final : public TypeData { // -------------------------------------------- -class Lexer; -TypePtr parse_type_from_tokens(Lexer& lex); -TypePtr parse_type_from_string(std::string_view text); - void type_system_init(); } // namespace tolk From ff5e49403a7cc7df6004c7088d27526731b58c27 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 29 Apr 2025 14:23:19 +0300 Subject: [PATCH 231/388] Fix level calculation in ProofStorageStat --- crypto/vm/boc.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index d3ef57b74..837624563 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -1255,6 +1255,7 @@ bool VmStorageStat::add_storage(const CellSlice& cs) { } void ProofStorageStat::add_loaded_cell(const Ref& cell, td::uint8 max_level) { + max_level = std::min(max_level, Cell::max_level); auto& [status, size] = cells_[cell->get_hash(max_level)]; if (status == c_loaded) { return; From 0f9dd15b9a5ab1474470b8e8ee04983ee4efd606 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 27 Feb 2025 18:48:20 +0400 Subject: [PATCH 232/388] [Tolk] Implement generic structs and aliases Now structures can be generic: Container and similar. The same goes for type aliases: type A = Container. Like generic functions, every struct is instantiated, "Container" and "Container" become different symbols, and on instantiation they walk though the pipeline. Type inferring and checking are done only upon instantiation. --- tolk-tester/tests/generics-1.tolk | 17 +- tolk-tester/tests/generics-2.tolk | 464 ++++++++++++++++++ tolk-tester/tests/generics-3.tolk | 146 ++++++ .../tests/invalid-declaration/err-1285.tolk | 14 + .../tests/invalid-declaration/err-1378.tolk | 11 + .../tests/invalid-declaration/err-1876.tolk | 11 + .../tests/invalid-semantics/err-4081.tolk | 20 + .../tests/invalid-semantics/err-4119.tolk | 16 + .../tests/invalid-semantics/err-4193.tolk | 2 +- .../tests/invalid-semantics/err-4218.tolk | 14 + .../tests/invalid-semantics/err-4260.tolk | 2 +- .../tests/invalid-semantics/err-4291.tolk | 7 + .../tests/invalid-semantics/err-4378.tolk | 11 + .../tests/invalid-semantics/err-4387.tolk | 17 + .../tests/invalid-semantics/err-4429.tolk | 20 + .../tests/invalid-semantics/err-4493.tolk | 13 + .../tests/invalid-semantics/err-4512.tolk | 8 + .../tests/invalid-semantics/err-4519.tolk | 8 +- .../tests/invalid-semantics/err-4530.tolk | 17 + .../tests/invalid-semantics/err-4580.tolk | 12 + .../tests/invalid-semantics/err-4628.tolk | 16 + .../tests/invalid-semantics/err-4630.tolk | 13 + .../tests/invalid-semantics/err-4711.tolk | 22 + .../tests/invalid-semantics/err-4862.tolk | 18 + .../tests/invalid-semantics/err-4880.tolk | 2 +- .../tests/invalid-semantics/err-4925.tolk | 20 + .../tests/invalid-syntax/err-3173.tolk | 8 + .../tests/invalid-syntax/err-3450.tolk | 12 + .../tests/invalid-typing/err-6070.tolk | 26 + .../tests/invalid-typing/err-6110.tolk | 2 +- .../tests/invalid-typing/err-6145.tolk | 20 + .../tests/invalid-typing/err-6369.tolk | 2 +- .../tests/invalid-typing/err-6409.tolk | 10 + .../tests/invalid-typing/err-6473.tolk | 18 + .../tests/invalid-typing/err-6480.tolk | 27 + .../tests/invalid-typing/err-6618.tolk | 17 + .../tests/invalid-typing/err-6620.tolk | 16 + .../tests/invalid-typing/err-6642.tolk | 21 + .../tests/invalid-typing/err-6919.tolk | 13 + .../tests/invalid-typing/err-6948.tolk | 16 + tolk-tester/tests/var-apply-tests.tolk | 18 + .../tests/warnings-not-errors/warnings-4.tolk | 15 + tolk/ast-from-tokens.cpp | 89 +++- tolk/ast-replicator.h | 248 +++++----- tolk/ast-stringifier.h | 2 + tolk/ast-visitor.h | 4 +- tolk/ast.h | 41 +- tolk/fwd-declarations.h | 2 + tolk/generics-helpers.cpp | 277 ++++++++--- tolk/generics-helpers.h | 70 +-- tolk/lexer.cpp | 6 + tolk/lexer.h | 1 + tolk/pipe-ast-to-legacy.cpp | 9 +- tolk/pipe-check-inferred-types.cpp | 2 +- tolk/pipe-constant-folding.cpp | 2 +- tolk/pipe-infer-types-and-calls.cpp | 284 ++++++++--- tolk/pipe-register-symbols.cpp | 64 ++- tolk/pipe-resolve-identifiers.cpp | 4 + tolk/pipe-resolve-types.cpp | 154 ++++-- tolk/pipeline.h | 9 +- tolk/smart-casts-cfg.cpp | 6 +- tolk/smart-casts-cfg.h | 1 + tolk/symtable.cpp | 14 + tolk/symtable.h | 50 +- tolk/type-system.cpp | 87 +++- tolk/type-system.h | 40 +- 66 files changed, 2227 insertions(+), 401 deletions(-) create mode 100644 tolk-tester/tests/generics-2.tolk create mode 100644 tolk-tester/tests/generics-3.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1285.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1378.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1876.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4081.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4119.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4218.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4291.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4378.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4387.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4429.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4493.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4512.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4530.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4580.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4628.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4630.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4711.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4862.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4925.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3173.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3450.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6070.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6145.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6409.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6473.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6480.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6618.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6620.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6642.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6919.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6948.tolk create mode 100644 tolk-tester/tests/warnings-not-errors/warnings-4.tolk diff --git a/tolk-tester/tests/generics-1.tolk b/tolk-tester/tests/generics-1.tolk index feb7a8393..ba46f3440 100644 --- a/tolk-tester/tests/generics-1.tolk +++ b/tolk-tester/tests/generics-1.tolk @@ -17,8 +17,15 @@ fun getTwo(): X { return 2 as X; } fun takeInt(a: int) { return a; } +struct Wrapper { + value: T; +} +struct Err { + errPayload: T +} + @method_id(102) -fun test102(): (int, int, int, [int, int]) { +fun test102(): (int, int, int, [int, int], Wrapper< Wrapper >) { var a: int = getTwo(); var _: int = getTwo(); var b = getTwo() as int; @@ -32,7 +39,10 @@ fun test102(): (int, int, int, [int, int]) { ab_tup.0 = getTwo(); ab_tup.1.1 = getTwo(); var ab_tens_al: Tensor2Int = (getTwo(), getTwo()); - return (eq1(a), eq2(b), takeInt(getTwo()), [getTwo(), ab_tens.1.1]); + var cint: Wrapper = { value: getTwo() }; + var cuni: Wrapper | Err = Wrapper { value: getTwo() }; + cuni = Err { errPayload: getTwo() }; + return (eq1(a), eq2(b), takeInt(getTwo()), [getTwo(), ab_tens.1.1], eq1({ value: { value: getTwo() } } as Wrapper< Wrapper >)); } @method_id(103) @@ -204,7 +214,7 @@ fun main(x: int): (int, [Tup2Int]) { /** @testcase | 0 | 1 | 1 [ [ 1 2 ] ] @testcase | 101 | 0 | 0 0 0 [ 0 0 ] 0 0 0 [ 0 0 ] 0 0 0 [] -@testcase | 102 | | 2 2 2 [ 2 2 ] +@testcase | 102 | | 2 2 2 [ 2 2 ] 2 @testcase | 103 | 0 | 0 100 100 @testcase | 104 | 0 | [ 1 (null) 2 ] [ 2 -1 0 ] @testcase | 105 | | 3 @@ -224,6 +234,7 @@ fun main(x: int): (int, [Tup2Int]) { @fif_codegen DECLPROC eq1<[int,int]> @fif_codegen DECLPROC eq1 @fif_codegen DECLPROC eq1 +@fif_codegen DECLPROC eq1>> @fif_codegen DECLPROC getTwo @fif_codegen_avoid DECLPROC eq1 diff --git a/tolk-tester/tests/generics-2.tolk b/tolk-tester/tests/generics-2.tolk new file mode 100644 index 000000000..cde9d9929 --- /dev/null +++ b/tolk-tester/tests/generics-2.tolk @@ -0,0 +1,464 @@ +struct Empty {} + +struct Wrapper { + value: T; +} + +type PairAlias = Pair; +struct Pair { + first: T1, + second: T2 +} + +type MyInt = int; +type WrappedInt = Wrapper; +type WrapperAlias = Wrapper; + +struct DeepNested { + wrapper: WrapperAlias>; + pair: PairAlias; +} + + +fun eq(v: T): T { return v; } + +fun swap(p: Pair): Pair { + return { first: p.second, second: p.first }; +} + +fun wrap(value: T): Wrapper { + return { value }; +} + +fun wrap2(value: T) { + return WrapperAlias { value }; +} + +fun makePair(a: A, b: B) { + var c: Pair = { first: a, second: b }; + return c; +} + + +@method_id(101) +fun test1() { + var c1: Wrapper = { value: 23 }; + __expect_type(c1, "Wrapper"); + var c2: WrapperAlias = { value: 23 }; + __expect_type(c2, "WrapperAlias"); + __expect_type(c2 as Wrapper, "Wrapper"); + __expect_type(c2 as WrappedInt, "WrappedInt"); + return (c1, c2); +} + +@method_id(102) +fun test2(c: Wrapper) { + var c2 = Wrapper { value: c.value + 1 }; + var c3 = Wrapper { value: c.value + c2.value }; + __expect_type(c3, "Wrapper"); + var c4: Wrapper? = c.value > 10 ? null : { value: c.value }; + return (c, c2, c3, c4); +} + +@pure +fun getWrappervalue1(c: Wrapper) { + return c.value; +} + +@pure +fun getWrappervalue2(c: T) { + return c.value; +} + +@method_id(103) +fun test3() { + var c1 = wrap(10); + var c2 = wrap(10 as int?); + var c3 = wrap2<(int, int)?>(null); + __expect_type(c1, "Wrapper"); + __expect_type(c2, "Wrapper"); + __expect_type(c3, "Wrapper<(int, int)?>"); + return (getWrappervalue1(c1), getWrappervalue1(c2), getWrappervalue1(c3), 777, getWrappervalue2(c1), getWrappervalue2(c2), getWrappervalue2(c3)); +} + +@method_id(104) +fun test4() { + var c1 = wrap(10); + var c2 = wrap(c1); + var c3 = wrap(Wrapper { value: beginCell() }); + __expect_type(c2, "Wrapper>"); + __expect_type(c3, "Wrapper>"); + __expect_type(c3.value, "Wrapper"); + __expect_type(c3.value.value, "builder"); + c3.value.value.storeInt(1, 32).storeInt(2, 32).storeInt(3, 32); + var c4 = wrap2(Wrapper { value: getWrappervalue1(c3.value).endCell().beginParse() }); + __expect_type(c4, "Wrapper>"); + var sumOf1 = 0; + sumOf1 += getWrappervalue1(getWrappervalue1(c4)).loadInt(32); // a function returns a copy + sumOf1 += getWrappervalue2(c4.value).loadInt(32); // so original slice isn't mutated + return (c1.value += c4.value.value.loadInt(32), c1.getWrappervalue1(), c1.value += c4.value.value.loadInt(32), c1.getWrappervalue2(), c1, sumOf1); +} + +@method_id(105) +fun test5(setNull: bool) { + var c1: Wrapper < WrappedInt >? = { value: { value: 80 } }; + __expect_type(c1, "Wrapper"); + if (setNull) { + c1 = null; + } + __expect_type(c1, "Wrapper?"); + return c1; +} + +fun createNestedWrapperOrNull(setNull: bool, initVal: T) { + if (setNull) { + return null; + } + var c1 = Wrapper> {value:{value:initVal}}; + var c2: Wrapper< Wrapper<(T)> > = {value:{value:initVal}}; + var c3: Wrapper< Wrapper<((T | T))> > = WrapperAlias {value:{value:initVal}}; + var c4: Wrapper< Wrapper<((T | T))> > = Wrapper> {value:c3.value}; + var c5: Wrapper< WrapperAlias<((T | T))> > = WrapperAlias> {value:c3.value}; + var c6: WrapperAlias< WrapperAlias<((T | T))> > = Wrapper> {value:c3.value}; + return c1; +} + +@method_id(106) +fun test6() { + var c1 = createNestedWrapperOrNull(true, (1, 2)); + var c2 = createNestedWrapperOrNull(false, (1, 2)); + __expect_type(c1, "Wrapper>?"); + getWrappervalue1(c2!); + getWrappervalue2(c2!); + return (c1, 777, c2); +} + +@method_id(107) +fun test7(i8: int8) { + var c1 = Pair { first: 10, second: i8 }; + var c2 = eq(c1); + __expect_type(c2, "Pair"); + var c3: Pair = { first: null, second: null }; + return (c2.first += 10, c2, swap(c2), 777, c3); +} + +@method_id(108) +fun test8(lastNull: bool) { + var c1 = Wrapper { value: () }; + var c2 = Pair> { first: {}, second: {first:{},second:{}} }; + var c3 = Pair { first: c1, second: Wrapper { value: Empty{} } }; + var c4 = Pair { first: c1, second: Wrapper { value: lastNull ? null : Empty{} } }; + __expect_type(c4, "Pair, Wrapper>"); + return (c1, c2, c3, 777, c4, 777, c4.first, c4.second.value); +} + +@method_id(109) +fun test9() { + var c1: WrappedInt = { value: 10 }; + var c2: WrappedInt = WrappedInt { value: 20 }; + var c3: WrappedInt = Wrapper { value: 30 }; + var c4: Wrapper = { value: 40 }; + c1 = c2 = c3 = c4; + __expect_type(c1, "WrappedInt"); + __expect_type(c1 as Wrapper, "Wrapper"); + + var c5: Pair, WrappedInt> = { first: {value:50}, second: {value:50} }; + c5.first = c1; c5.first = c4; + c5.second = c1; c5.second = c4; + + var c6 = Wrapper> { value: { value: 60 } }; + var c7 = Wrapper { value: { value: 70 } }; + c6 = c7; c7 = c6; + c6 as Wrapper; + c7 as Wrapper>; + + var p1: PairAlias = { first: 42, second: WrappedInt { value: 200 } }; + __expect_type(p1, "PairAlias"); + __expect_type(p1 as Pair>, "Pair>"); + var p2: Pair> = p1; + + __expect_type(DeepNested { + wrapper: wrap(wrap(999)), + pair: makePair(10, 20), + }, "DeepNested"); + + return (c5, c7); +} + +type Int16Or8 = int8 | int16; + +@method_id(110) +fun test10() { + var p1 = Pair { first: 5, second: 10 as int8 }; + var p2: Pair = { first: 10, second: 5 as int16 }; + var p3 = Pair { first: 20 as int|builder, second: (20 as int8) as Int16Or8 }; + __expect_type(p1, "Pair"); + __expect_type(p2, "Pair"); + __expect_type(p3, "Pair"); + p1 = p2; p1 = p3; p2 = p1; p2 = p3; p3 = p1; p3 = p2; + p2 as Pair; p3 as Pair; + p1 as Pair; p3 as Pair; + p1 as Pair; p3 as Pair; + return p3; +} + +@method_id(111) +fun test11() { + // type_id of these instantiations are the same + var c1 = { value: 5 } as int | Wrapper; + var c2 = { value: 5 } as int | Wrapper; + var c3 = { value: 5 } as int | Wrapper; + var c4 = { value: 5 } as int | Wrapper; + c1 = c1; c1 = c2; c1 = c3; c1 = c4; + c2 = c1; c2 = c2; c2 = c3; c2 = c4; + c3 = c1; c3 = c2; c3 = c3; c3 = c4; + c4 = c1; c4 = c2; c4 = c3; c4 = c4; + return (c1, c2, c3, c4); +} + +@method_id(112) +fun test12(is1: bool) { + var c1: Wrapper = { value: 10 }; + var c2: Wrapper = { value: 20 }; + var c3: Wrapper | WrapperAlias = is1 ? c1 : c2; + __expect_type(c3, "Wrapper"); + var m3: Wrapper | Wrapper = is1 ? c1 : Wrapper { value: null }; + __expect_type(m3, "Wrapper | Wrapper"); + return (c3, 777, m3); +} + +fun makeIdentity(): (T) -> T { + return eq; +} + +@method_id(113) +fun test13() { + return makeIdentity>()({ first: 30, second: true }); +} + +struct Storage { + Wrapper: Wrapper; + backup: T?; +} + +@method_id(114) +fun test14() { + var s1 = Storage { Wrapper: wrap(999), backup: null }; + var s2 = Storage { Wrapper: wrap(makePair(1, "")), backup: makePair(2, "") }; + __expect_type(s1, "Storage"); + __expect_type(s2, "Storage>"); + return (s1, s2.backup!.first); +} + +struct DualStorage { + primary: Wrapper; + secondary: Wrapper; +} + +type IntOrBuilder = int | builder; + +@method_id(115) +fun test15() { + var s1 = DualStorage { primary: wrap(10), secondary: wrap(false), }; + __expect_type(s1, "DualStorage"); + var s2: DualStorage = { primary: wrap(100 as IntOrBuilder), secondary: wrap(true), }; + var s3 = DualStorage { primary: wrap(200 as IntOrBuilder), secondary: wrap(true), }; + return (s1, s2 = s3, s3 = s2); +} + +type ComplexWrapPair = Wrapper>; + +@method_id(116) +fun test16() { + var c1: ComplexWrapPair = wrap(makePair(1, 2)); + __expect_type(c1, "ComplexWrapPair"); + var c2 = wrap(makePair(3, 4)); + __expect_type(c2, "Wrapper>"); + if (0) { c1 = c2; c2 = c1; } + c1 as Wrapper>; c2 as ComplexWrapPair; + return (c1, c2); +} + +fun transform(x: T, f: (T) -> U): U { + return f(x); +} + +@method_id(117) +fun test17() { + var id = makeIdentity(); + var result = transform(100 as MyInt, id); + __expect_type(result, "MyInt"); + var r2 = transform(wrap(123), wrap>); + __expect_type(r2, "Wrapper>"); + return (result, r2); +} + +struct UnionHolder { + item: Wrapper; + extra: Wrapper; +} + +@method_id(118) +fun test18() { + var uh: UnionHolder = { item: wrap(123 as int|slice), extra: wrap("" as int|slice|bool) }; + __expect_type(uh, "UnionHolder"); + __expect_type(uh.item, "Wrapper"); + __expect_type(uh.extra, "Wrapper"); + + var ub = UnionHolder { item: wrap(123 as int|bool), extra: wrap(true as int|bool) }; + __expect_type(ub.extra, "Wrapper"); + + return (uh.item, 777, ub); +} + +type StrangeInt = int; +type StrangeAlsoInt = StrangeInt<()>; + +@method_id(119) +fun test19() { + var i1: StrangeInt = 20; + var i2: StrangeInt = 30; + var i3: StrangeInt> = 40 as StrangeAlsoInt; + i1 = i2; i2 = i1; i1 = i3; i3 = i1; + return (i1 as StrangeInt<()>, i3 as StrangeAlsoInt, i1 is StrangeInt, i1 is StrangeInt, i3 is StrangeInt, i3 is StrangeAlsoInt); +} + +@method_id(120) +fun test20() { + var w1: Wrapper | int = { value: 10 }; + var w2: int | Wrapper = { value: 30 }; + var w3: slice | Wrapper<(int8, int16)> = { value: (50, 30) }; + __expect_type(w3.value, "(int8, int16)"); + var w4 = { value: null } as int | Wrapper; + var w5 = { value: null } as int | Wrapper?>; + var w6: int | Wrapper | null | slice | builder | (int, int) = { value: 10 }; + if (10 > 3) { + w6 = beginCell(); + } + __expect_type(w6, "builder | Wrapper"); + return (w4, 777, w5, w5 is Wrapper, w6 is builder); +} + +struct AnotherWrapper { + value: int; +} + +fun constructAnotherWrapper(value: slice): AnotherWrapper { + return AnotherWrapper { value }; +} + +fun test21(w: AnotherWrapper) { + var w2: AnotherWrapper = { value: beginCell() }; + var w3 = constructAnotherWrapper(beginCell()); + __expect_type(w2, "AnotherWrapper"); + __expect_type(w3, "AnotherWrapper"); + __expect_type(constructAnotherWrapper>, "(Wrapper) -> AnotherWrapper>"); + w.value = w2.value; + w3.value = beginCell(); + w = w2; +} + +fun test22(w: int | WrapperAlias | slice) { + match (w) { + int => {} + WrapperAlias => {} + slice => {} + } + __expect_type(w, "int | Wrapper | slice"); + match (w) { + MyInt => {} + slice => {} + WrapperAlias => {} + } +}; + +@method_id(123) +fun test23() { + var w1: Wrapper = { value: 10 }; + var w2: Wrapper> = { value: w1 }; + var w3: Wrapper>> = { value: w2 }; + var w4: Wrapper>>> = { value: w3 }; + var w5: Wrapper>>>> = { value: w4 }; + eq>(w1); + eq>>(w2); + eq>>>(w3); + return (w1, w2, w3, w4, w5); +} + +fun takeSomethingAndWrapper(something: T, w: Wrapper) { + w.value = something; +} + +struct HasSomethingAndWrapper { + something: T; + w: Wrapper; +} + +fun test24() { + // check that parameter w can be understood from context (after deducing parameter something) + takeSomethingAndWrapper(0, {value: 0}); + takeSomethingAndWrapper(null as slice?, {value: null}); + takeSomethingAndWrapper(Wrapper { value: 10 }, {value: { value: 20 }}); + // same for creating an object and deducing Ts one by one + HasSomethingAndWrapper { something: 0, w: { value: 0 } }; + val o = HasSomethingAndWrapper { something: 5 as uint8, w: { value: 5 } }; + __expect_type(o, "HasSomethingAndWrapper"); +} + + +fun main(c: Wrapper, d: WrappedInt) { + __expect_type(c, "Wrapper"); + __expect_type(c!, "Wrapper"); + __expect_type(d, "WrappedInt"); + __expect_type(d.value, "int"); + __expect_type(WrappedInt { value: 200 }, "Wrapper"); + __expect_type(Wrapper{ value: c.value }, "Wrapper"); + __expect_type(Wrapper{ value: c.value as bytes32 }, "Wrapper"); + __expect_type(Wrapper{ value: Wrapper { value: (c.value as bytes32?)! }.value }, "Wrapper"); + __expect_type(Wrapper{ value: c.value }, "Wrapper"); + __expect_type(Wrapper{ value: c.value.loadInt(32) }, "Wrapper"); + __expect_type(WrapperAlias{ value: 10 }, "Wrapper"); + __expect_type(WrapperAlias{ value: 10 } as WrapperAlias, "WrapperAlias"); + __expect_type(swap(Pair { first: 3, second: beginCell() }), "Pair"); +} + +/** +@testcase | 101 | | 23 23 +@testcase | 102 | 9 | 9 10 19 9 +@testcase | 102 | 19 | 19 20 39 (null) +@testcase | 103 | | 10 10 (null) (null) 0 777 10 10 (null) (null) 0 +@testcase | 104 | | 11 11 13 13 13 2 +@testcase | 105 | 0 | 80 +@testcase | 105 | -1 | (null) +@testcase | 106 | | (null) (null) 0 777 1 2 142 +@testcase | 107 | 5 | 20 20 5 5 20 777 (null) (null) (null) 0 +@testcase | 108 | 0 | 777 143 777 143 +@testcase | 108 | -1 | 777 0 777 0 +@testcase | 109 | | 40 40 70 +@testcase | 110 | | 20 1 20 42 +@testcase | 111 | | 5 1 132 5 1 132 5 1 132 5 1 132 +@testcase | 112 | -1 | 10 1 777 10 1 133 +@testcase | 112 | 0 | 20 1 777 (null) 0 134 +@testcase | 113 | | 30 -1 +@testcase | 114 | | 999 (null) 2 +@testcase | 115 | | 10 0 200 1 -1 200 1 -1 +@testcase | 116 | | 1 2 3 4 +@testcase | 117 | | 100 123 +@testcase | 118 | | 123 1 777 123 1 -1 2 +@testcase | 119 | | 40 40 -1 -1 -1 -1 +@testcase | 120 | | (null) 137 777 (null) (null) 0 139 -1 -1 +@testcase | 123 | | 10 10 10 10 10 + + +@fif_codegen +""" + test1 PROC:<{ + // + 23 PUSHINT // c1=23 + DUP // c1=23 c2=23 + }> +""" + +@fif_codegen DECLPROC getWrappervalue1> +@fif_codegen DECLPROC getWrappervalue2>> + */ diff --git a/tolk-tester/tests/generics-3.tolk b/tolk-tester/tests/generics-3.tolk new file mode 100644 index 000000000..4ec7b29b4 --- /dev/null +++ b/tolk-tester/tests/generics-3.tolk @@ -0,0 +1,146 @@ +struct Ok { + result: T; +} + +struct Err { + errPayload: T; +} + +type OkAlias = Ok; + +type Response = Ok | Err; + +@pure +fun getResponse(success: bool): Response { + return success ? Ok { result: 10 } : Err { errPayload: beginCell().storeInt(-1,32).endCell().beginParse() }; +} + +@method_id(101) +fun test1(getOk: bool) { + return match (var r = getResponse(getOk)) { + Ok => r.result, + Err => r.errPayload.loadInt(32) + }; +} + +fun test2() { + var r = getResponse(true); + match (r) { + Ok => r as OkAlias, + Err => r as Err, + } + match (r) { + OkAlias => r as Ok, + Err => {} + } + match (r) { + Ok => { __expect_type(r, "Ok"); } + Err => { __expect_type(r, "Err"); } + } + + if (r is Ok) { __expect_type(r, "Ok"); } + if (r is Ok) { __expect_type(r, "Ok"); } + if (r is OkAlias) { __expect_type(r, "Ok"); } + if (r !is OkAlias) { __expect_type(r, "Err"); } + if (r is Ok && r !is Err && r is Ok) { __expect_type(r, "Ok"); } +} + +@method_id(103) +fun test3() { + var r: Response = getResponse(true); + r = Err { errPayload: "" }; + __expect_type(r, "Err"); + r = OkAlias { result: 10 }; + __expect_type(r, "Ok"); + return (r, r is Ok, (r as Response) is Err); +} + +@method_id(104) +fun test4() { + var r1 = OkAlias { result: OkAlias { result: 10 } } as Response< Ok, slice >; + match (r1) { + Ok => { __expect_type(r1, "Ok>"); r1.result.result; } + Err => {} + } + var r2: Response< Response, slice > = Err { errPayload: "" }; + __expect_type(r2, "Err"); + r2 = 10>3 ? Ok { result: Ok { result: 12 } } : r2; + var mm1 = match (r2) { + OkAlias> => true, + Err => false + }; + var mm2 = match (r2) { + Ok => true, + Err => false + }; + return (r2, mm1, mm2, r2 is Ok>, r2 is OkAlias>, r2 is Err, r2 is Ok, r2 is Err, r2 !is Ok); +} + +@method_id(105) +fun test5() { + var o1: Ok = { result: 1 }; + var o2: OkAlias = { result: 2 }; + return (o1 is Ok, o1 is OkAlias, o2 is Ok, o2 is OkAlias, o1 is Ok, o1 is OkAlias); +} + +type OkInt = Ok; + +fun test6(w: Response | int) { + match (w) { + int => {} + Ok => {} + Err => {} + } + match (w) { + OkInt => {} + int => {} + Err => {} + } +} + +fun test7(w: OkInt | Err | int) { + match (w) { + Ok => {} + Err => {} + int => {} + } +} + +struct WithDef { + f1: T? = null; + f2: int? = null as int?; + f3: T? = null as T?; + price: coins = ton("0.05"), + slice: slice = stringHexToSlice("010203"), +} + +@method_id(108) +fun test8() { + var w1: WithDef = {}; + var w2: WithDef> = { price: ton("0.1"), f3: { result: 12 } }; + return (w1.price, w1.f1, w1.f3, w2.price, w2.f3!.result, w2.f3, w2.slice.getRemainingBitsCount()); +} + + +fun main() { + __expect_type(getResponse(true), "Response"); +} + +/** +@testcase | 101 | -1 | 10 +@testcase | 101 | 0 | -1 +@testcase | 103 | | 10 -1 0 +@testcase | 104 | | 12 129 132 -1 -1 -1 0 0 -1 0 0 +@testcase | 105 | | -1 -1 -1 -1 0 0 +@testcase | 108 | | 50000000 (null) (null) 100000000 12 12 24 + +@fif_codegen +""" + test3 PROC:<{ + // + 10 PUSHINT // r.USlot1=10 + -1 PUSHINT // r.USlot1=10 '11=-1 + FALSE // r.USlot1=10 '11=-1 '13 + }> +""" + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1285.tolk b/tolk-tester/tests/invalid-declaration/err-1285.tolk new file mode 100644 index 000000000..c647a1535 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1285.tolk @@ -0,0 +1,14 @@ +struct Pair { + first: T1; + second: T2; +} + +struct DoesntWork { + p: Pair; +} + +/** +@compilation_should_fail +@stderr struct `Pair` expects 2 type arguments, but 1 provided +@stderr p: Pair + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1378.tolk b/tolk-tester/tests/invalid-declaration/err-1378.tolk new file mode 100644 index 000000000..bd2750fcb --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1378.tolk @@ -0,0 +1,11 @@ +struct Wrapper { + value: T; +} + +type ErrWrapper = int | Wrapper; + +/** +@compilation_should_fail +@stderr type `Wrapper` is generic, you should provide type arguments +@stderr int | Wrapper; + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1876.tolk b/tolk-tester/tests/invalid-declaration/err-1876.tolk new file mode 100644 index 000000000..281a4af7d --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1876.tolk @@ -0,0 +1,11 @@ +struct Pair { + first: T1; + second: T2; + first: int; +} + +/** +@compilation_should_fail +@stderr redeclaration of field `first` +@stderr first: int + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4081.tolk b/tolk-tester/tests/invalid-semantics/err-4081.tolk new file mode 100644 index 000000000..4442cf2d5 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4081.tolk @@ -0,0 +1,20 @@ +struct Wrapper { + field: T; +} + +type MyInt = int; +type WrapperAlias = Wrapper; + +fun main(w: int | WrapperAlias | WrapperAlias) { + match (w) { + MyInt => {} + Wrapper => {} + WrapperAlias => {} + } +} + +/** +@compilation_should_fail +@stderr wrong pattern matching: duplicated `Wrapper` +@stderr WrapperAlias + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4119.tolk b/tolk-tester/tests/invalid-semantics/err-4119.tolk new file mode 100644 index 000000000..bb6839315 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4119.tolk @@ -0,0 +1,16 @@ +struct Wrapper { + value: T; +} + +type WrapperAlias = Wrapper; + +fun main() { + var w1 = WrapperAlias { value: 10 } as WrapperAlias | int | Wrapper; + w1 is Wrapper; +} + +/** +@compilation_should_fail +@stderr can not deduce type arguments for `Wrapper`, provide them manually +@stderr w1 is Wrapper + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4193.tolk b/tolk-tester/tests/invalid-semantics/err-4193.tolk index b0b31fa7d..6e7198c61 100644 --- a/tolk-tester/tests/invalid-semantics/err-4193.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4193.tolk @@ -8,6 +8,6 @@ fun cantInstantiateNonGenericStruct() { /** @compilation_should_fail -@stderr generic T not expected here +@stderr type `User` is not generic @stderr User */ diff --git a/tolk-tester/tests/invalid-semantics/err-4218.tolk b/tolk-tester/tests/invalid-semantics/err-4218.tolk new file mode 100644 index 000000000..baab5a2a3 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4218.tolk @@ -0,0 +1,14 @@ +struct Wrapper { + value: T; +} + +type nested = Wrapper>>; + +fun main() { + var w3: nested = {}; +} + +/** +@compilation_should_fail +@stderr field `value` missed in initialization of struct `Wrapper>>` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4260.tolk b/tolk-tester/tests/invalid-semantics/err-4260.tolk index 6f174a595..9c9a73b31 100644 --- a/tolk-tester/tests/invalid-semantics/err-4260.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4260.tolk @@ -10,6 +10,6 @@ fun cantInstantiateNonGenericStruct() { /** @compilation_should_fail -@stderr generic T not expected here +@stderr type `UserAlias` is not generic @stderr UserAlias */ diff --git a/tolk-tester/tests/invalid-semantics/err-4291.tolk b/tolk-tester/tests/invalid-semantics/err-4291.tolk new file mode 100644 index 000000000..907cc76c8 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4291.tolk @@ -0,0 +1,7 @@ +struct T1 { field: T1; } +struct CollisionName { item: T1; } + +/** +@compilation_should_fail +@stderr type `T1` is not generic + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4378.tolk b/tolk-tester/tests/invalid-semantics/err-4378.tolk new file mode 100644 index 000000000..7f1225670 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4378.tolk @@ -0,0 +1,11 @@ +struct Wrapper { + field: T; +} + +fun f(w: Wrapper) { +} + +/** +@compilation_should_fail +@type `Wrapper` is generic, you should provide type arguments + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4387.tolk b/tolk-tester/tests/invalid-semantics/err-4387.tolk new file mode 100644 index 000000000..50e02ed7b --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4387.tolk @@ -0,0 +1,17 @@ +struct Wrapper { + value: T; +} + +type nested = Wrapper>>; + +fun main(d: nested) { + var c1 = d; + + var c2 = Wrapper>>; +} + +/** +@compilation_should_fail +@stderr struct `Wrapper` can not be used as a value +@stderr c2 = Wrapper + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4429.tolk b/tolk-tester/tests/invalid-semantics/err-4429.tolk new file mode 100644 index 000000000..028ae1078 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4429.tolk @@ -0,0 +1,20 @@ +struct Wrapper { + field: T; +} + +type MyInt = int; + +fun main(w: int | Wrapper | Wrapper) { + w is Wrapper; + w is Wrapper; + w is Wrapper; + w is Wrapper; + + w is Wrapper; +} + +/** +@compilation_should_fail +@stderr can not deduce type arguments for `Wrapper`, provide them manually +@stderr Wrapper + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4493.tolk b/tolk-tester/tests/invalid-semantics/err-4493.tolk new file mode 100644 index 000000000..8a386f8c3 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4493.tolk @@ -0,0 +1,13 @@ +struct Wrapper { + value: T; +} + +fun my() { + var w: Wrapper = Wrapper { value: 30 }; +} + +/** +@compilation_should_fail +@stderr type `Wrapper` is generic, you should provide type arguments +@stderr var w: Wrapper = + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4512.tolk b/tolk-tester/tests/invalid-semantics/err-4512.tolk new file mode 100644 index 000000000..77c18cb89 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4512.tolk @@ -0,0 +1,8 @@ +fun f() { + var n: int = 10; +} + +/** +@compilation_should_fail +@stderr type `int` is not generic + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4519.tolk b/tolk-tester/tests/invalid-semantics/err-4519.tolk index b0377b719..a76d20a96 100644 --- a/tolk-tester/tests/invalid-semantics/err-4519.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4519.tolk @@ -2,11 +2,13 @@ struct value { id: int; } fun main() { var value = 100; - var obj = value { id: 100 }; + var value2 = 200; + var obj1 = value { id: 100 }; // "value" refs to a struct, it's parsed as a type + var obj2 = value2 { id: 200 }; // no struct "value2" } /** @compilation_should_fail -@stderr `value` does not name a struct -@stderr obj = value +@stderr unknown type name `value2` +@stderr obj2 = value2 */ diff --git a/tolk-tester/tests/invalid-semantics/err-4530.tolk b/tolk-tester/tests/invalid-semantics/err-4530.tolk new file mode 100644 index 000000000..c7adb10d7 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4530.tolk @@ -0,0 +1,17 @@ +struct Wrapper { + field: T; +} + +fun main() { + var w1: int | Wrapper | null | Wrapper | (int, int) = Wrapper { field: 10 }; + var w2: int | Wrapper | null | Wrapper | (int, int) = Wrapper { field: 10 }; + var w3: int | Wrapper | null | Wrapper | (int, int) = Wrapper { field: 10 }; + + var w4: int | Wrapper | null | Wrapper | (int, int) = { field: 20 }; +} + +/** +@compilation_should_fail +@stderr can not detect struct name +@stderr { field: 20 } + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4580.tolk b/tolk-tester/tests/invalid-semantics/err-4580.tolk new file mode 100644 index 000000000..bd414eeb6 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4580.tolk @@ -0,0 +1,12 @@ +struct Wrapper { + item: T; +} + +fun main() { + var c = Wrapper { }; +} + +/** +@compilation_should_fail +@stderr field `item` missed in initialization of struct `Wrapper` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4628.tolk b/tolk-tester/tests/invalid-semantics/err-4628.tolk new file mode 100644 index 000000000..3efe9069d --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4628.tolk @@ -0,0 +1,16 @@ +struct Wrapper { + item: T; + value: int; +} + +fun main() { + var c = Wrapper { + value: null + }; +} + +/** +@compilation_should_fail +@stderr field `item` missed in initialization of struct `Wrapper` +@stderr = Wrapper { + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4630.tolk b/tolk-tester/tests/invalid-semantics/err-4630.tolk new file mode 100644 index 000000000..d404adbde --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4630.tolk @@ -0,0 +1,13 @@ +struct Ok { result: T; } + +type OkInt = Ok; +type OkSlice = Ok; + +fun f(w: OkInt | int | OkSlice) { + if (w !is Ok) {} +} + +/** +@compilation_should_fail +@stderr can not deduce type arguments for `Ok`, provide them manually + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4711.tolk b/tolk-tester/tests/invalid-semantics/err-4711.tolk new file mode 100644 index 000000000..01f488487 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4711.tolk @@ -0,0 +1,22 @@ +struct Pair { + first: T1; + second: T2; +} + +fun main() { + var ok1: Pair = { first: null, second: null }; + var ok2: Pair = Pair { first: null, second: null }; + var ok3 = Pair { first: null, second: null }; + var ok4 = Pair { first: null as int?, second: null as (int, int)? }; + var ok5 = Pair { first: null as int?, second: null as slice? }; // ok since doesn't reach type checking + var ok6 = Pair { first: null, second: null }; + + var err = Pair { first: null, second: null }; +} + +/** +@compilation_should_fail +@stderr can not deduce T1 for generic struct `Pair` +@stderr instantiate it manually with `Pair<...>` +@stderr var err = + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4862.tolk b/tolk-tester/tests/invalid-semantics/err-4862.tolk new file mode 100644 index 000000000..490c9d2f3 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4862.tolk @@ -0,0 +1,18 @@ +struct Pair { + first: T1; + second: T2; +} + +fun cantProvideDuplicateField() { + var p: Pair = { + first: null, + second: null, + first: 3, + }; +} + +/** +@compilation_should_fail +@stderr duplicate field initialization +@stderr first: 3 + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4880.tolk b/tolk-tester/tests/invalid-semantics/err-4880.tolk index 0bbdeee67..4564a4a81 100644 --- a/tolk-tester/tests/invalid-semantics/err-4880.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4880.tolk @@ -6,5 +6,5 @@ fun failCantDeduceWithoutArgument() { /** @compilation_should_fail -@stderr too few arguments in call to `f`, expected 2, have 1 +@stderr too few arguments in call to `f`, expected 2, have 1 */ diff --git a/tolk-tester/tests/invalid-semantics/err-4925.tolk b/tolk-tester/tests/invalid-semantics/err-4925.tolk new file mode 100644 index 000000000..858ff6f77 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4925.tolk @@ -0,0 +1,20 @@ +struct Wrapper { + field: T; +} + +type MyInt = int; +type WrapperAlias = Wrapper; + +fun main(w: int | WrapperAlias | WrapperAlias) { + match (w) { + MyInt => {} + WrapperAlias => {} + WrapperAlias => {} + } +} + +/** +@compilation_should_fail +@stderr can not deduce type arguments for `WrapperAlias`, provide them manually +@stderr WrapperAlias => {} + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3173.tolk b/tolk-tester/tests/invalid-syntax/err-3173.tolk new file mode 100644 index 000000000..2c3e23b7d --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3173.tolk @@ -0,0 +1,8 @@ +struct Pair`, got `{` + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3450.tolk b/tolk-tester/tests/invalid-syntax/err-3450.tolk new file mode 100644 index 000000000..5075284ec --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3450.tolk @@ -0,0 +1,12 @@ +struct Wrapper { + value: T; +} + +fun main() { + var w3: Wrapper> = {}; +} + +/** +@compilation_should_fail +@stderr expected `>` or `,`, got `=` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6070.tolk b/tolk-tester/tests/invalid-typing/err-6070.tolk new file mode 100644 index 000000000..e10560210 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6070.tolk @@ -0,0 +1,26 @@ +struct Wrapper { + value: T; +} + +struct JustInt { + value: int; +} + +fun takeSmth(obj: T) { return obj.value; } +fun takeW(obj: Wrapper) { return obj.value; } + +fun cantPassJustIntAsWrapper() { + var w: Wrapper = { value: 10 }; + var j: JustInt = { value: 10 }; + + takeSmth(w); + takeSmth(j); + takeW(w); + takeW(j); +} + +/** +@compilation_should_fail +@stderr can not pass `JustInt` to `Wrapper` +@stderr takeW(j) + */ diff --git a/tolk-tester/tests/invalid-typing/err-6110.tolk b/tolk-tester/tests/invalid-typing/err-6110.tolk index fbef5abe6..49e9f30c7 100644 --- a/tolk-tester/tests/invalid-typing/err-6110.tolk +++ b/tolk-tester/tests/invalid-typing/err-6110.tolk @@ -10,6 +10,6 @@ fun cantDeduceDefiniteUnion(a: int | slice | builder) { /** @compilation_should_fail -@stderr can not deduce T1 for generic function `makeNullable` +@stderr can not deduce T1 for generic function `makeNullable` @stderr makeNullable(a); */ diff --git a/tolk-tester/tests/invalid-typing/err-6145.tolk b/tolk-tester/tests/invalid-typing/err-6145.tolk new file mode 100644 index 000000000..54893cb07 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6145.tolk @@ -0,0 +1,20 @@ +struct Wrapper { + value: T; +} + +fun analyze(w: Wrapper) {} + +type MInt = int; + +fun main() { + var w1: Wrapper = { value: 10 }; + var w2 = Wrapper { value: 10 as int7 }; + + analyze(w1); + analyze(w2); +} + +/** +@compilation_should_fail +@stderr can not pass `Wrapper` to `Wrapper` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6369.tolk b/tolk-tester/tests/invalid-typing/err-6369.tolk index 72b7df0ec..3d9391b11 100644 --- a/tolk-tester/tests/invalid-typing/err-6369.tolk +++ b/tolk-tester/tests/invalid-typing/err-6369.tolk @@ -6,6 +6,6 @@ fun failIncompatibleTypesForT() { /** @compilation_should_fail -@stderr T is both int and slice for generic function `f` +@stderr can not pass `slice` to `int` @stderr f(32 */ diff --git a/tolk-tester/tests/invalid-typing/err-6409.tolk b/tolk-tester/tests/invalid-typing/err-6409.tolk new file mode 100644 index 000000000..93c347edf --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6409.tolk @@ -0,0 +1,10 @@ +struct Wrapper { value: T } + +fun f(w: Wrapper | Wrapper) { + w = Wrapper { value: beginCell() }; +} + +/** +@compilation_should_fail +@stderr can not assign `Wrapper` to variable of type `Wrapper | Wrapper` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6473.tolk b/tolk-tester/tests/invalid-typing/err-6473.tolk new file mode 100644 index 000000000..a6859e03b --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6473.tolk @@ -0,0 +1,18 @@ +struct Wrapper { + value: T; +} + +type IntOrWrapped8 = Wrapper | int; + +fun main() { + var w1: IntOrWrapped8 | Wrapper = Wrapper { value: 10 as int8 }; + var w2: IntOrWrapped8 | Wrapper = Wrapper { value: 10 as int16 }; + + var w3: IntOrWrapped8 | Wrapper = Wrapper { value: 10 as int32 }; +} + +/** +@compilation_should_fail +@stderr can not assign `Wrapper` to variable of type `Wrapper | int | Wrapper` +@stderr var w3 + */ diff --git a/tolk-tester/tests/invalid-typing/err-6480.tolk b/tolk-tester/tests/invalid-typing/err-6480.tolk new file mode 100644 index 000000000..25a85e0bc --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6480.tolk @@ -0,0 +1,27 @@ +struct All { + f1: A; + f2: A; + f3: A; +} + +fun testInferGenericOneByOne(x: int?) { + // ok + __expect_type(All { + f1: x, + f2: 5, + f3: null + }, "All"); + // err + All { + f1: x, + f2: null, + f3: beginCell() // type checker error + }; +} + +/** +@compilation_should_fail +@stderr can not assign `builder` to field of type `int?` +@stderr f3: beginCell() + */ + diff --git a/tolk-tester/tests/invalid-typing/err-6618.tolk b/tolk-tester/tests/invalid-typing/err-6618.tolk new file mode 100644 index 000000000..ab505c124 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6618.tolk @@ -0,0 +1,17 @@ +struct Ok { result: T } +struct Err { errPayload: T } + +type Response = Ok | Err; + +fun f(w: Response | Response) { + match (w) { + Ok => {} + Err => {} + } +} + +/** +@compilation_should_fail +@stderr `match` does not cover all possible types +@stderr missing types are: `Ok`, `Err` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6620.tolk b/tolk-tester/tests/invalid-typing/err-6620.tolk new file mode 100644 index 000000000..47e30e99c --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6620.tolk @@ -0,0 +1,16 @@ +struct Wrapper { + value: T; +} + +fun main() { + var c1: Wrapper< Wrapper<(int,int)> > = {value:{value:(1,2)}}; // ok + var c2: Wrapper< Wrapper<(int?,int?)> > = {value:{value:(null,null)}}; // ok + + var c3: Wrapper< Wrapper<(int,int)> > = {value:{value:0}}; +} + +/** +@compilation_should_fail +@stderr can not assign `int` to field of type `(int, int)` +@stderr {value:0} + */ diff --git a/tolk-tester/tests/invalid-typing/err-6642.tolk b/tolk-tester/tests/invalid-typing/err-6642.tolk new file mode 100644 index 000000000..82563e322 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6642.tolk @@ -0,0 +1,21 @@ +struct Wrapper { + value: T; +} + +type MyInt = int; + +fun cantAssignFieldsOfDifferentTypes( + w1: Wrapper, w2: Wrapper, w3: Wrapper, w4: Wrapper<()> +) { + w1.value = w2.value; // ok + w1.value = w3.value; // ok + w3.value = w2.value; // ok + + w1.value = w4.value; +} + +/** +@compilation_should_fail +@stderr can not assign `()` to field of type `int` +@stderr w1.value = w4.value + */ diff --git a/tolk-tester/tests/invalid-typing/err-6919.tolk b/tolk-tester/tests/invalid-typing/err-6919.tolk new file mode 100644 index 000000000..f7a988567 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6919.tolk @@ -0,0 +1,13 @@ +struct Gen { + price: coins = (1 as T) + "", +} + +fun main() { + var c: Gen = {}; + return c; +} + +/** +@compilation_should_fail +@stderr can not apply operator `+` to `int8` and `slice` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6948.tolk b/tolk-tester/tests/invalid-typing/err-6948.tolk new file mode 100644 index 000000000..448d07a46 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6948.tolk @@ -0,0 +1,16 @@ +struct Wrapper { + field: T; +} + +struct AnotherWrapper { + field: T; +} + +fun main() { + var w: Wrapper = AnotherWrapper { field: 10 }; +} + +/** +@compilation_should_fail +@stderr can not assign `AnotherWrapper` to variable of type `Wrapper` + */ diff --git a/tolk-tester/tests/var-apply-tests.tolk b/tolk-tester/tests/var-apply-tests.tolk index aec77391f..9d9a0a0b8 100644 --- a/tolk-tester/tests/var-apply-tests.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -242,6 +242,23 @@ fun testCallableFields() { ); } +struct Wrapper { + value: T; +} + +fun wrap(value: T): Wrapper { + return{value}; +} + +@method_id(116) +fun testCallableGenericField() { + var w1: Wrapper<(int) -> int> = { value: justAdd2 }; + var w2: Wrapper = { value: justAdd2 }; + var w_f = wrapMInt>; + return (w1.value(5), w2.value!(9), wrap(w_f(justAdd2)).value.value(78)); +} + + fun main() {} /** @@ -267,4 +284,5 @@ fun main() {} @testcase | 114 | -1 | 2 3 131 @testcase | 114 | 0 | (null) 12 1 @testcase | 115 | | 7 6 7 +@testcase | 116 | | 7 11 80 */ diff --git a/tolk-tester/tests/warnings-not-errors/warnings-4.tolk b/tolk-tester/tests/warnings-not-errors/warnings-4.tolk new file mode 100644 index 000000000..bc843ecc7 --- /dev/null +++ b/tolk-tester/tests/warnings-not-errors/warnings-4.tolk @@ -0,0 +1,15 @@ +struct Wrapper { + value: T; +} + +fun main() { + var w = Wrapper { value: 16 as int16 }; + return (w is Wrapper, w is Wrapper, w is Wrapper, w !is Wrapper< Wrapper >); +} + +/** +@testcase | 0 | | -1 -1 0 -1 + +@stderr warning: variable `w` of type `Wrapper` can never be `Wrapper`, this condition is always false +@stderr warning: variable `w` of type `Wrapper` can never be `Wrapper>`, this condition is always true + */ diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 36c657c26..2999d1a10 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -168,14 +168,20 @@ static std::vector parse_nested_type_list(Lexer& lex, TokenType tok_op if (lex.tok() == tok_comma) { lex.next(); } else if (lex.tok() != tok_cl) { + // overcome the `>>` problem, like `Wrapper>`: + // treat token `>>` like two `>`; consume one here doing nothing (break) and leave the second `>` in a lexer + if (tok_cl == tok_gt && lex.tok() == tok_rshift) { + lex.hack_replace_rshift_with_one_triangle(); + break; + } lex.unexpected(s_cl); } } return sub_types; } -static std::vector parse_nested_type_list_in_parenthesis(Lexer& lex) { - return parse_nested_type_list(lex, tok_oppar, "`(`", tok_clpar, "`)` or `,`"); +static std::vector parse_nested_type_list_in_triangles(Lexer& lex) { + return parse_nested_type_list(lex, tok_lt, "`<`", tok_gt, "`>` or `,`"); } static AnyTypeV parse_simple_type(Lexer& lex) { @@ -189,7 +195,7 @@ static AnyTypeV parse_simple_type(Lexer& lex) { return createV(loc, text); } case tok_oppar: { - return createV(loc, parse_nested_type_list_in_parenthesis(lex)); + return createV(loc, parse_nested_type_list(lex, tok_oppar, "`(`", tok_clpar, "`)` or `,`")); } case tok_opbracket: { return createV(loc, parse_nested_type_list(lex, tok_opbracket, "`[`", tok_clbracket, "`]` or `,`")); @@ -202,6 +208,15 @@ static AnyTypeV parse_simple_type(Lexer& lex) { static AnyTypeV parse_type_nullable(Lexer& lex) { AnyTypeV result = parse_simple_type(lex); + if (lex.tok() == tok_lt) { + std::vector args = parse_nested_type_list_in_triangles(lex); + std::vector outer_and_args; + outer_and_args.reserve(1 + args.size()); + outer_and_args.push_back(result); + outer_and_args.insert(outer_and_args.end(), args.begin(), args.end()); + result = createV(result->loc, std::move(outer_and_args)); + } + if (lex.tok() == tok_question) { lex.next(); result = createV(result->loc, result); @@ -258,6 +273,24 @@ static AnyTypeV parse_type_from_tokens(Lexer& lex) { AnyExprV parse_expr(Lexer& lex); AnyV parse_statement(Lexer& lex); +static V parse_genericsT_list(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + std::vector genericsT_items; + lex.expect(tok_lt, "`<`"); + while (true) { + lex.check(tok_identifier, "T"); + std::string_view nameT = lex.cur_str(); + genericsT_items.emplace_back(createV(lex.cur_location(), nameT)); + lex.next(); + if (lex.tok() != tok_comma) { + break; + } + lex.next(); + } + lex.expect(tok_gt, "`>`"); + return createV{loc, std::move(genericsT_items)}; +} + static AnyV parse_parameter(Lexer& lex, bool is_first) { SrcLocation loc = lex.cur_location(); @@ -342,10 +375,16 @@ static AnyV parse_type_alias_declaration(Lexer& lex, const std::vector(lex.cur_location(), lex.cur_str()); lex.next(); + + V genericsT_list = nullptr; + if (lex.tok() == tok_lt) { // 'type Response' + genericsT_list = parse_genericsT_list(lex); + } + lex.expect(tok_assign, "`=`"); AnyTypeV underlying_type = parse_type_from_tokens(lex); lex.expect(tok_semicolon, "`;`"); - return createV(loc, v_ident, underlying_type); + return createV(loc, v_ident, genericsT_list, underlying_type); } static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable) { @@ -728,13 +767,22 @@ static AnyExprV parse_expr100(Lexer& lex) { v_instantiationTs = parse_maybe_instantiationTs_after_identifier(lex); } if (lex.tok() == tok_opbrace) { - auto explicit_ref = createV(loc, v_ident, v_instantiationTs); - return createV(loc, explicit_ref, parse_object_body(lex)); + AnyTypeV type_node = createV(v_ident->loc, v_ident->name); // `Pair { ... }` + if (v_instantiationTs) { // `Pair { ... }` + std::vector ident_and_args; + ident_and_args.reserve(1 + v_instantiationTs->size()); + ident_and_args.push_back(type_node); + for (int i = 0; i < v_instantiationTs->size(); ++i) { + ident_and_args.push_back(v_instantiationTs->get_item(i)->type_node); + } + type_node = createV(v_ident->loc, std::move(ident_and_args)); + } + return createV(loc, type_node, parse_object_body(lex)); } return createV(loc, v_ident, v_instantiationTs); } case tok_opbrace: - return createV(loc, createV(loc), parse_object_body(lex)); + return createV(loc, nullptr, parse_object_body(lex)); case tok_match: return parse_match_expression(lex); default: @@ -1195,24 +1243,6 @@ static AnyV parse_asm_func_body(Lexer& lex, V param_list) { return createV(loc, std::move(arg_order), std::move(ret_order), std::move(asm_commands)); } -static AnyV parse_genericsT_list(Lexer& lex) { - SrcLocation loc = lex.cur_location(); - std::vector genericsT_items; - lex.expect(tok_lt, "`<`"); - while (true) { - lex.check(tok_identifier, "T"); - std::string_view nameT = lex.cur_str(); - genericsT_items.emplace_back(createV(lex.cur_location(), nameT)); - lex.next(); - if (lex.tok() != tok_comma) { - break; - } - lex.next(); - } - lex.expect(tok_gt, "`>`"); - return createV{loc, std::move(genericsT_items)}; -} - static V parse_annotation(Lexer& lex) { SrcLocation loc = lex.cur_location(); lex.check(tok_annotation_at, "`@`"); @@ -1285,7 +1315,7 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector genericsT_list = nullptr; if (lex.tok() == tok_lt) { // 'fun f' - genericsT_list = parse_genericsT_list(lex)->as(); + genericsT_list = parse_genericsT_list(lex); } V v_param_list = parse_parameter_list(lex)->as(); @@ -1428,7 +1458,12 @@ static AnyV parse_struct_declaration(Lexer& lex, const std::vector(lex.cur_location(), lex.cur_str()); lex.next(); - return createV(loc, v_ident, parse_struct_body(lex)); + V genericsT_list = nullptr; + if (lex.tok() == tok_lt) { // 'struct Wrapper' + genericsT_list = parse_genericsT_list(lex); + } + + return createV(loc, v_ident, genericsT_list, parse_struct_body(lex)); } static AnyV parse_tolk_required_version(Lexer& lex) { diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index 3b0b458e4..42e691525 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -21,21 +21,8 @@ namespace tolk { -class ASTReplicator { -protected: - virtual AnyV clone(AnyV v) = 0; - virtual AnyExprV clone(AnyExprV v) = 0; - virtual AnyTypeV clone(AnyTypeV v) = 0; - -public: - virtual ~ASTReplicator() = default; -}; - -class ASTReplicatorFunction : public ASTReplicator { -protected: - using parent = ASTReplicatorFunction; - - std::vector clone(const std::vector& items) { +class ASTReplicator final { + static std::vector clone(const std::vector& items) { std::vector result; result.reserve(items.size()); for (AnyV item : items) { @@ -44,7 +31,7 @@ class ASTReplicatorFunction : public ASTReplicator { return result; } - std::vector clone(const std::vector& items) { + static std::vector clone(const std::vector& items) { std::vector result; result.reserve(items.size()); for (AnyExprV item : items) { @@ -53,7 +40,7 @@ class ASTReplicatorFunction : public ASTReplicator { return result; } - std::vector clone(const std::vector& items) { + static std::vector clone(const std::vector& items) { std::vector result; result.reserve(items.size()); for (AnyTypeV item : items) { @@ -64,195 +51,221 @@ class ASTReplicatorFunction : public ASTReplicator { // types - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->text); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_inner())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_items())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_items())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_params_and_return())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_variants())); } + static V clone(V v) { + return createV(v->loc, clone(v->get_inner_and_args())); + } // expressions - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_expr())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_block_statement())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_items())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_items())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_identifier()), v->has_instantiationTs() ? clone(v->get_instantiationTs()) : nullptr); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_identifier()), clone(v->type_node), v->is_immutable, v->marked_as_redef); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_expr())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->intval, v->orig_str); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->str_val); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->bool_val); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_expr()), v->passed_as_mutate); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_arguments())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_obj()), clone(v->get_identifier()), v->has_instantiationTs() ? clone(v->get_instantiationTs()) : nullptr); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_callee()), clone(v->get_arg_list())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_lhs()), clone(v->get_rhs())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->operator_name, v->tok, clone(v->get_lhs()), clone(v->get_rhs())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->operator_name, v->tok, clone(v->get_rhs())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->operator_name, v->tok, clone(v->get_lhs()), clone(v->get_rhs())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_cond()), clone(v->get_when_true()), clone(v->get_when_false())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_expr()), clone(v->type_node)); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_expr()), clone(v->type_node), v->is_negated); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_expr())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_all_children())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->pattern_kind, clone(v->pattern_type_node), clone(v->get_pattern_expr()), clone(v->get_body())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_field_identifier()), clone(v->get_init_val())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_all_fields())); } - virtual V clone(V v) { - return createV(v->loc, clone(v->get_maybe_reference()), clone(v->get_body())); + static V clone(V v) { + return createV(v->loc, clone(v->type_node), clone(v->get_body())); } // statements - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->loc_end, clone(v->get_items())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_return_value())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->is_ifnot, clone(v->get_cond()), clone(v->get_if_body()), clone(v->get_else_body())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_cond()), clone(v->get_body())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_cond()), clone(v->get_body())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_body()), clone(v->get_cond())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_thrown_code()), clone(v->get_thrown_arg())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_cond()), clone(v->get_thrown_code())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_try_body()), clone(v->get_catch_expr()), clone(v->get_catch_body())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->arg_order, v->ret_order, clone(v->get_asm_commands())); } - // other + // other (common) - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->name); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->nameT); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_items())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->type_node)); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_items())); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, v->param_name, clone(v->type_node), v->declared_as_mutate); } - virtual V clone(V v) { + static V clone(V v) { return createV(v->loc, clone(v->get_params())); } + static V clone(V v) { + return createV(v->loc, clone(v->get_identifier()), clone(v->get_default_value()), clone(v->type_node)); + } + static V clone(V v) { + return createV(v->loc, clone(v->get_all_fields())); + } - AnyTypeV clone(AnyTypeV v) final { - if (v == nullptr) { - return nullptr; - } + + static AnyV clone(AnyV v) { switch (v->kind) { - case ast_type_leaf_text: return clone(v->as()); - case ast_type_question_nullable: return clone(v->as()); - case ast_type_parenthesis_tensor: return clone(v->as()); - case ast_type_bracket_tuple: return clone(v->as()); - case ast_type_arrow_callable: return clone(v->as()); - case ast_type_vertical_bar_union: return clone(v->as()); - default: - throw UnexpectedASTNodeKind(v, "ASTReplicatorFunction::clone"); + case ast_empty_statement: return clone(v->as()); + case ast_block_statement: return clone(v->as()); + case ast_return_statement: return clone(v->as()); + case ast_if_statement: return clone(v->as()); + case ast_repeat_statement: return clone(v->as()); + case ast_while_statement: return clone(v->as()); + case ast_do_while_statement: return clone(v->as()); + case ast_throw_statement: return clone(v->as()); + case ast_assert_statement: return clone(v->as()); + case ast_try_catch_statement: return clone(v->as()); + case ast_asm_body: return clone(v->as()); + // other AST nodes that can be children of ast nodes of function/struct body + case ast_identifier: return clone(v->as()); + case ast_genericsT_item: return clone(v->as()); + case ast_genericsT_list: return clone(v->as()); + case ast_instantiationT_item: return clone(v->as()); + case ast_instantiationT_list: return clone(v->as()); + case ast_parameter: return clone(v->as()); + case ast_parameter_list: return clone(v->as()); + case ast_struct_field: return clone(v->as()); + case ast_struct_body: return clone(v->as()); + + default: { + // be very careful, don't forget to handle all statements/other (not expressions) above! + AnyExprV as_expr = reinterpret_cast(v); + return clone(as_expr); + } } } - AnyExprV clone(AnyExprV v) final { + static AnyExprV clone(AnyExprV v) { switch (v->kind) { case ast_empty_expression: return clone(v->as()); case ast_parenthesized_expression: return clone(v->as()); @@ -285,43 +298,30 @@ class ASTReplicatorFunction : public ASTReplicator { case ast_object_body: return clone(v->as()); case ast_object_literal: return clone(v->as()); default: - throw UnexpectedASTNodeKind(v, "ASTReplicatorFunction::clone"); + throw UnexpectedASTNodeKind(v, "ASTReplicatorFunction::clone(AnyExprV)"); } } - AnyV clone(AnyV v) final { + static AnyTypeV clone(AnyTypeV v) { + if (v == nullptr) { + return nullptr; + } switch (v->kind) { - case ast_empty_statement: return clone(v->as()); - case ast_block_statement: return clone(v->as()); - case ast_return_statement: return clone(v->as()); - case ast_if_statement: return clone(v->as()); - case ast_repeat_statement: return clone(v->as()); - case ast_while_statement: return clone(v->as()); - case ast_do_while_statement: return clone(v->as()); - case ast_throw_statement: return clone(v->as()); - case ast_assert_statement: return clone(v->as()); - case ast_try_catch_statement: return clone(v->as()); - case ast_asm_body: return clone(v->as()); - // other AST nodes that can be children of ast nodes of function body - case ast_identifier: return clone(v->as()); - case ast_genericsT_item: return clone(v->as()); - case ast_genericsT_list: return clone(v->as()); - case ast_instantiationT_item: return clone(v->as()); - case ast_instantiationT_list: return clone(v->as()); - case ast_parameter: return clone(v->as()); - case ast_parameter_list: return clone(v->as()); - - default: { - // be very careful, don't forget to handle all statements/other (not expressions) above! - AnyExprV as_expr = reinterpret_cast(v); - return clone(as_expr); - } + case ast_type_leaf_text: return clone(v->as()); + case ast_type_question_nullable: return clone(v->as()); + case ast_type_parenthesis_tensor: return clone(v->as()); + case ast_type_bracket_tuple: return clone(v->as()); + case ast_type_arrow_callable: return clone(v->as()); + case ast_type_vertical_bar_union: return clone(v->as()); + case ast_type_triangle_args: return clone(v->as()); + default: + throw UnexpectedASTNodeKind(v, "ASTReplicator::clone(AnyTypeV)"); } } public: // the cloned function becomes a deep copy, all AST nodes are copied, no previous pointers left - V clone_function_ast(V v_orig, V new_name_ident) { + static V clone_function_ast(V v_orig, V new_name_ident) { return createV( v_orig->loc, new_name_ident, @@ -333,6 +333,26 @@ class ASTReplicatorFunction : public ASTReplicator { v_orig->flags ); } + + // the cloned struct becomes a deep copy, all AST nodes are copied, no previous pointers left + static V clone_struct_ast(V v_orig, V new_name_ident) { + return createV( + v_orig->loc, + new_name_ident, + clone(v_orig->genericsT_list), + clone(v_orig->get_struct_body()) + ); + } + + // the cloned type alias becomes a deep copy, all AST nodes are copied, no previous pointers left + static V clone_type_alias_ast(V v_orig, V new_name_ident) { + return createV( + v_orig->loc, + new_name_ident, + clone(v_orig->genericsT_list), + clone(v_orig->underlying_type_node) + ); + } }; } // namespace tolk diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index ea991da20..c4af598d4 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -40,6 +40,7 @@ class ASTStringifier final : public ASTVisitor { {ast_type_bracket_tuple, "ast_type_bracket_tuple"}, {ast_type_arrow_callable, "ast_type_arrow_callable"}, {ast_type_vertical_bar_union, "ast_type_vertical_bar_union"}, + {ast_type_triangle_args, "ast_type_triangle_args"}, // expressions {ast_empty_expression, "ast_empty_expression"}, {ast_parenthesized_expression, "ast_parenthesized_expression"}, @@ -300,6 +301,7 @@ class ASTStringifier final : public ASTVisitor { case ast_type_bracket_tuple: return handle_vertex(v->as()); case ast_type_arrow_callable: return handle_vertex(v->as()); case ast_type_vertical_bar_union: return handle_vertex(v->as()); + case ast_type_triangle_args: return handle_vertex(v->as()); // expressions case ast_empty_expression: return handle_vertex(v->as()); case ast_parenthesized_expression: return handle_vertex(v->as()); diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h index 11ecbe80f..1bb72f44c 100644 --- a/tolk/ast-visitor.h +++ b/tolk/ast-visitor.h @@ -212,7 +212,9 @@ const std::vector& get_all_declared_structs(); template void visit_ast_of_all_functions() { BodyVisitorT visitor; - for (FunctionPtr fun_ref : get_all_not_builtin_functions()) { + const std::vector& all = get_all_not_builtin_functions(); + for (size_t i = 0; i < all.size(); ++i) { // NOLINT(*-loop-convert) + FunctionPtr fun_ref = all[i]; // not range-base loop to prevent iterator invalidation (push_back at generics) if (visitor.should_visit_function(fun_ref)) { visitor.start_visiting_function(fun_ref, fun_ref->ast_root->as()); } diff --git a/tolk/ast.h b/tolk/ast.h index dbe1e02b0..95677fcda 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -70,6 +70,7 @@ enum ASTNodeKind { ast_type_bracket_tuple, ast_type_arrow_callable, ast_type_vertical_bar_union, + ast_type_triangle_args, // expressions ast_empty_expression, ast_parenthesized_expression, @@ -448,6 +449,18 @@ struct Vertex final : ASTTypeVararg { }; +template<> +// ast_type_triangle_args is "T1" +// example: `type OkInt = Ok`, then "Ok" is triangle args, and at resolving, it's instantiated into TypeDataStruct +// example: `type A = Ok`, then "Ok" is triangle args, but at resolving, kept as TypeDataGenericTypeWithTs +struct Vertex final : ASTTypeVararg { + const std::vector& get_inner_and_args() const { return children; } + + Vertex(SrcLocation loc, std::vector&& inner_and_args) + : ASTTypeVararg(ast_type_triangle_args, loc, std::move(inner_and_args)) {} +}; + + // // --------------------------------------------------------- // expressions @@ -915,21 +928,21 @@ struct Vertex final : ASTExprVararg { template<> // ast_object_literal is creating an instance of a struct with initial values of fields, like objects in TypeScript -// example: `Point { ... }` (object creation has explicit ref and body) +// example: `Point { ... }` (object creation has type_node and body) // example: `var v: Point = { ... }` (object creation has only body, struct_ref is determined from the left) -struct Vertex final : ASTExprBinary { +// example: `Wrapper { ... }` (also type_node and body, this type_node is resolved as instantiated generic struct) +struct Vertex final : ASTExprUnary { StructPtr struct_ref = nullptr; // assigned at type inferring + AnyTypeV type_node; // not null for `T { ... }`, nullptr for plain `{ ... }` - bool has_explicit_ref() const { return lhs->kind != ast_empty_expression; } - auto get_explicit_ref() const { return lhs->as(); } - AnyExprV get_maybe_reference() const { return lhs; } // ast_empty_expression or ast_reference - auto get_body() const { return rhs->as(); } + auto get_body() const { return child->as(); } Vertex* mutate() const { return const_cast(this); } void assign_struct_ref(StructPtr struct_ref); - Vertex(SrcLocation loc, AnyExprV maybe_reference, V body) - : ASTExprBinary(ast_object_literal, loc, maybe_reference, body) {} + Vertex(SrcLocation loc, AnyTypeV type_node, V body) + : ASTExprUnary(ast_object_literal, loc, body) + , type_node(type_node) {} }; @@ -1255,6 +1268,7 @@ template<> // see TypeDataAlias in type-system.h struct Vertex final : ASTOtherVararg { AliasDefPtr alias_ref = nullptr; // filled after register + V genericsT_list; // exists for `type Response`; otherwise, nullptr AnyTypeV underlying_type_node; // at the right of `=` auto get_identifier() const { return children.at(0)->as(); } @@ -1262,9 +1276,9 @@ struct Vertex final : ASTOtherVararg { Vertex* mutate() const { return const_cast(this); } void assign_alias_ref(AliasDefPtr alias_ref); - Vertex(SrcLocation loc, V name_identifier, AnyTypeV underlying_type_node) + Vertex(SrcLocation loc, V name_identifier, V genericsT_list, AnyTypeV underlying_type_node) : ASTOtherVararg(ast_type_alias_declaration, loc, {name_identifier}) - , underlying_type_node(underlying_type_node) {} + , genericsT_list(genericsT_list), underlying_type_node(underlying_type_node) {} }; template<> @@ -1288,6 +1302,7 @@ template<> struct Vertex final : ASTOtherVararg { int get_num_fields() const { return size(); } auto get_field(int i) const { return children.at(i)->as(); } + const std::vector& get_all_fields() const { return children; } Vertex(SrcLocation loc, std::vector&& fields) : ASTOtherVararg(ast_struct_body, loc, std::move(fields)) {} @@ -1299,6 +1314,7 @@ template<> // currently, Tolk doesn't have "implements" or whatever, so struct declaration contains only body struct Vertex final : ASTOtherVararg { StructPtr struct_ref = nullptr; // filled after register + V genericsT_list; // exists for `Wrapper`; otherwise, nullptr auto get_identifier() const { return children.at(0)->as(); } auto get_struct_body() const { return children.at(1)->as(); } @@ -1306,8 +1322,9 @@ struct Vertex final : ASTOtherVararg { Vertex* mutate() const { return const_cast(this); } void assign_struct_ref(StructPtr struct_ref); - Vertex(SrcLocation loc, V name_identifier, V struct_body) - : ASTOtherVararg(ast_struct_declaration, loc, {name_identifier, struct_body}) {} + Vertex(SrcLocation loc, V name_identifier, V genericsT_list, V struct_body) + : ASTOtherVararg(ast_struct_declaration, loc, {name_identifier, struct_body}) + , genericsT_list(genericsT_list) {} }; template<> diff --git a/tolk/fwd-declarations.h b/tolk/fwd-declarations.h index cda17c424..97cba7200 100644 --- a/tolk/fwd-declarations.h +++ b/tolk/fwd-declarations.h @@ -48,6 +48,8 @@ using StructPtr = const StructData*; class TypeData; using TypePtr = const TypeData*; +struct GenericsSubstitutions; + struct SrcFile; } // namespace tolk diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index e73639b86..9983ca49d 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -24,49 +24,86 @@ namespace tolk { -// given orig = "(int, T)" and substitutions = [slice], return "(int, slice)" -static TypePtr replace_genericT_with_deduced(TypePtr orig, const GenericsDeclaration* genericTs, const std::vector& substitutionTs) { +// given orig `(int, T)` and substitutions [T=slice], return `(int, slice)` +static TypePtr replace_genericT_with_deduced(TypePtr orig, const GenericsSubstitutions* substitutedTs) { if (!orig || !orig->has_genericT_inside()) { return orig; } - tolk_assert(genericTs->size() == substitutionTs.size()); - return orig->replace_children_custom([genericTs, substitutionTs](TypePtr child) { + return orig->replace_children_custom([substitutedTs](TypePtr child) { if (const TypeDataGenericT* asT = child->try_as()) { - int idx = genericTs->find_nameT(asT->nameT); - if (idx == -1) { - throw Fatal("can not replace generic " + asT->nameT); + TypePtr typeT = substitutedTs->get_substitution_for_nameT(asT->nameT); + if (typeT == nullptr) { // T was not deduced yet, leave T as generic + typeT = child; } - if (substitutionTs[idx] == nullptr) { - throw GenericDeduceError("can not deduce " + asT->nameT); + return typeT; + } + if (const TypeDataGenericTypeWithTs* as_instTs = child->try_as(); as_instTs && !as_instTs->has_genericT_inside()) { + if (StructPtr struct_ref = as_instTs->struct_ref) { + struct_ref = instantiate_generic_struct(struct_ref, GenericsSubstitutions(struct_ref->genericTs, as_instTs->type_arguments)); + return TypeDataStruct::create(struct_ref); + } + if (AliasDefPtr alias_ref = as_instTs->alias_ref) { + alias_ref = instantiate_generic_alias(as_instTs->alias_ref, GenericsSubstitutions(alias_ref->genericTs, as_instTs->type_arguments)); + return TypeDataAlias::create(alias_ref); } - return substitutionTs[idx]; } return child; }); } -GenericSubstitutionsDeduceForCall::GenericSubstitutionsDeduceForCall(FunctionPtr fun_ref) - : fun_ref(fun_ref) { - substitutionTs.resize(fun_ref->genericTs->size()); // filled with nullptr (nothing deduced) +std::string GenericsSubstitutions::as_human_readable() const { + std::string result; + for (int i = 0; i < size(); ++i) { + if (!result.empty()) { + result += ", "; + } + result += genericTs->get_nameT(i); + if (valuesTs[i] == nullptr) { + result += "=nullptr"; + } else { + result += "=`"; + result += valuesTs[i]->as_human_readable(); + result += "`"; + } + } + return result; } -void GenericSubstitutionsDeduceForCall::provide_deducedT(const std::string& nameT, TypePtr deduced) { - if (deduced == TypeDataNullLiteral::create() || deduced->has_unknown_inside()) { - return; // just 'null' doesn't give sensible info +void GenericsSubstitutions::set_typeT(std::string_view nameT, TypePtr typeT) { + for (int i = 0; i < size(); ++i) { + if (genericTs->get_nameT(i) == nameT) { + if (valuesTs[i] == nullptr) { + if (typeT == TypeDataNullLiteral::create() || typeT->has_unknown_inside()) { + throw GenericDeduceError(nameT); + } + tolk_assert(!typeT->has_genericT_inside()); + valuesTs[i] = typeT; + } + break; + } } +} - int idx = fun_ref->genericTs->find_nameT(nameT); - if (substitutionTs[idx] == nullptr) { - substitutionTs[idx] = deduced; - } else if (substitutionTs[idx] != deduced) { - throw GenericDeduceError(nameT + " is both " + substitutionTs[idx]->as_human_readable() + " and " + deduced->as_human_readable()); +GenericsSubstitutions::GenericsSubstitutions(const GenericsDeclaration* genericTs, const std::vector& type_arguments) + : genericTs(genericTs) { + assert(genericTs != nullptr && genericTs->size() == static_cast(type_arguments.size())); + valuesTs.resize(genericTs->size()); + for (int i = 0; i < genericTs->size(); ++i) { + valuesTs[i] = type_arguments[i]; } } -void GenericSubstitutionsDeduceForCall::provide_manually_specified(std::vector&& substitutionTs) { - this->substitutionTs = std::move(substitutionTs); - this->manually_specified = true; +GenericSubstitutionsDeducing::GenericSubstitutionsDeducing(FunctionPtr fun_ref) + : fun_ref(fun_ref) + , struct_ref(nullptr) + , deducedTs(fun_ref->genericTs) { +} + +GenericSubstitutionsDeducing::GenericSubstitutionsDeducing(StructPtr struct_ref) + : fun_ref(nullptr) + , struct_ref(struct_ref) + , deducedTs(struct_ref->genericTs) { } // purpose: having `f(value: T)` and call `f(5)`, deduce T = int @@ -75,12 +112,18 @@ void GenericSubstitutionsDeduceForCall::provide_manually_specified(std::vector(a: int, b: T1, c: (T1, T2))` and call `f(6, 7, (8, cs))` // - `a` does not affect, it doesn't depend on generic Ts // - next condition: param_type = `T1`, arg_type = `int`, deduce T1 = int -// - next condition: param_type = `(T1, T2)`, arg_type = `(int, slice)`, deduce T1 = int, T2 = slice -// for call `f(6, cs, (8, cs))` T1 will be both `slice` and `int`, fired an error -void GenericSubstitutionsDeduceForCall::consider_next_condition(TypePtr param_type, TypePtr arg_type) { +// - next condition: param_type = `(T1, T2)` = `(int, T2)`, arg_type = `(int, slice)`, deduce T2 = slice +// for call `f(6, cs, (8, cs))` both T1 and T2 will become `slice`, firing a type mismatch error later +void GenericSubstitutionsDeducing::consider_next_condition(TypePtr param_type, TypePtr arg_type) { + // all Ts deduced up to this point are apriori + param_type = replace_genericT_with_deduced(param_type, &deducedTs); + if (!param_type->has_genericT_inside()) { + return; + } + if (const auto* asT = param_type->try_as()) { // `(arg: T)` called as `f([1, 2])` => T is [int, int] - provide_deducedT(asT->nameT, arg_type); + deducedTs.set_typeT(asT->nameT, arg_type); } else if (const auto* p_nullable = param_type->try_as(); p_nullable && p_nullable->or_null) { // `arg: T?` called as `f(nullableInt)` => T is int if (const auto* a_nullable = arg_type->unwrap_alias()->try_as(); a_nullable && a_nullable->or_null) { @@ -114,43 +157,90 @@ void GenericSubstitutionsDeduceForCall::consider_next_condition(TypePtr param_ty } } else if (const auto* p_union = param_type->try_as()) { // `arg: T1 | T2` called as `f(intOrBuilder)` => T1 is int, T2 is builder - if (const auto* a_union = arg_type->unwrap_alias()->try_as(); a_union && a_union->variants.size() == p_union->variants.size()) { - for (int i = 0; i < static_cast(p_union->variants.size()); ++i) { - consider_next_condition(p_union->variants[i], a_union->variants[i]); + // `arg: int | T1` called as `f(builderOrIntOrSlice)` => T1 is builder|slice + if (const auto* a_union = arg_type->unwrap_alias()->try_as()) { + std::vector p_generic; + std::vector a_sub_p = a_union->variants; + bool is_sub_correct = true; + for (TypePtr p_variant : p_union->variants) { + if (!p_variant->has_genericT_inside()) { + auto it = std::find(a_sub_p.begin(), a_sub_p.end(), p_variant); + if (it != a_sub_p.end()) { + a_sub_p.erase(it); + } else { + is_sub_correct = false; + } + } else { + p_generic.push_back(p_variant); + } + } + if (is_sub_correct && p_generic.size() == 1 && a_sub_p.size() > 1) { + consider_next_condition(p_generic[0], TypeDataUnion::create(std::move(a_sub_p))); + } else if (is_sub_correct && p_generic.size() == a_sub_p.size()) { + for (int i = 0; i < static_cast(p_generic.size()); ++i) { + consider_next_condition(p_generic[i], a_sub_p[i]); + } + } + } + } else if (const auto* p_instSt = param_type->try_as(); p_instSt && p_instSt->struct_ref) { + // `arg: Wrapper` called as `f(wrappedInt)` => T is int + if (const auto* a_struct = arg_type->try_as(); a_struct && a_struct->struct_ref->is_instantiation_of_generic_struct() && a_struct->struct_ref->base_struct_ref == p_instSt->struct_ref) { + tolk_assert(p_instSt->size() == a_struct->struct_ref->substitutedTs->size()); + for (int i = 0; i < p_instSt->size(); ++i) { + consider_next_condition(p_instSt->type_arguments[i], a_struct->struct_ref->substitutedTs->typeT_at(i)); + } + } + } else if (const auto* p_instAl = param_type->try_as(); p_instAl && p_instAl->alias_ref) { + // `arg: WrapperAlias` called as `f(wrappedInt)` => T is int + if (const auto* a_alias = arg_type->try_as(); a_alias && a_alias->alias_ref->is_instantiation_of_generic_alias() && a_alias->alias_ref->base_alias_ref == p_instAl->alias_ref) { + tolk_assert(p_instAl->size() == a_alias->alias_ref->substitutedTs->size()); + for (int i = 0; i < p_instAl->size(); ++i) { + consider_next_condition(p_instAl->type_arguments[i], a_alias->alias_ref->substitutedTs->typeT_at(i)); } } } } -TypePtr GenericSubstitutionsDeduceForCall::replace_by_manually_specified(TypePtr param_type) const { - return replace_genericT_with_deduced(param_type, fun_ref->genericTs, substitutionTs); +TypePtr GenericSubstitutionsDeducing::replace_Ts_with_currently_deduced(TypePtr orig) const { + return replace_genericT_with_deduced(orig, &deducedTs); } -TypePtr GenericSubstitutionsDeduceForCall::auto_deduce_from_argument(FunctionPtr cur_f, SrcLocation loc, TypePtr param_type, TypePtr arg_type) { +TypePtr GenericSubstitutionsDeducing::auto_deduce_from_argument(FunctionPtr cur_f, SrcLocation loc, TypePtr param_type, TypePtr arg_type) { try { - if (!manually_specified) { - consider_next_condition(param_type, arg_type); + consider_next_condition(param_type, arg_type); + param_type = replace_genericT_with_deduced(param_type, &deducedTs); + if (param_type->has_genericT_inside()) { + fire_error_can_not_deduce(cur_f, loc, get_first_not_deduced_nameT()); } - return replace_genericT_with_deduced(param_type, fun_ref->genericTs, substitutionTs); + return param_type; } catch (const GenericDeduceError& ex) { - throw ParseError(cur_f, loc, ex.message + " for generic function `" + fun_ref->as_human_readable() + "`; instantiate it manually with `" + fun_ref->name + "<...>()`"); + fire_error_can_not_deduce(cur_f, loc, ex.nameT); + return nullptr; } } -int GenericSubstitutionsDeduceForCall::get_first_not_deduced_idx() const { - for (int i = 0; i < static_cast(substitutionTs.size()); ++i) { - if (substitutionTs[i] == nullptr) { - return i; +std::string_view GenericSubstitutionsDeducing::get_first_not_deduced_nameT() const { + for (int i = 0; i < deducedTs.size(); ++i) { + if (deducedTs.typeT_at(i) == nullptr) { + return deducedTs.nameT_at(i); } } - return -1; + return ""; +} + +void GenericSubstitutionsDeducing::fire_error_can_not_deduce(FunctionPtr cur_f, SrcLocation loc, std::string_view nameT) const { + if (fun_ref) { + fire(cur_f, loc, "can not deduce " + static_cast(nameT) + " for generic function `" + fun_ref->as_human_readable() + "`; instantiate it manually with `" + fun_ref->name + "<...>()`"); + } else { + fire(cur_f, loc, "can not deduce " + static_cast(nameT) + " for generic struct `" + struct_ref->as_human_readable() + "`; instantiate it manually with `" + struct_ref->name + "<...>`"); + } } std::string GenericsDeclaration::as_human_readable() const { std::string result = "<"; for (const GenericsItem& item : itemsT) { if (result.size() > 1) { - result += ","; + result += ", "; } result += item.nameT; } @@ -167,36 +257,61 @@ int GenericsDeclaration::find_nameT(std::string_view nameT) const { return -1; } +bool GenericsSubstitutions::has_nameT(std::string_view nameT) const { + return genericTs->find_nameT(nameT) != -1; +} + +TypePtr GenericsSubstitutions::get_substitution_for_nameT(std::string_view nameT) const { + int idx = genericTs->find_nameT(nameT); + return idx == -1 ? nullptr : valuesTs[idx]; +} + +// given this= and rhs=, check that T1 is equal to T2 in terms of "equal_to" of TypePtr +// for example, `Wrapper>` / `Wrapper>` / `Wrapper` are equal +bool GenericsSubstitutions::equal_to(const GenericsSubstitutions* rhs) const { + if (size() != rhs->size()) { + return false; + } + for (int i = 0; i < size(); ++i) { + if (!valuesTs[i]->equal_to(rhs->valuesTs[i])) { + return false; + } + } + return true; +} + // when cloning `f`, original name is "f", we need a new name for symtable and output // name of an instantiated function will be "f" and similar (yes, with "<" symbol, it's okay to Fift) -static std::string generate_instantiated_name(const std::string& orig_name, const std::vector& substitutions) { +static std::string generate_instantiated_name(const std::string& orig_name, const GenericsSubstitutions& substitutedTs, bool allow_spaces) { // an instantiated function name will be "{orig_name}<{T1,T2,...}>" std::string name = orig_name; name += "<"; - for (TypePtr subs : substitutions) { + for (int i = 0; i < substitutedTs.size(); ++i) { if (name.size() > orig_name.size() + 1) { - name += ","; + name += ", "; } - name += subs->as_human_readable(); + name += substitutedTs.typeT_at(i)->as_human_readable(); + } + if (!allow_spaces) { + name.erase(std::remove(name.begin(), name.end(), ' '), name.end()); } - name.erase(std::remove(name.begin(), name.end(), ' '), name.end()); name += ">"; return name; } -FunctionPtr instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, std::vector&& substitutionTs) { +FunctionPtr instantiate_generic_function(FunctionPtr fun_ref, GenericsSubstitutions&& substitutedTs) { tolk_assert(fun_ref->is_generic_function() && !fun_ref->is_method_id_not_empty()); // fun_ref->name = "f", inst_name will be "f" and similar - std::string new_name = generate_instantiated_name(fun_ref->name, substitutionTs); + std::string new_name = generate_instantiated_name(fun_ref->name, substitutedTs, false); if (const Symbol* existing_sym = lookup_global_symbol(new_name)) { FunctionPtr existing_ref = existing_sym->try_as(); tolk_assert(existing_ref); return existing_ref; } - const GenericsInstantiation* instantiationTs = new GenericsInstantiation(loc, std::move(substitutionTs)); - ASTReplicatorFunction replicator; + // to store permanently, allocate an object in heap + const GenericsSubstitutions* allocatedTs = new GenericsSubstitutions(std::move(substitutedTs)); // built-in functions don't have AST to clone, types of parameters don't exist in AST, etc. // nevertheless, for outer code to follow a single algorithm, @@ -205,13 +320,14 @@ FunctionPtr instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, s std::vector new_parameters; new_parameters.reserve(fun_ref->get_num_params()); for (const LocalVarData& orig_p : fun_ref->parameters) { - TypePtr new_param_type = replace_genericT_with_deduced(orig_p.declared_type, fun_ref->genericTs, instantiationTs->substitutions); + TypePtr new_param_type = replace_genericT_with_deduced(orig_p.declared_type, allocatedTs); new_parameters.emplace_back(orig_p.name, orig_p.loc, new_param_type, orig_p.flags, orig_p.param_idx); } - TypePtr new_return_type = replace_genericT_with_deduced(fun_ref->declared_return_type, fun_ref->genericTs, instantiationTs->substitutions); - FunctionData* new_fun_ref = new FunctionData(new_name, fun_ref->loc, new_return_type, std::move(new_parameters), fun_ref->flags, nullptr, instantiationTs, fun_ref->body, fun_ref->ast_root); + TypePtr new_return_type = replace_genericT_with_deduced(fun_ref->declared_return_type, allocatedTs); + FunctionData* new_fun_ref = new FunctionData(new_name, fun_ref->loc, new_return_type, std::move(new_parameters), fun_ref->flags, nullptr, allocatedTs, fun_ref->body, fun_ref->ast_root); new_fun_ref->arg_order = fun_ref->arg_order; new_fun_ref->ret_order = fun_ref->ret_order; + new_fun_ref->base_fun_ref = fun_ref; G.symtable.add_function(new_fun_ref); return new_fun_ref; } @@ -221,9 +337,9 @@ FunctionPtr instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, s // it's instantiation, when resolving types, it substitutes T=int V orig_root = fun_ref->ast_root->as(); V new_name_ident = createV(orig_root->get_identifier()->loc, new_name); - V new_root = replicator.clone_function_ast(orig_root, new_name_ident); + V new_root = ASTReplicator::clone_function_ast(orig_root, new_name_ident); - FunctionPtr new_fun_ref = pipeline_register_instantiated_generic_function(new_root, instantiationTs); + FunctionPtr new_fun_ref = pipeline_register_instantiated_generic_function(fun_ref, new_root, allocatedTs); tolk_assert(new_fun_ref); // body of a cloned function (it's cloned at type inferring step) needs the previous pipeline to run // for example, all local vars need to be registered as symbols, etc. @@ -236,4 +352,49 @@ FunctionPtr instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, s return new_fun_ref; } +StructPtr instantiate_generic_struct(StructPtr struct_ref, GenericsSubstitutions&& substitutedTs) { + tolk_assert(struct_ref->is_generic_struct()); + + // if `Wrapper` was earlier instantiated, return it + std::string new_name = generate_instantiated_name(struct_ref->name, substitutedTs, true); + if (const Symbol* existing_sym = lookup_global_symbol(new_name)) { + StructPtr existing_ref = existing_sym->try_as(); + tolk_assert(existing_ref); + return existing_ref; + } + + const GenericsSubstitutions* allocatedTs = new GenericsSubstitutions(std::move(substitutedTs)); + V orig_root = struct_ref->ast_root->as(); + V new_name_ident = createV(orig_root->get_identifier()->loc, new_name); + V new_root = ASTReplicator::clone_struct_ast(orig_root, new_name_ident); + + StructPtr new_struct_ref = pipeline_register_instantiated_generic_struct(struct_ref, new_root, allocatedTs); + tolk_assert(new_struct_ref); + pipeline_resolve_identifiers_and_assign_symbols(new_struct_ref); + pipeline_resolve_types_and_aliases(new_struct_ref); + return new_struct_ref; +} + +AliasDefPtr instantiate_generic_alias(AliasDefPtr alias_ref, GenericsSubstitutions&& substitutedTs) { + tolk_assert(alias_ref->is_generic_alias()); + + // if `Response` was earlier instantiated, return it + std::string new_name = generate_instantiated_name(alias_ref->name, substitutedTs, true); + if (const Symbol* existing_sym = lookup_global_symbol(new_name)) { + AliasDefPtr existing_ref = existing_sym->try_as(); + tolk_assert(existing_ref); + return existing_ref; + } + + const GenericsSubstitutions* allocatedTs = new GenericsSubstitutions(std::move(substitutedTs)); + V orig_root = alias_ref->ast_root->as(); + V new_name_ident = createV(orig_root->get_identifier()->loc, new_name); + V new_root = ASTReplicator::clone_type_alias_ast(orig_root, new_name_ident); + + AliasDefPtr new_alias_ref = pipeline_register_instantiated_generic_alias(alias_ref, new_root, allocatedTs); + tolk_assert(new_alias_ref); + pipeline_resolve_types_and_aliases(new_alias_ref); + return new_alias_ref; +} + } // namespace tolk diff --git a/tolk/generics-helpers.h b/tolk/generics-helpers.h index 140d27815..7d324e625 100644 --- a/tolk/generics-helpers.h +++ b/tolk/generics-helpers.h @@ -40,61 +40,75 @@ struct GenericsDeclaration { std::string as_human_readable() const; - size_t size() const { return itemsT.size(); } + int size() const { return static_cast(itemsT.size()); } int find_nameT(std::string_view nameT) const; - std::string get_nameT(int idx) const { return static_cast(itemsT[idx].nameT); } + std::string_view get_nameT(int idx) const { return itemsT[idx].nameT; } }; // when a function call is `f()`, this "" is represented as this class -struct GenericsInstantiation { - const std::vector substitutions; // for genericTs - const SrcLocation loc; // first instantiation location +// same for `Wrapper`, "" is substitution +struct GenericsSubstitutions { +private: + const GenericsDeclaration* genericTs; // [T1, T2] + std::vector valuesTs; // [SomeStruct, int] - explicit GenericsInstantiation(SrcLocation loc, std::vector&& substitutions) - : substitutions(std::move(substitutions)) - , loc(loc) { +public: + explicit GenericsSubstitutions(const GenericsDeclaration* genericTs) + : genericTs(genericTs) + , valuesTs(genericTs == nullptr ? 0 : genericTs->size()) { } + explicit GenericsSubstitutions(const GenericsDeclaration* genericTs, const std::vector& type_arguments); + + std::string as_human_readable() const; + + int size() const { return static_cast(valuesTs.size()); } + bool has_nameT(std::string_view nameT) const; + TypePtr get_substitution_for_nameT(std::string_view nameT) const; + std::string_view nameT_at(int idx) const { return genericTs->get_nameT(idx); } + TypePtr typeT_at(int idx) const { return valuesTs.at(idx); } + bool equal_to(const GenericsSubstitutions* rhs) const; + + void set_typeT(std::string_view nameT, TypePtr typeT); }; // this class helps to deduce Ts on the fly // purpose: having `f(value: T)` and call `f(5)`, deduce T = int // while analyzing a call, arguments are handled one by one, by `auto_deduce_from_argument()` -// this class also handles manually specified substitutions like `f(5)` -class GenericSubstitutionsDeduceForCall { +// note, that manually specified substitutions like `f(5)` are NOT handled by this class, it's not deducing +class GenericSubstitutionsDeducing { FunctionPtr fun_ref; - std::vector substitutionTs; - bool manually_specified = false; + StructPtr struct_ref; + GenericsSubstitutions deducedTs; - void provide_deducedT(const std::string& nameT, TypePtr deduced); void consider_next_condition(TypePtr param_type, TypePtr arg_type); public: - explicit GenericSubstitutionsDeduceForCall(FunctionPtr fun_ref); - - bool is_manually_specified() const { - return manually_specified; - } + explicit GenericSubstitutionsDeducing(FunctionPtr fun_ref); + explicit GenericSubstitutionsDeducing(StructPtr struct_ref); - void provide_manually_specified(std::vector&& substitutionTs); - TypePtr replace_by_manually_specified(TypePtr param_type) const; + TypePtr replace_Ts_with_currently_deduced(TypePtr orig) const; TypePtr auto_deduce_from_argument(FunctionPtr cur_f, SrcLocation loc, TypePtr param_type, TypePtr arg_type); - int get_first_not_deduced_idx() const; + std::string_view get_first_not_deduced_nameT() const; + void fire_error_can_not_deduce(FunctionPtr cur_f, SrcLocation loc, std::string_view nameT) const; - std::vector&& flush() { - return std::move(substitutionTs); + GenericsSubstitutions&& flush() { + return std::move(deducedTs); } }; struct GenericDeduceError final : std::exception { - std::string message; - explicit GenericDeduceError(std::string message) - : message(std::move(message)) { } + std::string nameT; + + explicit GenericDeduceError(std::string_view nameT) + : nameT(nameT) { } const char* what() const noexcept override { - return message.c_str(); + return nameT.c_str(); } }; -FunctionPtr instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, std::vector&& substitutionTs); +FunctionPtr instantiate_generic_function(FunctionPtr fun_ref, GenericsSubstitutions&& substitutedTs); +StructPtr instantiate_generic_struct(StructPtr struct_ref, GenericsSubstitutions&& substitutedTs); +AliasDefPtr instantiate_generic_alias(AliasDefPtr alias_ref, GenericsSubstitutions&& substitutedTs); } // namespace tolk diff --git a/tolk/lexer.cpp b/tolk/lexer.cpp index 32989b3fa..d34a86399 100644 --- a/tolk/lexer.cpp +++ b/tolk/lexer.cpp @@ -591,6 +591,12 @@ void Lexer::restore_position(SavedPositionForLookahead saved) { location = saved.loc; } +void Lexer::hack_replace_rshift_with_one_triangle() { + // overcome the `>>` problem when parsing generics, leave only `>` here, see comments at usage + assert(cur_token.type == tok_rshift); + cur_token = Token(tok_gt, ">"); +} + void Lexer::error(const std::string& err_msg) const { throw ParseError(cur_location(), err_msg); } diff --git a/tolk/lexer.h b/tolk/lexer.h index 9c8545bf3..55bdf4edb 100644 --- a/tolk/lexer.h +++ b/tolk/lexer.h @@ -212,6 +212,7 @@ class Lexer { SavedPositionForLookahead save_parsing_position() const; void restore_position(SavedPositionForLookahead saved); + void hack_replace_rshift_with_one_triangle(); void check(TokenType next_tok, const char* str_expected) const { if (cur_token.type != next_tok) { diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 2139ae714..cefdb5b8d 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -681,7 +681,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectorvariants.size() >= o_union->variants.size()) { + if (t_union && o_union && t_union->size() >= o_union->size()) { tolk_assert(target_w >= orig_w && t_union->has_all_variants_of(o_union)); std::vector prepend_nulls; prepend_nulls.reserve(target_w - orig_w); @@ -830,6 +830,13 @@ static std::vector transition_expr_to_runtime_type_impl(std::vector> is ok to Wrapper> + if (original_type->try_as() && target_type->try_as()) { + tolk_assert(target_type->can_rhs_be_assigned(original_type) && orig_w == target_w); + return rvect; + } + throw Fatal("unhandled transition_expr_to_runtime_type_impl() combination"); } diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 8c42b3f54..1df682140 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -750,7 +750,7 @@ void pipeline_check_inferred_types() { } for (StructPtr struct_ref : get_all_declared_structs()) { for (StructFieldPtr field_ref : struct_ref->fields) { - if (field_ref->has_default_value()) { + if (field_ref->has_default_value() && !struct_ref->is_generic_struct()) { visitor.start_visiting_field_default(field_ref); } } diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp index 30d0ff562..32b7c28c2 100644 --- a/tolk/pipe-constant-folding.cpp +++ b/tolk/pipe-constant-folding.cpp @@ -170,7 +170,7 @@ void pipeline_constant_folding() { // do the same for default values of struct fields, they must be constant expressions for (StructPtr struct_ref : get_all_declared_structs()) { for (StructFieldPtr field_ref : struct_ref->fields) { - if (field_ref->has_default_value()) { + if (field_ref->has_default_value() && !struct_ref->is_generic_struct()) { check_expression_is_constant(field_ref->default_value); AnyExprV replaced = replacer.replace_in_expression(field_ref->default_value); field_ref->mutate()->assign_default_value(replaced); diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index f1e5b4c25..06e658ac9 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -120,6 +120,16 @@ static std::string to_string(FunctionPtr fun_ref) { return "`" + fun_ref->as_human_readable() + "`"; } +GNU_ATTRIBUTE_NOINLINE +static std::string to_string(StructPtr struct_ref) { + return "`" + struct_ref->as_human_readable() + "`"; +} + +GNU_ATTRIBUTE_NOINLINE +static std::string to_string(AliasDefPtr alias_ref) { + return "`" + alias_ref->as_human_readable() + "`"; +} + GNU_ATTRIBUTE_NOINLINE static std::string to_string(std::string_view string_view) { return static_cast(string_view); @@ -129,8 +139,8 @@ static std::string to_string(std::string_view string_view) { // asm functions generally can't handle it, they expect T to be a TVM primitive // (in FunC, `forall` type just couldn't be unified with non-primitives; in Tolk, generic T is expectedly inferred) GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_calling_asm_function_with_non1_stack_width_arg(FunctionPtr cur_f, SrcLocation loc, FunctionPtr fun_ref, const std::vector& substitutions, int arg_idx) { - fire(cur_f, loc, "can not call `" + fun_ref->as_human_readable() + "` with " + fun_ref->genericTs->get_nameT(arg_idx) + "=" + substitutions[arg_idx]->as_human_readable() + ", because it occupies " + std::to_string(substitutions[arg_idx]->get_width_on_stack()) + " stack slots in TVM, not 1"); +static void fire_error_calling_asm_function_with_non1_stack_width_arg(FunctionPtr cur_f, SrcLocation loc, FunctionPtr fun_ref, const GenericsSubstitutions& substitutions, int arg_idx) { + fire(cur_f, loc, "can not call `" + fun_ref->as_human_readable() + "` with " + to_string(substitutions.nameT_at(arg_idx)) + "=" + substitutions.typeT_at(arg_idx)->as_human_readable() + ", because it occupies " + std::to_string(substitutions.typeT_at(arg_idx)->get_width_on_stack()) + " stack slots in TVM, not 1"); } // fire an error on `untypedTupleVar.0` when used without a hint @@ -140,6 +150,51 @@ static void fire_error_cannot_deduce_untyped_tuple_access(FunctionPtr cur_f, Src fire(cur_f, loc, "can not deduce type of `" + idx_access + "`; either assign it to variable like `var c: int = " + idx_access + "` or cast the result like `" + idx_access + " as int`"); } +// helper function: given hint = `Ok | Err` and struct `Ok`, return `Ok` +// example: `match (...) { Ok => ... }` we need to deduce `Ok` based on subject +static TypePtr try_pick_instantiated_generic_from_hint(TypePtr hint, StructPtr lookup_ref) { + // example: `var w: Ok = Ok { ... }`, hint is `Ok`, lookup is `Ok` + if (const TypeDataStruct* h_struct = hint->unwrap_alias()->try_as()) { + if (lookup_ref == h_struct->struct_ref->base_struct_ref) { + return h_struct; + } + } + // example: `fun f(): Response { return Err { ... } }`, hint is `Ok | Err`, lookup is `Err` + if (const TypeDataUnion* h_union = hint->unwrap_alias()->try_as()) { + TypePtr only_variant = nullptr; // hint `Ok | Ok` is ambiguous + for (TypePtr h_variant : h_union->variants) { + if (const TypeDataStruct* variant_struct = h_variant->unwrap_alias()->try_as()) { + if (lookup_ref == variant_struct->struct_ref->base_struct_ref) { + if (only_variant) { + return nullptr; + } + only_variant = variant_struct; + } + } + } + return only_variant; + } + return nullptr; +} + +// helper function, similar to the above, but for generic type aliases +// example: `v is OkAlias`, need to deduce `OkAlias` based on type of v +static TypePtr try_pick_instantiated_generic_from_hint(TypePtr hint, AliasDefPtr lookup_ref) { + // when a generic type alias points to a generic struct actually: `type WrapperAlias = Wrapper` + if (const TypeDataGenericTypeWithTs* as_instT = lookup_ref->underlying_type->try_as()) { + return as_instT->struct_ref + ? try_pick_instantiated_generic_from_hint(hint, as_instT->struct_ref) + : try_pick_instantiated_generic_from_hint(hint, as_instT->alias_ref); + } + // it's something weird, when a generic alias refs non-generic type + // example: `type StrangeInt = int`, hint is `StrangeInt`, lookup `StrangeInt` + if (const TypeDataAlias* h_alias = hint->try_as()) { + if (lookup_ref == h_alias->alias_ref->base_alias_ref) { + return h_alias; + } + } + return nullptr; +} /* * This class handles all types of AST vertices and traverses them, filling all AnyExprV::inferred_type. @@ -626,7 +681,28 @@ class InferTypesAndCallsAndFieldsVisitor final { ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false); assign_inferred_type(v, TypeDataBool::create()); - TypePtr rhs_type = v->type_node->resolved_type->unwrap_alias(); + TypePtr rhs_type = v->type_node->resolved_type; + + if (const auto* t_struct = rhs_type->try_as(); t_struct && t_struct->struct_ref->is_generic_struct()) { + // `v is Wrapper`, detect T based on type of v (`Wrapper | int` => `Wrapper`) + if (TypePtr inst_rhs_type = try_pick_instantiated_generic_from_hint(v->get_expr()->inferred_type, t_struct->struct_ref)) { + rhs_type = inst_rhs_type; + v->type_node->mutate()->assign_resolved_type(rhs_type); + } else { + fire(cur_f, v->type_node->loc, "can not deduce type arguments for " + to_string(t_struct->struct_ref) + ", provide them manually"); + } + } + if (const auto* t_alias = rhs_type->try_as(); t_alias && t_alias->alias_ref->is_generic_alias()) { + // `v is WrapperAlias`, detect T similar to structures + if (TypePtr inst_rhs_type = try_pick_instantiated_generic_from_hint(v->get_expr()->inferred_type, t_alias->alias_ref)) { + rhs_type = inst_rhs_type; + v->type_node->mutate()->assign_resolved_type(rhs_type); + } else { + fire(cur_f, v->type_node->loc, "can not deduce type arguments for " + to_string(t_alias->alias_ref) + ", provide them manually"); + } + } + + rhs_type = rhs_type->unwrap_alias(); TypePtr expr_type = v->get_expr()->inferred_type->unwrap_alias(); TypePtr non_rhs_type = calculate_type_subtract_rhs_type(expr_type, rhs_type); if (expr_type->equal_to(rhs_type)) { // `expr is ` is always true @@ -712,8 +788,8 @@ class InferTypesAndCallsAndFieldsVisitor final { } else if (fun_ref->is_generic_function()) { // `genericFn` is valid, it's a reference to instantiation - std::vector substitutions = collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref, v_instantiationTs); - fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutions)); + GenericsSubstitutions substitutedTs = collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref->genericTs, v_instantiationTs); + fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutedTs)); v->mutate()->assign_sym(fun_ref); } else if (v_instantiationTs != nullptr && !fun_ref->is_instantiation_of_generic_function()) { @@ -727,10 +803,10 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(flow), used_as_condition); } else if (AliasDefPtr alias_ref = v->sym->try_as()) { - fire(cur_f, v->loc, "type `" + alias_ref->name + "` can not be used as a value"); + fire(cur_f, v->loc, "type " + to_string(alias_ref) + " can not be used as a value"); } else if (StructPtr struct_ref = v->sym->try_as()) { - fire(cur_f, v->loc, "struct `" + struct_ref->name + "` can not be used as a value"); + fire(cur_f, v->loc, "struct " + to_string(struct_ref) + " can not be used as a value"); } else { tolk_assert(false); @@ -745,19 +821,19 @@ class InferTypesAndCallsAndFieldsVisitor final { // given `genericF` / `t.tupleFirst` (the user manually specified instantiation Ts), // validate and collect them - // returns: [int, slice] / [cell] - std::vector collect_fun_generic_substitutions_from_manually_specified(SrcLocation loc, FunctionPtr fun_ref, V instantiationT_list) const { - if (fun_ref->genericTs->size() != instantiationT_list->get_items().size()) { - fire(cur_f, loc, "wrong count of generic T: expected " + std::to_string(fun_ref->genericTs->size()) + ", got " + std::to_string(instantiationT_list->size())); + // returns: [T1=int, T2=slice] / [T=cell] (with flag auto_deduced = false, meaning they are manually specified) + GenericsSubstitutions collect_fun_generic_substitutions_from_manually_specified(SrcLocation loc, const GenericsDeclaration* genericTs, V instantiationT_list) const { + if (instantiationT_list->size() != genericTs->size()) { + fire(cur_f, loc, "wrong count of generic T: expected " + std::to_string(genericTs->size()) + ", got " + std::to_string(instantiationT_list->size())); } - std::vector substitutions; - substitutions.reserve(instantiationT_list->size()); + std::vector type_arguments; + type_arguments.reserve(instantiationT_list->size()); for (int i = 0; i < instantiationT_list->size(); ++i) { - substitutions.push_back(instantiationT_list->get_item(i)->type_node->resolved_type); + type_arguments.push_back(instantiationT_list->get_item(i)->type_node->resolved_type); } - return substitutions; + return GenericsSubstitutions(genericTs, type_arguments); } // when generic Ts have been collected from user-specified or deduced from arguments, @@ -766,19 +842,19 @@ class InferTypesAndCallsAndFieldsVisitor final { // example: was `t.tuplePush(2)`, read , instantiate `tuplePush` (will later fail type check) // example: was `var cb = t.tupleFirst;` (used as reference, as non-call), instantiate `tupleFirst` // returns fun_ref to instantiated function - FunctionPtr check_and_instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, std::vector&& substitutionTs) const { + FunctionPtr check_and_instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, GenericsSubstitutions&& substitutedTs) const { // T for asm function must be a TVM primitive (width 1), otherwise, asm would act incorrectly if (fun_ref->is_asm_function() || fun_ref->is_builtin_function()) { - for (int i = 0; i < static_cast(substitutionTs.size()); ++i) { - if (substitutionTs[i]->get_width_on_stack() != 1) { - fire_error_calling_asm_function_with_non1_stack_width_arg(cur_f, loc, fun_ref, substitutionTs, i); + for (int i = 0; i < substitutedTs.size(); ++i) { + if (substitutedTs.typeT_at(i)->get_width_on_stack() != 1) { + fire_error_calling_asm_function_with_non1_stack_width_arg(cur_f, loc, fun_ref, substitutedTs, i); } } } - // make deep clone of `f` with substitutionTs + // make deep clone of `f` with substitutedTs // (if `f` was already instantiated, it will be immediately returned from a symbol table) - return instantiate_generic_function(loc, fun_ref, std::move(substitutionTs)); + return instantiate_generic_function(fun_ref, std::move(substitutedTs)); } ExprFlow infer_dot_access(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { @@ -889,8 +965,8 @@ class InferTypesAndCallsAndFieldsVisitor final { } else if (fun_ref->is_generic_function()) { // `t.tupleAt` is valid, it's a reference to instantiation - std::vector substitutions = collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref, v_instantiationTs); - fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutions)); + GenericsSubstitutions substitutedTs = collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref->genericTs, v_instantiationTs); + fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutedTs)); } else if (UNLIKELY(v_instantiationTs != nullptr)) { // non-generic method referenced like `var cb = c.cellHash;` @@ -915,7 +991,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (auto v_ref = callee->try_as()) { // `globalF()` / `globalF()` / `local_var()` / `SOME_CONST()` - fun_ref = v_ref->sym->try_as(); // not null for `globalF` + fun_ref = v_ref->sym->try_as(); // not null for `globalF` v_instantiationTs = v_ref->get_instantiationTs(); // present for `globalF()` } else if (auto v_dot = callee->try_as()) { @@ -970,29 +1046,36 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(flow), used_as_condition); } + // for a call `f(...)`, create and instantiate function "f" right now + // so that further, fun_ref will be not a generic, but an instantiated function + if (fun_ref->is_generic_function() && v_instantiationTs) { + GenericsSubstitutions substitutedTs = collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref->genericTs, v_instantiationTs); + fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutedTs)); + } else if (UNLIKELY(v_instantiationTs != nullptr)) { + // `c.cellHash()` / `beginCell()` + fire(cur_f, v_instantiationTs->loc, "calling a not generic function with generic T"); + } + // so, we have a call `f(args)` or `obj.f(args)`, f is a global function (fun_ref) (code / asm / builtin) - // we're going to iterate over passed arguments, and (if generic) infer substitutionTs + // we're going to iterate over passed arguments, and (if generic) infer substitutedTs // at first, check arguments count (Tolk doesn't have optional parameters, so just compare counts) int n_arguments = v->get_num_args() + delta_self; int n_parameters = fun_ref->get_num_params(); if (!n_parameters && dot_obj) { - fire(cur_f, v->loc, "`" + fun_ref->name + "` has no parameters and can not be called as method"); + fire(cur_f, v->loc, to_string(fun_ref) + " has no parameters and can not be called as method"); } if (n_parameters < n_arguments) { - fire(cur_f, v->loc, "too many arguments in call to `" + fun_ref->name + "`, expected " + std::to_string(n_parameters - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); + fire(cur_f, v->loc, "too many arguments in call to " + to_string(fun_ref) + ", expected " + std::to_string(n_parameters - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); } if (n_arguments < n_parameters) { - fire(cur_f, v->loc, "too few arguments in call to `" + fun_ref->name + "`, expected " + std::to_string(n_parameters - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); + fire(cur_f, v->loc, "too few arguments in call to " + to_string(fun_ref) + ", expected " + std::to_string(n_parameters - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); } // now, for every passed argument, we need to infer its type // for regular functions, it's obvious - // but for generic functions, we need to infer type arguments (substitutionTs) on the fly + // but for generic functions, we need to infer type arguments (substitutedTs) on the fly // (unless Ts are specified by a user like `f(args)` / `t.tupleAt()`, take them) - GenericSubstitutionsDeduceForCall* deducingTs = fun_ref->is_generic_function() ? new GenericSubstitutionsDeduceForCall(fun_ref) : nullptr; - if (deducingTs && v_instantiationTs) { - deducingTs->provide_manually_specified(collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref, v_instantiationTs)); - } + GenericSubstitutionsDeducing deducingTs(fun_ref); // loop over every argument, for `obj.method()` obj is the first one // if genericT deducing has a conflict, ParseError is thrown @@ -1002,7 +1085,7 @@ class InferTypesAndCallsAndFieldsVisitor final { const LocalVarData& param_0 = fun_ref->parameters[0]; TypePtr param_type = param_0.declared_type; if (param_type->has_genericT_inside()) { - param_type = deducingTs->auto_deduce_from_argument(cur_f, dot_obj->loc, param_type, dot_obj->inferred_type); + param_type = deducingTs.auto_deduce_from_argument(cur_f, dot_obj->loc, param_type, dot_obj->inferred_type); } if (param_0.is_mutate_parameter() && dot_obj->inferred_type->unwrap_alias() != param_type->unwrap_alias()) { if (SinkExpression s_expr = extract_sink_expression_from_vertex(dot_obj)) { @@ -1015,16 +1098,12 @@ class InferTypesAndCallsAndFieldsVisitor final { const LocalVarData& param_i = fun_ref->parameters[delta_self + i]; AnyExprV arg_i = v->get_arg(i)->get_expr(); TypePtr param_type = param_i.declared_type; - if (param_type->has_genericT_inside() && deducingTs->is_manually_specified()) { // `f(a)` - param_type = deducingTs->replace_by_manually_specified(param_type); + if (param_type->has_genericT_inside()) { + param_type = deducingTs.replace_Ts_with_currently_deduced(param_type); } + flow = infer_any_expr(arg_i, std::move(flow), false, param_type).out_flow; if (param_type->has_genericT_inside()) { // `f(a)` where f is generic: use `a` to infer param type - // then arg_i is inferred without any hint - flow = infer_any_expr(arg_i, std::move(flow), false).out_flow; - param_type = deducingTs->auto_deduce_from_argument(cur_f, arg_i->loc, param_type, arg_i->inferred_type); - } else { - // param_type is hint, helps infer arg_i - flow = infer_any_expr(arg_i, std::move(flow), false, param_type).out_flow; + param_type = deducingTs.auto_deduce_from_argument(cur_f, arg_i->loc, param_type, arg_i->inferred_type); } assign_inferred_type(v->get_arg(i), arg_i); // arg itself is an expression if (param_i.is_mutate_parameter() && arg_i->inferred_type->unwrap_alias() != param_type->unwrap_alias()) { @@ -1040,22 +1119,17 @@ class InferTypesAndCallsAndFieldsVisitor final { if (fun_ref->is_generic_function()) { // if `f(args)` was called, Ts were inferred; check that all of them are known - int idx = deducingTs->get_first_not_deduced_idx(); - if (idx != -1 && hint && fun_ref->declared_return_type->has_genericT_inside()) { + std::string_view nameT_unknown = deducingTs.get_first_not_deduced_nameT(); + if (!nameT_unknown.empty() && hint && fun_ref->declared_return_type && fun_ref->declared_return_type->has_genericT_inside()) { // example: `t.tupleFirst()`, T doesn't depend on arguments, but is determined by return type // if used like `var x: int = t.tupleFirst()` / `t.tupleFirst() as int` / etc., use hint - deducingTs->auto_deduce_from_argument(cur_f, v->loc, fun_ref->declared_return_type, hint); - idx = deducingTs->get_first_not_deduced_idx(); + deducingTs.auto_deduce_from_argument(cur_f, v->loc, fun_ref->declared_return_type, hint); + nameT_unknown = deducingTs.get_first_not_deduced_nameT(); } - if (idx != -1) { - fire(cur_f, v->loc, "can not deduce " + fun_ref->genericTs->get_nameT(idx)); + if (!nameT_unknown.empty()) { + deducingTs.fire_error_can_not_deduce(cur_f, v->get_arg_list()->loc, nameT_unknown); } - fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, deducingTs->flush()); - delete deducingTs; - - } else if (UNLIKELY(v_instantiationTs != nullptr)) { - // non-generic function/method called with type arguments, like `c.cellHash()` / `beginCell()` - fire(cur_f, v_instantiationTs->loc, "calling a not generic function with generic T"); + fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, deducingTs.flush()); } v->mutate()->assign_fun_ref(fun_ref); @@ -1118,7 +1192,26 @@ class InferTypesAndCallsAndFieldsVisitor final { auto v_arm = v->get_arm(i); FlowContext arm_flow = infer_any_expr(v_arm->get_pattern_expr(), arms_entry_facts.clone(), false, nullptr).out_flow; if (s_expr && v_arm->pattern_kind == MatchArmKind::exact_type) { - arm_flow.register_known_type(s_expr, v_arm->pattern_type_node->resolved_type); + TypePtr exact_type = v_arm->pattern_type_node->resolved_type; + if (const auto* t_struct = exact_type->try_as(); t_struct && t_struct->struct_ref->is_generic_struct()) { + // `Wrapper => ...`, detect T based on type of subject (`Wrapper | int` => `Wrapper`) + if (TypePtr inst_exact_type = try_pick_instantiated_generic_from_hint(v->get_subject()->inferred_type, t_struct->struct_ref)) { + exact_type = inst_exact_type; + v_arm->pattern_type_node->mutate()->assign_resolved_type(exact_type); + } else { + fire(cur_f, v_arm->loc, "can not deduce type arguments for " + to_string(t_struct->struct_ref) + ", provide them manually"); + } + } + if (const auto* t_alias = exact_type->try_as(); t_alias && t_alias->alias_ref->is_generic_alias()) { + // `WrapperAlias => ...`, detect T similar to structures + if (TypePtr inst_exact_type = try_pick_instantiated_generic_from_hint(v->get_subject()->inferred_type, t_alias->alias_ref)) { + exact_type = inst_exact_type; + v_arm->pattern_type_node->mutate()->assign_resolved_type(exact_type); + } else { + fire(cur_f, v_arm->loc, "can not deduce type arguments for " + to_string(t_alias->alias_ref) + ", provide them manually"); + } + } + arm_flow.register_known_type(s_expr, exact_type); } arm_flow = infer_any_expr(v_arm->get_body(), std::move(arm_flow), false, hint).out_flow; match_out_flow = i == 0 ? std::move(arm_flow) : FlowContext::merge_flow(std::move(match_out_flow), std::move(arm_flow)); @@ -1147,22 +1240,26 @@ class InferTypesAndCallsAndFieldsVisitor final { } ExprFlow infer_object_literal(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + // the goal is to detect struct_ref + // either by lhs hint `var u: User = { ... }, or by explicitly provided ref `User { ... }` StructPtr struct_ref = nullptr; - if (v->has_explicit_ref()) { - V v_ref = v->get_explicit_ref(); - struct_ref = v_ref->sym->try_as(); - if (!struct_ref) { - if (AliasDefPtr as_alias = v_ref->sym->try_as()) { - if (const TypeDataStruct* as_struct = as_alias->underlying_type->unwrap_alias()->try_as()) { - struct_ref = as_struct->struct_ref; - } - } + + // `User { ... }` / `UserAlias { ... }` / `Wrapper { ... }` / `Wrapper { ... }` + if (v->type_node) { + TypePtr provided_type = v->type_node->resolved_type->unwrap_alias(); + if (const TypeDataStruct* hint_struct = provided_type->try_as()) { + struct_ref = hint_struct->struct_ref; // `Wrapper` / `Wrapper` + } else if (const TypeDataGenericTypeWithTs* hint_instTs = provided_type->try_as()) { + struct_ref = hint_instTs->struct_ref; // if `type WAlias = Wrapper`, here `Wrapper` (generic struct) } if (!struct_ref) { - fire(cur_f, v_ref->loc, "`" + to_string(v_ref->get_name()) + "` does not name a struct"); + fire(cur_f, v->type_node->loc, to_string(v->type_node->resolved_type) + " does not name a struct"); } - if (UNLIKELY(v_ref->has_instantiationTs())) { - fire(cur_f, v_ref->get_instantiationTs()->loc, "generic T not expected here"); + // example: `var v: Ok = Ok { ... }`, now struct_ref is "Ok", take "Ok" from hint + if (struct_ref->is_generic_struct() && hint) { + if (TypePtr inst_explicit_ref = try_pick_instantiated_generic_from_hint(hint, struct_ref)) { + struct_ref = inst_explicit_ref->try_as()->struct_ref; + } } } if (!struct_ref && hint) { @@ -1187,26 +1284,55 @@ class InferTypesAndCallsAndFieldsVisitor final { fire(cur_f, v->loc, "can not detect struct name; use either `var v: StructName = { ... }` or `var v = StructName { ... }`"); } + // so, we have struct_ref, so we can check field names and infer values + // if it's a generic struct, we need to deduce Ts by field values, like for a function call + GenericSubstitutionsDeducing deducingTs(struct_ref); + uint64_t occurred_mask = 0; for (int i = 0; i < v->get_body()->get_num_fields(); ++i) { - auto v_field = v->get_body()->get_field(i); - std::string_view field_name = v_field->get_field_name(); + auto field_i = v->get_body()->get_field(i); + std::string_view field_name = field_i->get_field_name(); StructFieldPtr field_ref = struct_ref->find_field(field_name); if (!field_ref) { - fire(cur_f, v_field->loc, "field `" + to_string(field_name) + "` not found in struct `" + struct_ref->name + "`"); + fire(cur_f, field_i->loc, "field `" + to_string(field_name) + "` not found in struct " + to_string(struct_ref)); } - v_field->mutate()->assign_field_ref(field_ref); + field_i->mutate()->assign_field_ref(field_ref); if (occurred_mask & (1ULL << field_ref->field_idx)) { - fire(cur_f, v_field->loc, "duplicate field initialization"); + fire(cur_f, field_i->loc, "duplicate field initialization"); } occurred_mask |= 1ULL << field_ref->field_idx; - flow = infer_any_expr(v_field->get_init_val(), std::move(flow), false, field_ref->declared_type).out_flow; - assign_inferred_type(v_field, v_field->get_init_val()); + + AnyExprV val_i = field_i->get_init_val(); + TypePtr field_type = field_ref->declared_type; + if (field_type->has_genericT_inside()) { + field_type = deducingTs.replace_Ts_with_currently_deduced(field_type); + } + if (field_type->has_genericT_inside()) { // `item: v` where field `item` is generic: use `v` to infer T + flow = infer_any_expr(val_i, std::move(flow), false).out_flow; + deducingTs.auto_deduce_from_argument(cur_f, field_i->loc, field_type, val_i->inferred_type); + } else { + // field_type is hint, helps infer val_i + flow = infer_any_expr(val_i, std::move(flow), false, field_type).out_flow; + } + assign_inferred_type(field_i, val_i); } for (StructFieldPtr field_ref : struct_ref->fields) { if (!(occurred_mask & (1ULL << field_ref->field_idx)) && !field_ref->has_default_value()) { - fire(cur_f, v->get_body()->loc, "field `" + field_ref->name + "` missed in initialization of struct `" + struct_ref->name + "`"); + fire(cur_f, v->get_body()->loc, "field `" + field_ref->name + "` missed in initialization of struct " + to_string(struct_ref)); + } + } + + // if it's a generic struct `Wrapper`, we need to instantiate it, like `Wrapper` + if (struct_ref->is_generic_struct()) { + if (std::string_view nameT = deducingTs.get_first_not_deduced_nameT(); !nameT.empty()) { + deducingTs.fire_error_can_not_deduce(cur_f, v->loc, nameT); + } + struct_ref = instantiate_generic_struct(struct_ref, deducingTs.flush()); + // re-assign field_ref (it was earlier assigned into a field of a generic struct) + for (int i = 0; i < v->get_body()->get_num_fields(); ++i) { + auto field_i = v->get_body()->get_field(i); + field_i->mutate()->assign_field_ref(struct_ref->find_field(field_i->get_field_name())); } } @@ -1406,13 +1532,13 @@ class InferTypesAndCallsAndFieldsVisitor final { if (has_void_returns && has_non_void_returns) { for (AnyExprV return_expr : return_statements) { if (return_expr->inferred_type == TypeDataVoid::create()) { - fire(fun_ref, return_expr->loc, "mixing void and non-void returns in function `" + fun_ref->as_human_readable() + "`"); + fire(fun_ref, return_expr->loc, "mixing void and non-void returns in function " + to_string(fun_ref)); } } } if (return_unifier.is_union_of_different_types()) { // `return intVar` + `return sliceVar` results in `int | slice`, probably unexpected - fire(fun_ref, v_function->get_body()->loc, "function `" + fun_ref->as_human_readable() + "` calculated return type is " + to_string(inferred_return_type) + "; probably, it's not what you expected; declare `fun (...): ` manually"); + fire(fun_ref, v_function->get_body()->loc, "function " + to_string(fun_ref) + " calculated return type is " + to_string(inferred_return_type) + "; probably, it's not what you expected; declare `fun (...): ` manually"); } } @@ -1514,7 +1640,9 @@ void pipeline_infer_types_and_calls_and_fields() { // infer types for default values in structs for (StructPtr struct_ref : get_all_declared_structs()) { - visitor.start_visiting_struct_fields(struct_ref); + if (!struct_ref->is_generic_struct()) { + visitor.start_visiting_struct_fields(struct_ref); + } } } diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index 7a09defae..29754580b 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -109,45 +109,68 @@ static const GenericsDeclaration* construct_genericTs(V v_li return new GenericsDeclaration(std::move(itemsT)); } -static void register_constant(V v) { +static GlobalConstPtr register_constant(V v) { GlobalConstData* c_sym = new GlobalConstData(static_cast(v->get_identifier()->name), v->loc, v->type_node, v->get_init_value()); G.symtable.add_global_const(c_sym); G.all_constants.push_back(c_sym); v->mutate()->assign_const_ref(c_sym); + return c_sym; } -static void register_global_var(V v) { +static GlobalVarPtr register_global_var(V v) { GlobalVarData* g_sym = new GlobalVarData(static_cast(v->get_identifier()->name), v->loc, v->type_node); G.symtable.add_global_var(g_sym); G.all_global_vars.push_back(g_sym); v->mutate()->assign_glob_ref(g_sym); + return g_sym; } -static void register_type_alias(V v) { - AliasDefData* a_sym = new AliasDefData(static_cast(v->get_identifier()->name), v->loc, v->underlying_type_node); +static AliasDefPtr register_type_alias(V v, AliasDefPtr base_alias_ref = nullptr, const GenericsSubstitutions* substitutedTs = nullptr) { + const GenericsDeclaration* genericTs = nullptr; + if (v->genericsT_list && !substitutedTs) { + genericTs = construct_genericTs(v->genericsT_list); + } + + AliasDefData* a_sym = new AliasDefData(static_cast(v->get_identifier()->name), v->loc, v->underlying_type_node, genericTs, substitutedTs, v); + a_sym->base_alias_ref = base_alias_ref; // for `Response`, here is `Response` G.symtable.add_type_alias(a_sym); v->mutate()->assign_alias_ref(a_sym); + return a_sym; } -static void register_struct(V v) { +static StructPtr register_struct(V v, StructPtr base_struct_ref = nullptr, const GenericsSubstitutions* substitutedTs = nullptr) { auto v_body = v->get_struct_body(); std::vector fields; fields.reserve(v_body->get_num_fields()); for (int i = 0; i < v_body->get_num_fields(); ++i) { auto v_field = v_body->get_field(i); + std::string field_name = static_cast(v_field->get_identifier()->name); AnyExprV default_value = v_field->has_default_value() ? v_field->get_default_value() : nullptr; + + for (StructFieldPtr prev : fields) { + if (UNLIKELY(prev->name == field_name)) { + v_field->error("redeclaration of field `" + field_name + "`"); + } + } fields.emplace_back(new StructFieldData(static_cast(v_field->get_identifier()->name), v_field->loc, i, v_field->type_node, default_value)); } - StructData* s_sym = new StructData(static_cast(v->get_identifier()->name), v->loc, std::move(fields)); + const GenericsDeclaration* genericTs = nullptr; + if (v->genericsT_list && !substitutedTs) { + genericTs = construct_genericTs(v->genericsT_list); + } + + StructData* s_sym = new StructData(static_cast(v->get_identifier()->name), v->loc, std::move(fields), genericTs, substitutedTs, v); + s_sym->base_struct_ref = base_struct_ref; // for `Container`, here is `Container` G.symtable.add_struct(s_sym); G.all_structs.push_back(s_sym); v->mutate()->assign_struct_ref(s_sym); + return s_sym; } static LocalVarData register_parameter(V v, int idx) { @@ -165,7 +188,7 @@ static LocalVarData register_parameter(V v, int idx) { return LocalVarData(static_cast(v->param_name), v->loc, v->type_node, flags, idx); } -static void register_function(V v, const GenericsInstantiation* instantiationTs = nullptr) { +static FunctionPtr register_function(V v, FunctionPtr base_fun_ref = nullptr, const GenericsSubstitutions* substitutedTs = nullptr) { std::string_view func_name = v->get_identifier()->name; std::vector parameters; @@ -184,12 +207,17 @@ static void register_function(V v, const GenericsInsta v->error("`builtin` used for non-builtin function"); } v->mutate()->assign_fun_ref(fun_ref); - return; + return fun_ref; + } + + const GenericsDeclaration* genericTs = nullptr; + if (v->genericsT_list && !substitutedTs) { + genericTs = construct_genericTs(v->genericsT_list); } - const GenericsDeclaration* genericTs = v->genericsT_list ? construct_genericTs(v->genericsT_list) : nullptr; FunctionBody f_body = v->get_body()->kind == ast_block_statement ? static_cast(new FunctionBodyCode) : static_cast(new FunctionBodyAsm); - FunctionData* f_sym = new FunctionData(static_cast(func_name), v->loc, v->return_type_node, std::move(parameters), 0, genericTs, instantiationTs, f_body, v); + FunctionData* f_sym = new FunctionData(static_cast(func_name), v->loc, v->return_type_node, std::move(parameters), 0, genericTs, substitutedTs, f_body, v); + f_sym->base_fun_ref = base_fun_ref; // for `f`, here is `f` if (auto v_asm = v->get_body()->try_as()) { if (!v->return_type_node) { @@ -223,6 +251,7 @@ static void register_function(V v, const GenericsInsta G.all_get_methods.push_back(f_sym); } v->mutate()->assign_fun_ref(f_sym); + return f_sym; } static void iterate_through_file_symbols(const SrcFile* file) { @@ -267,10 +296,19 @@ void pipeline_register_global_symbols() { } } -FunctionPtr pipeline_register_instantiated_generic_function(AnyV cloned_v, const GenericsInstantiation* instantiationTs) { +FunctionPtr pipeline_register_instantiated_generic_function(FunctionPtr base_fun_ref, AnyV cloned_v, const GenericsSubstitutions* substitutedTs) { auto v = cloned_v->as(); - register_function(v, instantiationTs); - return lookup_global_symbol(v->get_identifier()->name)->try_as(); + return register_function(v, base_fun_ref, substitutedTs); +} + +StructPtr pipeline_register_instantiated_generic_struct(StructPtr base_struct_ref, AnyV cloned_v, const GenericsSubstitutions* substitutedTs) { + auto v = cloned_v->as(); + return register_struct(v, base_struct_ref, substitutedTs); +} + +AliasDefPtr pipeline_register_instantiated_generic_alias(AliasDefPtr base_alias_ref, AnyV cloned_v, const GenericsSubstitutions* substitutedTs) { + auto v = cloned_v->as(); + return register_type_alias(v, base_alias_ref, substitutedTs); } } // namespace tolk diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 9f089b064..667172034 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -305,4 +305,8 @@ void pipeline_resolve_identifiers_and_assign_symbols(FunctionPtr fun_ref) { } } +void pipeline_resolve_identifiers_and_assign_symbols(StructPtr struct_ref) { + AssignSymInsideFunctionVisitor().start_visiting_struct_fields(struct_ref); +} + } // namespace tolk diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 3af32fa96..8f805f741 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -34,13 +34,17 @@ namespace tolk { * Example: `fun f(a: cell): (int, User)` param to TypeDataCell, return type to TypeDataTensor(TypeDataInt, TypeDataStruct) * Example: `var x: T = 0` to TypeDataGenericT inside `f` * Example: `f()` to TypeDataAlias inside instantiation list + * Example: `arg: Wrapper` instantiates "Wrapper" right here and returns TypeDataStruct to it * Example: `fun f(): KKK` fires an error "unknown type name" * * Types resolving is done everywhere: inside functions bodies, in struct fields, inside globals declaration, etc. * See finalize_type_node(). * * Note, that resolving T to TypeDataGenericT (and replacing T with substitution when instantiating a generic type) - * is also done here, see genericTs and instantiationTs. + * is also done here, see genericTs and substitutedTs. + * Note, that instantiating generic structs and aliases is also done here (if they don't have generic Ts inside). + * Example: `type OkInt = Ok`, struct "Ok" is instantiated (as a clone of `Ok` substituting T=int) + * Example: `type A = Ok`, then `Ok` is not ready yet, it's left as TypeDataGenericTypeWithTs. */ GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD @@ -59,6 +63,11 @@ static void fire_void_type_not_allowed_inside_union(FunctionPtr cur_f, SrcLocati fire(cur_f, loc, "type `" + disallowed_variant->as_human_readable() + "` is not allowed inside a union"); } +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_generic_type_used_without_T(FunctionPtr cur_f, SrcLocation loc, const std::string& type_name_with_Ts) { + fire(cur_f, loc, "type `" + type_name_with_Ts + "` is generic, you should provide type arguments"); +} + static TypePtr parse_intN(std::string_view strN, bool is_unsigned) { int n; auto result = std::from_chars(strN.data() + 3 + static_cast(is_unsigned), strN.data() + strN.size(), n); @@ -137,31 +146,30 @@ static TypePtr try_parse_predefined_type(std::string_view str) { class TypeNodesVisitorResolver { FunctionPtr cur_f; // exists if we're inside its body const GenericsDeclaration* genericTs; // `` if we're inside `f` or `f` - const GenericsInstantiation* instantiationTs; // `` if we're inside `f` + const GenericsSubstitutions* substitutedTs; // `T=int` if we're inside `f` - TypePtr parse_ast_type_node(AnyTypeV v) { + TypePtr parse_ast_type_node(AnyTypeV v, bool allow_without_type_arguments) { switch (v->kind) { case ast_type_leaf_text: { std::string_view text = v->as()->text; - if (TypePtr predefined_type = try_parse_predefined_type(text)) { - return predefined_type; - } - if (genericTs) { + SrcLocation loc = v->as()->loc; + if (genericTs && genericTs->find_nameT(text) != -1) { // if we're inside `f`, replace "T" with TypeDataGenericT - // if we're inside `f`, replace "T" with TypeDataInt (substitution) - if (int idx = genericTs->find_nameT(text); idx != -1) { - if (instantiationTs) { - return instantiationTs->substitutions[idx]; - } - return TypeDataGenericT::create(static_cast(text)); - } + return TypeDataGenericT::create(static_cast(text)); + } + if (substitutedTs && substitutedTs->has_nameT(text)) { + // if we're inside `f`, replace "T" with TypeDataInt + return substitutedTs->get_substitution_for_nameT(text); } if (const Symbol* sym = lookup_global_symbol(text)) { - if (TypePtr struct_or_other = try_resolve_user_defined_type(sym)) { - return struct_or_other; + if (TypePtr custom_type = try_resolve_user_defined_type(cur_f, loc, sym, allow_without_type_arguments)) { + return custom_type; } } - fire_error_unknown_type_name(cur_f, v->as()->loc, text); + if (TypePtr predefined_type = try_parse_predefined_type(text)) { + return predefined_type; + } + fire_error_unknown_type_name(cur_f, loc, text); } case ast_type_question_nullable: { @@ -202,19 +210,41 @@ class TypeNodesVisitorResolver { return result; } + case ast_type_triangle_args: { + const std::vector& inner_and_args = v->as()->get_inner_and_args(); + TypePtr inner = finalize_type_node(inner_and_args.front(), true); + std::vector type_arguments; + type_arguments.reserve(inner_and_args.size() - 1); + for (size_t i = 1; i < inner_and_args.size(); ++i) { + type_arguments.push_back(finalize_type_node(inner_and_args[i])); + } + return instantiate_generic_type_or_fire(cur_f, v->loc, inner, std::move(type_arguments)); + } + default: - throw UnexpectedASTNodeKind(v, "resolve_ast_type_node"); + throw UnexpectedASTNodeKind(v, "parse_ast_type_node"); } } - static TypePtr try_resolve_user_defined_type(const Symbol* sym) { + // given `dict` / `User` / `Wrapper` / `WrapperAlias`, find it in a symtable + // for generic types, like `Wrapper`, fire that it's used without type arguments (unless allowed) + // example: `var w: Wrapper = ...`, here will be an error of generic usage without T + // example: `w is Wrapper`, here not, it's allowed (instantiated at type inferring later) + // example: `var w: KKK`, nullptr will be returned + static TypePtr try_resolve_user_defined_type(FunctionPtr cur_f, SrcLocation loc, const Symbol* sym, bool allow_without_type_arguments) { if (AliasDefPtr alias_ref = sym->try_as()) { + if (alias_ref->is_generic_alias() && !allow_without_type_arguments) { + fire_error_generic_type_used_without_T(cur_f, loc, alias_ref->as_human_readable()); + } if (!alias_ref->was_visited_by_resolver()) { visit_symbol(alias_ref); } return TypeDataAlias::create(alias_ref); } if (StructPtr struct_ref = sym->try_as()) { + if (struct_ref->is_generic_struct() && !allow_without_type_arguments) { + fire_error_generic_type_used_without_T(cur_f, loc, struct_ref->as_human_readable()); + } if (!struct_ref->was_visited_by_resolver()) { visit_symbol(struct_ref); } @@ -223,6 +253,40 @@ class TypeNodesVisitorResolver { return nullptr; } + // given `Wrapper` / `Pair` / `Response`, instantiate a generic struct/alias + // an error for invalid usage `Pair` / `cell` is also here + static TypePtr instantiate_generic_type_or_fire(FunctionPtr cur_f, SrcLocation loc, TypePtr type_to_instantiate, std::vector&& type_arguments) { + // example: `type WrapperAlias = Wrapper`, we are at `Wrapper`, type_arguments = `` + // they contain generics, so the struct is not ready to be instantiated yet + bool is_still_generic = false; + for (TypePtr argT : type_arguments) { + is_still_generic |= argT->has_genericT_inside(); + } + + if (const TypeDataStruct* t_struct = type_to_instantiate->try_as(); t_struct && t_struct->struct_ref->is_generic_struct()) { + StructPtr struct_ref = t_struct->struct_ref; + if (struct_ref->genericTs->size() != static_cast(type_arguments.size())) { + fire(cur_f, loc, "struct `" + struct_ref->as_human_readable() + "` expects " + std::to_string(struct_ref->genericTs->size()) + " type arguments, but " + std::to_string(type_arguments.size()) + " provided"); + } + if (is_still_generic) { + return TypeDataGenericTypeWithTs::create(struct_ref, nullptr, std::move(type_arguments)); + } + return TypeDataStruct::create(instantiate_generic_struct(struct_ref, GenericsSubstitutions(struct_ref->genericTs, type_arguments))); + } + if (const TypeDataAlias* t_alias = type_to_instantiate->try_as(); t_alias && t_alias->alias_ref->is_generic_alias()) { + AliasDefPtr alias_ref = t_alias->alias_ref; + if (alias_ref->genericTs->size() != static_cast(type_arguments.size())) { + fire(cur_f, loc, "type `" + alias_ref->as_human_readable() + "` expects " + std::to_string(alias_ref->genericTs->size()) + " type arguments, but " + std::to_string(type_arguments.size()) + " provided"); + } + if (is_still_generic) { + return TypeDataGenericTypeWithTs::create(nullptr, alias_ref, std::move(type_arguments)); + } + return TypeDataAlias::create(instantiate_generic_alias(alias_ref, GenericsSubstitutions(alias_ref->genericTs, type_arguments))); + } + // `User` / `cell` + fire(cur_f, loc, "type `" + type_to_instantiate->as_human_readable() + "` is not generic"); + } + static void validate_resulting_union_type(const TypeDataUnion* t_union, FunctionPtr cur_f, SrcLocation loc) { for (TypePtr variant : t_union->variants) { if (variant == TypeDataVoid::create() || variant == TypeDataNever::create()) { @@ -233,16 +297,16 @@ class TypeNodesVisitorResolver { public: - TypeNodesVisitorResolver(FunctionPtr cur_f, const GenericsDeclaration* genericTs, const GenericsInstantiation* instantiationTs) + TypeNodesVisitorResolver(FunctionPtr cur_f, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs) : cur_f(cur_f) , genericTs(genericTs) - , instantiationTs(instantiationTs) {} + , substitutedTs(substitutedTs) {} - TypePtr finalize_type_node(AnyTypeV type_node) { + TypePtr finalize_type_node(AnyTypeV type_node, bool allow_without_type_arguments = false) { #ifdef TOLK_DEBUG tolk_assert(type_node != nullptr); #endif - TypePtr resolved_type = parse_ast_type_node(type_node); + TypePtr resolved_type = parse_ast_type_node(type_node, allow_without_type_arguments); type_node->mutate()->assign_resolved_type(resolved_type); return resolved_type; } @@ -282,7 +346,7 @@ class TypeNodesVisitorResolver { } called_stack.push_back(alias_ref); - TypeNodesVisitorResolver visitor(nullptr, nullptr, nullptr); + TypeNodesVisitorResolver visitor(nullptr, alias_ref->genericTs, alias_ref->substitutedTs); TypePtr underlying_type = visitor.finalize_type_node(alias_ref->underlying_type_node); alias_ref->mutate()->assign_resolved_type(underlying_type); alias_ref->mutate()->assign_visited_by_resolver(); @@ -301,7 +365,7 @@ class TypeNodesVisitorResolver { } called_stack.push_back(struct_ref); - TypeNodesVisitorResolver visitor(nullptr, nullptr, nullptr); + TypeNodesVisitorResolver visitor(nullptr, struct_ref->genericTs, struct_ref->substitutedTs); for (int i = 0; i < struct_ref->get_num_fields(); ++i) { StructFieldPtr field_ref = struct_ref->get_field(i); TypePtr declared_type = visitor.finalize_type_node(field_ref->type_node); @@ -313,10 +377,10 @@ class TypeNodesVisitorResolver { }; class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { - static TypeNodesVisitorResolver type_nodes_visitor; + TypeNodesVisitorResolver type_nodes_visitor{nullptr, nullptr, nullptr}; - static TypePtr finalize_type_node(AnyTypeV type_node) { - return type_nodes_visitor.finalize_type_node(type_node); + TypePtr finalize_type_node(AnyTypeV type_node, bool allow_without_type_arguments = false) { + return type_nodes_visitor.finalize_type_node(type_node, allow_without_type_arguments); } protected: @@ -329,6 +393,8 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { } void visit(V v) override { + tolk_assert(v->sym != nullptr); + // for `f` / `f`, resolve "MyAlias" and "T" // (for function call `f()`, this v (ast_reference `f`) is callee) if (auto v_instantiationTs = v->get_instantiationTs()) { @@ -340,7 +406,9 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { void visit(V v) override { if (v->pattern_type_node) { - finalize_type_node(v->pattern_type_node); + // before `=>` we allow referencing generic types, type inferring will guess + // example: `struct Ok` + `type Response = Ok | ErrCode` + `match (resp) { Ok => ... }` + finalize_type_node(v->pattern_type_node, true); } parent::visit(v->get_pattern_expr()); parent::visit(v->get_body()); @@ -363,17 +431,24 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { } void visit(V v) override { - finalize_type_node(v->type_node); + finalize_type_node(v->type_node, true); parent::visit(v->get_expr()); } + void visit(V v) override { + if (v->type_node) { + finalize_type_node(v->type_node, true); + } + parent::visit(v->get_body()); + } + public: bool should_visit_function(FunctionPtr fun_ref) override { return !fun_ref->is_builtin_function(); } void start_visiting_function(FunctionPtr fun_ref, V v) override { - type_nodes_visitor = TypeNodesVisitorResolver(fun_ref, fun_ref->genericTs, fun_ref->instantiationTs); + type_nodes_visitor = TypeNodesVisitorResolver(fun_ref, fun_ref->genericTs, fun_ref->substitutedTs); for (int i = 0; i < v->get_num_params(); ++i) { const LocalVarData& param_var = fun_ref->parameters[i]; @@ -388,17 +463,19 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { if (fun_ref->is_code_function()) { parent::visit(v->get_body()->as()); } - - type_nodes_visitor = TypeNodesVisitorResolver(nullptr, nullptr, nullptr); } void start_visiting_constant(GlobalConstPtr const_ref) { + type_nodes_visitor = TypeNodesVisitorResolver(nullptr, nullptr, nullptr); + // `const a = 0 as int8`, resolve types there // same for struct field `v: int8 = 0 as int8` parent::visit(const_ref->init_value); } void start_visiting_struct_fields(StructPtr struct_ref) { + type_nodes_visitor = TypeNodesVisitorResolver(nullptr, struct_ref->genericTs, struct_ref->substitutedTs); + // same for struct field `v: int8 = 0 as int8` for (StructFieldPtr field_ref : struct_ref->fields) { if (field_ref->has_default_value()) { @@ -408,8 +485,6 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { } }; -TypeNodesVisitorResolver ResolveTypesInsideFunctionVisitor::type_nodes_visitor(nullptr, nullptr, nullptr); - void pipeline_resolve_types_and_aliases() { ResolveTypesInsideFunctionVisitor visitor; @@ -452,4 +527,13 @@ void pipeline_resolve_types_and_aliases(FunctionPtr fun_ref) { } } +void pipeline_resolve_types_and_aliases(StructPtr struct_ref) { + ResolveTypesInsideFunctionVisitor().start_visiting_struct_fields(struct_ref); + TypeNodesVisitorResolver::visit_symbol(struct_ref); +} + +void pipeline_resolve_types_and_aliases(AliasDefPtr alias_ref) { + TypeNodesVisitorResolver::visit_symbol(alias_ref); +} + } // namespace tolk diff --git a/tolk/pipeline.h b/tolk/pipeline.h index acd0fd87a..c0a51b096 100644 --- a/tolk/pipeline.h +++ b/tolk/pipeline.h @@ -50,12 +50,19 @@ void pipeline_generate_fif_output_to_std_cout(); // these pipes also can be called per-function individually // they are called for instantiated generics functions, when `f` is deeply cloned as `f` -FunctionPtr pipeline_register_instantiated_generic_function(AnyV cloned_v, const GenericsInstantiation* instantiationTs); +FunctionPtr pipeline_register_instantiated_generic_function(FunctionPtr base_fun_ref, AnyV cloned_v, const GenericsSubstitutions* substitutedTs); + void pipeline_resolve_identifiers_and_assign_symbols(FunctionPtr); void pipeline_resolve_types_and_aliases(FunctionPtr); void pipeline_calculate_rvalue_lvalue(FunctionPtr); void pipeline_detect_unreachable_statements(FunctionPtr); void pipeline_infer_types_and_calls_and_fields(FunctionPtr); +StructPtr pipeline_register_instantiated_generic_struct(StructPtr base_struct_ref, AnyV cloned_v, const GenericsSubstitutions* substitutedTs); +void pipeline_resolve_identifiers_and_assign_symbols(StructPtr); +void pipeline_resolve_types_and_aliases(StructPtr); + +AliasDefPtr pipeline_register_instantiated_generic_alias(AliasDefPtr base_alias_ref, AnyV cloned_v, const GenericsSubstitutions* substitutedTs); +void pipeline_resolve_types_and_aliases(AliasDefPtr); } // namespace tolk diff --git a/tolk/smart-casts-cfg.cpp b/tolk/smart-casts-cfg.cpp index dd6a5abbf..31b19558f 100644 --- a/tolk/smart-casts-cfg.cpp +++ b/tolk/smart-casts-cfg.cpp @@ -368,7 +368,7 @@ TypePtr calculate_type_subtract_rhs_type(TypePtr type, TypePtr subtract_type) { if (const TypeDataUnion* sub_union = subtract_type->try_as()) { if (lhs_union->has_all_variants_of(sub_union)) { - rest_variants.reserve(lhs_union->variants.size() - sub_union->variants.size()); + rest_variants.reserve(lhs_union->size() - sub_union->size()); for (TypePtr lhs_variant : lhs_union->variants) { if (!sub_union->has_variant_with_type_id(lhs_variant)) { rest_variants.push_back(lhs_variant); @@ -376,7 +376,7 @@ TypePtr calculate_type_subtract_rhs_type(TypePtr type, TypePtr subtract_type) { } } } else if (lhs_union->has_variant_with_type_id(subtract_type)) { - rest_variants.reserve(lhs_union->variants.size() - 1); + rest_variants.reserve(lhs_union->size() - 1); for (TypePtr lhs_variant : lhs_union->variants) { if (lhs_variant->get_type_id() != subtract_type->get_type_id()) { rest_variants.push_back(lhs_variant); @@ -496,7 +496,7 @@ TypePtr calc_smart_cast_type_on_assignment(TypePtr lhs_declared_type, TypePtr rh for (TypePtr rhs_variant : rhs_union->variants) { lhs_has_all_variants_of_rhs &= lhs_union->has_variant_with_type_id(rhs_variant); } - if (lhs_has_all_variants_of_rhs && rhs_union->variants.size() < lhs_union->variants.size()) { + if (lhs_has_all_variants_of_rhs && rhs_union->size() < lhs_union->size()) { std::vector subtypes_of_lhs; for (TypePtr lhs_variant : lhs_union->variants) { if (rhs_union->has_variant_with_type_id(lhs_variant)) { diff --git a/tolk/smart-casts-cfg.h b/tolk/smart-casts-cfg.h index 31d4ebe0d..1e24a7b69 100644 --- a/tolk/smart-casts-cfg.h +++ b/tolk/smart-casts-cfg.h @@ -18,6 +18,7 @@ #include "fwd-declarations.h" #include "type-system.h" +#include #include #include diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index e2a0a1836..2449de59d 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -28,6 +28,20 @@ std::string FunctionData::as_human_readable() const { return name + genericTs->as_human_readable(); } +std::string AliasDefData::as_human_readable() const { + if (!is_generic_alias()) { + return name; + } + return name + genericTs->as_human_readable(); +} + +std::string StructData::as_human_readable() const { + if (!is_generic_struct()) { + return name; + } + return name + genericTs->as_human_readable(); +} + bool FunctionData::does_need_codegen() const { // when a function is declared, but not referenced from code in any way, don't generate its body if (!is_really_used() && G.settings.remove_unused_functions) { diff --git a/tolk/symtable.h b/tolk/symtable.h index 4252386c0..cde3a8397 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -86,7 +86,6 @@ struct FunctionBodyCode; struct FunctionBodyAsm; struct FunctionBodyBuiltin; struct GenericsDeclaration; -struct GenericsInstantiation; typedef std::variant< FunctionBodyCode*, @@ -124,28 +123,29 @@ struct FunctionData final : Symbol { TypePtr inferred_full_type = nullptr; // assigned on type inferring, it's TypeDataFunCallable(params -> return) const GenericsDeclaration* genericTs; - const GenericsInstantiation* instantiationTs; + const GenericsSubstitutions* substitutedTs; + FunctionPtr base_fun_ref = nullptr; // for `f`, here is `f` FunctionBody body; AnyV ast_root; // V for user-defined (not builtin) - FunctionData(std::string name, SrcLocation loc, AnyTypeV return_type_node, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsInstantiation* instantiationTs, FunctionBody body, AnyV ast_root) + FunctionData(std::string name, SrcLocation loc, AnyTypeV return_type_node, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) : Symbol(std::move(name), loc) , flags(initial_flags) , parameters(std::move(parameters)) , return_type_node(return_type_node) , genericTs(genericTs) - , instantiationTs(instantiationTs) + , substitutedTs(substitutedTs) , body(body) , ast_root(ast_root) { } - FunctionData(std::string name, SrcLocation loc, TypePtr declared_return_type, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsInstantiation* instantiationTs, FunctionBody body, AnyV ast_root) + FunctionData(std::string name, SrcLocation loc, TypePtr declared_return_type, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) : Symbol(std::move(name), loc) , flags(initial_flags) , parameters(std::move(parameters)) , return_type_node(nullptr) // for built-in functions, defined in sources , declared_return_type(declared_return_type) , genericTs(genericTs) - , instantiationTs(instantiationTs) + , substitutedTs(substitutedTs) , body(body) , ast_root(ast_root) { } @@ -166,8 +166,8 @@ struct FunctionData final : Symbol { bool is_asm_function() const { return std::holds_alternative(body); } bool is_builtin_function() const { return ast_root == nullptr; } - bool is_generic_function() const { return genericTs != nullptr && instantiationTs == nullptr; } - bool is_instantiation_of_generic_function() const { return instantiationTs != nullptr; } + bool is_generic_function() const { return genericTs != nullptr; } + bool is_instantiation_of_generic_function() const { return substitutedTs != nullptr; } bool is_inline() const { return flags & flagInline; } bool is_inline_ref() const { return flags & flagInlineRef; } @@ -245,11 +245,24 @@ struct AliasDefData final : Symbol { TypePtr underlying_type = nullptr; // = resolved underlying_type_node int flags = 0; - AliasDefData(std::string name, SrcLocation loc, AnyTypeV underlying_type_node) + const GenericsDeclaration* genericTs; + const GenericsSubstitutions* substitutedTs; + AliasDefPtr base_alias_ref = nullptr; // for `Response`, here is `Response` + AnyV ast_root; // V + + AliasDefData(std::string name, SrcLocation loc, AnyTypeV underlying_type_node, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, AnyV ast_root) : Symbol(std::move(name), loc) - , underlying_type_node(underlying_type_node) { + , underlying_type_node(underlying_type_node) + , genericTs(genericTs) + , substitutedTs(substitutedTs) + , ast_root(ast_root) { } + std::string as_human_readable() const; + + bool is_generic_alias() const { return genericTs != nullptr; } + bool is_instantiation_of_generic_alias() const { return substitutedTs != nullptr; } + bool was_visited_by_resolver() const { return flags & flagVisitedByResolver; } AliasDefData* mutate() const { return const_cast(this); } @@ -285,19 +298,32 @@ struct StructData final : Symbol { std::vector fields; int flags = 0; + const GenericsDeclaration* genericTs; + const GenericsSubstitutions* substitutedTs; + StructPtr base_struct_ref = nullptr; // for `Container`, here is `Container` + AnyV ast_root; // V + int get_num_fields() const { return static_cast(fields.size()); } StructFieldPtr get_field(int i) const { return fields.at(i); } StructFieldPtr find_field(std::string_view field_name) const; + bool is_generic_struct() const { return genericTs != nullptr; } + bool is_instantiation_of_generic_struct() const { return substitutedTs != nullptr; } + bool was_visited_by_resolver() const { return flags & flagVisitedByResolver; } StructData* mutate() const { return const_cast(this); } void assign_visited_by_resolver(); - StructData(std::string name, SrcLocation loc, std::vector&& fields) + StructData(std::string name, SrcLocation loc, std::vector&& fields, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, AnyV ast_root) : Symbol(std::move(name), loc) - , fields(std::move(fields)) { + , fields(std::move(fields)) + , genericTs(genericTs) + , substitutedTs(substitutedTs) + , ast_root(ast_root) { } + + std::string as_human_readable() const; }; class GlobalSymbolTable { diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index 7f2101ca4..39431b4a3 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -15,8 +15,8 @@ along with TON Blockchain Library. If not, see . */ #include "type-system.h" -#include "lexer.h" #include "platform-utils.h" +#include "generics-helpers.h" #include "compiler-state.h" #include #include @@ -81,6 +81,7 @@ class TypeDataHasherForUnique { class TypeIdCalculation { static int last_type_id; static std::unordered_map map_ptr_to_type_id; + static std::vector instantiated_structs; public: static int assign_type_id(TypePtr self) { @@ -91,6 +92,21 @@ class TypeIdCalculation { return it->second; } + // make `Wrapper>` and `Wrapper>` and `Wrapper` have equal type_id + if (const TypeDataStruct* t_struct = self->try_as(); t_struct && t_struct->struct_ref->is_instantiation_of_generic_struct()) { + StructPtr struct_ref = t_struct->struct_ref; + instantiated_structs.push_back(struct_ref); + for (StructPtr another_ref : instantiated_structs) { + if (struct_ref != another_ref && self->equal_to(TypeDataStruct::create(another_ref))) { + auto it = map_ptr_to_type_id.find(TypeDataStruct::create(another_ref)); + assert(it != map_ptr_to_type_id.end()); + int type_id = it->second; + map_ptr_to_type_id[self] = type_id; + return type_id; + } + } + } + int type_id = ++last_type_id; map_ptr_to_type_id[self] = type_id; return type_id; @@ -110,6 +126,7 @@ class TypeIdCalculation { int TypeIdCalculation::last_type_id = 128; // below 128 reserved for built-in types std::unordered_map TypeDataHasherForUnique::all_unique_occurred_types; std::unordered_map TypeIdCalculation::map_ptr_to_type_id; +std::vector TypeIdCalculation::instantiated_structs; TypePtr TypeDataInt::singleton; TypePtr TypeDataBool::singleton; @@ -156,6 +173,12 @@ bool TypeData::equal_to_slow_path(TypePtr lhs, TypePtr rhs) { return lhs_union->variants.size() == rhs_union->variants.size() && lhs_union->has_all_variants_of(rhs_union); } } + if (const TypeDataStruct* lhs_struct = lhs->try_as(); lhs_struct && lhs_struct->struct_ref->is_instantiation_of_generic_struct()) { + if (const TypeDataStruct* rhs_struct = rhs->try_as(); rhs_struct && rhs_struct->struct_ref->is_instantiation_of_generic_struct()) { + return lhs_struct->struct_ref->base_struct_ref == rhs_struct->struct_ref->base_struct_ref + && lhs_struct->struct_ref->substitutedTs->equal_to(rhs_struct->struct_ref->substitutedTs); + } + } return false; } @@ -218,6 +241,22 @@ TypePtr TypeDataGenericT::create(std::string&& nameT) { return hash.register_unique(new TypeDataGenericT(std::move(nameT))); } +TypePtr TypeDataGenericTypeWithTs::create(StructPtr struct_ref, AliasDefPtr alias_ref, std::vector&& type_arguments) { + TypeDataHasherForUnique hash(7451094818554079348ULL); // use it only to calculate children_flags + if (struct_ref) { + assert(alias_ref == nullptr && struct_ref->is_generic_struct()); + hash.feed_string(struct_ref->name); + } else { + assert(struct_ref == nullptr && alias_ref->is_generic_alias()); + hash.feed_string(alias_ref->name); + } + for (TypePtr argT : type_arguments) { + hash.feed_child(argT); + } + + return new TypeDataGenericTypeWithTs(hash.children_flags(), struct_ref, alias_ref, std::move(type_arguments)); +} + TypePtr TypeDataStruct::create(StructPtr struct_ref) { TypeDataHasherForUnique hash(8315986401043319583ULL); hash.feed_string(struct_ref->name); @@ -225,7 +264,7 @@ TypePtr TypeDataStruct::create(StructPtr struct_ref) { if (TypePtr existing = hash.get_existing()) { return existing; } - int width_on_stack = 0; + int width_on_stack = 0; // if a struct is generic, it will be incorrect, it's okay for (StructFieldPtr field_ref : struct_ref->fields) { width_on_stack += field_ref->declared_type->get_width_on_stack(); } @@ -377,6 +416,7 @@ TypePtr TypeDataUnion::create_nullable(TypePtr nullable) { // int TypeDataAlias::get_type_id() const { + assert(!alias_ref->is_generic_alias()); return underlying_type->get_type_id(); } @@ -389,7 +429,13 @@ int TypeDataGenericT::get_type_id() const { throw Fatal("unexpected get_type_id() call"); } +int TypeDataGenericTypeWithTs::get_type_id() const { + assert(false); // `Wrapper` has to be resolved in advance + throw Fatal("unexpected get_type_id() call"); +} + int TypeDataStruct::get_type_id() const { + assert(!struct_ref->is_generic_struct()); return TypeIdCalculation::assign_type_id(this); } @@ -454,6 +500,19 @@ std::string TypeDataFunCallable::as_human_readable() const { return result; } +std::string TypeDataGenericTypeWithTs::as_human_readable() const { + std::string result = struct_ref ? struct_ref->name : alias_ref->name; + result += '<'; + for (TypePtr argT : type_arguments) { + if (result[result.size() - 1] != '<') { + result += ", "; + } + result += argT->as_human_readable(); + } + result += '>'; + return result; +} + std::string TypeDataStruct::as_human_readable() const { return struct_ref->name; } @@ -536,6 +595,15 @@ TypePtr TypeDataFunCallable::replace_children_custom(const ReplacerCallbackT& ca return callback(create(std::move(mapped), return_type->replace_children_custom(callback))); } +TypePtr TypeDataGenericTypeWithTs::replace_children_custom(const ReplacerCallbackT& callback) const { + std::vector mapped; + mapped.reserve(type_arguments.size()); + for (TypePtr argT : type_arguments) { + mapped.push_back(argT->replace_children_custom(callback)); + } + return callback(create(struct_ref, alias_ref, std::move(mapped))); +} + TypePtr TypeDataTensor::replace_children_custom(const ReplacerCallbackT& callback) const { std::vector mapped; mapped.reserve(items.size()); @@ -692,7 +760,10 @@ bool TypeDataFunCallable::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataGenericT::can_rhs_be_assigned(TypePtr rhs) const { - assert(false); + return false; +} + +bool TypeDataGenericTypeWithTs::can_rhs_be_assigned(TypePtr rhs) const { return false; } @@ -703,6 +774,9 @@ bool TypeDataStruct::can_rhs_be_assigned(TypePtr rhs) const { if (const TypeDataAlias* rhs_alias = rhs->try_as()) { return can_rhs_be_assigned(rhs_alias->underlying_type); } + if (const TypeDataStruct* rhs_struct = rhs->try_as()) { // C> = C + return equal_to(rhs_struct); + } return rhs == TypeDataNever::create(); } @@ -956,6 +1030,10 @@ bool TypeDataGenericT::can_be_casted_with_as_operator(TypePtr cast_to) const { return true; } +bool TypeDataGenericTypeWithTs::can_be_casted_with_as_operator(TypePtr cast_to) const { + return true; +} + bool TypeDataStruct::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataUnion* to_union = cast_to->try_as()) { return can_be_casted_to_union(this, to_union); @@ -963,6 +1041,9 @@ bool TypeDataStruct::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } + if (const TypeDataStruct* to_struct = cast_to->try_as()) { // C> as C + return equal_to(to_struct); + } return cast_to == this; } diff --git a/tolk/type-system.h b/tolk/type-system.h index cd0c34581..40b291015 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -103,6 +103,8 @@ class TypeData { * It never occurs at runtime: at IR generation it's erased, replaced by an underlying type. * But until IR generation, aliases exists, and `var t: MyTensor2 = (1,2)` is alias "MyTensor", not tensor (int,int). * That's why lots of code comparing types use `type->unwrap_alias()` or `try_as`. + * Note, that generic aliases, when instantiated, are inserted into in symtable (like structs and functions), + * so for `WrapperAlias` alias_ref points to a generic alias, and for `WrapperAlias` to an instantiated one. */ class TypeDataAlias final : public TypeData { explicit TypeDataAlias(int children_flags, AliasDefPtr alias_ref, TypePtr underlying_type) @@ -301,8 +303,9 @@ class TypeDataFunCallable final : public TypeData { }; /* - * `T` inside generic functions is TypeDataGenericT. + * `T` inside generic functions and structs is TypeDataGenericT. * Example: `fun f(a: X, b: Y): [X, Y]` (here X and Y are). + * Example: `struct Wrapper { value: T }` (type of field is generic T). * On instantiation like `f(1,"")`, a new function `f` is created with type `fun(int,slice)->[int,slice]`. */ class TypeDataGenericT final : public TypeData { @@ -322,9 +325,40 @@ class TypeDataGenericT final : public TypeData { }; /* - * `A`, `User`, `SomeStruct` is TypeDataStruct. At TVM level, structs are tensors. + * `Wrapper` when T is a generic (a struct is not ready to instantiate) is TypeDataGenericTypeWithTs. + * `Wrapper` is NOT here, it's an instantiated struct. Here is only when type arguments contain generics. + * Example: `type WrapperAlias = Wrapper`, then `Wrapper` (underlying type of alias) is here. + * Since structs and type aliases both can be generic, either struct_ref of alias_ref is filled. + */ +class TypeDataGenericTypeWithTs final : public TypeData { + TypeDataGenericTypeWithTs(int children_flags, StructPtr struct_ref, AliasDefPtr alias_ref, std::vector&& type_arguments) + : TypeData(children_flags, -999999) + , struct_ref(struct_ref) + , alias_ref(alias_ref) + , type_arguments(std::move(type_arguments)) {} + +public: + const StructPtr struct_ref; // for `Wrapper`, then alias_ref = nullptr + const AliasDefPtr alias_ref; // for `PairAlias`, then struct_ref = nullptr + const std::vector type_arguments; // ``, ``, at least one of them contains generic T + + static TypePtr create(StructPtr struct_ref, AliasDefPtr alias_ref, std::vector&& type_arguments); + + int size() const { return static_cast(type_arguments.size()); } + + int get_type_id() const override; + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; +}; + +/* + * `A`, `User`, `SomeStruct`, `Wrapper` is TypeDataStruct. At TVM level, structs are tensors. * In the code, creating a struct is either `var v: A = { ... }` (by hint) or `var v = A { ... }`. * Fields of a struct have their own types (accessed by struct_ref). + * Note, that instantiated structs like "Wrapper" exist in symtable (like aliases and functions), + * so for `Wrapper` struct_ref points to a generic struct, and for `Wrapper` to an instantiated one. */ class TypeDataStruct final : public TypeData { TypeDataStruct(int width_on_stack, StructPtr struct_ref) @@ -487,6 +521,8 @@ class TypeDataUnion final : public TypeData { static TypePtr create(std::vector&& variants); static TypePtr create_nullable(TypePtr nullable); + int size() const { return static_cast(variants.size()); } + // "primitive nullable" is `T?` which holds TVM NULL in the same slot (it other words, has no UTag slot) // true : `int?`, `slice?`, `StructWith1IntField?` // false: `(int, int)?`, `ComplexStruct?`, `()?` From 720a173d0e8afdd8f919de265998d7394dad87b0 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Mon, 28 Apr 2025 20:35:22 +0400 Subject: [PATCH 233/388] [Tolk] Output original .tolk lines into Fift as comments This update greatly enhances reading Fift output: 1) stack comments are aligned 2) original lines from .tolk files are output as comments, grouping asm instructions Internally, every AsmOp now has SrcLocation. When outputting them one by one, an original .tolk line is inserted if locations differ. This can be optimized later by storing an index to fast mapping of location (offset) into a line in a file. --- .../tests/allow-post-modification.tolk | 8 +- tolk-tester/tests/assignment-tests.tolk | 19 +- tolk-tester/tests/bit-operators.tolk | 35 +- tolk-tester/tests/cells-slices.tolk | 1 - tolk-tester/tests/codegen-check-demo.tolk | 4 +- tolk-tester/tests/comments-tests.tolk | 42 ++ tolk-tester/tests/generics-2.tolk | 1 - tolk-tester/tests/generics-3.tolk | 1 - tolk-tester/tests/if-else-tests.tolk | 14 +- tolk-tester/tests/indexed-access.tolk | 8 +- tolk-tester/tests/intN-tests.tolk | 1 - .../tests/invalid-declaration/err-1918.tolk | 2 +- tolk-tester/tests/logical-operators.tolk | 19 +- tolk-tester/tests/match-by-expr-tests.tolk | 10 +- tolk-tester/tests/mutate-methods.tolk | 8 +- tolk-tester/tests/no-spaces.tolk | 3 - tolk-tester/tests/null-keyword.tolk | 3 +- tolk-tester/tests/nullable-tensors.tolk | 6 +- tolk-tester/tests/op-priority.tolk | 21 +- tolk-tester/tests/self-keyword.tolk | 19 +- tolk-tester/tests/smart-cast-tests.tolk | 1 - tolk-tester/tests/some-tests-2.tolk | 3 - tolk-tester/tests/struct-tests.tolk | 8 +- tolk-tester/tests/try-catch-tests.tolk | 9 +- tolk-tester/tests/type-aliases-tests.tolk | 7 +- tolk-tester/tests/union-types-tests.tolk | 9 +- tolk-tester/tests/use-before-declare.tolk | 2 - tolk-tester/tolk-tester.js | 11 +- tolk-tester/tolk-tester.py | 5 + tolk/abscode.cpp | 3 - tolk/analyzer.cpp | 20 +- tolk/asmops.cpp | 209 +++++---- tolk/ast-aux-data.h | 38 ++ tolk/ast-from-tokens.cpp | 2 +- tolk/ast-replacer.h | 2 + tolk/ast-replicator.h | 4 + tolk/ast-stringifier.h | 2 + tolk/ast-visitor.h | 2 + tolk/ast.h | 20 + tolk/builtins.cpp | 416 +++++++++--------- tolk/codegen.cpp | 277 ++++++------ tolk/compiler-state.h | 1 + tolk/optimize.cpp | 72 ++- tolk/pipe-ast-to-legacy.cpp | 31 +- tolk/pipe-generate-fif-output.cpp | 24 +- tolk/pipe-resolve-identifiers.cpp | 2 +- tolk/src-file.cpp | 55 ++- tolk/src-file.h | 4 +- tolk/tolk-main.cpp | 6 +- tolk/tolk-wasm.cpp | 2 + tolk/tolk.h | 305 ++++++------- 51 files changed, 944 insertions(+), 833 deletions(-) create mode 100644 tolk-tester/tests/comments-tests.tolk create mode 100644 tolk/ast-aux-data.h diff --git a/tolk-tester/tests/allow-post-modification.tolk b/tolk-tester/tests/allow-post-modification.tolk index 10c243740..49a2eaec5 100644 --- a/tolk-tester/tests/allow-post-modification.tolk +++ b/tolk-tester/tests/allow-post-modification.tolk @@ -146,16 +146,14 @@ fun main() { @fif_codegen """ - ~inc PROC:<{ - // self y - inc CALLDICT // self newY + ~inc PROC:<{ // self y + inc CALLDICT // self newY }> """ @fif_codegen """ - test_assign_tensor_global PROC:<{ - // x.0 x.1 + test_assign_tensor_global PROC:<{ // x.0 x.1 """ @code_hash 6737917279814799680932710799951154408447028229503449454536845752635763933556 diff --git a/tolk-tester/tests/assignment-tests.tolk b/tolk-tester/tests/assignment-tests.tolk index f4d2c0d78..2301c9b70 100644 --- a/tolk-tester/tests/assignment-tests.tolk +++ b/tolk-tester/tests/assignment-tests.tolk @@ -245,16 +245,15 @@ fun main(value: int, ) { @fif_codegen """ - test116 PROC:<{ - // - 1 PUSHINT // '10=1 - 2 PUSHINT // '10=1 '11=2 - 3 PUSHINT // '10=1 '11=2 '12=3 - 4 PUSHINT // '10=1 '11=2 '12=3 '13=4 - 4 TUPLE // rhs - DUP // rhs rhs - 4 UNTUPLE // rhs2 a b c d - 4 ROLL // a b c d rhs2 + test116 PROC:<{ // + 1 PUSHINT // '10=1 + 2 PUSHINT // '10=1 '11=2 + 3 PUSHINT // '10=1 '11=2 '12=3 + 4 PUSHINT // '10=1 '11=2 '12=3 '13=4 + 4 TUPLE // rhs + DUP // rhs rhs + 4 UNTUPLE // rhs2 a b c d + 4 ROLL // a b c d rhs2 }> """ */ diff --git a/tolk-tester/tests/bit-operators.tolk b/tolk-tester/tests/bit-operators.tolk index b01068839..9ae04090c 100644 --- a/tolk-tester/tests/bit-operators.tolk +++ b/tolk-tester/tests/bit-operators.tolk @@ -126,18 +126,16 @@ fun testBoolCompareOptimized(x: bool) { @fif_codegen """ boolWithBitwiseConst PROC:<{ - // - 0 PUSHINT // '3 - -1 PUSHINT // '3 '5 - 0 PUSHINT // '3 '5 '7 - -1 PUSHINT // '3 '5 '7 '8 + 0 PUSHINT // '3 + -1 PUSHINT // '3 '5 + 0 PUSHINT // '3 '5 '7 + -1 PUSHINT // '3 '5 '7 '8 }> """ @fif_codegen """ - testDoUntilCodegen PROC:<{ - // i n + testDoUntilCodegen PROC:<{ // i n 0 PUSHINT // i n cnt=0 UNTIL:<{ INC // i n cnt @@ -164,7 +162,6 @@ fun testBoolCompareOptimized(x: bool) { @fif_codegen """ testConstNegateCodegen PROC:<{ - // TRUE // '0 FALSE // '0 '1 FALSE // '0 '1 '2 @@ -176,26 +173,24 @@ fun testBoolCompareOptimized(x: bool) { @fif_codegen """ - testBoolNegateOptimized PROC:<{ - // x - DUP // x x - NOT // x '1 - OVER // x '1 x - NOT // x '1 '2 + testBoolNegateOptimized PROC:<{ // x + DUP // x x + NOT // x '1 + OVER // x '1 x + NOT // x '1 '2 s2 s(-1) PUXC - TRUE // x '1 x '2 '3 + TRUE // x '1 x '2 '3 }> """ @fif_codegen """ - testBoolCompareOptimized PROC:<{ - // x - DUP // x x - NOT // x '1 + testBoolCompareOptimized PROC:<{ // x + DUP // x x + NOT // x '1 OVER // x '1 x eqX CALLDICT // x '1 '2 - NOT // x '1 '3 + NOT // x '1 '3 s2 PUSH // x '1 '3 x eqX CALLDICT // x '1 '3 '4 s3 PUSH // x '1 '3 '4 x diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index 772812eb1..e900268bb 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -216,7 +216,6 @@ Note, that since 'compute-asm-ltr' became on be default, chaining methods codege @fif_codegen """ test6 PROC:<{ - // NEWC // '0 1 PUSHINT // '0 '1=1 SWAP // '1=1 '0 diff --git a/tolk-tester/tests/codegen-check-demo.tolk b/tolk-tester/tests/codegen-check-demo.tolk index 5b46c0935..ff2ca3c2c 100644 --- a/tolk-tester/tests/codegen-check-demo.tolk +++ b/tolk-tester/tests/codegen-check-demo.tolk @@ -33,8 +33,7 @@ Below, I just give examples of @fif_codegen tag: @fif_codegen """ -main PROC:<{ - // s +main PROC:<{ // s 17 PUSHINT // s '3=17 OVER // s z=17 t WHILE:<{ @@ -78,7 +77,6 @@ main PROC:<{ @fif_codegen """ test1 PROC:<{ -// FALSE }> """ diff --git a/tolk-tester/tests/comments-tests.tolk b/tolk-tester/tests/comments-tests.tolk new file mode 100644 index 000000000..3d00a0b1a --- /dev/null +++ b/tolk-tester/tests/comments-tests.tolk @@ -0,0 +1,42 @@ + +struct WithDef { + f1: int, + f2: int = 10, + f3: int, + f4: int, +} + +const C_20 = 20; + +fun demo_fields_def(x: int) { + var w: WithDef = { + f1: x + C_20, + f3: 100, + f4: C_20 + }; + return (w.f4 + 5, w.f3, w.f2, w.f1); +} + +fun main() { + return 1; +} + +/** +@testcase | 0 | | 1 + +@fif_codegen_enable_comments +@fif_codegen +""" + demo_fields_def PROC:<{ // x + // 13: f1: x + C_20 + 20 ADDCONST // '6 + 10 PUSHINT // '6 '7=10 + // 14: f3: 100 + 100 PUSHINT // w.f1 w.f2=10 w.f3=100 + // 17: return (w.f4 + 5, w.f3, w.f2, w.f1) + 25 PUSHINT // w.f1 w.f2=10 w.f3=100 '11 + s2 s3 XCHG2 // '11 w.f3=100 w.f2=10 w.f1 + }> +""" + +*/ \ No newline at end of file diff --git a/tolk-tester/tests/generics-2.tolk b/tolk-tester/tests/generics-2.tolk index cde9d9929..b3e0ea5b9 100644 --- a/tolk-tester/tests/generics-2.tolk +++ b/tolk-tester/tests/generics-2.tolk @@ -453,7 +453,6 @@ fun main(c: Wrapper, d: WrappedInt) { @fif_codegen """ test1 PROC:<{ - // 23 PUSHINT // c1=23 DUP // c1=23 c2=23 }> diff --git a/tolk-tester/tests/generics-3.tolk b/tolk-tester/tests/generics-3.tolk index 4ec7b29b4..77d046113 100644 --- a/tolk-tester/tests/generics-3.tolk +++ b/tolk-tester/tests/generics-3.tolk @@ -137,7 +137,6 @@ fun main() { @fif_codegen """ test3 PROC:<{ - // 10 PUSHINT // r.USlot1=10 -1 PUSHINT // r.USlot1=10 '11=-1 FALSE // r.USlot1=10 '11=-1 '13 diff --git a/tolk-tester/tests/if-else-tests.tolk b/tolk-tester/tests/if-else-tests.tolk index f40336a9e..8c314b8ed 100644 --- a/tolk-tester/tests/if-else-tests.tolk +++ b/tolk-tester/tests/if-else-tests.tolk @@ -128,15 +128,14 @@ fun main() { @fif_codegen """ - test3 PROC:<{ - // x - DUP // x x + test3 PROC:<{ // x + DUP // x x 20 NEQINT // x '2 IFNOTJMP:<{ // x - DROP // + DROP // 20 PUSHINT // '3=20 - }> // x - DUP // x x + }> // x + DUP // x x 50 EQINT // x '5 IFNOTJMP:<{ // x """ @@ -144,7 +143,6 @@ fun main() { @fif_codegen """ withNoElse PROC:<{ - // op DUP 123 EQINT IFJMP:<{ @@ -170,7 +168,6 @@ fun main() { @fif_codegen """ withElse PROC:<{ - // op DUP 123 EQINT IFJMP:<{ @@ -196,7 +193,6 @@ fun main() { @fif_codegen """ withMatch PROC:<{ - // op DUP 123 EQINT IFJMP:<{ diff --git a/tolk-tester/tests/indexed-access.tolk b/tolk-tester/tests/indexed-access.tolk index 4b3a0a4b2..f6a376ad3 100644 --- a/tolk-tester/tests/indexed-access.tolk +++ b/tolk-tester/tests/indexed-access.tolk @@ -286,7 +286,6 @@ fun main(){} @fif_codegen """ testCodegenSimple PROC:<{ - // 1 PUSHINT // '2=1 SINGLE // t1 2 PUSHINT // t1 '3=2 @@ -336,15 +335,13 @@ fun main(){} @fif_codegen """ testCodegenNoPureIndexedAccess PROC:<{ - // 0 PUSHINT // '8=0 }> """ @fif_codegen """ - testCodegenIndexPostfix1 PROC:<{ - // x.0 x.1 + testCodegenIndexPostfix1 PROC:<{ // x.0 x.1 // ab.1 ab.0 SWAP // ab.0 ab.1 }> @@ -352,8 +349,7 @@ fun main(){} @fif_codegen """ - testCodegenIndexPostfix2 PROC:<{ - // x.0 x.1.0 x.1.1 x.2 + testCodegenIndexPostfix2 PROC:<{ // x.0 x.1.0 x.1.1 x.2 s2 POP // y.0 y.2 y.1.1 s1 s2 XCHG // y.2 y.0 y.1.1 }> diff --git a/tolk-tester/tests/intN-tests.tolk b/tolk-tester/tests/intN-tests.tolk index d12436f5e..4b2fccb25 100644 --- a/tolk-tester/tests/intN-tests.tolk +++ b/tolk-tester/tests/intN-tests.tolk @@ -254,7 +254,6 @@ fun main() { @fif_codegen """ test12 PROC:<{ - // 100000000 PUSHINT // '4=100000000 }> """ diff --git a/tolk-tester/tests/invalid-declaration/err-1918.tolk b/tolk-tester/tests/invalid-declaration/err-1918.tolk index ae801de26..8fec98925 100644 --- a/tolk-tester/tests/invalid-declaration/err-1918.tolk +++ b/tolk-tester/tests/invalid-declaration/err-1918.tolk @@ -8,5 +8,5 @@ fun hello(): int { @compilation_should_fail @stderr fun hello() @stderr redefinition of symbol, previous was at -@stderr err-1918.tolk:1:1 +@stderr err-1918.tolk:1 */ diff --git a/tolk-tester/tests/logical-operators.tolk b/tolk-tester/tests/logical-operators.tolk index 7e3168c7c..2fd62db5b 100644 --- a/tolk-tester/tests/logical-operators.tolk +++ b/tolk-tester/tests/logical-operators.tolk @@ -218,7 +218,6 @@ fun main() { @fif_codegen """ simpleAllConst PROC:<{ - // TRUE 0 PUSHINT TRUE @@ -233,8 +232,7 @@ fun main() { @fif_codegen """ - compileTimeEval1 PROC:<{ - // x + compileTimeEval1 PROC:<{ // x DUP // x x 0 EQINT // x '1 FALSE // x '1 '4 @@ -250,9 +248,9 @@ fun main() { @fif_codegen """ - withIfNot PROC:<{ + withIfNot PROC:<{ // x y c2 SAVE - SAMEALTSAVE // x y + SAMEALTSAVE OVER // x y x IFNOTJMP:<{ // x y 2DROP // @@ -271,7 +269,6 @@ fun main() { @fif_codegen """ testAndConstCodegen PROC:<{ - // FALSE 0 PUSHINT DUP @@ -294,7 +291,6 @@ fun main() { @fif_codegen """ testOrConstCodegen PROC:<{ - // -1 PUSHINT TRUE FALSE @@ -319,8 +315,7 @@ For example, `a && b` can be expressed without IFs. These are moments of future optimizations. For now, it's more than enough. @fif_codegen """ - testAndSimpleCodegen PROC:<{ - // a b + testAndSimpleCodegen PROC:<{ // a b SWAP // b a IF:<{ // b 0 NEQINT // '2 @@ -333,8 +328,7 @@ These are moments of future optimizations. For now, it's more than enough. @fif_codegen """ - testOrSimpleCodegen PROC:<{ - // a b + testOrSimpleCodegen PROC:<{ // a b SWAP // b a 0 GTINT // b '3 IF:<{ // b @@ -349,8 +343,7 @@ These are moments of future optimizations. For now, it's more than enough. @fif_codegen """ - testConvertIfToIfnot PROC:<{ - // x + testConvertIfToIfnot PROC:<{ // x DUP // x x 100 THROWIF DUP // x x diff --git a/tolk-tester/tests/match-by-expr-tests.tolk b/tolk-tester/tests/match-by-expr-tests.tolk index 7941b4600..559919353 100644 --- a/tolk-tester/tests/match-by-expr-tests.tolk +++ b/tolk-tester/tests/match-by-expr-tests.tolk @@ -157,8 +157,7 @@ fun main() { @fif_codegen """ - test1 PROC:<{ - // x + test1 PROC:<{ // x DUP // x x 1 EQINT // x '3 IF:<{ // x @@ -185,15 +184,13 @@ fun main() { @fif_codegen """ test2 PROC:<{ - // 300 PUSHINT }> """ @fif_codegen """ - test3 PROC:<{ - // init + test3 PROC:<{ // init DUP // init init isGt10 CALLDICT // init '2 -1 EQINT // init '5 @@ -206,8 +203,7 @@ fun main() { @fif_codegen """ - test8 PROC:<{ - // x + test8 PROC:<{ // x DROP // 30 PUSHINT // r1 }> diff --git a/tolk-tester/tests/mutate-methods.tolk b/tolk-tester/tests/mutate-methods.tolk index 9ebf8b1d4..59da68484 100644 --- a/tolk-tester/tests/mutate-methods.tolk +++ b/tolk-tester/tests/mutate-methods.tolk @@ -295,8 +295,7 @@ fun main(){} @fif_codegen """ - incrementInPlace PROC:<{ - // self byValue + incrementInPlace PROC:<{ // self byValue ADD // self }> """ @@ -315,8 +314,7 @@ fun main(){} @fif_codegen """ - load_next PROC:<{ - // cs + load_next PROC:<{ // cs 32 LDI // '4 cs SWAP // cs '4 }> @@ -325,7 +323,6 @@ fun main(){} @fif_codegen """ testStoreUintPureUnusedResult PROC:<{ - // 0 PUSHINT // '11=0 }> """ @@ -333,7 +330,6 @@ fun main(){} @fif_codegen """ testStoreUintImpureUnusedResult PROC:<{ - // NEWC // b STIX // '2 DROP // diff --git a/tolk-tester/tests/no-spaces.tolk b/tolk-tester/tests/no-spaces.tolk index 3fe2573a9..33959917e 100644 --- a/tolk-tester/tests/no-spaces.tolk +++ b/tolk-tester/tests/no-spaces.tolk @@ -80,7 +80,6 @@ fun`main`(){} @fif_codegen """ get_+-+1 PROC:<{ - // -1 PUSHINT }> """ @@ -88,7 +87,6 @@ fun`main`(){} @fif_codegen """ unary+bitwise-constant PROC:<{ - // -4 PUSHINT 6 PUSHINT -4 PUSHINT @@ -99,7 +97,6 @@ fun`main`(){} @fif_codegen """ unary_const_check PROC:<{ - // -1 PUSHINT // fst1=-1 DUP // fst1=-1 snd1=-1 2 PUSHINT // fst1=-1 snd1=-1 trd1=2 diff --git a/tolk-tester/tests/null-keyword.tolk b/tolk-tester/tests/null-keyword.tolk index 65890a92c..db54973e9 100644 --- a/tolk-tester/tests/null-keyword.tolk +++ b/tolk-tester/tests/null-keyword.tolk @@ -89,10 +89,10 @@ fun main() { @testcase | 103 | 15 | -1 @testcase | 104 | | (null) @testcase | 107 | | -11 + @fif_codegen """ test1 PROC:<{ - // PUSHNULL // numbers 1 PUSHINT // numbers '2=1 SWAP // '2=1 numbers @@ -114,7 +114,6 @@ fun main() { @fif_codegen """ main PROC:<{ - // 1 PUSHINT // '3=1 }> """ diff --git a/tolk-tester/tests/nullable-tensors.tolk b/tolk-tester/tests/nullable-tensors.tolk index 41b3d62b8..af051ce93 100644 --- a/tolk-tester/tests/nullable-tensors.tolk +++ b/tolk-tester/tests/nullable-tensors.tolk @@ -530,9 +530,8 @@ fun main(){} @fif_codegen """ - isTensorNull PROC:<{ - // t.0 t.1 t.UTag - 2 1 BLKDROP2 // t.UTag + isTensorNull PROC:<{ // t.0 t.1 t.UTag + 2 1 BLKDROP2 // t.UTag 0 EQINT // '3 }> """ @@ -540,7 +539,6 @@ fun main(){} @fif_codegen """ test113 PROC:<{ - // 1 PUSHINT // '2=1 PUSHNULL // '2=1 '3 PAIR // t diff --git a/tolk-tester/tests/op-priority.tolk b/tolk-tester/tests/op-priority.tolk index e3c6bb6e8..3a1dee21e 100644 --- a/tolk-tester/tests/op-priority.tolk +++ b/tolk-tester/tests/op-priority.tolk @@ -92,28 +92,25 @@ fun main() { @fif_codegen """ - unary_minus_1 PROC:<{ - // a b c + unary_minus_1 PROC:<{ // a b c -ROT // c a b - ADD // c '3 + ADD // c '3 NEGATE // c '4 SWAP // '4 c - MUL // '5 + MUL // '5 }> - unary_minus_2 PROC:<{ - // a b c + unary_minus_2 PROC:<{ // a b c -ROT // c a b - ADD // c '3 + ADD // c '3 NEGATE // c '4 SWAP // '4 c - MUL // '5 + MUL // '5 }> - unary_minus_3 PROC:<{ - // a b c + unary_minus_3 PROC:<{ // a b c -ROT // c a b - ADD // c '3 + ADD // c '3 SWAP // '3 c - MUL // '4 + MUL // '4 NEGATE // '5 }> """ diff --git a/tolk-tester/tests/self-keyword.tolk b/tolk-tester/tests/self-keyword.tolk index b0567696d..37f3cb178 100644 --- a/tolk-tester/tests/self-keyword.tolk +++ b/tolk-tester/tests/self-keyword.tolk @@ -222,29 +222,24 @@ fun main() { } @fif_codegen """ - incChained PROC:<{ - // self - INC // self + incChained PROC:<{ // self + INC // self }> - incChained2 PROC:<{ - // self + incChained2 PROC:<{ // self incChained CALLDICT // self }> - incChained3 PROC:<{ - // self + incChained3 PROC:<{ // self incChained CALLDICT // self }> - incChained4 PROC:<{ - // self + incChained4 PROC:<{ // self incChained CALLDICT // self }> """ @fif_codegen """ - testIncChainedCodegen PROC:<{ - // x - incChained CALLDICT // x + testIncChainedCodegen PROC:<{ // x + incChained CALLDICT // x incChained2 CALLDICT // x incChained3 CALLDICT // x incChained4 CALLDICT // x diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index aac1028f3..24baa34f1 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -748,7 +748,6 @@ fun main(x: int?): int { @fif_codegen """ test60 PROC:<{ - // 101 PUSHINT // cc1 109 PUSHINT // cc1 cc2 }> diff --git a/tolk-tester/tests/some-tests-2.tolk b/tolk-tester/tests/some-tests-2.tolk index 9d24f38da..4e46f2633 100644 --- a/tolk-tester/tests/some-tests-2.tolk +++ b/tolk-tester/tests/some-tests-2.tolk @@ -106,7 +106,6 @@ fun test95() { @fif_codegen """ touchCodegen2 PROC:<{ - // get10 CALLDICT // f }> """ @@ -125,7 +124,6 @@ fun test95() { @fif_codegen """ testStartBalanceCodegen1 PROC:<{ - // BALANCE // t FIRST // first }> @@ -134,7 +132,6 @@ fun test95() { @fif_codegen """ testStartBalanceCodegen2 PROC:<{ - // BALANCE FIRST // first }> diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index 8dd2f3455..81f3b9f29 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -517,7 +517,6 @@ type PointAlias = Point; @fif_codegen """ test6 PROC:<{ - // 10 PUSHINT // p.x 20 PUSHINT // p.x p.y }> @@ -525,8 +524,7 @@ type PointAlias = Point; @fif_codegen """ - sumXY PROC:<{ - // point.x point.y + sumXY PROC:<{ // point.x point.y ADD // '2 }> """ @@ -534,8 +532,7 @@ type PointAlias = Point; @fif_codegen """ - test23 PROC:<{ - // p.x p.y + test23 PROC:<{ // p.x p.y 2DROP 20 PUSHINT // '10=20 }> @@ -544,7 +541,6 @@ type PointAlias = Point; @fif_codegen """ test29 PROC:<{ - // PUSHNULL // '5 }> """ diff --git a/tolk-tester/tests/try-catch-tests.tolk b/tolk-tester/tests/try-catch-tests.tolk index 4ac86d966..b0a50aacb 100644 --- a/tolk-tester/tests/try-catch-tests.tolk +++ b/tolk-tester/tests/try-catch-tests.tolk @@ -269,8 +269,7 @@ fun main() { @fif_codegen """ - testCodegen1 PROC:<{ - // x + testCodegen1 PROC:<{ // x DUP // x x 10 GTINT // x '2 IFJMP:<{ // x @@ -288,8 +287,7 @@ fun main() { @fif_codegen """ - testCodegen2 PROC:<{ - // x + testCodegen2 PROC:<{ // x DUP // x x 10 GTINT // x '2 IFJMP:<{ // x @@ -307,8 +305,7 @@ fun main() { @fif_codegen """ - testCodegen3 PROC:<{ - // numberId paramVal + testCodegen3 PROC:<{ // numberId paramVal SWAP -1000 PUSHINT // paramVal numberId '2=-1000 EQUAL // paramVal '3 diff --git a/tolk-tester/tests/type-aliases-tests.tolk b/tolk-tester/tests/type-aliases-tests.tolk index 68935733c..d8ab0782c 100644 --- a/tolk-tester/tests/type-aliases-tests.tolk +++ b/tolk-tester/tests/type-aliases-tests.tolk @@ -184,12 +184,11 @@ fun main(x: MInt, y: MInt?) { @fif_codegen """ - test9 PROC:<{ - // b - DUP // b b + test9 PROC:<{ // b + DUP // b b IFNOTJMP:<{ // b NOT // '2 - }> // b + }> // b DROP // TRUE // '5 }> diff --git a/tolk-tester/tests/union-types-tests.tolk b/tolk-tester/tests/union-types-tests.tolk index 0bc88c366..11dae3f8c 100644 --- a/tolk-tester/tests/union-types-tests.tolk +++ b/tolk-tester/tests/union-types-tests.tolk @@ -854,17 +854,15 @@ fun main() { @fif_codegen """ test26 PROC:<{ - // 42 PUSHINT // result }> """ @fif_codegen """ - checkSimple PROC:<{ - // a.USlot1 a.UTag - NIP // a.UTag - DUP // a.UTag a.UTag + checkSimple PROC:<{ // a.USlot1 a.UTag + NIP // a.UTag + DUP // a.UTag a.UTag 1 EQINT // a.UTag '3 IF:<{ // a.UTag DROP // @@ -883,7 +881,6 @@ fun main() { @fif_codegen """ test61 PROC:<{ - // i.USlot1 i.UTag NIP DUP 46 EQINT diff --git a/tolk-tester/tests/use-before-declare.tolk b/tolk-tester/tests/use-before-declare.tolk index f444c48ef..baae0b400 100644 --- a/tolk-tester/tests/use-before-declare.tolk +++ b/tolk-tester/tests/use-before-declare.tolk @@ -49,7 +49,6 @@ const first = 1; @fif_codegen """ test1 PROC:<{ - // 30 PUSHINT // '10 }> """ @@ -57,7 +56,6 @@ const first = 1; @fif_codegen """ test2 PROC:<{ - // 2 PUSHINT // '2 }> """ diff --git a/tolk-tester/tolk-tester.js b/tolk-tester/tolk-tester.js index 0aaf0738f..8b7000724 100644 --- a/tolk-tester/tolk-tester.js +++ b/tolk-tester/tolk-tester.js @@ -273,6 +273,8 @@ class TolkTestFile { this.expected_hash = null /** @type {string | null} */ this.experimental_options = null + /** @type {boolean} */ + this.enable_tolk_lines_comments = false } parse_input_from_tolk_file() { @@ -292,6 +294,8 @@ class TolkTestFile { this.stderr_includes.push(new TolkTestCaseStderr(this.parse_string_value(lines), false)) } else if (line.startsWith("@fif_codegen_avoid")) { this.fif_codegen.push(new TolkTestCaseFifCodegen(this.parse_string_value(lines), true)) + } else if (line.startsWith("@fif_codegen_enable_comments")) { + this.enable_tolk_lines_comments = true } else if (line.startsWith("@fif_codegen")) { this.fif_codegen.push(new TolkTestCaseFifCodegen(this.parse_string_value(lines), false)) } else if (line.startsWith("@code_hash")) { @@ -345,9 +349,9 @@ class TolkTestFile { async run_and_check() { const wasmModule = await compileWasm(TOLKFIFTLIB_MODULE, TOLKFIFTLIB_WASM) - let res = compileFile(wasmModule, this.tolk_filename, this.experimental_options) + let res = compileFile(wasmModule, this.tolk_filename, this.experimental_options, this.enable_tolk_lines_comments) let exit_code = res.status === 'ok' ? 0 : 1 - let stderr = res.message + let stderr = res.message || res.stderr let stdout = '' if (exit_code === 0 && this.compilation_should_fail) @@ -488,7 +492,7 @@ function copyFromCString(mod, ptr) { } /** @return {{status: string, message: string, fiftCode: string, codeBoc: string, codeHashHex: string}} */ -function compileFile(mod, filename, experimentalOptions) { +function compileFile(mod, filename, experimentalOptions, withSrcLineComments) { // see tolk-wasm.cpp: typedef void (*WasmFsReadCallback)(int, char const*, char**, char**) const callbackPtr = mod.addFunction((kind, dataPtr, destContents, destError) => { if (kind === 0) { // realpath @@ -520,6 +524,7 @@ function compileFile(mod, filename, experimentalOptions) { const config = { optimizationLevel: 2, withStackComments: true, + withSrcLineComments: withSrcLineComments, experimentalOptions: experimentalOptions || undefined, entrypointFileName: filename }; diff --git a/tolk-tester/tolk-tester.py b/tolk-tester/tolk-tester.py index f8a7a5512..126b07930 100644 --- a/tolk-tester/tolk-tester.py +++ b/tolk-tester/tolk-tester.py @@ -260,6 +260,7 @@ def __init__(self, tolk_filename: str, artifacts_folder: str): self.fif_codegen: List[TolkTestCaseFifCodegen] = [] self.expected_hash: TolkTestCaseExpectedHash | None = None self.experimental_options: str | None = None + self.enable_tolk_lines_comments = False def parse_input_from_tolk_file(self): with open(self.tolk_filename, "r") as fd: @@ -279,6 +280,8 @@ def parse_input_from_tolk_file(self): self.stderr_includes.append(TolkTestCaseStderr(self.parse_string_value(lines), False)) elif line.startswith("@fif_codegen_avoid"): self.fif_codegen.append(TolkTestCaseFifCodegen(self.parse_string_value(lines), True)) + elif line.startswith("@fif_codegen_enable_comments"): + self.enable_tolk_lines_comments = True elif line.startswith("@fif_codegen"): self.fif_codegen.append(TolkTestCaseFifCodegen(self.parse_string_value(lines), False)) elif line.startswith("@code_hash"): @@ -326,6 +329,8 @@ def run_and_check(self): cmd_args = [TOLK_EXECUTABLE, "-o", self.get_compiled_fif_filename()] if self.experimental_options: cmd_args = cmd_args + ["-x", self.experimental_options] + if not self.enable_tolk_lines_comments: + cmd_args = cmd_args + ["-L"] res = subprocess.run(cmd_args + [self.tolk_filename], capture_output=True, timeout=10) exit_code = res.returncode stderr = str(res.stderr, "utf-8") diff --git a/tolk/abscode.cpp b/tolk/abscode.cpp index 984d6ffc8..02215d884 100644 --- a/tolk/abscode.cpp +++ b/tolk/abscode.cpp @@ -202,9 +202,6 @@ void Op::show(std::ostream& os, const std::vector& vars, std::string pfx dis += " "; } switch (cl) { - case _Undef: - os << pfx << dis << "???\n"; - break; case _Nop: os << pfx << dis << "NOP\n"; break; diff --git a/tolk/analyzer.cpp b/tolk/analyzer.cpp index 935c8b02c..480165a16 100644 --- a/tolk/analyzer.cpp +++ b/tolk/analyzer.cpp @@ -501,7 +501,7 @@ bool Op::compute_used_vars(const CodeBlob& code, bool edit) { } default: std::cerr << "fatal: unknown operation in compute_used_vars()\n"; - throw ParseError{where, "unknown operation"}; + throw ParseError(loc, "unknown operation"); } } @@ -583,7 +583,7 @@ bool prune_unreachable(std::unique_ptr& ops) { op.cl = Op::_If; std::unique_ptr new_op = std::move(op.block0); op.block0 = std::move(op.block1); - op.block1 = std::make_unique(op.next->where, Op::_Nop); + op.block1 = std::make_unique(op.next->loc, Op::_Nop); new_op->last().next = std::move(ops); ops = std::move(new_op); } @@ -629,7 +629,7 @@ bool prune_unreachable(std::unique_ptr& ops) { } default: std::cerr << "fatal: unknown operation \n"; - throw ParseError{op.where, "unknown operation in prune_unreachable()"}; + throw ParseError(op.loc, "unknown operation in prune_unreachable()"); } if (reach) { return prune_unreachable(op.next); @@ -643,7 +643,7 @@ bool prune_unreachable(std::unique_ptr& ops) { void CodeBlob::prune_unreachable_code() { if (prune_unreachable(ops)) { - throw ParseError{loc, "control reaches end of function"}; + throw ParseError(fun_ref->loc, "control reaches end of function"); } } @@ -704,10 +704,8 @@ VarDescrList Op::fwd_analyze(VarDescrList values) { res.emplace_back(i); } AsmOpList tmp; - if (f_sym->is_asm_function()) { - std::get(f_sym->body)->compile(tmp); // abstract interpretation of res := f (args) - } else { - std::get(f_sym->body)->compile(tmp, res, args, where); + if (!f_sym->is_asm_function()) { + std::get(f_sym->body)->compile(tmp, res, args, loc); } int j = 0; for (var_idx_t i : left) { @@ -822,7 +820,7 @@ VarDescrList Op::fwd_analyze(VarDescrList values) { } default: std::cerr << "fatal: unknown operation \n"; - throw ParseError{where, "unknown operation in fwd_analyze()"}; + throw ParseError(loc, "unknown operation in fwd_analyze()"); } if (next) { return next->fwd_analyze(std::move(values)); @@ -892,7 +890,7 @@ bool Op::mark_noreturn() { } last_in_block1->next = std::move(next); next = std::move(block1); - block1 = std::make_unique(where, Op::_Nop); + block1 = std::make_unique(loc, Op::_Nop); block1->var_info = std::move(block1_var_info); } else { block1->mark_noreturn(); @@ -916,7 +914,7 @@ bool Op::mark_noreturn() { return set_noreturn(next->mark_noreturn()); default: std::cerr << "fatal: unknown operation \n"; - throw ParseError{where, "unknown operation in mark_noreturn()"}; + throw ParseError(loc, "unknown operation in mark_noreturn()"); } } diff --git a/tolk/asmops.cpp b/tolk/asmops.cpp index 2618ed26e..44887633b 100644 --- a/tolk/asmops.cpp +++ b/tolk/asmops.cpp @@ -52,30 +52,50 @@ std::ostream& operator<<(std::ostream& os, AsmOp::SReg stack_reg) { } } -AsmOp AsmOp::Const(int arg, const std::string& push_op) { +// mirror the above operator<< formatting, but calculate resulting strlen +// used to align comments in Fift output +int AsmOp::SReg::calc_out_strlen() const { + int i = idx; + if (i >= 0) { + if (i < 10) { + return 2; + } else if (i < 16) { + return 3; + } else { + return 6; + } + } else if (i >= -2) { + return 5; + } else { + return 6; + } +} + + +AsmOp AsmOp::Const(SrcLocation loc, int arg, const std::string& push_op) { std::ostringstream os; os << arg << ' ' << push_op; - return AsmOp::Const(os.str()); + return AsmOp::Const(loc, os.str()); } -AsmOp AsmOp::make_stk2(int a, int b, const char* str, int delta) { +AsmOp AsmOp::make_stk2(SrcLocation loc, int a, int b, const char* str, int delta) { std::ostringstream os; os << SReg(a) << ' ' << SReg(b) << ' ' << str; int c = std::max(a, b) + 1; - return AsmOp::Custom(os.str(), c, c + delta); + return AsmOp::Custom(loc, os.str(), c, c + delta); } -AsmOp AsmOp::make_stk3(int a, int b, int c, const char* str, int delta) { +AsmOp AsmOp::make_stk3(SrcLocation loc, int a, int b, int c, const char* str, int delta) { std::ostringstream os; os << SReg(a) << ' ' << SReg(b) << ' ' << SReg(c) << ' ' << str; int m = std::max(a, std::max(b, c)) + 1; - return AsmOp::Custom(os.str(), m, m + delta); + return AsmOp::Custom(loc, os.str(), m, m + delta); } -AsmOp AsmOp::BlkSwap(int a, int b) { +AsmOp AsmOp::BlkSwap(SrcLocation loc, int a, int b) { std::ostringstream os; if (a == 1 && b == 1) { - return AsmOp::Xchg(0, 1); + return AsmOp::Xchg(loc, 0, 1); } else if (a == 1) { if (b == 2) { os << "ROT"; @@ -91,125 +111,125 @@ AsmOp AsmOp::BlkSwap(int a, int b) { } else { os << a << " " << b << " BLKSWAP"; } - return AsmOp::Custom(os.str(), a + b, a + b); + return AsmOp::Custom(loc, os.str(), a + b, a + b); } -AsmOp AsmOp::BlkPush(int a, int b) { +AsmOp AsmOp::BlkPush(SrcLocation loc, int a, int b) { std::ostringstream os; if (a == 1) { - return AsmOp::Push(b); + return AsmOp::Push(loc, b); } else if (a == 2 && b == 1) { os << "2DUP"; } else { os << a << " " << b << " BLKPUSH"; } - return AsmOp::Custom(os.str(), b + 1, a + b + 1); + return AsmOp::Custom(loc, os.str(), b + 1, a + b + 1); } -AsmOp AsmOp::BlkDrop(int a) { +AsmOp AsmOp::BlkDrop(SrcLocation loc, int a) { std::ostringstream os; if (a == 1) { - return AsmOp::Pop(); + return AsmOp::Pop(loc, 0); } else if (a == 2) { os << "2DROP"; } else { os << a << " BLKDROP"; } - return AsmOp::Custom(os.str(), a, 0); + return AsmOp::Custom(loc, os.str(), a, 0); } -AsmOp AsmOp::BlkDrop2(int a, int b) { +AsmOp AsmOp::BlkDrop2(SrcLocation loc, int a, int b) { if (!b) { - return BlkDrop(a); + return BlkDrop(loc, a); } std::ostringstream os; os << a << " " << b << " BLKDROP2"; - return AsmOp::Custom(os.str(), a + b, b); + return AsmOp::Custom(loc, os.str(), a + b, b); } -AsmOp AsmOp::BlkReverse(int a, int b) { +AsmOp AsmOp::BlkReverse(SrcLocation loc, int a, int b) { std::ostringstream os; os << a << " " << b << " REVERSE"; - return AsmOp::Custom(os.str(), a + b, a + b); + return AsmOp::Custom(loc, os.str(), a + b, a + b); } -AsmOp AsmOp::Tuple(int a) { +AsmOp AsmOp::Tuple(SrcLocation loc, int a) { switch (a) { case 1: - return AsmOp::Custom("SINGLE", 1, 1); + return AsmOp::Custom(loc, "SINGLE", 1, 1); case 2: - return AsmOp::Custom("PAIR", 2, 1); + return AsmOp::Custom(loc, "PAIR", 2, 1); case 3: - return AsmOp::Custom("TRIPLE", 3, 1); + return AsmOp::Custom(loc, "TRIPLE", 3, 1); } std::ostringstream os; os << a << " TUPLE"; - return AsmOp::Custom(os.str(), a, 1); + return AsmOp::Custom(loc, os.str(), a, 1); } -AsmOp AsmOp::UnTuple(int a) { +AsmOp AsmOp::UnTuple(SrcLocation loc, int a) { switch (a) { case 1: - return AsmOp::Custom("UNSINGLE", 1, 1); + return AsmOp::Custom(loc, "UNSINGLE", 1, 1); case 2: - return AsmOp::Custom("UNPAIR", 1, 2); + return AsmOp::Custom(loc, "UNPAIR", 1, 2); case 3: - return AsmOp::Custom("UNTRIPLE", 1, 3); + return AsmOp::Custom(loc, "UNTRIPLE", 1, 3); } std::ostringstream os; os << a << " UNTUPLE"; - return AsmOp::Custom(os.str(), 1, a); + return AsmOp::Custom(loc, os.str(), 1, a); } -AsmOp AsmOp::IntConst(const td::RefInt256& x) { +AsmOp AsmOp::IntConst(SrcLocation loc, const td::RefInt256& x) { if (x->signed_fits_bits(8)) { - return AsmOp::Const(dec_string(x) + " PUSHINT"); + return AsmOp::Const(loc, dec_string(x) + " PUSHINT"); } if (!x->is_valid()) { - return AsmOp::Const("PUSHNAN"); + return AsmOp::Const(loc, "PUSHNAN"); } int k = is_pos_pow2(x); if (k >= 0) { - return AsmOp::Const(k, "PUSHPOW2"); + return AsmOp::Const(loc, k, "PUSHPOW2"); } k = is_pos_pow2(x + 1); if (k >= 0) { - return AsmOp::Const(k, "PUSHPOW2DEC"); + return AsmOp::Const(loc, k, "PUSHPOW2DEC"); } k = is_pos_pow2(-x); if (k >= 0) { - return AsmOp::Const(k, "PUSHNEGPOW2"); + return AsmOp::Const(loc, k, "PUSHNEGPOW2"); } if (!x->mod_pow2_short(23)) { - return AsmOp::Const(dec_string(x) + " PUSHINTX"); + return AsmOp::Const(loc, dec_string(x) + " PUSHINTX"); } - return AsmOp::Const(dec_string(x) + " PUSHINT"); + return AsmOp::Const(loc, dec_string(x) + " PUSHINT"); } -AsmOp AsmOp::BoolConst(bool f) { - return AsmOp::Const(f ? "TRUE" : "FALSE"); +AsmOp AsmOp::BoolConst(SrcLocation loc, bool f) { + return AsmOp::Const(loc, f ? "TRUE" : "FALSE"); } -AsmOp AsmOp::Parse(const std::string& custom_op) { +AsmOp AsmOp::Parse(SrcLocation loc, const std::string& custom_op) { if (custom_op == "NOP") { - return AsmOp::Nop(); + return AsmOp::Nop(loc); } else if (custom_op == "SWAP") { - return AsmOp::Xchg(1); + return AsmOp::Xchg(loc, 1); } else if (custom_op == "DROP") { - return AsmOp::Pop(0); + return AsmOp::Pop(loc, 0); } else if (custom_op == "NIP") { - return AsmOp::Pop(1); + return AsmOp::Pop(loc, 1); } else if (custom_op == "DUP") { - return AsmOp::Push(0); + return AsmOp::Push(loc, 0); } else if (custom_op == "OVER") { - return AsmOp::Push(1); + return AsmOp::Push(loc, 1); } else { - return AsmOp::Custom(custom_op); + return AsmOp::Custom(loc, custom_op); } } -AsmOp AsmOp::Parse(std::string custom_op, int args, int retv) { - auto res = Parse(custom_op); +AsmOp AsmOp::Parse(SrcLocation loc, std::string custom_op, int args, int retv) { + auto res = Parse(loc, custom_op); if (res.is_custom()) { res.a = args; res.b = retv; @@ -217,48 +237,50 @@ AsmOp AsmOp::Parse(std::string custom_op, int args, int retv) { return res; } -void AsmOp::out(std::ostream& os) const { +int AsmOp::out(std::ostream& os) const { if (!op.empty()) { os << op; - return; + return static_cast(op.size()); // return strlen to align a comment at the right } switch (t) { - case a_none: - break; + case a_nop: + case a_comment: + return 0; case a_xchg: if (!a && !(b & -2)) { os << (b ? "SWAP" : "NOP"); - break; + return b ? 4 : 3; } os << SReg(a) << ' ' << SReg(b) << " XCHG"; - break; + return SReg(a).calc_out_strlen() + 1 + SReg(b).calc_out_strlen() + 5; case a_push: if (!(a & -2)) { os << (a ? "OVER" : "DUP"); - break; + return a ? 4 : 3; } os << SReg(a) << " PUSH"; - break; + return SReg(a).calc_out_strlen() + 5; case a_pop: if (!(a & -2)) { os << (a ? "NIP" : "DROP"); - break; + return a ? 3 : 4; } os << SReg(a) << " POP"; - break; + return SReg(a).calc_out_strlen() + 4; default: - throw Fatal{"unknown assembler operation"}; + throw Fatal("unknown assembler operation"); } } -void AsmOp::out_indent_nl(std::ostream& os, bool no_eol) const { - for (int i = 0; i < indent; i++) { - os << " "; +int AsmOp::out_indented(std::ostream& os, bool print_src_line_above) const { + static int last_line_no = -1; + if (loc.is_defined() && print_src_line_above) { + loc.show_line_to_fif_output(os, indent, &last_line_no); } - out(os); - if (!no_eol) { - os << std::endl; + for (int i = 0; i < indent * 2; i++) { + os << ' '; } + return out(os) + indent * 2; } std::string AsmOp::to_string() const { @@ -271,16 +293,7 @@ std::string AsmOp::to_string() const { } } -bool AsmOpList::append(const std::vector& ops) { - for (const auto& op : ops) { - if (!append(op)) { - return false; - } - } - return true; -} - -const_idx_t AsmOpList::register_const(Const new_const) { +const_idx_t AsmOpList::register_const(td::RefInt256 new_const) { if (new_const.is_null()) { return not_const; } @@ -294,7 +307,7 @@ const_idx_t AsmOpList::register_const(Const new_const) { return (const_idx_t)idx; } -Const AsmOpList::get_const(const_idx_t idx) { +td::RefInt256 AsmOpList::get_const(const_idx_t idx) { if ((unsigned)idx < constants_.size()) { return constants_[idx]; } else { @@ -316,25 +329,27 @@ void AsmOpList::show_var_ext(std::ostream& os, std::pair } void AsmOpList::out(std::ostream& os, int mode) const { - if (!(mode & 2)) { - for (const auto& op : list_) { - op.out_indent_nl(os); - } - } else { - std::size_t n = list_.size(); - for (std::size_t i = 0; i < n; i++) { - const auto& op = list_[i]; - if (!op.is_comment() && i + 1 < n && list_[i + 1].is_comment()) { - op.out_indent_nl(os, true); - os << '\t'; - do { - i++; - } while (i + 1 < n && list_[i + 1].is_comment()); - list_[i].out(os); - os << std::endl; - } else { - op.out_indent_nl(os, false); + std::size_t n = list_.size(); + for (std::size_t i = 0; i < n; i++) { + const AsmOp& op = list_[i]; + if (!op.is_comment() && i + 1 < n && list_[i + 1].is_comment()) { + int len = op.out_indented(os, mode & Stack::_LineComments); + while (len < 28) { // align stack comments at the right + os << ' '; + len++; } + os << '\t'; + do { + i++; + } while (i + 1 < n && list_[i + 1].is_comment()); + list_[i].out(os); + os << std::endl; + } else if (op.is_comment()) { + op.out(os); + os << std::endl; + } else { + op.out_indented(os, mode & Stack::_LineComments); + os << std::endl; } } } @@ -344,7 +359,7 @@ bool apply_op(StackTransform& trans, const AsmOp& op) { return false; } switch (op.t) { - case AsmOp::a_none: + case AsmOp::a_nop: return true; case AsmOp::a_xchg: return trans.apply_xchg(op.a, op.b, true); diff --git a/tolk/ast-aux-data.h b/tolk/ast-aux-data.h new file mode 100644 index 000000000..59006b1fc --- /dev/null +++ b/tolk/ast-aux-data.h @@ -0,0 +1,38 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "ast.h" + +/* + * This file contains a schema of aux_data inside ast_artificial_aux_vertex + * (it's a compiler-inserted vertex that can't occur in source code). + */ + +namespace tolk { + +// AuxData_ForceFiftLocation is created when transforming AST to IR; +// it wraps constants to force codegen location point to usage, not to init_val AST nodes +struct AuxData_ForceFiftLocation final : ASTAuxData { + SrcLocation forced_loc; + + explicit AuxData_ForceFiftLocation(SrcLocation forced_loc) + : forced_loc(forced_loc) { + } +}; + +} // namespace tolk diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 2999d1a10..74ecbd282 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -1209,7 +1209,7 @@ static AnyV parse_asm_func_body(Lexer& lex, V param_list) { lex.expect(tok_asm, "`asm`"); size_t n_params = param_list->size(); if (n_params > 16) { - throw ParseError{loc, "assembler built-in function can have at most 16 arguments"}; + throw ParseError(loc, "assembler built-in function can have at most 16 arguments"); } std::vector arg_order, ret_order; if (lex.tok() == tok_oppar) { diff --git a/tolk/ast-replacer.h b/tolk/ast-replacer.h index 54d1f8e26..382f2bd71 100644 --- a/tolk/ast-replacer.h +++ b/tolk/ast-replacer.h @@ -95,6 +95,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } virtual AnyExprV replace(V v) { return replace_children(v); } @@ -139,6 +140,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { case ast_empty_expression: return replace(v->as()); case ast_parenthesized_expression: return replace(v->as()); case ast_braced_expression: return replace(v->as()); + case ast_artificial_aux_vertex: return replace(v->as()); case ast_tensor: return replace(v->as()); case ast_bracket_tuple: return replace(v->as()); case ast_reference: return replace(v->as()); diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index 42e691525..b056db314 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -84,6 +84,9 @@ class ASTReplicator final { static V clone(V v) { return createV(v->loc, clone(v->get_block_statement())); } + static V clone(V v) { + return createV(v->loc, clone(v->get_wrapped_expr()), v->aux_data, v->inferred_type); + } static V clone(V v) { return createV(v->loc, clone(v->get_items())); } @@ -270,6 +273,7 @@ class ASTReplicator final { case ast_empty_expression: return clone(v->as()); case ast_parenthesized_expression: return clone(v->as()); case ast_braced_expression: return clone(v->as()); + case ast_artificial_aux_vertex: return clone(v->as()); case ast_tensor: return clone(v->as()); case ast_bracket_tuple: return clone(v->as()); case ast_reference: return clone(v->as()); diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index c4af598d4..4e6f795e9 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -45,6 +45,7 @@ class ASTStringifier final : public ASTVisitor { {ast_empty_expression, "ast_empty_expression"}, {ast_parenthesized_expression, "ast_parenthesized_expression"}, {ast_braced_expression, "ast_braced_expression"}, + {ast_artificial_aux_vertex, "ast_artificial_aux_vertex"}, {ast_tensor, "ast_tensor"}, {ast_bracket_tuple, "ast_bracket_tuple"}, {ast_reference, "ast_reference"}, @@ -306,6 +307,7 @@ class ASTStringifier final : public ASTVisitor { case ast_empty_expression: return handle_vertex(v->as()); case ast_parenthesized_expression: return handle_vertex(v->as()); case ast_braced_expression: return handle_vertex(v->as()); + case ast_artificial_aux_vertex: return handle_vertex(v->as()); case ast_tensor: return handle_vertex(v->as()); case ast_bracket_tuple: return handle_vertex(v->as()); case ast_reference: return handle_vertex(v->as()); diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h index 1bb72f44c..926ec5808 100644 --- a/tolk/ast-visitor.h +++ b/tolk/ast-visitor.h @@ -104,6 +104,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } virtual void visit(V v) { return visit_children(v); } @@ -149,6 +150,7 @@ class ASTVisitorFunctionBody : public ASTVisitor { case ast_empty_expression: return visit(v->as()); case ast_parenthesized_expression: return visit(v->as()); case ast_braced_expression: return visit(v->as()); + case ast_artificial_aux_vertex: return visit(v->as()); case ast_tensor: return visit(v->as()); case ast_bracket_tuple: return visit(v->as()); case ast_reference: return visit(v->as()); diff --git a/tolk/ast.h b/tolk/ast.h index 95677fcda..927be5b71 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -75,6 +75,7 @@ enum ASTNodeKind { ast_empty_expression, ast_parenthesized_expression, ast_braced_expression, + ast_artificial_aux_vertex, ast_tensor, ast_bracket_tuple, ast_reference, @@ -149,6 +150,10 @@ enum class MatchArmKind { // for `match` expression, each of arms `pattern => else_branch, // `else => body` }; +struct ASTAuxData { // base class for data in ast_artificial_aux_vertex, see ast-aux-data.h + virtual ~ASTAuxData() = default; +}; + template struct Vertex; @@ -498,6 +503,21 @@ struct Vertex final : ASTExprBlockOfStatements { : ASTExprBlockOfStatements(ast_braced_expression, loc, child_block_statement) {} }; +template<> +// ast_artificial_aux_vertex is a compiler-inserted vertex that can't occur in source code +// example: special vertex to force location in Fift output at constant usage +struct Vertex final : ASTExprUnary { + const ASTAuxData* aux_data; // custom payload, see ast-aux-data.h + + AnyExprV get_wrapped_expr() const { return child; } + + Vertex(SrcLocation loc, AnyExprV wrapped_expr, const ASTAuxData* aux_data, TypePtr inferred_type) + : ASTExprUnary(ast_artificial_aux_vertex, loc, wrapped_expr) + , aux_data(aux_data) { + assign_inferred_type(inferred_type); + } +}; + template<> // ast_tensor is a set of expressions embraced by (parenthesis) // in most languages, it's called "tuple", but in TVM, "tuple" is a TVM primitive, that's why "tensor" diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 75ead3458..7de0b3d84 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -44,11 +44,6 @@ static void define_builtin_func(const std::string& name, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const AsmOp& macro, int flags) { - auto* f_sym = new FunctionData(name, {}, return_type, define_builtin_parameters(params_types, flags), flags, genericTs, nullptr, new FunctionBodyBuiltin(make_simple_compile(macro)), nullptr); - G.symtable.add_function(f_sym); -} - static void define_builtin_func(const std::string& name, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const simple_compile_func_t& func, int flags, std::initializer_list arg_order, std::initializer_list ret_order) { auto* f_sym = new FunctionData(name, {}, return_type, define_builtin_parameters(params_types, flags), flags, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); @@ -58,12 +53,16 @@ static void define_builtin_func(const std::string& name, const std::vector& out, std::vector& in, - SrcLocation where) const { - dest.append(simple_compile(out, in, where)); + SrcLocation loc) const { + dest << simple_compile(out, in, loc); } -void FunctionBodyAsm::compile(AsmOpList& dest) const { - dest.append(ops); +void FunctionBodyAsm::compile(AsmOpList& dest, SrcLocation loc) const { + for (const AsmOp& op : ops) { + AsmOp copy = op; + copy.loc = loc; + dest << std::move(copy); + } } @@ -331,101 +330,89 @@ bool VarDescr::always_neq(const VarDescr& other) const { (always_odd() && other.always_even()); } -AsmOp exec_op(std::string op) { - return AsmOp::Custom(op); -} - -AsmOp exec_op(std::string op, int args, int retv = 1) { - return AsmOp::Custom(op, args, retv); +AsmOp exec_op(SrcLocation loc, std::string op) { + return AsmOp::Custom(loc, op); } -AsmOp exec_arg_op(std::string op, long long arg) { - std::ostringstream os; - os << arg << ' ' << op; - return AsmOp::Custom(os.str()); -} - -AsmOp exec_arg_op(std::string op, long long arg, int args, int retv) { - std::ostringstream os; - os << arg << ' ' << op; - return AsmOp::Custom(os.str(), args, retv); +AsmOp exec_op(SrcLocation loc, std::string op, int args, int retv = 1) { + return AsmOp::Custom(loc, op, args, retv); } -AsmOp exec_arg_op(std::string op, td::RefInt256 arg) { +AsmOp exec_arg_op(SrcLocation loc, std::string op, long long arg, int args, int retv) { std::ostringstream os; os << arg << ' ' << op; - return AsmOp::Custom(os.str()); + return AsmOp::Custom(loc, os.str(), args, retv); } -AsmOp exec_arg_op(std::string op, td::RefInt256 arg, int args, int retv) { +AsmOp exec_arg_op(SrcLocation loc, std::string op, td::RefInt256 arg, int args, int retv) { std::ostringstream os; os << arg << ' ' << op; - return AsmOp::Custom(os.str(), args, retv); + return AsmOp::Custom(loc, os.str(), args, retv); } -AsmOp exec_arg2_op(std::string op, long long imm1, long long imm2, int args, int retv) { +AsmOp exec_arg2_op(SrcLocation loc, std::string op, long long imm1, long long imm2, int args, int retv) { std::ostringstream os; os << imm1 << ' ' << imm2 << ' ' << op; - return AsmOp::Custom(os.str(), args, retv); + return AsmOp::Custom(loc, os.str(), args, retv); } -AsmOp push_const(td::RefInt256 x) { - return AsmOp::IntConst(std::move(x)); +AsmOp push_const(SrcLocation loc, td::RefInt256 x) { + return AsmOp::IntConst(loc, std::move(x)); } -AsmOp compile_add(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_add(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const + y.int_const); if (!r.int_const->is_valid()) { - throw ParseError(where, "integer overflow"); + throw ParseError(loc, "integer overflow"); } x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_add(x.val, y.val); if (y.is_int_const() && y.int_const->signed_fits_bits(8)) { y.unused(); if (y.always_zero()) { - return AsmOp::Nop(); + return AsmOp::Nop(loc); } if (*y.int_const == 1) { - return exec_op("INC", 1); + return exec_op(loc, "INC", 1); } if (*y.int_const == -1) { - return exec_op("DEC", 1); + return exec_op(loc, "DEC", 1); } - return exec_arg_op("ADDCONST", y.int_const, 1); + return exec_arg_op(loc, "ADDCONST", y.int_const, 1); } if (x.is_int_const() && x.int_const->signed_fits_bits(8)) { x.unused(); if (x.always_zero()) { - return AsmOp::Nop(); + return AsmOp::Nop(loc); } if (*x.int_const == 1) { - return exec_op("INC", 1); + return exec_op(loc, "INC", 1); } if (*x.int_const == -1) { - return exec_op("DEC", 1); + return exec_op(loc, "DEC", 1); } - return exec_arg_op("ADDCONST", x.int_const, 1); + return exec_arg_op(loc, "ADDCONST", x.int_const, 1); } - return exec_op("ADD", 2); + return exec_op(loc, "ADD", 2); } -AsmOp compile_sub(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_sub(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const - y.int_const); if (!r.int_const->is_valid()) { - throw ParseError(where, "integer overflow"); + throw ParseError(loc, "integer overflow"); } x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_sub(x.val, y.val); if (y.is_int_const() && (-y.int_const)->signed_fits_bits(8)) { @@ -434,121 +421,121 @@ AsmOp compile_sub(std::vector& res, std::vector& args, SrcLo return {}; } if (*y.int_const == 1) { - return exec_op("DEC", 1); + return exec_op(loc, "DEC", 1); } if (*y.int_const == -1) { - return exec_op("INC", 1); + return exec_op(loc, "INC", 1); } - return exec_arg_op("ADDCONST", -y.int_const, 1); + return exec_arg_op(loc, "ADDCONST", -y.int_const, 1); } if (x.always_zero()) { x.unused(); - return exec_op("NEGATE", 1); + return exec_op(loc, "NEGATE", 1); } - return exec_op("SUB", 2); + return exec_op(loc, "SUB", 2); } -AsmOp compile_unary_minus(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_unary_minus(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 1); VarDescr &r = res[0], &x = args[0]; if (x.is_int_const()) { r.set_const(-x.int_const); if (!r.int_const->is_valid()) { - throw ParseError(where, "integer overflow"); + throw ParseError(loc, "integer overflow"); } x.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_negate(x.val); - return exec_op("NEGATE", 1); + return exec_op(loc, "NEGATE", 1); } -AsmOp compile_unary_plus(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_unary_plus(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 1); VarDescr &r = res[0], &x = args[0]; if (x.is_int_const()) { r.set_const(x.int_const); x.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = x.val; - return AsmOp::Nop(); + return AsmOp::Nop(loc); } -AsmOp compile_logical_not(std::vector& res, std::vector& args, SrcLocation where, bool for_int_arg) { +static AsmOp compile_logical_not(std::vector& res, std::vector& args, SrcLocation loc, bool for_int_arg) { tolk_assert(res.size() == 1 && args.size() == 1); VarDescr &r = res[0], &x = args[0]; if (x.is_int_const()) { r.set_const(x.int_const == 0 ? -1 : 0); x.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = VarDescr::ValBool; // for integers, `!var` is `var != 0` // for booleans, `!var` can be shortened to `~var` (works the same for 0/-1 but consumes less) - return for_int_arg ? exec_op("0 EQINT", 1) : exec_op("NOT", 1); + return for_int_arg ? exec_op(loc, "0 EQINT", 1) : exec_op(loc, "NOT", 1); } -AsmOp compile_bitwise_and(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_bitwise_and(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const & y.int_const); x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_bitwise_and(x.val, y.val); - return exec_op("AND", 2); + return exec_op(loc, "AND", 2); } -AsmOp compile_bitwise_or(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_bitwise_or(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const | y.int_const); x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_bitwise_or(x.val, y.val); - return exec_op("OR", 2); + return exec_op(loc, "OR", 2); } -AsmOp compile_bitwise_xor(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_bitwise_xor(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const ^ y.int_const); x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_bitwise_xor(x.val, y.val); - return exec_op("XOR", 2); + return exec_op(loc, "XOR", 2); } -AsmOp compile_bitwise_not(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_bitwise_not(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 1); VarDescr &r = res[0], &x = args[0]; if (x.is_int_const()) { r.set_const(~x.int_const); x.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_bitwise_not(x.val); - return exec_op("NOT", 1); + return exec_op(loc, "NOT", 1); } -AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation where) { +static AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation loc) { if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const * y.int_const); if (!r.int_const->is_valid()) { - throw ParseError(where, "integer overflow"); + throw ParseError(loc, "integer overflow"); } x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_mul(x.val, y.val); if (y.is_int_const()) { @@ -559,23 +546,23 @@ AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation wh // dubious optimization: NaN * 0 = ? r.set_const(y.int_const); x.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } if (*y.int_const == 1 && x.always_finite()) { - return AsmOp::Nop(); + return AsmOp::Nop(loc); } if (*y.int_const == -1) { - return exec_op("NEGATE", 1); + return exec_op(loc, "NEGATE", 1); } - return exec_arg_op("MULCONST", y.int_const, 1); + return exec_arg_op(loc, "MULCONST", y.int_const, 1); } if (k > 0) { y.unused(); - return exec_arg_op("LSHIFT#", k, 1); + return exec_arg_op(loc, "LSHIFT#", k, 1); } if (k == 0) { y.unused(); - return AsmOp::Nop(); + return AsmOp::Nop(loc); } } if (x.is_int_const()) { @@ -586,48 +573,48 @@ AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation wh // dubious optimization: NaN * 0 = ? r.set_const(x.int_const); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } if (*x.int_const == 1 && y.always_finite()) { - return AsmOp::Nop(); + return AsmOp::Nop(loc); } if (*x.int_const == -1) { - return exec_op("NEGATE", 1); + return exec_op(loc, "NEGATE", 1); } - return exec_arg_op("MULCONST", x.int_const, 1); + return exec_arg_op(loc, "MULCONST", x.int_const, 1); } if (k > 0) { x.unused(); - return exec_arg_op("LSHIFT#", k, 1); + return exec_arg_op(loc, "LSHIFT#", k, 1); } if (k == 0) { x.unused(); - return AsmOp::Nop(); + return AsmOp::Nop(loc); } } - return exec_op("MUL", 2); + return exec_op(loc, "MUL", 2); } -AsmOp compile_mul(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_mul(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 2); - return compile_mul_internal(res[0], args[0], args[1], where); + return compile_mul_internal(res[0], args[0], args[1], loc); } -AsmOp compile_lshift(std::vector& res, std::vector& args, SrcLocation where) { +static AsmOp compile_lshift(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (y.is_int_const()) { auto yv = y.int_const->to_long(); if (yv < 0 || yv > 256) { - throw ParseError(where, "lshift argument is out of range"); + throw ParseError(loc, "lshift argument is out of range"); } else if (x.is_int_const()) { r.set_const(x.int_const << (int)yv); if (!r.int_const->is_valid()) { - throw ParseError(where, "integer overflow"); + throw ParseError(loc, "integer overflow"); } x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } } r.val = emulate_lshift(x.val, y.val); @@ -636,38 +623,38 @@ AsmOp compile_lshift(std::vector& res, std::vector& args, Sr if (!k /* && x.always_finite() */) { // dubious optimization: what if x=NaN ? y.unused(); - return AsmOp::Nop(); + return AsmOp::Nop(loc); } y.unused(); - return exec_arg_op("LSHIFT#", k, 1); + return exec_arg_op(loc, "LSHIFT#", k, 1); } if (x.is_int_const()) { auto xv = x.int_const->to_long(); if (xv == 1) { x.unused(); - return exec_op("POW2", 1); + return exec_op(loc, "POW2", 1); } if (xv == -1) { x.unused(); - return exec_op("-1 PUSHINT SWAP LSHIFT", 1); + return exec_op(loc, "-1 PUSHINT SWAP LSHIFT", 1); } } - return exec_op("LSHIFT", 2); + return exec_op(loc, "LSHIFT", 2); } -AsmOp compile_rshift(std::vector& res, std::vector& args, SrcLocation where, +static AsmOp compile_rshift(std::vector& res, std::vector& args, SrcLocation loc, int round_mode) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (y.is_int_const()) { auto yv = y.int_const->to_long(); if (yv < 0 || yv > 256) { - throw ParseError(where, "rshift argument is out of range"); + throw ParseError(loc, "rshift argument is out of range"); } else if (x.is_int_const()) { r.set_const(td::rshift(x.int_const, (int)yv, round_mode)); x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } } r.val = emulate_rshift(x.val, y.val); @@ -677,36 +664,36 @@ AsmOp compile_rshift(std::vector& res, std::vector& args, Sr if (!k /* && x.always_finite() */) { // dubious optimization: what if x=NaN ? y.unused(); - return AsmOp::Nop(); + return AsmOp::Nop(loc); } y.unused(); - return exec_arg_op(rshift + "#", k, 1); + return exec_arg_op(loc, rshift + "#", k, 1); } - return exec_op(rshift, 2); + return exec_op(loc, rshift, 2); } -AsmOp compile_div_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation where, int round_mode) { +static AsmOp compile_div_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation loc, int round_mode) { if (x.is_int_const() && y.is_int_const()) { r.set_const(div(x.int_const, y.int_const, round_mode)); if (!r.int_const->is_valid()) { - throw ParseError(where, *y.int_const == 0 ? "division by zero" : "integer overflow"); + throw ParseError(loc, *y.int_const == 0 ? "division by zero" : "integer overflow"); } x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_div(x.val, y.val); if (y.is_int_const()) { if (*y.int_const == 0) { - throw ParseError(where, "division by zero"); + throw ParseError(loc, "division by zero"); } if (*y.int_const == 1 && x.always_finite()) { y.unused(); - return AsmOp::Nop(); + return AsmOp::Nop(loc); } if (*y.int_const == -1) { y.unused(); - return exec_op("NEGATE", 1); + return exec_op(loc, "NEGATE", 1); } int k = is_pos_pow2(y.int_const); if (k > 0) { @@ -715,44 +702,44 @@ AsmOp compile_div_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation wh if (round_mode >= 0) { op += (round_mode > 0 ? 'C' : 'R'); } - return exec_arg_op(op + '#', k, 1); + return exec_arg_op(loc, op + '#', k, 1); } } std::string op = "DIV"; if (round_mode >= 0) { op += (round_mode > 0 ? 'C' : 'R'); } - return exec_op(op, 2); + return exec_op(loc, op, 2); } -AsmOp compile_div(std::vector& res, std::vector& args, SrcLocation where, int round_mode) { +static AsmOp compile_div(std::vector& res, std::vector& args, SrcLocation loc, int round_mode) { tolk_assert(res.size() == 1 && args.size() == 2); - return compile_div_internal(res[0], args[0], args[1], where, round_mode); + return compile_div_internal(res[0], args[0], args[1], loc, round_mode); } -AsmOp compile_mod(std::vector& res, std::vector& args, SrcLocation where, +static AsmOp compile_mod(std::vector& res, std::vector& args, SrcLocation loc, int round_mode) { tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(mod(x.int_const, y.int_const, round_mode)); if (!r.int_const->is_valid()) { - throw ParseError(where, *y.int_const == 0 ? "division by zero" : "integer overflow"); + throw ParseError(loc, *y.int_const == 0 ? "division by zero" : "integer overflow"); } x.unused(); y.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } r.val = emulate_mod(x.val, y.val); if (y.is_int_const()) { if (*y.int_const == 0) { - throw ParseError(where, "division by zero"); + throw ParseError(loc, "division by zero"); } if ((*y.int_const == 1 || *y.int_const == -1) && x.always_finite()) { x.unused(); y.unused(); r.set_const(td::zero_refint()); - return push_const(r.int_const); + return push_const(loc, r.int_const); } int k = is_pos_pow2(y.int_const); if (k > 0) { @@ -761,29 +748,29 @@ AsmOp compile_mod(std::vector& res, std::vector& args, SrcLo if (round_mode >= 0) { op += (round_mode > 0 ? 'C' : 'R'); } - return exec_arg_op(op + '#', k, 1); + return exec_arg_op(loc, op + '#', k, 1); } } std::string op = "MOD"; if (round_mode >= 0) { op += (round_mode > 0 ? 'C' : 'R'); } - return exec_op(op, 2); + return exec_op(loc, op, 2); } -AsmOp compile_muldiv(std::vector& res, std::vector& args, SrcLocation where, +static AsmOp compile_muldiv(std::vector& res, std::vector& args, SrcLocation loc, int round_mode) { tolk_assert(res.size() == 1 && args.size() == 3); VarDescr &r = res[0], &x = args[0], &y = args[1], &z = args[2]; if (x.is_int_const() && y.is_int_const() && z.is_int_const()) { r.set_const(muldiv(x.int_const, y.int_const, z.int_const, round_mode)); if (!r.int_const->is_valid()) { - throw ParseError(where, *z.int_const == 0 ? "division by zero" : "integer overflow"); + throw ParseError(loc, *z.int_const == 0 ? "division by zero" : "integer overflow"); } x.unused(); y.unused(); z.unused(); - return push_const(r.int_const); + return push_const(loc, r.int_const); } if (x.always_zero() || y.always_zero()) { // dubious optimization for z=0... @@ -791,26 +778,26 @@ AsmOp compile_muldiv(std::vector& res, std::vector& args, Sr y.unused(); z.unused(); r.set_const(td::make_refint(0)); - return push_const(r.int_const); + return push_const(loc, r.int_const); } char c = (round_mode < 0) ? 0 : (round_mode > 0 ? 'C' : 'R'); r.val = emulate_div(emulate_mul(x.val, y.val), z.val); if (z.is_int_const()) { if (*z.int_const == 0) { - throw ParseError(where, "division by zero"); + throw ParseError(loc, "division by zero"); } if (*z.int_const == 1) { z.unused(); - return compile_mul_internal(r, x, y, where); + return compile_mul_internal(r, x, y, loc); } } if (y.is_int_const() && *y.int_const == 1) { y.unused(); - return compile_div_internal(r, x, z, where, round_mode); + return compile_div_internal(r, x, z, loc, round_mode); } if (x.is_int_const() && *x.int_const == 1) { x.unused(); - return compile_div_internal(r, y, z, where, round_mode); + return compile_div_internal(r, y, z, loc, round_mode); } if (z.is_int_const()) { int k = is_pos_pow2(z.int_const); @@ -820,7 +807,7 @@ AsmOp compile_muldiv(std::vector& res, std::vector& args, Sr if (c) { op += c; } - return exec_arg_op(op + '#', k, 2); + return exec_arg_op(loc, op + '#', k, 2); } } if (y.is_int_const()) { @@ -831,7 +818,7 @@ AsmOp compile_muldiv(std::vector& res, std::vector& args, Sr if (c) { op += c; } - return exec_arg_op(op, k, 2); + return exec_arg_op(loc, op, k, 2); } } if (x.is_int_const()) { @@ -842,17 +829,22 @@ AsmOp compile_muldiv(std::vector& res, std::vector& args, Sr if (c) { op += c; } - return exec_arg_op(op, k, 2); + return exec_arg_op(loc, op, k, 2); } } std::string op = "MULDIV"; if (c) { op += c; } - return exec_op(op, 3); + return exec_op(loc, op, 3); } -int compute_compare(td::RefInt256 x, td::RefInt256 y, int mode) { +// fun mulDivMod(x: int, y: int, z: int): (int, int) asm "MULDIVMOD"; +static AsmOp compile_muldivmod(std::vector&, std::vector&, SrcLocation loc) { + return AsmOp::Custom(loc, "MULDIVMOD", 3, 2); +} + +static int compute_compare(td::RefInt256 x, td::RefInt256 y, int mode) { int s = td::cmp(x, y); if (mode == 7) { return s; @@ -866,7 +858,7 @@ int compute_compare(td::RefInt256 x, td::RefInt256 y, int mode) { // 2 -> constant 0 // 1 -> constant -1 // 3 -> 0 or -1 -int compute_compare(const VarDescr& x, const VarDescr& y, int mode) { +static int compute_compare(const VarDescr& x, const VarDescr& y, int mode) { switch (mode) { case 1: // > return x.always_greater(y) ? 1 : (x.always_leq(y) ? 2 : 3); @@ -893,7 +885,7 @@ int compute_compare(const VarDescr& x, const VarDescr& y, int mode) { } } -AsmOp compile_cmp_int(std::vector& res, std::vector& args, int mode) { +static AsmOp compile_cmp_int(std::vector& res, std::vector& args, SrcLocation loc, int mode) { tolk_assert(mode >= 1 && mode <= 7); tolk_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; @@ -902,7 +894,7 @@ AsmOp compile_cmp_int(std::vector& res, std::vector& args, i r.set_const(v); x.unused(); y.unused(); - return mode == 7 ? push_const(r.int_const) : AsmOp::BoolConst(v != 0); + return mode == 7 ? push_const(loc, r.int_const) : AsmOp::BoolConst(loc, v != 0); } int v = compute_compare(x, y, mode); // std::cerr << "compute_compare(" << x << ", " << y << ", " << mode << ") = " << v << std::endl; @@ -911,7 +903,7 @@ AsmOp compile_cmp_int(std::vector& res, std::vector& args, i r.set_const(v - (v >> 2) - 2); x.unused(); y.unused(); - return mode == 7 ? push_const(r.int_const) : AsmOp::BoolConst(v & 1); + return mode == 7 ? push_const(loc, r.int_const) : AsmOp::BoolConst(loc, v & 1); } r.val = ~0; if (v & 1) { @@ -930,29 +922,29 @@ AsmOp compile_cmp_int(std::vector& res, std::vector& args, i if (mode != 7) { if (y.is_int_const() && y.int_const >= -128 && y.int_const <= 127) { y.unused(); - return exec_arg_op(cmp_int_names[mode], y.int_const + cmp_int_delta[mode], 1); + return exec_arg_op(loc, cmp_int_names[mode], y.int_const + cmp_int_delta[mode], 1); } if (x.is_int_const() && x.int_const >= -128 && x.int_const <= 127) { x.unused(); mode = ((mode & 4) >> 2) | (mode & 2) | ((mode & 1) << 2); - return exec_arg_op(cmp_int_names[mode], x.int_const + cmp_int_delta[mode], 1); + return exec_arg_op(loc, cmp_int_names[mode], x.int_const + cmp_int_delta[mode], 1); } } - return exec_op(cmp_names[mode], 2); + return exec_op(loc, cmp_names[mode], 2); } -AsmOp compile_throw(std::vector& res, std::vector& args, SrcLocation) { +static AsmOp compile_throw(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.empty() && args.size() == 1); VarDescr& x = args[0]; if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { x.unused(); - return exec_arg_op("THROW", x.int_const, 0, 0); + return exec_arg_op(loc, "THROW", x.int_const, 0, 0); } else { - return exec_op("THROWANY", 1, 0); + return exec_op(loc, "THROWANY", 1, 0); } } -AsmOp compile_throw_if_unless(std::vector& res, std::vector& args, SrcLocation) { +static AsmOp compile_throw_if_unless(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.empty() && args.size() == 3); VarDescr &x = args[0], &y = args[1], &z = args[2]; if (!z.always_true() && !z.always_false()) { @@ -967,40 +959,40 @@ AsmOp compile_throw_if_unless(std::vector& res, std::vector& skip_cond = true; if (y.always_true() != mode) { x.unused(); - return AsmOp::Nop(); + return AsmOp::Nop(loc); } } if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { x.unused(); - return skip_cond ? exec_arg_op("THROW", x.int_const, 0, 0) : exec_arg_op("THROW"s + suff, x.int_const, 1, 0); + return skip_cond ? exec_arg_op(loc, "THROW", x.int_const, 0, 0) : exec_arg_op(loc, "THROW"s + suff, x.int_const, 1, 0); } else { - return skip_cond ? exec_op("THROWANY", 1, 0) : exec_op("THROWANY"s + suff, 2, 0); + return skip_cond ? exec_op(loc, "THROWANY", 1, 0) : exec_op(loc, "THROWANY"s + suff, 2, 0); } } -AsmOp compile_throw_arg(std::vector& res, std::vector& args, SrcLocation) { +static AsmOp compile_throw_arg(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.empty() && args.size() == 2); VarDescr &x = args[1]; if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { x.unused(); - return exec_arg_op("THROWARG", x.int_const, 1, 0); + return exec_arg_op(loc, "THROWARG", x.int_const, 1, 0); } else { - return exec_op("THROWARGANY", 2, 0); + return exec_op(loc, "THROWARGANY", 2, 0); } } -AsmOp compile_bool_const(std::vector& res, std::vector& args, bool val) { +static AsmOp compile_bool_const(std::vector& res, std::vector& args, SrcLocation loc, bool val) { tolk_assert(res.size() == 1 && args.empty()); VarDescr& r = res[0]; r.set_const(val ? -1 : 0); - return AsmOp::Const(val ? "TRUE" : "FALSE"); + return AsmOp::Const(loc, val ? "TRUE" : "FALSE"); } // fun loadInt (mutate s: slice, len: int): int asm(s len -> 1 0) "LDIX"; // fun loadUint (mutate s: slice, len: int): int asm( -> 1 0) "LDUX"; // fun preloadInt (s: slice, len: int): int asm "PLDIX"; // fun preloadUint(s: slice, len: int): int asm "PLDUX"; -AsmOp compile_fetch_int(std::vector& res, std::vector& args, bool fetch, bool sgnd) { +static AsmOp compile_fetch_int(std::vector& res, std::vector& args, SrcLocation loc, bool fetch, bool sgnd) { tolk_assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); auto &y = args[1], &r = res.back(); r.val = (sgnd ? VarDescr::FiniteInt : VarDescr::FiniteUInt); @@ -1015,27 +1007,27 @@ AsmOp compile_fetch_int(std::vector& res, std::vector& args, } if (v > 0) { y.unused(); - return exec_arg_op((fetch ? "LD"s : "PLD"s) + (sgnd ? 'I' : 'U'), v, 1, 1 + (unsigned)fetch); + return exec_arg_op(loc, (fetch ? "LD"s : "PLD"s) + (sgnd ? 'I' : 'U'), v, 1, 1 + (unsigned)fetch); } } - return exec_op((fetch ? "LD"s : "PLD"s) + (sgnd ? "IX" : "UX"), 2, 1 + (unsigned)fetch); + return exec_op(loc, (fetch ? "LD"s : "PLD"s) + (sgnd ? "IX" : "UX"), 2, 1 + (unsigned)fetch); } // fun storeInt (mutate self: builder, x: int, len: int): self asm(x b len) "STIX"; // fun storeUint (mutate self: builder, x: int, len: int): self asm(x b len) "STUX"; -AsmOp compile_store_int(std::vector& res, std::vector& args, bool sgnd) { +static AsmOp compile_store_int(std::vector& res, std::vector& args, SrcLocation loc, bool sgnd) { tolk_assert(args.size() == 3 && res.size() == 1); auto& z = args[2]; if (z.is_int_const() && z.int_const > 0 && z.int_const <= 256) { z.unused(); - return exec_arg_op("ST"s + (sgnd ? 'I' : 'U'), z.int_const, 2, 1); + return exec_arg_op(loc, "ST"s + (sgnd ? 'I' : 'U'), z.int_const, 2, 1); } - return exec_op("ST"s + (sgnd ? "IX" : "UX"), 3, 1); + return exec_op(loc, "ST"s + (sgnd ? "IX" : "UX"), 3, 1); } // fun loadBits (mutate self: slice, len: int): self asm(s len -> 1 0) "LDSLICEX" // fun preloadBits(self: slice, len: int): slice asm(s len -> 1 0) "PLDSLICEX" -AsmOp compile_fetch_slice(std::vector& res, std::vector& args, bool fetch) { +static AsmOp compile_fetch_slice(std::vector& res, std::vector& args, SrcLocation loc, bool fetch) { tolk_assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); auto& y = args[1]; int v = -1; @@ -1043,47 +1035,73 @@ AsmOp compile_fetch_slice(std::vector& res, std::vector& arg v = (int)y.int_const->to_long(); if (v > 0) { y.unused(); - return exec_arg_op(fetch ? "LDSLICE" : "PLDSLICE", v, 1, 1 + (unsigned)fetch); + return exec_arg_op(loc, fetch ? "LDSLICE" : "PLDSLICE", v, 1, 1 + (unsigned)fetch); } } - return exec_op(fetch ? "LDSLICEX" : "PLDSLICEX", 2, 1 + (unsigned)fetch); + return exec_op(loc, fetch ? "LDSLICEX" : "PLDSLICEX", 2, 1 + (unsigned)fetch); } // fun tupleAt(t: tuple, index: int): X asm "INDEXVAR"; -AsmOp compile_tuple_at(std::vector& res, std::vector& args, SrcLocation) { +static AsmOp compile_tuple_at(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(args.size() == 2 && res.size() == 1); auto& y = args[1]; if (y.is_int_const() && y.int_const >= 0 && y.int_const < 16) { y.unused(); - return exec_arg_op("INDEX", y.int_const, 1, 1); + return exec_arg_op(loc, "INDEX", y.int_const, 1, 1); } - return exec_op("INDEXVAR", 2, 1); + return exec_op(loc, "INDEXVAR", 2, 1); } // fun tupleSetAt(mutate self: tuple, value: X, index: int): void asm "SETINDEXVAR"; -AsmOp compile_tuple_set_at(std::vector& res, std::vector& args, SrcLocation) { +static AsmOp compile_tuple_set_at(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(args.size() == 3 && res.size() == 1); auto& y = args[2]; if (y.is_int_const() && y.int_const >= 0 && y.int_const < 16) { y.unused(); - return exec_arg_op("SETINDEX", y.int_const, 1, 1); + return exec_arg_op(loc, "SETINDEX", y.int_const, 1, 1); } - return exec_op("SETINDEXVAR", 2, 1); + return exec_op(loc, "SETINDEXVAR", 2, 1); +} + +// fun debugDumpStack(): void asm "DUMPSTK"; +static AsmOp compile_dumpstk(std::vector&, std::vector&, SrcLocation loc) { + return AsmOp::Custom(loc, "DUMPSTK", 0, 0); +} + +// fun debugPrintString(x: T): void asm "STRDUMP"; +static AsmOp compile_strdump(std::vector&, std::vector&, SrcLocation loc) { + return AsmOp::Custom(loc, "STRDUMP DROP", 1, 1); +} + +// fun debugPrint(x: T): void; +static AsmOp compile_debug_print_to_string(std::vector&, std::vector&, SrcLocation loc) { + return AsmOp::Custom(loc, "s0 DUMP DROP", 1, 1); } // fun ton(amount: slice): coins; ton("0.05") replaced by 50000000 at compile-time // same for stringCrc32(constString: slice) and others -AsmOp compile_time_only_function(std::vector&, std::vector&, SrcLocation) { +AsmOp compile_time_only_function(std::vector&, std::vector&, SrcLocation loc) { // all ton() invocations are constants, replaced by integers; no dynamic values allowed, no work at runtime tolk_assert(false); - return AsmOp::Nop(); + return AsmOp::Nop(loc); +} + +// `null` literal is under the hood transformed to PUSHNULL +static AsmOp compile_push_null(std::vector&, std::vector&, SrcLocation loc) { + return AsmOp::Const(loc, "PUSHNULL"); } // fun __isNull(X arg): bool -AsmOp compile_is_null(std::vector& res, std::vector& args, SrcLocation) { +static AsmOp compile_is_null(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(args.size() == 1 && res.size() == 1); res[0].val = VarDescr::ValBool; - return exec_op("ISNULL", 1, 1); + return exec_op(loc, "ISNULL", 1, 1); +} + +// fun __expect_type(, ""): void; +static AsmOp compile_expect_type(std::vector&, std::vector&, SrcLocation loc) { + // handled by type checker, does nothing at runtime + return AsmOp::Nop(loc); } @@ -1175,37 +1193,37 @@ void define_builtins() { compile_bitwise_xor, FunctionData::flagMarkedAsPure); define_builtin_func("_==_", ParamsInt2, Int, nullptr, // also works for bool - std::bind(compile_cmp_int, _1, _2, 2), + std::bind(compile_cmp_int, _1, _2, _3, 2), FunctionData::flagMarkedAsPure); define_builtin_func("_!=_", ParamsInt2, Int, nullptr, // also works for bool - std::bind(compile_cmp_int, _1, _2, 5), + std::bind(compile_cmp_int, _1, _2, _3, 5), FunctionData::flagMarkedAsPure); define_builtin_func("_<_", ParamsInt2, Int, nullptr, - std::bind(compile_cmp_int, _1, _2, 4), + std::bind(compile_cmp_int, _1, _2, _3, 4), FunctionData::flagMarkedAsPure); define_builtin_func("_>_", ParamsInt2, Int, nullptr, - std::bind(compile_cmp_int, _1, _2, 1), + std::bind(compile_cmp_int, _1, _2, _3, 1), FunctionData::flagMarkedAsPure); define_builtin_func("_<=_", ParamsInt2, Int, nullptr, - std::bind(compile_cmp_int, _1, _2, 6), + std::bind(compile_cmp_int, _1, _2, _3, 6), FunctionData::flagMarkedAsPure); define_builtin_func("_>=_", ParamsInt2, Int, nullptr, - std::bind(compile_cmp_int, _1, _2, 3), + std::bind(compile_cmp_int, _1, _2, _3, 3), FunctionData::flagMarkedAsPure); define_builtin_func("_<=>_", ParamsInt2, Int, nullptr, - std::bind(compile_cmp_int, _1, _2, 7), + std::bind(compile_cmp_int, _1, _2, _3, 7), FunctionData::flagMarkedAsPure); // special function used for internal compilation of some lexical constructs // for example, `throw 123;` is actually calling `__throw(123)` define_builtin_func("__true", {}, Bool, nullptr, /* AsmOp::Const("TRUE") */ - std::bind(compile_bool_const, _1, _2, true), + std::bind(compile_bool_const, _1, _2, _3, true), FunctionData::flagMarkedAsPure); define_builtin_func("__false", {}, Bool, nullptr, /* AsmOp::Const("FALSE") */ - std::bind(compile_bool_const, _1, _2, false), + std::bind(compile_bool_const, _1, _2, _3, false), FunctionData::flagMarkedAsPure); define_builtin_func("__null", {}, typeT, declGenericT, - AsmOp::Const("PUSHNULL"), + compile_push_null, FunctionData::flagMarkedAsPure); define_builtin_func("__isNull", {typeT}, Bool, declGenericT, compile_is_null, @@ -1260,35 +1278,35 @@ void define_builtins() { std::bind(compile_muldiv, _1, _2, _3, 1), FunctionData::flagMarkedAsPure); define_builtin_func("mulDivMod", ParamsInt3, TypeDataTensor::create({Int, Int}), nullptr, - AsmOp::Custom("MULDIVMOD", 3, 2), + compile_muldivmod, FunctionData::flagMarkedAsPure); define_builtin_func("loadInt", ParamsSliceInt, Int, nullptr, - std::bind(compile_fetch_int, _1, _2, true, true), + std::bind(compile_fetch_int, _1, _2, _3, true, true), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, {}, {1, 0}); define_builtin_func("loadUint", ParamsSliceInt, Int, nullptr, - std::bind(compile_fetch_int, _1, _2, true, false), + std::bind(compile_fetch_int, _1, _2, _3, true, false), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, {}, {1, 0}); define_builtin_func("loadBits", ParamsSliceInt, Slice, nullptr, - std::bind(compile_fetch_slice, _1, _2, true), + std::bind(compile_fetch_slice, _1, _2, _3, true), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, {}, {1, 0}); define_builtin_func("preloadInt", ParamsSliceInt, Int, nullptr, - std::bind(compile_fetch_int, _1, _2, false, true), + std::bind(compile_fetch_int, _1, _2, _3, false, true), FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); define_builtin_func("preloadUint", ParamsSliceInt, Int, nullptr, - std::bind(compile_fetch_int, _1, _2, false, false), + std::bind(compile_fetch_int, _1, _2, _3, false, false), FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); define_builtin_func("preloadBits", ParamsSliceInt, Slice, nullptr, - std::bind(compile_fetch_slice, _1, _2, false), + std::bind(compile_fetch_slice, _1, _2, _3, false), FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); define_builtin_func("storeInt", {Builder, Int, Int}, Unit, nullptr, - std::bind(compile_store_int, _1, _2, true), + std::bind(compile_store_int, _1, _2, _3, true), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf, {1, 0, 2}, {}); define_builtin_func("storeUint", {Builder, Int, Int}, Unit, nullptr, - std::bind(compile_store_int, _1, _2, false), + std::bind(compile_store_int, _1, _2, _3, false), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf, {1, 0, 2}, {}); define_builtin_func("tupleAt", {Tuple, Int}, typeT, declGenericT, @@ -1298,20 +1316,20 @@ void define_builtins() { compile_tuple_set_at, FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf); define_builtin_func("debugPrint", {typeT}, Unit, declGenericT, - AsmOp::Custom("s0 DUMP DROP", 1, 1), + compile_debug_print_to_string, 0); define_builtin_func("debugPrintString", {typeT}, Unit, declGenericT, - AsmOp::Custom("STRDUMP DROP", 1, 1), + compile_strdump, 0); define_builtin_func("debugDumpStack", {}, Unit, nullptr, - AsmOp::Custom("DUMPSTK", 0, 0), + compile_dumpstk, 0); // functions not presented in stdlib at all // used in tolk-tester to check/expose internal compiler state // each of them is handled in a special way, search by its name define_builtin_func("__expect_type", {TypeDataUnknown::create(), Slice}, Unit, nullptr, - AsmOp::Nop(), + compile_expect_type, FunctionData::flagMarkedAsPure); } diff --git a/tolk/codegen.cpp b/tolk/codegen.cpp index ac1cf6391..6c970269e 100644 --- a/tolk/codegen.cpp +++ b/tolk/codegen.cpp @@ -82,38 +82,41 @@ void Stack::forget_const() { } } -void Stack::issue_pop(int i) { +void Stack::issue_pop(SrcLocation loc, int i) { validate(i); if (output_enabled()) { - o << AsmOp::Pop(i); + o << AsmOp::Pop(loc, i); } at(i) = get(0); s.pop_back(); modified(); + opt_show(); } -void Stack::issue_push(int i) { +void Stack::issue_push(SrcLocation loc, int i) { validate(i); if (output_enabled()) { - o << AsmOp::Push(i); + o << AsmOp::Push(loc, i); } s.push_back(get(i)); modified(); + opt_show(); } -void Stack::issue_xchg(int i, int j) { +void Stack::issue_xchg(SrcLocation loc, int i, int j) { validate(i); validate(j); if (i != j && get(i) != get(j)) { if (output_enabled()) { - o << AsmOp::Xchg(i, j); + o << AsmOp::Xchg(loc, i, j); } std::swap(at(i), at(j)); modified(); + opt_show(); } } -int Stack::drop_vars_except(const VarDescrList& var_info, int excl_var) { +int Stack::drop_vars_except(SrcLocation loc, const VarDescrList& var_info, int excl_var) { int dropped = 0, changes; do { changes = 0; @@ -122,7 +125,7 @@ int Stack::drop_vars_except(const VarDescrList& var_info, int excl_var) { var_idx_t idx = at(i).first; if (((!var_info[idx] || var_info[idx]->is_unused()) && idx != excl_var) || find(idx, 0, i - 1) >= 0) { // unneeded - issue_pop(i); + issue_pop(loc, i); changes = 1; break; } @@ -138,7 +141,7 @@ void Stack::show() { os << ' '; o.show_var_ext(os, i); } - o << AsmOp::Comment(os.str()); + o << AsmOp::Comment({}, os.str()); mode |= _Shown; } @@ -172,17 +175,17 @@ void Stack::assign_var(var_idx_t new_idx, var_idx_t old_idx) { } } -void Stack::do_copy_var(var_idx_t new_idx, var_idx_t old_idx) { +void Stack::do_copy_var(SrcLocation loc, var_idx_t new_idx, var_idx_t old_idx) { int i = find(old_idx); tolk_assert(i >= 0 && "variable not found in stack"); if (find(old_idx, i + 1) < 0) { - issue_push(i); + issue_push(loc, i); tolk_assert(at(0).first == old_idx); } assign_var(new_idx, old_idx); } -void Stack::enforce_state(const StackLayout& req_stack) { +void Stack::enforce_state(SrcLocation loc, const StackLayout& req_stack) { int k = (int)req_stack.size(); for (int i = 0; i < k; i++) { var_idx_t x = req_stack[i]; @@ -191,18 +194,18 @@ void Stack::enforce_state(const StackLayout& req_stack) { } while (depth() > 0 && std::find(req_stack.cbegin(), req_stack.cend(), get(0).first) == req_stack.cend()) { // current TOS entry is unused in req_stack, drop it - issue_pop(0); + issue_pop(loc, 0); } int j = find(x); if (j >= depth() - i) { - issue_push(j); + issue_push(loc, j); j = 0; } - issue_xchg(j, depth() - i - 1); + issue_xchg(loc, j, depth() - i - 1); tolk_assert(s[i].first == x); } while (depth() > k) { - issue_pop(0); + issue_pop(loc, 0); } tolk_assert(depth() == k); for (int i = 0; i < k; i++) { @@ -220,12 +223,12 @@ void Stack::merge_const(const Stack& req_stack) { } } -void Stack::merge_state(const Stack& req_stack) { - enforce_state(req_stack.vars()); +void Stack::merge_state(SrcLocation loc, const Stack& req_stack) { + enforce_state(loc, req_stack.vars()); merge_const(req_stack); } -void Stack::rearrange_top(const StackLayout& top, std::vector last) { +void Stack::rearrange_top(SrcLocation loc, const StackLayout& top, std::vector last) { while (last.size() < top.size()) { last.push_back(false); } @@ -250,24 +253,24 @@ void Stack::rearrange_top(const StackLayout& top, std::vector last) { int j = find_outside(x, ss, ss + i); if (last[i]) { // rearrange x to be at s(ss-1) - issue_xchg(--ss, j); + issue_xchg(loc, --ss, j); tolk_assert(get(ss).first == x); } else { // create a new copy of x - issue_push(j); - issue_xchg(0, ss); + issue_push(loc, j); + issue_xchg(loc, 0, ss); tolk_assert(get(ss).first == x); } } tolk_assert(!ss); } -void Stack::rearrange_top(var_idx_t top, bool last) { +void Stack::rearrange_top(SrcLocation loc, var_idx_t top, bool last) { int i = find(top); if (last) { - issue_xchg(0, i); + issue_xchg(loc, 0, i); } else { - issue_push(i); + issue_push(loc, i); } tolk_assert(get(0).first == top); } @@ -280,7 +283,7 @@ bool Op::generate_code_step(Stack& stack) { bool will_now_immediate_throw = (cl == _Call && f_sym->is_builtin_function() && f_sym->name == "__throw") || (cl == _IntConst && next->cl == _Call && next->f_sym->is_builtin_function() && next->f_sym->name == "__throw"); if (!will_now_immediate_throw) { - stack.drop_vars_except(var_info); + stack.drop_vars_except(loc, var_info); stack.opt_show(); } @@ -290,9 +293,9 @@ bool Op::generate_code_step(Stack& stack) { case _Import: return true; case _Return: { - stack.enforce_state(left); + stack.enforce_state(loc, left); if (stack.o.retalt_ && (stack.mode & Stack::_NeedRetAlt)) { - stack.o << "RETALT"; + stack.o << AsmOp::Custom(loc, "RETALT"); stack.o.retalt_inserted_ = true; } stack.opt_show(); @@ -306,11 +309,11 @@ bool Op::generate_code_step(Stack& stack) { auto cidx = stack.o.register_const(int_const); int i = stack.find_const(cidx); if (i < 0) { - stack.o << push_const(int_const); + stack.o << push_const(loc, int_const); stack.push_new_const(left[0], cidx); } else { tolk_assert(stack.at(i).second == cidx); - stack.do_copy_var(left[0], stack[i]); + stack.do_copy_var(loc, left[0], stack[i]); } return true; } @@ -319,7 +322,7 @@ bool Op::generate_code_step(Stack& stack) { if (!p || p->is_unused()) { return true; } - stack.o << AsmOp::Const("x{" + str_const + "} PUSHSLICE"); + stack.o << AsmOp::Const(loc, "x{" + str_const + "} PUSHSLICE"); stack.push_new_var(left[0]); return true; } @@ -335,10 +338,10 @@ bool Op::generate_code_step(Stack& stack) { if (!used || disabled()) { return true; } - stack.o << AsmOp::Custom(g_sym->name + " GETGLOB", 0, 1); + stack.o << AsmOp::Custom(loc, g_sym->name + " GETGLOB", 0, 1); if (left.size() != 1) { tolk_assert(left.size() <= 15); - stack.o << AsmOp::UnTuple((int)left.size()); + stack.o << AsmOp::UnTuple(loc, (int)left.size()); } for (auto i : left) { stack.push_new_var(i); @@ -350,7 +353,7 @@ bool Op::generate_code_step(Stack& stack) { if (!p || p->is_unused() || disabled()) { return true; } - stack.o << "CONT:<{"; + stack.o << AsmOp::Custom(loc, "CONT:<{"); stack.o.indent(); if (f_sym->is_asm_function() || f_sym->is_builtin_function()) { // TODO: create and compile a true lambda instead of this (so that arg_order and ret_order would work correctly) @@ -368,15 +371,15 @@ bool Op::generate_code_step(Stack& stack) { args0.emplace_back(0); } if (f_sym->is_asm_function()) { - std::get(f_sym->body)->compile(stack.o); // compile res := f (args0) + std::get(f_sym->body)->compile(stack.o, loc); // compile res := f (args0) } else { - std::get(f_sym->body)->compile(stack.o, res, args0, where); // compile res := f (args0) + std::get(f_sym->body)->compile(stack.o, res, args0, loc); // compile res := f (args0) } } else { - stack.o << AsmOp::Custom(f_sym->name + " CALLDICT", (int)right.size(), (int)left.size()); + stack.o << AsmOp::Custom(loc, f_sym->name + " CALLDICT", (int)right.size(), (int)left.size()); } stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); stack.push_new_var(left.at(0)); return true; } @@ -408,7 +411,7 @@ bool Op::generate_code_step(Stack& stack) { if (is_last) { stack.assign_var(--i, x); } else { - stack.do_copy_var(--i, x); + stack.do_copy_var(loc, --i, x); } } i = 0; @@ -428,15 +431,15 @@ bool Op::generate_code_step(Stack& stack) { for (var_idx_t x : right) { last.push_back(var_info[x] && var_info[x]->is_last()); } - stack.rearrange_top(right, std::move(last)); + stack.rearrange_top(loc, right, std::move(last)); stack.opt_show(); int k = (int)stack.depth() - (int)right.size(); tolk_assert(k >= 0); if (cl == _Tuple) { - stack.o << AsmOp::Tuple((int)right.size()); + stack.o << AsmOp::Tuple(loc, (int)right.size()); tolk_assert(left.size() == 1); } else { - stack.o << AsmOp::UnTuple((int)left.size()); + stack.o << AsmOp::UnTuple(loc, (int)left.size()); tolk_assert(right.size() == 1); } stack.s.resize(k); @@ -475,24 +478,21 @@ bool Op::generate_code_step(Stack& stack) { for (var_idx_t x : right1) { last.push_back(var_info[x] && var_info[x]->is_last()); } - stack.rearrange_top(right1, std::move(last)); + stack.rearrange_top(loc, right1, std::move(last)); stack.opt_show(); int k = (int)stack.depth() - (int)right1.size(); tolk_assert(k >= 0); for (int i = 0; i < (int)right1.size(); i++) { - if (stack.s[k + i].first != right1[i]) { - std::cerr << stack.o; - } tolk_assert(stack.s[k + i].first == right1[i]); } auto exec_callxargs = [&](int args, int ret) { if (args <= 15 && ret <= 15) { - stack.o << exec_arg2_op("CALLXARGS", args, ret, args + 1, ret); + stack.o << exec_arg2_op(loc, "CALLXARGS", args, ret, args + 1, ret); } else { tolk_assert(args <= 254 && ret <= 254); - stack.o << AsmOp::Const(PSTRING() << args << " PUSHINT"); - stack.o << AsmOp::Const(PSTRING() << ret << " PUSHINT"); - stack.o << AsmOp::Custom("CALLXVARARGS", args + 3, ret); + stack.o << AsmOp::Const(loc, PSTRING() << args << " PUSHINT"); + stack.o << AsmOp::Const(loc, PSTRING() << ret << " PUSHINT"); + stack.o << AsmOp::Custom(loc, "CALLXVARARGS", args + 3, ret); } }; if (cl == _CallInd) { @@ -504,20 +504,21 @@ bool Op::generate_code_step(Stack& stack) { res.emplace_back(i); } if (f_sym->is_asm_function()) { - std::get(f_sym->body)->compile(stack.o); // compile res := f (args) + std::get(f_sym->body)->compile(stack.o, loc); // compile res := f (args) } else { - std::get(f_sym->body)->compile(stack.o, res, args, where); // compile res := f (args) + std::get(f_sym->body)->compile(stack.o, res, args, loc); // compile res := f (args) } } else { if (f_sym->is_inline() || f_sym->is_inline_ref()) { - stack.o << AsmOp::Custom(f_sym->name + " INLINECALLDICT", (int)right.size(), (int)left.size()); + stack.o << AsmOp::Custom(loc, f_sym->name + " INLINECALLDICT", (int)right.size(), (int)left.size()); } else if (f_sym->is_code_function() && std::get(f_sym->body)->code->require_callxargs) { - stack.o << AsmOp::Custom(f_sym->name + (" PREPAREDICT"), 0, 2); + stack.o << AsmOp::Custom(loc, f_sym->name + (" PREPAREDICT"), 0, 2); exec_callxargs((int)right.size() + 1, (int)left.size()); } else { - stack.o << AsmOp::Custom(f_sym->name + " CALLDICT", (int)right.size(), (int)left.size()); + stack.o << AsmOp::Custom(loc, f_sym->name + " CALLDICT", (int)right.size(), (int)left.size()); } } + stack.modified(); stack.s.resize(k); for (int i = 0; i < (int)left.size(); i++) { int j = ret_order ? ret_order->at(i) : i; @@ -531,21 +532,19 @@ bool Op::generate_code_step(Stack& stack) { for (var_idx_t x : right) { last.push_back(var_info[x] && var_info[x]->is_last()); } - stack.rearrange_top(right, std::move(last)); + stack.rearrange_top(loc, right, std::move(last)); stack.opt_show(); int k = (int)stack.depth() - (int)right.size(); tolk_assert(k >= 0); for (int i = 0; i < (int)right.size(); i++) { - if (stack.s[k + i].first != right[i]) { - std::cerr << stack.o; - } tolk_assert(stack.s[k + i].first == right[i]); } if (right.size() > 1) { - stack.o << AsmOp::Tuple((int)right.size()); + stack.o << AsmOp::Tuple(loc, (int)right.size()); } if (!right.empty()) { - stack.o << AsmOp::Custom(g_sym->name + " SETGLOB", 1, 0); + stack.o << AsmOp::Custom(loc, g_sym->name + " SETGLOB", 1, 0); + stack.modified(); } stack.s.resize(k); return true; @@ -558,7 +557,7 @@ bool Op::generate_code_step(Stack& stack) { stack.o.retalt_ = true; } var_idx_t x = left[0]; - stack.rearrange_top(x, var_info[x] && var_info[x]->is_last()); + stack.rearrange_top(loc, x, var_info[x] && var_info[x]->is_last()); tolk_assert(stack[0] == x); stack.opt_show(); stack.s.pop_back(); @@ -568,19 +567,19 @@ bool Op::generate_code_step(Stack& stack) { Op* block_noreturn = is0 ? block0.get() : block1.get(); Op* block_other = is0 ? block1.get() : block0.get(); stack.mode &= ~Stack::_InlineFunc; - stack.o << (is0 ? "IF:<{" : "IFNOT:<{"); + stack.o << AsmOp::Custom(loc, is0 ? "IF:<{" : "IFNOT:<{"); stack.o.indent(); Stack stack_copy{stack}; block_noreturn->generate_code_all(stack_copy); stack.o.undent(); - stack.o << "}>ELSE<{"; + stack.o << AsmOp::Custom({}, "}>ELSE<{"); stack.o.indent(); block_other->generate_code_all(stack); if (!block_other->noreturn()) { next->generate_code_all(stack); } stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); return false; } if (block1->is_empty() || block0->is_empty()) { @@ -589,88 +588,88 @@ bool Op::generate_code_step(Stack& stack) { // if (left) block0; ... // if (!left) block1; ... if (block->noreturn()) { - stack.o << (is0 ? "IFJMP:<{" : "IFNOTJMP:<{"); + stack.o << AsmOp::Custom(loc, is0 ? "IFJMP:<{" : "IFNOTJMP:<{"); stack.o.indent(); Stack stack_copy{stack}; stack_copy.mode &= ~Stack::_InlineFunc; stack_copy.mode |= next->noreturn() ? 0 : Stack::_NeedRetAlt; block->generate_code_all(stack_copy); stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); return true; } - stack.o << (is0 ? "IF:<{" : "IFNOT:<{"); + stack.o << AsmOp::Custom(loc, is0 ? "IF:<{" : "IFNOT:<{"); stack.o.indent(); Stack stack_copy{stack}, stack_target{stack}; stack_target.disable_output(); - stack_target.drop_vars_except(next->var_info); + stack_target.drop_vars_except(loc, next->var_info); stack_copy.mode &= ~Stack::_InlineFunc; block->generate_code_all(stack_copy); - stack_copy.drop_vars_except(var_info); + stack_copy.drop_vars_except(loc, var_info); stack_copy.opt_show(); if ((is0 && stack_copy == stack) || (!is0 && stack_copy.vars() == stack.vars())) { stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); if (!is0) { stack.merge_const(stack_copy); } return true; } // stack_copy.drop_vars_except(next->var_info); - stack_copy.enforce_state(stack_target.vars()); + stack_copy.enforce_state(loc, stack_target.vars()); stack_copy.opt_show(); if (stack_copy.vars() == stack.vars()) { stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); stack.merge_const(stack_copy); return true; } stack.o.undent(); - stack.o << "}>ELSE<{"; + stack.o << AsmOp::Custom({}, "}>ELSE<{"); stack.o.indent(); - stack.merge_state(stack_copy); + stack.merge_state(loc, stack_copy); stack.opt_show(); stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); return true; } if (block0->noreturn() || block1->noreturn()) { bool is0 = block0->noreturn(); Op* block_noreturn = is0 ? block0.get() : block1.get(); Op* block_other = is0 ? block1.get() : block0.get(); - stack.o << (is0 ? "IFJMP:<{" : "IFNOTJMP:<{"); + stack.o << AsmOp::Custom(loc, is0 ? "IFJMP:<{" : "IFNOTJMP:<{"); stack.o.indent(); Stack stack_copy{stack}; stack_copy.mode &= ~Stack::_InlineFunc; stack_copy.mode |= (block_other->noreturn() || next->noreturn()) ? 0 : Stack::_NeedRetAlt; block_noreturn->generate_code_all(stack_copy); stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); block_other->generate_code_all(stack); return !block_other->noreturn(); } - stack.o << "IF:<{"; + stack.o << AsmOp::Custom(loc, "IF:<{"); stack.o.indent(); Stack stack_copy{stack}; stack_copy.mode &= ~Stack::_InlineFunc; block0->generate_code_all(stack_copy); - stack_copy.drop_vars_except(next->var_info); + stack_copy.drop_vars_except(loc, next->var_info); stack_copy.opt_show(); stack.o.undent(); - stack.o << "}>ELSE<{"; + stack.o << AsmOp::Custom({}, "}>ELSE<{"); stack.o.indent(); stack.mode &= ~Stack::_InlineFunc; block1->generate_code_all(stack); - stack.merge_state(stack_copy); + stack.merge_state(loc, stack_copy); stack.opt_show(); stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); return true; } case _Repeat: { var_idx_t x = left[0]; //stack.drop_vars_except(block0->var_info, x); - stack.rearrange_top(x, var_info[x] && var_info[x]->is_last()); + stack.rearrange_top(loc, x, var_info[x] && var_info[x]->is_last()); tolk_assert(stack[0] == x); stack.opt_show(); stack.s.pop_back(); @@ -679,7 +678,7 @@ bool Op::generate_code_step(Stack& stack) { stack.o.retalt_ = true; } if (true || !next->is_empty()) { - stack.o << "REPEAT:<{"; + stack.o << AsmOp::Custom(loc, "REPEAT:<{"); stack.o.indent(); stack.forget_const(); if (block0->noreturn()) { @@ -693,47 +692,47 @@ bool Op::generate_code_step(Stack& stack) { stack.mode &= ~Stack::_InlineFunc; stack.mode |= Stack::_NeedRetAlt; block0->generate_code_all(stack); - stack.enforce_state(std::move(layout1)); + stack.enforce_state(loc, std::move(layout1)); stack.opt_show(); } stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); return true; } else { - stack.o << "REPEATEND"; + stack.o << AsmOp::Custom(loc, "REPEATEND"); stack.forget_const(); StackLayout layout1 = stack.vars(); block0->generate_code_all(stack); - stack.enforce_state(std::move(layout1)); + stack.enforce_state(loc, std::move(layout1)); stack.opt_show(); return false; } } case _Again: { - stack.drop_vars_except(block0->var_info); + stack.drop_vars_except(loc, block0->var_info); stack.opt_show(); if (block0->noreturn()) { stack.o.retalt_ = true; } if (!next->is_empty() || inline_func) { - stack.o << "AGAIN:<{"; + stack.o << AsmOp::Custom(loc, "AGAIN:<{"); stack.o.indent(); stack.forget_const(); StackLayout layout1 = stack.vars(); stack.mode &= ~Stack::_InlineFunc; stack.mode |= Stack::_NeedRetAlt; block0->generate_code_all(stack); - stack.enforce_state(std::move(layout1)); + stack.enforce_state(loc, std::move(layout1)); stack.opt_show(); stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); return true; } else { - stack.o << "AGAINEND"; + stack.o << AsmOp::Custom(loc, "AGAINEND"); stack.forget_const(); StackLayout layout1 = stack.vars(); block0->generate_code_all(stack); - stack.enforce_state(std::move(layout1)); + stack.enforce_state(loc, std::move(layout1)); stack.opt_show(); return false; } @@ -745,7 +744,7 @@ bool Op::generate_code_step(Stack& stack) { stack.o.retalt_ = true; } if (true || !next->is_empty()) { - stack.o << "UNTIL:<{"; + stack.o << AsmOp::Custom(loc, "UNTIL:<{"); stack.o.indent(); stack.forget_const(); auto layout1 = stack.vars(); @@ -753,20 +752,20 @@ bool Op::generate_code_step(Stack& stack) { stack.mode |= Stack::_NeedRetAlt; block0->generate_code_all(stack); layout1.push_back(left[0]); - stack.enforce_state(std::move(layout1)); + stack.enforce_state(loc, std::move(layout1)); stack.opt_show(); stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); stack.s.pop_back(); stack.modified(); return true; } else { - stack.o << "UNTILEND"; + stack.o << AsmOp::Custom(loc, "UNTILEND"); stack.forget_const(); StackLayout layout1 = stack.vars(); block0->generate_code_all(stack); layout1.push_back(left[0]); - stack.enforce_state(std::move(layout1)); + stack.enforce_state(loc, std::move(layout1)); stack.opt_show(); return false; } @@ -774,36 +773,36 @@ bool Op::generate_code_step(Stack& stack) { case _While: { // while (block0 | left) block1; ...next var_idx_t x = left[0]; - stack.drop_vars_except(block0->var_info); + stack.drop_vars_except(loc, block0->var_info); stack.opt_show(); StackLayout layout1 = stack.vars(); bool next_empty = false && next->is_empty(); if (block0->noreturn()) { stack.o.retalt_ = true; } - stack.o << "WHILE:<{"; + stack.o << AsmOp::Custom(loc, "WHILE:<{"); stack.o.indent(); stack.forget_const(); stack.mode &= ~Stack::_InlineFunc; stack.mode |= Stack::_NeedRetAlt; block0->generate_code_all(stack); - stack.rearrange_top(x, !next->var_info[x] && !block1->var_info[x]); + stack.rearrange_top(loc, x, !next->var_info[x] && !block1->var_info[x]); stack.opt_show(); stack.s.pop_back(); stack.modified(); stack.o.undent(); Stack stack_copy{stack}; - stack.o << (next_empty ? "}>DO:" : "}>DO<{"); + stack.o << AsmOp::Custom(loc, next_empty ? "}>DO:" : "}>DO<{"); if (!next_empty) { stack.o.indent(); } stack_copy.opt_show(); block1->generate_code_all(stack_copy); - stack_copy.enforce_state(std::move(layout1)); + stack_copy.enforce_state(loc, std::move(layout1)); stack_copy.opt_show(); if (!next_empty) { stack.o.undent(); - stack.o << "}>"; + stack.o << AsmOp::Custom({}, "}>"); return true; } else { return false; @@ -816,7 +815,7 @@ bool Op::generate_code_step(Stack& stack) { if (block0->noreturn() || block1->noreturn()) { stack.o.retalt_ = true; } - Stack catch_stack{stack.o}; + Stack catch_stack{stack.o, 0}; std::vector catch_vars; std::vector catch_last; for (const VarDescr& var : block1->var_info.list) { @@ -834,37 +833,37 @@ bool Op::generate_code_step(Stack& stack) { } catch_stack.push_new_var(left[0]); catch_stack.push_new_var(left[1]); - stack.rearrange_top(catch_vars, catch_last); + stack.rearrange_top(loc, catch_vars, catch_last); stack.opt_show(); - stack.o << "c1 PUSH"; - stack.o << "c3 PUSH"; - stack.o << "c4 PUSH"; - stack.o << "c5 PUSH"; - stack.o << "c7 PUSH"; - stack.o << "<{"; + stack.o << AsmOp::Custom(loc, "c1 PUSH"); + stack.o << AsmOp::Custom(loc, "c3 PUSH"); + stack.o << AsmOp::Custom(loc, "c4 PUSH"); + stack.o << AsmOp::Custom(loc, "c5 PUSH"); + stack.o << AsmOp::Custom(loc, "c7 PUSH"); + stack.o << AsmOp::Custom(loc, "<{"); stack.o.indent(); if (block1->noreturn()) { catch_stack.mode |= Stack::_NeedRetAlt; } block1->generate_code_all(catch_stack); - catch_stack.drop_vars_except(next->var_info); + catch_stack.drop_vars_except(loc, next->var_info); catch_stack.opt_show(); stack.o.undent(); - stack.o << "}>CONT"; - stack.o << "c7 SETCONT"; - stack.o << "c5 SETCONT"; - stack.o << "c4 SETCONT"; - stack.o << "c3 SETCONT"; - stack.o << "c1 SETCONT"; + stack.o << AsmOp::Custom({}, "}>CONT"); + stack.o << AsmOp::Custom(loc, "c7 SETCONT"); + stack.o << AsmOp::Custom(loc, "c5 SETCONT"); + stack.o << AsmOp::Custom(loc, "c4 SETCONT"); + stack.o << AsmOp::Custom(loc, "c3 SETCONT"); + stack.o << AsmOp::Custom(loc, "c1 SETCONT"); for (size_t begin = catch_vars.size(), end = begin; end > 0; end = begin) { begin = end >= block_size ? end - block_size : 0; - stack.o << std::to_string(end - begin) + " PUSHINT"; - stack.o << "-1 PUSHINT"; - stack.o << "SETCONTVARARGS"; + stack.o << AsmOp::Custom(loc, std::to_string(end - begin) + " PUSHINT"); + stack.o << AsmOp::Custom(loc, "-1 PUSHINT"); + stack.o << AsmOp::Custom(loc, "SETCONTVARARGS"); } stack.s.erase(stack.s.end() - catch_vars.size(), stack.s.end()); stack.modified(); - stack.o << "<{"; + stack.o << AsmOp::Custom(loc, "<{"); stack.o.indent(); if (block0->noreturn()) { stack.mode |= Stack::_NeedRetAlt; @@ -873,20 +872,20 @@ bool Op::generate_code_step(Stack& stack) { if (block0->noreturn()) { stack.s = std::move(catch_stack.s); } else if (!block1->noreturn()) { - stack.merge_state(catch_stack); + stack.merge_state(loc, catch_stack); } stack.opt_show(); stack.o.undent(); - stack.o << "}>CONT"; - stack.o << "c1 PUSH"; - stack.o << "COMPOSALT"; - stack.o << "SWAP"; - stack.o << "TRY"; + stack.o << AsmOp::Custom({}, "}>CONT"); + stack.o << AsmOp::Custom(loc, "c1 PUSH"); + stack.o << AsmOp::Custom(loc, "COMPOSALT"); + stack.o << AsmOp::Custom(loc, "SWAP"); + stack.o << AsmOp::Custom(loc, "TRY"); return true; } default: std::cerr << "fatal: unknown operation \n"; - throw ParseError{where, "unknown operation in generate_code()"}; + throw ParseError(loc, "unknown operation in generate_code()"); } } @@ -899,20 +898,16 @@ void Op::generate_code_all(Stack& stack) { } } -void CodeBlob::generate_code(AsmOpList& out, int mode) { - Stack stack{out, mode}; +void CodeBlob::generate_code(std::ostream& os, int mode, int indent) const { + AsmOpList out_list(indent, &vars); + Stack stack{out_list, mode}; tolk_assert(ops && ops->cl == Op::_Import); - auto args = (int)ops->left.size(); + int n_import_width = static_cast(ops->left.size()); for (var_idx_t x : ops->left) { stack.push_new_var(x); } ops->generate_code_all(stack); - stack.apply_wrappers(require_callxargs && (mode & Stack::_InlineAny) ? args : -1); -} - -void CodeBlob::generate_code(std::ostream& os, int mode, int indent) { - AsmOpList out_list(indent, &vars); - generate_code(out_list, mode); + stack.apply_wrappers(fun_ref->loc, require_callxargs && (mode & Stack::_InlineAny) ? n_import_width : -1); if (G.settings.optimization_level >= 2) { optimize_code(out_list); } diff --git a/tolk/compiler-state.h b/tolk/compiler-state.h index b39e62fb3..9b67293b5 100644 --- a/tolk/compiler-state.h +++ b/tolk/compiler-state.h @@ -52,6 +52,7 @@ struct CompilerSettings { int verbosity = 0; int optimization_level = 2; bool stack_layout_comments = true; + bool tolk_src_as_line_comments = true; std::string output_filename; std::string boc_output_filename; diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp index 76d756386..332069c03 100644 --- a/tolk/optimize.cpp +++ b/tolk/optimize.cpp @@ -125,13 +125,6 @@ void Optimizer::show_right() const { std::cerr << std::endl; } -bool Optimizer::say(std::string str) const { - if (debug_) { - std::cerr << str << std::endl; - } - return true; -} - bool Optimizer::find_const_op(int* op_idx, int cst) { for (int i = 0; i < l2_; i++) { if (op_[i]->is_gconst() && tr_[i].get(0) == cst) { @@ -157,7 +150,7 @@ bool Optimizer::rewrite_push_const(int i, int c) { show_left(); oq_[1] = std::move(op_[idx]); oq_[0] = std::move(op_[!idx]); - *oq_[0] = AsmOp::Push(i); + *oq_[0] = AsmOp::Push(oq_[0]->loc, i); show_right(); return true; } @@ -176,7 +169,7 @@ bool Optimizer::rewrite_const_rot(int c) { show_left(); oq_[0] = std::move(op_[idx]); oq_[1] = std::move(op_[!idx]); - *oq_[1] = AsmOp::Custom("ROT", 3, 3); + *oq_[1] = AsmOp::Custom(oq_[0]->loc, "ROT", 3, 3); show_right(); return true; } @@ -195,7 +188,7 @@ bool Optimizer::rewrite_const_pop(int c, int i) { show_left(); oq_[0] = std::move(op_[idx]); oq_[1] = std::move(op_[!idx]); - *oq_[1] = AsmOp::Pop(i); + *oq_[1] = AsmOp::Pop(oq_[0]->loc, i); show_right(); return true; } @@ -553,40 +546,41 @@ bool Optimizer::find_at_least(int pb) { pb_ = pb; // show_stack_transforms(); int i, j, k, l, c; + SrcLocation loc; // for asm ops inserted by optimizer, leave location empty (in fift output, it'll be attached to above) return (is_push_const(&i, &c) && rewrite_push_const(i, c)) || (is_nop() && rewrite_nop()) || (!(mode_ & 1) && is_const_rot(&c) && rewrite_const_rot(c)) || (is_const_push_xchgs() && rewrite_const_push_xchgs()) || (is_const_pop(&c, &i) && rewrite_const_pop(c, i)) || - (is_xchg(&i, &j) && rewrite(AsmOp::Xchg(i, j))) || (is_push(&i) && rewrite(AsmOp::Push(i))) || - (is_pop(&i) && rewrite(AsmOp::Pop(i))) || (is_pop_pop(&i, &j) && rewrite(AsmOp::Pop(i), AsmOp::Pop(j))) || - (is_xchg_xchg(&i, &j, &k, &l) && rewrite(AsmOp::Xchg(i, j), AsmOp::Xchg(k, l))) || + (is_xchg(&i, &j) && rewrite(AsmOp::Xchg(loc, i, j))) || (is_push(&i) && rewrite(AsmOp::Push(loc, i))) || + (is_pop(&i) && rewrite(AsmOp::Pop(loc, i))) || (is_pop_pop(&i, &j) && rewrite(AsmOp::Pop(loc, i), AsmOp::Pop(loc, j))) || + (is_xchg_xchg(&i, &j, &k, &l) && rewrite(AsmOp::Xchg(loc, i, j), AsmOp::Xchg(loc, k, l))) || (!(mode_ & 1) && - ((is_rot() && rewrite(AsmOp::Custom("ROT", 3, 3))) || (is_rotrev() && rewrite(AsmOp::Custom("-ROT", 3, 3))) || - (is_2dup() && rewrite(AsmOp::Custom("2DUP", 2, 4))) || - (is_2swap() && rewrite(AsmOp::Custom("2SWAP", 2, 4))) || - (is_2over() && rewrite(AsmOp::Custom("2OVER", 2, 4))) || - (is_tuck() && rewrite(AsmOp::Custom("TUCK", 2, 3))) || - (is_2drop() && rewrite(AsmOp::Custom("2DROP", 2, 0))) || (is_xchg2(&i, &j) && rewrite(AsmOp::Xchg2(i, j))) || - (is_xcpu(&i, &j) && rewrite(AsmOp::XcPu(i, j))) || (is_puxc(&i, &j) && rewrite(AsmOp::PuXc(i, j))) || - (is_push2(&i, &j) && rewrite(AsmOp::Push2(i, j))) || (is_blkswap(&i, &j) && rewrite(AsmOp::BlkSwap(i, j))) || - (is_blkpush(&i, &j) && rewrite(AsmOp::BlkPush(i, j))) || (is_blkdrop(&i) && rewrite(AsmOp::BlkDrop(i))) || - (is_push_rot(&i) && rewrite(AsmOp::Push(i), AsmOp::Custom("ROT"))) || - (is_push_rotrev(&i) && rewrite(AsmOp::Push(i), AsmOp::Custom("-ROT"))) || - (is_push_xchg(&i, &j, &k) && rewrite(AsmOp::Push(i), AsmOp::Xchg(j, k))) || - (is_reverse(&i, &j) && rewrite(AsmOp::BlkReverse(i, j))) || - (is_blkdrop2(&i, &j) && rewrite(AsmOp::BlkDrop2(i, j))) || - (is_nip_seq(&i, &j) && rewrite(AsmOp::Xchg(i, j), AsmOp::BlkDrop(i))) || - (is_pop_blkdrop(&i, &k) && rewrite(AsmOp::Pop(i), AsmOp::BlkDrop(k))) || + ((is_rot() && rewrite(AsmOp::Custom(loc, "ROT", 3, 3))) || (is_rotrev() && rewrite(AsmOp::Custom(loc, "-ROT", 3, 3))) || + (is_2dup() && rewrite(AsmOp::Custom(loc, "2DUP", 2, 4))) || + (is_2swap() && rewrite(AsmOp::Custom(loc, "2SWAP", 2, 4))) || + (is_2over() && rewrite(AsmOp::Custom(loc, "2OVER", 2, 4))) || + (is_tuck() && rewrite(AsmOp::Custom(loc, "TUCK", 2, 3))) || + (is_2drop() && rewrite(AsmOp::Custom(loc, "2DROP", 2, 0))) || (is_xchg2(&i, &j) && rewrite(AsmOp::Xchg2(loc, i, j))) || + (is_xcpu(&i, &j) && rewrite(AsmOp::XcPu(loc, i, j))) || (is_puxc(&i, &j) && rewrite(AsmOp::PuXc(loc, i, j))) || + (is_push2(&i, &j) && rewrite(AsmOp::Push2(loc, i, j))) || (is_blkswap(&i, &j) && rewrite(AsmOp::BlkSwap(loc, i, j))) || + (is_blkpush(&i, &j) && rewrite(AsmOp::BlkPush(loc, i, j))) || (is_blkdrop(&i) && rewrite(AsmOp::BlkDrop(loc, i))) || + (is_push_rot(&i) && rewrite(AsmOp::Push(loc, i), AsmOp::Custom(loc, "ROT"))) || + (is_push_rotrev(&i) && rewrite(AsmOp::Push(loc, i), AsmOp::Custom(loc, "-ROT"))) || + (is_push_xchg(&i, &j, &k) && rewrite(AsmOp::Push(loc, i), AsmOp::Xchg(loc, j, k))) || + (is_reverse(&i, &j) && rewrite(AsmOp::BlkReverse(loc, i, j))) || + (is_blkdrop2(&i, &j) && rewrite(AsmOp::BlkDrop2(loc, i, j))) || + (is_nip_seq(&i, &j) && rewrite(AsmOp::Xchg(loc, i, j), AsmOp::BlkDrop(loc, i))) || + (is_pop_blkdrop(&i, &k) && rewrite(AsmOp::Pop(loc, i), AsmOp::BlkDrop(loc, k))) || (is_2pop_blkdrop(&i, &j, &k) && (k >= 3 && k <= 13 && i != j + 1 && i <= 15 && j <= 14 - ? rewrite(AsmOp::Xchg2(j + 1, i), AsmOp::BlkDrop(k + 2)) - : rewrite(AsmOp::Pop(i), AsmOp::Pop(j), AsmOp::BlkDrop(k)))) || - (is_xchg3(&i, &j, &k) && rewrite(AsmOp::Xchg3(i, j, k))) || - (is_xc2pu(&i, &j, &k) && rewrite(AsmOp::Xc2Pu(i, j, k))) || - (is_xcpuxc(&i, &j, &k) && rewrite(AsmOp::XcPuXc(i, j, k))) || - (is_xcpu2(&i, &j, &k) && rewrite(AsmOp::XcPu2(i, j, k))) || - (is_puxc2(&i, &j, &k) && rewrite(AsmOp::PuXc2(i, j, k))) || - (is_puxcpu(&i, &j, &k) && rewrite(AsmOp::PuXcPu(i, j, k))) || - (is_pu2xc(&i, &j, &k) && rewrite(AsmOp::Pu2Xc(i, j, k))) || - (is_push3(&i, &j, &k) && rewrite(AsmOp::Push3(i, j, k))))); + ? rewrite(AsmOp::Xchg2(loc, j + 1, i), AsmOp::BlkDrop(loc, k + 2)) + : rewrite(AsmOp::Pop(loc, i), AsmOp::Pop(loc, j), AsmOp::BlkDrop(loc, k)))) || + (is_xchg3(&i, &j, &k) && rewrite(AsmOp::Xchg3(loc, i, j, k))) || + (is_xc2pu(&i, &j, &k) && rewrite(AsmOp::Xc2Pu(loc, i, j, k))) || + (is_xcpuxc(&i, &j, &k) && rewrite(AsmOp::XcPuXc(loc, i, j, k))) || + (is_xcpu2(&i, &j, &k) && rewrite(AsmOp::XcPu2(loc, i, j, k))) || + (is_puxc2(&i, &j, &k) && rewrite(AsmOp::PuXc2(loc, i, j, k))) || + (is_puxcpu(&i, &j, &k) && rewrite(AsmOp::PuXcPu(loc, i, j, k))) || + (is_pu2xc(&i, &j, &k) && rewrite(AsmOp::Pu2Xc(loc, i, j, k))) || + (is_push3(&i, &j, &k) && rewrite(AsmOp::Push3(loc, i, j, k))))); } bool Optimizer::find() { diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index cefdb5b8d..53447277b 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -17,6 +17,7 @@ #include "tolk.h" #include "src-file.h" #include "ast.h" +#include "ast-aux-data.h" #include "ast-visitor.h" #include "type-system.h" #include "common/refint.h" @@ -883,7 +884,9 @@ std::vector pre_compile_symbol(SrcLocation loc, const Symbol* sym, Co } if (GlobalConstPtr const_ref = sym->try_as()) { tolk_assert(lval_ctx == nullptr); - return pre_compile_expr(const_ref->init_value, code, const_ref->declared_type); + ASTAuxData* aux_data = new AuxData_ForceFiftLocation(loc); + auto v_force_loc = createV(loc, const_ref->init_value, aux_data, const_ref->inferred_type); + return pre_compile_expr(v_force_loc, code, const_ref->declared_type); } if (FunctionPtr fun_ref = sym->try_as()) { std::vector rvect = code.create_tmp_var(fun_ref->inferred_full_type, loc, "(glob-var-fun)"); @@ -1327,6 +1330,7 @@ static std::vector process_object_literal(V v, Co // an object (an instance of a struct) is actually a tensor at low-level // for example, `struct User { id: int; name: slice; }` occupies 2 slots // fields of a tensor are placed in order of declaration (in a literal they might be shuffled) + SrcLocation prev_loc = v->loc; std::vector tensor_items; std::vector target_types; tensor_items.reserve(v->struct_ref->get_num_fields()); @@ -1342,10 +1346,13 @@ static std::vector process_object_literal(V v, Co } if (!v_init_val) { tolk_assert(field_ref->has_default_value()); - v_init_val = field_ref->default_value; // it may be a complex expression, but it's constant, checked earlier + ASTAuxData *aux_data = new AuxData_ForceFiftLocation(prev_loc); + auto v_force_loc = createV(v->loc, field_ref->default_value, aux_data, field_ref->declared_type); + v_init_val = v_force_loc; // it may be a complex expression, but it's a constant, checked earlier } tensor_items.push_back(v_init_val); target_types.push_back(field_ref->declared_type); + prev_loc = v_init_val->loc; } const auto* tensor_target_type = TypeDataTensor::create(std::move(target_types))->try_as(); std::vector rvect = pre_compile_tensor(code, tensor_items, lval_ctx, tensor_target_type); @@ -1406,6 +1413,18 @@ static std::vector process_empty_expression(V v return transition_to_target_type(std::move(empty_rvect), code, target_type, v); } +static std::vector process_artificial_aux_vertex(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + if (const auto* data = dynamic_cast(v->aux_data)) { + SrcLocation backup = code.forced_loc; + code.forced_loc = data->forced_loc; + std::vector rvect = pre_compile_expr(v->get_wrapped_expr(), code, target_type, lval_ctx); + code.forced_loc = backup; + return rvect; + } + + tolk_assert(false); +} + std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { switch (v->kind) { case ast_reference: @@ -1458,6 +1477,8 @@ std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr targ return process_underscore(v->as(), code); case ast_empty_expression: return process_empty_expression(v->as(), code, target_type); + case ast_artificial_aux_vertex: + return process_artificial_aux_vertex(v->as(), code, target_type, lval_ctx); default: throw UnexpectedASTNodeKind(v, "pre_compile_expr"); } @@ -1681,7 +1702,7 @@ void process_any_statement(AnyV v, CodeBlob& code) { static void convert_function_body_to_CodeBlob(FunctionPtr fun_ref, FunctionBodyCode* code_body) { auto v_body = fun_ref->ast_root->as()->get_body()->as(); - CodeBlob* blob = new CodeBlob{fun_ref->name, fun_ref->loc, fun_ref}; + CodeBlob* blob = new CodeBlob(fun_ref); std::vector rvect_import; int total_arg_width = 0; @@ -1720,7 +1741,7 @@ static void convert_asm_body_to_AsmOp(FunctionPtr fun_ref, FunctionBodyAsm* asm_ for (char c : ops) { if (c == '\n' || c == '\r') { if (!op.empty()) { - asm_ops.push_back(AsmOp::Parse(op, cnt, width)); + asm_ops.push_back(AsmOp::Parse({}, op, cnt, width)); if (asm_ops.back().is_custom()) { cnt = width; } @@ -1731,7 +1752,7 @@ static void convert_asm_body_to_AsmOp(FunctionPtr fun_ref, FunctionBodyAsm* asm_ } } if (!op.empty()) { - asm_ops.push_back(AsmOp::Parse(op, cnt, width)); + asm_ops.push_back(AsmOp::Parse({}, op, cnt, width)); if (asm_ops.back().is_custom()) { cnt = width; } diff --git a/tolk/pipe-generate-fif-output.cpp b/tolk/pipe-generate-fif-output.cpp index 57f481f07..b40ce6860 100644 --- a/tolk/pipe-generate-fif-output.cpp +++ b/tolk/pipe-generate-fif-output.cpp @@ -84,10 +84,24 @@ static void generate_output_func(FunctionPtr fun_ref) { } else if (fun_ref->is_inline_ref()) { modifier = "REF"; } - std::cout << std::string(2, ' ') << fun_ref->name << " PROC" << modifier << ":<{\n"; + if (G.settings.tolk_src_as_line_comments) { + std::cout << " // " << fun_ref->loc << std::endl; + } + std::cout << " " << fun_ref->name << " PROC" << modifier << ":<{"; int mode = 0; if (G.settings.stack_layout_comments) { - mode |= Stack::_StkCmt | Stack::_CptStkCmt; + mode |= Stack::_StackComments; + size_t len = 2 + fun_ref->name.size() + 5 + std::strlen(modifier) + 3; + while (len < 28) { // a bit weird, but okay for now: + std::cout << ' '; // insert space after "xxx PROC" before `// stack state` + len++; // (the first AsmOp-comment that will be code generated) + } // space is the same as used to align comments in asmops.cpp + std::cout << '\t'; + } else { + std::cout << std::endl; + } + if (G.settings.tolk_src_as_line_comments) { + mode |= Stack::_LineComments; } if (fun_ref->is_inline() && code->ops->noreturn()) { mode |= Stack::_InlineFunc; @@ -96,7 +110,7 @@ static void generate_output_func(FunctionPtr fun_ref) { mode |= Stack::_InlineAny; } code->generate_code(std::cout, mode, 2); - std::cout << std::string(2, ' ') << "}>\n"; + std::cout << " " << "}>\n"; if (G.is_verbosity(2)) { std::cerr << "--------------\n"; } @@ -131,7 +145,7 @@ void pipeline_generate_fif_output_to_std_cout() { has_main_procedure = true; } - std::cout << std::string(2, ' '); + std::cout << " "; if (fun_ref->is_method_id_not_empty()) { std::cout << fun_ref->method_id << " DECLMETHOD " << fun_ref->name << "\n"; } else { @@ -151,7 +165,7 @@ void pipeline_generate_fif_output_to_std_cout() { continue; } - std::cout << std::string(2, ' ') << "DECLGLOBVAR " << var_ref->name << "\n"; + std::cout << " " << "DECLGLOBVAR " << var_ref->name << "\n"; } for (FunctionPtr fun_ref : G.all_functions) { diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 667172034..d4cda33fb 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -86,7 +86,7 @@ struct NameAndScopeResolver { void close_scope([[maybe_unused]] SrcLocation loc) { // std::cerr << "close_scope " << scopes.size() << " at " << loc << std::endl; if (UNLIKELY(scopes.empty())) { - throw Fatal{"cannot close the outer scope"}; + throw Fatal("cannot close the outer scope"); } scopes.pop_back(); } diff --git a/tolk/src-file.cpp b/tolk/src-file.cpp index 1286c1f9c..d97608560 100644 --- a/tolk/src-file.cpp +++ b/tolk/src-file.cpp @@ -24,15 +24,6 @@ namespace tolk { static_assert(sizeof(SrcLocation) == 8); -const SrcFile* AllRegisteredSrcFiles::find_file(int file_id) const { - for (const SrcFile* file : all_src_files) { - if (file->file_id == file_id) { - return file; - } - } - return nullptr; -} - const SrcFile* AllRegisteredSrcFiles::find_file(const std::string& abs_filename) const { for (const SrcFile* file : all_src_files) { if (file->abs_filename == abs_filename) { @@ -64,7 +55,8 @@ const SrcFile* AllRegisteredSrcFiles::locate_and_register_source_file(const std: throw Fatal("Failed to read " + rel_filename + ": " + text.move_as_error().message().str()); } - SrcFile* created = new SrcFile(++last_registered_file_id, rel_filename, std::move(abs_filename), text.move_as_ok()); + int file_id = static_cast(all_src_files.size()); // SrcFile::file_id is the index in all files + SrcFile* created = new SrcFile(file_id, rel_filename, std::move(abs_filename), text.move_as_ok()); if (G.is_verbosity(1)) { std::cerr << "register file_id " << created->file_id << " " << created->abs_filename << std::endl; } @@ -73,6 +65,7 @@ const SrcFile* AllRegisteredSrcFiles::locate_and_register_source_file(const std: } SrcFile* AllRegisteredSrcFiles::get_next_unparsed_file() { + int last_registered_file_id = static_cast(all_src_files.size() - 1); if (last_parsed_file_id >= last_registered_file_id) { return nullptr; } @@ -93,6 +86,10 @@ SrcFile::SrcPosition SrcFile::convert_offset(int offset) const { return SrcPosition{offset, -1, -1, "invalid offset"}; } + // currently, converting offset to line number is O(N): just read file contents char by char and detect lines + // since original Tolk src lines are now printed into Fift output, this is invoked for every asm instruction + // but anyway, it consumes a small amount of time relative to other work of the compiler + // in the future, it can be optimized by making lines index aside just std::string_view text int line_idx = 0; int char_idx = 0; int line_offset = 0; @@ -129,7 +126,7 @@ std::ostream& operator<<(std::ostream& os, const Fatal& fatal) { } const SrcFile* SrcLocation::get_src_file() const { - return G.all_src_files.find_file(file_id); + return G.all_src_files.get_file(file_id); } void SrcLocation::show(std::ostream& os) const { @@ -137,7 +134,10 @@ void SrcLocation::show(std::ostream& os) const { os << src_file; if (src_file && src_file->is_offset_valid(char_offset)) { SrcFile::SrcPosition pos = src_file->convert_offset(char_offset); - os << ':' << pos.line_no << ':' << pos.char_no; + os << ':' << pos.line_no; + if (pos.char_no != 1) { + os << ':' << pos.char_no; + } } } @@ -157,6 +157,37 @@ void SrcLocation::show_context(std::ostream& os) const { os << '^' << "\n"; } +// when generating Fift output, every block of asm instructions originated from the same Tolk line, +// is preceded by an original line as a comment +void SrcLocation::show_line_to_fif_output(std::ostream& os, int indent, int* last_line_no) const { + SrcFile::SrcPosition pos = G.all_src_files.get_file(file_id)->convert_offset(char_offset); + + // avoid duplicating one line multiple times in fift output + if (pos.line_no == *last_line_no) { + return; + } + *last_line_no = pos.line_no; + + // trim some characters from start and end to see `else if (x)` not `} else if (x) {` + std::string_view s = pos.line_str; + int b = 0, e = static_cast(s.size() - 1); + while (std::isspace(s[b]) || s[b] == '}') { + if (b < e) b++; + else break; + } + while (std::isspace(s[e]) || s[e] == '{' || s[e] == ';' || s[e] == ',') { + if (e > b) e--; + else break; + } + + if (b < e) { + for (int i = 0; i < indent * 2; ++i) { + os << ' '; + } + os << "// " << pos.line_no << ": " << s.substr(b, e - b + 1) << std::endl; + } +} + std::string SrcLocation::to_string() const { std::ostringstream os; show(os); diff --git a/tolk/src-file.h b/tolk/src-file.h index b0f9cba37..1f3eb030e 100644 --- a/tolk/src-file.h +++ b/tolk/src-file.h @@ -85,6 +85,7 @@ class SrcLocation { void show(std::ostream& os) const; void show_context(std::ostream& os) const; + void show_line_to_fif_output(std::ostream& os, int indent, int* last_line_no) const; std::string to_string() const; void show_general_error(std::ostream& os, const std::string& message, const std::string& err_type) const; @@ -97,11 +98,10 @@ std::ostream& operator<<(std::ostream& os, SrcLocation loc); class AllRegisteredSrcFiles { std::vector all_src_files; - int last_registered_file_id = -1; int last_parsed_file_id = -1; public: - const SrcFile* find_file(int file_id) const; + const SrcFile* get_file(int file_id) const { return all_src_files.at(file_id); } const SrcFile* find_file(const std::string& abs_filename) const; const SrcFile* locate_and_register_source_file(const std::string& rel_filename, SrcLocation included_from); diff --git a/tolk/tolk-main.cpp b/tolk/tolk-main.cpp index 7f939670b..4e9c1f4d8 100644 --- a/tolk/tolk-main.cpp +++ b/tolk/tolk-main.cpp @@ -50,6 +50,7 @@ void usage(const char* progname) { "-O\tSets optimization level (2 by default)\n" "-x\tEnables experimental options, comma-separated\n" "-S\tDon't include stack layout comments into Fift output\n" + "-L\tDon't include original lines from Tolk src into Fift output\n" "-e\tIncreases verbosity level (extra output into stderr)\n" "-v\tOutput version of Tolk and exit\n"; std::exit(2); @@ -214,7 +215,7 @@ class StdCoutRedirectToFile { int main(int argc, char* const argv[]) { int i; - while ((i = getopt(argc, argv, "o:b:O:x:Sevh")) != -1) { + while ((i = getopt(argc, argv, "o:b:O:x:SLevh")) != -1) { switch (i) { case 'o': G.settings.output_filename = optarg; @@ -231,6 +232,9 @@ int main(int argc, char* const argv[]) { case 'S': G.settings.stack_layout_comments = false; break; + case 'L': + G.settings.tolk_src_as_line_comments = false; + break; case 'e': G.settings.verbosity++; break; diff --git a/tolk/tolk-wasm.cpp b/tolk/tolk-wasm.cpp index e74589ce8..3c8cb87dd 100644 --- a/tolk/tolk-wasm.cpp +++ b/tolk/tolk-wasm.cpp @@ -40,12 +40,14 @@ static td::Result compile_internal(char *config_json) { TRY_RESULT(opt_level, td::get_json_object_int_field(config, "optimizationLevel", true, 2)); TRY_RESULT(stack_comments, td::get_json_object_bool_field(config, "withStackComments", true, false)); + TRY_RESULT(src_line_comments, td::get_json_object_bool_field(config, "withSrcLineComments", true, false)); TRY_RESULT(entrypoint_filename, td::get_json_object_string_field(config, "entrypointFileName", false)); TRY_RESULT(experimental_options, td::get_json_object_string_field(config, "experimentalOptions", true)); G.settings.verbosity = 0; G.settings.optimization_level = std::max(0, opt_level); G.settings.stack_layout_comments = stack_comments; + G.settings.tolk_src_as_line_comments = src_line_comments; if (!experimental_options.empty()) { G.settings.parse_experimental_options_cmd_arg(experimental_options.c_str()); } diff --git a/tolk/tolk.h b/tolk/tolk.h index 12bc6b694..9fb39dae2 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -267,7 +267,6 @@ struct Stack; struct Op { enum OpKind { - _Undef, _Nop, _Call, _CallInd, @@ -293,48 +292,48 @@ struct Op { std::unique_ptr next; FunctionPtr f_sym = nullptr; GlobalVarPtr g_sym = nullptr; - SrcLocation where; + SrcLocation loc; VarDescrList var_info; std::vector args; std::vector left, right; std::unique_ptr block0, block1; td::RefInt256 int_const; std::string str_const; - Op(SrcLocation _where = {}, OpKind _cl = _Undef) : cl(_cl), flags(0), f_sym(nullptr), where(_where) { + Op(SrcLocation loc, OpKind cl) : cl(cl), flags(0), loc(loc) { } - Op(SrcLocation _where, OpKind _cl, const std::vector& _left) - : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(_left) { + Op(SrcLocation loc, OpKind cl, const std::vector& left) + : cl(cl), flags(0), loc(loc), left(left) { } - Op(SrcLocation _where, OpKind _cl, std::vector&& _left) - : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(std::move(_left)) { + Op(SrcLocation loc, OpKind cl, std::vector&& left) + : cl(cl), flags(0), loc(loc), left(std::move(left)) { } - Op(SrcLocation _where, OpKind _cl, const std::vector& _left, td::RefInt256 _const) - : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(_left), int_const(_const) { + Op(SrcLocation loc, OpKind cl, const std::vector& left, td::RefInt256 int_const) + : cl(cl), flags(0), loc(loc), left(left), int_const(std::move(int_const)) { } - Op(SrcLocation _where, OpKind _cl, const std::vector& _left, std::string _const) - : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(_left), str_const(_const) { + Op(SrcLocation loc, OpKind cl, const std::vector& left, std::string str_const) + : cl(cl), flags(0), loc(loc), left(left), str_const(std::move(str_const)) { } - Op(SrcLocation _where, OpKind _cl, const std::vector& _left, const std::vector& _right) - : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(_left), right(_right) { + Op(SrcLocation loc, OpKind cl, const std::vector& left, const std::vector& right) + : cl(cl), flags(0), loc(loc), left(left), right(right) { } - Op(SrcLocation _where, OpKind _cl, std::vector&& _left, std::vector&& _right) - : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(std::move(_left)), right(std::move(_right)) { + Op(SrcLocation loc, OpKind cl, std::vector&& left, std::vector&& right) + : cl(cl), flags(0), loc(loc), left(std::move(left)), right(std::move(right)) { } - Op(SrcLocation _where, OpKind _cl, const std::vector& _left, const std::vector& _right, + Op(SrcLocation loc, OpKind cl, const std::vector& left, const std::vector& right, FunctionPtr _fun) - : cl(_cl), flags(0), f_sym(_fun), where(_where), left(_left), right(_right) { + : cl(cl), flags(0), f_sym(_fun), loc(loc), left(left), right(right) { } - Op(SrcLocation _where, OpKind _cl, std::vector&& _left, std::vector&& _right, - FunctionPtr _fun) - : cl(_cl), flags(0), f_sym(_fun), where(_where), left(std::move(_left)), right(std::move(_right)) { + Op(SrcLocation loc, OpKind cl, std::vector&& left, std::vector&& right, + FunctionPtr fun_ref) + : cl(cl), flags(0), f_sym(fun_ref), loc(loc), left(std::move(left)), right(std::move(right)) { } - Op(SrcLocation _where, OpKind _cl, const std::vector& _left, const std::vector& _right, - GlobalVarPtr _gvar) - : cl(_cl), flags(0), g_sym(_gvar), where(_where), left(_left), right(_right) { + Op(SrcLocation loc, OpKind cl, const std::vector& left, const std::vector& right, + GlobalVarPtr glob_ref) + : cl(cl), flags(0), g_sym(glob_ref), loc(loc), left(left), right(right) { } - Op(SrcLocation _where, OpKind _cl, std::vector&& _left, std::vector&& _right, + Op(SrcLocation loc, OpKind cl, std::vector&& left, std::vector&& right, GlobalVarPtr _gvar) - : cl(_cl), flags(0), g_sym(_gvar), where(_where), left(std::move(_left)), right(std::move(_right)) { + : cl(cl), flags(0), g_sym(_gvar), loc(loc), left(std::move(left)), right(std::move(right)) { } bool disabled() const { return flags & _Disabled; } @@ -408,45 +407,46 @@ typedef std::vector StackLayout; typedef std::pair var_const_idx_t; typedef std::vector StackLayoutExt; constexpr const_idx_t not_const = -1; -using Const = td::RefInt256; struct AsmOp { - enum Type { a_none, a_xchg, a_push, a_pop, a_const, a_custom, a_magic }; - Type t{a_none}; + enum Type { a_nop, a_comment, a_xchg, a_push, a_pop, a_const, a_custom }; + Type t; + SrcLocation loc; int indent{0}; int a, b; bool gconst{false}; std::string op; struct SReg { int idx; - SReg(int _idx) : idx(_idx) { + explicit SReg(int _idx) : idx(_idx) { } + int calc_out_strlen() const; }; AsmOp() = default; - AsmOp(Type _t) : t(_t) { + AsmOp(Type t, SrcLocation loc) : t(t), loc(loc) { } - AsmOp(Type _t, std::string _op) : t(_t), op(std::move(_op)) { + AsmOp(Type t, SrcLocation loc, std::string _op) : t(t), loc(loc), op(std::move(_op)) { } - AsmOp(Type _t, int _a) : t(_t), a(_a) { + AsmOp(Type t, SrcLocation loc, int a) : t(t), loc(loc), a(a) { } - AsmOp(Type _t, int _a, std::string _op) : t(_t), a(_a), op(std::move(_op)) { + AsmOp(Type t, SrcLocation loc, int a, std::string _op) : t(t), loc(loc), a(a), op(std::move(_op)) { } - AsmOp(Type _t, int _a, int _b) : t(_t), a(_a), b(_b) { + AsmOp(Type t, SrcLocation loc, int a, int b) : t(t), loc(loc), a(a), b(b) { } - AsmOp(Type _t, int _a, int _b, std::string _op) : t(_t), a(_a), b(_b), op(std::move(_op)) { + AsmOp(Type t, SrcLocation loc, int a, int b, std::string op) : t(t), loc(loc), a(a), b(b), op(std::move(op)) { compute_gconst(); } - void out(std::ostream& os) const; - void out_indent_nl(std::ostream& os, bool no_nl = false) const; + int out(std::ostream& os) const; + int out_indented(std::ostream& os, bool print_src_line_above) const; std::string to_string() const; void compute_gconst() { gconst = (is_custom() && (op == "PUSHNULL" || op == "NEWC" || op == "NEWB" || op == "TRUE" || op == "FALSE" || op == "NOW")); } bool is_nop() const { - return t == a_none && op.empty(); + return t == a_nop; } bool is_comment() const { - return t == a_none && !op.empty(); + return t == a_comment; } bool is_custom() const { return t == a_custom; @@ -493,80 +493,80 @@ struct AsmOp { bool is_gconst() const { return !a && b == 1 && (t == a_const || gconst); } - static AsmOp Nop() { - return AsmOp(a_none); + static AsmOp Nop(SrcLocation loc) { + return AsmOp(a_nop, loc); } - static AsmOp Xchg(int a, int b = 0) { - return a == b ? AsmOp(a_none) : (a < b ? AsmOp(a_xchg, a, b) : AsmOp(a_xchg, b, a)); + static AsmOp Xchg(SrcLocation loc, int a, int b = 0) { + return a == b ? AsmOp(a_nop, loc) : (a < b ? AsmOp(a_xchg, loc, a, b) : AsmOp(a_xchg, loc, b, a)); } - static AsmOp Push(int a) { - return AsmOp(a_push, a); + static AsmOp Push(SrcLocation loc, int a) { + return AsmOp(a_push, loc, a); } - static AsmOp Pop(int a = 0) { - return AsmOp(a_pop, a); + static AsmOp Pop(SrcLocation loc, int a) { + return AsmOp(a_pop, loc, a); } - static AsmOp Xchg2(int a, int b) { - return make_stk2(a, b, "XCHG2", 0); + static AsmOp Xchg2(SrcLocation loc, int a, int b) { + return make_stk2(loc, a, b, "XCHG2", 0); } - static AsmOp XcPu(int a, int b) { - return make_stk2(a, b, "XCPU", 1); + static AsmOp XcPu(SrcLocation loc, int a, int b) { + return make_stk2(loc, a, b, "XCPU", 1); } - static AsmOp PuXc(int a, int b) { - return make_stk2(a, b, "PUXC", 1); + static AsmOp PuXc(SrcLocation loc, int a, int b) { + return make_stk2(loc, a, b, "PUXC", 1); } - static AsmOp Push2(int a, int b) { - return make_stk2(a, b, "PUSH2", 2); + static AsmOp Push2(SrcLocation loc, int a, int b) { + return make_stk2(loc, a, b, "PUSH2", 2); } - static AsmOp Xchg3(int a, int b, int c) { - return make_stk3(a, b, c, "XCHG3", 0); + static AsmOp Xchg3(SrcLocation loc, int a, int b, int c) { + return make_stk3(loc, a, b, c, "XCHG3", 0); } - static AsmOp Xc2Pu(int a, int b, int c) { - return make_stk3(a, b, c, "XC2PU", 1); + static AsmOp Xc2Pu(SrcLocation loc, int a, int b, int c) { + return make_stk3(loc, a, b, c, "XC2PU", 1); } - static AsmOp XcPuXc(int a, int b, int c) { - return make_stk3(a, b, c, "XCPUXC", 1); + static AsmOp XcPuXc(SrcLocation loc, int a, int b, int c) { + return make_stk3(loc, a, b, c, "XCPUXC", 1); } - static AsmOp XcPu2(int a, int b, int c) { - return make_stk3(a, b, c, "XCPU2", 3); + static AsmOp XcPu2(SrcLocation loc, int a, int b, int c) { + return make_stk3(loc, a, b, c, "XCPU2", 3); } - static AsmOp PuXc2(int a, int b, int c) { - return make_stk3(a, b, c, "PUXC2", 3); + static AsmOp PuXc2(SrcLocation loc, int a, int b, int c) { + return make_stk3(loc, a, b, c, "PUXC2", 3); } - static AsmOp PuXcPu(int a, int b, int c) { - return make_stk3(a, b, c, "PUXCPU", 3); + static AsmOp PuXcPu(SrcLocation loc, int a, int b, int c) { + return make_stk3(loc, a, b, c, "PUXCPU", 3); } - static AsmOp Pu2Xc(int a, int b, int c) { - return make_stk3(a, b, c, "PU2XC", 3); + static AsmOp Pu2Xc(SrcLocation loc, int a, int b, int c) { + return make_stk3(loc, a, b, c, "PU2XC", 3); } - static AsmOp Push3(int a, int b, int c) { - return make_stk3(a, b, c, "PUSH3", 3); + static AsmOp Push3(SrcLocation loc, int a, int b, int c) { + return make_stk3(loc, a, b, c, "PUSH3", 3); } - static AsmOp BlkSwap(int a, int b); - static AsmOp BlkPush(int a, int b); - static AsmOp BlkDrop(int a); - static AsmOp BlkDrop2(int a, int b); - static AsmOp BlkReverse(int a, int b); - static AsmOp make_stk2(int a, int b, const char* str, int delta); - static AsmOp make_stk3(int a, int b, int c, const char* str, int delta); - static AsmOp IntConst(const td::RefInt256& x); - static AsmOp BoolConst(bool f); - static AsmOp Const(std::string push_op) { - return AsmOp(a_const, 0, 1, std::move(push_op)); + static AsmOp BlkSwap(SrcLocation loc, int a, int b); + static AsmOp BlkPush(SrcLocation loc, int a, int b); + static AsmOp BlkDrop(SrcLocation loc, int a); + static AsmOp BlkDrop2(SrcLocation loc, int a, int b); + static AsmOp BlkReverse(SrcLocation loc, int a, int b); + static AsmOp make_stk2(SrcLocation loc, int a, int b, const char* str, int delta); + static AsmOp make_stk3(SrcLocation loc, int a, int b, int c, const char* str, int delta); + static AsmOp IntConst(SrcLocation loc, const td::RefInt256& x); + static AsmOp BoolConst(SrcLocation loc, bool f); + static AsmOp Const(SrcLocation loc, std::string push_op) { + return AsmOp(a_const, loc, 0, 1, std::move(push_op)); } - static AsmOp Const(int arg, const std::string& push_op); - static AsmOp Comment(const std::string& comment) { - return AsmOp(a_none, std::string{"// "} + comment); + static AsmOp Const(SrcLocation loc, int arg, const std::string& push_op); + static AsmOp Comment(SrcLocation loc, const std::string& comment) { + return AsmOp(a_comment, loc, std::string{"// "} + comment); } - static AsmOp Custom(const std::string& custom_op) { - return AsmOp(a_custom, 255, 255, custom_op); + static AsmOp Custom(SrcLocation loc, const std::string& custom_op) { + return AsmOp(a_custom, loc, 255, 255, custom_op); } - static AsmOp Parse(const std::string& custom_op); - static AsmOp Custom(const std::string& custom_op, int args, int retv = 1) { - return AsmOp(a_custom, args, retv, custom_op); + static AsmOp Parse(SrcLocation loc, const std::string& custom_op); + static AsmOp Custom(SrcLocation loc, const std::string& custom_op, int args, int retv = 1) { + return AsmOp(a_custom, loc, args, retv, custom_op); } - static AsmOp Parse(std::string custom_op, int args, int retv = 1); - static AsmOp Tuple(int a); - static AsmOp UnTuple(int a); + static AsmOp Parse(SrcLocation loc, std::string custom_op, int args, int retv = 1); + static AsmOp Tuple(SrcLocation loc, int a); + static AsmOp UnTuple(SrcLocation loc, int a); }; inline std::ostream& operator<<(std::ostream& os, const AsmOp& op) { @@ -581,38 +581,19 @@ struct AsmOpList { std::vector list_; int indent_{0}; const std::vector* var_names_{nullptr}; - std::vector constants_; + std::vector constants_; bool retalt_{false}; bool retalt_inserted_{false}; void out(std::ostream& os, int mode = 0) const; AsmOpList(int indent = 0, const std::vector* var_names = nullptr) : indent_(indent), var_names_(var_names) { } - template - AsmOpList& add(Args&&... args) { - append(AsmOp(std::forward(args)...)); + AsmOpList& operator<<(AsmOp&& op) { + list_.emplace_back(op); adjust_last(); return *this; } - bool append(const AsmOp& op) { - list_.push_back(op); - adjust_last(); - return true; - } - bool append(const std::vector& ops); - bool append(std::initializer_list ops) { - return append(std::vector(std::move(ops))); - } - AsmOpList& operator<<(const AsmOp& op) { - return add(op); - } - AsmOpList& operator<<(AsmOp&& op) { - return add(std::move(op)); - } - AsmOpList& operator<<(std::string str) { - return add(AsmOp::Type::a_custom, 255, 255, str); - } - const_idx_t register_const(Const new_const); - Const get_const(const_idx_t idx); + const_idx_t register_const(td::RefInt256 new_const); + td::RefInt256 get_const(const_idx_t idx); void show_var_ext(std::ostream& os, std::pair idx_pair) const; void adjust_last() { if (list_.back().is_nop()) { @@ -627,11 +608,8 @@ struct AsmOpList { void undent() { --indent_; } - void set_indent(int new_indent) { - indent_ = new_indent; - } - void insert(size_t pos, std::string str) { - insert(pos, AsmOp(AsmOp::a_custom, 255, 255, str)); + void insert(size_t pos, SrcLocation loc, std::string str) { + insert(pos, AsmOp(AsmOp::a_custom, loc, 255, 255, std::move(str))); } void insert(size_t pos, const AsmOp& op) { auto ip = list_.begin() + pos; @@ -645,11 +623,6 @@ struct AsmOpList { } }; -inline std::ostream& operator<<(std::ostream& os, const AsmOpList& op_list) { - op_list.out(os); - return os; -} - struct AsmOpCons { std::unique_ptr car; std::unique_ptr cdr; @@ -884,7 +857,6 @@ struct Optimizer { bool find(); bool optimize(); bool compute_stack_transforms(); - bool say(std::string str) const; bool show_stack_transforms() const; void show_head() const; void show_left() const; @@ -960,17 +932,17 @@ struct Stack { StackLayoutExt s; AsmOpList& o; enum { - _StkCmt = 1, _CptStkCmt = 2, _DisableOut = 128, _Shown = 256, + _StackComments = 1, _LineComments = 2, _DisableOut = 128, _Shown = 256, _InlineFunc = 512, _NeedRetAlt = 1024, _InlineAny = 2048, _ModeSave = _InlineFunc | _NeedRetAlt | _InlineAny, _Garbage = -0x10000 }; int mode; - Stack(AsmOpList& _o, int _mode = 0) : o(_o), mode(_mode) { + Stack(AsmOpList& _o, int _mode) : o(_o), mode(_mode) { } - Stack(AsmOpList& _o, const StackLayoutExt& _s, int _mode = 0) : s(_s), o(_o), mode(_mode) { + Stack(AsmOpList& _o, const StackLayoutExt& _s, int _mode) : s(_s), o(_o), mode(_mode) { } - Stack(AsmOpList& _o, StackLayoutExt&& _s, int _mode = 0) : s(std::move(_s)), o(_o), mode(_mode) { + Stack(AsmOpList& _o, StackLayoutExt&& _s, int _mode) : s(std::move(_s)), o(_o), mode(_mode) { } int depth() const { return (int)s.size(); @@ -1007,55 +979,56 @@ struct Stack { void forget_const(); void validate(int i) const { if (i > 255) { - throw Fatal{"Too deep stack"}; + throw Fatal("Too deep stack"); } tolk_assert(i >= 0 && i < depth() && "invalid stack reference"); } void modified() { mode &= ~_Shown; } - void issue_pop(int i); - void issue_push(int i); - void issue_xchg(int i, int j); - int drop_vars_except(const VarDescrList& var_info, int excl_var = 0x80000000); + void issue_pop(SrcLocation loc, int i); + void issue_push(SrcLocation loc, int i); + void issue_xchg(SrcLocation loc, int i, int j); + int drop_vars_except(SrcLocation loc, const VarDescrList& var_info, int excl_var = 0x80000000); void forget_var(var_idx_t idx); void push_new_var(var_idx_t idx); void push_new_const(var_idx_t idx, const_idx_t cidx); void assign_var(var_idx_t new_idx, var_idx_t old_idx); - void do_copy_var(var_idx_t new_idx, var_idx_t old_idx); - void enforce_state(const StackLayout& req_stack); - void rearrange_top(const StackLayout& top, std::vector last); - void rearrange_top(var_idx_t top, bool last); + void do_copy_var(SrcLocation loc, var_idx_t new_idx, var_idx_t old_idx); + void enforce_state(SrcLocation loc, const StackLayout& req_stack); + void rearrange_top(SrcLocation loc, const StackLayout& top, std::vector last); + void rearrange_top(SrcLocation loc, var_idx_t top, bool last); void merge_const(const Stack& req_stack); - void merge_state(const Stack& req_stack); + void merge_state(SrcLocation loc, const Stack& req_stack); void show(); void opt_show() { - if ((mode & (_StkCmt | _Shown)) == _StkCmt) { + if ((mode & (_StackComments | _Shown)) == _StackComments) { show(); } } bool operator==(const Stack& y) const & { return s == y.s; } - void apply_wrappers(int callxargs_count) { + void apply_wrappers(SrcLocation loc, int callxargs_count) { + int pos0 = (mode & _StackComments && !o.list_.empty() && o.list_[0].is_comment()) ? 1 : 0; bool is_inline = mode & _InlineFunc; if (o.retalt_inserted_) { - o.insert(0, "SAMEALTSAVE"); - o.insert(0, "c2 SAVE"); + o.insert(pos0, loc, "SAMEALTSAVE"); + o.insert(pos0, loc, "c2 SAVE"); } if (callxargs_count != -1 || (is_inline && o.retalt_)) { o.indent_all(); - o.insert(0, "CONT:<{"); - o << "}>"; + o.insert(pos0, loc, "CONT:<{"); + o << AsmOp::Custom(loc, "}>"); if (callxargs_count != -1) { if (callxargs_count <= 15) { - o << AsmOp::Custom(PSTRING() << callxargs_count << " -1 CALLXARGS"); + o << AsmOp::Custom(loc, PSTRING() << callxargs_count << " -1 CALLXARGS"); } else { tolk_assert(callxargs_count <= 254); - o << AsmOp::Custom(PSTRING() << callxargs_count << " PUSHINT -1 PUSHINT CALLXVARARGS"); + o << AsmOp::Custom(loc, PSTRING() << callxargs_count << " PUSHINT -1 PUSHINT CALLXVARARGS"); } } else { - o << "EXECUTE"; + o << AsmOp::Custom(loc, "EXECUTE"); } } } @@ -1070,31 +1043,27 @@ struct Stack { typedef std::function&, std::vector&, SrcLocation)> simple_compile_func_t; -inline simple_compile_func_t make_simple_compile(AsmOp op) { - return [op](std::vector& out, std::vector& in, SrcLocation) -> AsmOp { return op; }; -} - struct FunctionBodyBuiltin { simple_compile_func_t simple_compile; explicit FunctionBodyBuiltin(simple_compile_func_t compile) : simple_compile(std::move(compile)) {} - void compile(AsmOpList& dest, std::vector& out, std::vector& in, SrcLocation where) const; + void compile(AsmOpList& dest, std::vector& out, std::vector& in, SrcLocation loc) const; }; struct FunctionBodyAsm { std::vector ops; void set_code(std::vector&& code); - void compile(AsmOpList& dest) const; + void compile(AsmOpList& dest, SrcLocation loc) const; }; struct CodeBlob { int var_cnt, in_var_cnt; FunctionPtr fun_ref; std::string name; - SrcLocation loc; + SrcLocation forced_loc; std::vector vars; std::unique_ptr ops; std::unique_ptr* cur_ops; @@ -1103,12 +1072,15 @@ struct CodeBlob { #endif std::stack*> cur_ops_stack; bool require_callxargs = false; - CodeBlob(std::string name, SrcLocation loc, FunctionPtr fun_ref) - : var_cnt(0), in_var_cnt(0), fun_ref(fun_ref), name(std::move(name)), loc(loc), cur_ops(&ops) { + explicit CodeBlob(FunctionPtr fun_ref) + : var_cnt(0), in_var_cnt(0), fun_ref(fun_ref), name(fun_ref->name), cur_ops(&ops) { } template Op& emplace_back(Args&&... args) { Op& res = *(*cur_ops = std::make_unique(args...)); + if (forced_loc.is_defined()) { + res.loc = forced_loc; + } cur_ops = &(res.next); #ifdef TOLK_DEBUG _vector_of_ops.push_back(&res); @@ -1146,17 +1118,14 @@ struct CodeBlob { void prune_unreachable_code(); void fwd_analyze(); void mark_noreturn(); - void generate_code(AsmOpList& out_list, int mode = 0); - void generate_code(std::ostream& os, int mode = 0, int indent = 0); + void generate_code(std::ostream& os, int mode = 0, int indent = 0) const; }; // defined in builtins.cpp -AsmOp exec_arg_op(std::string op, long long arg); -AsmOp exec_arg_op(std::string op, long long arg, int args, int retv = 1); -AsmOp exec_arg_op(std::string op, td::RefInt256 arg); -AsmOp exec_arg_op(std::string op, td::RefInt256 arg, int args, int retv = 1); -AsmOp exec_arg2_op(std::string op, long long imm1, long long imm2, int args, int retv = 1); -AsmOp push_const(td::RefInt256 x); +AsmOp exec_arg_op(SrcLocation loc, std::string op, long long arg, int args, int retv = 1); +AsmOp exec_arg_op(SrcLocation loc, std::string op, td::RefInt256 arg, int args, int retv = 1); +AsmOp exec_arg2_op(SrcLocation loc, std::string op, long long imm1, long long imm2, int args, int retv = 1); +AsmOp push_const(SrcLocation loc, td::RefInt256 x); void define_builtins(); From 578df21a28720cf26b2f8811c42479f1de3a5a96 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Wed, 19 Mar 2025 21:50:30 +0400 Subject: [PATCH 234/388] [Tolk] Implement methods for any types as extension functions In FunC, any function could be called `f(arg)` or `arg.f()`. You had to call `cell.cell_hash()` / `slice.slice_hash()`: these were two different global-scope functions. Now, Tolk, as other languages, separates functions from methods. It drops the ability to call a function via dot; you can only call a method (for an object type). > fun cell.hash(self): int ... > fun slice.hash(self): int ... With methods, stdlib functions became short: `t.size()` instead of `t.tupleSize()`, and so on. Methods can be declared for any type, including generics. Calling a method, the compiler does type matching to detect the only method from many equally named. This could be generalized to functions overloading some day. --- crypto/smartcont/tolk-stdlib/common.tolk | 325 ++++++++------- .../smartcont/tolk-stdlib/gas-payments.tolk | 18 +- crypto/smartcont/tolk-stdlib/lisp-lists.tolk | 8 +- crypto/smartcont/tolk-stdlib/tvm-dicts.tolk | 134 +++--- .../smartcont/tolk-stdlib/tvm-lowlevel.tolk | 2 +- .../tests/allow-post-modification.tolk | 10 +- tolk-tester/tests/asm-arg-order.tolk | 7 +- tolk-tester/tests/assignment-tests.tolk | 22 +- tolk-tester/tests/bytesN-tests.tolk | 8 +- tolk-tester/tests/cells-slices.tolk | 50 ++- tolk-tester/tests/constants-tests.tolk | 4 +- tolk-tester/tests/dicts-demo.tolk | 10 +- tolk-tester/tests/generics-1.tolk | 27 +- tolk-tester/tests/generics-2.tolk | 21 + tolk-tester/tests/generics-3.tolk | 2 +- tolk-tester/tests/if-else-tests.tolk | 2 +- tolk-tester/tests/indexed-access.tolk | 55 ++- tolk-tester/tests/inference-tests.tolk | 10 +- tolk-tester/tests/intN-tests.tolk | 23 +- .../tests/invalid-declaration/err-1074.tolk | 6 + .../tests/invalid-declaration/err-1140.tolk | 3 +- .../tests/invalid-declaration/err-1193.tolk | 7 + .../tests/invalid-declaration/err-1226.tolk | 14 + .../tests/invalid-declaration/err-1458.tolk | 13 + .../tests/invalid-declaration/err-1556.tolk | 7 + .../tests/invalid-declaration/err-1601.tolk | 8 - .../tests/invalid-declaration/err-1663.tolk | 11 + .../tests/invalid-declaration/err-1701.tolk | 12 +- .../tests/invalid-declaration/err-1790.tolk | 2 +- .../tests/invalid-declaration/err-1794.tolk | 4 +- .../tests/invalid-declaration/err-1912.tolk | 9 + .../tests/invalid-declaration/err-1973.tolk | 7 + .../tests/invalid-semantics/err-4057.tolk | 17 + .../tests/invalid-semantics/err-4064.tolk | 7 +- .../tests/invalid-semantics/err-4080.tolk | 4 +- .../tests/invalid-semantics/err-4083.tolk | 2 +- .../tests/invalid-semantics/err-4104.tolk | 18 +- .../tests/invalid-semantics/err-4115.tolk | 14 + .../tests/invalid-semantics/err-4127.tolk | 2 +- .../tests/invalid-semantics/err-4190.tolk | 2 +- .../tests/invalid-semantics/err-4195.tolk | 4 + .../tests/invalid-semantics/err-4256.tolk | 18 + .../tests/invalid-semantics/err-4266.tolk | 4 +- .../tests/invalid-semantics/err-4270.tolk | 2 +- .../tests/invalid-semantics/err-4291.tolk | 3 +- .../tests/invalid-semantics/err-4304.tolk | 2 +- .../tests/invalid-semantics/err-4319.tolk | 17 + .../tests/invalid-semantics/err-4385.tolk | 10 +- .../tests/invalid-semantics/err-4387.tolk | 2 +- .../tests/invalid-semantics/err-4413.tolk | 10 +- .../tests/invalid-semantics/err-4473.tolk | 2 +- .../tests/invalid-semantics/err-4480.tolk | 10 + .../tests/invalid-semantics/err-4506.tolk | 16 - .../tests/invalid-semantics/err-4510.tolk | 2 +- .../tests/invalid-semantics/err-4520.tolk | 2 +- .../tests/invalid-semantics/err-4542.tolk | 4 +- .../tests/invalid-semantics/err-4588.tolk | 17 + .../tests/invalid-semantics/err-4598.tolk | 16 + .../tests/invalid-semantics/err-4604.tolk | 2 +- .../tests/invalid-semantics/err-4610.tolk | 13 + .../tests/invalid-semantics/err-4682.tolk | 2 +- .../tests/invalid-semantics/err-4711.tolk | 22 - .../tests/invalid-semantics/err-4739.tolk | 12 + .../tests/invalid-semantics/err-4828.tolk | 12 - .../tests/invalid-semantics/err-4841.tolk | 12 - .../tests/invalid-semantics/err-4847.tolk | 16 + .../tests/invalid-semantics/err-4851.tolk | 2 +- .../tests/invalid-semantics/err-4884.tolk | 13 + .../tests/invalid-semantics/err-4899.tolk | 8 - .../tests/invalid-semantics/err-4941.tolk | 10 - .../tests/invalid-semantics/err-4951.tolk | 2 +- .../tests/invalid-semantics/err-4956.tolk | 2 +- .../tests/invalid-symbol/err-2188.tolk | 2 +- .../tests/invalid-symbol/err-2374.tolk | 4 +- .../tests/invalid-symbol/err-2619.tolk | 8 + .../tests/invalid-symbol/err-2788.tolk | 9 + .../tests/invalid-symbol/err-2914.tolk | 10 - .../tests/invalid-syntax/err-3407.tolk | 2 +- .../tests/invalid-syntax/err-3967.tolk | 2 +- .../tests/invalid-typing/err-6013.tolk | 18 + .../tests/invalid-typing/err-6080.tolk | 2 +- .../tests/invalid-typing/err-6085.tolk | 17 - .../tests/invalid-typing/err-6087.tolk | 2 +- .../tests/invalid-typing/err-6134.tolk | 4 +- .../tests/invalid-typing/err-6148.tolk | 2 +- .../tests/invalid-typing/err-6149.tolk | 18 + .../tests/invalid-typing/err-6204.tolk | 8 +- .../tests/invalid-typing/err-6249.tolk | 6 +- .../tests/invalid-typing/err-6257.tolk | 4 +- .../tests/invalid-typing/err-6281.tolk | 12 + .../tests/invalid-typing/err-6288.tolk | 2 +- .../tests/invalid-typing/err-6337.tolk | 13 - .../tests/invalid-typing/err-6386.tolk | 10 - .../tests/invalid-typing/err-6391.tolk | 2 +- .../tests/invalid-typing/err-6393.tolk | 3 + .../tests/invalid-typing/err-6407.tolk | 2 +- .../tests/invalid-typing/err-6519.tolk | 13 - .../tests/invalid-typing/err-6629.tolk | 2 +- .../tests/invalid-typing/err-6637.tolk | 2 +- .../tests/invalid-typing/err-6659.tolk | 18 +- .../tests/invalid-typing/err-6731.tolk | 2 +- .../tests/invalid-typing/err-6820.tolk | 4 +- .../tests/invalid-typing/err-6839.tolk | 6 +- .../tests/invalid-typing/err-6844.tolk | 2 +- .../tests/invalid-typing/err-6942.tolk | 4 +- .../tests/invalid-typing/err-6981.tolk | 10 +- tolk-tester/tests/logical-operators.tolk | 2 +- tolk-tester/tests/methods-tests.tolk | 311 ++++++++++++++ tolk-tester/tests/mutate-methods.tolk | 66 ++- tolk-tester/tests/no-spaces.tolk | 2 +- tolk-tester/tests/null-keyword.tolk | 4 +- tolk-tester/tests/nullable-tensors.tolk | 26 +- tolk-tester/tests/nullable-types.tolk | 13 +- tolk-tester/tests/parse-address.tolk | 4 +- tolk-tester/tests/pure-functions.tolk | 4 +- tolk-tester/tests/self-keyword.tolk | 66 +-- tolk-tester/tests/smart-cast-tests.tolk | 74 ++-- tolk-tester/tests/some-tests-2.tolk | 28 +- tolk-tester/tests/strings-tests.tolk | 6 +- tolk-tester/tests/struct-tests.tolk | 25 +- tolk-tester/tests/test-math.tolk | 12 +- tolk-tester/tests/type-aliases-tests.tolk | 23 +- tolk-tester/tests/union-types-tests.tolk | 4 +- tolk-tester/tests/use-before-declare.tolk | 2 +- tolk-tester/tests/var-apply-tests.tolk | 90 ++++- .../tests/warnings-not-errors/warnings-1.tolk | 2 +- tolk/ast-from-tokens.cpp | 82 ++-- tolk/ast-replicator.h | 9 +- tolk/ast-stringifier.h | 7 +- tolk/ast.cpp | 3 +- tolk/ast.h | 20 +- tolk/builtins.cpp | 87 ++-- tolk/compiler-state.h | 5 +- tolk/generics-helpers.cpp | 195 ++++++--- tolk/generics-helpers.h | 47 +-- tolk/lexer.cpp | 1 - tolk/lexer.h | 1 - tolk/pipe-ast-to-legacy.cpp | 19 +- tolk/pipe-check-inferred-types.cpp | 22 +- tolk/pipe-check-pure-impure.cpp | 23 +- tolk/pipe-check-rvalue-lvalue.cpp | 4 +- tolk/pipe-find-unused-symbols.cpp | 2 +- tolk/pipe-generate-fif-output.cpp | 4 +- tolk/pipe-infer-types-and-calls.cpp | 382 ++++++++++-------- tolk/pipe-refine-lvalue-for-mutate.cpp | 77 ++-- tolk/pipe-register-symbols.cpp | 104 ++--- tolk/pipe-resolve-identifiers.cpp | 33 +- tolk/pipe-resolve-types.cpp | 93 ++++- tolk/pipeline.h | 6 +- tolk/src-file.cpp | 22 +- tolk/symtable.cpp | 13 + tolk/symtable.h | 36 +- tolk/tolk.h | 1 + 153 files changed, 2236 insertions(+), 1227 deletions(-) create mode 100644 tolk-tester/tests/invalid-declaration/err-1074.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1193.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1226.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1458.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1556.tolk delete mode 100644 tolk-tester/tests/invalid-declaration/err-1601.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1663.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1912.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1973.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4057.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4115.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4256.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4319.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4480.tolk delete mode 100644 tolk-tester/tests/invalid-semantics/err-4506.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4588.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4598.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4610.tolk delete mode 100644 tolk-tester/tests/invalid-semantics/err-4711.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4739.tolk delete mode 100644 tolk-tester/tests/invalid-semantics/err-4828.tolk delete mode 100644 tolk-tester/tests/invalid-semantics/err-4841.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4847.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4884.tolk delete mode 100644 tolk-tester/tests/invalid-semantics/err-4899.tolk delete mode 100644 tolk-tester/tests/invalid-semantics/err-4941.tolk create mode 100644 tolk-tester/tests/invalid-symbol/err-2619.tolk create mode 100644 tolk-tester/tests/invalid-symbol/err-2788.tolk delete mode 100644 tolk-tester/tests/invalid-symbol/err-2914.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6013.tolk delete mode 100644 tolk-tester/tests/invalid-typing/err-6085.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6149.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6281.tolk delete mode 100644 tolk-tester/tests/invalid-typing/err-6337.tolk delete mode 100644 tolk-tester/tests/invalid-typing/err-6386.tolk delete mode 100644 tolk-tester/tests/invalid-typing/err-6519.tolk create mode 100644 tolk-tester/tests/methods-tests.tolk diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 6b737ef3e..f22f4695f 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -10,9 +10,9 @@ tolk 0.11 type dict = cell?; /** - Tuple manipulation primitives. - Elements of a tuple can be of arbitrary type. - Note that atomic type `tuple` can't be cast to composite tuple type (e.g. `[int, cell]`) and vise versa. + Tuple manipulation primitives. + Elements of a tuple can be of arbitrary type. + Note that atomic type `tuple` can't be cast to composite tuple type (e.g. `[int, cell]`) and vise versa. */ /// Creates a tuple with zero elements. @@ -23,36 +23,36 @@ fun createEmptyTuple(): tuple /// Appends a value to tuple, resulting in `Tuple t' = (x1, ..., xn, value)`. /// If its size exceeds 255, throws a type check exception. @pure -fun tuplePush(mutate self: tuple, value: T): void +fun tuple.push(mutate self, value: T): void asm "TPUSH"; /// Returns the first element of a non-empty tuple. -/// `t.0` is actually the same as `t.tupleFirst()` +/// `t.0` is actually the same as `t.first()` @pure -fun tupleFirst(self: tuple): T +fun tuple.first(self): T asm "FIRST"; /// Returns the [`index`]-th element of a tuple. -/// `t.i` is actually the same as `t.tupleAt(i)` +/// `t.i` is actually the same as `t.get(i)` @pure -fun tupleAt(self: tuple, index: int): T +fun tuple.get(self, index: int): T builtin; /// Sets the [`index`]-th element of a tuple to a specified value /// (element with this index must already exist, a new element isn't created). -/// `t.i = value` is actually the same as `t.tupleSetAt(value, i)` +/// `t.i = value` is actually the same as `t.set(value, i)` @pure -fun tupleSetAt(mutate self: tuple, value: T, index: int): void +fun tuple.set(mutate self, value: T, index: int): void builtin; /// Returns the size of a tuple (elements count in it). @pure -fun tupleSize(self: tuple): int +fun tuple.size(self): int asm "TLEN"; /// Returns the last element of a non-empty tuple. @pure -fun tupleLast(self: tuple): T +fun tuple.last(self): T asm "LAST"; @@ -78,6 +78,7 @@ fun max(x: int, y: int): int asm "MAX"; /// Sorts two integers. +/// Example: `minMax(x, y)` with (x=20, y=10) and with (x=10, y=20) returns (10, 20) @pure fun minMax(x: int, y: int): (int, int) asm "MINMAX"; @@ -125,70 +126,83 @@ fun mulDivMod(x: int, y: int, z: int): (int, int) /** - Global getters of environment and contract state. + Global getters and setters of current contract state. */ -const MASTERCHAIN = -1; -const BASECHAIN = 0; - -/// Returns current Unix timestamp (in seconds). -@pure -fun now(): int - asm "NOW"; +/// `contract` is a built-in struct, it has only static methods. +/// Example: `contract.getCode()` and other methods. +struct contract {} /// Returns the internal address of the current smart contract as a Slice with a `MsgAddressInt`. /// If necessary, it can be parsed further using primitives such as [parseStandardAddress]. @pure -fun getMyAddress(): slice +fun contract.getAddress(): slice asm "MYADDR"; /// Returns the balance (in nanotoncoins) of the smart contract at the start of Computation Phase. /// Note that RAW primitives such as [sendMessage] do not update this field. @pure -fun getMyOriginalBalance(): int +fun contract.getOriginalBalance(): coins asm "BALANCE" "FIRST"; -/// Same as [getMyOriginalBalance], but returns a tuple: +/// Same as [contract.getOriginalBalance], but returns a tuple: /// `int` — balance in nanotoncoins; /// `dict` — a dictionary with 32-bit keys representing the balance of "extra currencies". @pure -fun getMyOriginalBalanceWithExtraCurrencies(): [int, dict] +fun contract.getOriginalBalanceWithExtraCurrencies(): [coins, dict] asm "BALANCE"; -/// Returns the logical time of the current transaction. -@pure -fun getLogicalTime(): int - asm "LTIME"; - -/// Returns the starting logical time of the current block. -@pure -fun getCurrentBlockLogicalTime(): int - asm "BLOCKLT"; - -/// Returns the value of the global configuration parameter with integer index `i` as a `cell` or `null` value. -@pure -fun getBlockchainConfigParam(x: int): cell? - asm "CONFIGOPTPARAM"; - /// Returns the persistent contract storage cell. It can be parsed or modified with slice and builder primitives later. @pure -fun getContractData(): cell +fun contract.getData(): cell asm "c4 PUSH"; /// Sets `cell` [c] as persistent contract data. You can update persistent contract storage with this primitive. -fun setContractData(c: cell): void +fun contract.setData(c: cell): void asm "c4 POP"; /// Retrieves code of smart-contract from c7 @pure -fun getContractCode(): cell +fun contract.getCode(): cell asm "MYCODE"; /// Creates an output action that would change this smart contract code to that given by cell [newCode]. /// Notice that this change will take effect only after the successful termination of the current run of the smart contract. -fun setContractCodePostponed(newCode: cell): void +fun contract.setCodePostponed(newCode: cell): void asm "SETCODE"; + +/** + Global getters of current blockchain (environment) state. +*/ + +const MASTERCHAIN = -1; +const BASECHAIN = 0; + +/// `blockchain` is a built-in struct, it has only static methods. +/// Example: `blockchain.configParam(16)` and other methods. +struct blockchain {} + +/// Returns current Unix timestamp (in seconds). +@pure +fun blockchain.now(): int + asm "NOW"; + +/// Returns the logical time of the current transaction. +@pure +fun blockchain.logicalTime(): int + asm "LTIME"; + +/// Returns the starting logical time of the current block. +@pure +fun blockchain.currentBlockLogicalTime(): int + asm "BLOCKLT"; + +/// Returns the value of the global configuration parameter with integer index `i` as a `cell` or `null` value. +@pure +fun blockchain.configParam(x: int): cell? + asm "CONFIGOPTPARAM"; + /// Commits the current state of registers `c4` (“persistent data”) and `c5` (“actions”) /// so that the current execution is considered “successful” with the saved values even if an exception /// in Computation Phase is thrown later. @@ -197,7 +211,7 @@ fun commitContractDataAndActions(): void /** - Signature checks, hashing, cryptography. + Signature checks, hashing, cryptography. */ /// Compile-time function that calculates crc32 of a constant string. @@ -236,23 +250,23 @@ fun stringSha256_32(constString: slice): int fun stringToBase256(constString: slice): int builtin; -/// Computes the representation hash of a `cell` [c] and returns it as a 256-bit unsigned integer `x`. +/// Computes the representation hash of a `cell` and returns it as a 256-bit unsigned integer `x`. /// Useful for signing and checking signatures of arbitrary entities represented by a tree of cells. @pure -fun cellHash(c: cell): int +fun cell.hash(self): int asm "HASHCU"; -/// Computes the hash of a `slice s` and returns it as a 256-bit unsigned integer `x`. +/// Computes the hash of a `slice` and returns it as a 256-bit unsigned integer `x`. /// The result is the same as if an ordinary cell containing only data and references from `s` had been created -/// and its hash computed by [cellHash]. +/// and its hash computed by [cell.hash]. @pure -fun sliceHash(s: slice): int +fun slice.hash(self): int asm "HASHSU"; -/// Computes sha256 of the data bits of `slice` [s]. If the bit length of `s` is not divisible by eight, +/// Computes sha256 of the data bits of a `slice`. If the bit length of `s` is not divisible by eight, /// throws a cell underflow exception. The hash value is returned as a 256-bit unsigned integer `x`. @pure -fun sliceBitsHash(s: slice): int +fun slice.bitsHash(self): int asm "SHA256U"; /// Checks the Ed25519-`signature` of a `hash` (a 256-bit unsigned integer, usually computed as the hash of some data) @@ -275,36 +289,43 @@ fun isSignatureValid(hash: int, signature: slice, publicKey: int): bool fun isSliceSignatureValid(data: slice, signature: slice, publicKey: int): bool asm "CHKSIGNS"; +/// `random` is a built-in struct, it has only static methods. +/// Example: `random.uint256()` and other methods. +struct random {} + /// Generates a new pseudo-random unsigned 256-bit integer x. -fun random(): int +/// Ensure you've called [random.initialize] in advance to make it unpredictable! +fun random.uint256(): uint256 asm "RANDU256"; -/// Generates a new pseudo-random integer z in the range 0..range−1 (or range..−1, if range < 0). -/// More precisely, an unsigned random value x is generated as in random; then z := x * range / 2^256 is computed. -fun randomRange(range: int): int +/// Generates a new pseudo-random integer z in the range 0..limit−1 (or limit..−1, if upto < 0). +/// More precisely, an unsigned random value x is generated as in random; then z := x * limit / 2^256 is computed. +/// Ensure you've called [random.initialize] in advance to make it unpredictable! +fun random.range(limit: int): int asm "RAND"; -/// Returns the current random seed as an unsigned 256-bit integer. +/// Returns the current random seed used to generate pseudo-random numbers. @pure -fun randomGetSeed(): int +fun random.getSeed(): uint256 asm "RANDSEED"; -/// Sets the random seed to unsigned 256-bit seed. -fun randomSetSeed(seed: int): void +/// Sets the random seed to the provided value. +fun random.setSeed(seed: uint256): void asm "SETRAND"; -/// Initializes (mixes) random seed with unsigned 256-bit integer x. -fun randomizeBy(x: int): void +/// Initializes (mixes) random seed with the provided value. +fun random.initializeBy(mixSeedWith: uint256): void asm "ADDRAND"; -/// Initializes random seed using current time. Don't forget to call this before calling `random`! -fun randomizeByLogicalTime(): void +/// Initializes random seed with current time to make random generation unpredictable. +/// Typically, you call this function once before calling [random.uint256] / [random.range]. +fun random.initialize(): void asm "LTIME" "ADDRAND"; /** - Size computation primitives. - They may be useful for computing storage fees of user-provided data. + Size computation primitives. + They may be useful for computing storage fees of user-provided data. */ /// Returns `(x, y, z, -1)` or `(null, null, null, 0)`. @@ -317,69 +338,73 @@ fun randomizeByLogicalTime(): void /// otherwise the computation is aborted before visiting the `(maxCells + 1)`-st cell and /// a zero flag is returned to indicate failure. If [c] is `null`, returns `x = y = z = 0`. @pure -fun calculateCellSize(c: cell, maxCells: int): (int, int, int, bool) +fun cell.calculateSize(self, maxCells: int): (int, int, int, bool) asm "CDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT"; -/// Similar to [calculateCellSize], but accepting a `slice` [s] instead of a `cell`. +/// Similar to [cell.calculateSize], but accepting a `slice` [s] instead of a `cell`. /// The returned value of `x` does not take into account the cell that contains the `slice` [s] itself; /// however, the data bits and the cell references of [s] are accounted for in `y` and `z`. @pure -fun calculateSliceSize(s: slice, maxCells: int): (int, int, int, bool) +fun slice.calculateSize(self, maxCells: int): (int, int, int, bool) asm "SDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT"; -/// A non-quiet version of [calculateCellSize] that throws a cell overflow exception (`8`) on failure. -fun calculateCellSizeStrict(c: cell, maxCells: int): (int, int, int) +/// A non-quiet version of [cell.calculateSize] that throws a cell overflow exception (`8`) on failure. +fun cell.calculateSizeStrict(self, maxCells: int): (int, int, int) asm "CDATASIZE"; -/// A non-quiet version of [calculateSliceSize] that throws a cell overflow exception (`8`) on failure. -fun calculateSliceSizeStrict(s: slice, maxCells: int): (int, int, int) +/// A non-quiet version of [cell.calculateSize] that throws a cell overflow exception (`8`) on failure. +fun slice.calculateSizeStrict(self, maxCells: int): (int, int, int) asm "SDATASIZE"; -/// Returns the depth of `cell` [c]. +/// Returns the depth of a `cell`. /// If [c] has no references, then return `0`; /// otherwise the returned value is one plus the maximum of depths of cells referred to from [c]. /// If [c] is a `null` instead of a cell, returns zero. @pure -fun getCellDepth(c: cell?): int +fun cell?.depth(self): int asm "CDEPTH"; -/// Returns the depth of `slice` [s]. +/// Returns the depth of a `slice`. /// If [s] has no references, then returns `0`; /// otherwise the returned value is one plus the maximum of depths of cells referred to from [s]. @pure -fun getSliceDepth(s: slice): int +fun slice.depth(self): int asm "SDEPTH"; -/// Returns the depth of `builder` [b]. +/// Returns the depth of a `builder`. /// If no cell references are stored in [b], then returns 0; /// otherwise the returned value is one plus the maximum of depths of cells referred to from [b]. @pure -fun getBuilderDepth(b: builder): int +fun builder.depth(self): int asm "BDEPTH"; /** - Debug primitives. - Only works for local TVM execution with debug level verbosity. + Debug primitives. + Only works for local TVM execution with debug level verbosity. */ +/// `debug` is a built-in struct, it has only static methods. +/// Example: `debug.print(v)` and other methods. +struct debug {} + /// Dump a variable [x] to the debug log. -fun debugPrint(x: T): void +fun debug.print(x: T): void builtin; /// Dump a string [x] to the debug log. -fun debugPrintString(x: T): void +fun debug.printString(x: T): void builtin; /// Dumps the stack (at most the top 255 values) and shows the total stack depth. -fun debugDumpStack(): void +fun debug.dumpStack(): void builtin; /** - Slice primitives: parsing cells. - When you _load_ some data, you mutate the slice (shifting an internal pointer on the stack). - When you _preload_ some data, you just get the result without mutating the slice. + Slice primitives: parsing cells. + When you _load_ some data, you mutate the slice (shifting an internal pointer on the stack). + When you _preload_ some data, you just get the result without mutating the slice. */ /// Compile-time function that converts a constant string to TL-encoded MsgAddressInt (TVM slice). @@ -398,124 +423,124 @@ fun stringAddressToSlice(constStringAddress: slice): slice fun stringHexToSlice(constStringBytesHex: slice): slice builtin; -/// Converts a `cell` [c] into a `slice`. Notice that [c] must be either an ordinary cell, +/// Converts a `cell` into a `slice`. Notice that [c] must be either an ordinary cell, /// or an exotic cell (see [TVM.pdf](https://ton-blockchain.github.io/docs/tvm.pdf), 3.1.2) /// which is automatically loaded to yield an ordinary cell `c'`, converted into a `slice` afterwards. @pure -fun beginParse(c: cell): slice +fun cell.beginParse(self): slice asm "CTOS"; /// Checks if slice is empty. If not, throws an exception. -fun assertEndOfSlice(self: slice): void +fun slice.assertEnd(self): void asm "ENDS"; /// Loads the next reference from the slice. @pure -fun loadRef(mutate self: slice): cell +fun slice.loadRef(mutate self): cell asm( -> 1 0) "LDREF"; /// Preloads the next reference from the slice. @pure -fun preloadRef(self: slice): cell +fun slice.preloadRef(self): cell asm "PLDREF"; /// Loads a signed [len]-bit integer from a slice. @pure -fun loadInt(mutate self: slice, len: int): int +fun slice.loadInt(mutate self, len: int): int builtin; /// Loads an unsigned [len]-bit integer from a slice. @pure -fun loadUint(mutate self: slice, len: int): int +fun slice.loadUint(mutate self, len: int): int builtin; /// Loads the first `0 ≤ len ≤ 1023` bits from slice [s] into a separate slice `s''`. @pure -fun loadBits(mutate self: slice, len: int): slice +fun slice.loadBits(mutate self, len: int): slice builtin; /// Preloads a signed [len]-bit integer from a slice. @pure -fun preloadInt(self: slice, len: int): int +fun slice.preloadInt(self, len: int): int builtin; /// Preloads an unsigned [len]-bit integer from a slice. @pure -fun preloadUint(self: slice, len: int): int +fun slice.preloadUint(self, len: int): int builtin; /// Preloads the first `0 ≤ len ≤ 1023` bits from slice [s] into a separate slice. @pure -fun preloadBits(self: slice, len: int): slice +fun slice.preloadBits(self, len: int): slice builtin; /// Loads serialized amount of Toncoins (any unsigned integer up to `2^120 - 1`). @pure -fun loadCoins(mutate self: slice): coins +fun slice.loadCoins(mutate self): coins asm( -> 1 0) "LDGRAMS"; /// Loads bool (-1 or 0) from a slice @pure -fun loadBool(mutate self: slice): bool +fun slice.loadBool(mutate self): bool asm( -> 1 0) "1 LDI"; /// Shifts a slice pointer to [len] bits forward, mutating the slice. @pure -fun skipBits(mutate self: slice, len: int): self +fun slice.skipBits(mutate self, len: int): self asm "SDSKIPFIRST"; /// Returns the first `0 ≤ len ≤ 1023` bits of a slice. @pure -fun getFirstBits(self: slice, len: int): slice +fun slice.getFirstBits(self, len: int): slice asm "SDCUTFIRST"; /// Returns all but the last `0 ≤ len ≤ 1023` bits of a slice. @pure -fun removeLastBits(mutate self: slice, len: int): self +fun slice.removeLastBits(mutate self, len: int): self asm "SDSKIPLAST"; /// Returns the last `0 ≤ len ≤ 1023` bits of a slice. @pure -fun getLastBits(self: slice, len: int): slice +fun slice.getLastBits(self, len: int): slice asm "SDCUTLAST"; /// Loads a dictionary (TL HashMapE structure, represented as TVM cell) from a slice. /// Returns `null` if `nothing` constructor is used. @pure -fun loadDict(mutate self: slice): dict +fun slice.loadDict(mutate self): dict asm( -> 1 0) "LDDICT"; /// Preloads a dictionary (cell) from a slice. @pure -fun preloadDict(self: slice): dict +fun slice.preloadDict(self): dict asm "PLDDICT"; -/// Loads a dictionary as [loadDict], but returns only the remainder of the slice. +/// Loads a dictionary as [slice.loadDict], but returns only the remainder of the slice. @pure -fun skipDict(mutate self: slice): self +fun slice.skipDict(mutate self): self asm "SKIPDICT"; /// Loads (Maybe ^Cell) from a slice. /// In other words, loads 1 bit: if it's true, loads the first ref, otherwise returns `null`. @pure -fun loadMaybeRef(mutate self: slice): cell? +fun slice.loadMaybeRef(mutate self): cell? asm( -> 1 0) "LDOPTREF"; /// Preloads (Maybe ^Cell) from a slice. @pure -fun preloadMaybeRef(self: slice): cell? +fun slice.preloadMaybeRef(self): cell? asm "PLDOPTREF"; /// Loads (Maybe ^Cell), but returns only the remainder of the slice. @pure -fun skipMaybeRef(mutate self: slice): self +fun slice.skipMaybeRef(mutate self): self asm "SKIPOPTREF"; /** - Builder primitives: constructing cells. - When you _store_ some data, you mutate the builder (shifting an internal pointer on the stack). - All the primitives below first check whether there is enough space in the `builder`, - and only then check the range of the value being serialized. + Builder primitives: constructing cells. + When you _store_ some data, you mutate the builder (shifting an internal pointer on the stack). + All the primitives below first check whether there is enough space in the `builder`, + and only then check the range of the value being serialized. */ /// Creates a new empty builder. @@ -525,116 +550,116 @@ fun beginCell(): builder /// Converts a builder into an ordinary `cell`. @pure -fun endCell(self: builder): cell +fun builder.endCell(self): cell asm "ENDC"; /// Stores a reference to a cell into a builder. @pure -fun storeRef(mutate self: builder, c: cell): self +fun builder.storeRef(mutate self, c: cell): self asm(c self) "STREF"; /// Stores a signed [len]-bit integer into a builder (`0 ≤ len ≤ 257`). @pure -fun storeInt(mutate self: builder, x: int, len: int): self +fun builder.storeInt(mutate self, x: int, len: int): self builtin; /// Stores an unsigned [len]-bit integer into a builder (`0 ≤ len ≤ 256`). @pure -fun storeUint(mutate self: builder, x: int, len: int): self +fun builder.storeUint(mutate self, x: int, len: int): self builtin; /// Stores a slice into a builder. @pure -fun storeSlice(mutate self: builder, s: slice): self +fun builder.storeSlice(mutate self, s: slice): self asm "STSLICER"; /// Stores amount of Toncoins into a builder. @pure -fun storeCoins(mutate self: builder, x: coins): self +fun builder.storeCoins(mutate self, x: coins): self asm "STGRAMS"; /// Stores bool (-1 or 0) into a builder. /// Attention: true value is `-1`, not 1! If you pass `1` here, TVM will throw an exception. @pure -fun storeBool(mutate self: builder, x: bool): self +fun builder.storeBool(mutate self, x: bool): self asm(x self) "1 STI"; /// Stores dictionary (represented by TVM `cell` or `null`) into a builder. /// In other words, stores a `1`-bit and a reference to [c] if [c] is not `null` and `0`-bit otherwise. @pure -fun storeDict(mutate self: builder, c: dict): self +fun builder.storeDict(mutate self, c: dict): self asm(c self) "STDICT"; /// Stores (Maybe ^Cell) into a builder. /// In other words, if cell is `null`, store '0' bit; otherwise, store '1' and a ref to [c]. @pure -fun storeMaybeRef(mutate self: builder, c: cell?): self +fun builder.storeMaybeRef(mutate self, c: cell?): self asm(c self) "STOPTREF"; /// Concatenates two builders. @pure -fun storeBuilder(mutate self: builder, from: builder): self +fun builder.storeBuilder(mutate self, from: builder): self asm "STBR"; /// Stores a slice representing TL addr_none$00 (two `0` bits). @pure -fun storeAddressNone(mutate self: builder): self +fun builder.storeAddressNone(mutate self): self asm "b{00} STSLICECONST"; /** - Slice size primitives. + Slice size primitives. */ /// Returns the number of references in a slice. @pure -fun getRemainingRefsCount(self: slice): int +fun slice.remainingRefsCount(self): int asm "SREFS"; /// Returns the number of data bits in a slice. @pure -fun getRemainingBitsCount(self: slice): int +fun slice.remainingBitsCount(self): int asm "SBITS"; /// Returns both the number of data bits and the number of references in a slice. @pure -fun getRemainingBitsAndRefsCount(self: slice): (int, int) +fun slice.remainingBitsAndRefsCount(self): (int, int) asm "SBITREFS"; /// Checks whether a slice is empty (i.e., contains no bits of data and no cell references). @pure -fun isEndOfSlice(self: slice): bool +fun slice.isEnd(self): bool asm "SEMPTY"; /// Checks whether a slice has no bits of data. @pure -fun isEndOfSliceBits(self: slice): bool +fun slice.isEndOfBits(self): bool asm "SDEMPTY"; /// Checks whether a slice has no references. @pure -fun isEndOfSliceRefs(self: slice): bool +fun slice.isEndOfRefs(self): bool asm "SREMPTY"; /// Checks whether data parts of two slices coinside. @pure -fun isSliceBitsEqual(self: slice, b: slice): bool +fun slice.bitsEqual(self, b: slice): bool asm "SDEQ"; /// Returns the number of cell references already stored in a builder. @pure -fun getBuilderRefsCount(self: builder): int +fun builder.refsCount(self): int asm "BREFS"; /// Returns the number of data bits already stored in a builder. @pure -fun getBuilderBitsCount(self: builder): int +fun builder.bitsCount(self): int asm "BBITS"; /** - Address manipulation primitives. - The address manipulation primitives listed below serialize and deserialize values according to the following TL-B scheme: + Address manipulation primitives. + The address manipulation primitives listed below serialize and deserialize values according to the following TL-B scheme: ```TL-B addr_none$00 = MsgAddressExt; addr_extern$01 len:(## 8) external_address:(bits len) @@ -672,7 +697,7 @@ fun getBuilderBitsCount(self: builder): int /// Loads from slice [s] the only prefix that is a valid `MsgAddress`, /// and returns both this prefix `s'` and the remainder `s''` of [s] as slices. @pure -fun loadAddress(mutate self: slice): slice +fun slice.loadAddress(mutate self): slice asm( -> 1 0) "LDMSGADDR"; /// Decomposes slice [s] containing a valid `MsgAddress` into a `tuple t` with separate fields of this `MsgAddress`. @@ -703,7 +728,7 @@ fun addressIsNone(s: slice): bool /** - Reserving Toncoins on balance and its flags. + Reserving Toncoins on balance and its flags. */ /// mode = 0: Reserve exact amount of nanotoncoins @@ -721,12 +746,12 @@ const RESERVE_MODE_BOUNCE_ON_ACTION_FAIL = 16; /// Creates an output action which would reserve Toncoins on balance. /// For [reserveMode] consider constants above. -fun reserveToncoinsOnBalance(nanoTonCoins: int, reserveMode: int): void +fun reserveToncoinsOnBalance(nanoTonCoins: coins, reserveMode: int): void asm "RAWRESERVE"; /// Similar to [reserveToncoinsOnBalance], but also accepts a dictionary extraAmount (represented by a cell or null) /// with extra currencies. In this way currencies other than Toncoin can be reserved. -fun reserveExtraCurrenciesOnBalance(nanoTonCoins: int, extraAmount: dict, reserveMode: int): void +fun reserveExtraCurrenciesOnBalance(nanoTonCoins: coins, extraAmount: dict, reserveMode: int): void asm "RAWRESERVEX"; @@ -745,7 +770,7 @@ const NON_BOUNCEABLE = 0x10; /// Load msgFlags from incoming message body (4 bits). @pure -fun loadMessageFlags(mutate self: slice): int +fun slice.loadMessageFlags(mutate self): int asm( -> 1 0) "4 LDU"; /// Having msgFlags (4 bits), check that a message is bounced. @@ -756,33 +781,33 @@ fun isMessageBounced(msgFlags: int): bool /// Skip 0xFFFFFFFF prefix (when a message is bounced). @pure -fun skipBouncedPrefix(mutate self: slice): self +fun slice.skipBouncedPrefix(mutate self): self asm "32 PUSHINT" "SDSKIPFIRST"; /// The guideline recommends to start the body of an internal message with uint32 `op` and uint64 `queryId`. @pure -fun loadMessageOp(mutate self: slice): int +fun slice.loadMessageOp(mutate self): int asm( -> 1 0) "32 LDU"; @pure -fun skipMessageOp(mutate self: slice): self +fun slice.skipMessageOp(mutate self): self asm "32 PUSHINT" "SDSKIPFIRST"; @pure -fun storeMessageOp(mutate self: builder, op: int): self +fun builder.storeMessageOp(mutate self, op: int): self asm(op self) "32 STU"; /// The guideline recommends that uint64 `queryId` should follow uint32 `op`. @pure -fun loadMessageQueryId(mutate self: slice): int +fun slice.loadMessageQueryId(mutate self): int asm( -> 1 0) "64 LDU"; @pure -fun skipMessageQueryId(mutate self: slice): self +fun slice.skipMessageQueryId(mutate self): self asm "64 PUSHINT" "SDSKIPFIRST"; @pure -fun storeMessageQueryId(mutate self: builder, queryId: int): self +fun builder.storeMessageQueryId(mutate self, queryId: int): self asm(queryId self) "64 STU"; /// SEND MODES - https://docs.ton.org/tvm.pdf page 137, SENDRAWMSG @@ -809,8 +834,6 @@ const SEND_MODE_ESTIMATE_FEE_ONLY = 1024; /// Sends a raw message — a correctly serialized TL object `Message X`. /// For `mode`, see constants above (except SEND_MODE_ESTIMATE_FEE_ONLY). -/// This function is still available, but deprecated: consider using [sendMessage]. -@deprecated fun sendRawMessage(msg: cell, mode: int): void asm "SENDRAWMSG"; diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk index 0e8305e1f..b0cab4de8 100644 --- a/crypto/smartcont/tolk-stdlib/gas-payments.tolk +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -6,7 +6,7 @@ tolk 0.11 */ /// Returns amount of gas (in gas units) consumed in current Computation Phase. -fun getGasConsumedAtTheMoment(): int +fun getGasConsumedAtTheMoment(): coins asm "GASCONSUMED"; /// This function is required to be called when you process an external message (from an outer world) @@ -33,37 +33,37 @@ fun setGasLimit(limit: int): void asm "SETGASLIMIT"; /// Calculates fee (amount in nanotoncoins to be paid) for a transaction which consumed [gasUsed] gas units. -fun calculateGasFee(workchain: int, gasUsed: int): int +fun calculateGasFee(workchain: int8, gasUsed: int): coins asm(gasUsed workchain) "GETGASFEE"; /// Same as [calculateGasFee], but without flat price (you have supposed to read https://docs.ton.org/develop/howto/fees-low-level) -fun calculateGasFeeWithoutFlatPrice(workchain: int, gasUsed: int): int +fun calculateGasFeeWithoutFlatPrice(workchain: int8, gasUsed: coins): coins asm(gasUsed workchain) "GETGASFEESIMPLE"; /// Calculates amount of nanotoncoins you should pay for storing a contract of provided size for [seconds]. /// [bits] and [cells] represent contract state (code + data). -fun calculateStorageFee(workchain: int, seconds: int, bits: int, cells: int): int +fun calculateStorageFee(workchain: int8, seconds: int, bits: int, cells: int): coins asm(cells bits seconds workchain) "GETSTORAGEFEE"; /// Calculates amount of nanotoncoins you should pay to send a message of specified size. -fun calculateMessageFee(workchain: int, bits: int, cells: int): int +fun calculateMessageFee(workchain: int8, bits: int, cells: int): coins asm(cells bits workchain) "GETFORWARDFEE"; /// Same as [calculateMessageFee], but without lump price (you have supposed to read https://docs.ton.org/develop/howto/fees-low-level) -fun calculateMessageFeeWithoutLumpPrice(workchain: int, bits: int, cells: int): int +fun calculateMessageFeeWithoutLumpPrice(workchain: int8, bits: int, cells: int): coins asm(cells bits workchain) "GETFORWARDFEESIMPLE"; /// Calculates fee that was paid by the sender of an incoming internal message. -fun calculateOriginalMessageFee(workchain: int, incomingFwdFee: int): int +fun calculateOriginalMessageFee(workchain: int8, incomingFwdFee: coins): coins asm(incomingFwdFee workchain) "GETORIGINALFWDFEE"; /// Returns the amount of nanotoncoins current contract debts for storage. ("due" and "debt" are synonyms) /// If it has no debt, `0` is returned. -fun getMyStorageDuePayment(): int +fun contract.getStorageDuePayment(): coins asm "DUEPAYMENT"; /// Returns the amount of nanotoncoins charged for storage. /// (during storage phase preceeding to current computation phase) @pure -fun getMyStoragePaidPayment(): int +fun contract.getStoragePaidPayment(): coins asm "STORAGEFEES"; diff --git a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk index a08b2c11c..9dd6c1da9 100644 --- a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk +++ b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk @@ -2,7 +2,7 @@ tolk 0.11 /** - Lisp-style lists are nested 2-elements tuples: `(1, (2, (3, null)))` represents list `[1, 2, 3]`. + Lisp-style lists are nested 2-elements tuples: `[1, [2, [3, null]]]` represents list `[1, 2, 3]`. Elements of a list can be of different types. Empty list is conventionally represented as TVM `null` value. */ @@ -22,12 +22,6 @@ fun listPrepend(head: X, tail: tuple?): tuple fun listSplit(list: tuple): (X, tuple?) asm "UNCONS"; -/// Extracts the tail and the head of lisp-style list. -/// After extracting the last element, tuple is assigned to null. -@pure -fun listNext(mutate self: tuple?): X - asm( -> 1 0) "UNCONS"; - /// Returns the head of lisp-style list. @pure fun listGetHead(list: tuple): X diff --git a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk index 00ae594c6..5de989aed 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk @@ -24,278 +24,278 @@ fun createEmptyDict(): dict /// Checks whether a dictionary is empty. @pure -fun dictIsEmpty(self: dict): bool +fun dict.dictIsEmpty(self): bool asm "DICTEMPTY"; @pure -fun iDictGet(self: dict, keyLen: int, key: int): (slice?, bool) +fun dict.iDictGet(self, keyLen: int, key: int): (slice?, bool) asm(key self keyLen) "DICTIGET" "NULLSWAPIFNOT"; @pure -fun uDictGet(self: dict, keyLen: int, key: int): (slice?, bool) +fun dict.uDictGet(self, keyLen: int, key: int): (slice?, bool) asm(key self keyLen) "DICTUGET" "NULLSWAPIFNOT"; @pure -fun sDictGet(self: dict, keyLen: int, key: slice): (slice?, bool) +fun dict.sDictGet(self, keyLen: int, key: slice): (slice?, bool) asm(key self keyLen) "DICTGET" "NULLSWAPIFNOT"; @pure -fun iDictSet(mutate self: dict, keyLen: int, key: int, value: slice): void +fun dict.iDictSet(mutate self, keyLen: int, key: int, value: slice): void asm(value key self keyLen) "DICTISET"; @pure -fun uDictSet(mutate self: dict, keyLen: int, key: int, value: slice): void +fun dict.uDictSet(mutate self, keyLen: int, key: int, value: slice): void asm(value key self keyLen) "DICTUSET"; @pure -fun sDictSet(mutate self: dict, keyLen: int, key: slice, value: slice): void +fun dict.sDictSet(mutate self, keyLen: int, key: slice, value: slice): void asm(value key self keyLen) "DICTSET"; @pure -fun iDictSetRef(mutate self: dict, keyLen: int, key: int, value: cell): void +fun dict.iDictSetRef(mutate self, keyLen: int, key: int, value: cell): void asm(value key self keyLen) "DICTISETREF"; @pure -fun uDictSetRef(mutate self: dict, keyLen: int, key: int, value: cell): void +fun dict.uDictSetRef(mutate self, keyLen: int, key: int, value: cell): void asm(value key self keyLen) "DICTUSETREF"; @pure -fun sDictSetRef(mutate self: dict, keyLen: int, key: slice, value: cell): void +fun dict.sDictSetRef(mutate self, keyLen: int, key: slice, value: cell): void asm(value key self keyLen) "DICTSETREF"; @pure -fun iDictSetIfNotExists(mutate self: dict, keyLen: int, key: int, value: slice): bool +fun dict.iDictSetIfNotExists(mutate self, keyLen: int, key: int, value: slice): bool asm(value key self keyLen) "DICTIADD"; @pure -fun uDictSetIfNotExists(mutate self: dict, keyLen: int, key: int, value: slice): bool +fun dict.uDictSetIfNotExists(mutate self, keyLen: int, key: int, value: slice): bool asm(value key self keyLen) "DICTUADD"; @pure -fun iDictSetIfExists(mutate self: dict, keyLen: int, key: int, value: slice): bool +fun dict.iDictSetIfExists(mutate self, keyLen: int, key: int, value: slice): bool asm(value key self keyLen) "DICTIREPLACE"; @pure -fun uDictSetIfExists(mutate self: dict, keyLen: int, key: int, value: slice): bool +fun dict.uDictSetIfExists(mutate self, keyLen: int, key: int, value: slice): bool asm(value key self keyLen) "DICTUREPLACE"; @pure -fun iDictGetRef(self: dict, keyLen: int, key: int): (dict, bool) +fun dict.iDictGetRef(self, keyLen: int, key: int): (dict, bool) asm(key self keyLen) "DICTIGETREF" "NULLSWAPIFNOT"; @pure -fun uDictGetRef(self: dict, keyLen: int, key: int): (dict, bool) +fun dict.uDictGetRef(self, keyLen: int, key: int): (dict, bool) asm(key self keyLen) "DICTUGETREF" "NULLSWAPIFNOT"; @pure -fun sDictGetRef(self: dict, keyLen: int, key: slice): (dict, bool) +fun dict.sDictGetRef(self, keyLen: int, key: slice): (dict, bool) asm(key self keyLen) "DICTGETREF" "NULLSWAPIFNOT"; @pure -fun iDictGetRefOrNull(self: dict, keyLen: int, key: int): dict +fun dict.iDictGetRefOrNull(self, keyLen: int, key: int): dict asm(key self keyLen) "DICTIGETOPTREF"; @pure -fun uDictGetRefOrNull(self: dict, keyLen: int, key: int): dict +fun dict.uDictGetRefOrNull(self, keyLen: int, key: int): dict asm(key self keyLen) "DICTUGETOPTREF"; @pure -fun sDictGetRefOrNull(self: dict, keyLen: int, key: slice): dict +fun dict.sDictGetRefOrNull(self, keyLen: int, key: slice): dict asm(key self keyLen) "DICTGETOPTREF"; @pure -fun iDictDelete(mutate self: dict, keyLen: int, key: int): bool +fun dict.iDictDelete(mutate self, keyLen: int, key: int): bool asm(key self keyLen) "DICTIDEL"; @pure -fun uDictDelete(mutate self: dict, keyLen: int, key: int): bool +fun dict.uDictDelete(mutate self, keyLen: int, key: int): bool asm(key self keyLen) "DICTUDEL"; @pure -fun sDictDelete(mutate self: dict, keyLen: int, key: slice): bool +fun dict.sDictDelete(mutate self, keyLen: int, key: slice): bool asm(key self keyLen) "DICTDEL"; @pure -fun iDictSetAndGet(mutate self: dict, keyLen: int, key: int, value: slice): (slice?, bool) +fun dict.iDictSetAndGet(mutate self, keyLen: int, key: int, value: slice): (slice?, bool) asm(value key self keyLen) "DICTISETGET" "NULLSWAPIFNOT"; @pure -fun uDictSetAndGet(mutate self: dict, keyLen: int, key: int, value: slice): (slice?, bool) +fun dict.uDictSetAndGet(mutate self, keyLen: int, key: int, value: slice): (slice?, bool) asm(value key self keyLen) "DICTUSETGET" "NULLSWAPIFNOT"; @pure -fun sDictSetAndGet(mutate self: dict, keyLen: int, key: slice, value: slice): (slice?, bool) +fun dict.sDictSetAndGet(mutate self, keyLen: int, key: slice, value: slice): (slice?, bool) asm(value key self keyLen) "DICTSETGET" "NULLSWAPIFNOT"; @pure -fun iDictSetAndGetRefOrNull(mutate self: dict, keyLen: int, key: int, value: cell): dict +fun dict.iDictSetAndGetRefOrNull(mutate self, keyLen: int, key: int, value: cell): dict asm(value key self keyLen) "DICTISETGETOPTREF"; @pure -fun uDictSetAndGetRefOrNull(mutate self: dict, keyLen: int, key: int, value: cell): dict +fun dict.uDictSetAndGetRefOrNull(mutate self, keyLen: int, key: int, value: cell): dict asm(value key self keyLen) "DICTUSETGETOPTREF"; @pure -fun iDictDeleteAndGet(mutate self: dict, keyLen: int, key: int): (slice?, bool) +fun dict.iDictDeleteAndGet(mutate self, keyLen: int, key: int): (slice?, bool) asm(key self keyLen) "DICTIDELGET" "NULLSWAPIFNOT"; @pure -fun uDictDeleteAndGet(mutate self: dict, keyLen: int, key: int): (slice?, bool) +fun dict.uDictDeleteAndGet(mutate self, keyLen: int, key: int): (slice?, bool) asm(key self keyLen) "DICTUDELGET" "NULLSWAPIFNOT"; @pure -fun sDictDeleteAndGet(mutate self: dict, keyLen: int, key: slice): (slice?, bool) +fun dict.sDictDeleteAndGet(mutate self, keyLen: int, key: slice): (slice?, bool) asm(key self keyLen) "DICTDELGET" "NULLSWAPIFNOT"; @pure -fun iDictSetBuilder(mutate self: dict, keyLen: int, key: int, value: builder): void +fun dict.iDictSetBuilder(mutate self, keyLen: int, key: int, value: builder): void asm(value key self keyLen) "DICTISETB"; @pure -fun uDictSetBuilder(mutate self: dict, keyLen: int, key: int, value: builder): void +fun dict.uDictSetBuilder(mutate self, keyLen: int, key: int, value: builder): void asm(value key self keyLen) "DICTUSETB"; @pure -fun sDictSetBuilder(mutate self: dict, keyLen: int, key: slice, value: builder): void +fun dict.sDictSetBuilder(mutate self, keyLen: int, key: slice, value: builder): void asm(value key self keyLen) "DICTSETB"; @pure -fun iDictSetBuilderIfNotExists(mutate self: dict, keyLen: int, key: int, value: builder): bool +fun dict.iDictSetBuilderIfNotExists(mutate self, keyLen: int, key: int, value: builder): bool asm(value key self keyLen) "DICTIADDB"; @pure -fun uDictSetBuilderIfNotExists(mutate self: dict, keyLen: int, key: int, value: builder): bool +fun dict.uDictSetBuilderIfNotExists(mutate self, keyLen: int, key: int, value: builder): bool asm(value key self keyLen) "DICTUADDB"; @pure -fun iDictSetBuilderIfExists(mutate self: dict, keyLen: int, key: int, value: builder): bool +fun dict.iDictSetBuilderIfExists(mutate self, keyLen: int, key: int, value: builder): bool asm(value key self keyLen) "DICTIREPLACEB"; @pure -fun uDictSetBuilderIfExists(mutate self: dict, keyLen: int, key: int, value: builder): bool +fun dict.uDictSetBuilderIfExists(mutate self, keyLen: int, key: int, value: builder): bool asm(value key self keyLen) "DICTUREPLACEB"; @pure -fun iDictDeleteFirstAndGet(mutate self: dict, keyLen: int): (int?, slice?, bool) +fun dict.iDictDeleteFirstAndGet(mutate self, keyLen: int): (int?, slice?, bool) asm(-> 0 2 1 3) "DICTIREMMIN" "NULLSWAPIFNOT2"; @pure -fun uDictDeleteFirstAndGet(mutate self: dict, keyLen: int): (int?, slice?, bool) +fun dict.uDictDeleteFirstAndGet(mutate self, keyLen: int): (int?, slice?, bool) asm(-> 0 2 1 3) "DICTUREMMIN" "NULLSWAPIFNOT2"; @pure -fun sDictDeleteFirstAndGet(mutate self: dict, keyLen: int): (slice?, slice?, bool) +fun dict.sDictDeleteFirstAndGet(mutate self, keyLen: int): (slice?, slice?, bool) asm(-> 0 2 1 3) "DICTREMMIN" "NULLSWAPIFNOT2"; @pure -fun iDictDeleteLastAndGet(mutate self: dict, keyLen: int): (int?, slice?, bool) +fun dict.iDictDeleteLastAndGet(mutate self, keyLen: int): (int?, slice?, bool) asm(-> 0 2 1 3) "DICTIREMMAX" "NULLSWAPIFNOT2"; @pure -fun uDictDeleteLastAndGet(mutate self: dict, keyLen: int): (int?, slice?, bool) +fun dict.uDictDeleteLastAndGet(mutate self, keyLen: int): (int?, slice?, bool) asm(-> 0 2 1 3) "DICTUREMMAX" "NULLSWAPIFNOT2"; @pure -fun sDictDeleteLastAndGet(mutate self: dict, keyLen: int): (slice?, slice?, bool) +fun dict.sDictDeleteLastAndGet(mutate self, keyLen: int): (slice?, slice?, bool) asm(-> 0 2 1 3) "DICTREMMAX" "NULLSWAPIFNOT2"; @pure -fun iDictGetFirst(self: dict, keyLen: int): (int?, slice?, bool) +fun dict.iDictGetFirst(self, keyLen: int): (int?, slice?, bool) asm (-> 1 0 2) "DICTIMIN" "NULLSWAPIFNOT2"; @pure -fun uDictGetFirst(self: dict, keyLen: int): (int?, slice?, bool) +fun dict.uDictGetFirst(self, keyLen: int): (int?, slice?, bool) asm (-> 1 0 2) "DICTUMIN" "NULLSWAPIFNOT2"; @pure -fun sDictGetFirst(self: dict, keyLen: int): (slice?, slice?, bool) +fun dict.sDictGetFirst(self, keyLen: int): (slice?, slice?, bool) asm (-> 1 0 2) "DICTMIN" "NULLSWAPIFNOT2"; @pure -fun iDictGetFirstAsRef(self: dict, keyLen: int): (int?, dict, bool) +fun dict.iDictGetFirstAsRef(self, keyLen: int): (int?, dict, bool) asm (-> 1 0 2) "DICTIMINREF" "NULLSWAPIFNOT2"; @pure -fun uDictGetFirstAsRef(self: dict, keyLen: int): (int?, dict, bool) +fun dict.uDictGetFirstAsRef(self, keyLen: int): (int?, dict, bool) asm (-> 1 0 2) "DICTUMINREF" "NULLSWAPIFNOT2"; @pure -fun sDictGetFirstAsRef(self: dict, keyLen: int): (slice?, dict, bool) +fun dict.sDictGetFirstAsRef(self, keyLen: int): (slice?, dict, bool) asm (-> 1 0 2) "DICTMINREF" "NULLSWAPIFNOT2"; @pure -fun iDictGetLast(self: dict, keyLen: int): (int?, slice?, bool) +fun dict.iDictGetLast(self, keyLen: int): (int?, slice?, bool) asm (-> 1 0 2) "DICTIMAX" "NULLSWAPIFNOT2"; @pure -fun uDictGetLast(self: dict, keyLen: int): (int?, slice?, bool) +fun dict.uDictGetLast(self, keyLen: int): (int?, slice?, bool) asm (-> 1 0 2) "DICTUMAX" "NULLSWAPIFNOT2"; @pure -fun sDictGetLast(self: dict, keyLen: int): (slice?, slice?, bool) +fun dict.sDictGetLast(self, keyLen: int): (slice?, slice?, bool) asm (-> 1 0 2) "DICTMAX" "NULLSWAPIFNOT2"; @pure -fun iDictGetLastAsRef(self: dict, keyLen: int): (int?, dict, bool) +fun dict.iDictGetLastAsRef(self, keyLen: int): (int?, dict, bool) asm (-> 1 0 2) "DICTIMAXREF" "NULLSWAPIFNOT2"; @pure -fun uDictGetLastAsRef(self: dict, keyLen: int): (int?, dict, bool) +fun dict.uDictGetLastAsRef(self, keyLen: int): (int?, dict, bool) asm (-> 1 0 2) "DICTUMAXREF" "NULLSWAPIFNOT2"; @pure -fun sDictGetLastAsRef(self: dict, keyLen: int): (slice?, dict, bool) +fun dict.sDictGetLastAsRef(self, keyLen: int): (slice?, dict, bool) asm (-> 1 0 2) "DICTMAXREF" "NULLSWAPIFNOT2"; @pure -fun iDictGetNext(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) +fun dict.iDictGetNext(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXT" "NULLSWAPIFNOT2"; @pure -fun uDictGetNext(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) +fun dict.uDictGetNext(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXT" "NULLSWAPIFNOT2"; @pure -fun iDictGetNextOrEqual(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) +fun dict.iDictGetNextOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXTEQ" "NULLSWAPIFNOT2"; @pure -fun uDictGetNextOrEqual(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) +fun dict.uDictGetNextOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXTEQ" "NULLSWAPIFNOT2"; @pure -fun iDictGetPrev(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) +fun dict.iDictGetPrev(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTIGETPREV" "NULLSWAPIFNOT2"; @pure -fun uDictGetPrev(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) +fun dict.uDictGetPrev(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTUGETPREV" "NULLSWAPIFNOT2"; @pure -fun iDictGetPrevOrEqual(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) +fun dict.iDictGetPrevOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTIGETPREVEQ" "NULLSWAPIFNOT2"; @pure -fun uDictGetPrevOrEqual(self: dict, keyLen: int, pivot: int): (int?, slice?, bool) +fun dict.uDictGetPrevOrEqual(self, keyLen: int, pivot: int): (int?, slice?, bool) asm(pivot self keyLen -> 1 0 2) "DICTUGETPREVEQ" "NULLSWAPIFNOT2"; @@ -304,13 +304,13 @@ fun uDictGetPrevOrEqual(self: dict, keyLen: int, pivot: int): (int?, slice?, boo */ @pure -fun prefixDictGet(self: dict, keyLen: int, key: slice): (slice, slice?, slice?, bool) +fun dict.prefixDictGet(self, keyLen: int, key: slice): (slice, slice?, slice?, bool) asm(key self keyLen) "PFXDICTGETQ" "NULLSWAPIFNOT2"; @pure -fun prefixDictSet(mutate self: dict, keyLen: int, key: slice, value: slice): bool +fun dict.prefixDictSet(mutate self, keyLen: int, key: slice, value: slice): bool asm(value key self keyLen) "PFXDICTSET"; @pure -fun prefixDictDelete(mutate self: dict, keyLen: int, key: slice): bool +fun dict.prefixDictDelete(mutate self, keyLen: int, key: slice): bool asm(key self keyLen) "PFXDICTDEL"; diff --git a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk index a7f158248..075514539 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk @@ -21,5 +21,5 @@ fun transformSliceToContinuation(s: slice): continuation /// Moves a variable or a value [x] to the top of the stack. @pure -fun stackMoveToTop(mutate self: X): void +fun T.stackMoveToTop(mutate self): void asm "NOP"; diff --git a/tolk-tester/tests/allow-post-modification.tolk b/tolk-tester/tests/allow-post-modification.tolk index 49a2eaec5..6d3abf057 100644 --- a/tolk-tester/tests/allow-post-modification.tolk +++ b/tolk-tester/tests/allow-post-modification.tolk @@ -4,7 +4,7 @@ fun unsafe_tuple(x: X): tuple fun inc(x: int, y: int): (int, int) { return (x + y, y * 10); } -fun `~inc`(mutate self: int, y: int): int { +fun int.`~inc`(mutate self, y: int): int { val (newX, newY) = inc(self, y); self = newX; return newY; @@ -13,7 +13,7 @@ fun `~inc`(mutate self: int, y: int): int { fun eq(v: X): X { return v; } fun eq2(v: (int, int)) { return v; } fun mul2(mutate dest: int, v: int): int { dest = v*2; return dest; } -fun multens(mutate self: (int, int), v: (int, int)): (int, int) { var (f, s) = self; var (m1, m2) = v; self = (f*m1, s*m2); return self; } +fun (int, int).multens(mutate self, v: (int, int)): (int, int) { var (f, s) = self; var (m1, m2) = v; self = (f*m1, s*m2); return self; } @method_id(11) fun test_return(x: int): (int, int, int, int, int, int, int) { @@ -102,14 +102,14 @@ fun test_assign_with_mutate(x: int) { @method_id(23) fun test_assign_tensor(x: (int, int)) { var fs = (0, 0); - return (x, x = (20, 30), fs = x.multens((1, 2)), fs.multens(multens(mutate x, (-1, -1))), x, fs); + return (x, x = (20, 30), fs = x.multens((1, 2)), fs.multens(x.multens((-1, -1))), x, fs); } global fs: (int, int); @method_id(24) fun test_assign_tensor_global(x: (int, int)) { fs = (0, 0); - return (x, x = (20, 30), fs = x.multens((1, 2)), fs.multens(multens(mutate x, (-1, -1))), x, fs); + return (x, x = (20, 30), fs = x.multens((1, 2)), fs.multens(x.multens((-1, -1))), x, fs); } struct Point1D { @@ -146,7 +146,7 @@ fun main() { @fif_codegen """ - ~inc PROC:<{ // self y + int.~inc PROC:<{ // self y inc CALLDICT // self newY }> """ diff --git a/tolk-tester/tests/asm-arg-order.tolk b/tolk-tester/tests/asm-arg-order.tolk index b96e09ecb..841dc9564 100644 --- a/tolk-tester/tests/asm-arg-order.tolk +++ b/tolk-tester/tests/asm-arg-order.tolk @@ -2,7 +2,7 @@ fun empty_tuple2(): tuple asm "NIL"; @pure -fun tpush2(mutate self: tuple, x: X): void +fun tuple.tpush2(mutate self, x: X): void asm "TPUSH"; @pure @@ -15,11 +15,14 @@ asm (z y x -> 0) "3 TUPLE"; fun asm_func_3(x: int, y: int, z: int): tuple asm (y z x -> 0) "3 TUPLE"; @pure +fun int.asm_func_3(self, y: int, z: int): tuple +asm (y z self -> 0) "3 TUPLE"; +@pure fun asm_func_4(a: int, b: (int, (int, int)), c: int): tuple asm (b a c -> 0) "5 TUPLE"; @pure -fun asm_func_modify(mutate self: tuple, b: int, c: int): void +fun tuple.asm_func_modify(mutate self, b: int, c: int): void asm (c b self) "SWAP TPUSH SWAP TPUSH"; global t: tuple; diff --git a/tolk-tester/tests/assignment-tests.tolk b/tolk-tester/tests/assignment-tests.tolk index 2301c9b70..a52b3ca50 100644 --- a/tolk-tester/tests/assignment-tests.tolk +++ b/tolk-tester/tests/assignment-tests.tolk @@ -21,7 +21,7 @@ fun typesAsIdentifiers(builder: builder) { { var cell: cell = cell; var tuple: tuple = createEmptyTuple(); - var bool: bool = tuple.tupleAt(0) > 0; + var bool: bool = tuple.get(0) > 0; } return int; } @@ -29,37 +29,37 @@ fun typesAsIdentifiers(builder: builder) { global callOrder: tuple; fun getTensor_12() { - callOrder.tuplePush(100); + callOrder.push(100); return (1, 2); } fun getTensor_1X(x: int) { - callOrder.tuplePush(101); + callOrder.push(101); return (1, x); } fun getTuple_12() { - callOrder.tuplePush(110); + callOrder.push(110); return [1, 2, ]; } fun getTuple_1X(x: int) { - callOrder.tuplePush(111); + callOrder.push(111); return [1, x]; } fun getUntypedTuple_12() { - callOrder.tuplePush(120); - var t = createEmptyTuple(); t.tuplePush(1); t.tuplePush(2); + callOrder.push(120); + var t = createEmptyTuple(); t.push(1); t.push(2); return t; } fun getUntypedTuple_1X(x: int) { - callOrder.tuplePush(121); - var t = createEmptyTuple(); t.tuplePush(1); t.tuplePush(x); + callOrder.push(121); + var t = createEmptyTuple(); t.push(1); t.push(x); return t; } fun getIntValue5() { - callOrder.tuplePush(10); + callOrder.push(10); return 5; } fun getIntValueX(x: int) { - callOrder.tuplePush(11); + callOrder.push(11); return x; } diff --git a/tolk-tester/tests/bytesN-tests.tolk b/tolk-tester/tests/bytesN-tests.tolk index 32b975bf7..7f1a76e2f 100644 --- a/tolk-tester/tests/bytesN-tests.tolk +++ b/tolk-tester/tests/bytesN-tests.tolk @@ -6,8 +6,8 @@ const someBytes = "asdf" as bytes16; const anotherBytes: bytes16 = "asdf" as bytes16; fun autoInferBytes16() { - if (random()) { return someBytes; } - else if (random()) { return true ? "" as bytes16 : anotherBytes; } + if (random.uint256()) { return someBytes; } + else if (random.uint256()) { return true ? "" as bytes16 : anotherBytes; } else { return someBytes!; } } @@ -20,7 +20,7 @@ fun test1() { __expect_type(ss as bytes128, "bytes128"); __expect_type(someBytes, "bytes16"); __expect_type(10>3 ? (null, ss as bits1) : (ss as bytes1, null), "(bytes1?, bits1?)"); - return (getRemainingBitsCount(ss as slice), loadInt(mutate ss as slice, 32), (ss as slice).loadInt(32)); + return ((ss as slice).remainingBitsCount(), (ss as slice).loadInt(32), (ss as slice).loadInt(32)); } fun test2(a: bytes8, b: bits64) { @@ -35,7 +35,7 @@ fun test3() { var slice = beginCell().storeInt(1, 32).storeInt(2, 32).endCell().beginParse(); var bq = slice as bits16?; if (bq != null) { - return (bq as slice).loadInt(32 as uint8) + (slice = bq as slice).getRemainingBitsCount(); + return (bq as slice).loadInt(32 as uint8) + (slice = bq as slice).remainingBitsCount(); } return -1; } diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index e900268bb..63a823ed9 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -1,15 +1,21 @@ -fun store_u32(mutate self: builder, value: int): self { +fun builder.store_u32(mutate self, value: int): self { return self.storeUint(value, 32); } -fun load_u32(mutate self: slice): int { +fun slice.load_u32(mutate self): int { return self.loadUint(32); } -fun myLoadInt(mutate self: slice, len: int): int +fun slice.myLoadInt(mutate self, len: int): int asm(-> 1 0) "LDIX"; -fun myStoreInt(mutate self: builder, x: int, len: int): self +fun builder.myStoreInt(mutate self, x: int, len: int): self asm(x self len) "STIX"; +@pure +fun endCell(b: builder): cell + asm "ENDC"; +@pure +fun beginParse(c: cell): slice + asm "CTOS"; @method_id(101) fun test1(): [int,int,int,int,int] { @@ -80,7 +86,7 @@ fun test7() { var uri_builder = beginCell(); var uri_slice = uri_builder.storeSlice(".json").endCell().beginParse(); var image_slice = uri_builder.storeSlice(".png").endCell().beginParse(); - return (uri_builder.getBuilderBitsCount(), uri_slice.getRemainingBitsCount(), image_slice.getRemainingBitsCount()); + return (uri_builder.bitsCount(), uri_slice.remainingBitsCount(), image_slice.remainingBitsCount()); } @method_id(108) @@ -90,13 +96,13 @@ fun test8() { var uri_slice = fresh.storeSlice(".json").endCell().beginParse(); var fresh redef = uri_builder; var image_slice = fresh.storeSlice(".png").endCell().beginParse(); - return (uri_builder.getBuilderBitsCount(), uri_slice.getRemainingBitsCount(), image_slice.getRemainingBitsCount()); + return (uri_builder.bitsCount(), uri_slice.remainingBitsCount(), image_slice.remainingBitsCount()); } -fun sumNumbersInSlice(mutate self: slice): int { +fun slice.sumNumbersInSlice(mutate self): int { var result = 0; - while (!self.isEndOfSliceBits()) { + while (!self.isEndOfBits()) { result += self.loadUint(32); } return result; @@ -106,27 +112,27 @@ fun sumNumbersInSlice(mutate self: slice): int { fun test10() { var ref = beginCell().storeInt(100, 32).endCell(); var s: slice = beginCell().storeInt(1, 32).storeInt(2, 32).storeRef(ref).endCell().beginParse(); - var result = (getRemainingBitsCount(s), s.sumNumbersInSlice(), getRemainingBitsCount(s), isEndOfSlice(s), isEndOfSliceBits(s), isEndOfSliceRefs(s)); + var result = (s.remainingBitsCount(), s.sumNumbersInSlice(), s.remainingBitsCount(), s.isEnd(), s.isEndOfBits(), s.isEndOfRefs()); var ref2: cell = s.loadRef(); var s2: slice = ref2.beginParse(); - s.assertEndOfSlice(); - return (result, s2.loadInt(32), s2.isEndOfSlice()); + s.assertEnd(); + return (result, s2.loadInt(32), s2.isEnd()); } @method_id(111) fun test11() { var s: slice = beginCell().storeInt(1, 32).storeInt(2, 32).storeInt(3, 32).storeInt(4, 32).storeInt(5, 32).storeInt(6, 32).storeInt(7, 32).endCell().beginParse(); - var size1 = getRemainingBitsCount(s); + var size1 = s.remainingBitsCount(); s!.skipBits(32); var s1: slice = s.getFirstBits(64); var n1 = s1.loadInt(32); - var size2 = getRemainingBitsCount(s); + var size2 = s.remainingBitsCount(); s.loadInt(32); - var size3 = getRemainingBitsCount(s); + var size3 = s.remainingBitsCount(); s.removeLastBits(32); - var size4 = getRemainingBitsCount(s); + var size4 = s.remainingBitsCount(); var n2 = s.loadInt(32); - var size5 = getRemainingBitsCount(s); + var size5 = s.remainingBitsCount(); return (n1, n2, size1, size2, size3, size4, size5); } @@ -134,13 +140,13 @@ fun test11() { fun test12() { var (result1, result2) = (0, 0); try { - beginCell().storeRef(beginCell().endCell()).endCell().beginParse().assertEndOfSlice(); + beginCell().storeRef(beginCell().endCell()).endCell().beginParse().assertEnd(); result1 = 100; } catch (code) { result1 = code; } try { - beginCell().endCell().beginParse().assertEndOfSlice(); + beginCell().endCell().beginParse().assertEnd(); result2 = 100; } catch (code) { result2 = code; @@ -153,12 +159,12 @@ fun test13() { var ref2 = beginCell().storeInt(1, 32).endCell(); var ref1 = beginCell().storeInt(1, 32).storeRef(ref2).endCell(); var c = beginCell().storeInt(444, 32).storeRef(ref1).storeRef(ref1).storeRef(ref1).storeRef(ref2).storeInt(4, 32).endCell(); - var (n_cells1, n_bits1, n_refs1) = c.calculateCellSizeStrict(10); + var (n_cells1, n_bits1, n_refs1) = c.calculateSizeStrict(10); var s = c.beginParse(); s.loadRef(); s.loadRef(); var n = s.loadInt(32); - var (n_cells2, n_bits2, n_refs2) = s.calculateSliceSizeStrict(10); + var (n_cells2, n_bits2, n_refs2) = s.calculateSizeStrict(10); return ([n_cells1, n_bits1, n_refs1], [n_cells2, n_bits2, n_refs2], n); } @@ -177,7 +183,7 @@ fun test111() { .endCell().beginParse(); var op1 = s.loadUint(32); var q1 = s.loadUint(64); - if (s.addressIsNone()) { + if (addressIsNone(s)) { s.skipBits(2); } if (s.loadBool() == false) { @@ -187,7 +193,7 @@ fun test111() { var op2 = s.loadMessageOp(); var q2 = s.loadMessageQueryId(); s.skipBits(64); - s.assertEndOfSlice(); + s.assertEnd(); assert(isMessageBounced(0x001) && !isMessageBounced(0x002)) throw 444; return (op1, q1, op2, q2); } diff --git a/tolk-tester/tests/constants-tests.tolk b/tolk-tester/tests/constants-tests.tolk index 128a5f7ea..ba213093b 100644 --- a/tolk-tester/tests/constants-tests.tolk +++ b/tolk-tester/tests/constants-tests.tolk @@ -53,13 +53,13 @@ fun iget240(): MInt { return int240; } fun newc(): builder asm "NEWC"; @pure -fun endcs(b: builder): slice +fun builder.endcs(self): slice asm "ENDC" "CTOS"; @pure fun sdeq(s1: slice, s2: slice): MInt asm "SDEQ"; @pure -fun stslicer(b: builder, s: slice): builder +fun builder.stslicer(self, s: slice): builder asm "STSLICER"; @method_id(101) diff --git a/tolk-tester/tests/dicts-demo.tolk b/tolk-tester/tests/dicts-demo.tolk index 606318cb3..1ff4cd8c3 100644 --- a/tolk-tester/tests/dicts-demo.tolk +++ b/tolk-tester/tests/dicts-demo.tolk @@ -1,6 +1,6 @@ import "@stdlib/tvm-dicts" -fun addIntToIDict(mutate self: cell?, key: int, number: int): void { +fun cell?.addIntToIDict(mutate self, key: int, number: int): void { return self.iDictSetBuilder(32, key, beginCell().storeInt(number, 32)); } @@ -14,7 +14,7 @@ fun calculateDictLen(d: cell?) { return len; } -fun loadTwoDigitNumberFromSlice(mutate self: slice): int { +fun slice.loadTwoDigitNumberFromSlice(mutate self): int { var n1 = self.loadInt(8); var n2 = self.loadInt(8); return (n1 - 48) * 10 + (n2 - 48); @@ -47,7 +47,7 @@ fun test102() { while (!shouldBreak) { var (kDel, kVal, wasDel) = dict.iDictDeleteLastAndGet(32); if (wasDel) { - deleted.tuplePush([kDel, kVal!.loadInt(32)]); + deleted.push([kDel, kVal!.loadInt(32)]); } else { shouldBreak = true; } @@ -82,8 +82,8 @@ fun test104() { var (old2, _) = dict.sDictDeleteAndGet(32, "key1"); var (restK, restV, _) = dict.sDictGetFirst(32); var (restK1, restV1, _) = dict.sDictDeleteLastAndGet(32); - assert (restK!.isSliceBitsEqual(restK1!)) throw 123; - assert (restV!.isSliceBitsEqual(restV1!)) throw 123; + assert (restK!.bitsEqual(restK1!)) throw 123; + assert (restV!.bitsEqual(restV1!)) throw 123; return ( old1!.loadTwoDigitNumberFromSlice(), old2!.loadTwoDigitNumberFromSlice(), diff --git a/tolk-tester/tests/generics-1.tolk b/tolk-tester/tests/generics-1.tolk index ba46f3440..0984dcd51 100644 --- a/tolk-tester/tests/generics-1.tolk +++ b/tolk-tester/tests/generics-1.tolk @@ -7,6 +7,10 @@ fun eq2(value: X) { return value; } fun eq3(value: X): X { var cp: [X] = [eq1(value)]; var ((([v: X]))) = cp; return v; } fun eq4(value: X) { return eq1(value); } +@pure +fun tuplePush(mutate t: tuple, value: T): void + asm "TPUSH"; + @method_id(101) fun test101(x: int) { var (a, b, c) = (x, (x,x), [x,x]); @@ -49,12 +53,12 @@ fun test102(): (int, int, int, [int, int], Wrapper< Wrapper >) { fun test103(first: int): (int, int, int) { var t = createEmptyTuple(); var cs = beginCell().storeInt(100, 32).endCell().beginParse(); - t.tuplePush(first); - t.tuplePush(2); - t.tuplePush(cs); - cs = t.tupleAt(2); - cs = t.tupleAt(2) as slice; - return (t.tupleAt(0), cs.loadInt(32), t.tupleAt(2).loadInt(32)); + t.push(first); + t.push(2); + t.push(cs); + cs = t.get(2); + cs = t.get(2) as slice; + return (t.get(0), cs.loadInt(32), t.get(2).loadInt(32)); } fun manyEq(a: T1, b: T2, c: T3): [T1, T2, T3] { @@ -101,10 +105,11 @@ fun test106() { ]; } -fun callTupleFirst(t: X): Y { return t.tupleFirst(); } -fun callTuplePush(mutate self: T, v1: V, v2: V): self { self.tuplePush(v1); tuplePush(mutate self, v2); return self; } -fun getTupleLastInt(t: tuple) { return t.tupleLast(); } -fun getTupleSize(t: MTuple) { return t.tupleSize(); } +fun X.callTupleFirst(self): Y { return self.first(); } +fun callTupleFirst(t: X): Y { return t.first(); } +fun T.callTuplePush(mutate self, v1: V, v2: V): self { self.push(v1); tuplePush(mutate self, v2); return self; } +fun getTupleLastInt(t: tuple) { return t.last(); } +fun getTupleSize(t: MTuple) { return t.size(); } fun callAnyFn(f: (TObj) -> TResult, arg: TObj) { return f(arg); } fun callAnyFn2(f: TCallback, arg: tuple) { return f(arg); } @@ -114,7 +119,7 @@ type MTuple = tuple; @method_id(107) fun test107() { t107 = createEmptyTuple(); - callTuplePush(mutate t107, 1, 2); + t107.callTuplePush(1, 2); t107.callTuplePush(3, 4).callTuplePush(5, 6); var first: int = t107.callTupleFirst(); return ( diff --git a/tolk-tester/tests/generics-2.tolk b/tolk-tester/tests/generics-2.tolk index b3e0ea5b9..8e95d884b 100644 --- a/tolk-tester/tests/generics-2.tolk +++ b/tolk-tester/tests/generics-2.tolk @@ -65,11 +65,21 @@ fun getWrappervalue1(c: Wrapper) { return c.value; } +@pure +fun Wrapper.getWrappervalue1(self) { + return self.value; +} + @pure fun getWrappervalue2(c: T) { return c.value; } +@pure +fun T.getWrappervalue2(self) { + return self.value; +} + @method_id(103) fun test3() { var c1 = wrap(10); @@ -405,6 +415,17 @@ fun test24() { __expect_type(o, "HasSomethingAndWrapper"); } +fun test25() { + var p1 = Pair { first: null, second: null }; + __expect_type(p1, "Pair"); + + try {} + catch(e, second) { + var p2 = Pair { first: null, second }; + __expect_type(p2, "Pair"); + } +} + fun main(c: Wrapper, d: WrappedInt) { __expect_type(c, "Wrapper"); diff --git a/tolk-tester/tests/generics-3.tolk b/tolk-tester/tests/generics-3.tolk index 77d046113..4539d6144 100644 --- a/tolk-tester/tests/generics-3.tolk +++ b/tolk-tester/tests/generics-3.tolk @@ -118,7 +118,7 @@ struct WithDef { fun test8() { var w1: WithDef = {}; var w2: WithDef> = { price: ton("0.1"), f3: { result: 12 } }; - return (w1.price, w1.f1, w1.f3, w2.price, w2.f3!.result, w2.f3, w2.slice.getRemainingBitsCount()); + return (w1.price, w1.f1, w1.f3, w2.price, w2.f3!.result, w2.f3, w2.slice.remainingBitsCount()); } diff --git a/tolk-tester/tests/if-else-tests.tolk b/tolk-tester/tests/if-else-tests.tolk index 8c314b8ed..159a86bd5 100644 --- a/tolk-tester/tests/if-else-tests.tolk +++ b/tolk-tester/tests/if-else-tests.tolk @@ -68,7 +68,7 @@ fun doNothing(): void {} @method_id(107) fun test7() { - if (random()) { return doNothing() } + if (random.uint256()) { return doNothing() } // here we test that no "missing return" error } diff --git a/tolk-tester/tests/indexed-access.tolk b/tolk-tester/tests/indexed-access.tolk index f6a376ad3..f84d2d93e 100644 --- a/tolk-tester/tests/indexed-access.tolk +++ b/tolk-tester/tests/indexed-access.tolk @@ -2,10 +2,21 @@ type Tup2Int = [int, MInt]; type Tensor2Int = (int, MInt); type MInt = int; -fun increment(mutate self: MInt) { +@pure +fun tuplePush(mutate t: tuple, value: T): void + asm "TPUSH"; +@pure +fun getBuilderBitsCount(b: builder): int + asm "BBITS"; + +fun MInt.increment(mutate self) { self += 1; } +fun increment(mutate v: MInt) { + v += 1; +} + fun increment2(mutate a: int, mutate b: int) { a += 1; b += 1; @@ -16,12 +27,18 @@ fun assign1020(mutate a: int, mutate b: MInt) { b = 20; } -fun plus(mutate self: int, y: int): int { +fun int.plus(mutate self, y: int): int { val newVals = (self + y, y * 10); self = newVals.0; return newVals.1; } +fun plus(mutate x: int, y: int): int { + val newVals = (x + y, y * 10); + x = newVals.0; + return newVals.1; +} + fun eq(v: X): X { return v; } global gTup: [int]; @@ -31,16 +48,16 @@ global gTens: (int, int); fun testCodegenSimple() { var t1 = [1]; t1.0 = 2; - debugPrintString(""); + debug.printString(""); var t2 = [[1]]; t2.0.0 = 2; - debugPrintString(""); + debug.printString(""); gTup = [1]; gTup.0 = 2; - debugPrintString(""); + debug.printString(""); gTens = (1,2); gTens.1 = 4; - debugPrintString(""); + debug.printString(""); return (t1, t2, gTup, gTens); } @@ -90,8 +107,8 @@ fun test104() { @method_id(105) fun test105(x: int, y: int): (tuple, int, (int?, int), int, int) { - var ab = (createEmptyTuple(), (x as int?, y), tupleSize); - ab.0.tuplePush(1); + var ab = (createEmptyTuple(), (x as int?, y), tuple.size); + ab.0.push(1); tuplePush(mutate ab.0, 2); ab.1.0 = null; ab.1.1 += 10; @@ -101,8 +118,8 @@ fun test105(x: int, y: int): (tuple, int, (int?, int), int, int) { @method_id(106) fun test106(x: int, y: int) { - var ab = [createEmptyTuple(), [x as int?, y], tupleSize]; - ab.0.tuplePush(1); + var ab = [createEmptyTuple(), [x as int?, y], tuple.size]; + ab.0.push(1); tuplePush(mutate ab.0, 2); ab.1.0 = null; ab.1.1 += 10; @@ -113,8 +130,8 @@ fun test106(x: int, y: int) { @method_id(107) fun test107() { var ab = createEmptyTuple(); - ab.tuplePush(1); - ab.tuplePush(beginCell().storeInt(1, 32)); + ab.push(1); + ab.push(beginCell().storeInt(1, 32)); return (ab.0 as int, getBuilderBitsCount(ab.1)); } @@ -187,7 +204,7 @@ fun test115() { @method_id(116) fun test116() { var t = createEmptyTuple(); - t.tuplePush(1); + t.push(1); try { return t.100500 as int; } catch(excNo) { @@ -198,7 +215,7 @@ fun test116() { @method_id(117) fun test117() { var t = createEmptyTuple(); - t.tuplePush(1); + t.push(1); try { return (t.0 as tuple).0 as MInt; } catch(excNo) { @@ -228,14 +245,18 @@ fun test120() { @method_id(121) fun test121(zero: int) { var t = createEmptyTuple(); - t.tuplePush(-100); - t.tupleSetAt(0, zero); + t.push(-100); + t.set(0, zero); (t.0 as int).increment(); (((t.0) as int) as int).increment(); increment(mutate t.0 as int); return t; } +fun (T1, T2).isFirstComponentGt0(self): bool { + return self.0 > 0; +} + fun isFirstComponentGt0(t: (T1, T2)): bool { return t.0 > 0; } @@ -244,7 +265,7 @@ fun isFirstComponentGt0(t: (T1, T2)): bool { fun test122(x: (int, int)) { return ( isFirstComponentGt0(x), isFirstComponentGt0((2, beginCell())), isFirstComponentGt0((0, null)), - x.isFirstComponentGt0(), (2, beginCell()).isFirstComponentGt0(), (0, null).isFirstComponentGt0() + x.isFirstComponentGt0(), (2, beginCell()).isFirstComponentGt0(), (0, null as slice?).isFirstComponentGt0() ); } diff --git a/tolk-tester/tests/inference-tests.tolk b/tolk-tester/tests/inference-tests.tolk index affe42ee1..9c836baf4 100644 --- a/tolk-tester/tests/inference-tests.tolk +++ b/tolk-tester/tests/inference-tests.tolk @@ -15,7 +15,7 @@ fun test1(x: int, y: int) { __expect_type(x = x, "int"); __expect_type(x += x, "int"); __expect_type(x &= x, "int"); - __expect_type(random() ? x : y, "int"); + __expect_type(random.uint256() ? x : y, "int"); __expect_type(eq(x), "int"); __expect_type(eq(x), "int"); __expect_type(eq(null), "int?"); @@ -47,7 +47,7 @@ fun test2(x: int, y: bool) { __expect_type(x <= x, "bool"); __expect_type(x <=> x, "bool"); __expect_type(x <=> x, "bool"); - __expect_type(!random(), "bool"); + __expect_type(!random.uint256(), "bool"); __expect_type(!!(x != null), "bool"); __expect_type(x ? x != null : null == x, "bool"); __expect_type(y & true, "bool"); @@ -60,7 +60,7 @@ fun test2(x: int, y: bool) { fun test3() { __expect_type(true as int, "int"); - __expect_type(!random() as int, "int"); + __expect_type(!random.uint256() as int, "int"); } fun test4(x: int) { @@ -75,13 +75,13 @@ fun test5(x: int) { __expect_type([x, x >= 1], "[int, bool]"); __expect_type([x, x >= 1, null as slice?], "[int, bool, slice?]"); __expect_type((x, [x], [[x], x]), "(int, [int], [[int], int])"); - __expect_type(getMyOriginalBalanceWithExtraCurrencies(), "[int, dict]"); + __expect_type(contract.getOriginalBalanceWithExtraCurrencies(), "[coins, dict]"); } fun test6() { var t = createEmptyTuple(); __expect_type(t, "tuple"); - t.tuplePush(1); + t.push(1); __expect_type(t, "tuple"); } diff --git a/tolk-tester/tests/intN-tests.tolk b/tolk-tester/tests/intN-tests.tolk index 4b2fccb25..6f44ed030 100644 --- a/tolk-tester/tests/intN-tests.tolk +++ b/tolk-tester/tests/intN-tests.tolk @@ -10,28 +10,29 @@ const someN_coins = 10 as coins; const cost1 = ton("1.234"); const cost2 = ton("0.05") + ton("0.001"); +fun rand(): uint256 { return random.uint256(); } fun autoInferInt8(x: int8) { - if (random()) { return x; } - else if (random()) { return 0 as int8; } - else if (random()) { return x!; } + if (rand()) { return x; } + else if (rand()) { return 0 as int8; } + else if (rand()) { return x!; } else { return x += x; } } fun autoInferUInt16Opt(x: uint16) { - if (random()) { return autoInferInt8(x as int8) as uint16; } + if (rand()) { return autoInferInt8(x as int8) as uint16; } return null; } fun inferInt_v1(): int { - if (random()) { return 0; } - else if (random()) { return 0 as uint16; } + if (rand()) { return 0; } + else if (rand()) { return 0 as uint16; } else { return 0 as int8; } } fun inferInt_v2(): int { - if (random()) { return 0 as uint1; } - else if (random()) { return 0 as int123; } + if (rand()) { return 0 as uint1; } + else if (rand()) { return 0 as int123; } else { return 0 as int8; } } @@ -218,9 +219,9 @@ fun assign0(mutate v: T) { v = 0; } fun main() { var t = createEmptyTuple(); - t.tuplePush(1); - t.tuplePush(2); - t.tuplePush(3); + t.push(1); + t.push(2); + t.push(3); assign0(mutate t.0 as int8); assign0(mutate t.1 as uint16); assign0(mutate t.2 as int); diff --git a/tolk-tester/tests/invalid-declaration/err-1074.tolk b/tolk-tester/tests/invalid-declaration/err-1074.tolk new file mode 100644 index 000000000..b3298436f --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1074.tolk @@ -0,0 +1,6 @@ +fun Container.create() {} + +/** +@compilation_should_fail +@stderr unknown type name `Container` + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1140.tolk b/tolk-tester/tests/invalid-declaration/err-1140.tolk index eb1ca04e5..ab902fc26 100644 --- a/tolk-tester/tests/invalid-declaration/err-1140.tolk +++ b/tolk-tester/tests/invalid-declaration/err-1140.tolk @@ -21,6 +21,7 @@ fun autoInferUnionType(a: int8, b: int16) { /** @compilation_should_fail -@stderr function `autoInferUnionType` calculated return type is `int8 | int16`; probably, it's not what you expected; declare `fun (...): ` manually +@stderr function `autoInferUnionType` calculated return type is `int8 | int16`; probably, it's not what you expected +@stderr declare `fun (...): ` manually @stderr fun autoInferUnionType */ diff --git a/tolk-tester/tests/invalid-declaration/err-1193.tolk b/tolk-tester/tests/invalid-declaration/err-1193.tolk new file mode 100644 index 000000000..45124998e --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1193.tolk @@ -0,0 +1,7 @@ +fun int.smth(self: int) { +} + +/** +@compilation_should_fail +@stderr `self` parameter should not have a type + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1226.tolk b/tolk-tester/tests/invalid-declaration/err-1226.tolk new file mode 100644 index 000000000..12620289c --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1226.tolk @@ -0,0 +1,14 @@ +type MInt = int; +type MInt2 = MInt; + +fun int.check(self) {} +fun MInt.check(self) {} +fun MInt2.check(self) {} +fun `MInt`.`check`(self) {} + +/** +@compilation_should_fail +@stderr redefinition of symbol +@stderr err-1226.tolk:7 +@stderr err-1226.tolk:5 + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1458.tolk b/tolk-tester/tests/invalid-declaration/err-1458.tolk new file mode 100644 index 000000000..c1d24dfa2 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1458.tolk @@ -0,0 +1,13 @@ +struct Pair { + first: A; + second: B; +} + +type PairAlias = Pair; + +fun PairAlias.create() {} + +/** +@compilation_should_fail +@stderr type `PairAlias` expects 2 type arguments, but 1 provided + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1556.tolk b/tolk-tester/tests/invalid-declaration/err-1556.tolk new file mode 100644 index 000000000..831ab2d49 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1556.tolk @@ -0,0 +1,7 @@ +fun nothing(self) { +} + +/** +@compilation_should_fail +@stderr `self` can only be the first parameter of a method + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1601.tolk b/tolk-tester/tests/invalid-declaration/err-1601.tolk deleted file mode 100644 index 588c70ab2..000000000 --- a/tolk-tester/tests/invalid-declaration/err-1601.tolk +++ /dev/null @@ -1,8 +0,0 @@ -fun increment(x: int, self: int): int { - return x + self; -} - -/** -@compilation_should_fail -@stderr `self` can only be the first parameter - */ diff --git a/tolk-tester/tests/invalid-declaration/err-1663.tolk b/tolk-tester/tests/invalid-declaration/err-1663.tolk new file mode 100644 index 000000000..0c41e4c2c --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1663.tolk @@ -0,0 +1,11 @@ +struct Container { + item: T; +} + +fun Container.wrap(item: T) {} + +/** +@compilation_should_fail +@stderr duplicate generic parameter `T` +@stderr fun Container.wrap + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1701.tolk b/tolk-tester/tests/invalid-declaration/err-1701.tolk index 6a7f1ca7f..e942ca470 100644 --- a/tolk-tester/tests/invalid-declaration/err-1701.tolk +++ b/tolk-tester/tests/invalid-declaration/err-1701.tolk @@ -1,10 +1,12 @@ +// try to cheat fun moddiv2(x: int, y: int): (int, int) builtin; +// but actually, it remains unknown symbol + +fun main() { + moddiv2(1, 2); +} /** @compilation_should_fail -@stderr -""" -`builtin` used for non-builtin function -fun moddiv2 -""" +@stderr undefined symbol `moddiv2` */ diff --git a/tolk-tester/tests/invalid-declaration/err-1790.tolk b/tolk-tester/tests/invalid-declaration/err-1790.tolk index 75ebb450b..4a845db8d 100644 --- a/tolk-tester/tests/invalid-declaration/err-1790.tolk +++ b/tolk-tester/tests/invalid-declaration/err-1790.tolk @@ -1,7 +1,7 @@ // this function is declared incorrectly, // since it should return 2 values onto a stack (1 for returned slice, 1 for mutated int) // but contains not 2 numbers in asm ret_order -fun loadAddress2(mutate self: int): slice +fun int.loadAddress2(mutate self): slice asm( -> 1 0 2) "LDMSGADDR"; fun main(){} diff --git a/tolk-tester/tests/invalid-declaration/err-1794.tolk b/tolk-tester/tests/invalid-declaration/err-1794.tolk index 42cb7b953..a064a633c 100644 --- a/tolk-tester/tests/invalid-declaration/err-1794.tolk +++ b/tolk-tester/tests/invalid-declaration/err-1794.tolk @@ -1,8 +1,8 @@ -get seqno(self: int) { +get int.seqno(self) { return 0; } /** @compilation_should_fail -@stderr get methods can't have `mutate` and `self` params +@stderr invalid declaration of a get method */ diff --git a/tolk-tester/tests/invalid-declaration/err-1912.tolk b/tolk-tester/tests/invalid-declaration/err-1912.tolk new file mode 100644 index 000000000..2674580ca --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1912.tolk @@ -0,0 +1,9 @@ +struct Point { x: int, y: int } + +fun Point.create() {} + +/** +@compilation_should_fail +@stderr type `Point` is not generic +@stderr fun Point + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1973.tolk b/tolk-tester/tests/invalid-declaration/err-1973.tolk new file mode 100644 index 000000000..d00819e4b --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1973.tolk @@ -0,0 +1,7 @@ +fun int.nothing(zero: int, self: slice) { +} + +/** +@compilation_should_fail +@stderr `self` can only be the first parameter of a method + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4057.tolk b/tolk-tester/tests/invalid-semantics/err-4057.tolk new file mode 100644 index 000000000..a5de199dc --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4057.tolk @@ -0,0 +1,17 @@ +fun int16.plus1(self) { return self + 100; } +fun int8.plus1(self) { return self + 1; } + +fun main() { + (10 as int16).plus1(); // ok + (10 as int8).plus1(); // ok + + 10.plus1(); +} + +/** +@compilation_should_fail +@stderr call to method `plus1` for type `int` is ambiguous +@stderr candidate function: `int16.plus1` +@stderr candidate function: `int8.plus1` +@stderr 10.plus1 + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4064.tolk b/tolk-tester/tests/invalid-semantics/err-4064.tolk index 4f0e9142a..e2336ab3b 100644 --- a/tolk-tester/tests/invalid-semantics/err-4064.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4064.tolk @@ -12,9 +12,6 @@ fun main(): int { /** @compilation_should_fail -@stderr -""" -an impure operation in a pure function -return f_impure(); -""" +@stderr an impure operation in a pure function +@stderr in function `f_pure` */ diff --git a/tolk-tester/tests/invalid-semantics/err-4080.tolk b/tolk-tester/tests/invalid-semantics/err-4080.tolk index c8f7dcebf..df569995f 100644 --- a/tolk-tester/tests/invalid-semantics/err-4080.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4080.tolk @@ -1,4 +1,4 @@ -fun methodWith1Param(self: int, param: int) { +fun int.methodWith1Param(self, param: int) { } @@ -9,5 +9,5 @@ fun main() { /** @compilation_should_fail -@stderr too many arguments in call to `methodWith1Param`, expected 1, have 2 +@stderr too many arguments in call to `int.methodWith1Param`, expected 1, have 2 */ diff --git a/tolk-tester/tests/invalid-semantics/err-4083.tolk b/tolk-tester/tests/invalid-semantics/err-4083.tolk index 9da6e2534..aac7417bd 100644 --- a/tolk-tester/tests/invalid-semantics/err-4083.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4083.tolk @@ -1,6 +1,6 @@ fun cantCallMutatingFunctionWithAssignmentLValue() { var t: tuple = createEmptyTuple(); - (t = createEmptyTuple()).tuplePush(1); + (t = createEmptyTuple()).push(1); } /** diff --git a/tolk-tester/tests/invalid-semantics/err-4104.tolk b/tolk-tester/tests/invalid-semantics/err-4104.tolk index 73e6403fd..3ca7ef1bc 100644 --- a/tolk-tester/tests/invalid-semantics/err-4104.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4104.tolk @@ -1,10 +1,20 @@ -fun eq(t: X) { return t; } +struct Container { + item: T; +} + +fun Container.createFrom(item: U): Container { + return { item }; +} + +fun cantUseAsNonCall() { + Container.createFrom; // ok + Container.createFrom>; // ok -fun failUsingGenericFunctionPartially() { - var cb = createEmptyTuple().eq().eq().tuplePush; + Container.createFrom; } /** @compilation_should_fail -@stderr can not use a generic function `tuplePush` as non-call +@stderr can not use a generic function `Container.createFrom` as non-call +@stderr Container.createFrom; */ diff --git a/tolk-tester/tests/invalid-semantics/err-4115.tolk b/tolk-tester/tests/invalid-semantics/err-4115.tolk new file mode 100644 index 000000000..26bcf0c1f --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4115.tolk @@ -0,0 +1,14 @@ +fun (T|int).equals(self, v: U) { return self == v; } +fun T?.equals(self, v: U) { return self == v; } + +fun main() { + (10 as int?).equals(20); +} + +/** +@compilation_should_fail +@stderr error: call to method `equals` for type `int?` is ambiguous +@stderr candidate function: `(T|int).equals` with T=`null` +@stderr candidate function: `T?.equals` with T=`int` +@stderr (10 as int?).equals(20); + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4127.tolk b/tolk-tester/tests/invalid-semantics/err-4127.tolk index 339addfb5..cae958ab6 100644 --- a/tolk-tester/tests/invalid-semantics/err-4127.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4127.tolk @@ -3,5 +3,5 @@ const asdf = ttt; /** @compilation_should_fail -@stderr generic T not expected here +@stderr type arguments not expected here */ diff --git a/tolk-tester/tests/invalid-semantics/err-4190.tolk b/tolk-tester/tests/invalid-semantics/err-4190.tolk index 70355bf98..97497bd62 100644 --- a/tolk-tester/tests/invalid-semantics/err-4190.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4190.tolk @@ -1,6 +1,6 @@ struct A { ok: coins = ton("0.05"), - err: int = now(), + err: int = blockchain.now(), } /** diff --git a/tolk-tester/tests/invalid-semantics/err-4195.tolk b/tolk-tester/tests/invalid-semantics/err-4195.tolk index bb8cde058..a4170ffdd 100644 --- a/tolk-tester/tests/invalid-semantics/err-4195.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4195.tolk @@ -1,5 +1,9 @@ fun getNullableTuple(): tuple? { return createEmptyTuple(); } +@pure +fun tuplePush(mutate t: tuple, value: T): void + asm "TPUSH"; + fun cantUseLValueUnwrappedNotNull() { tuplePush(mutate getNullableTuple()!, 1); } diff --git a/tolk-tester/tests/invalid-semantics/err-4256.tolk b/tolk-tester/tests/invalid-semantics/err-4256.tolk new file mode 100644 index 000000000..6b29b8aab --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4256.tolk @@ -0,0 +1,18 @@ +type MInt = int; +type IntOrSlice = MInt|slice; + +fun (int | slice).method(self) {} +fun (slice | MInt).method(self) {} +fun IntOrSlice.method(self) {} + +fun main(p: int|slice) { + p.method(); +} + +/** +@compilation_should_fail +@stderr call to method `method` for type `int | slice` is ambiguous +@stderr candidate function: `(int|slice).method` +@stderr candidate function: `(slice|MInt).method` +@stderr candidate function: `IntOrSlice.method` +*/ diff --git a/tolk-tester/tests/invalid-semantics/err-4266.tolk b/tolk-tester/tests/invalid-semantics/err-4266.tolk index c7f72bf4d..6d6ec061b 100644 --- a/tolk-tester/tests/invalid-semantics/err-4266.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4266.tolk @@ -1,9 +1,9 @@ fun invalidReferencingGenericMethodWithoutGeneric() { var t = createEmptyTuple(); - var cb = t.tupleLast; + var cb = t.last; } /** @compilation_should_fail -@stderr can not use a generic function `tupleLast` as non-call +@stderr can not use a generic function `tuple.last` as non-call */ diff --git a/tolk-tester/tests/invalid-semantics/err-4270.tolk b/tolk-tester/tests/invalid-semantics/err-4270.tolk index 14aef95ca..455c16345 100644 --- a/tolk-tester/tests/invalid-semantics/err-4270.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4270.tolk @@ -1,5 +1,5 @@ struct A { - tens: (int, int) = (1, 2 + now()), + tens: (int, int) = (1, 2 + blockchain.now()), } /** diff --git a/tolk-tester/tests/invalid-semantics/err-4291.tolk b/tolk-tester/tests/invalid-semantics/err-4291.tolk index 907cc76c8..a7fadaa2d 100644 --- a/tolk-tester/tests/invalid-semantics/err-4291.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4291.tolk @@ -3,5 +3,6 @@ struct CollisionName { item: T1; } /** @compilation_should_fail -@stderr type `T1` is not generic +@stderr unknown type name `T1` +@stderr item: T1 */ diff --git a/tolk-tester/tests/invalid-semantics/err-4304.tolk b/tolk-tester/tests/invalid-semantics/err-4304.tolk index 963693414..032586aaa 100644 --- a/tolk-tester/tests/invalid-semantics/err-4304.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4304.tolk @@ -6,5 +6,5 @@ fun main() { /** @compilation_should_fail -@stderr type `MInt` can not be used as a value +@stderr `MInt` only refers to a type, but is being used as a value here */ diff --git a/tolk-tester/tests/invalid-semantics/err-4319.tolk b/tolk-tester/tests/invalid-semantics/err-4319.tolk new file mode 100644 index 000000000..f1438ea33 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4319.tolk @@ -0,0 +1,17 @@ +type MInt = int; + +fun (int | slice).method(self) {} +fun (MInt|null).method(self) {} +fun int8?.method(self) {} + +fun main(p: int8) { + p.method(); +} + +/** +@compilation_should_fail +@stderr call to method `method` for type `int8` is ambiguous +@stderr candidate function: `(int|slice).method` +@stderr candidate function: `MInt?.method` +@stderr candidate function: `int8?.method` +*/ diff --git a/tolk-tester/tests/invalid-semantics/err-4385.tolk b/tolk-tester/tests/invalid-semantics/err-4385.tolk index 31d4f0213..767f524f6 100644 --- a/tolk-tester/tests/invalid-semantics/err-4385.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4385.tolk @@ -1,6 +1,6 @@ @pure fun validate_input(input: cell): (int, int) { - var (x, y, z, correct) = calculateCellSize(input, 10); + var (x, y, z, correct) = input.calculateSize(10); assert(correct) throw 102; return (x, y); } @@ -16,9 +16,7 @@ fun main() {} /** @compilation_should_fail -@stderr -""" -an impure operation in a pure function -assert(correct) -""" +@stderr an impure operation in a pure function +@stderr in function `validate_input` +@stderr assert(correct) */ diff --git a/tolk-tester/tests/invalid-semantics/err-4387.tolk b/tolk-tester/tests/invalid-semantics/err-4387.tolk index 50e02ed7b..0de2871ac 100644 --- a/tolk-tester/tests/invalid-semantics/err-4387.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4387.tolk @@ -12,6 +12,6 @@ fun main(d: nested) { /** @compilation_should_fail -@stderr struct `Wrapper` can not be used as a value +@stderr `Wrapper` only refers to a type, but is being used as a value here @stderr c2 = Wrapper */ diff --git a/tolk-tester/tests/invalid-semantics/err-4413.tolk b/tolk-tester/tests/invalid-semantics/err-4413.tolk index 213206834..6ee576dcc 100644 --- a/tolk-tester/tests/invalid-semantics/err-4413.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4413.tolk @@ -15,9 +15,7 @@ fun main(): int { /** @compilation_should_fail -@stderr -""" -an impure operation in a pure function -g = g + 1; -""" -*/ +@stderr an impure operation in a pure function +@stderr in function `f_pure` +@stderr g = g + 1; + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4473.tolk b/tolk-tester/tests/invalid-semantics/err-4473.tolk index dfc69851c..9d4ad1cc0 100644 --- a/tolk-tester/tests/invalid-semantics/err-4473.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4473.tolk @@ -1,4 +1,4 @@ -fun load32(self: slice): int { +fun slice.load32(self): int { return self.loadUint(32); } diff --git a/tolk-tester/tests/invalid-semantics/err-4480.tolk b/tolk-tester/tests/invalid-semantics/err-4480.tolk new file mode 100644 index 000000000..a0f0da32d --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4480.tolk @@ -0,0 +1,10 @@ +fun main() { + Point2.create(); +} + +/** +@compilation_should_fail +@stderr unknown type name `Point2` +@stderr in function `main` +@stderr Point2.create() + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4506.tolk b/tolk-tester/tests/invalid-semantics/err-4506.tolk deleted file mode 100644 index 8cd37c517..000000000 --- a/tolk-tester/tests/invalid-semantics/err-4506.tolk +++ /dev/null @@ -1,16 +0,0 @@ -fun increment(mutate x: int) { - x = x + 1; -} - -fun cantCallMutatingAsAMember() { - var x = 0; - x.increment(); - return x; -} - -/** -@compilation_should_fail -@stderr function `increment` mutates parameter `x` -@stderr consider calling `increment(mutate x)`, not `x.increment`() -@stderr alternatively, rename parameter to `self` to make it a method - */ diff --git a/tolk-tester/tests/invalid-semantics/err-4510.tolk b/tolk-tester/tests/invalid-semantics/err-4510.tolk index a399bc917..82aaaaccf 100644 --- a/tolk-tester/tests/invalid-semantics/err-4510.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4510.tolk @@ -7,5 +7,5 @@ fun main() { /** @compilation_should_fail -@stderr generic T not expected here +@stderr type arguments not expected here */ diff --git a/tolk-tester/tests/invalid-semantics/err-4520.tolk b/tolk-tester/tests/invalid-semantics/err-4520.tolk index 199aa681a..994440163 100644 --- a/tolk-tester/tests/invalid-semantics/err-4520.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4520.tolk @@ -6,5 +6,5 @@ fun main() { /** @compilation_should_fail -@stderr type `int` is not indexable +@stderr method `3` not found for type `int` */ diff --git a/tolk-tester/tests/invalid-semantics/err-4542.tolk b/tolk-tester/tests/invalid-semantics/err-4542.tolk index 9327f07d8..e1ef833e5 100644 --- a/tolk-tester/tests/invalid-semantics/err-4542.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4542.tolk @@ -1,5 +1,5 @@ @pure -fun tupleMut(mutate self: tuple): int +fun tuple.tupleMut(mutate self): int asm "TLEN"; fun main() { @@ -9,5 +9,5 @@ fun main() { /** @compilation_should_fail -@stderr saving `tupleMut` into a variable is impossible, since it has `mutate` parameters +@stderr saving `tuple.tupleMut` into a variable is impossible, since it has `mutate` parameters */ diff --git a/tolk-tester/tests/invalid-semantics/err-4588.tolk b/tolk-tester/tests/invalid-semantics/err-4588.tolk new file mode 100644 index 000000000..9c4ded675 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4588.tolk @@ -0,0 +1,17 @@ +struct Point { + x: int; + y: int; +} + +fun getX(i: int) { return i; } +fun Point.getX(self) { return self.x; } + +fun main() { + var Point = 10; + Point.getX(); +} + +/** +@compilation_should_fail +@stderr method `getX` not found for type `int` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4598.tolk b/tolk-tester/tests/invalid-semantics/err-4598.tolk new file mode 100644 index 000000000..3ef699380 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4598.tolk @@ -0,0 +1,16 @@ +struct Container { + item: T; +} + +fun Container.wrap(item: T) {} + +fun main(c: Container) { + c.wrap(); +} + +/** +@compilation_should_fail +@stderr method `Container.wrap` can not be called via dot +@stderr it's a static method, it does not accept `self` +@stderr c.wrap(); + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4604.tolk b/tolk-tester/tests/invalid-semantics/err-4604.tolk index 73fd6f87c..07637d9da 100644 --- a/tolk-tester/tests/invalid-semantics/err-4604.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4604.tolk @@ -4,5 +4,5 @@ fun invalidProvidingGenericTsToNotGeneric() { /** @compilation_should_fail -@stderr calling a not generic function with generic T +@stderr type arguments not expected here */ diff --git a/tolk-tester/tests/invalid-semantics/err-4610.tolk b/tolk-tester/tests/invalid-semantics/err-4610.tolk new file mode 100644 index 000000000..2c495602d --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4610.tolk @@ -0,0 +1,13 @@ +fun int.create(value: int) { + return value; +} + +fun main() { + 10.create(); +} + +/** +@compilation_should_fail +@stderr method `int.create` can not be called via dot +@stderr it's a static method, it does not accept `self` + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4682.tolk b/tolk-tester/tests/invalid-semantics/err-4682.tolk index 054a6a403..0a042411b 100644 --- a/tolk-tester/tests/invalid-semantics/err-4682.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4682.tolk @@ -8,6 +8,6 @@ fun main() { /** @compilation_should_fail -@stderr struct `User` can not be used as a value +@stderr `User` only refers to a type, but is being used as a value here @stderr absk(User) */ diff --git a/tolk-tester/tests/invalid-semantics/err-4711.tolk b/tolk-tester/tests/invalid-semantics/err-4711.tolk deleted file mode 100644 index 01f488487..000000000 --- a/tolk-tester/tests/invalid-semantics/err-4711.tolk +++ /dev/null @@ -1,22 +0,0 @@ -struct Pair { - first: T1; - second: T2; -} - -fun main() { - var ok1: Pair = { first: null, second: null }; - var ok2: Pair = Pair { first: null, second: null }; - var ok3 = Pair { first: null, second: null }; - var ok4 = Pair { first: null as int?, second: null as (int, int)? }; - var ok5 = Pair { first: null as int?, second: null as slice? }; // ok since doesn't reach type checking - var ok6 = Pair { first: null, second: null }; - - var err = Pair { first: null, second: null }; -} - -/** -@compilation_should_fail -@stderr can not deduce T1 for generic struct `Pair` -@stderr instantiate it manually with `Pair<...>` -@stderr var err = - */ diff --git a/tolk-tester/tests/invalid-semantics/err-4739.tolk b/tolk-tester/tests/invalid-semantics/err-4739.tolk new file mode 100644 index 000000000..58fa5efe5 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4739.tolk @@ -0,0 +1,12 @@ +fun T.pairWith(self, value: U) { + return [self, value]; +} + +fun main() { + 10.pairWith(); +} + +/** +@compilation_should_fail +@stderr too few arguments in call to `T.pairWith`, expected 1, have 0 + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4828.tolk b/tolk-tester/tests/invalid-semantics/err-4828.tolk deleted file mode 100644 index f6874fb8c..000000000 --- a/tolk-tester/tests/invalid-semantics/err-4828.tolk +++ /dev/null @@ -1,12 +0,0 @@ -fun asdf(mutate cs: slice) {} - -fun main(cs: slice) { - cs.asdf(); -} - -/** -@compilation_should_fail -@stderr function `asdf` mutates parameter `cs` -@stderr consider calling `asdf(mutate cs)`, not `cs.asdf`() -@stderr alternatively, rename parameter to `self` to make it a method - */ diff --git a/tolk-tester/tests/invalid-semantics/err-4841.tolk b/tolk-tester/tests/invalid-semantics/err-4841.tolk deleted file mode 100644 index cbf598066..000000000 --- a/tolk-tester/tests/invalid-semantics/err-4841.tolk +++ /dev/null @@ -1,12 +0,0 @@ -fun nothing() { -} - -fun main() { - val x = 0; - return x.nothing(); -} - -/** -@compilation_should_fail -@stderr `nothing` has no parameters and can not be called as method - */ diff --git a/tolk-tester/tests/invalid-semantics/err-4847.tolk b/tolk-tester/tests/invalid-semantics/err-4847.tolk new file mode 100644 index 000000000..215c245c3 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4847.tolk @@ -0,0 +1,16 @@ +fun increment(mutate x: int) { + x += 1; +} + +fun testCantCallWithoutMutate() { + var x = 0; + increment(x); +} + +/** +@compilation_should_fail +@stderr function `increment` mutates parameter `x` +@stderr you need to specify `mutate` when passing an argument, like `mutate x` +@stderr in function `testCantCallWithoutMutate` +@stderr increment(x) + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4851.tolk b/tolk-tester/tests/invalid-semantics/err-4851.tolk index 3489a2889..5195da070 100644 --- a/tolk-tester/tests/invalid-semantics/err-4851.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4851.tolk @@ -1,4 +1,4 @@ -fun increment(self: int) { +fun int.increment(self) { self = self + 1; } diff --git a/tolk-tester/tests/invalid-semantics/err-4884.tolk b/tolk-tester/tests/invalid-semantics/err-4884.tolk new file mode 100644 index 000000000..0515a4b2e --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4884.tolk @@ -0,0 +1,13 @@ +fun T.copy(self) { + return self; +} + +fun main() { + int.copy(); +} + +/** +@compilation_should_fail +@stderr too few arguments in call to `int.copy`, expected 1, have 0 +@stderr int.copy(); + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4899.tolk b/tolk-tester/tests/invalid-semantics/err-4899.tolk deleted file mode 100644 index 2ba645d13..000000000 --- a/tolk-tester/tests/invalid-semantics/err-4899.tolk +++ /dev/null @@ -1,8 +0,0 @@ -fun main(cs: slice) { - return loadInt(cs, 32); -} - -/** -@compilation_should_fail -@stderr `loadInt` is a mutating method; consider calling `cs.loadInt()`, not `loadInt(cs)` - */ diff --git a/tolk-tester/tests/invalid-semantics/err-4941.tolk b/tolk-tester/tests/invalid-semantics/err-4941.tolk deleted file mode 100644 index bb8cde058..000000000 --- a/tolk-tester/tests/invalid-semantics/err-4941.tolk +++ /dev/null @@ -1,10 +0,0 @@ -fun getNullableTuple(): tuple? { return createEmptyTuple(); } - -fun cantUseLValueUnwrappedNotNull() { - tuplePush(mutate getNullableTuple()!, 1); -} - -/** -@compilation_should_fail -@stderr function call can not be used as lvalue - */ diff --git a/tolk-tester/tests/invalid-semantics/err-4951.tolk b/tolk-tester/tests/invalid-semantics/err-4951.tolk index 87eb61e84..6766b2490 100644 --- a/tolk-tester/tests/invalid-semantics/err-4951.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4951.tolk @@ -6,5 +6,5 @@ fun main() { /** @compilation_should_fail -@stderr calling a not generic function with generic T +@stderr type arguments not expected here */ diff --git a/tolk-tester/tests/invalid-semantics/err-4956.tolk b/tolk-tester/tests/invalid-semantics/err-4956.tolk index 5a8c9fa5d..de5309dc9 100644 --- a/tolk-tester/tests/invalid-semantics/err-4956.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4956.tolk @@ -10,5 +10,5 @@ fun main() { /** @compilation_should_fail -@stderr `mutate` used for non-mutate argument +@stderr `mutate` used for non-mutate parameter */ diff --git a/tolk-tester/tests/invalid-symbol/err-2188.tolk b/tolk-tester/tests/invalid-symbol/err-2188.tolk index cf8c788c1..6511fd40d 100644 --- a/tolk-tester/tests/invalid-symbol/err-2188.tolk +++ b/tolk-tester/tests/invalid-symbol/err-2188.tolk @@ -9,6 +9,6 @@ fun main() { /** @compilation_should_fail -@stderr non-existing method `storeUnexisting` of type `builder` +@stderr method `storeUnexisting` not found for type `builder` @stderr .storeUnexisting() */ diff --git a/tolk-tester/tests/invalid-symbol/err-2374.tolk b/tolk-tester/tests/invalid-symbol/err-2374.tolk index 08a86f176..3a3ba567c 100644 --- a/tolk-tester/tests/invalid-symbol/err-2374.tolk +++ b/tolk-tester/tests/invalid-symbol/err-2374.tolk @@ -4,11 +4,11 @@ fun main(x: int): int { } else { var y: slice = "20"; } - debugPrint(y); + debug.print(y); } /** @compilation_should_fail -@stderr debugPrint(y); +@stderr debug.print(y); @stderr undefined symbol `y` */ diff --git a/tolk-tester/tests/invalid-symbol/err-2619.tolk b/tolk-tester/tests/invalid-symbol/err-2619.tolk new file mode 100644 index 000000000..8eef3bc07 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2619.tolk @@ -0,0 +1,8 @@ +fun main() { + return random(); +} + +/** +@compilation_should_fail +@stderr `random` is not a function, you probably want `random.uint256()` + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2788.tolk b/tolk-tester/tests/invalid-symbol/err-2788.tolk new file mode 100644 index 000000000..64135e701 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2788.tolk @@ -0,0 +1,9 @@ +fun main() { + var f = stringHexToSlice; + f("asdf"); +} + +/** +@compilation_should_fail +@stderr can not get reference to this function, it's compile-time only + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2914.tolk b/tolk-tester/tests/invalid-symbol/err-2914.tolk deleted file mode 100644 index 7435bb3cf..000000000 --- a/tolk-tester/tests/invalid-symbol/err-2914.tolk +++ /dev/null @@ -1,10 +0,0 @@ -const asdf = 1; - -fun main(x: int) { - return x.asdf(); -} - -/** -@compilation_should_fail -@stderr non-existing method `asdf` of type `int` - */ diff --git a/tolk-tester/tests/invalid-syntax/err-3407.tolk b/tolk-tester/tests/invalid-syntax/err-3407.tolk index 807e7be88..e37402deb 100644 --- a/tolk-tester/tests/invalid-syntax/err-3407.tolk +++ b/tolk-tester/tests/invalid-syntax/err-3407.tolk @@ -7,5 +7,5 @@ not nested /** @compilation_should_fail -@stderr error: expected fun or get, got `*` +@stderr error: expected top-level declaration, got `*` */ diff --git a/tolk-tester/tests/invalid-syntax/err-3967.tolk b/tolk-tester/tests/invalid-syntax/err-3967.tolk index 3edc09fda..9a158a390 100644 --- a/tolk-tester/tests/invalid-syntax/err-3967.tolk +++ b/tolk-tester/tests/invalid-syntax/err-3967.tolk @@ -4,5 +4,5 @@ int main() { /** @compilation_should_fail -@stderr expected fun or get, got `int` +@stderr expected top-level declaration, got `int` */ diff --git a/tolk-tester/tests/invalid-typing/err-6013.tolk b/tolk-tester/tests/invalid-typing/err-6013.tolk new file mode 100644 index 000000000..c5ec6783d --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6013.tolk @@ -0,0 +1,18 @@ +struct Pair { + first: A; + second: B; +} + +fun Pair.demoInvalid(self) { + return self.first == (null as int?); +} + +fun main(p: Pair) { + p.demoInvalid(); +} + +/** +@compilation_should_fail +@stderr can not apply operator `==` to `int` and `int?` +@stderr in function `Pair.demoInvalid` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6080.tolk b/tolk-tester/tests/invalid-typing/err-6080.tolk index cf985addf..d02675097 100644 --- a/tolk-tester/tests/invalid-typing/err-6080.tolk +++ b/tolk-tester/tests/invalid-typing/err-6080.tolk @@ -1,5 +1,5 @@ fun incrementOrSetNull(mutate x: int?) { - if (random()) { x! += 1; } + if (random.uint256()) { x! += 1; } else { x = null; } } diff --git a/tolk-tester/tests/invalid-typing/err-6085.tolk b/tolk-tester/tests/invalid-typing/err-6085.tolk deleted file mode 100644 index eb3adc921..000000000 --- a/tolk-tester/tests/invalid-typing/err-6085.tolk +++ /dev/null @@ -1,17 +0,0 @@ -fun eq(v: X) {} - -fun cantDeduceWhenNotInferred() { - // at type inferring (before type checking) they are unknown - var (x, y) = 2; - - eq(x as int); // ok (since execution doesn't reach type checking) - eq(x); // ok (since execution doesn't reach type checking) - eq(x); -} - -/** -@compilation_should_fail -@stderr in function `cantDeduceWhenNotInferred` -@stderr can not deduce X for generic function `eq` -@stderr eq(x); - */ diff --git a/tolk-tester/tests/invalid-typing/err-6087.tolk b/tolk-tester/tests/invalid-typing/err-6087.tolk index 463ba4871..aabd956ea 100644 --- a/tolk-tester/tests/invalid-typing/err-6087.tolk +++ b/tolk-tester/tests/invalid-typing/err-6087.tolk @@ -5,5 +5,5 @@ fun cantAutoCastBytesNToSlice() { /** @compilation_should_fail -@stderr can not call method for `slice` with object of type `bits32` +@stderr method `loadInt` not found for type `bits32` */ diff --git a/tolk-tester/tests/invalid-typing/err-6134.tolk b/tolk-tester/tests/invalid-typing/err-6134.tolk index 1ee71290c..b3972a7af 100644 --- a/tolk-tester/tests/invalid-typing/err-6134.tolk +++ b/tolk-tester/tests/invalid-typing/err-6134.tolk @@ -1,4 +1,4 @@ -fun incNotChained(mutate self: int) { +fun int.incNotChained(mutate self) { self = self + 1; } @@ -8,5 +8,5 @@ fun cantCallNotChainedMethodsInAChain(x: int) { /** @compilation_should_fail -@stderr can not call method for `int` with object of type `void` +@stderr method `incNotChained` not found for type `void` */ diff --git a/tolk-tester/tests/invalid-typing/err-6148.tolk b/tolk-tester/tests/invalid-typing/err-6148.tolk index 1621bab19..cbd305f21 100644 --- a/tolk-tester/tests/invalid-typing/err-6148.tolk +++ b/tolk-tester/tests/invalid-typing/err-6148.tolk @@ -9,6 +9,6 @@ fun testSmartCastsDropAfterMutate() { /** @compilation_should_fail -@stderr type `(int, int)?` is not indexable +@stderr field `1` doesn't exist in type `(int, int)?` @stderr return x.1 */ diff --git a/tolk-tester/tests/invalid-typing/err-6149.tolk b/tolk-tester/tests/invalid-typing/err-6149.tolk new file mode 100644 index 000000000..7fd34b646 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6149.tolk @@ -0,0 +1,18 @@ +struct Pair { + first: A; + second: B; +} + +fun Pair.compareWith(self, f: U, s: V) { + return self.first == f && self.second == s; +} + +fun main(p: Pair) { + p.compareWith(null, null); +} + +/** +@compilation_should_fail +@stderr can not apply operator `==` to `int` and `int?` +@stderr in function `Pair.compareWith` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6204.tolk b/tolk-tester/tests/invalid-typing/err-6204.tolk index 8f2b540da..50c191f1f 100644 --- a/tolk-tester/tests/invalid-typing/err-6204.tolk +++ b/tolk-tester/tests/invalid-typing/err-6204.tolk @@ -1,9 +1,9 @@ fun cantUnifyCoinsAndUInt8(n: int8, c: coins) { - __expect_type(random() ? n : c as int8, "int8"); - __expect_type(random() ? n as int16 : c as int16, "int16"); + __expect_type(random.uint256() ? n : c as int8, "int8"); + __expect_type(random.uint256() ? n as int16 : c as int16, "int16"); - var withHint: int = random() ? n : c; // ok - var withoutHint = random() ? n : c; // error + var withHint: int = random.uint256() ? n : c; // ok + var withoutHint = random.uint256() ? n : c; // error } /** diff --git a/tolk-tester/tests/invalid-typing/err-6249.tolk b/tolk-tester/tests/invalid-typing/err-6249.tolk index 274f51b4c..1553bf75e 100644 --- a/tolk-tester/tests/invalid-typing/err-6249.tolk +++ b/tolk-tester/tests/invalid-typing/err-6249.tolk @@ -1,7 +1,7 @@ fun cantUnifyBytesNAndSlice(n: slice, c: bytes16) { - __expect_type(random() ? n : c as slice, "slice"); - __expect_type(random() ? n : c as slice?, "slice?"); - random() ? n : c; + __expect_type(random.uint256() ? n : c as slice, "slice"); + __expect_type(random.uint256() ? n : c as slice?, "slice?"); + random.uint256() ? n : c; } /** diff --git a/tolk-tester/tests/invalid-typing/err-6257.tolk b/tolk-tester/tests/invalid-typing/err-6257.tolk index a007a93c2..5e18b376c 100644 --- a/tolk-tester/tests/invalid-typing/err-6257.tolk +++ b/tolk-tester/tests/invalid-typing/err-6257.tolk @@ -1,9 +1,9 @@ -fun increment(mutate self: int): self { +fun int.increment(mutate self): self { self = self + 1; return self; } -fun cantReturnAnotherSelf(mutate self: int): self { +fun int.cantReturnAnotherSelf(mutate self): self { self = self + 1; var x = 0; return x.increment(); diff --git a/tolk-tester/tests/invalid-typing/err-6281.tolk b/tolk-tester/tests/invalid-typing/err-6281.tolk new file mode 100644 index 000000000..7d77f411f --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6281.tolk @@ -0,0 +1,12 @@ +struct Wrapper { item: T; } + +fun Wrapper.create(initial: T): Wrapper { return { item: initial }; } + +fun main() { + Wrapper>.create(123); +} + +/** +@compilation_should_fail +@stderr can not pass `int` to `Wrapper` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6288.tolk b/tolk-tester/tests/invalid-typing/err-6288.tolk index 40b54f163..3ea5c02a1 100644 --- a/tolk-tester/tests/invalid-typing/err-6288.tolk +++ b/tolk-tester/tests/invalid-typing/err-6288.tolk @@ -1,4 +1,4 @@ -fun cantReturnFromSelf(mutate self: int): self { +fun int.cantReturnFromSelf(mutate self): self { return 2; } diff --git a/tolk-tester/tests/invalid-typing/err-6337.tolk b/tolk-tester/tests/invalid-typing/err-6337.tolk deleted file mode 100644 index 5c9cbdc4c..000000000 --- a/tolk-tester/tests/invalid-typing/err-6337.tolk +++ /dev/null @@ -1,13 +0,0 @@ -@pure -fun myDictDeleteStrict(mutate self: cell, keyLen: int, key: int): bool - asm(key self keyLen) "DICTIDEL"; - - -fun testCantCallDictMethodsOnNullable(c: cell) { - c.beginParse().loadDict().myDictDeleteStrict(16, 1); -} - -/** -@compilation_should_fail -@stderr can not call method for `cell` with object of type `dict` - */ diff --git a/tolk-tester/tests/invalid-typing/err-6386.tolk b/tolk-tester/tests/invalid-typing/err-6386.tolk deleted file mode 100644 index 155944338..000000000 --- a/tolk-tester/tests/invalid-typing/err-6386.tolk +++ /dev/null @@ -1,10 +0,0 @@ -fun f(v: int, x: T) {} - -fun failCantDeduceWithPlainNull() { - return f(0, null); -} - -/** -@compilation_should_fail -@stderr can not deduce T for generic function `f` - */ diff --git a/tolk-tester/tests/invalid-typing/err-6391.tolk b/tolk-tester/tests/invalid-typing/err-6391.tolk index f6eb2f9fd..1da57d883 100644 --- a/tolk-tester/tests/invalid-typing/err-6391.tolk +++ b/tolk-tester/tests/invalid-typing/err-6391.tolk @@ -1,4 +1,4 @@ -fun acceptMutateNullableTensor(mutate self: (int, int)?) { +fun (int, int)?.acceptMutateNullableTensor(mutate self) { } fun cantModifyTupleIndexWithTypeTransition() { diff --git a/tolk-tester/tests/invalid-typing/err-6393.tolk b/tolk-tester/tests/invalid-typing/err-6393.tolk index a0d5ee04e..28a9f76ca 100644 --- a/tolk-tester/tests/invalid-typing/err-6393.tolk +++ b/tolk-tester/tests/invalid-typing/err-6393.tolk @@ -1,3 +1,6 @@ +fun tupleLast(t: tuple): T + asm "LAST"; + fun getTupleLastGetter(): tuple -> X { return tupleLast; } diff --git a/tolk-tester/tests/invalid-typing/err-6407.tolk b/tolk-tester/tests/invalid-typing/err-6407.tolk index 68c6c8043..8f0265de2 100644 --- a/tolk-tester/tests/invalid-typing/err-6407.tolk +++ b/tolk-tester/tests/invalid-typing/err-6407.tolk @@ -1,5 +1,5 @@ fun invalidNever(): never { - if (random()) { throw 123; } + if (random.uint256()) { throw 123; } } /** diff --git a/tolk-tester/tests/invalid-typing/err-6519.tolk b/tolk-tester/tests/invalid-typing/err-6519.tolk deleted file mode 100644 index 9d8cd480d..000000000 --- a/tolk-tester/tests/invalid-typing/err-6519.tolk +++ /dev/null @@ -1,13 +0,0 @@ -fun incNotChained(mutate self: int) { - self = self + 1; -} - -fun failWhenReturnANotChainedValue(x: int): int { - return x.incNotChained(); -} - -/** -@compilation_should_fail -@stderr x.incNotChained() -@stderr can not convert type `void` to return type `int` - */ diff --git a/tolk-tester/tests/invalid-typing/err-6629.tolk b/tolk-tester/tests/invalid-typing/err-6629.tolk index d2c24e532..c68082fa6 100644 --- a/tolk-tester/tests/invalid-typing/err-6629.tolk +++ b/tolk-tester/tests/invalid-typing/err-6629.tolk @@ -6,6 +6,6 @@ fun wrongTCountPassed() { /** @compilation_should_fail -@stderr wrong count of generic T: expected 2, got 1 +@stderr expected 2 type arguments, got 1 @stderr */ diff --git a/tolk-tester/tests/invalid-typing/err-6637.tolk b/tolk-tester/tests/invalid-typing/err-6637.tolk index 657ab5f4a..26c7ce108 100644 --- a/tolk-tester/tests/invalid-typing/err-6637.tolk +++ b/tolk-tester/tests/invalid-typing/err-6637.tolk @@ -1,6 +1,6 @@ fun autoGetIntOrNull() { - if (random()) { return 1; } + if (random.uint256()) { return 1; } return null; } diff --git a/tolk-tester/tests/invalid-typing/err-6659.tolk b/tolk-tester/tests/invalid-typing/err-6659.tolk index 82fa462b6..326e239d8 100644 --- a/tolk-tester/tests/invalid-typing/err-6659.tolk +++ b/tolk-tester/tests/invalid-typing/err-6659.tolk @@ -1,15 +1,17 @@ +fun rand() { return random.uint256() } + fun cantAutoInferUnionInTernary() { - var dd: int | slice = random() ? 1 : ""; // ok - var cc = (random() ? 1 : "") as int|slice; // ok - random() ? 1 as int|slice : "" as int|slice; // ok - random() ? 1 as int|slice : ""; // ok - random() ? 1 : "" as int|slice|builder; // ok - random() ? 1 as int|slice|null : "" as slice?; // ok + var dd: int | slice = rand() ? 1 : ""; // ok + var cc = (rand() ? 1 : "") as int|slice; // ok + rand() ? 1 as int|slice : "" as int|slice; // ok + rand() ? 1 as int|slice : ""; // ok + rand() ? 1 : "" as int|slice|builder; // ok + rand() ? 1 as int|slice|null : "" as slice?; // ok - var sub = random() ? (1, 2 as int|slice) : (1, ""); + var sub = rand() ? (1, 2 as int|slice) : (1, ""); __expect_type(sub, "(int, int | slice)"); - random() ? 1 : ""; + rand() ? 1 : ""; } /** diff --git a/tolk-tester/tests/invalid-typing/err-6731.tolk b/tolk-tester/tests/invalid-typing/err-6731.tolk index 6a33e6968..461e48d55 100644 --- a/tolk-tester/tests/invalid-typing/err-6731.tolk +++ b/tolk-tester/tests/invalid-typing/err-6731.tolk @@ -4,5 +4,5 @@ fun main(cs: slice) { /** @compilation_should_fail -@stderr referencing a method for `tuple` with object of type `slice` +@stderr field `tupleSize` doesn't exist in type `slice` */ diff --git a/tolk-tester/tests/invalid-typing/err-6820.tolk b/tolk-tester/tests/invalid-typing/err-6820.tolk index 47ebb1a2f..092858290 100644 --- a/tolk-tester/tests/invalid-typing/err-6820.tolk +++ b/tolk-tester/tests/invalid-typing/err-6820.tolk @@ -1,6 +1,6 @@ fun cantUnifyBytesNAndBytesM(n: bytes8, c: bytes16) { - __expect_type(random() ? n : c as bytes8, "bytes8"); - random() ? n : c; + __expect_type(random.uint256() ? n : c as bytes8, "bytes8"); + random.uint256() ? n : c; } /** diff --git a/tolk-tester/tests/invalid-typing/err-6839.tolk b/tolk-tester/tests/invalid-typing/err-6839.tolk index ac019a421..91cca9c4f 100644 --- a/tolk-tester/tests/invalid-typing/err-6839.tolk +++ b/tolk-tester/tests/invalid-typing/err-6839.tolk @@ -1,9 +1,9 @@ -fun incInt(mutate self: int): self { +fun int.incInt(mutate self): self { self += 1; return self; } -fun appendBuilder(mutate self: builder): self { +fun builder.appendBuilder(mutate self): self { self.storeUint(1, 32); return self; } @@ -15,5 +15,5 @@ fun cantMixDifferentThis() { /** @compilation_should_fail -@stderr can not call method for `builder` with object of type `int` +@stderr method `appendBuilder` not found for type `int` */ diff --git a/tolk-tester/tests/invalid-typing/err-6844.tolk b/tolk-tester/tests/invalid-typing/err-6844.tolk index 0be6b9e4d..3f719f204 100644 --- a/tolk-tester/tests/invalid-typing/err-6844.tolk +++ b/tolk-tester/tests/invalid-typing/err-6844.tolk @@ -1,4 +1,4 @@ -fun cantReturnNothingFromSelf(mutate self: int): self { +fun int.cantReturnNothingFromSelf(mutate self): self { self = self + 1; } diff --git a/tolk-tester/tests/invalid-typing/err-6942.tolk b/tolk-tester/tests/invalid-typing/err-6942.tolk index 0ffe5c218..95d0b9ad3 100644 --- a/tolk-tester/tests/invalid-typing/err-6942.tolk +++ b/tolk-tester/tests/invalid-typing/err-6942.tolk @@ -1,14 +1,12 @@ struct Point { x: int; y: int; } -fun methodOverNullable(p: Point?) {} -fun method(p: Point) {} +fun Point?.methodOverNullable(self) {} fun cantAssignNullableObjectField(a: Point?) { if (a != null) { a.x = 10; // ok } a.methodOverNullable(); // ok - a.method(); // also ok, would have failed on type checking, but doesn't reach type checking a.y = 20; } diff --git a/tolk-tester/tests/invalid-typing/err-6981.tolk b/tolk-tester/tests/invalid-typing/err-6981.tolk index 62a6f5da4..af3bebb8c 100644 --- a/tolk-tester/tests/invalid-typing/err-6981.tolk +++ b/tolk-tester/tests/invalid-typing/err-6981.tolk @@ -1,15 +1,15 @@ -fun getTwo(): X { return 2; } +fun getTwo(): X { return 2 as X; } fun cantDeduceNonArgumentGeneric() { var t1: [int] = [0]; t1.0 = getTwo(); // ok var t2 = createEmptyTuple(); - t2.tuplePush(0); - t2.0 = getTwo(); // error, can't decude X + t2.push(0); + t2.0 = getTwo(); // deduced X = unknown } /** @compilation_should_fail -@stderr can not deduce X for generic function `getTwo` -@stderr t2.0 = getTwo(); +@stderr in function `getTwo` +@stderr type `int` can not be cast to `unknown` */ diff --git a/tolk-tester/tests/logical-operators.tolk b/tolk-tester/tests/logical-operators.tolk index 2fd62db5b..c925b8c1a 100644 --- a/tolk-tester/tests/logical-operators.tolk +++ b/tolk-tester/tests/logical-operators.tolk @@ -120,7 +120,7 @@ fun testLogicalOps2(first: int) { (10 == s.loadUint(32)) || (20 == s.loadUint(32)) || (3 == s.loadUint(32)) || (4 == s.loadUint(32)); sum += s.loadUint(32); } - return (s.getRemainingBitsCount(), sum); + return (s.remainingBitsCount(), sum); } @method_id(112) diff --git a/tolk-tester/tests/methods-tests.tolk b/tolk-tester/tests/methods-tests.tolk new file mode 100644 index 000000000..a54fe9415 --- /dev/null +++ b/tolk-tester/tests/methods-tests.tolk @@ -0,0 +1,311 @@ +import "@stdlib/common.tolk" +type MInt = int; + +global callOrder: tuple; + +fun int.zero() { return 0; } +fun int.plus1(self) { return self + 1; } +fun MInt.plusN(self, n: int) { return self + n; } +fun int.main(self) { return self; } + +struct Point { + x: int; + y: int; +} + +fun Point.create0() { return Point.create(0, 0); } +fun Point.create(x: int, y: int): Point { return {x,y}; } + +fun Point.getMaxCoord(self) { return self.x > self.y ? self.x : self.y; } +fun Point.incX(mutate self) { self.x += 1; } +fun Point.incXMutateY(mutate self, mutate curY: int) { self.x += 1; curY = self.y; } + +fun Point?.isNull(self) { return self == null; } + +struct Pair { + first: A; + second: B; +} + +fun Pair.create0(): Pair { + return { first: 0, second: 0 }; +} +fun Pair.create0(): Pair { + return { first: 0, second: 0 }; +} +fun Pair.createFrom(first: U, second: V): Pair { + return { first: first as A, second: second as B }; +} +fun Pair.is0(self) { + return self.first == 0 && self.second == 0; +} +fun Pair.compareWith(self, f: U, s: V) { + return self.first == f && self.second == s; +} + +type Tuple2Int = [int, int]; +fun [int, int].getLast(self) { return self.1; } +fun [int, MInt].assignLast(mutate self, v: int) { self.1 = v; } +fun Tuple2Int.getSum(self) { return self.0 + self.1; } + +fun int.calcExactF(self) { return self; } +fun int8.calcExactF(self) { return self * 2; } +fun int8?.calcExactF(self) { return self == null ? 0 : self * 3; } + +fun T.copy(self) { return self; } +fun T.assign(mutate self, value: T2) { self = value; } + +struct Wrapper { item: T; } + +fun Wrapper.create(initial: T): Wrapper { return { item: initial }; } +fun Wrapper.getItem(self) { return self.item; } +fun Wrapper.getItem(self) { return 100500; } +fun createWrapper(item: T) { return Wrapper.create(item); } + +@method_id(101) +fun test1() { + var i = int.zero() + MInt.zero(); + i = i.plus1(); + return (i.plus1(), i, i.plusN(5), i.plusN(i.plus1())); +} + +@method_id(102) +fun test2(a: int8, b: int16) { + var c = b as coins?; + return (a.plus1(), b.plusN(a), c!.plus1()); +} + +@method_id(103) +fun test3() { + var p = Point.create0(); + var yy = 10; + p.incX(); + Point.incX(mutate p); + p.incXMutateY(mutate yy); + p.y = p.x; + Point.incXMutateY(mutate p, mutate yy); + return (p, p.getMaxCoord(), Point.getMaxCoord(p), yy); +} + +@method_id(104) +fun test4() { + var p0 = Point.create0(); + var p5 = Point.create(5,5) as Point?; + var pN = null as Point?; + return (p0.isNull(), p5.isNull(), pN.isNull(), Point.isNull(null), Point.isNull({x:3,y:3})); +} + +@method_id(105) +fun test5(a: MInt, b: int) { + var t = [a,b]; + return (t.getSum(), t.assignLast(a+b), t.getSum(), Tuple2Int.getLast(t)); +} + +@method_id(106) +fun test6() { + var x = 5; + var x8 = x as int8; + return ( + x.calcExactF(), x8.calcExactF(), (8 as int8?).calcExactF(), (null as int8?).calcExactF(), + (5 as int16).calcExactF(), (x8 as coins).calcExactF(), (x8 as MInt).calcExactF(), + int8.calcExactF(x) + ); +} + +@method_id(107) +fun test7() { + __expect_type(5.copy(), "int"); + __expect_type((5 as int99).copy(), "int99"); + __expect_type(beginCell().storeInt(1,32).copy(), "builder"); + __expect_type(((null, null) as (MInt?,bool?)).copy(), "(MInt?, bool?)"); + return ( + beginCell().storeInt(1,32).endCell().beginParse().loadInt(32).copy() + 1, + stringHexToSlice("01").copy().loadInt(8), + stringHexToSlice("FF").loadInt(8), + ); +} + +@method_id(108) +fun test8(x: int32) { + var cp = x.copy(); + x.assign(10); + int32.assign(mutate x, x+cp); + (x as int18).assign(x+2); + __expect_type(x, "int32"); + return (x, cp); +} + +@method_id(109) +fun test9() { + var w1 = Wrapper.create(5); + var w2 = Wrapper.create(10); + var w3 = createWrapper(20 as int16); + var w4 = Wrapper>.create({item:beginCell().storeInt(30,32).storeInt(31,32).endCell().beginParse()}); + __expect_type(w1, "Wrapper"); + __expect_type(w2, "Wrapper"); + __expect_type(w3, "Wrapper"); + __expect_type(w4, "Wrapper>"); + __expect_type(w4.getItem(), "Wrapper"); + __expect_type(w4.getItem().getItem(), "slice"); + return ( + w1.getItem(), w2.getItem(), w3.getItem(), w4.item.item.loadInt(32), + Wrapper.getItem(w1), w1.getItem(), Wrapper>.getItem(w4).getItem().loadInt(32) + ); +} + + +fun T.isAnotherNull(self) { callOrder.push(1); return self == null; } +fun int.isAnotherNull(self) { callOrder.push(2); return false; } +fun int?.isAnotherNull(self) { callOrder.push(3); return false; } + +@method_id(110) +fun test10() { + callOrder = createEmptyTuple(); + ( + 5.isAnotherNull(), (5 as int?).isAnotherNull(), beginCell().isAnotherNull(), (beginCell() as builder?).isAnotherNull(), + null.isAnotherNull(), true.isAnotherNull(), ((1,null) as (int,slice?)).isAnotherNull() + ); + return (callOrder, null.isNull()); +} + +fun int.anyMethod(self) { callOrder.push(1); } +fun slice?.anyMethod(self) { callOrder.push(2); } +fun T.anyMethod(self) { callOrder.push(3); } +fun callAnyMethod(obj: T) { return obj.anyMethod(); } + +@method_id(111) +fun test11() { + callOrder = createEmptyTuple(); + (callAnyMethod(10), callAnyMethod(null as slice?), callAnyMethod(null), + callAnyMethod(null), callAnyMethod(null)); + return callOrder; +} + +fun int?.arbitraryMethod(self) { return self; } +fun Wrapper.arbitraryMethod(self) { return self; } +fun Wrapper.arbitraryMethod(self) { return beginCell(); } +fun Wrapper>.arbitraryMethod(self) { return beginCell(); } +fun T?.arbitraryMethod(self): never { throw 123; } + +fun test12() { + __expect_type(10.arbitraryMethod(), "int?"); + __expect_type((10 as int8?).arbitraryMethod(), "int?"); + __expect_type((10 as coins?)!.arbitraryMethod(), "int?"); + + __expect_type(null.arbitraryMethod(), "int?"); + __expect_type((10 as coins?).arbitraryMethod(), "int?"); + + __expect_type(Wrapper { item: 10 }.arbitraryMethod(), "Wrapper"); + __expect_type(Wrapper { item: 10 as int8 }.arbitraryMethod(), "Wrapper"); + + __expect_type(Wrapper { item: 10 as int8 }.arbitraryMethod(), "builder"); + __expect_type(Wrapper { item: Wrapper { item: 10 } }.arbitraryMethod(), "builder"); + + __expect_type((Wrapper { item: 10 } as Wrapper?).arbitraryMethod(), "never"); + __expect_type((null as slice?).arbitraryMethod(), "never"); +} + +type BuilderOrInt = builder|int; +fun (int|builder).isInt(self) { return self is int; } + +@method_id(113) +fun test13() { + var u1 = beginCell() as builder|int; + var u2 = 10 as int|builder; + var u3 = 20 as BuilderOrInt; + return (u1.isInt(), u2.isInt(), u3.isInt(), 10.isInt(), beginCell().isInt(), (8 as int8).isInt()); +} + +fun (int|builder|T).isBuilder(self) { return self is builder; } + +@method_id(114) +fun test14() { + var b1 = beginCell() as BuilderOrInt | slice; + var b2 = beginCell() as builder|slice|int|Point; + var i1 = 5 as int|builder|continuation|(int, int); + var c = beginCell().endCell() as cell?|builder|int; + var cn = null as cell?|int|int8|builder; + return (b1.isBuilder(), b2.isBuilder(), i1.isBuilder(), c.isBuilder(), cn.isBuilder()); +} + +@method_id(115) +fun test15() { + var p1 = Pair.create0(); + __expect_type(p1, "Pair"); + var p2 = Pair.create0(); + __expect_type(p2, "Pair"); + var p3 = Pair.createFrom(10, (20 as coins?)!); + __expect_type(p3, "Pair"); + return (p2.is0(), p2.compareWith(p3.first, p3.second)); +} + +struct CounterIncrement { inc_by: int; } +struct CounterReset { initial_value: int; } +type CounterMsg = CounterIncrement | CounterReset; + +fun CounterReset.onInternalMessage(self) { + throw 123; +} +fun CounterMsg.onInternalMessage(self) { + return match (self) { + CounterIncrement => self.inc_by, + CounterReset => self.initial_value, + }; +} + +@method_id(116) +fun test16() { + var inc: CounterIncrement = { inc_by: 100 }; + var msg = inc as CounterMsg; + return (inc.onInternalMessage(), msg.onInternalMessage(), CounterMsg.onInternalMessage(CounterReset{initial_value:5})); +} + +fun int.`0`() { return 0; } +fun int.`1`(self) { return self + 1; } + +@method_id(117) +fun test17() { + var zero = int.0(); + return zero.1() + int.0().1() + int.1(int.0()); +} + +fun Wrapper.createNull(): Wrapper { return { item: null }; } +fun Wrapper.createFrom(item: U): Wrapper { return {item}; } + +@method_id(118) +fun test18() { + __expect_type(10.copy, "(int) -> int"); + __expect_type(Wrapper{item:null as int8?}.copy, "(Wrapper) -> Wrapper"); + __expect_type(Wrapper.createNull, "() -> Wrapper"); + __expect_type(Wrapper?>.createNull, "() -> Wrapper?>"); + __expect_type(Wrapper.createFrom, "(int8) -> Wrapper"); + + var cb = Wrapper.createFrom; + return cb(10); +} + +fun main() {} + +/** +@testcase | 101 | | 2 1 6 3 +@testcase | 102 | 5 6 | 6 11 7 +@testcase | 103 | | 4 3 4 4 3 +@testcase | 104 | | 0 0 -1 -1 0 +@testcase | 105 | 5 6 | 11 16 11 +@testcase | 106 | | 5 10 24 0 5 5 5 10 +@testcase | 107 | | 2 1 -1 +@testcase | 108 | 5 | 17 5 +@testcase | 109 | | 5 100500 20 30 5 5 31 +@testcase | 110 | | [ 2 3 1 1 3 1 1 ] -1 +@testcase | 111 | | [ 1 2 3 3 2 ] +@testcase | 113 | | 0 -1 -1 -1 0 -1 +@testcase | 114 | | -1 -1 0 0 0 +@testcase | 115 | | -1 0 +@testcase | 116 | | 100 100 5 +@testcase | 117 | | 3 +@testcase | 118 | | 10 + +@fif_codegen DECLPROC Pair.createFrom +@fif_codegen DECLPROC Pair.compareWith +@fif_codegen DECLPROC CounterMsg.onInternalMessage + */ diff --git a/tolk-tester/tests/mutate-methods.tolk b/tolk-tester/tests/mutate-methods.tolk index 59da68484..18cb64404 100644 --- a/tolk-tester/tests/mutate-methods.tolk +++ b/tolk-tester/tests/mutate-methods.tolk @@ -1,8 +1,18 @@ -fun incrementInPlace(mutate self: int, byValue: int): void { +fun incrementInPlace(mutate x: int, byValue: int): void { + x = x + byValue; +} + +fun int.incrementInPlace(mutate self, byValue: int): void { self = self + byValue; } -fun incrementTwoInPlace(mutate self: int, mutate y: int, byValue: int): int { +fun incrementTwoInPlace(mutate x: int, mutate y: int, byValue: int): int { + x.incrementInPlace(byValue); + y += byValue; + return x + y; +} + +fun int.incrementTwoInPlace(mutate self, mutate y: int, byValue: int): int { self.incrementInPlace(byValue); y += byValue; return self + y; @@ -31,12 +41,12 @@ fun testIncrement2() { fun load_next(mutate cs: slice): int { - return loadInt(mutate cs, 32); + return cs.loadInt(32); } -fun myLoadInt(mutate self: slice, len: int): int +fun slice.myLoadInt(mutate self, len: int): int asm(-> 1 0) "LDIX"; -fun myStoreInt(mutate self: builder, x: int, len: int): self +fun builder.myStoreInt(mutate self, x: int, len: int): self asm(x self len) "STIX"; @inline_ref @@ -58,7 +68,7 @@ fun testSlices1() { return (first, cs.myLoadInt(32), cs.loadInt(32)); } -fun load_decimal_symbol(mutate self: slice): int { +fun slice.load_decimal_symbol(mutate self): int { // load decimal from bits using utf-8 table var n: int = self.loadUint(8); n = n - 48; @@ -113,10 +123,15 @@ fun testNameShadowing() { return (x, sum); } -fun updateTwoItems(mutate self: (int, int), byValue: int) { +type Pair2 = (int, int); +fun Pair2.updateTwoItems(mutate self, byValue: int) { val (first, second) = self; self = (first + byValue, second + byValue); } +fun updateTwoItems(mutate pair: Pair2, byValue: int) { + val (first, second) = pair; + pair = (first + byValue, second + byValue); +} global t107_1: int; global t107_2: int; @@ -134,18 +149,29 @@ fun testMutableTensor() { } @pure -fun myStoreUint(mutate self: builder, x: int, len: int): self +fun builder.myStoreUint(mutate self, x: int, len: int): self asm(x self len) "STIX"; @pure -fun myStoreU32(mutate self: builder, x: int): self { +fun myStoreUint(mutate b: builder, x: int, len: int): void + asm(x b len) "STUX"; +@pure +fun storeUint(mutate b: builder, x: int, len: int): void + asm(x b len) "STUX"; + +@pure +fun builder.myStoreU32(mutate self, x: int): self { return self.storeUint(x, 32); } +@pure +fun myStoreU32(mutate b: builder, x: int) { + b.storeUint(x, 32); +} fun getSumOfNumbersInCell(c: cell): int { var sum = 0; var s = c.beginParse(); - var n_numbers = s.getRemainingBitsCount() / 32; + var n_numbers = s.remainingBitsCount() / 32; repeat (n_numbers) { sum += s.loadUint(32); } @@ -161,7 +187,7 @@ fun testStoreChaining() { b = b.storeUint(8, 32); b = b.storeUint(9, 32).storeUint(10, 32); - return getBuilderBitsCount(b); + return b.bitsCount(); } @method_id(111) @@ -185,7 +211,7 @@ fun testStoreChainingCustom() { return (sum1, sum2); } -fun myStoreU32_and_mutate_x(mutate self: builder, mutate x: int): void { +fun builder.myStoreU32_and_mutate_x(mutate self, mutate x: int): void { return myStoreUint(mutate self, x += 10, 32); } @@ -215,7 +241,7 @@ fun testStoreChainingForGlobal() { ccc = ccc.myStoreU32(8); ccc = ccc.storeUint(9, 32).myStoreUint(10, 32); - return getBuilderBitsCount(ccc); + return ccc.bitsCount(); } fun alwaysThrows(): int { throw 123; return 123; } @@ -233,10 +259,10 @@ fun testLoadIntForTemporaryObject() { } @pure -fun myStoreUint_pure(mutate self: builder): void +fun builder.myStoreUint_pure(mutate self): void asm "STIX"; -fun myStoreUint_impure(mutate self: builder): void +fun builder.myStoreUint_impure(mutate self): void asm "STIX"; fun testStoreUintPureUnusedResult() { @@ -257,11 +283,11 @@ fun testStoreUintImpureUnusedResult() { global counter: int; -fun writeNext2(mutate self: builder): self { +fun builder.writeNext2(mutate self): self { return self.storeUint(counter += 1, 32).storeUint(counter += 1, 32); } -fun resetCounter(mutate self: builder): self { +fun builder.resetCounter(mutate self): self { counter = 0; return self; } @@ -270,7 +296,7 @@ fun resetCounter(mutate self: builder): self { fun testExplicitReturn() { counter = 0; return ( - beginCell().writeNext2().writeNext2().resetCounter().writeNext2().endCell().getSumOfNumbersInCell(), + getSumOfNumbersInCell(beginCell().writeNext2().writeNext2().resetCounter().writeNext2().endCell()), counter ); } @@ -295,7 +321,7 @@ fun main(){} @fif_codegen """ - incrementInPlace PROC:<{ // self byValue + int.incrementInPlace PROC:<{ // self byValue ADD // self }> """ @@ -307,7 +333,7 @@ fun main(){} incrementTwoInPlace CALLDICT // x y sum1 -ROT 10 PUSHINT // sum1 x y '11=10 - incrementTwoInPlace CALLDICT // sum1 x y sum2 + int.incrementTwoInPlace CALLDICT // sum1 x y sum2 s1 s3 s0 XCHG3 // x y sum1 sum2 }> """ diff --git a/tolk-tester/tests/no-spaces.tolk b/tolk-tester/tests/no-spaces.tolk index 33959917e..50b2761d7 100644 --- a/tolk-tester/tests/no-spaces.tolk +++ b/tolk-tester/tests/no-spaces.tolk @@ -47,7 +47,7 @@ fun add3(a: int, b: int, c: int) { return a+b+c; } return [add3(fst2,snd2,trd2),add3(fst1,snd1,trd1)]; } -fun `load:u32`(mutate self: slice): int { +fun slice.`load:u32`(mutate self): int { return self.loadUint(32) } diff --git a/tolk-tester/tests/null-keyword.tolk b/tolk-tester/tests/null-keyword.tolk index db54973e9..98b90811f 100644 --- a/tolk-tester/tests/null-keyword.tolk +++ b/tolk-tester/tests/null-keyword.tolk @@ -14,8 +14,8 @@ fun test1() { (_, _) = (null, null); var t = createEmptyTuple(); do { - var num: int = numbers.listNext(); - t.tuplePush(num); + var (num: int, numbers redef) = listSplit(numbers!); + t.push(num); } while (numbers != null); return (h, numbers == null, t); diff --git a/tolk-tester/tests/nullable-tensors.tolk b/tolk-tester/tests/nullable-tensors.tolk index af051ce93..4811c4144 100644 --- a/tolk-tester/tests/nullable-tensors.tolk +++ b/tolk-tester/tests/nullable-tensors.tolk @@ -13,7 +13,18 @@ fun isTensorNull(t: (int, int)?) { return t == null; } -fun incrementNullableTensorComponents(mutate self: (int, int)?): self { +fun (int, int)?.isTensorNull(self) { + return self == null; +} + +fun incrementNullableTensorComponents(mutate t: (int, int)?) { + if (t != null) { + t!.0 += 1; + t!.1 += 1; + } +} + +fun (int, int)?.incrementNullableTensorComponents(mutate self): self { if (self != null) { self!.0 += 1; self!.1 += 1; @@ -21,7 +32,12 @@ fun incrementNullableTensorComponents(mutate self: (int, int)?): self { return self; } -fun incrementTensorComponents(mutate self: (int, int)): self { +fun incrementTensorComponents(mutate t: (int, int)) { + t.0 += 1; + t.1 += 1; +} + +fun (int, int).incrementTensorComponents(mutate self): self { self.0 += 1; self.1 += 1; return self; @@ -120,6 +136,10 @@ fun isTensorNullGen(t: (T1, T2)?) { return t == null; } +fun (T1, T2)?.isTensorNullGen(self) { + return self == null; +} + @method_id(109) fun test109() { var x1 = (1, 2); @@ -128,7 +148,7 @@ fun test109() { return ( isTensorNullGen(x1), isTensorNullGen(x2), isTensorNullGen(null), isTensorNullGen(x1), isTensorNullGen(x3), - x1.isTensorNullGen(), x2.isTensorNullGen(), x3.isTensorNullGen(), null.isTensorNullGen() + x1.isTensorNullGen(), x2.isTensorNullGen(), x3.isTensorNullGen(), (null as (int,int)?).isTensorNullGen() ); } diff --git a/tolk-tester/tests/nullable-types.tolk b/tolk-tester/tests/nullable-types.tolk index 74e8f1b1b..39906eae4 100644 --- a/tolk-tester/tests/nullable-types.tolk +++ b/tolk-tester/tests/nullable-types.tolk @@ -9,6 +9,13 @@ fun eq(x: T) { return x } fun unwrap(x: T?): T { return x!; } fun intOr0(x: int?): int { return null == x ? 0 : x!; } +@pure +fun storeInt(mutate b: builder, x: int, len: int): void + asm(x b len) "STIX"; +@pure +fun tuplePush(mutate t: tuple, value: T): void + asm "TPUSH"; + @method_id(101) fun test101(x: int) { var re = x == 0 ? null : 100; @@ -61,8 +68,8 @@ fun test106() { gInt106! += 5; var int106: int? = 0; var gTup106 = createEmptyTuple(); - gTup106!.tuplePush(createEmptyTuple()); - (gTup106!.0 as tuple?)!.tuplePush(0 as int?); + gTup106!.push(createEmptyTuple()); + (gTup106!.0 as tuple?)!.push(0 as int?); tuplePush(mutate gTup106!, gInt106); tuplePush(mutate gTup106!.0, int106! += 1); return (gTup106 == null, null != gTup106, gTup106, gTup106!.0 as tuple?); @@ -75,7 +82,7 @@ fun test107() { b = b!.storeInt(3, 32); storeInt(mutate b!, 4, 32); (b! as builder).storeInt(5, 32); - return b!.getBuilderBitsCount(); + return b!.bitsCount(); } @method_id(108) diff --git a/tolk-tester/tests/parse-address.tolk b/tolk-tester/tests/parse-address.tolk index 64ca5d67e..912fb84c2 100644 --- a/tolk-tester/tests/parse-address.tolk +++ b/tolk-tester/tests/parse-address.tolk @@ -2,7 +2,7 @@ const cc1 = stringAddressToSlice("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce const cc2 = stringAddressToSlice("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"); fun verifyAddr(addr: slice, workchain: int, number: int) { - assert (addr.getRemainingBitsCount() == 3 + 8 + 256) throw 112; + assert (addr.remainingBitsCount() == 3 + 8 + 256) throw 112; addr.skipBits(3); assert (addr.loadUint(8) == workchain) throw 111; assert (addr.loadUint(256) == number) throw 111; @@ -105,7 +105,7 @@ fun main() { verifyAddr(stringAddressToSlice("99:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 99, 37304138005561100291416421295333982606153966175434134130332440738068913455320); verifyAddr(stringAddressToSlice("-1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 255, 37304138005561100291416421295333982606153966175434134130332440738068913455320); - return cc1.isSliceBitsEqual(cc2); + return cc1.bitsEqual(cc2); } /** diff --git a/tolk-tester/tests/pure-functions.tolk b/tolk-tester/tests/pure-functions.tolk index bb1a7d593..285320dec 100644 --- a/tolk-tester/tests/pure-functions.tolk +++ b/tolk-tester/tests/pure-functions.tolk @@ -11,7 +11,7 @@ fun f_pure2(): int { @pure fun get_contract_data(): (int, int) { - var c: cell = getContractData(); + var c: cell = contract.getData(); var cs: slice = c.beginParse(); cs.loadBits(32); var value: int = cs.loadUint(16); @@ -20,7 +20,7 @@ fun get_contract_data(): (int, int) { fun save_contract_data(value: int) { var b: builder = beginCell().storeInt(1, 32).storeUint(value, 16); - setContractData(b.endCell()); + contract.setData(b.endCell()); } @pure diff --git a/tolk-tester/tests/self-keyword.tolk b/tolk-tester/tests/self-keyword.tolk index 37f3cb178..e504b5c48 100644 --- a/tolk-tester/tests/self-keyword.tolk +++ b/tolk-tester/tests/self-keyword.tolk @@ -1,18 +1,18 @@ -fun incChained(mutate self: int): self { +fun int.incChained(mutate self): self { self = self + 1; return self; } -fun incChained2(mutate self: int): self { +fun int.incChained2(mutate self): self { return self.incChained(); } -fun incChained3(mutate self: int): self { - incChained(mutate self); +fun int.incChained3(mutate self): self { + self.incChained(); return self; } -fun incChained4(mutate self: int): self { +fun int.incChained4(mutate self): self { self.incChained(); return self; } @@ -25,7 +25,7 @@ fun testIncChainedCodegen(x: int) { @method_id(102) fun testIncChained() { var x: int = 10; - incChained(mutate x); + x.incChained(); x.incChained(); x.incChained2(); x.incChained2().incChained(); @@ -34,7 +34,7 @@ fun testIncChained() { return x.incChained(); } -fun incChainedWithMiddleReturn(mutate self: int, maxValue: int): self { +fun int.incChainedWithMiddleReturn(mutate self, maxValue: int): self { if (self >= maxValue) { return self; } @@ -49,7 +49,7 @@ fun testIncChainedWithMiddleReturn(x: int) { return x.incChainedWithMiddleReturn(10).incChainedWithMiddleReturn(999); } -fun incChainedMutatingBoth(mutate self: int, mutate y: int): self { +fun int.incChainedMutatingBoth(mutate self, mutate y: int): self { self += 1; y += 1; return self; @@ -62,12 +62,12 @@ fun testIncChainedMutatingBoth() { var (x, y) = (0, 0); c104 = 0; x.incChainedMutatingBoth(mutate y).incChainedMutatingBoth(mutate y); - incChainedMutatingBoth(mutate x, mutate y); + x.incChainedMutatingBoth(mutate y); x = x.incChainedMutatingBoth(mutate c104).incChainedMutatingBoth(mutate c104).incChainedMutatingBoth(mutate y); return (x, y, c104); } -fun incTensorChained(mutate self: (int, int)): self { +fun (int, int).incTensorChained(mutate self): self { val (f, s) = self; self = (f + 1, s + 1); return self; @@ -80,7 +80,7 @@ fun testIncTensorChained(f: int, s: int) { return tens.incTensorChained().incTensorChained(); } -fun incConditionalChainable(mutate self: int, mutate another: int, ifLessThan: int): self { +fun int.incConditionalChainable(mutate self, mutate another: int, ifLessThan: int): self { another += 1; return self.incChained() < ifLessThan ? self.incChained().incChained() : self; } @@ -93,7 +93,7 @@ fun testIncConditionalChainable(x: int) { return (x.incConditionalChainable(mutate y, 5), y); } -fun checkNotEq(self: int, throwIfEq: int): void { +fun int.checkNotEq(self, throwIfEq: int): void { if (self == throwIfEq) { throw 100 + throwIfEq; } @@ -113,7 +113,7 @@ fun testNotMutatingSelf(arg: int) { global c108: int; -fun checkNotEqChainable(self: int, throwIfEq: int): self { +fun int.checkNotEqChainable(self, throwIfEq: int): self { c108 += 1; if (self != throwIfEq) { return self; @@ -136,7 +136,7 @@ fun testNotMutatingChainableSelf(arg: int) { global onceFailed109: int; -fun checkNotEqChainableMutateAnother(self: int, throwIfEq: int, mutate toInc: int): self { +fun int.checkNotEqChainableMutateAnother(self, throwIfEq: int, mutate toInc: int): self { if (onceFailed109) { return self; } toInc += 1; try { return self.checkNotEqChainable(throwIfEq); } @@ -158,9 +158,9 @@ fun testNotMutatingChainableSelfMutateAnother(initial: int) { return (arg, c108, c109, x); } -fun pickG110(mutate self: int, mutate pushTo: tuple): self { +fun int.pickG110(mutate self, mutate pushTo: tuple): self { self += 10; - pushTo.tuplePush(c110); + pushTo.push(c110); return self; } @@ -175,13 +175,13 @@ fun testMutateGlobalsLValue(init: int) { return (c110, tup110); } -fun myTuplePush(mutate self: tuple, value: T): self { - self.tuplePush(value); +fun tuple.myTuplePush(mutate self, value: T): self { + self.push(value); return self; } -fun myTupleAt(self: tuple, idx: int): T { - return self.tupleAt(idx); +fun tuple.myTupleAt(self, idx: int): T { + return self.get(idx); } global tup111: tuple; @@ -192,7 +192,7 @@ fun testForallFunctionsWithSelf(): (int, int, tuple) { tup111 = createEmptyTuple(); t.myTuplePush(10); tup111.myTuplePush(1).myTuplePush(2).myTuplePush(3); - return (t.myTupleAt(0), tup111.myTupleAt(tup111.tupleSize() - 1), tup111); + return (t.myTupleAt(0), tup111.myTupleAt(tup111.size() - 1), tup111); } @@ -222,27 +222,27 @@ fun main() { } @fif_codegen """ - incChained PROC:<{ // self - INC // self + int.incChained PROC:<{ // self + INC // self }> - incChained2 PROC:<{ // self - incChained CALLDICT // self + int.incChained2 PROC:<{ // self + int.incChained CALLDICT // self }> - incChained3 PROC:<{ // self - incChained CALLDICT // self + int.incChained3 PROC:<{ // self + int.incChained CALLDICT // self }> - incChained4 PROC:<{ // self - incChained CALLDICT // self + int.incChained4 PROC:<{ // self + int.incChained CALLDICT // self }> """ @fif_codegen """ testIncChainedCodegen PROC:<{ // x - incChained CALLDICT // x - incChained2 CALLDICT // x - incChained3 CALLDICT // x - incChained4 CALLDICT // x + int.incChained CALLDICT // x + int.incChained2 CALLDICT // x + int.incChained3 CALLDICT // x + int.incChained4 CALLDICT // x }> """ */ diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index 24baa34f1..106d93840 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -12,13 +12,19 @@ struct PointOfNulls { sub: Point?; } +fun rand(): int + asm "RANDU256"; + fun getNullableInt(): int? { return 5; } fun getNullableSlice(): slice? { return null; } fun takeNullableInt(a: int?) {} fun takeNullableSlice(a: slice?) {} -fun increment(mutate self: int) { self += 1; } -fun assignToInt(mutate self: int, value: int) { self = value; } -fun assignToNullableInt(mutate self: int?, value: int) { self = value; } +fun increment(mutate x: int) { x += 1; } +fun int.increment(mutate self) { self += 1; } +fun assignToInt(mutate x: int, value: int) { x = value; } +fun int.assignToInt(mutate self, value: int) { self = value; } +fun assignToNullableInt(mutate x: int?, value: int) { x = value; } +fun int?.assignToNullableInt(mutate self, value: int) { self = value; } fun sameTensor(t: (int, int)) { return t; } fun sameTensor2(t: (int?, Pair4N)) { return t; } fun eq(v: T) { return v; } @@ -58,7 +64,7 @@ fun test3(): int { } return x; } - if (random() > -1) { + if (rand() > -1) { if (y == null) { return -1 } else { return y } } @@ -88,7 +94,7 @@ fun test5() { var (a, (b, c)) = (getNullableInt(), (getNullableInt(), getNullableInt())); if (a == null) { return -1; } if (!(b != null)) { return -2; } - if (random() ? c == null && c == null : c == null) { return -3; } + if (rand() ? c == null && c == null : c == null) { return -3; } return a + b + c; } @@ -99,10 +105,10 @@ fun test6() { __expect_type(a == null ? "" : a, "int"); takeNullableInt(a); __expect_type(a, "int"); - if (random()) { + if (rand()) { a = null; } else { - if (random()) { a = null; } + if (rand()) { a = null; } else { a = null; } } __expect_type(a, "null"); @@ -121,7 +127,7 @@ fun test7() { if (a == null && true) { return -1; } if (true && true && 1 && !0 && b == null) { return -2; } if (true ? c == null && (((c))) == null && true : false) { return -3; } - if (!true ? random() > 0 : a != null && (d == null && b != null)) { return -4; } + if (!true ? rand() > 0 : a != null && (d == null && b != null)) { return -4; } return a + b + c + d; } @@ -129,7 +135,7 @@ fun test8(x: int?, y: int?) { var allGt1 = x != null && x > 1 && y != null && y > 1; var xGtY = x != null && y != null && x > y; var xLtEq0 = x == null || x < 0; - (x = 0) < random() || x > 10; + (x = 0) < rand() || x > 10; return x + 0; } @@ -170,7 +176,7 @@ fun test11() { fun test12() { var (x, y) = (getNullableInt(), getNullableInt()); - if (random() ? x == null || y == null : x == null || y == null) { return -1; } + if (rand() ? x == null || y == null : x == null || y == null) { return -1; } __expect_type(x, "int"); __expect_type(y, "int"); return x + y; @@ -192,7 +198,7 @@ fun test14() { x = 0; } if (y == null) { - if (random()) { return 0; } + if (rand()) { return 0; } else { y = 0; } } return x + y; @@ -262,45 +268,45 @@ fun test24(x: int?) { fun test25() { var x = (getNullableInt(), getNullableInt(), getNullableInt()); - x.0 = x.2 = random(); + x.0 = x.2 = rand(); return (x.0) + ((x.2)); } fun test26() { var x = [getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt()]; - if (~(x.0 = random())) { return; } - if ((x.1 = random()) < (x.2 = random())) { return; } - else if (!(x.2 <=> (x.3 = random()))) { return; } - x.5 = (x.4 = random()) ? (x.6 = random()) : (x.6 = random()); - if ((x.7 = random()) as int) { return; } - if (((((x.8 = random()) != null)))) { return; } - if ([x.1, (x.9 = random())!].1) { return; } + if (~(x.0 = rand())) { return; } + if ((x.1 = rand()) < (x.2 = rand())) { return; } + else if (!(x.2 <=> (x.3 = rand()))) { return; } + x.5 = (x.4 = rand()) ? (x.6 = rand()) : (x.6 = rand()); + if ((x.7 = rand()) as int) { return; } + if (((((x.8 = rand()) != null)))) { return; } + if ([x.1, (x.9 = rand())!].1) { return; } val result = x.0+x.1+x.2+x.3+x.4+x.5+x.6+x.7+x.8+x.9; } fun test27() { var (x, _) = ([getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt()], []); - +(x.0 = random()); - x.0 += [((x.1 = random()) < (x.2 = random() + x.1)) as int].0; - !(x.2 <=> (x.3 = random() + x.2)); - x.5 = (x.4 = random()) ? (x.6 = random()) : (x.6 = random()); - (x.7 = random()) as int; - (((((x.8 = random()) != null)))); - [x.1, (x.9 = random())!].1; + +(x.0 = rand()); + x.0 += [((x.1 = rand()) < (x.2 = rand() + x.1)) as int].0; + !(x.2 <=> (x.3 = rand() + x.2)); + x.5 = (x.4 = rand()) ? (x.6 = rand()) : (x.6 = rand()); + (x.7 = rand()) as int; + (((((x.8 = rand()) != null)))); + [x.1, (x.9 = rand())!].1; return x.0+x.1+x.2+x.3+x.4+x.5+x.6+x.7+x.8+x.9; } fun test28() { var x = (getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt()); - __expect_type((x.0 = random(), x.0 += (x.1 = random()) as int, !(x.1 <=> (x.2 = random() + x.0)) == null, (x.3 = random()) ? x.3 : (!x.3) as int), + __expect_type((x.0 = rand(), x.0 += (x.1 = rand()) as int, !(x.1 <=> (x.2 = rand() + x.0)) == null, (x.3 = rand()) ? x.3 : (!x.3) as int), "(int, int, bool, int)"); } fun test29() { var x = (getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt()); - __expect_type([x.0 = random(), ((x.0 += (x.1 = random()) as int)), !(x.1 <=> (x.2 = random() + x.0)) == null, (x.3 = random()) ? x.3 : (!x.3) as int], + __expect_type([x.0 = rand(), ((x.0 += (x.1 = rand()) as int)), !(x.1 <=> (x.2 = rand() + x.0)) == null, (x.3 = rand()) ? x.3 : (!x.3) as int], "[int, int, bool, int]"); } @@ -357,7 +363,7 @@ fun test33(): int { fun test34() { var (x, y) = (getNullableInt(), getNullableInt()); - if (random()) { throw (x = 1, y = 2) } + if (rand()) { throw (x = 1, y = 2) } else { throw (x = 3, y = (1, getNullableInt()!).1) } return x + y; } @@ -411,8 +417,8 @@ fun test38() { assignNull2(mutate t.0, mutate t.1); __expect_type(t.0, "int?"); __expect_type(t.1, "slice?"); - t.0 != null && t.1 != null ? t.0 + loadInt(mutate t.1, 32) : -1; - return t.0 != null && t.1 != null ? t.0 + loadInt(mutate t.1, 32) : -1; + t.0 != null && t.1 != null ? t.0 + t.1.loadInt(32) : -1; + return t.0 != null && t.1 != null ? t.0 + t.1.loadInt(32) : -1; } @method_id(139) @@ -585,7 +591,7 @@ fun test54() { x1 = getNullableInt(); __expect_type(x1, "int?"); assignToNullableInt(mutate x2, 5); - if (random()) { x3.assignToNullableInt(5); } + if (rand()) { x3.assignToNullableInt(5); } x11 = 10; assignToInt(mutate x12, 5); } while (x1 != null); @@ -606,7 +612,7 @@ fun test55() { // (checked via codegen) eq55(x); __expect_type(x, "int?"); // types are checked (unlike generics instantiated) after inferring - x = random() ? 1 : null; + x = rand() ? 1 : null; } __expect_type(x, "int?"); } @@ -694,7 +700,7 @@ fun test62() { __expect_type(p.sub, "Point"); __expect_type(p!!.sub!!, "Point"); p.sub.x + p.x! + p.y!; - return random() ? null : p; + return rand() ? null : p; } fun test63() { diff --git a/tolk-tester/tests/some-tests-2.tolk b/tolk-tester/tests/some-tests-2.tolk index 4e46f2633..b3b756b19 100644 --- a/tolk-tester/tests/some-tests-2.tolk +++ b/tolk-tester/tests/some-tests-2.tolk @@ -1,6 +1,6 @@ import "@stdlib/tvm-lowlevel" -fun pair_first(p: [X, Y]): X asm "FIRST"; +fun [X, Y].pair_first(self): X asm "FIRST"; fun one(dummy: tuple?) { return 1; @@ -37,11 +37,11 @@ fun test88(x: int) { @method_id(89) fun test89(last: int): (int, int, int, int) { var t: tuple = createEmptyTuple(); - t.tuplePush(1); - t.tuplePush(2); - t.tuplePush(3); - t.tuplePush(last); - return (t.tupleAt(0), t.tupleAt(t.tupleSize() - 1), t.tupleFirst(), t.tupleLast()); + t.push(1); + t.push(2); + t.push(3); + t.push(last); + return (t.get(0), t.get(t.size() - 1), t.first(), t.last()); } @pure fun get10() { return 10; } @@ -56,25 +56,25 @@ fun touchCodegen2() { @method_id(92) fun testDumpDontPolluteStack() { var f = get10(); - f.debugPrint(); - debugPrint(10); + debug.print(f); + debug.print(10); var s = "asdf"; - s.debugPrintString(); - debugDumpStack(); - debugPrintString("my"); - return (f, getRemainingBitsCount(s)); + debug.printString(s); + debug.dumpStack(); + debug.printString("my"); + return (f, s.remainingBitsCount()); } @method_id(93) fun testStartBalanceCodegen1() { - var t = getMyOriginalBalanceWithExtraCurrencies(); + var t = contract.getOriginalBalanceWithExtraCurrencies(); var first = t.pair_first(); return first; } @method_id(94) fun testStartBalanceCodegen2() { - var first = getMyOriginalBalance(); + var first = contract.getOriginalBalance(); return first; } diff --git a/tolk-tester/tests/strings-tests.tolk b/tolk-tester/tests/strings-tests.tolk index af8c6edeb..131659571 100644 --- a/tolk-tester/tests/strings-tests.tolk +++ b/tolk-tester/tests/strings-tests.tolk @@ -33,7 +33,7 @@ get fun string_crc16(): int { @pure fun newc(): builder asm "NEWC"; -fun endcs(b: builder): slice +fun builder.endcs(self): slice asm "ENDC" "CTOS"; @pure fun sdeq(s1: slice, s2: slice): int @@ -42,8 +42,8 @@ asm "SDEQ"; fun calcSliceBytes(slice: slice): tuple { var t = createEmptyTuple(); var n = 0; - while (n = slice.getRemainingBitsCount()) { - t.tuplePush(slice.loadUint(min(8, n))); + while (n = slice.remainingBitsCount()) { + t.push(slice.loadUint(min(8, n))); } return t; } diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index 81f3b9f29..c7c8205e4 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -41,6 +41,10 @@ fun sumCoords(p: Point) { return p.x + p.y; } +fun Point.sumCoords(self) { + return self.x + self.y; +} + fun getStorage1(x: int): Storage { val owner = User { id: 1, name: "u" }; return { @@ -53,7 +57,12 @@ fun generatePoint(x: int, y: int): Point { return { x, y }; } -fun assignCoords(mutate self: Point) { +fun assignCoords(mutate p: Point) { + p.x = 10; + p.y = 20; +} + +fun Point.assignCoords(mutate self) { self.x = 10; self.y = 20; } @@ -149,6 +158,10 @@ fun maxCoord(p: Point): int8 { return p.x > p.y ? p.x : p.y; } +fun Point.maxCoord(self): int8 { + return self.x > self.y ? self.x : self.y; +} + @method_id(109) fun test9() { var p = Point { x: 80, y: 100 }; @@ -222,7 +235,7 @@ fun test15() { } fun pushZero(mutate t: tuple) { - t.tuplePush(0); + t.push(0); } @method_id(116) @@ -237,7 +250,7 @@ fun test16(): WithTensorInside? { wt.coords.1 -= 2; wt.tup.0 += 5; wt.otherCoords!.0 = 7; - wt.otherTup.tuplePush(0); + wt.otherTup.push(0); pushZero(mutate wt.otherTup); (wt.otherTup.1, wt.otherTup.0) = (11, 10); return wt; @@ -300,7 +313,7 @@ fun asm_func(x1: JustInt, x2: int, x3: JustIntAlias, x4: JustIntAlias, x5: int, (int, JustInt, int, JustInt, int, int, JustIntAlias2) asm (x4 x5 x6 x7 x1 x2 x3->0 1 2 3 4 5 6) "NOP"; -fun someOp(mutate self: JustInt, y: int): int { +fun JustInt.someOp(mutate self, y: int): int { val (newX, newY) = (self.value + y, y * 10); self = { value: newX }; return newY; @@ -453,9 +466,9 @@ struct WithDefaults { @method_id(133) fun test33(): WithDefaults { var w1: WithDefaults = { f2: 0 }; - assert(w1.f1.0 && w1.f1.1 == 0 && w1.f3!.getRemainingBitsCount() == 24 && w1.f4 == null && w1.f5 is int32) throw 100; + assert(w1.f1.0 && w1.f1.1 == 0 && w1.f3!.remainingBitsCount() == 24 && w1.f4 == null && w1.f5 is int32) throw 100; var w2: WithDefaults? = { f1: (false, 55), f2: 10, f5: 8 as int64 }; - assert(w2.f1.0 != true && w2.f3!.getRemainingBitsCount() == 24 && w2.f4 == null && w2.f5 is int64 && w2.f5 == 8) throw 100; + assert(w2.f1.0 != true && w2.f3!.remainingBitsCount() == 24 && w2.f4 == null && w2.f5 is int64 && w2.f5 == 8) throw 100; var w3: (int, WithDefaults) = (0, { f2: 7, f4: {y: 20} }); assert(w3.1.f4 != null && w3.1.f4.x == 0 && w3.1.f4.y == 20) throw 100; return { f2: 5, f3: null }; diff --git a/tolk-tester/tests/test-math.tolk b/tolk-tester/tests/test-math.tolk index 6b147c776..55c518800 100644 --- a/tolk-tester/tests/test-math.tolk +++ b/tolk-tester/tests/test-math.tolk @@ -921,7 +921,7 @@ fun nrand_f252(): int { var (x, s, t, A, B, r0) = (nan(), 29483 << 236, -3167 << 239, 12845, 16693, 9043); // 4/sqrt(e*Pi) = 1.369 loop iterations on average do { - var (u, v) = (random() / 16 + 1, mulDivRound(random() - (1 << 255), 7027, 1 << 16)); // fixed252; 7027=ceil(sqrt(8/e)*2^12) + var (u, v) = (random.uint256() / 16 + 1, mulDivRound(random.uint256() - (1 << 255), 7027, 1 << 16)); // fixed252; 7027=ceil(sqrt(8/e)*2^12) var va: int = abs(v); var (u1, v1) = (u - s, va - t); // (u - 29483/2^16, abs(v) + 3167/2^13) as fixed252 // Q := u1^2 + v1 * (A*v1 - B*u1) as fixed252 where A=12845/2^16, B=16693/2^16 @@ -952,7 +952,7 @@ fun nrand_f252(): int { fun nrand_fast_f252(): int { var t: int = -3 << 253; // -6. as fixed252 repeat (12) { - t += random() / 16; // add together 12 uniformly random numbers + t += random.uint256() / 16; // add together 12 uniformly random numbers } return t; } @@ -961,7 +961,7 @@ fun nrand_fast_f252(): int { /// fixed248 random(); @inline fun fixed248_random(): int { - return random() >> 8; + return random.uint256() >> 8; } /// random number with standard normal distribution @@ -979,7 +979,7 @@ fun fixed248_nrand_fast(): int { } @pure -fun tset(mutate self: tuple, idx: int, value: X): void +fun tuple.tset(mutate self, idx: int, value: X): void asm(self value idx) "SETINDEXVAR"; // computes 1-acos(x)/Pi by a very simple, extremely slow (~70k gas) and imprecise method @@ -1014,12 +1014,12 @@ fun asin_slow_f255(x: int): int { fun test_nrand(n: int): tuple { var t: tuple = createEmptyTuple(); repeat (255) { - t.tuplePush(0); + t.push(0); } repeat (n) { var x: int = fixed248_nrand(); var bucket: int = (abs(x) >> 243); // 255 buckets starting from x=0, each 1/32 wide - var at_bucket: int = t.tupleAt(bucket); + var at_bucket: int = t.get(bucket); t.tset(bucket, at_bucket + 1); } return t; diff --git a/tolk-tester/tests/type-aliases-tests.tolk b/tolk-tester/tests/type-aliases-tests.tolk index d8ab0782c..ed0cca11d 100644 --- a/tolk-tester/tests/type-aliases-tests.tolk +++ b/tolk-tester/tests/type-aliases-tests.tolk @@ -11,6 +11,9 @@ struct Point { x: int; y: int } type PointAlias = Point; type PointAlias2 = PointAlias; +fun rand(): uint256 + asm "RANDU256"; + fun test1(x: MInt): MVoid { var y = x; var z: MInt = 2; @@ -24,14 +27,14 @@ fun test1(x: MInt): MVoid { __expect_type(~x, "int"); __expect_type(x as int, "int"); __expect_type(x as int8, "int8"); - __expect_type(random() ? x : y, "MInt"); + __expect_type(rand() ? x : y, "MInt"); __expect_type((x, 1, y), "(MInt, int, MInt)"); - __expect_type(random() ? (1, 2) : (1, 2) as Pair2_v1, "(int, int)"); - __expect_type(random() ? (1, 2) : (1, 2) as Pair2_v2, "(int, int)"); - __expect_type(random() ? (1, 2) as Pair2_v1 : (1, 2), "Pair2_v1"); - __expect_type(random() ? (1, 2) as Pair2_v1 : (1, 2) as Pair2_v2, "(int, int)"); - __expect_type(random() ? (1, 2) as Pair2_v2 : (1, 2) as Pair2_v2, "Pair2_v2"); + __expect_type(rand() ? (1, 2) : (1, 2) as Pair2_v1, "(int, int)"); + __expect_type(rand() ? (1, 2) : (1, 2) as Pair2_v2, "(int, int)"); + __expect_type(rand() ? (1, 2) as Pair2_v1 : (1, 2), "Pair2_v1"); + __expect_type(rand() ? (1, 2) as Pair2_v1 : (1, 2) as Pair2_v2, "(int, int)"); + __expect_type(rand() ? (1, 2) as Pair2_v2 : (1, 2) as Pair2_v2, "Pair2_v2"); __expect_type(!x, "bool"); @@ -73,10 +76,10 @@ fun test4(x: MIntN, y: MIntN) { } __expect_type(x, "MInt?"); __expect_type(x!, "MInt"); - __expect_type(random() ? x : y, "MInt?"); - __expect_type(random() ? x : 0, "MInt?"); - __expect_type(random() ? 0 : x, "int?"); - __expect_type(random() ? 0 : x!, "int"); + __expect_type(rand() ? x : y, "MInt?"); + __expect_type(rand() ? x : 0, "MInt?"); + __expect_type(rand() ? 0 : x, "int?"); + __expect_type(rand() ? 0 : x!, "int"); return y!; } diff --git a/tolk-tester/tests/union-types-tests.tolk b/tolk-tester/tests/union-types-tests.tolk index 11dae3f8c..692c06e1c 100644 --- a/tolk-tester/tests/union-types-tests.tolk +++ b/tolk-tester/tests/union-types-tests.tolk @@ -596,7 +596,7 @@ fun test50() { if (match(b) { int => 1, null => null } !is int) { return 100 } - return (b! + (match (a) { int => a, slice => a, null => a } as int|slice?).test49()); + return (b! + test49(match (a) { int => a, slice => a, null => a } as int|slice?)); } @method_id(151) @@ -671,7 +671,7 @@ fun test54(): int|slice { __expect_type(a, "int"); a = beginCell().storeInt(100, 32).endCell().beginParse(); __expect_type(a, "slice"); - if (a.getRemainingBitsCount() < 100) { a = a.loadInt(32); } + if (a.remainingBitsCount() < 100) { a = a.loadInt(32); } else { a = 0; } __expect_type(a, "int"); return a; diff --git a/tolk-tester/tests/use-before-declare.tolk b/tolk-tester/tests/use-before-declare.tolk index baae0b400..42afe8b00 100644 --- a/tolk-tester/tests/use-before-declare.tolk +++ b/tolk-tester/tests/use-before-declare.tolk @@ -9,7 +9,7 @@ fun main(): int { fun my_begin_cell(): builder asm "NEWC"; @pure -fun my_end_cell(b: builder): cell +fun builder.my_end_cell(self): cell asm "ENDC"; @pure fun my_begin_parse(c: cell): slice diff --git a/tolk-tester/tests/var-apply-tests.tolk b/tolk-tester/tests/var-apply-tests.tolk index 9d9a0a0b8..1c0f164cb 100644 --- a/tolk-tester/tests/var-apply-tests.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -3,6 +3,14 @@ type MIntN = int?; type MBuilder = builder; type CallbackIntToInt = int -> int; +@pure +fun beginParse(c: cell): slice + asm "CTOS"; + +@pure +fun endCell(b: builder): cell + asm "ENDC"; + fun getBeginCell() { return beginCell } @@ -71,24 +79,24 @@ fun demo_handler(op: int, query_id: int, a: int, b: int): int { fun testVarApplyInTernary() { var t: tuple = createEmptyTuple(); try { - t.tuplePush(demo_handler(0xF2, 122, 100, 200)); + t.push(demo_handler(0xF2, 122, 100, 200)); } catch(code) { - t.tuplePush(code); + t.push(code); } try { - t.tuplePush(demo_handler(0xF4, 122, 100, 200)) + t.push(demo_handler(0xF4, 122, 100, 200)) } catch(code) { - t.tuplePush(code) + t.push(code) } try { - t.tuplePush(demo_handler(0xF2, 122, 10, 10)) + t.push(demo_handler(0xF2, 122, 10, 10)) } catch(code) { - t.tuplePush(code) + t.push(code) } try { - t.tuplePush(demo_handler(0xF2, 123, 10, 10)) + t.push(demo_handler(0xF2, 123, 10, 10)) } catch(code) { - t.tuplePush(code) + t.push(code) } return t; } @@ -114,22 +122,22 @@ fun testGlobalVarApply() { fun testVarApply2() { var creator = createEmptyTuple; var t = creator(); - t.tuplePush(1); - var sizer = t.tupleSize; + t.push(1); + var sizer = t.size; return sizer(t); } fun getTupleLastGetter(): (tuple) -> X { - return tupleLast; + return X.last; } @method_id(106) fun testVarApply3() { var t = createEmptyTuple(); - t.tuplePush(1); - t.tuplePush([2]); - var getIntAt = t.tupleAt; - var getTupleFirstInt = createEmptyTuple().tupleFirst; + t.push(1); + t.push([2]); + var getIntAt = t.get; + var getTupleFirstInt = createEmptyTuple().first; var getTupleLastTuple = getTupleLastGetter(); return (getIntAt(t, 0), getTupleFirstInt(t), getTupleLastTuple(t), getTupleLastGetter()(t)); } @@ -227,7 +235,7 @@ struct WithCallbacks { beginCell: builder -> cell, } -fun doEndCellActually(w: WithCallbacks, b: builder) { return w.beginCell(b); } +fun WithCallbacks.doEndCellActually(self, b: builder) { return self.beginCell(b); } @method_id(115) fun testCallableFields() { @@ -258,6 +266,54 @@ fun testCallableGenericField() { return (w1.value(5), w2.value!(9), wrap(w_f(justAdd2)).value.value(78)); } +struct Point { + x: int; + y: int; +} + +fun Point.createXY(x: int, y: int): Point { return {x,y}; } +fun Point.createFrom(x: U, y: V): Point { return {x,y}; } + +fun Point.maxCoord(self) { return self.x > self.y ? self.x : self.y; } + +@method_id(117) +fun testStaticAndInstanceMethods() { + var fPointCreateXY = Point.createXY; + var p1 = fPointCreateXY(5,6); + var fPointCreateFrom = Point.createFrom; + var p2 = fPointCreateFrom(7,8); + __expect_type(p1, "Point"); + __expect_type(p2, "Point"); + var getM1 = Point.maxCoord; + var getM2 = p1.maxCoord; + return (getM1(p1), (getM2 = p2.maxCoord)(p2)); +} + +fun Wrapper.createFromNull(): Wrapper { + return {value: null}; +} +fun Wrapper.createFrom(value: U): Wrapper { + return {value: value as T}; +} +fun Wrapper.equalsNotNull(self, anotherW: Wrapper) { + return self.value != null && anotherW.value != null && self.value == anotherW.value; +} + +@method_id(118) +fun testMethodsOfGenericStruct() { + var creator1 = Wrapper.createFrom; + var w1 = creator1(10); + __expect_type(w1, "Wrapper"); + var creator2 = Wrapper.createFrom; + var w2 = creator2(null as int16?); + __expect_type(w2, "Wrapper"); + var eq = Wrapper.equalsNotNull; + __expect_type(eq, "(Wrapper, Wrapper) -> bool"); + var creator3 = Wrapper.createFromNull; + __expect_type(creator3, "() -> Wrapper"); + return (eq(w1, w2), w2.value = w1.value as int, eq(w1, w2), creator3()); +} + fun main() {} @@ -285,4 +341,6 @@ fun main() {} @testcase | 114 | 0 | (null) 12 1 @testcase | 115 | | 7 6 7 @testcase | 116 | | 7 11 80 +@testcase | 117 | | 6 8 +@testcase | 118 | | 0 10 -1 (null) */ diff --git a/tolk-tester/tests/warnings-not-errors/warnings-1.tolk b/tolk-tester/tests/warnings-not-errors/warnings-1.tolk index 5dc3acdc3..ec77a2a66 100644 --- a/tolk-tester/tests/warnings-not-errors/warnings-1.tolk +++ b/tolk-tester/tests/warnings-not-errors/warnings-1.tolk @@ -6,7 +6,7 @@ fun main() { if (c == null) {} var d: int? = c; - if (((d)) != null && tupleSize(createEmptyTuple())) {} + if (((d)) != null && createEmptyTuple().size()) {} var e: int? = getNullableInt(); if (e != null) { diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 74ecbd282..73791e3dd 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -291,7 +291,7 @@ static V parse_genericsT_list(Lexer& lex) { return createV{loc, std::move(genericsT_items)}; } -static AnyV parse_parameter(Lexer& lex, bool is_first) { +static AnyV parse_parameter(Lexer& lex, AnyTypeV self_type) { SrcLocation loc = lex.cur_location(); // optional keyword `mutate` meaning that a function will mutate a passed argument (like passed by reference) @@ -303,21 +303,28 @@ static AnyV parse_parameter(Lexer& lex, bool is_first) { // parameter name (or underscore for an unnamed parameter) std::string_view param_name; + bool is_self = false; if (lex.tok() == tok_identifier) { param_name = lex.cur_str(); } else if (lex.tok() == tok_self) { - if (!is_first) { - lex.error("`self` can only be the first parameter"); + if (!self_type) { + lex.error("`self` can only be the first parameter of a method"); } + is_self = true; param_name = "self"; } else if (lex.tok() != tok_underscore) { lex.unexpected("parameter name"); } lex.next(); - // parameter type after colon are mandatory - lex.expect(tok_colon, "`: `"); - AnyTypeV param_type = parse_type_from_tokens(lex); + // parameter type after colon is mandatory + AnyTypeV param_type = self_type; + if (!is_self) { + lex.expect(tok_colon, "`: `"); + param_type = parse_type_from_tokens(lex); + } else if (lex.tok() == tok_colon) { + lex.error("`self` parameter should not have a type"); + } return createV(loc, param_name, param_type, declared_as_mutate); } @@ -459,18 +466,19 @@ static AnyExprV parse_local_vars_declaration_assignment(Lexer& lex) { } // "parameters" are at function declaration: `fun f(param1: int, mutate param2: slice)` -static V parse_parameter_list(Lexer& lex) { +// for methods like `fun builder.storeUint(self, i: int)`, receiver_type = builder (type of self) +static V parse_parameter_list(Lexer& lex, AnyTypeV receiver_type) { SrcLocation loc = lex.cur_location(); std::vector params; lex.expect(tok_oppar, "parameter list"); if (lex.tok() != tok_clpar) { - params.push_back(parse_parameter(lex, true)); + params.push_back(parse_parameter(lex, receiver_type)); while (lex.tok() == tok_comma) { lex.next(); if (lex.tok() == tok_clpar) { // trailing comma break; } - params.push_back(parse_parameter(lex, false)); + params.push_back(parse_parameter(lex, nullptr)); } } lex.expect(tok_clpar, "`)`"); @@ -1273,12 +1281,14 @@ static V parse_annotation(Lexer& lex) { case AnnotationKind::inline_simple: case AnnotationKind::inline_ref: case AnnotationKind::pure: - case AnnotationKind::deprecated: if (v_arg) { throw ParseError(v_arg->loc, "arguments aren't allowed for " + static_cast(name)); } v_arg = createV(loc, {}); break; + case AnnotationKind::deprecated: + // allowed with and without arguments; it's IDE-only, the compiler doesn't analyze @deprecated + break; case AnnotationKind::method_id: if (!v_arg || v_arg->size() != 1 || v_arg->get_item(0)->kind != ast_int_const) { throw ParseError(loc, "expecting `(number)` after " + static_cast(name)); @@ -1291,21 +1301,31 @@ static V parse_annotation(Lexer& lex) { static AnyV parse_function_declaration(Lexer& lex, const std::vector>& annotations) { SrcLocation loc = lex.cur_location(); - bool is_get_method = lex.tok() == tok_get; + bool is_contract_getter = lex.cur_str() == "get"; lex.next(); - if (is_get_method && lex.tok() == tok_fun) { + if (is_contract_getter && lex.tok() == tok_fun) { lex.next(); // 'get f()' and 'get fun f()' both correct } + AnyTypeV receiver_type = nullptr; + auto backup = lex.save_parsing_position(); + try { + receiver_type = parse_type_expression(lex); + lex.expect(tok_dot, ""); + } catch (const ParseError&) { + receiver_type = nullptr; + lex.restore_position(backup); + } + lex.check(tok_identifier, "function name identifier"); std::string_view f_name = lex.cur_str(); - bool is_entrypoint = + bool is_entrypoint = !receiver_type && ( f_name == "main" || f_name == "onInternalMessage" || f_name == "onExternalMessage" || - f_name == "onRunTickTock" || f_name == "onSplitPrepare" || f_name == "onSplitInstall"; - bool is_FunC_entrypoint = + f_name == "onRunTickTock" || f_name == "onSplitPrepare" || f_name == "onSplitInstall"); + bool is_FunC_entrypoint = !receiver_type && ( f_name == "recv_internal" || f_name == "recv_external" || - f_name == "run_ticktock" || f_name == "split_prepare" || f_name == "split_install"; + f_name == "run_ticktock" || f_name == "split_prepare" || f_name == "split_install"); if (is_FunC_entrypoint) { lex.error("this is a reserved FunC/Fift identifier; you need `onInternalMessage`"); } @@ -1318,7 +1338,7 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector v_param_list = parse_parameter_list(lex)->as(); + V v_param_list = parse_parameter_list(lex, receiver_type)->as(); bool accepts_self = !v_param_list->empty() && v_param_list->get_param(0)->param_name == "self"; int n_mutate_params = v_param_list->get_mutate_params_count(); @@ -1338,11 +1358,11 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vectorkind) { case AnnotationKind::inline_simple: @@ -1389,14 +1409,14 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vectorerror("@method_id can be specified only for regular functions"); } auto v_int = v_annotation->get_arg()->get_item(0)->as(); if (v_int->intval.is_null() || !v_int->intval->signed_fits_bits(32)) { v_int->error("invalid integer constant"); } - method_id = v_int->intval; + tvm_method_id = v_int->intval; break; } case AnnotationKind::deprecated: @@ -1408,7 +1428,7 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector(loc, v_ident, v_param_list, v_body, ret_type, genericsT_list, std::move(method_id), flags); + return createV(loc, v_ident, v_param_list, v_body, receiver_type, ret_type, genericsT_list, std::move(tvm_method_id), flags); } static AnyV parse_struct_field(Lexer& lex) { @@ -1541,7 +1561,6 @@ AnyV parse_src_file_to_ast(const SrcFile* file) { annotations.clear(); break; case tok_fun: - case tok_get: toplevel_declarations.push_back(parse_function_declaration(lex, annotations)); annotations.clear(); break; @@ -1556,8 +1575,15 @@ AnyV parse_src_file_to_ast(const SrcFile* file) { case tok_infix: lex.error("`" + static_cast(lex.cur_str()) +"` is not supported yet"); + case tok_identifier: + if (lex.cur_str() == "get") { // tok-level "get", contract getter + toplevel_declarations.push_back(parse_function_declaration(lex, annotations)); + annotations.clear(); + break; + } + // fallthrough default: - lex.unexpected("fun or get"); + lex.unexpected("top-level declaration"); } } diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index b056db314..b3e1702eb 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -325,15 +325,16 @@ class ASTReplicator final { public: // the cloned function becomes a deep copy, all AST nodes are copied, no previous pointers left - static V clone_function_ast(V v_orig, V new_name_ident) { + static V clone_function_ast(V v_orig) { return createV( v_orig->loc, - new_name_ident, + clone(v_orig->get_identifier()), clone(v_orig->get_param_list()), clone(v_orig->get_body()), + clone(v_orig->receiver_type_node), clone(v_orig->return_type_node), - clone(v_orig->genericsT_list), - v_orig->method_id, + v_orig->genericsT_list ? clone(v_orig->genericsT_list) : nullptr, + v_orig->tvm_method_id, v_orig->flags ); } diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index 4e6f795e9..8ced76a0b 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -215,7 +215,12 @@ class ASTStringifier final : public ASTVisitor { param_names += ","; param_names += v->as()->get_param(i)->param_name; } - return "fun " + static_cast(v->as()->get_identifier()->name) + "(" + param_names + ")"; + std::string decl = "fun "; + if (auto receiver_node = v->as()->receiver_type_node) { + decl += specific_str(receiver_node); + decl += "."; + } + return decl + static_cast(v->as()->get_identifier()->name) + "(" + param_names + ")"; } case ast_local_var_lhs: { std::string str_type = v->as()->inferred_type ? v->as()->inferred_type->as_human_readable() : ast_type_node_to_string(v->as()->type_node); diff --git a/tolk/ast.cpp b/tolk/ast.cpp index 24c797bd2..6e725b781 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -134,8 +134,9 @@ void Vertex::assign_literal_value(std::string&& literal_value) this->literal_value = std::move(literal_value); } -void Vertex::assign_fun_ref(FunctionPtr fun_ref) { +void Vertex::assign_fun_ref(FunctionPtr fun_ref, bool dot_obj_is_self) { this->fun_maybe = fun_ref; + this->dot_obj_is_self = dot_obj_is_self; } void Vertex::assign_is_negated(bool is_negated) { diff --git a/tolk/ast.h b/tolk/ast.h index 927be5b71..013ac2a78 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -684,7 +684,7 @@ struct Vertex final : ASTExprVararg { template<> // ast_dot_access is "object before dot, identifier + optional after dot" -// examples: `tensorVar.0` / `obj.field` / `getObj().method` / `t.tupleFirst` +// examples: `tensorVar.0` / `obj.field` / `getObj().method` / `t.tupleFirst` / `Point.create` // from traversing point of view, it's an unary expression: only obj is expression, field name is not // note, that `obj.method()` is a function call with "dot access `obj.method`" callee struct Vertex final : ASTExprUnary { @@ -725,19 +725,20 @@ template<> // example: `globalF()` then callee is reference (with instantiation Ts filled) // example: `local_var()` then callee is reference (points to local var, filled at resolve identifiers) // example: `getF()()` then callee is another func call (which type is TypeDataFunCallable) -// example: `obj.method()` then callee is dot access (resolved while type inferring) +// example: `obj.method()` then callee is dot access, self_obj = obj +// example: `Point.create()` then callee is dot access, self_obj = nullptr struct Vertex final : ASTExprBinary { FunctionPtr fun_maybe = nullptr; // filled while type inferring for `globalF()` / `obj.f()`; remains nullptr for `local_var()` / `getF()()` + bool dot_obj_is_self = false; // true for `obj.method()` (obj will be `self` in method); false for `globalF()` / `Point.create()` AnyExprV get_callee() const { return lhs; } - bool is_dot_call() const { return lhs->kind == ast_dot_access; } - AnyExprV get_dot_obj() const { return lhs->as()->get_obj(); } + AnyExprV get_self_obj() const { return dot_obj_is_self ? lhs->as()->get_obj() : nullptr; } auto get_arg_list() const { return rhs->as(); } int get_num_args() const { return rhs->as()->size(); } auto get_arg(int i) const { return rhs->as()->get_arg(i); } Vertex* mutate() const { return const_cast(this); } - void assign_fun_ref(FunctionPtr fun_ref); + void assign_fun_ref(FunctionPtr fun_ref, bool dot_obj_is_self); Vertex(SrcLocation loc, AnyExprV lhs_f, V arguments) : ASTExprBinary(ast_function_call, loc, lhs_f, arguments) {} @@ -1166,7 +1167,7 @@ struct Vertex final : ASTOtherVararg { std::vector get_items() const { return children; } auto get_item(int i) const { return children.at(i)->as(); } - Vertex(SrcLocation loc, std::vector instantiationTs) + Vertex(SrcLocation loc, std::vector&& instantiationTs) : ASTOtherVararg(ast_instantiationT_list, loc, std::move(instantiationTs)) {} }; @@ -1229,9 +1230,10 @@ struct Vertex final : ASTOtherVararg { AnyV get_body() const { return children.at(2); } // ast_block_statement / ast_asm_body FunctionPtr fun_ref = nullptr; // filled after register + AnyTypeV receiver_type_node; // for `fun builder.storeInt`, here is `builder` AnyTypeV return_type_node; // if unspecified (nullptr), means "auto infer" V genericsT_list; // for non-generics it's nullptr - td::RefInt256 method_id; // specified via @method_id annotation + td::RefInt256 tvm_method_id; // specified via @method_id annotation int flags; // from enum in FunctionData bool is_asm_function() const { return children.at(2)->kind == ast_asm_body; } @@ -1241,9 +1243,9 @@ struct Vertex final : ASTOtherVararg { Vertex* mutate() const { return const_cast(this); } void assign_fun_ref(FunctionPtr fun_ref); - Vertex(SrcLocation loc, V name_identifier, V parameters, AnyV body, AnyTypeV return_type_node, V genericsT_list, td::RefInt256 method_id, int flags) + Vertex(SrcLocation loc, V name_identifier, V parameters, AnyV body, AnyTypeV receiver_type_node, AnyTypeV return_type_node, V genericsT_list, td::RefInt256 tvm_method_id, int flags) : ASTOtherVararg(ast_function_declaration, loc, {name_identifier, parameters, body}) - , return_type_node(return_type_node), genericsT_list(genericsT_list), method_id(std::move(method_id)), flags(flags) {} + , receiver_type_node(receiver_type_node), return_type_node(return_type_node), genericsT_list(genericsT_list), tvm_method_id(std::move(tvm_method_id)), flags(flags) {} }; template<> diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 7de0b3d84..5b962f604 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -40,16 +40,18 @@ static std::vector define_builtin_parameters(const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const simple_compile_func_t& func, int flags) { - auto* f_sym = new FunctionData(name, {}, return_type, define_builtin_parameters(params_types, flags), flags, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); + auto* f_sym = new FunctionData(name, {}, "", nullptr, return_type, define_builtin_parameters(params_types, flags), flags, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); G.symtable.add_function(f_sym); } -static void define_builtin_func(const std::string& name, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const simple_compile_func_t& func, int flags, - std::initializer_list arg_order, std::initializer_list ret_order) { - auto* f_sym = new FunctionData(name, {}, return_type, define_builtin_parameters(params_types, flags), flags, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); +static void define_builtin_method(const std::string& name, TypePtr receiver_type, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const simple_compile_func_t& func, int flags, + std::initializer_list arg_order = {}, std::initializer_list ret_order = {}) { + std::string method_name = name.substr(name.find('.') + 1); + auto* f_sym = new FunctionData(name, {}, std::move(method_name), receiver_type, return_type, define_builtin_parameters(params_types, flags), flags, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); f_sym->arg_order = arg_order; f_sym->ret_order = ret_order; G.symtable.add_function(f_sym); + G.all_methods.push_back(f_sym); } void FunctionBodyBuiltin::compile(AsmOpList& dest, std::vector& out, std::vector& in, @@ -988,10 +990,10 @@ static AsmOp compile_bool_const(std::vector& res, std::vector 1 0) "LDIX"; -// fun loadUint (mutate s: slice, len: int): int asm( -> 1 0) "LDUX"; -// fun preloadInt (s: slice, len: int): int asm "PLDIX"; -// fun preloadUint(s: slice, len: int): int asm "PLDUX"; +// fun slice.loadInt (mutate self, len: int): int asm(s len -> 1 0) "LDIX"; +// fun slice.loadUint (mutate self, len: int): int asm( -> 1 0) "LDUX"; +// fun slice.preloadInt (self, len: int): int asm "PLDIX"; +// fun slice.preloadUint(self, len: int): int asm "PLDUX"; static AsmOp compile_fetch_int(std::vector& res, std::vector& args, SrcLocation loc, bool fetch, bool sgnd) { tolk_assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); auto &y = args[1], &r = res.back(); @@ -1013,8 +1015,8 @@ static AsmOp compile_fetch_int(std::vector& res, std::vector return exec_op(loc, (fetch ? "LD"s : "PLD"s) + (sgnd ? "IX" : "UX"), 2, 1 + (unsigned)fetch); } -// fun storeInt (mutate self: builder, x: int, len: int): self asm(x b len) "STIX"; -// fun storeUint (mutate self: builder, x: int, len: int): self asm(x b len) "STUX"; +// fun builder.storeInt (mutate self, x: int, len: int): self asm(x b len) "STIX"; +// fun builder.storeUint (mutate self, x: int, len: int): self asm(x b len) "STUX"; static AsmOp compile_store_int(std::vector& res, std::vector& args, SrcLocation loc, bool sgnd) { tolk_assert(args.size() == 3 && res.size() == 1); auto& z = args[2]; @@ -1025,8 +1027,8 @@ static AsmOp compile_store_int(std::vector& res, std::vector return exec_op(loc, "ST"s + (sgnd ? "IX" : "UX"), 3, 1); } -// fun loadBits (mutate self: slice, len: int): self asm(s len -> 1 0) "LDSLICEX" -// fun preloadBits(self: slice, len: int): slice asm(s len -> 1 0) "PLDSLICEX" +// fun slice.loadBits (mutate self, len: int): self asm(s len -> 1 0) "LDSLICEX" +// fun slice.preloadBits(self, len: int): slice asm(s len -> 1 0) "PLDSLICEX" static AsmOp compile_fetch_slice(std::vector& res, std::vector& args, SrcLocation loc, bool fetch) { tolk_assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); auto& y = args[1]; @@ -1041,8 +1043,8 @@ static AsmOp compile_fetch_slice(std::vector& res, std::vector(t: tuple, index: int): X asm "INDEXVAR"; -static AsmOp compile_tuple_at(std::vector& res, std::vector& args, SrcLocation loc) { +// fun tuple.get(t: tuple, index: int): X asm "INDEXVAR"; +static AsmOp compile_tuple_get(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(args.size() == 2 && res.size() == 1); auto& y = args[1]; if (y.is_int_const() && y.int_const >= 0 && y.int_const < 16) { @@ -1052,7 +1054,7 @@ static AsmOp compile_tuple_at(std::vector& res, std::vector& return exec_op(loc, "INDEXVAR", 2, 1); } -// fun tupleSetAt(mutate self: tuple, value: X, index: int): void asm "SETINDEXVAR"; +// fun tuple.set(mutate self: tuple, value: X, index: int): void asm "SETINDEXVAR"; static AsmOp compile_tuple_set_at(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(args.size() == 3 && res.size() == 1); auto& y = args[2]; @@ -1063,17 +1065,17 @@ static AsmOp compile_tuple_set_at(std::vector& res, std::vector&, std::vector&, SrcLocation loc) { return AsmOp::Custom(loc, "DUMPSTK", 0, 0); } -// fun debugPrintString(x: T): void asm "STRDUMP"; +// fun debug.printString(x: T): void asm "STRDUMP"; static AsmOp compile_strdump(std::vector&, std::vector&, SrcLocation loc) { return AsmOp::Custom(loc, "STRDUMP DROP", 1, 1); } -// fun debugPrint(x: T): void; +// fun debug.print(x: T): void; static AsmOp compile_debug_print_to_string(std::vector&, std::vector&, SrcLocation loc) { return AsmOp::Custom(loc, "s0 DUMP DROP", 1, 1); } @@ -1116,16 +1118,18 @@ void define_builtins() { TypePtr Tuple = TypeDataTuple::create(); TypePtr Never = TypeDataNever::create(); - std::vector itemsT; - itemsT.emplace_back("T"); TypePtr typeT = TypeDataGenericT::create("T"); - const GenericsDeclaration* declGenericT = new GenericsDeclaration(std::move(itemsT)); + const GenericsDeclaration* declGenericT = new GenericsDeclaration(std::vector{"T"}, 0); std::vector ParamsInt1 = {Int}; std::vector ParamsInt2 = {Int, Int}; std::vector ParamsInt3 = {Int, Int, Int}; std::vector ParamsSliceInt = {Slice, Int}; + // these types are defined in stdlib, currently unknown + // see patch_builtins_after_stdlib_loaded() below + TypePtr debug = TypeDataUnknown::create(); + // builtin operators // they are internally stored as functions, because at IR level, there is no difference // between calling `userAdd(a,b)` and `_+_(a,b)` @@ -1280,48 +1284,48 @@ void define_builtins() { define_builtin_func("mulDivMod", ParamsInt3, TypeDataTensor::create({Int, Int}), nullptr, compile_muldivmod, FunctionData::flagMarkedAsPure); - define_builtin_func("loadInt", ParamsSliceInt, Int, nullptr, + define_builtin_method("slice.loadInt", Slice, ParamsSliceInt, Int, nullptr, std::bind(compile_fetch_int, _1, _2, _3, true, true), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, {}, {1, 0}); - define_builtin_func("loadUint", ParamsSliceInt, Int, nullptr, + define_builtin_method("slice.loadUint", Slice, ParamsSliceInt, Int, nullptr, std::bind(compile_fetch_int, _1, _2, _3, true, false), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, {}, {1, 0}); - define_builtin_func("loadBits", ParamsSliceInt, Slice, nullptr, + define_builtin_method("slice.loadBits", Slice, ParamsSliceInt, Slice, nullptr, std::bind(compile_fetch_slice, _1, _2, _3, true), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, {}, {1, 0}); - define_builtin_func("preloadInt", ParamsSliceInt, Int, nullptr, + define_builtin_method("slice.preloadInt", Slice, ParamsSliceInt, Int, nullptr, std::bind(compile_fetch_int, _1, _2, _3, false, true), FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); - define_builtin_func("preloadUint", ParamsSliceInt, Int, nullptr, + define_builtin_method("slice.preloadUint", Slice, ParamsSliceInt, Int, nullptr, std::bind(compile_fetch_int, _1, _2, _3, false, false), FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); - define_builtin_func("preloadBits", ParamsSliceInt, Slice, nullptr, + define_builtin_method("slice.preloadBits", Slice, ParamsSliceInt, Slice, nullptr, std::bind(compile_fetch_slice, _1, _2, _3, false), FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); - define_builtin_func("storeInt", {Builder, Int, Int}, Unit, nullptr, + define_builtin_method("builder.storeInt", Builder, {Builder, Int, Int}, Unit, nullptr, std::bind(compile_store_int, _1, _2, _3, true), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf, {1, 0, 2}, {}); - define_builtin_func("storeUint", {Builder, Int, Int}, Unit, nullptr, + define_builtin_method("builder.storeUint", Builder, {Builder, Int, Int}, Unit, nullptr, std::bind(compile_store_int, _1, _2, _3, false), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf, {1, 0, 2}, {}); - define_builtin_func("tupleAt", {Tuple, Int}, typeT, declGenericT, - compile_tuple_at, + define_builtin_method("tuple.get", Tuple, {Tuple, Int}, typeT, declGenericT, + compile_tuple_get, FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); - define_builtin_func("tupleSetAt", {Tuple, typeT, Int}, Unit, declGenericT, + define_builtin_method("tuple.set", Tuple, {Tuple, typeT, Int}, Unit, declGenericT, compile_tuple_set_at, FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf); - define_builtin_func("debugPrint", {typeT}, Unit, declGenericT, + define_builtin_method("debug.print", debug, {typeT}, Unit, declGenericT, compile_debug_print_to_string, 0); - define_builtin_func("debugPrintString", {typeT}, Unit, declGenericT, + define_builtin_method("debug.printString", debug, {typeT}, Unit, declGenericT, compile_strdump, 0); - define_builtin_func("debugDumpStack", {}, Unit, nullptr, + define_builtin_method("debug.dumpStack", debug, {}, Unit, nullptr, compile_dumpstk, 0); @@ -1333,4 +1337,17 @@ void define_builtins() { FunctionData::flagMarkedAsPure); } +// there are some built-in functions that operate on types declared in stdlib (like Cell) +// that's why these symbols were undefined, and when builtins were registered, they were set to unknown +// after all files have been loaded, symbols have been registered, and aliases exist, +// we patch that earlier registered built-in functions providing types that now exist +void patch_builtins_after_stdlib_loaded() { + StructPtr struct_debug = lookup_global_symbol("debug")->try_as(); + TypePtr debug = TypeDataStruct::create(struct_debug); + + lookup_function("debug.print")->mutate()->receiver_type = debug; + lookup_function("debug.printString")->mutate()->receiver_type = debug; + lookup_function("debug.dumpStack")->mutate()->receiver_type = debug; +} + } // namespace tolk diff --git a/tolk/compiler-state.h b/tolk/compiler-state.h index 9b67293b5..2193503a2 100644 --- a/tolk/compiler-state.h +++ b/tolk/compiler-state.h @@ -96,8 +96,9 @@ struct CompilerState { GlobalSymbolTable symtable; PersistentHeapAllocator persistent_mem; - std::vector all_functions; // all user-defined (not built-in) functions, with generic instantiations - std::vector all_get_methods; + std::vector all_functions; // all user-defined (not built-in) global-scope functions, with generic instantiations + std::vector all_methods; // all user-defined and built-in extension methods for arbitrary types (receivers) + std::vector all_contract_getters; std::vector all_global_vars; std::vector all_constants; std::vector all_structs; diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index 9983ca49d..0f8e6356c 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -24,7 +24,7 @@ namespace tolk { -// given orig `(int, T)` and substitutions [T=slice], return `(int, slice)` +// given orig `(int, T)` and substitutions [slice], return `(int, slice)` static TypePtr replace_genericT_with_deduced(TypePtr orig, const GenericsSubstitutions* substitutedTs) { if (!orig || !orig->has_genericT_inside()) { return orig; @@ -52,9 +52,18 @@ static TypePtr replace_genericT_with_deduced(TypePtr orig, const GenericsSubstit }); } -std::string GenericsSubstitutions::as_human_readable() const { +GenericsSubstitutions::GenericsSubstitutions(const GenericsDeclaration* genericTs, const std::vector& type_arguments) + : genericTs(genericTs) + , valuesTs(genericTs->size()) { + provide_type_arguments(type_arguments); +} + +std::string GenericsSubstitutions::as_human_readable(bool show_nullptr) const { std::string result; for (int i = 0; i < size(); ++i) { + if (valuesTs[i] == nullptr && !show_nullptr) { + continue;; + } if (!result.empty()) { result += ", "; } @@ -74,9 +83,6 @@ void GenericsSubstitutions::set_typeT(std::string_view nameT, TypePtr typeT) { for (int i = 0; i < size(); ++i) { if (genericTs->get_nameT(i) == nameT) { if (valuesTs[i] == nullptr) { - if (typeT == TypeDataNullLiteral::create() || typeT->has_unknown_inside()) { - throw GenericDeduceError(nameT); - } tolk_assert(!typeT->has_genericT_inside()); valuesTs[i] = typeT; } @@ -85,12 +91,12 @@ void GenericsSubstitutions::set_typeT(std::string_view nameT, TypePtr typeT) { } } -GenericsSubstitutions::GenericsSubstitutions(const GenericsDeclaration* genericTs, const std::vector& type_arguments) - : genericTs(genericTs) { - assert(genericTs != nullptr && genericTs->size() == static_cast(type_arguments.size())); - valuesTs.resize(genericTs->size()); - for (int i = 0; i < genericTs->size(); ++i) { - valuesTs[i] = type_arguments[i]; +void GenericsSubstitutions::provide_type_arguments(const std::vector& type_arguments) { + tolk_assert(genericTs != nullptr); + int start_from = genericTs->n_from_receiver; // for `Container.wrap` user should specify only U + tolk_assert(static_cast(type_arguments.size()) + start_from == genericTs->size()); + for (int i = start_from; i < genericTs->size(); ++i) { + valuesTs[i] = type_arguments[i - start_from]; } } @@ -205,18 +211,17 @@ TypePtr GenericSubstitutionsDeducing::replace_Ts_with_currently_deduced(TypePtr return replace_genericT_with_deduced(orig, &deducedTs); } +TypePtr GenericSubstitutionsDeducing::auto_deduce_from_argument(TypePtr param_type, TypePtr arg_type) { + consider_next_condition(param_type, arg_type); + return replace_genericT_with_deduced(param_type, &deducedTs); +} + TypePtr GenericSubstitutionsDeducing::auto_deduce_from_argument(FunctionPtr cur_f, SrcLocation loc, TypePtr param_type, TypePtr arg_type) { - try { - consider_next_condition(param_type, arg_type); - param_type = replace_genericT_with_deduced(param_type, &deducedTs); - if (param_type->has_genericT_inside()) { - fire_error_can_not_deduce(cur_f, loc, get_first_not_deduced_nameT()); - } - return param_type; - } catch (const GenericDeduceError& ex) { - fire_error_can_not_deduce(cur_f, loc, ex.nameT); - return nullptr; + param_type = auto_deduce_from_argument(param_type, arg_type); + if (param_type->has_genericT_inside()) { + fire_error_can_not_deduce(cur_f, loc, get_first_not_deduced_nameT()); } + return param_type; } std::string_view GenericSubstitutionsDeducing::get_first_not_deduced_nameT() const { @@ -236,27 +241,43 @@ void GenericSubstitutionsDeducing::fire_error_can_not_deduce(FunctionPtr cur_f, } } -std::string GenericsDeclaration::as_human_readable() const { - std::string result = "<"; - for (const GenericsItem& item : itemsT) { - if (result.size() > 1) { - result += ", "; +std::string GenericsDeclaration::as_human_readable(bool include_from_receiver) const { + std::string result; + if (int start_from = include_from_receiver ? 0 : n_from_receiver; start_from < size()) { + result += '<'; + for (int i = start_from; i < size(); ++i) { + if (result.size() > 1) { + result += ", "; + } + result += namesT[i]; } - result += item.nameT; + result += '>'; } - result += ">"; return result; } int GenericsDeclaration::find_nameT(std::string_view nameT) const { - for (int i = 0; i < static_cast(itemsT.size()); ++i) { - if (itemsT[i].nameT == nameT) { + for (int i = 0; i < static_cast(namesT.size()); ++i) { + if (namesT[i] == nameT) { return i; } } return -1; } +void GenericsDeclaration::append_ast_list_checking_duplicates(AnyV v_genericsT_list, std::vector& out) { + for (int i = 0; i < v_genericsT_list->as()->size(); ++i) { + auto v_item = v_genericsT_list->as()->get_item(i); + auto it_existing = std::find_if(out.begin(), out.end(), [v_item](const std::string_view& prevT) { + return prevT == v_item->nameT; + }); + if (it_existing != out.end()) { + v_item->error("duplicate generic parameter `" + static_cast(v_item->nameT) + "`"); + } + out.emplace_back(v_item->nameT); + } +} + bool GenericsSubstitutions::has_nameT(std::string_view nameT) const { return genericTs->find_nameT(nameT) != -1; } @@ -282,28 +303,34 @@ bool GenericsSubstitutions::equal_to(const GenericsSubstitutions* rhs) const { // when cloning `f`, original name is "f", we need a new name for symtable and output // name of an instantiated function will be "f" and similar (yes, with "<" symbol, it's okay to Fift) -static std::string generate_instantiated_name(const std::string& orig_name, const GenericsSubstitutions& substitutedTs, bool allow_spaces) { +static std::string generate_instantiated_name(const std::string& orig_name, const GenericsSubstitutions& substitutedTs, bool allow_spaces, int size_from_receiver = 0) { // an instantiated function name will be "{orig_name}<{T1,T2,...}>" std::string name = orig_name; - name += "<"; - for (int i = 0; i < substitutedTs.size(); ++i) { - if (name.size() > orig_name.size() + 1) { - name += ", "; + if (size_from_receiver < substitutedTs.size()) { + name += '<'; + for (int i = size_from_receiver; i < substitutedTs.size(); ++i) { + if (name.size() > orig_name.size() + 1) { + name += ", "; + } + name += substitutedTs.typeT_at(i)->as_human_readable(); } - name += substitutedTs.typeT_at(i)->as_human_readable(); + name += '>'; } if (!allow_spaces) { name.erase(std::remove(name.begin(), name.end(), ' '), name.end()); } - name += ">"; return name; } FunctionPtr instantiate_generic_function(FunctionPtr fun_ref, GenericsSubstitutions&& substitutedTs) { - tolk_assert(fun_ref->is_generic_function() && !fun_ref->is_method_id_not_empty()); + tolk_assert(fun_ref->is_generic_function() && !fun_ref->has_tvm_method_id()); // fun_ref->name = "f", inst_name will be "f" and similar - std::string new_name = generate_instantiated_name(fun_ref->name, substitutedTs, false); + std::string fun_name = fun_ref->name; + if (fun_ref->is_method() && fun_ref->receiver_type->has_genericT_inside()) { + fun_name = replace_genericT_with_deduced(fun_ref->receiver_type, &substitutedTs)->as_human_readable() + "." + fun_ref->method_name; + } + std::string new_name = generate_instantiated_name(fun_name, substitutedTs, false, fun_ref->genericTs->n_from_receiver); if (const Symbol* existing_sym = lookup_global_symbol(new_name)) { FunctionPtr existing_ref = existing_sym->try_as(); tolk_assert(existing_ref); @@ -324,7 +351,8 @@ FunctionPtr instantiate_generic_function(FunctionPtr fun_ref, GenericsSubstituti new_parameters.emplace_back(orig_p.name, orig_p.loc, new_param_type, orig_p.flags, orig_p.param_idx); } TypePtr new_return_type = replace_genericT_with_deduced(fun_ref->declared_return_type, allocatedTs); - FunctionData* new_fun_ref = new FunctionData(new_name, fun_ref->loc, new_return_type, std::move(new_parameters), fun_ref->flags, nullptr, allocatedTs, fun_ref->body, fun_ref->ast_root); + TypePtr new_receiver_type = replace_genericT_with_deduced(fun_ref->receiver_type, allocatedTs); + FunctionData* new_fun_ref = new FunctionData(new_name, fun_ref->loc, fun_ref->method_name, new_receiver_type, new_return_type, std::move(new_parameters), fun_ref->flags, nullptr, allocatedTs, fun_ref->body, fun_ref->ast_root); new_fun_ref->arg_order = fun_ref->arg_order; new_fun_ref->ret_order = fun_ref->ret_order; new_fun_ref->base_fun_ref = fun_ref; @@ -336,10 +364,9 @@ FunctionPtr instantiate_generic_function(FunctionPtr fun_ref, GenericsSubstituti // it means, that types still contain T: `f(v: T): T`, but since type resolving knows // it's instantiation, when resolving types, it substitutes T=int V orig_root = fun_ref->ast_root->as(); - V new_name_ident = createV(orig_root->get_identifier()->loc, new_name); - V new_root = ASTReplicator::clone_function_ast(orig_root, new_name_ident); + V new_root = ASTReplicator::clone_function_ast(orig_root); - FunctionPtr new_fun_ref = pipeline_register_instantiated_generic_function(fun_ref, new_root, allocatedTs); + FunctionPtr new_fun_ref = pipeline_register_instantiated_generic_function(fun_ref, new_root, std::move(new_name), allocatedTs); tolk_assert(new_fun_ref); // body of a cloned function (it's cloned at type inferring step) needs the previous pipeline to run // for example, all local vars need to be registered as symbols, etc. @@ -368,7 +395,7 @@ StructPtr instantiate_generic_struct(StructPtr struct_ref, GenericsSubstitutions V new_name_ident = createV(orig_root->get_identifier()->loc, new_name); V new_root = ASTReplicator::clone_struct_ast(orig_root, new_name_ident); - StructPtr new_struct_ref = pipeline_register_instantiated_generic_struct(struct_ref, new_root, allocatedTs); + StructPtr new_struct_ref = pipeline_register_instantiated_generic_struct(struct_ref, new_root, std::move(new_name), allocatedTs); tolk_assert(new_struct_ref); pipeline_resolve_identifiers_and_assign_symbols(new_struct_ref); pipeline_resolve_types_and_aliases(new_struct_ref); @@ -391,10 +418,88 @@ AliasDefPtr instantiate_generic_alias(AliasDefPtr alias_ref, GenericsSubstitutio V new_name_ident = createV(orig_root->get_identifier()->loc, new_name); V new_root = ASTReplicator::clone_type_alias_ast(orig_root, new_name_ident); - AliasDefPtr new_alias_ref = pipeline_register_instantiated_generic_alias(alias_ref, new_root, allocatedTs); + AliasDefPtr new_alias_ref = pipeline_register_instantiated_generic_alias(alias_ref, new_root, std::move(new_name), allocatedTs); tolk_assert(new_alias_ref); pipeline_resolve_types_and_aliases(new_alias_ref); return new_alias_ref; } +// find `builder.storeInt` for called_receiver = "builder" and called_name = "storeInt" +// most practical case, when a direct method for receiver exists +FunctionPtr match_exact_method_for_call_not_generic(TypePtr called_receiver, std::string_view called_name) { + FunctionPtr exact_found = nullptr; + + for (FunctionPtr method_ref : G.all_methods) { + if (method_ref->method_name == called_name && !method_ref->receiver_type->has_genericT_inside()) { + if (method_ref->receiver_type->equal_to(called_receiver)) { + if (exact_found) { + return nullptr; + } + exact_found = method_ref; + } + } + } + + return exact_found; +} + +// find `int?.copy` / `T.copy` for called_receiver = "int" and called_name = "copy" +std::vector match_methods_for_call_including_generic(TypePtr called_receiver, std::string_view called_name) { + std::vector candidates; + + // step1: find all methods where a receiver equals to provided, e.g. `MInt.copy` + for (FunctionPtr method_ref : G.all_methods) { + if (method_ref->method_name == called_name && !method_ref->receiver_type->has_genericT_inside()) { + if (method_ref->receiver_type->equal_to(called_receiver)) { + candidates.emplace_back(method_ref, GenericsSubstitutions(method_ref->genericTs)); + } + } + } + if (!candidates.empty()) { + return candidates; + } + + // step2: find all methods where a receiver can accept provided, e.g. `int8.copy` / `int?.copy` / `(int|slice).copy` + for (FunctionPtr method_ref : G.all_methods) { + if (method_ref->method_name == called_name && !method_ref->receiver_type->has_genericT_inside()) { + if (method_ref->receiver_type->can_rhs_be_assigned(called_receiver)) { + candidates.emplace_back(method_ref, GenericsSubstitutions(method_ref->genericTs)); + } + } + } + if (!candidates.empty()) { + return candidates; + } + + // step 3: try to match generic receivers, e.g. `Container.copy` / `(T?|slice).copy` but NOT `T.copy` + for (FunctionPtr method_ref : G.all_methods) { + if (method_ref->method_name == called_name && method_ref->receiver_type->has_genericT_inside() && !method_ref->receiver_type->try_as()) { + try { + GenericSubstitutionsDeducing deducingTs(method_ref); + TypePtr replaced = deducingTs.auto_deduce_from_argument(method_ref->receiver_type, called_receiver); + if (!replaced->has_genericT_inside()) { + candidates.emplace_back(method_ref, deducingTs.flush()); + } + } catch (...) {} + } + } + if (!candidates.empty()) { + return candidates; + } + + // step 4: try to match `T.copy` + for (FunctionPtr method_ref : G.all_methods) { + if (method_ref->method_name == called_name && method_ref->receiver_type->try_as()) { + try { + GenericSubstitutionsDeducing deducingTs(method_ref); + TypePtr replaced = deducingTs.auto_deduce_from_argument(method_ref->receiver_type, called_receiver); + if (!replaced->has_genericT_inside()) { + candidates.emplace_back(method_ref, deducingTs.flush()); + } + } catch (...) {} + } + } + return candidates; +} + } // namespace tolk diff --git a/tolk/generics-helpers.h b/tolk/generics-helpers.h index 7d324e625..08dfcc981 100644 --- a/tolk/generics-helpers.h +++ b/tolk/generics-helpers.h @@ -24,25 +24,26 @@ namespace tolk { // when a function is declared `f`, this "" is represented as this class -// (not at AST, but at symbol storage level) +// - `fun tuple.push(self, value:T)` namesT = [T] +// - `struct Pair` namesT = [A,B] +// - `fun Container.compareWith` namesT = [T,U], n_from_receiver = 1 +// - `fun Pair.createFrom` namesT = [U,V] +// - `fun Pair.create` namesT = [A,B], n_from_receiver = 2 struct GenericsDeclaration { - struct GenericsItem { - std::string_view nameT; + explicit GenericsDeclaration(std::vector&& namesT, int n_from_receiver) + : namesT(std::move(namesT)) + , n_from_receiver(n_from_receiver) {} - explicit GenericsItem(std::string_view nameT) - : nameT(nameT) {} - }; + const std::vector namesT; + const int n_from_receiver; - explicit GenericsDeclaration(std::vector&& itemsT) - : itemsT(std::move(itemsT)) {} + std::string as_human_readable(bool include_from_receiver = false) const; - const std::vector itemsT; - - std::string as_human_readable() const; - - int size() const { return static_cast(itemsT.size()); } + int size() const { return static_cast(namesT.size()); } int find_nameT(std::string_view nameT) const; - std::string_view get_nameT(int idx) const { return itemsT[idx].nameT; } + std::string_view get_nameT(int idx) const { return namesT[idx]; } + + static void append_ast_list_checking_duplicates(AnyV v_genericsT_list, std::vector& out); }; // when a function call is `f()`, this "" is represented as this class @@ -59,7 +60,7 @@ struct GenericsSubstitutions { } explicit GenericsSubstitutions(const GenericsDeclaration* genericTs, const std::vector& type_arguments); - std::string as_human_readable() const; + std::string as_human_readable(bool show_nullptr) const; int size() const { return static_cast(valuesTs.size()); } bool has_nameT(std::string_view nameT) const; @@ -69,6 +70,7 @@ struct GenericsSubstitutions { bool equal_to(const GenericsSubstitutions* rhs) const; void set_typeT(std::string_view nameT, TypePtr typeT); + void provide_type_arguments(const std::vector& type_arguments); }; // this class helps to deduce Ts on the fly @@ -87,6 +89,7 @@ class GenericSubstitutionsDeducing { explicit GenericSubstitutionsDeducing(StructPtr struct_ref); TypePtr replace_Ts_with_currently_deduced(TypePtr orig) const; + TypePtr auto_deduce_from_argument(TypePtr param_type, TypePtr arg_type); TypePtr auto_deduce_from_argument(FunctionPtr cur_f, SrcLocation loc, TypePtr param_type, TypePtr arg_type); std::string_view get_first_not_deduced_nameT() const; void fire_error_can_not_deduce(FunctionPtr cur_f, SrcLocation loc, std::string_view nameT) const; @@ -96,19 +99,13 @@ class GenericSubstitutionsDeducing { } }; -struct GenericDeduceError final : std::exception { - std::string nameT; - - explicit GenericDeduceError(std::string_view nameT) - : nameT(nameT) { } - - const char* what() const noexcept override { - return nameT.c_str(); - } -}; +typedef std::pair MethodCallCandidate; FunctionPtr instantiate_generic_function(FunctionPtr fun_ref, GenericsSubstitutions&& substitutedTs); StructPtr instantiate_generic_struct(StructPtr struct_ref, GenericsSubstitutions&& substitutedTs); AliasDefPtr instantiate_generic_alias(AliasDefPtr alias_ref, GenericsSubstitutions&& substitutedTs); +FunctionPtr match_exact_method_for_call_not_generic(TypePtr called_receiver, std::string_view called_name); +std::vector match_methods_for_call_including_generic(TypePtr called_receiver, std::string_view called_name); + } // namespace tolk diff --git a/tolk/lexer.cpp b/tolk/lexer.cpp index d34a86399..c31f48762 100644 --- a/tolk/lexer.cpp +++ b/tolk/lexer.cpp @@ -344,7 +344,6 @@ struct ChunkIdentifierOrKeyword final : ChunkLexerBase { if (str == "var") return tok_var; if (str == "fun") return tok_fun; if (str == "asm") return tok_asm; - if (str == "get") return tok_get; if (str == "try") return tok_try; if (str == "val") return tok_val; break; diff --git a/tolk/lexer.h b/tolk/lexer.h index 55bdf4edb..19ccb5c10 100644 --- a/tolk/lexer.h +++ b/tolk/lexer.h @@ -26,7 +26,6 @@ enum TokenType { tok_empty, tok_fun, - tok_get, tok_type, tok_enum, tok_struct, diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 53447277b..0f86b2ad1 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -205,7 +205,7 @@ class LValContext { code.emplace_back(loc, Op::_IntConst, index_ir_idx, td::make_refint(index_at)); vars_modification_watcher.trigger_callbacks(tuple_ir_idx, loc); - FunctionPtr builtin_sym = lookup_global_symbol("tupleSetAt")->try_as(); + FunctionPtr builtin_sym = lookup_global_symbol("tuple.set")->try_as(); code.emplace_back(loc, Op::_Call, std::vector{tuple_ir_idx}, std::vector{tuple_ir_idx[0], lval_ir_idx[0], index_ir_idx[0]}, builtin_sym); local_lval.after_let(std::move(tuple_ir_idx), code, loc); } @@ -1167,13 +1167,13 @@ static std::vector process_dot_access(V v, CodeBlob& lval_ctx->capture_tuple_index_modification(v->get_obj(), index_at, lval_ir_idx); return lval_ir_idx; } - // `tupleVar.0` as rvalue: the same as "tupleAt(tupleVar, 0)" written in terms of IR vars + // `tupleVar.0` as rvalue: the same as "tuple.get(tupleVar, 0)" written in terms of IR vars std::vector tuple_ir_idx = pre_compile_expr(v->get_obj(), code); std::vector index_ir_idx = code.create_tmp_var(TypeDataInt::create(), v->get_identifier()->loc, "(tuple-idx)"); code.emplace_back(v->loc, Op::_IntConst, index_ir_idx, td::make_refint(index_at)); std::vector field_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(tuple-field)"); tolk_assert(tuple_ir_idx.size() == 1 && field_ir_idx.size() == 1); // tuples contain only 1-slot values - FunctionPtr builtin_sym = lookup_global_symbol("tupleAt")->try_as(); + FunctionPtr builtin_sym = lookup_global_symbol("tuple.get")->try_as(); code.emplace_back(v->loc, Op::_Call, field_ir_idx, std::vector{tuple_ir_idx[0], index_ir_idx[0]}, builtin_sym); if (lval_ctx && calc_sink_leftmost_obj(v)) { // `tupleVar.0.1 = rhs`, then `tupleVar.0` is rval inside lval lval_ctx->capture_tuple_index_modification(v->get_obj(), index_at, field_ir_idx); @@ -1221,15 +1221,14 @@ static std::vector process_function_call(V v, Code return transition_to_target_type(std::move(rvect), code, target_type, v); } - int delta_self = v->is_dot_call(); - AnyExprV obj_leftmost = nullptr; + AnyExprV obj_leftmost = v->get_self_obj(); + int delta_self = obj_leftmost != nullptr; std::vector args; args.reserve(delta_self + v->get_num_args()); if (delta_self) { - args.push_back(v->get_dot_obj()); - obj_leftmost = v->get_dot_obj(); - while (obj_leftmost->kind == ast_function_call && obj_leftmost->as()->is_dot_call() && obj_leftmost->as()->fun_maybe && obj_leftmost->as()->fun_maybe->does_return_self()) { - obj_leftmost = obj_leftmost->as()->get_dot_obj(); + args.push_back(obj_leftmost); + while (obj_leftmost->kind == ast_function_call && obj_leftmost->as()->get_self_obj() && obj_leftmost->as()->fun_maybe && obj_leftmost->as()->fun_maybe->does_return_self()) { + obj_leftmost = obj_leftmost->as()->get_self_obj(); } } for (int i = 0; i < v->get_num_args(); ++i) { @@ -1243,7 +1242,7 @@ static std::vector process_function_call(V v, Code TypePtr op_call_type = v->inferred_type; TypePtr real_ret_type = v->inferred_type; - if (delta_self && fun_ref->does_return_self()) { + if (obj_leftmost && fun_ref->does_return_self()) { real_ret_type = TypeDataVoid::create(); if (!fun_ref->parameters[0].is_mutate_parameter()) { op_call_type = TypeDataVoid::create(); diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 1df682140..a863fc592 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -98,7 +98,7 @@ static void check_function_argument_mutate_back(FunctionPtr cur_f, TypePtr param GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_assign_always_null_to_variable(FunctionPtr cur_f, SrcLocation loc, LocalVarPtr assigned_var, bool is_assigned_null_literal) { std::string var_name = assigned_var->name; - fire(cur_f, loc, "can not infer type of `" + var_name + "`, it's always null; specify its type with `" + var_name + ": `" + (is_assigned_null_literal ? " or use `null as `" : "")); + fire(cur_f, loc, "can not infer type of `" + var_name + "`, it's always null\nspecify its type with `" + var_name + ": `" + (is_assigned_null_literal ? " or use `null as `" : "")); } // fire an error on `untypedTupleVar.0` when inferred as (int,int), or `[int, (int,int)]`, or other non-1 width in a tuple @@ -334,20 +334,16 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { return; } - // so, we have a call `f(args)` or `obj.f(args)`, f is a global function (fun_ref) (code / asm / builtin) - int delta_self = 0; - AnyExprV dot_obj = nullptr; - if (auto v_dot = v->get_callee()->try_as()) { - delta_self = 1; - dot_obj = v_dot->get_obj(); - } + // so, we have a call `f(args)` or `obj.f(args)`, fun_ref is a function/method (code / asm / builtin) + AnyExprV self_obj = v->get_self_obj(); + int delta_self = self_obj != nullptr; - if (dot_obj) { + if (self_obj) { const LocalVarData& param_0 = fun_ref->parameters[0]; TypePtr param_type = param_0.declared_type; - check_function_argument_passed(cur_f, param_type, dot_obj, true); + check_function_argument_passed(cur_f, param_type, self_obj, true); if (param_0.is_mutate_parameter()) { - check_function_argument_mutate_back(cur_f, param_type, dot_obj, true); + check_function_argument_mutate_back(cur_f, param_type, self_obj, true); } } for (int i = 0; i < v->get_num_args(); ++i) { @@ -483,8 +479,8 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { return true; } // `return self.someMethod()` - if (auto v_call = return_expr->try_as(); v_call && v_call->is_dot_call()) { - return v_call->fun_maybe && v_call->fun_maybe->does_return_self() && is_expr_valid_as_return_self(v_call->get_dot_obj()); + if (auto v_call = return_expr->try_as(); v_call && v_call->get_self_obj()) { + return v_call->fun_maybe && v_call->fun_maybe->does_return_self() && is_expr_valid_as_return_self(v_call->get_self_obj()); } // `return cond ? ... : ...` if (auto v_ternary = return_expr->try_as()) { diff --git a/tolk/pipe-check-pure-impure.cpp b/tolk/pipe-check-pure-impure.cpp index 366ff1607..1a9143875 100644 --- a/tolk/pipe-check-pure-impure.cpp +++ b/tolk/pipe-check-pure-impure.cpp @@ -27,15 +27,17 @@ namespace tolk { GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_impure_operation_inside_pure_function(AnyV v) { - v->error("an impure operation in a pure function"); +static void fire_error_impure_operation_inside_pure_function(FunctionPtr cur_f, SrcLocation loc) { + fire(cur_f, loc, "an impure operation in a pure function"); } class CheckImpureOperationsInPureFunctionVisitor final : public ASTVisitorFunctionBody { - static void fire_if_global_var(AnyExprV v) { + FunctionPtr cur_f = nullptr; + + void fire_if_global_var(AnyExprV v) const { if (auto v_ident = v->try_as()) { if (v_ident->sym->try_as()) { - fire_error_impure_operation_inside_pure_function(v); + fire_error_impure_operation_inside_pure_function(cur_f, v->loc); } } } @@ -54,11 +56,11 @@ class CheckImpureOperationsInPureFunctionVisitor final : public ASTVisitorFuncti // v is `globalF(args)` / `globalF(args)` / `obj.method(args)` / `local_var(args)` / `getF()(args)` if (!v->fun_maybe) { // `local_var(args)` is always impure, no considerations about what's there at runtime - fire_error_impure_operation_inside_pure_function(v); + fire_error_impure_operation_inside_pure_function(cur_f, v->loc); } if (!v->fun_maybe->is_marked_as_pure()) { - fire_error_impure_operation_inside_pure_function(v); + fire_error_impure_operation_inside_pure_function(cur_f, v->loc); } parent::visit(v); @@ -73,17 +75,22 @@ class CheckImpureOperationsInPureFunctionVisitor final : public ASTVisitorFuncti } void visit(V v) override { - fire_error_impure_operation_inside_pure_function(v); + fire_error_impure_operation_inside_pure_function(cur_f, v->loc); } void visit(V v) override { - fire_error_impure_operation_inside_pure_function(v); + fire_error_impure_operation_inside_pure_function(cur_f, v->loc); } public: bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function() && fun_ref->is_marked_as_pure(); } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + parent::visit(v_function->get_body()); + } }; void pipeline_check_pure_impure_operations() { diff --git a/tolk/pipe-check-rvalue-lvalue.cpp b/tolk/pipe-check-rvalue-lvalue.cpp index aa274a436..be4db2749 100644 --- a/tolk/pipe-check-rvalue-lvalue.cpp +++ b/tolk/pipe-check-rvalue-lvalue.cpp @@ -183,8 +183,8 @@ class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { } // for `f()` don't visit ast_reference `f`, to detect `f` usage as non-call, like `var cb = f` // same for `obj.method()`, don't visit ast_reference method, visit only obj - if (v->is_dot_call()) { - parent::visit(v->get_dot_obj()); + if (AnyExprV self_obj = v->get_self_obj()) { + parent::visit(self_obj); } for (int i = 0; i < v->get_num_args(); ++i) { diff --git a/tolk/pipe-find-unused-symbols.cpp b/tolk/pipe-find-unused-symbols.cpp index 2b7e55578..cce03cc90 100644 --- a/tolk/pipe-find-unused-symbols.cpp +++ b/tolk/pipe-find-unused-symbols.cpp @@ -67,7 +67,7 @@ static void mark_function_used_dfs(const std::unique_ptr& op) { void pipeline_find_unused_symbols() { for (FunctionPtr fun_ref : G.all_functions) { - if (fun_ref->is_method_id_not_empty()) { // get methods, main and other entrypoints, regular functions with @method_id + if (fun_ref->has_tvm_method_id()) { // get methods, main and other entrypoints, regular functions with @method_id mark_function_used(fun_ref); } } diff --git a/tolk/pipe-generate-fif-output.cpp b/tolk/pipe-generate-fif-output.cpp index b40ce6860..6cfd50683 100644 --- a/tolk/pipe-generate-fif-output.cpp +++ b/tolk/pipe-generate-fif-output.cpp @@ -146,8 +146,8 @@ void pipeline_generate_fif_output_to_std_cout() { } std::cout << " "; - if (fun_ref->is_method_id_not_empty()) { - std::cout << fun_ref->method_id << " DECLMETHOD " << fun_ref->name << "\n"; + if (fun_ref->has_tvm_method_id()) { + std::cout << fun_ref->tvm_method_id << " DECLMETHOD " << fun_ref->name << "\n"; } else { std::cout << "DECLPROC " << fun_ref->name << "\n"; } diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 06e658ac9..2f324f561 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -147,7 +147,7 @@ static void fire_error_calling_asm_function_with_non1_stack_width_arg(FunctionPt GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_cannot_deduce_untyped_tuple_access(FunctionPtr cur_f, SrcLocation loc, int index) { std::string idx_access = "." + std::to_string(index); - fire(cur_f, loc, "can not deduce type of `" + idx_access + "`; either assign it to variable like `var c: int = " + idx_access + "` or cast the result like `" + idx_access + " as int`"); + fire(cur_f, loc, "can not deduce type of `" + idx_access + "`\neither assign it to variable like `var c: int = " + idx_access + "` or cast the result like `" + idx_access + " as int`"); } // helper function: given hint = `Ok | Err` and struct `Ok`, return `Ok` @@ -196,6 +196,39 @@ static TypePtr try_pick_instantiated_generic_from_hint(TypePtr hint, AliasDefPtr return nullptr; } +// given `p.create` (called_receiver = Point, called_name = "create") +// look up a corresponding method (it may be `Point.create` / `Point?.create` / `T.create`) +static MethodCallCandidate choose_only_method_to_call(FunctionPtr cur_f, SrcLocation loc, TypePtr called_receiver, std::string_view called_name) { + // most practical cases: `builder.storeInt` etc., when a direct method for receiver exists + if (FunctionPtr exact_method = match_exact_method_for_call_not_generic(called_receiver, called_name)) { + return {exact_method, GenericsSubstitutions(exact_method->genericTs)}; + } + + // else, try to match, for example `10.copy` with `int?.copy`, with `T.copy`, etc. + std::vector candidates = match_methods_for_call_including_generic(called_receiver, called_name); + if (candidates.size() == 1) { + return candidates[0]; + } + if (candidates.empty()) { // return nullptr, the caller side decides how to react on this + return {nullptr, GenericsSubstitutions(nullptr)}; + } + + std::ostringstream msg; + msg << "call to method `" << called_name << "` for type `" << called_receiver << "` is ambiguous\n"; + for (const auto& [method_ref, substitutedTs] : candidates) { + msg << "candidate function: `" << method_ref->as_human_readable() << "`"; + if (method_ref->is_generic_function()) { + msg << " with " << substitutedTs.as_human_readable(false); + } + if (method_ref->loc.is_defined()) { + msg << " (declared at " << method_ref->loc << ")\n"; + } else if (method_ref->is_builtin_function()) { + msg << " (builtin)\n"; + } + } + fire(cur_f, loc, msg.str()); +} + /* * This class handles all types of AST vertices and traverses them, filling all AnyExprV::inferred_type. * Note, that it isn't derived from ASTVisitor, it has manual `switch` over all existing vertex types. @@ -656,7 +689,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // `... ? intVar : sliceVar` results in `int | slice`, probably it's not what the user expected // example: `var v = ternary`, show an inference error // do NOT show an error for `var v: T = ternary` (T is hint); it will be checked by type checker later - if (hint == nullptr || hint == TypeDataUnknown::create()) { + if (hint == nullptr || hint == TypeDataUnknown::create() || hint->has_genericT_inside()) { fire(cur_f, v->loc, "types of ternary branches are incompatible: " + to_string(v->get_when_true()) + " and " + to_string(v->get_when_false())); } } @@ -762,11 +795,54 @@ class InferTypesAndCallsAndFieldsVisitor final { return ExprFlow(std::move(flow), used_as_condition); } - ExprFlow infer_reference(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) const { + // given `genericF` / `t.tupleFirst` (the user manually specified instantiation Ts), + // validate and collect them + // returns: [int, slice] / [cell] + std::vector collect_type_arguments_for_fun(SrcLocation loc, const GenericsDeclaration* genericTs, V instantiationT_list) const { + // for `genericF` user should provide two Ts + // for `Container.wrap` — one U (and one T is implicitly from receiver) + if (instantiationT_list->size() != genericTs->size() - genericTs->n_from_receiver) { + fire(cur_f, loc, "expected " + std::to_string(genericTs->size() - genericTs->n_from_receiver) + " type arguments, got " + std::to_string(instantiationT_list->size())); + } + + std::vector type_arguments; + type_arguments.reserve(instantiationT_list->size()); + for (int i = 0; i < instantiationT_list->size(); ++i) { + type_arguments.push_back(instantiationT_list->get_item(i)->type_node->resolved_type); + } + + return type_arguments; + } + + // when substitutedTs have been collected from `f` or deduced from arguments, instantiate a generic function `f` + // example: was `t.push(2)`, deduced , instantiate `tuple.push` + // example: was `t.push(2)`, collected , instantiate `tuple.push` (will later fail type check) + // example: was `var cb = t.first;` (used as reference, as non-call), instantiate `tuple.first` + // returns fun_ref to instantiated function + FunctionPtr check_and_instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, GenericsSubstitutions&& substitutedTs) const { + // T for asm function must be a TVM primitive (width 1), otherwise, asm would act incorrectly + if (fun_ref->is_asm_function() || fun_ref->is_builtin_function()) { + for (int i = 0; i < substitutedTs.size(); ++i) { + if (substitutedTs.typeT_at(i)->get_width_on_stack() != 1) { + fire_error_calling_asm_function_with_non1_stack_width_arg(cur_f, loc, fun_ref, substitutedTs, i); + } + } + } + + // make deep clone of `f` with substitutedTs (or immediately return from symtable if already instantiated) + return instantiate_generic_function(fun_ref, std::move(substitutedTs)); + } + + ExprFlow infer_reference(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint, FunctionPtr* out_f_called = nullptr) const { + // at current point, v is a reference: + // - either a standalone: `local_var` / `SOME_CONST` / `globalF` / `genericFn` + // - or inside a call: `globalF()` / `genericFn()` / `genericFn()` / `local_var()` + if (LocalVarPtr var_ref = v->sym->try_as()) { TypePtr declared_or_smart_casted = flow.smart_cast_if_exists(SinkExpression(var_ref)); tolk_assert(declared_or_smart_casted != nullptr); // all local vars are presented in flow assign_inferred_type(v, declared_or_smart_casted); + // it might be `local_var()` also, don't fill out_f_called, we have no fun_ref, it's a call of arbitrary expression } else if (GlobalConstPtr const_ref = v->sym->try_as()) { if (!const_ref->inferred_type) { @@ -779,98 +855,83 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, glob_ref->declared_type); } else if (FunctionPtr fun_ref = v->sym->try_as()) { - // it's `globalF` / `globalF` - references to functions used as non-call + // it's `globalF` / `globalF` / `globalF()` / `globalF()` + // if it's a call, then out_f_called is present, we should fill it V v_instantiationTs = v->get_instantiationTs(); - if (fun_ref->is_generic_function() && !v_instantiationTs) { + if (fun_ref->is_generic_function() && !v_instantiationTs && !out_f_called) { // `genericFn` is invalid as non-call, can't be used without fire(cur_f, v->loc, "can not use a generic function " + to_string(fun_ref) + " as non-call"); - } else if (fun_ref->is_generic_function()) { - // `genericFn` is valid, it's a reference to instantiation - GenericsSubstitutions substitutedTs = collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref->genericTs, v_instantiationTs); + } else if (fun_ref->is_generic_function() && v_instantiationTs) { + // `genericFn` is valid as non-call, it's a reference to instantiation + // `genericFn()` is also ok, we'll assign an instantiated fun_ref to out_f_called + GenericsSubstitutions substitutedTs(fun_ref->genericTs); + substitutedTs.provide_type_arguments(collect_type_arguments_for_fun(v->loc, fun_ref->genericTs, v_instantiationTs)); fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutedTs)); v->mutate()->assign_sym(fun_ref); - } else if (v_instantiationTs != nullptr && !fun_ref->is_instantiation_of_generic_function()) { + } else if (UNLIKELY(v_instantiationTs != nullptr)) { // non-generic function referenced like `return beginCell;` - fire(cur_f, v_instantiationTs->loc, "not generic function used with generic T"); + fire(cur_f, v_instantiationTs->loc, "type arguments not expected here"); } - fun_ref->mutate()->assign_is_used_as_noncall(); - get_or_infer_return_type(fun_ref); - assign_inferred_type(v, fun_ref->inferred_full_type); + if (out_f_called) { // so, it's `globalF()` / `genericFn()` / `genericFn()` + *out_f_called = fun_ref; // (it's still may be a generic one, then Ts will be deduced from arguments) + } else { // so, it's `globalF` / `genericFn` as a reference + if (fun_ref->is_compile_time_only()) { + fire(cur_f, v->loc, "can not get reference to this function, it's compile-time only"); + } + fun_ref->mutate()->assign_is_used_as_noncall(); + get_or_infer_return_type(fun_ref); + assign_inferred_type(v, fun_ref->inferred_full_type); + } return ExprFlow(std::move(flow), used_as_condition); - } else if (AliasDefPtr alias_ref = v->sym->try_as()) { - fire(cur_f, v->loc, "type " + to_string(alias_ref) + " can not be used as a value"); - - } else if (StructPtr struct_ref = v->sym->try_as()) { - fire(cur_f, v->loc, "struct " + to_string(struct_ref) + " can not be used as a value"); - } else { tolk_assert(false); } // for non-functions: `local_var` and similar not allowed if (UNLIKELY(v->has_instantiationTs())) { - fire(cur_f, v->get_instantiationTs()->loc, "generic T not expected here"); + fire(cur_f, v->get_instantiationTs()->loc, "type arguments not expected here"); } return ExprFlow(std::move(flow), used_as_condition); } - // given `genericF` / `t.tupleFirst` (the user manually specified instantiation Ts), - // validate and collect them - // returns: [T1=int, T2=slice] / [T=cell] (with flag auto_deduced = false, meaning they are manually specified) - GenericsSubstitutions collect_fun_generic_substitutions_from_manually_specified(SrcLocation loc, const GenericsDeclaration* genericTs, V instantiationT_list) const { - if (instantiationT_list->size() != genericTs->size()) { - fire(cur_f, loc, "wrong count of generic T: expected " + std::to_string(genericTs->size()) + ", got " + std::to_string(instantiationT_list->size())); - } - - std::vector type_arguments; - type_arguments.reserve(instantiationT_list->size()); - for (int i = 0; i < instantiationT_list->size(); ++i) { - type_arguments.push_back(instantiationT_list->get_item(i)->type_node->resolved_type); - } + ExprFlow infer_dot_access(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint, FunctionPtr* out_f_called = nullptr, AnyExprV* out_dot_obj = nullptr) { + // at current point, v is a dot access to a field / index / method: + // - either a standalone: `user.id` / `getUser().id` / `var.0` / `t.size` / `Point.create` / `t.tupleAt` + // - or inside a call: `user.getId()` / `.method()` / `Point.create()` / `t.tupleAt(1)` - return GenericsSubstitutions(genericTs, type_arguments); - } + AnyExprV dot_obj = nullptr; // to be filled for `.(field/index/method)`, nullptr for `Point.create` + FunctionPtr fun_ref = nullptr; // to be filled for `.method` / `Point.create` (both standalone or in a call) + GenericsSubstitutions substitutedTs(nullptr); + V v_ident = v->get_identifier(); // field/method name vertex - // when generic Ts have been collected from user-specified or deduced from arguments, - // instantiate a generic function - // example: was `t.tuplePush(2)`, deduced , instantiate `tuplePush` - // example: was `t.tuplePush(2)`, read , instantiate `tuplePush` (will later fail type check) - // example: was `var cb = t.tupleFirst;` (used as reference, as non-call), instantiate `tupleFirst` - // returns fun_ref to instantiated function - FunctionPtr check_and_instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, GenericsSubstitutions&& substitutedTs) const { - // T for asm function must be a TVM primitive (width 1), otherwise, asm would act incorrectly - if (fun_ref->is_asm_function() || fun_ref->is_builtin_function()) { - for (int i = 0; i < substitutedTs.size(); ++i) { - if (substitutedTs.typeT_at(i)->get_width_on_stack() != 1) { - fire_error_calling_asm_function_with_non1_stack_width_arg(cur_f, loc, fun_ref, substitutedTs, i); + // handle `Point.create` / `Container.wrap`: no dot_obj expression actually, lhs is a type, looking up a method + if (auto obj_ref = v->get_obj()->try_as()) { + if (const auto* obj_as_type = obj_ref->sym->try_as()) { + TypePtr receiver_type = obj_as_type->resolved_type; + std::tie(fun_ref, substitutedTs) = choose_only_method_to_call(cur_f, v_ident->loc, receiver_type, v->get_field_name()); + if (!fun_ref) { + fire(cur_f, v_ident->loc, "method `" + to_string(v->get_field_name()) + "` not found for type " + to_string(receiver_type)); } } } + // handle other (most, actually) cases: `.field` / `.method` + if (!fun_ref) { + dot_obj = v->get_obj(); + flow = infer_any_expr(dot_obj, std::move(flow), false).out_flow; + } - // make deep clone of `f` with substitutedTs - // (if `f` was already instantiated, it will be immediately returned from a symbol table) - return instantiate_generic_function(fun_ref, std::move(substitutedTs)); - } - - ExprFlow infer_dot_access(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { - // it's NOT a method call `t.tupleSize()` (since such cases are handled by infer_function_call) - // it's `t.0`, `getUser().id`, and `t.tupleSize` (as a reference, not as a call) - flow = infer_any_expr(v->get_obj(), std::move(flow), false).out_flow; - - TypePtr obj_type = v->get_obj()->inferred_type; - TypePtr unwrapped_obj_type = obj_type->unwrap_alias(); - // our goal is to fill v->target knowing type of obj - V v_ident = v->get_identifier(); // field/method name vertex + // our goal is to fill v->target (field/index/method) knowing type of obj + TypePtr obj_type = dot_obj ? dot_obj->inferred_type->unwrap_alias() : TypeDataUnknown::create(); V v_instantiationTs = v->get_instantiationTs(); std::string_view field_name = v_ident->name; // check for field access (`user.id`), when obj is a struct - if (const TypeDataStruct* obj_struct = unwrapped_obj_type->try_as()) { + if (const TypeDataStruct* obj_struct = obj_type->try_as()) { if (StructFieldPtr field_ref = obj_struct->struct_ref->find_field(field_name)) { v->mutate()->assign_target(field_ref); TypePtr inferred_type = field_ref->declared_type; @@ -882,13 +943,13 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, inferred_type); return ExprFlow(std::move(flow), used_as_condition); } - // if field_name doesn't exist, don't fire an error now — maybe, it's `user.globalFunction()` + // if field_name doesn't exist, don't fire an error now — maybe, it's `user.method()` } // check for indexed access (`tensorVar.0` / `tupleVar.1`) - if (field_name[0] >= '0' && field_name[0] <= '9') { + if (!fun_ref && field_name[0] >= '0' && field_name[0] <= '9') { int index_at = std::stoi(std::string(field_name)); - if (const auto* t_tensor = unwrapped_obj_type->try_as()) { + if (const auto* t_tensor = obj_type->try_as()) { if (index_at >= t_tensor->size()) { fire(cur_f, v_ident->loc, "invalid tensor index, expected 0.." + std::to_string(t_tensor->items.size() - 1)); } @@ -902,7 +963,7 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, inferred_type); return ExprFlow(std::move(flow), used_as_condition); } - if (const auto* t_tuple = unwrapped_obj_type->try_as()) { + if (const auto* t_tuple = obj_type->try_as()) { if (index_at >= t_tuple->size()) { fire(cur_f, v_ident->loc, "invalid tuple index, expected 0.." + std::to_string(t_tuple->items.size() - 1)); } @@ -916,12 +977,12 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, inferred_type); return ExprFlow(std::move(flow), used_as_condition); } - if (unwrapped_obj_type->try_as()) { + if (obj_type->try_as()) { TypePtr item_type = nullptr; if (v->is_lvalue && !hint) { // left side of assignment item_type = TypeDataUnknown::create(); } else { - if (hint == nullptr) { + if (hint == nullptr || hint == TypeDataUnknown::create() || hint->has_genericT_inside()) { fire_error_cannot_deduce_untyped_tuple_access(cur_f, v->loc, index_at); } item_type = hint; @@ -930,53 +991,62 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, item_type); return ExprFlow(std::move(flow), used_as_condition); } - // numeric field not found, fire an error - if (unwrapped_obj_type->try_as()) { // `user.100500` - fire(cur_f, v_ident->loc, "field `" + static_cast(field_name) + "` doesn't exist in type " + to_string(obj_type)); - } else { - fire(cur_f, v_ident->loc, "type " + to_string(obj_type) + " is not indexable"); - } } - // check for method (`t.tupleSize` / `user.globalFunction`) - // for now, Tolk doesn't have object-scoped methods; `t.tupleSize` is a global function `tupleSize` - const Symbol* sym = lookup_global_symbol(field_name); - FunctionPtr fun_ref = sym ? sym->try_as() : nullptr; + // check for method (`t.size` / `user.getId`); even `i.0()` can be here if `fun int.0(self)` exists + // for `T.copy` / `Container.create`, substitution for T is also returned + if (!fun_ref) { + std::tie(fun_ref, substitutedTs) = choose_only_method_to_call(cur_f, dot_obj->loc, obj_type, field_name); + } // not a field, not a method — fire an error if (!fun_ref) { // as a special case, handle accessing fields of nullable objects, to show a more precise error - if (const TypeDataUnion* as_union = unwrapped_obj_type->try_as(); as_union && as_union->or_null) { + if (const TypeDataUnion* as_union = obj_type->try_as(); as_union && as_union->or_null) { if (const TypeDataStruct* n_struct = as_union->or_null->try_as(); n_struct && n_struct->struct_ref->find_field(field_name)) { - fire(cur_f, v_ident->loc, "can not access field `" + static_cast(field_name) + "` of a possibly nullable object " + to_string(obj_type) + "; check it via `obj != null` or use non-null assertion `obj!` operator"); + fire(cur_f, v_ident->loc, "can not access field `" + static_cast(field_name) + "` of a possibly nullable object " + to_string(dot_obj) + "\ncheck it via `obj != null` or use non-null assertion `obj!` operator"); } } - fire(cur_f, v_ident->loc, "field `" + static_cast(field_name) + "` doesn't exist in type " + to_string(obj_type)); + if (out_f_called) { + fire(cur_f, v_ident->loc, "method `" + to_string(v->get_field_name()) + "` not found for type " + to_string(obj_type)); + } else { + fire(cur_f, v_ident->loc, "field `" + static_cast(field_name) + "` doesn't exist in type " + to_string(dot_obj)); + } } - // `t.tupleSize` is ok, `cs.tupleSize` not - if (!fun_ref->parameters[0].declared_type->can_rhs_be_assigned(obj_type)) { - fire(cur_f, v_ident->loc, "referencing a method for " + to_string(fun_ref->parameters[0].declared_type) + " with object of type " + to_string(obj_type)); + // if `fun T.copy(self)` and reference `int.copy` — all generic parameters are determined by the receiver, we know it + if (fun_ref->is_generic_function() && fun_ref->genericTs->size() == fun_ref->genericTs->n_from_receiver) { + tolk_assert(substitutedTs.typeT_at(0) != nullptr); + fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutedTs)); } - if (fun_ref->is_generic_function() && !v_instantiationTs) { - // `genericFn` and `t.tupleAt` are invalid as non-call, they can't be used without + if (fun_ref->is_generic_function() && !v_instantiationTs && !out_f_called) { + // `t.tupleAt` is invalid as non-call, can't be used without fire(cur_f, v->loc, "can not use a generic function " + to_string(fun_ref) + " as non-call"); - } else if (fun_ref->is_generic_function()) { - // `t.tupleAt` is valid, it's a reference to instantiation - GenericsSubstitutions substitutedTs = collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref->genericTs, v_instantiationTs); + } else if (fun_ref->is_generic_function() && v_instantiationTs) { + // `t.tupleAt` is valid as non-call, it's a reference to instantiation + // `t.tupleAt()` is also ok, we'll assign an instantiated fun_ref to out_f_called + substitutedTs.provide_type_arguments(collect_type_arguments_for_fun(v->loc, fun_ref->genericTs, v_instantiationTs)); fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutedTs)); } else if (UNLIKELY(v_instantiationTs != nullptr)) { - // non-generic method referenced like `var cb = c.cellHash;` - fire(cur_f, v_instantiationTs->loc, "not generic function used with generic T"); + // non-generic method referenced like `var cb = c.hash;` + fire(cur_f, v_instantiationTs->loc, "type arguments not expected here"); } - fun_ref->mutate()->assign_is_used_as_noncall(); - v->mutate()->assign_target(fun_ref); - get_or_infer_return_type(fun_ref); - assign_inferred_type(v, fun_ref->inferred_full_type); // type of `t.tupleSize` is TypeDataFunCallable + if (out_f_called) { // so, it's `user.method()` / `t.tupleAt()` / `t.tupleAt()` / `Point.create` + *out_f_called = fun_ref; // (it's still may be a generic one, then Ts will be deduced from arguments) + *out_dot_obj = dot_obj; + } else { // so, it's `user.method` / `t.tupleAt` as a reference + if (fun_ref->is_compile_time_only()) { + fire(cur_f, v->get_identifier()->loc, "can not get reference to this method, it's compile-time only"); + } + fun_ref->mutate()->assign_is_used_as_noncall(); + v->mutate()->assign_target(fun_ref); + get_or_infer_return_type(fun_ref); + assign_inferred_type(v, fun_ref->inferred_full_type); + } return ExprFlow(std::move(flow), used_as_condition); } @@ -984,50 +1054,20 @@ class InferTypesAndCallsAndFieldsVisitor final { AnyExprV callee = v->get_callee(); // v is `globalF(args)` / `globalF(args)` / `obj.method(args)` / `local_var(args)` / `getF()(args)` - int delta_self = 0; - AnyExprV dot_obj = nullptr; + AnyExprV self_obj = nullptr; // for `obj.method()`, obj will be here (but for `Point.create()`, no obj exists) FunctionPtr fun_ref = nullptr; - V v_instantiationTs = nullptr; - - if (auto v_ref = callee->try_as()) { - // `globalF()` / `globalF()` / `local_var()` / `SOME_CONST()` - fun_ref = v_ref->sym->try_as(); // not null for `globalF` - v_instantiationTs = v_ref->get_instantiationTs(); // present for `globalF()` - - } else if (auto v_dot = callee->try_as()) { - // `obj.someMethod()` / `obj.someMethod()` / `getF().someMethod()` / `obj.SOME_CONST()` - // note, that dot_obj->target is not filled yet, since callee was not inferred yet - delta_self = 1; - dot_obj = v_dot->get_obj(); - v_instantiationTs = v_dot->get_instantiationTs(); // present for `obj.someMethod()` - flow = infer_any_expr(dot_obj, std::move(flow), false).out_flow; - - // it can be indexed access (`tensorVar.0()`, `tupleVar.1()`) or a method (`t.tupleSize()`) - std::string_view field_name = v_dot->get_field_name(); - if (const TypeDataStruct* t_struct = dot_obj->inferred_type->unwrap_alias()->try_as(); t_struct && t_struct->struct_ref->find_field(field_name)) { - // `t.someField`, it's a field, probably a callable, fun_ref remains nullptr - } else if (field_name[0] >= '0' && field_name[0] <= '9') { - // indexed access `ab.2()`, then treat `ab.2` just like an expression, fun_ref remains nullptr - // infer_dot_access() will be called for a callee, it will check index correctness - } else { - // for now, Tolk doesn't have fields and object-scoped methods; `t.tupleSize` is a global function `tupleSize` - const Symbol* sym = lookup_global_symbol(field_name); - fun_ref = sym ? sym->try_as() : nullptr; - if (!fun_ref) { - fire(cur_f, v_dot->get_identifier()->loc, "non-existing method `" + to_string(field_name) + "` of type " + to_string(dot_obj)); - } - } + if (callee->kind == ast_reference) { + flow = infer_reference(callee->as(), std::move(flow), false, nullptr, &fun_ref).out_flow; + } else if (callee->kind == ast_dot_access) { + flow = infer_dot_access(callee->as(), std::move(flow), false, nullptr, &fun_ref, &self_obj).out_flow; } else { - // `getF()()` / `5()` - // fun_ref remains nullptr + flow = infer_any_expr(callee, std::move(flow), false).out_flow; } // handle `local_var()` / `getF()()` / `5()` / `SOME_CONST()` / `obj.method()()()` / `tensorVar.0()` if (!fun_ref) { - // treat callee like a usual expression - flow = infer_any_expr(callee, std::move(flow), false).out_flow; - // it must have "callable" inferred type + // callee must have "callable" inferred type const TypeDataFunCallable* f_callable = callee->inferred_type->unwrap_alias()->try_as(); if (!f_callable) { // `5()` / `SOME_CONST()` / `null()` fire(cur_f, v->loc, "calling a non-function " + to_string(callee->inferred_type)); @@ -1041,28 +1081,18 @@ class InferTypesAndCallsAndFieldsVisitor final { flow = infer_any_expr(arg_i, std::move(flow), false, f_callable->params_types[i]).out_flow; assign_inferred_type(v->get_arg(i), arg_i); } - v->mutate()->assign_fun_ref(nullptr); // no fun_ref to a global function assign_inferred_type(v, f_callable->return_type); return ExprFlow(std::move(flow), used_as_condition); } - // for a call `f(...)`, create and instantiate function "f" right now - // so that further, fun_ref will be not a generic, but an instantiated function - if (fun_ref->is_generic_function() && v_instantiationTs) { - GenericsSubstitutions substitutedTs = collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref->genericTs, v_instantiationTs); - fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutedTs)); - } else if (UNLIKELY(v_instantiationTs != nullptr)) { - // `c.cellHash()` / `beginCell()` - fire(cur_f, v_instantiationTs->loc, "calling a not generic function with generic T"); - } - - // so, we have a call `f(args)` or `obj.f(args)`, f is a global function (fun_ref) (code / asm / builtin) + // so, we have a call `f(args)` or `obj.f(args)`, f is fun_ref (function / method) (code / asm / builtin) // we're going to iterate over passed arguments, and (if generic) infer substitutedTs // at first, check arguments count (Tolk doesn't have optional parameters, so just compare counts) + int delta_self = self_obj != nullptr; int n_arguments = v->get_num_args() + delta_self; int n_parameters = fun_ref->get_num_params(); - if (!n_parameters && dot_obj) { - fire(cur_f, v->loc, to_string(fun_ref) + " has no parameters and can not be called as method"); + if (!fun_ref->does_accept_self() && self_obj) { // static method `Point.create(...)` called as `p.create()` + fire(cur_f, v->loc, "method " + to_string(fun_ref) + " can not be called via dot\n(it's a static method, it does not accept `self`)"); } if (n_parameters < n_arguments) { fire(cur_f, v->loc, "too many arguments in call to " + to_string(fun_ref) + ", expected " + std::to_string(n_parameters - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); @@ -1071,34 +1101,38 @@ class InferTypesAndCallsAndFieldsVisitor final { fire(cur_f, v->loc, "too few arguments in call to " + to_string(fun_ref) + ", expected " + std::to_string(n_parameters - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); } - // now, for every passed argument, we need to infer its type - // for regular functions, it's obvious - // but for generic functions, we need to infer type arguments (substitutedTs) on the fly - // (unless Ts are specified by a user like `f(args)` / `t.tupleAt()`, take them) + // for every passed argument, we need to infer its type + // for generic functions, we need to infer type arguments (substitutedTs) on the fly + // (if they are specified by a user like `f(args)` / `t.tupleAt()`, fun_ref is already instantiated) GenericSubstitutionsDeducing deducingTs(fun_ref); - // loop over every argument, for `obj.method()` obj is the first one - // if genericT deducing has a conflict, ParseError is thrown - // note, that deducing Ts one by one is important to manage control flow (mutate params work like assignments) - // a corner case, e.g. `f(v1:T?, v2:T?)` and `f(null,2)` will fail on first argument, won't try the second one - if (dot_obj) { + // for `obj.method()` obj is the first argument (passed to `self` parameter) + if (self_obj) { const LocalVarData& param_0 = fun_ref->parameters[0]; TypePtr param_type = param_0.declared_type; if (param_type->has_genericT_inside()) { - param_type = deducingTs.auto_deduce_from_argument(cur_f, dot_obj->loc, param_type, dot_obj->inferred_type); + param_type = deducingTs.auto_deduce_from_argument(cur_f, self_obj->loc, param_type, self_obj->inferred_type); } - if (param_0.is_mutate_parameter() && dot_obj->inferred_type->unwrap_alias() != param_type->unwrap_alias()) { - if (SinkExpression s_expr = extract_sink_expression_from_vertex(dot_obj)) { - assign_inferred_type(dot_obj, calc_declared_type_before_smart_cast(dot_obj)); + if (param_0.is_mutate_parameter() && self_obj->inferred_type->unwrap_alias() != param_type->unwrap_alias()) { + if (SinkExpression s_expr = extract_sink_expression_from_vertex(self_obj)) { + assign_inferred_type(self_obj, calc_declared_type_before_smart_cast(self_obj)); flow.register_known_type(s_expr, param_type); } } } + // for static generic method call `Container.create` of fun_ref = `Container.create`, gather T=int + if (!self_obj && fun_ref->receiver_type && fun_ref->receiver_type->has_genericT_inside() && callee->kind == ast_dot_access && callee->as()->get_obj()->kind == ast_reference) { + const auto* t_static = callee->as()->get_obj()->as()->sym->try_as(); + tolk_assert(t_static); + deducingTs.auto_deduce_from_argument(cur_f, v->loc, fun_ref->receiver_type, t_static->resolved_type); + } + + // loop over every argument, one by one, like control flow goes for (int i = 0; i < v->get_num_args(); ++i) { const LocalVarData& param_i = fun_ref->parameters[delta_self + i]; AnyExprV arg_i = v->get_arg(i)->get_expr(); TypePtr param_type = param_i.declared_type; - if (param_type->has_genericT_inside()) { + if (param_type->has_genericT_inside()) { // `fun f(a:T, b:T)` T was fixated on `a`, use it as hint for `b` param_type = deducingTs.replace_Ts_with_currently_deduced(param_type); } flow = infer_any_expr(arg_i, std::move(flow), false, param_type).out_flow; @@ -1120,7 +1154,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (fun_ref->is_generic_function()) { // if `f(args)` was called, Ts were inferred; check that all of them are known std::string_view nameT_unknown = deducingTs.get_first_not_deduced_nameT(); - if (!nameT_unknown.empty() && hint && fun_ref->declared_return_type && fun_ref->declared_return_type->has_genericT_inside()) { + if (!nameT_unknown.empty() && hint && fun_ref->declared_return_type) { // example: `t.tupleFirst()`, T doesn't depend on arguments, but is determined by return type // if used like `var x: int = t.tupleFirst()` / `t.tupleFirst() as int` / etc., use hint deducingTs.auto_deduce_from_argument(cur_f, v->loc, fun_ref->declared_return_type, hint); @@ -1132,20 +1166,15 @@ class InferTypesAndCallsAndFieldsVisitor final { fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, deducingTs.flush()); } - v->mutate()->assign_fun_ref(fun_ref); - // since for `t.tupleAt()`, infer_dot_access() not called for callee = "t.tupleAt", assign its target here - if (v->is_dot_call()) { - v->get_callee()->as()->mutate()->assign_target(fun_ref); - } // get return type either from user-specified declaration or infer here on demand traversing its body + v->mutate()->assign_fun_ref(fun_ref, self_obj != nullptr); get_or_infer_return_type(fun_ref); - TypePtr inferred_type = dot_obj && fun_ref->does_return_self() ? dot_obj->inferred_type : fun_ref->inferred_return_type; + TypePtr inferred_type = fun_ref->does_return_self() ? self_obj->inferred_type : fun_ref->inferred_return_type; assign_inferred_type(v, inferred_type); assign_inferred_type(callee, fun_ref->inferred_full_type); if (inferred_type == TypeDataNever::create()) { flow.mark_unreachable(UnreachableKind::CallNeverReturnFunction); } - // note, that mutate params don't affect typing, they are handled when converting to IR return ExprFlow(std::move(flow), used_as_condition); } @@ -1229,8 +1258,8 @@ class InferTypesAndCallsAndFieldsVisitor final { } if (branches_unifier.is_union_of_different_types()) { // same as in ternary: `match (...) { t1 => someSlice, t2 => someInt }` is `int|slice`, probably unexpected - if (hint == nullptr || hint == TypeDataUnknown::create()) { - fire(cur_f, v->loc, "type of `match` was inferred as " + to_string(branches_unifier.get_result()) + "; probably, it's not what you expected; assign it to a variable `var v: = match (...) { ... }` manually"); + if (hint == nullptr || hint == TypeDataUnknown::create() || hint->has_genericT_inside()) { + fire(cur_f, v->loc, "type of `match` was inferred as " + to_string(branches_unifier.get_result()) + "; probably, it's not what you expected\nassign it to a variable `var v: = match (...) { ... }` manually"); } } assign_inferred_type(v, branches_unifier.get_result()); @@ -1281,7 +1310,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } } if (!struct_ref) { - fire(cur_f, v->loc, "can not detect struct name; use either `var v: StructName = { ... }` or `var v = StructName { ... }`"); + fire(cur_f, v->loc, "can not detect struct name\nuse either `var v: StructName = { ... }` or `var v = StructName { ... }`"); } // so, we have struct_ref, so we can check field names and infer values @@ -1308,12 +1337,9 @@ class InferTypesAndCallsAndFieldsVisitor final { if (field_type->has_genericT_inside()) { field_type = deducingTs.replace_Ts_with_currently_deduced(field_type); } - if (field_type->has_genericT_inside()) { // `item: v` where field `item` is generic: use `v` to infer T - flow = infer_any_expr(val_i, std::move(flow), false).out_flow; + flow = infer_any_expr(val_i, std::move(flow), false, field_type).out_flow; + if (field_type->has_genericT_inside()) { deducingTs.auto_deduce_from_argument(cur_f, field_i->loc, field_type, val_i->inferred_type); - } else { - // field_type is hint, helps infer val_i - flow = infer_any_expr(val_i, std::move(flow), false, field_type).out_flow; } assign_inferred_type(field_i, val_i); } @@ -1538,7 +1564,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } if (return_unifier.is_union_of_different_types()) { // `return intVar` + `return sliceVar` results in `int | slice`, probably unexpected - fire(fun_ref, v_function->get_body()->loc, "function " + to_string(fun_ref) + " calculated return type is " + to_string(inferred_return_type) + "; probably, it's not what you expected; declare `fun (...): ` manually"); + fire(fun_ref, v_function->get_body()->loc, "function " + to_string(fun_ref) + " calculated return type is " + to_string(inferred_return_type) + "; probably, it's not what you expected\ndeclare `fun (...): ` manually"); } } @@ -1598,7 +1624,7 @@ static void infer_and_save_return_type_of_function(FunctionPtr fun_ref) { // prevent recursion of untyped functions, like `fun f() { return g(); } fun g() { return f(); }` bool contains = std::find(called_stack.begin(), called_stack.end(), fun_ref) != called_stack.end(); if (contains) { - fire(fun_ref, fun_ref->loc, "could not infer return type of " + to_string(fun_ref) + ", because it appears in a recursive call chain; specify `: ` manually"); + fire(fun_ref, fun_ref->loc, "could not infer return type of " + to_string(fun_ref) + ", because it appears in a recursive call chain\ndeclare `fun (...): ` manually"); } // dig into g's body; it's safe, since the compiler is single-threaded diff --git a/tolk/pipe-refine-lvalue-for-mutate.cpp b/tolk/pipe-refine-lvalue-for-mutate.cpp index 5187457b2..7692b0a2b 100644 --- a/tolk/pipe-refine-lvalue-for-mutate.cpp +++ b/tolk/pipe-refine-lvalue-for-mutate.cpp @@ -34,30 +34,22 @@ namespace tolk { GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD -static void fire_error_invalid_mutate_arg_passed(AnyExprV v, FunctionPtr fun_ref, const LocalVarData& p_sym, bool called_as_method, bool arg_passed_as_mutate, AnyV arg_expr) { +static void fire_error_invalid_mutate_arg_passed(FunctionPtr cur_f, SrcLocation loc, FunctionPtr fun_ref, const LocalVarData& p_sym, bool arg_passed_as_mutate, AnyV arg_expr) { std::string arg_str(arg_expr->kind == ast_reference ? arg_expr->as()->get_name() : "obj"); - // case: `loadInt(cs, 32)`; suggest: `cs.loadInt(32)` - if (p_sym.is_mutate_parameter() && !arg_passed_as_mutate && !called_as_method && p_sym.param_idx == 0 && fun_ref->does_accept_self()) { - v->error("`" + fun_ref->name + "` is a mutating method; consider calling `" + arg_str + "." + fun_ref->name + "()`, not `" + fun_ref->name + "(" + arg_str + ")`"); - } - // case: `cs.mutating_function()`; suggest: `mutating_function(mutate cs)` or make it a method - if (p_sym.is_mutate_parameter() && called_as_method && p_sym.param_idx == 0 && !fun_ref->does_accept_self()) { - v->error("function `" + fun_ref->name + "` mutates parameter `" + p_sym.name + "`; consider calling `" + fun_ref->name + "(mutate " + arg_str + ")`, not `" + arg_str + "." + fun_ref->name + "`(); alternatively, rename parameter to `self` to make it a method"); - } - // case: `mutating_function(arg)`; suggest: `mutate arg` if (p_sym.is_mutate_parameter() && !arg_passed_as_mutate) { - v->error("function `" + fun_ref->name + "` mutates parameter `" + p_sym.name + "`; you need to specify `mutate` when passing an argument, like `mutate " + arg_str + "`"); + // called `mutating_function(arg)`; suggest: `mutate arg` + fire(cur_f, loc, "function `" + fun_ref->as_human_readable() + "` mutates parameter `" + p_sym.name + "`\nyou need to specify `mutate` when passing an argument, like `mutate " + arg_str + "`"); + } else { + // called `usual_function(mutate arg)` + fire(cur_f, loc, "incorrect `mutate`, since `" + fun_ref->as_human_readable() + "` does not mutate this parameter"); } - // case: `usual_function(mutate arg)` - if (!p_sym.is_mutate_parameter() && arg_passed_as_mutate) { - v->error("incorrect `mutate`, since `" + fun_ref->name + "` does not mutate this parameter"); - } - throw Fatal("unreachable"); } class RefineLvalueForMutateArgumentsVisitor final : public ASTVisitorFunctionBody { + FunctionPtr cur_f = nullptr; + void visit(V v) override { // v is `globalF(args)` / `globalF(args)` / `obj.method(args)` / `local_var(args)` / `getF()(args)` FunctionPtr fun_ref = v->fun_maybe; @@ -66,41 +58,35 @@ class RefineLvalueForMutateArgumentsVisitor final : public ASTVisitorFunctionBod for (int i = 0; i < v->get_num_args(); ++i) { auto v_arg = v->get_arg(i); if (v_arg->passed_as_mutate) { - v_arg->error("`mutate` used for non-mutate argument"); + v_arg->error("`mutate` used for non-mutate parameter"); } } return; } - int delta_self = v->is_dot_call(); + int delta_self = v->get_self_obj() != nullptr; tolk_assert(fun_ref->get_num_params() == delta_self + v->get_num_args()); - if (v->is_dot_call()) { - if (fun_ref->does_mutate_self()) { - // for `b.storeInt()`, `b` should become lvalue, since `storeInt` is a method mutating self - // but: `beginCell().storeInt()`, then `beginCell()` is not lvalue - // (it will be extracted as tmp var when transforming AST to IR) - AnyExprV leftmost_obj = v->get_dot_obj(); - while (true) { - if (auto as_par = leftmost_obj->try_as()) { - leftmost_obj = as_par->get_expr(); - } else if (auto as_cast = leftmost_obj->try_as()) { - leftmost_obj = as_cast->get_expr(); - } else if (auto as_nn = leftmost_obj->try_as()) { - leftmost_obj = as_nn->get_expr(); - } else { - break; - } - } - bool will_be_extracted_as_tmp_var = leftmost_obj->kind == ast_function_call; - if (!will_be_extracted_as_tmp_var) { - leftmost_obj->mutate()->assign_lvalue_true(); - v->get_dot_obj()->mutate()->assign_lvalue_true(); + if (delta_self && fun_ref->does_mutate_self()) { + // for `b.storeInt()`, `b` should become lvalue, since `storeInt` is a method mutating self + // but: `beginCell().storeInt()`, then `beginCell()` is not lvalue + // (it will be extracted as tmp var when transforming AST to IR) + AnyExprV leftmost_obj = v->get_self_obj(); + while (true) { + if (auto as_par = leftmost_obj->try_as()) { + leftmost_obj = as_par->get_expr(); + } else if (auto as_cast = leftmost_obj->try_as()) { + leftmost_obj = as_cast->get_expr(); + } else if (auto as_nn = leftmost_obj->try_as()) { + leftmost_obj = as_nn->get_expr(); + } else { + break; } } - - if (!fun_ref->does_accept_self() && fun_ref->parameters[0].is_mutate_parameter()) { - fire_error_invalid_mutate_arg_passed(v, fun_ref, fun_ref->parameters[0], true, false, v->get_dot_obj()); + bool will_be_extracted_as_tmp_var = leftmost_obj->kind == ast_function_call; + if (!will_be_extracted_as_tmp_var) { + leftmost_obj->mutate()->assign_lvalue_true(); + v->get_self_obj()->mutate()->assign_lvalue_true(); } } @@ -108,7 +94,7 @@ class RefineLvalueForMutateArgumentsVisitor final : public ASTVisitorFunctionBod const LocalVarData& p_sym = fun_ref->parameters[delta_self + i]; auto arg_i = v->get_arg(i); if (p_sym.is_mutate_parameter() != arg_i->passed_as_mutate) { - fire_error_invalid_mutate_arg_passed(arg_i, fun_ref, p_sym, false, arg_i->passed_as_mutate, arg_i->get_expr()); + fire_error_invalid_mutate_arg_passed(cur_f, arg_i->loc, fun_ref, p_sym, arg_i->passed_as_mutate, arg_i->get_expr()); } parent::visit(arg_i); } @@ -119,6 +105,11 @@ class RefineLvalueForMutateArgumentsVisitor final : public ASTVisitorFunctionBod bool should_visit_function(FunctionPtr fun_ref) override { return fun_ref->is_code_function() && !fun_ref->is_generic_function(); } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + parent::visit(v_function->get_body()); + } }; void pipeline_refine_lvalue_for_mutate_arguments() { diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index 29754580b..69e24c1a8 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -35,7 +35,7 @@ namespace tolk { -static int calculate_method_id_for_entrypoint(std::string_view func_name) { +static int calculate_tvm_method_id_for_entrypoint(std::string_view func_name) { if (func_name == "main" || func_name == "onInternalMessage") { return 0; } @@ -54,7 +54,7 @@ static int calculate_method_id_for_entrypoint(std::string_view func_name) { tolk_assert(false); } -static int calculate_method_id_by_func_name(std::string_view func_name) { +static int calculate_tvm_method_id_by_func_name(std::string_view func_name) { unsigned int crc = td::crc16(static_cast(func_name)); return static_cast(crc & 0xffff) | 0x10000; } @@ -92,21 +92,10 @@ static void validate_arg_ret_order_of_asm_function(V v_body, int n } static const GenericsDeclaration* construct_genericTs(V v_list) { - std::vector itemsT; - itemsT.reserve(v_list->size()); - - for (int i = 0; i < v_list->size(); ++i) { - auto v_item = v_list->get_item(i); - auto it_existing = std::find_if(itemsT.begin(), itemsT.end(), [v_item](const GenericsDeclaration::GenericsItem& prev) { - return prev.nameT == v_item->nameT; - }); - if (it_existing != itemsT.end()) { - v_item->error("duplicate generic parameter `" + static_cast(v_item->nameT) + "`"); - } - itemsT.emplace_back(v_item->nameT); - } - - return new GenericsDeclaration(std::move(itemsT)); + std::vector namesT; + namesT.reserve(v_list->size()); + GenericsDeclaration::append_ast_list_checking_duplicates(v_list, namesT); + return new GenericsDeclaration(std::move(namesT), 0); } static GlobalConstPtr register_constant(V v) { @@ -127,13 +116,17 @@ static GlobalVarPtr register_global_var(V v) { return g_sym; } -static AliasDefPtr register_type_alias(V v, AliasDefPtr base_alias_ref = nullptr, const GenericsSubstitutions* substitutedTs = nullptr) { +static AliasDefPtr register_type_alias(V v, AliasDefPtr base_alias_ref = nullptr, std::string override_name = {}, const GenericsSubstitutions* substitutedTs = nullptr) { const GenericsDeclaration* genericTs = nullptr; if (v->genericsT_list && !substitutedTs) { genericTs = construct_genericTs(v->genericsT_list); } - AliasDefData* a_sym = new AliasDefData(static_cast(v->get_identifier()->name), v->loc, v->underlying_type_node, genericTs, substitutedTs, v); + std::string name = std::move(override_name); + if (name.empty()) { + name = v->get_identifier()->name; + } + AliasDefData* a_sym = new AliasDefData(std::move(name), v->loc, v->underlying_type_node, genericTs, substitutedTs, v); a_sym->base_alias_ref = base_alias_ref; // for `Response`, here is `Response` G.symtable.add_type_alias(a_sym); @@ -141,7 +134,7 @@ static AliasDefPtr register_type_alias(V v, AliasDef return a_sym; } -static StructPtr register_struct(V v, StructPtr base_struct_ref = nullptr, const GenericsSubstitutions* substitutedTs = nullptr) { +static StructPtr register_struct(V v, StructPtr base_struct_ref = nullptr, std::string override_name = {}, const GenericsSubstitutions* substitutedTs = nullptr) { auto v_body = v->get_struct_body(); std::vector fields; @@ -164,7 +157,11 @@ static StructPtr register_struct(V v, StructPtr base_str genericTs = construct_genericTs(v->genericsT_list); } - StructData* s_sym = new StructData(static_cast(v->get_identifier()->name), v->loc, std::move(fields), genericTs, substitutedTs, v); + std::string name = std::move(override_name); + if (name.empty()) { + name = v->get_identifier()->name; + } + StructData* s_sym = new StructData(std::move(name), v->loc, std::move(fields), genericTs, substitutedTs, v); s_sym->base_struct_ref = base_struct_ref; // for `Container`, here is `Container` G.symtable.add_struct(s_sym); @@ -188,8 +185,12 @@ static LocalVarData register_parameter(V v, int idx) { return LocalVarData(static_cast(v->param_name), v->loc, v->type_node, flags, idx); } -static FunctionPtr register_function(V v, FunctionPtr base_fun_ref = nullptr, const GenericsSubstitutions* substitutedTs = nullptr) { - std::string_view func_name = v->get_identifier()->name; +static FunctionPtr register_function(V v, FunctionPtr base_fun_ref = nullptr, std::string override_name = {}, const GenericsSubstitutions* substitutedTs = nullptr) { + if (v->is_builtin_function()) { + return nullptr; + } + + std::string_view f_identifier = v->get_identifier()->name; // function or method name std::vector parameters; int n_mutate_params = 0; @@ -200,23 +201,22 @@ static FunctionPtr register_function(V v, FunctionPtr n_mutate_params += static_cast(v_param->declared_as_mutate); } - if (v->is_builtin_function()) { - const Symbol* sym = lookup_global_symbol(func_name); - FunctionPtr fun_ref = sym ? sym->try_as() : nullptr; - if (!fun_ref || !fun_ref->is_builtin_function()) { - v->error("`builtin` used for non-builtin function"); - } - v->mutate()->assign_fun_ref(fun_ref); - return fun_ref; - } - const GenericsDeclaration* genericTs = nullptr; if (v->genericsT_list && !substitutedTs) { genericTs = construct_genericTs(v->genericsT_list); } + std::string method_name; + if (v->receiver_type_node) { + method_name = f_identifier; + } + std::string name = std::move(override_name); + if (name.empty()) { + name = f_identifier; + } + FunctionBody f_body = v->get_body()->kind == ast_block_statement ? static_cast(new FunctionBodyCode) : static_cast(new FunctionBodyAsm); - FunctionData* f_sym = new FunctionData(static_cast(func_name), v->loc, v->return_type_node, std::move(parameters), 0, genericTs, substitutedTs, f_body, v); + FunctionData* f_sym = new FunctionData(std::move(name), v->loc, std::move(method_name), v->receiver_type_node, v->return_type_node, std::move(parameters), 0, genericTs, substitutedTs, f_body, v); f_sym->base_fun_ref = base_fun_ref; // for `f`, here is `f` if (auto v_asm = v->get_body()->try_as()) { @@ -228,27 +228,31 @@ static FunctionPtr register_function(V v, FunctionPtr f_sym->ret_order = v_asm->ret_order; } - if (v->method_id.not_null()) { - f_sym->method_id = static_cast(v->method_id->to_long()); - } else if (v->flags & FunctionData::flagGetMethod) { - f_sym->method_id = calculate_method_id_by_func_name(func_name); - for (FunctionPtr other : G.all_get_methods) { - if (other->method_id == f_sym->method_id) { + if (v->tvm_method_id.not_null()) { + f_sym->tvm_method_id = static_cast(v->tvm_method_id->to_long()); + } else if (v->flags & FunctionData::flagContractGetter) { + f_sym->tvm_method_id = calculate_tvm_method_id_by_func_name(f_identifier); + for (FunctionPtr other : G.all_contract_getters) { + if (other->tvm_method_id == f_sym->tvm_method_id) { v->error(PSTRING() << "GET methods hash collision: `" << other->name << "` and `" << f_sym->name << "` produce the same hash. Consider renaming one of these functions."); } } } else if (v->flags & FunctionData::flagIsEntrypoint) { - f_sym->method_id = calculate_method_id_for_entrypoint(func_name); + f_sym->tvm_method_id = calculate_tvm_method_id_for_entrypoint(f_identifier); } f_sym->flags |= v->flags; if (n_mutate_params) { f_sym->flags |= FunctionData::flagHasMutateParams; } - G.symtable.add_function(f_sym); + if (!f_sym->receiver_type_node) { + G.symtable.add_function(f_sym); + } else if (!substitutedTs) { + G.all_methods.push_back(f_sym); + } G.all_functions.push_back(f_sym); - if (f_sym->is_get_method()) { - G.all_get_methods.push_back(f_sym); + if (f_sym->is_contract_getter()) { + G.all_contract_getters.push_back(f_sym); } v->mutate()->assign_fun_ref(f_sym); return f_sym; @@ -296,19 +300,19 @@ void pipeline_register_global_symbols() { } } -FunctionPtr pipeline_register_instantiated_generic_function(FunctionPtr base_fun_ref, AnyV cloned_v, const GenericsSubstitutions* substitutedTs) { +FunctionPtr pipeline_register_instantiated_generic_function(FunctionPtr base_fun_ref, AnyV cloned_v, std::string&& name, const GenericsSubstitutions* substitutedTs) { auto v = cloned_v->as(); - return register_function(v, base_fun_ref, substitutedTs); + return register_function(v, base_fun_ref, std::move(name), substitutedTs); } -StructPtr pipeline_register_instantiated_generic_struct(StructPtr base_struct_ref, AnyV cloned_v, const GenericsSubstitutions* substitutedTs) { +StructPtr pipeline_register_instantiated_generic_struct(StructPtr base_struct_ref, AnyV cloned_v, std::string&& name, const GenericsSubstitutions* substitutedTs) { auto v = cloned_v->as(); - return register_struct(v, base_struct_ref, substitutedTs); + return register_struct(v, base_struct_ref, std::move(name), substitutedTs); } -AliasDefPtr pipeline_register_instantiated_generic_alias(AliasDefPtr base_alias_ref, AnyV cloned_v, const GenericsSubstitutions* substitutedTs) { +AliasDefPtr pipeline_register_instantiated_generic_alias(AliasDefPtr base_alias_ref, AnyV cloned_v, std::string&& name, const GenericsSubstitutions* substitutedTs) { auto v = cloned_v->as(); - return register_type_alias(v, base_alias_ref, substitutedTs); + return register_type_alias(v, base_alias_ref, std::move(name), substitutedTs); } } // namespace tolk diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index d4cda33fb..4452ea573 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -49,9 +49,18 @@ namespace tolk { GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_undefined_symbol(FunctionPtr cur_f, V v) { if (v->name == "self") { - throw ParseError(cur_f, v->loc, "using `self` in a non-member function (it does not accept the first `self` parameter)"); + fire(cur_f, v->loc, "using `self` in a non-member function (it does not accept the first `self` parameter)"); } else { - throw ParseError(cur_f, v->loc, "undefined symbol `" + static_cast(v->name) + "`"); + fire(cur_f, v->loc, "undefined symbol `" + static_cast(v->name) + "`"); + } +} + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_type_used_as_symbol(FunctionPtr cur_f, V v) { + if (v->name == "random") { // calling `random()`, but it's a struct, correct is `random.uint256()` + fire(cur_f, v->loc, "`random` is not a function, you probably want `random.uint256()`"); + } else { + fire(cur_f, v->loc, "`" + static_cast(v->name) + "` only refers to a type, but is being used as a value here"); } } @@ -164,6 +173,9 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { if (!sym) { fire_error_undefined_symbol(cur_f, v->get_identifier()); } + if (sym->try_as() || sym->try_as()) { + fire_error_type_used_as_symbol(cur_f, v->get_identifier()); + } v->mutate()->assign_sym(sym); // for global functions, global vars and constants, `import` must exist @@ -172,6 +184,19 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { } } + void visit(V v) override { + try { + parent::visit(v->get_obj()); + } catch (const ParseError&) { + if (v->get_obj()->kind == ast_reference) { + // for `Point.create` / `int.zero`, "undefined symbol" is fired for Point/int + // suppress this exception till a later pipe, it will be tried to be resolved as a type + return; + } + throw; + } + } + void visit(V v) override { current_scope.open_scope(v->loc); parent::visit(v->get_block_statement()); @@ -248,7 +273,7 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { auto v_block = v->get_body()->as(); current_scope.open_scope(v->loc); - for (int i = 0; i < v->get_num_params(); ++i) { + for (int i = 0; i < fun_ref->get_num_params(); ++i) { current_scope.add_local_var(&fun_ref->parameters[i]); } parent::visit(v_block); @@ -280,7 +305,7 @@ void pipeline_resolve_identifiers_and_assign_symbols() { AssignSymInsideFunctionVisitor visitor; for (const SrcFile* file : G.all_src_files) { for (AnyV v : file->ast->as()->get_toplevel_declarations()) { - if (auto v_func = v->try_as()) { + if (auto v_func = v->try_as(); v_func && !v_func->is_builtin_function()) { tolk_assert(v_func->fun_ref); if (visitor.should_visit_function(v_func->fun_ref)) { visitor.start_visiting_function(v_func->fun_ref, v_func); diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 8f805f741..084a48c91 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -143,10 +143,33 @@ static TypePtr try_parse_predefined_type(std::string_view str) { return nullptr; } +static const GenericsDeclaration* combine_genericTs_from_receiver_and_declaration(TypePtr receiver_type, V v_list) { + std::vector namesT; + receiver_type->replace_children_custom([&namesT](TypePtr child) { + if (const TypeDataGenericT* asT = child->try_as()) { + auto it_existing = std::find_if(namesT.begin(), namesT.end(), [asT](const std::string_view& prevT) { + return prevT == asT->nameT; + }); + if (it_existing == namesT.end()) { + namesT.emplace_back(asT->nameT); + } + } + return child; + }); + int n_from_receiver = static_cast(namesT.size()); + + if (v_list) { + GenericsDeclaration::append_ast_list_checking_duplicates(v_list, namesT); + } + + return new GenericsDeclaration(std::move(namesT), n_from_receiver); +} + class TypeNodesVisitorResolver { FunctionPtr cur_f; // exists if we're inside its body const GenericsDeclaration* genericTs; // `` if we're inside `f` or `f` const GenericsSubstitutions* substitutedTs; // `T=int` if we're inside `f` + bool treat_unresolved_as_genericT; // used for receivers `fun Container.create()`, T becomes generic TypePtr parse_ast_type_node(AnyTypeV v, bool allow_without_type_arguments) { switch (v->kind) { @@ -169,6 +192,9 @@ class TypeNodesVisitorResolver { if (TypePtr predefined_type = try_parse_predefined_type(text)) { return predefined_type; } + if (treat_unresolved_as_genericT) { + return TypeDataGenericT::create(static_cast(text)); + } fire_error_unknown_type_name(cur_f, loc, text); } @@ -283,6 +309,9 @@ class TypeNodesVisitorResolver { } return TypeDataAlias::create(instantiate_generic_alias(alias_ref, GenericsSubstitutions(alias_ref->genericTs, type_arguments))); } + if (const TypeDataGenericT* asT = type_to_instantiate->try_as()) { + fire_error_unknown_type_name(cur_f, loc, asT->nameT); + } // `User` / `cell` fire(cur_f, loc, "type `" + type_to_instantiate->as_human_readable() + "` is not generic"); } @@ -297,10 +326,11 @@ class TypeNodesVisitorResolver { public: - TypeNodesVisitorResolver(FunctionPtr cur_f, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs) + TypeNodesVisitorResolver(FunctionPtr cur_f, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, bool treat_unresolved_as_genericT) : cur_f(cur_f) , genericTs(genericTs) - , substitutedTs(substitutedTs) {} + , substitutedTs(substitutedTs) + , treat_unresolved_as_genericT(treat_unresolved_as_genericT) {} TypePtr finalize_type_node(AnyTypeV type_node, bool allow_without_type_arguments = false) { #ifdef TOLK_DEBUG @@ -321,7 +351,7 @@ class TypeNodesVisitorResolver { } static void visit_symbol(GlobalVarPtr glob_ref) { - TypeNodesVisitorResolver visitor(nullptr, nullptr, nullptr); + TypeNodesVisitorResolver visitor(nullptr, nullptr, nullptr, false); TypePtr declared_type = visitor.finalize_type_node(glob_ref->type_node); glob_ref->mutate()->assign_resolved_type(declared_type); } @@ -331,7 +361,7 @@ class TypeNodesVisitorResolver { return; } - TypeNodesVisitorResolver visitor(nullptr, nullptr, nullptr); + TypeNodesVisitorResolver visitor(nullptr, nullptr, nullptr, false); TypePtr declared_type = visitor.finalize_type_node(const_ref->type_node); const_ref->mutate()->assign_resolved_type(declared_type); } @@ -346,7 +376,7 @@ class TypeNodesVisitorResolver { } called_stack.push_back(alias_ref); - TypeNodesVisitorResolver visitor(nullptr, alias_ref->genericTs, alias_ref->substitutedTs); + TypeNodesVisitorResolver visitor(nullptr, alias_ref->genericTs, alias_ref->substitutedTs, false); TypePtr underlying_type = visitor.finalize_type_node(alias_ref->underlying_type_node); alias_ref->mutate()->assign_resolved_type(underlying_type); alias_ref->mutate()->assign_visited_by_resolver(); @@ -365,7 +395,7 @@ class TypeNodesVisitorResolver { } called_stack.push_back(struct_ref); - TypeNodesVisitorResolver visitor(nullptr, struct_ref->genericTs, struct_ref->substitutedTs); + TypeNodesVisitorResolver visitor(nullptr, struct_ref->genericTs, struct_ref->substitutedTs, false); for (int i = 0; i < struct_ref->get_num_fields(); ++i) { StructFieldPtr field_ref = struct_ref->get_field(i); TypePtr declared_type = visitor.finalize_type_node(field_ref->type_node); @@ -377,7 +407,7 @@ class TypeNodesVisitorResolver { }; class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { - TypeNodesVisitorResolver type_nodes_visitor{nullptr, nullptr, nullptr}; + TypeNodesVisitorResolver type_nodes_visitor{nullptr, nullptr, nullptr, false}; TypePtr finalize_type_node(AnyTypeV type_node, bool allow_without_type_arguments = false) { return type_nodes_visitor.finalize_type_node(type_node, allow_without_type_arguments); @@ -415,6 +445,27 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { } void visit(V v) override { + // for static method calls, like "int.zero()" or "Point.create()", dot obj symbol is unresolved for now + // so, resolve it as a type and store as a "type reference symbol" + if (auto obj_ref = v->get_obj()->try_as()) { + if (obj_ref->sym == nullptr) { + std::string_view obj_type_name = obj_ref->get_identifier()->name; + AnyTypeV obj_type_node = createV(obj_ref->loc, obj_type_name); + if (obj_ref->has_instantiationTs()) { // Container.create + std::vector inner_and_args; + inner_and_args.reserve(1 + obj_ref->get_instantiationTs()->size()); + inner_and_args.push_back(obj_type_node); + for (int i = 0; i < obj_ref->get_instantiationTs()->size(); ++i) { + inner_and_args.push_back(obj_ref->get_instantiationTs()->get_item(i)->type_node); + } + obj_type_node = createV(obj_ref->loc, std::move(inner_and_args)); + } + TypePtr type_as_reference = finalize_type_node(obj_type_node); + const Symbol* type_as_symbol = new TypeReferenceUsedAsSymbol(static_cast(obj_type_name), obj_ref->loc, type_as_reference); + obj_ref->mutate()->assign_sym(type_as_symbol); + } + } + // for `t.tupleAt` / `obj.method`, resolve "MyAlias" and "T" // (for function call `t.tupleAt()`, this v (ast_dot_access `t.tupleAt`) is callee) if (auto v_instantiationTs = v->get_instantiationTs()) { @@ -448,9 +499,25 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { } void start_visiting_function(FunctionPtr fun_ref, V v) override { - type_nodes_visitor = TypeNodesVisitorResolver(fun_ref, fun_ref->genericTs, fun_ref->substitutedTs); + if (fun_ref->receiver_type_node) { + TypeNodesVisitorResolver receiver_visitor(fun_ref, fun_ref->genericTs, fun_ref->substitutedTs, true); + TypePtr receiver_type = receiver_visitor.finalize_type_node(fun_ref->receiver_type_node); + const GenericsDeclaration* genericTs = fun_ref->genericTs; + if (receiver_type->has_genericT_inside()) { + genericTs = combine_genericTs_from_receiver_and_declaration(receiver_type, v->genericsT_list); + } + std::string name_prefix = receiver_type->as_human_readable(); + bool embrace = receiver_type->try_as() && !receiver_type->try_as()->or_null; + if (embrace) { + name_prefix = "(" + name_prefix + ")"; + } + fun_ref->mutate()->assign_resolved_receiver_type(receiver_type, genericTs, std::move(name_prefix)); + G.symtable.add_function(fun_ref); + } - for (int i = 0; i < v->get_num_params(); ++i) { + type_nodes_visitor = TypeNodesVisitorResolver(fun_ref, fun_ref->genericTs, fun_ref->substitutedTs, false); + + for (int i = 0; i < fun_ref->get_num_params(); ++i) { const LocalVarData& param_var = fun_ref->parameters[i]; TypePtr declared_type = finalize_type_node(param_var.type_node); param_var.mutate()->assign_resolved_type(declared_type); @@ -466,7 +533,7 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { } void start_visiting_constant(GlobalConstPtr const_ref) { - type_nodes_visitor = TypeNodesVisitorResolver(nullptr, nullptr, nullptr); + type_nodes_visitor = TypeNodesVisitorResolver(nullptr, nullptr, nullptr, false); // `const a = 0 as int8`, resolve types there // same for struct field `v: int8 = 0 as int8` @@ -474,7 +541,7 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { } void start_visiting_struct_fields(StructPtr struct_ref) { - type_nodes_visitor = TypeNodesVisitorResolver(nullptr, struct_ref->genericTs, struct_ref->substitutedTs); + type_nodes_visitor = TypeNodesVisitorResolver(nullptr, struct_ref->genericTs, struct_ref->substitutedTs, false); // same for struct field `v: int8 = 0 as int8` for (StructFieldPtr field_ref : struct_ref->fields) { @@ -490,7 +557,7 @@ void pipeline_resolve_types_and_aliases() { for (const SrcFile* file : G.all_src_files) { for (AnyV v : file->ast->as()->get_toplevel_declarations()) { - if (auto v_func = v->try_as()) { + if (auto v_func = v->try_as(); v_func && !v_func->is_builtin_function()) { tolk_assert(v_func->fun_ref); if (visitor.should_visit_function(v_func->fun_ref)) { visitor.start_visiting_function(v_func->fun_ref, v_func); @@ -518,6 +585,8 @@ void pipeline_resolve_types_and_aliases() { } } } + + patch_builtins_after_stdlib_loaded(); } void pipeline_resolve_types_and_aliases(FunctionPtr fun_ref) { diff --git a/tolk/pipeline.h b/tolk/pipeline.h index c0a51b096..a46e3e037 100644 --- a/tolk/pipeline.h +++ b/tolk/pipeline.h @@ -50,7 +50,7 @@ void pipeline_generate_fif_output_to_std_cout(); // these pipes also can be called per-function individually // they are called for instantiated generics functions, when `f` is deeply cloned as `f` -FunctionPtr pipeline_register_instantiated_generic_function(FunctionPtr base_fun_ref, AnyV cloned_v, const GenericsSubstitutions* substitutedTs); +FunctionPtr pipeline_register_instantiated_generic_function(FunctionPtr base_fun_ref, AnyV cloned_v, std::string&& name, const GenericsSubstitutions* substitutedTs); void pipeline_resolve_identifiers_and_assign_symbols(FunctionPtr); void pipeline_resolve_types_and_aliases(FunctionPtr); @@ -58,11 +58,11 @@ void pipeline_calculate_rvalue_lvalue(FunctionPtr); void pipeline_detect_unreachable_statements(FunctionPtr); void pipeline_infer_types_and_calls_and_fields(FunctionPtr); -StructPtr pipeline_register_instantiated_generic_struct(StructPtr base_struct_ref, AnyV cloned_v, const GenericsSubstitutions* substitutedTs); +StructPtr pipeline_register_instantiated_generic_struct(StructPtr base_struct_ref, AnyV cloned_v, std::string&& name, const GenericsSubstitutions* substitutedTs); void pipeline_resolve_identifiers_and_assign_symbols(StructPtr); void pipeline_resolve_types_and_aliases(StructPtr); -AliasDefPtr pipeline_register_instantiated_generic_alias(AliasDefPtr base_alias_ref, AnyV cloned_v, const GenericsSubstitutions* substitutedTs); +AliasDefPtr pipeline_register_instantiated_generic_alias(AliasDefPtr base_alias_ref, AnyV cloned_v, std::string&& name, const GenericsSubstitutions* substitutedTs); void pipeline_resolve_types_and_aliases(AliasDefPtr); } // namespace tolk diff --git a/tolk/src-file.cpp b/tolk/src-file.cpp index d97608560..ab54ce7b0 100644 --- a/tolk/src-file.cpp +++ b/tolk/src-file.cpp @@ -226,7 +226,27 @@ std::ostream& operator<<(std::ostream& os, const ParseError& error) { } void ParseError::show(std::ostream& os) const { - os << loc << ": error: " << message << std::endl; + if (message.find('\n') == std::string::npos) { + // just print a single-line message + os << loc << ": error: " << message << std::endl; + } else { + // print "location: line1 \n (spaces) line2 \n ..." + std::string_view message = this->message; + std::string loc_text = loc.to_string(); + std::string loc_spaces(std::min(static_cast(loc_text.size()), 30), ' '); + size_t start = 0, end; + os << loc_text << ": error: "; + while ((end = message.find('\n', start)) != std::string::npos) { + if (start > 0) { + os << loc_spaces << " "; + } + os << message.substr(start, end - start) << std::endl; + start = end + 1; + } + if (start < message.size()) { + os << loc_spaces << " " << message.substr(start) << std::endl; + } + } if (current_function) { os << " // in function `" << current_function->as_human_readable() << "`" << std::endl; } diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 2449de59d..2e9fbabe5 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -66,6 +66,15 @@ bool FunctionData::does_need_codegen() const { return true; } +void FunctionData::assign_resolved_receiver_type(TypePtr receiver_type, const GenericsDeclaration* genericTs, std::string&& name_prefix) { + this->receiver_type = receiver_type; + if (!this->substitutedTs) { // after receiver has been resolve, update name to "receiver.method" + name_prefix.erase(std::remove(name_prefix.begin(), name_prefix.end(), ' '), name_prefix.end()); + this->name = name_prefix + "." + this->method_name; + this->genericTs = genericTs; + } +} + void FunctionData::assign_resolved_type(TypePtr declared_return_type) { this->declared_return_type = declared_return_type; } @@ -212,4 +221,8 @@ const Symbol* lookup_global_symbol(std::string_view name) { return G.symtable.lookup(name); } +FunctionPtr lookup_function(std::string_view name) { + return G.symtable.lookup(name)->try_as(); +} + } // namespace tolk diff --git a/tolk/symtable.h b/tolk/symtable.h index cde3a8397..9c80d1992 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -94,7 +94,7 @@ typedef std::variant< > FunctionBody; struct FunctionData final : Symbol { - static constexpr int EMPTY_METHOD_ID = -10; + static constexpr int EMPTY_TVM_METHOD_ID = -10; enum { flagInline = 1, // marked `@inline` @@ -103,7 +103,7 @@ struct FunctionData final : Symbol { flagUsedAsNonCall = 8, // used not only as `f()`, but as a 1-st class function (assigned to var, pushed to tuple, etc.) flagMarkedAsPure = 16, // declared as `pure`, can't call impure and access globals, unused invocations are optimized out flagImplicitReturn = 32, // control flow reaches end of function, so it needs implicit return at the end - flagGetMethod = 64, // was declared via `get func(): T`, method_id is auto-assigned + flagContractGetter = 64, // was declared via `get func(): T`, tvm_method_id is auto-assigned flagIsEntrypoint = 128, // it's `main` / `onExternalMessage` / etc. flagHasMutateParams = 256, // has parameters declared as `mutate` flagAcceptsSelf = 512, // is a member function (has `self` first parameter) @@ -112,9 +112,13 @@ struct FunctionData final : Symbol { flagCompileTimeOnly = 4096, // calculated only at compile-time for constant arguments: `ton("0.05")`, `stringCrc32`, and others }; - int method_id = EMPTY_METHOD_ID; + int tvm_method_id = EMPTY_TVM_METHOD_ID; int flags; + std::string method_name; // for `fun Container.store` here is "store" + AnyTypeV receiver_type_node; // for `fun Container.store` here is `Container` + TypePtr receiver_type = nullptr; // = resolved receiver_type_node + std::vector parameters; std::vector arg_order, ret_order; AnyTypeV return_type_node; // may be nullptr, meaning "auto infer" @@ -128,9 +132,11 @@ struct FunctionData final : Symbol { FunctionBody body; AnyV ast_root; // V for user-defined (not builtin) - FunctionData(std::string name, SrcLocation loc, AnyTypeV return_type_node, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) + FunctionData(std::string name, SrcLocation loc, std::string method_name, AnyTypeV receiver_type_node, AnyTypeV return_type_node, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) : Symbol(std::move(name), loc) , flags(initial_flags) + , method_name(std::move(method_name)) + , receiver_type_node(receiver_type_node) , parameters(std::move(parameters)) , return_type_node(return_type_node) , genericTs(genericTs) @@ -138,9 +144,12 @@ struct FunctionData final : Symbol { , body(body) , ast_root(ast_root) { } - FunctionData(std::string name, SrcLocation loc, TypePtr declared_return_type, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) + FunctionData(std::string name, SrcLocation loc, std::string method_name, TypePtr receiver_type, TypePtr declared_return_type, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, FunctionBody body, AnyV ast_root) : Symbol(std::move(name), loc) , flags(initial_flags) + , method_name(std::move(method_name)) + , receiver_type_node(nullptr) + , receiver_type(receiver_type) , parameters(std::move(parameters)) , return_type_node(nullptr) // for built-in functions, defined in sources , declared_return_type(declared_return_type) @@ -165,6 +174,8 @@ struct FunctionData final : Symbol { bool is_code_function() const { return std::holds_alternative(body); } bool is_asm_function() const { return std::holds_alternative(body); } bool is_builtin_function() const { return ast_root == nullptr; } + bool is_method() const { return !method_name.empty(); } + bool is_static_method() const { return is_method() && !does_accept_self(); } bool is_generic_function() const { return genericTs != nullptr; } bool is_instantiation_of_generic_function() const { return substitutedTs != nullptr; } @@ -175,8 +186,8 @@ struct FunctionData final : Symbol { bool is_used_as_noncall() const { return flags & flagUsedAsNonCall; } bool is_marked_as_pure() const { return flags & flagMarkedAsPure; } bool is_implicit_return() const { return flags & flagImplicitReturn; } - bool is_get_method() const { return flags & flagGetMethod; } - bool is_method_id_not_empty() const { return method_id != EMPTY_METHOD_ID; } + bool is_contract_getter() const { return flags & flagContractGetter; } + bool has_tvm_method_id() const { return tvm_method_id != EMPTY_TVM_METHOD_ID; } bool is_entrypoint() const { return flags & flagIsEntrypoint; } bool has_mutate_params() const { return flags & flagHasMutateParams; } bool does_accept_self() const { return flags & flagAcceptsSelf; } @@ -188,6 +199,7 @@ struct FunctionData final : Symbol { bool does_need_codegen() const; FunctionData* mutate() const { return const_cast(this); } + void assign_resolved_receiver_type(TypePtr receiver_type, const GenericsDeclaration* genericTs, std::string&& name_prefix); void assign_resolved_type(TypePtr declared_return_type); void assign_inferred_type(TypePtr inferred_return_type, TypePtr inferred_full_type); void assign_is_used_as_noncall(); @@ -326,6 +338,15 @@ struct StructData final : Symbol { std::string as_human_readable() const; }; +struct TypeReferenceUsedAsSymbol final : Symbol { + TypePtr resolved_type; + + TypeReferenceUsedAsSymbol(std::string name, SrcLocation loc, TypePtr resolved_type) + : Symbol(std::move(name), loc) + , resolved_type(resolved_type) { + } +}; + class GlobalSymbolTable { std::unordered_map entries; @@ -347,5 +368,6 @@ class GlobalSymbolTable { }; const Symbol* lookup_global_symbol(std::string_view name); +FunctionPtr lookup_function(std::string_view name); } // namespace tolk diff --git a/tolk/tolk.h b/tolk/tolk.h index 9fb39dae2..55d9d7677 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -1128,6 +1128,7 @@ AsmOp exec_arg2_op(SrcLocation loc, std::string op, long long imm1, long long im AsmOp push_const(SrcLocation loc, td::RefInt256 x); void define_builtins(); +void patch_builtins_after_stdlib_loaded(); From 76c9f9eaaf0679e2299167ab7744320c5a9ee5a3 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Mon, 28 Apr 2025 20:40:22 +0400 Subject: [PATCH 235/388] [Tolk] Support defaults for generic type arguments This is now valid: > struct Container { item: T = null } > Container{} // does Container Defaults for Ts work both for functions and structs. Their main purpose is to have the `never` default: > struct WithOptField { f: T; } I've added a rule that if a struct has the `never` field, it can be missed out from a literal while creation (like it doesn't exist at all). This will be used in stdlib later. --- tolk-tester/tests/generics-4.tolk | 181 ++++++++++++++++++ .../tests/invalid-semantics/err-4628.tolk | 2 +- .../tests/invalid-semantics/err-4629.tolk | 16 ++ .../tests/invalid-semantics/err-4907.tolk | 25 +++ .../tests/invalid-typing/err-6380.tolk | 14 ++ tolk-tester/tests/struct-tests.tolk | 13 ++ tolk/ast-from-tokens.cpp | 8 +- tolk/ast-replicator.h | 2 +- tolk/ast.h | 6 +- tolk/builtins.cpp | 2 +- tolk/generics-helpers.cpp | 58 ++++-- tolk/generics-helpers.h | 35 ++-- tolk/pipe-ast-to-legacy.cpp | 3 + tolk/pipe-infer-types-and-calls.cpp | 35 +++- tolk/pipe-register-symbols.cpp | 25 +-- tolk/pipe-resolve-types.cpp | 80 +++++--- tolk/symtable.cpp | 19 +- tolk/symtable.h | 5 +- tolk/type-system.cpp | 2 +- 19 files changed, 434 insertions(+), 97 deletions(-) create mode 100644 tolk-tester/tests/generics-4.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4629.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4907.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6380.tolk diff --git a/tolk-tester/tests/generics-4.tolk b/tolk-tester/tests/generics-4.tolk new file mode 100644 index 000000000..30ce19e35 --- /dev/null +++ b/tolk-tester/tests/generics-4.tolk @@ -0,0 +1,181 @@ +fun eqUnusedT(v: int) { return v; } +fun eqUnusedU(v: T) { if (v is int123) { __expect_type(v as U, "never"); } return v; } + +fun dup(x: T1, y: T2): (T1, T2) { return (x, y) } + +struct Container1 { + item: T; +} + +fun getItemOf(v: Container1) { + return v.item; +} + +struct WithDef1 { + body: T? = null; +} + +fun getBodyOf(o: WithDef1) { + return o.body; +} + +struct WithNever { + f1: int; + f2: T; +} + +fun eqNever(o: WithNever) { return o; } + +struct MyInit { + value: coins; + data: TInit; +} + +fun MyInit.getValue(self) { + return self.value; +} + +struct Parameters { + bounce: bool; + body: TBody; + init: builder | MyInit | null = null; +} + +fun createParameters(bounce: bool, body: TBody, data: TInit): Parameters { + return { bounce, body, init: { value: ton("0"), data } }; +} + +fun mySend(p: Parameters): int { + var total = 0; + if (p.bounce) { + total += 1; + } + if (p.body !is never) { + assert(p.body is TBody, 101); + total += 10; + } + if (p.init is MyInit) { + total += 100 + p.init.getValue(); + } + return total; +} + +fun test1() { + eqUnusedT(100); + eqUnusedU(100); + eqUnusedU(beginCell()); + + __expect_type(dup(null, null), "(null, null)"); + __expect_type(dup(createEmptyTuple(), 6), "(tuple, int)"); +} + +fun test2() { + var w1 = Container1 { item: 123 }; + var w2 = Container1 { item: null }; + __expect_type(w1, "Container1"); + __expect_type(w2, "Container1"); + __expect_type(getItemOf(w1), "int"); + __expect_type(getItemOf(w2), "null"); + + __expect_type(getItemOf({item: null as slice?}), "slice?"); + __expect_type(getItemOf({item: null}), "null"); +} + +@method_id(103) +fun test3() { + __expect_type(WithNever{f1:10}, "WithNever"); + __expect_type(WithNever{f1:10,f2:20}, "WithNever"); + + __expect_type(eqNever({f1:10}), "WithNever"); + __expect_type(eqNever({f1:10,f2:20}), "WithNever"); + __expect_type(eqNever({f1:10,f2:null}), "WithNever"); + + var a: WithNever = {f1:10}; + return (a, WithNever{f1:20}, eqNever({f1:30}), 777, eqNever({f1:40,f2:40})); +} + +@method_id(104) +fun test4() { + __expect_type(getBodyOf({body: 123}), "int?"); + __expect_type(getBodyOf({body: null}), "null"); + __expect_type(getBodyOf({}), "null"); + + __expect_type(WithDef1{}, "WithDef1"); + __expect_type(WithDef1{}.body, "null"); + __expect_type(WithDef1{body: null}.body, "null"); + __expect_type(WithDef1{body: 123}.body, "int?"); + + return (getBodyOf({body: null}), getBodyOf({}), WithDef1{}); +} + +@method_id(105) +fun test5() { + __expect_type(Parameters { bounce: true }, "Parameters"); + __expect_type(Parameters { bounce: false, body: 179 }, "Parameters"); + __expect_type(Parameters { bounce: true, init: beginCell() }, "Parameters"); + __expect_type(Parameters { bounce: false, body: beginCell(), init: { value: 123, data: 123 } }, "Parameters"); + + __expect_type(createParameters(true, null, null), "Parameters"); + __expect_type(createParameters(true, beginCell(), "123"), "Parameters"); + + __expect_type(createParameters(true, null, null).body, "null"); + __expect_type(createParameters(true, 123, null).body, "int"); + __expect_type(createParameters(true, null, null).init, "builder | MyInit | null"); + __expect_type(createParameters(true, 123, 456).init, "builder | MyInit | null"); + + return (createParameters(true, null, null), 777, createParameters(false, 123, 456)); +} + +@method_id(106) +fun test6() { + var p: Parameters = { + bounce: true, + body: 123, + init: { value: ton("0"), data: beginCell().endCell() } + }; + return (p.body is int, p.init is cell, p.init is MyInit, p.init is MyInit && p.init.data.depth() == 0); +} + +@method_id(107) +fun test7() { + __expect_type(Parameters{ bounce: false, body: 123, init: { value: ton("0"), data: beginCell() }}, "Parameters"); + var v1 = mySend({ bounce: true }); + var v2 = mySend({ bounce: false, body: 123, init: beginCell() }); + var v3 = mySend({ bounce: false, body: 123, init: { value: ton("0"), data: beginCell() }}); + var v4 = mySend({ bounce: true, init: { value: 16, data: null as [int]? }}); + return (v1, v2, v3, v4); +} + +fun main() { +} + +/** +@testcase | 103 | | 10 20 30 777 40 40 +@testcase | 104 | | (null) (null) (null) +@testcase | 105 | | -1 (null) 0 (null) 133 777 0 123 0 456 132 +@testcase | 106 | | -1 0 -1 -1 +@testcase | 107 | | 1 10 110 117 + +@fif_codegen DECLPROC eqUnusedT +@fif_codegen DECLPROC eqUnusedU +@fif_codegen DECLPROC mySend +@fif_codegen DECLPROC mySend +@fif_codegen DECLPROC mySend +@fif_codegen DECLPROC mySend + +@fif_codegen +""" + test6 PROC:<{ // + NEWC // '8 + ENDC // p.init.USlot2 + -1 PUSHINT // p.init.USlot2 '11=-1 + FALSE // p.init.USlot2 '11=-1 '12 + TRUE // p.init.USlot2 '11=-1 '12 '14 + s0 s3 XCHG // '14 '11=-1 '12 p.init.USlot2 + CDEPTH // '14 '11=-1 '12 '19 + 0 EQINT // '14 '11=-1 '12 '21 + 0 NEQINT // '14 '11=-1 '12 '18 + s1 s3 s0 XCHG3 // '11=-1 '12 '14 '18 + }> +""" + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4628.tolk b/tolk-tester/tests/invalid-semantics/err-4628.tolk index 3efe9069d..1f7debb5e 100644 --- a/tolk-tester/tests/invalid-semantics/err-4628.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4628.tolk @@ -11,6 +11,6 @@ fun main() { /** @compilation_should_fail -@stderr field `item` missed in initialization of struct `Wrapper` +@stderr can not deduce T for generic struct `Wrapper` @stderr = Wrapper { */ diff --git a/tolk-tester/tests/invalid-semantics/err-4629.tolk b/tolk-tester/tests/invalid-semantics/err-4629.tolk new file mode 100644 index 000000000..dc69a83be --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4629.tolk @@ -0,0 +1,16 @@ +struct Wrapper { + item: T; + value: int; +} + +fun main() { + var c = Wrapper { + value: null + }; +} + +/** +@compilation_should_fail +@stderr field `item` missed in initialization of struct `Wrapper` +@stderr = Wrapper { + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4907.tolk b/tolk-tester/tests/invalid-semantics/err-4907.tolk new file mode 100644 index 000000000..0674c933b --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4907.tolk @@ -0,0 +1,25 @@ +struct Data { + data: T; +} + +struct Parameters { + bounce: bool, + dest: slice | Data, + body: TBody, +} + +fun f(params: Parameters) { +} + +fun main() { + f({ + dest: "sss", + body: 123, + }) +} + +/** +@compilation_should_fail +@stderr can not deduce TData for generic struct `Parameters` +@stderr dest: "sss" + */ diff --git a/tolk-tester/tests/invalid-typing/err-6380.tolk b/tolk-tester/tests/invalid-typing/err-6380.tolk new file mode 100644 index 000000000..b7ab81a33 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6380.tolk @@ -0,0 +1,14 @@ +struct A { + f1: never; +} + +fun main() { + var a: A = {}; // f1 is never, can be omitted + a.f1 = 100; +} + +/** +@compilation_should_fail +@stderr can not assign `int` to field of type `never` +@stderr a.f1 = 100 + */ diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index c7c8205e4..a484475a2 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -474,6 +474,18 @@ fun test33(): WithDefaults { return { f2: 5, f3: null }; } +struct WithNever { + f1: int; + f2: never; + f3: int; +} + +@method_id(134) +fun test34() { + var o1: WithNever = { f1: 10, f3: 20 }; // f2 is `never`, it can be omitted + __expect_type(o1.f2, "never"); + return o1; +} fun main(x: int8, y: MInt) { @@ -526,6 +538,7 @@ type PointAlias = Point; @testcase | 131 | | 5 141 (null) 141 (null) 0 777 0 0 -1 777 0 -1 @testcase | 132 | | 10 20 10 0 0 20 0 0 0 0 @testcase | 133 | | -1 0 5 (null) (null) (null) 0 0 46 +@testcase | 134 | | 10 20 @fif_codegen """ diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 73791e3dd..ff13ed386 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -279,9 +279,15 @@ static V parse_genericsT_list(Lexer& lex) { lex.expect(tok_lt, "`<`"); while (true) { lex.check(tok_identifier, "T"); + SrcLocation locT = lex.cur_location(); std::string_view nameT = lex.cur_str(); - genericsT_items.emplace_back(createV(lex.cur_location(), nameT)); lex.next(); + AnyTypeV default_type = nullptr; + if (lex.tok() == tok_assign) { // + lex.next(); + default_type = parse_type_expression(lex); + } + genericsT_items.emplace_back(createV(locT, nameT, default_type)); if (lex.tok() != tok_comma) { break; } diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index b3e1702eb..fc1dafa22 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -211,7 +211,7 @@ class ASTReplicator final { return createV(v->loc, v->name); } static V clone(V v) { - return createV(v->loc, v->nameT); + return createV(v->loc, v->nameT, clone(v->default_type_node)); } static V clone(V v) { return createV(v->loc, clone(v->get_items())); diff --git a/tolk/ast.h b/tolk/ast.h index 013ac2a78..ee550064c 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -1127,12 +1127,14 @@ struct Vertex final : ASTStatementVararg { template<> // ast_genericsT_item is generics T at declaration // example: `fun f` has a list of 2 generic Ts +// example: `struct Params` has 1 generic T with default struct Vertex final : ASTOtherLeaf { std::string_view nameT; + AnyTypeV default_type_node; // exists for ``, nullptr otherwise - Vertex(SrcLocation loc, std::string_view nameT) + Vertex(SrcLocation loc, std::string_view nameT, AnyTypeV default_type_node) : ASTOtherLeaf(ast_genericsT_item, loc) - , nameT(nameT) {} + , nameT(nameT), default_type_node(default_type_node) {} }; template<> diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 5b962f604..70944cd4c 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1119,7 +1119,7 @@ void define_builtins() { TypePtr Never = TypeDataNever::create(); TypePtr typeT = TypeDataGenericT::create("T"); - const GenericsDeclaration* declGenericT = new GenericsDeclaration(std::vector{"T"}, 0); + const GenericsDeclaration* declGenericT = new GenericsDeclaration(std::vector{{"T", nullptr}}, 0); std::vector ParamsInt1 = {Int}; std::vector ParamsInt2 = {Int, Int}; diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index 0f8e6356c..5c0cc8867 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -25,16 +25,22 @@ namespace tolk { // given orig `(int, T)` and substitutions [slice], return `(int, slice)` -static TypePtr replace_genericT_with_deduced(TypePtr orig, const GenericsSubstitutions* substitutedTs) { +static TypePtr replace_genericT_with_deduced(TypePtr orig, const GenericsSubstitutions* substitutedTs, bool apply_defaultTs = false, std::string_view* out_unknownT = nullptr) { if (!orig || !orig->has_genericT_inside()) { return orig; } - return orig->replace_children_custom([substitutedTs](TypePtr child) { + return orig->replace_children_custom([substitutedTs, apply_defaultTs, &out_unknownT](TypePtr child) { if (const TypeDataGenericT* asT = child->try_as()) { TypePtr typeT = substitutedTs->get_substitution_for_nameT(asT->nameT); + if (typeT == nullptr && apply_defaultTs) { + typeT = substitutedTs->get_default_for_nameT(asT->nameT); + } if (typeT == nullptr) { // T was not deduced yet, leave T as generic typeT = child; + if (out_unknownT && out_unknownT->empty()) { + *out_unknownT = asT->nameT; + } } return typeT; } @@ -100,6 +106,14 @@ void GenericsSubstitutions::provide_type_arguments(const std::vector& t } } +void GenericsSubstitutions::rewrite_missing_with_defaults() { + for (int i = 0; i < size(); ++i) { + if (valuesTs[i] == nullptr) { + valuesTs[i] = genericTs->get_defaultT(i); // if no default, left nullptr + } + } +} + GenericSubstitutionsDeducing::GenericSubstitutionsDeducing(FunctionPtr fun_ref) : fun_ref(fun_ref) , struct_ref(nullptr) @@ -188,6 +202,12 @@ void GenericSubstitutionsDeducing::consider_next_condition(TypePtr param_type, T } } } + // `arg: int | MyData` called as `f(MyData)` => T is int + else { + for (TypePtr p_variant : p_union->variants) { + consider_next_condition(p_variant, arg_type); + } + } } else if (const auto* p_instSt = param_type->try_as(); p_instSt && p_instSt->struct_ref) { // `arg: Wrapper` called as `f(wrappedInt)` => T is int if (const auto* a_struct = arg_type->try_as(); a_struct && a_struct->struct_ref->is_instantiation_of_generic_struct() && a_struct->struct_ref->base_struct_ref == p_instSt->struct_ref) { @@ -217,9 +237,11 @@ TypePtr GenericSubstitutionsDeducing::auto_deduce_from_argument(TypePtr param_ty } TypePtr GenericSubstitutionsDeducing::auto_deduce_from_argument(FunctionPtr cur_f, SrcLocation loc, TypePtr param_type, TypePtr arg_type) { - param_type = auto_deduce_from_argument(param_type, arg_type); + std::string_view unknown_nameT; + consider_next_condition(param_type, arg_type); + param_type = replace_genericT_with_deduced(param_type, &deducedTs, true, &unknown_nameT); if (param_type->has_genericT_inside()) { - fire_error_can_not_deduce(cur_f, loc, get_first_not_deduced_nameT()); + fire_error_can_not_deduce(cur_f, loc, unknown_nameT); } return param_type; } @@ -233,6 +255,10 @@ std::string_view GenericSubstitutionsDeducing::get_first_not_deduced_nameT() con return ""; } +void GenericSubstitutionsDeducing::apply_defaults_from_declaration() { + deducedTs.rewrite_missing_with_defaults(); +} + void GenericSubstitutionsDeducing::fire_error_can_not_deduce(FunctionPtr cur_f, SrcLocation loc, std::string_view nameT) const { if (fun_ref) { fire(cur_f, loc, "can not deduce " + static_cast(nameT) + " for generic function `" + fun_ref->as_human_readable() + "`; instantiate it manually with `" + fun_ref->name + "<...>()`"); @@ -249,7 +275,7 @@ std::string GenericsDeclaration::as_human_readable(bool include_from_receiver) c if (result.size() > 1) { result += ", "; } - result += namesT[i]; + result += itemsT[i].nameT; } result += '>'; } @@ -257,27 +283,14 @@ std::string GenericsDeclaration::as_human_readable(bool include_from_receiver) c } int GenericsDeclaration::find_nameT(std::string_view nameT) const { - for (int i = 0; i < static_cast(namesT.size()); ++i) { - if (namesT[i] == nameT) { + for (int i = 0; i < static_cast(itemsT.size()); ++i) { + if (itemsT[i].nameT == nameT) { return i; } } return -1; } -void GenericsDeclaration::append_ast_list_checking_duplicates(AnyV v_genericsT_list, std::vector& out) { - for (int i = 0; i < v_genericsT_list->as()->size(); ++i) { - auto v_item = v_genericsT_list->as()->get_item(i); - auto it_existing = std::find_if(out.begin(), out.end(), [v_item](const std::string_view& prevT) { - return prevT == v_item->nameT; - }); - if (it_existing != out.end()) { - v_item->error("duplicate generic parameter `" + static_cast(v_item->nameT) + "`"); - } - out.emplace_back(v_item->nameT); - } -} - bool GenericsSubstitutions::has_nameT(std::string_view nameT) const { return genericTs->find_nameT(nameT) != -1; } @@ -287,6 +300,11 @@ TypePtr GenericsSubstitutions::get_substitution_for_nameT(std::string_view nameT return idx == -1 ? nullptr : valuesTs[idx]; } +TypePtr GenericsSubstitutions::get_default_for_nameT(std::string_view nameT) const { + int idx = genericTs->find_nameT(nameT); + return idx == -1 ? nullptr : genericTs->get_defaultT(idx); +} + // given this= and rhs=, check that T1 is equal to T2 in terms of "equal_to" of TypePtr // for example, `Wrapper>` / `Wrapper>` / `Wrapper` are equal bool GenericsSubstitutions::equal_to(const GenericsSubstitutions* rhs) const { diff --git a/tolk/generics-helpers.h b/tolk/generics-helpers.h index 08dfcc981..c8f46ff27 100644 --- a/tolk/generics-helpers.h +++ b/tolk/generics-helpers.h @@ -24,26 +24,34 @@ namespace tolk { // when a function is declared `f`, this "" is represented as this class -// - `fun tuple.push(self, value:T)` namesT = [T] -// - `struct Pair` namesT = [A,B] -// - `fun Container.compareWith` namesT = [T,U], n_from_receiver = 1 -// - `fun Pair.createFrom` namesT = [U,V] -// - `fun Pair.create` namesT = [A,B], n_from_receiver = 2 +// - `fun tuple.push(self, value:T)` itemsT = [T] +// - `struct Pair` itemsT = [A,B] +// - `fun Container.compareWith` itemsT = [T,U], n_from_receiver = 1 +// - `fun Pair.createFrom` itemsT = [U,V] +// - `fun Pair.create` itemsT = [A,B], n_from_receiver = 2 +// - `struct Opts` itemsT = [T default_type=never] struct GenericsDeclaration { - explicit GenericsDeclaration(std::vector&& namesT, int n_from_receiver) - : namesT(std::move(namesT)) + struct ItemT { + std::string_view nameT; + TypePtr default_type; // exists for ``, nullptr otherwise + + ItemT(std::string_view nameT, TypePtr default_type) + : nameT(nameT), default_type(default_type) {} + }; + + explicit GenericsDeclaration(std::vector&& itemsT, int n_from_receiver) + : itemsT(std::move(itemsT)) , n_from_receiver(n_from_receiver) {} - const std::vector namesT; + const std::vector itemsT; const int n_from_receiver; std::string as_human_readable(bool include_from_receiver = false) const; - int size() const { return static_cast(namesT.size()); } + int size() const { return static_cast(itemsT.size()); } int find_nameT(std::string_view nameT) const; - std::string_view get_nameT(int idx) const { return namesT[idx]; } - - static void append_ast_list_checking_duplicates(AnyV v_genericsT_list, std::vector& out); + std::string_view get_nameT(int idx) const { return itemsT[idx].nameT; } + TypePtr get_defaultT(int idx) const { return itemsT[idx].default_type; } }; // when a function call is `f()`, this "" is represented as this class @@ -65,12 +73,14 @@ struct GenericsSubstitutions { int size() const { return static_cast(valuesTs.size()); } bool has_nameT(std::string_view nameT) const; TypePtr get_substitution_for_nameT(std::string_view nameT) const; + TypePtr get_default_for_nameT(std::string_view nameT) const; std::string_view nameT_at(int idx) const { return genericTs->get_nameT(idx); } TypePtr typeT_at(int idx) const { return valuesTs.at(idx); } bool equal_to(const GenericsSubstitutions* rhs) const; void set_typeT(std::string_view nameT, TypePtr typeT); void provide_type_arguments(const std::vector& type_arguments); + void rewrite_missing_with_defaults(); }; // this class helps to deduce Ts on the fly @@ -92,6 +102,7 @@ class GenericSubstitutionsDeducing { TypePtr auto_deduce_from_argument(TypePtr param_type, TypePtr arg_type); TypePtr auto_deduce_from_argument(FunctionPtr cur_f, SrcLocation loc, TypePtr param_type, TypePtr arg_type); std::string_view get_first_not_deduced_nameT() const; + void apply_defaults_from_declaration(); void fire_error_can_not_deduce(FunctionPtr cur_f, SrcLocation loc, std::string_view nameT) const; GenericsSubstitutions&& flush() { diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 0f86b2ad1..ecabb5c71 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -1344,6 +1344,9 @@ static std::vector process_object_literal(V v, Co } } if (!v_init_val) { + if (field_ref->declared_type == TypeDataNever::create()) { + continue; // field of `never` type can be missed out of object literal (useful in generics defaults) + } // (it occupies 0 slots, nothing is assignable to it — like this field is missing from a struct) tolk_assert(field_ref->has_default_value()); ASTAuxData *aux_data = new AuxData_ForceFiftLocation(prev_loc); auto v_force_loc = createV(v->loc, field_ref->default_value, aux_data, field_ref->declared_type); diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 2f324f561..abeaffa1f 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -1160,6 +1160,10 @@ class InferTypesAndCallsAndFieldsVisitor final { deducingTs.auto_deduce_from_argument(cur_f, v->loc, fun_ref->declared_return_type, hint); nameT_unknown = deducingTs.get_first_not_deduced_nameT(); } + if (!nameT_unknown.empty()) { + deducingTs.apply_defaults_from_declaration(); + nameT_unknown = deducingTs.get_first_not_deduced_nameT(); + } if (!nameT_unknown.empty()) { deducingTs.fire_error_can_not_deduce(cur_f, v->get_arg_list()->loc, nameT_unknown); } @@ -1303,11 +1307,18 @@ class InferTypesAndCallsAndFieldsVisitor final { last_struct = variant_struct->struct_ref; n_structs++; } + if (const TypeDataGenericTypeWithTs* hint_withTs = variant->unwrap_alias()->try_as()) { + last_struct = hint_withTs->struct_ref; + n_structs++; + } } if (n_structs == 1) { struct_ref = last_struct; } } + if (const TypeDataGenericTypeWithTs* hint_withTs = hint->unwrap_alias()->try_as()) { + struct_ref = hint_withTs->struct_ref; + } } if (!struct_ref) { fire(cur_f, v->loc, "can not detect struct name\nuse either `var v: StructName = { ... }` or `var v = StructName { ... }`"); @@ -1343,16 +1354,16 @@ class InferTypesAndCallsAndFieldsVisitor final { } assign_inferred_type(field_i, val_i); } - for (StructFieldPtr field_ref : struct_ref->fields) { - if (!(occurred_mask & (1ULL << field_ref->field_idx)) && !field_ref->has_default_value()) { - fire(cur_f, v->get_body()->loc, "field `" + field_ref->name + "` missed in initialization of struct " + to_string(struct_ref)); - } - } // if it's a generic struct `Wrapper`, we need to instantiate it, like `Wrapper` if (struct_ref->is_generic_struct()) { - if (std::string_view nameT = deducingTs.get_first_not_deduced_nameT(); !nameT.empty()) { - deducingTs.fire_error_can_not_deduce(cur_f, v->loc, nameT); + std::string_view nameT_unknown = deducingTs.get_first_not_deduced_nameT(); + if (!nameT_unknown.empty()) { + deducingTs.apply_defaults_from_declaration(); + nameT_unknown = deducingTs.get_first_not_deduced_nameT(); + } + if (!nameT_unknown.empty()) { + deducingTs.fire_error_can_not_deduce(cur_f, v->loc, nameT_unknown); } struct_ref = instantiate_generic_struct(struct_ref, deducingTs.flush()); // re-assign field_ref (it was earlier assigned into a field of a generic struct) @@ -1362,6 +1373,16 @@ class InferTypesAndCallsAndFieldsVisitor final { } } + // check that all fields are present (do it after potential generic instantiation, when types are known) + for (StructFieldPtr field_ref : struct_ref->fields) { + if (!(occurred_mask & (1ULL << field_ref->field_idx))) { + bool allow_missing = field_ref->has_default_value() || field_ref->declared_type == TypeDataNever::create(); + if (!allow_missing) { + fire(cur_f, v->get_body()->loc, "field `" + field_ref->name + "` missed in initialization of struct " + to_string(struct_ref)); + } + } + } + v->mutate()->assign_struct_ref(struct_ref); assign_inferred_type(v, TypeDataStruct::create(struct_ref)); assign_inferred_type(v->get_body(), v); diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index 69e24c1a8..bbb105e42 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -91,13 +91,6 @@ static void validate_arg_ret_order_of_asm_function(V v_body, int n } } -static const GenericsDeclaration* construct_genericTs(V v_list) { - std::vector namesT; - namesT.reserve(v_list->size()); - GenericsDeclaration::append_ast_list_checking_duplicates(v_list, namesT); - return new GenericsDeclaration(std::move(namesT), 0); -} - static GlobalConstPtr register_constant(V v) { GlobalConstData* c_sym = new GlobalConstData(static_cast(v->get_identifier()->name), v->loc, v->type_node, v->get_init_value()); @@ -117,15 +110,11 @@ static GlobalVarPtr register_global_var(V v) { } static AliasDefPtr register_type_alias(V v, AliasDefPtr base_alias_ref = nullptr, std::string override_name = {}, const GenericsSubstitutions* substitutedTs = nullptr) { - const GenericsDeclaration* genericTs = nullptr; - if (v->genericsT_list && !substitutedTs) { - genericTs = construct_genericTs(v->genericsT_list); - } - std::string name = std::move(override_name); if (name.empty()) { name = v->get_identifier()->name; } + const GenericsDeclaration* genericTs = nullptr; // at registering it's null; will be assigned after types resolving AliasDefData* a_sym = new AliasDefData(std::move(name), v->loc, v->underlying_type_node, genericTs, substitutedTs, v); a_sym->base_alias_ref = base_alias_ref; // for `Response`, here is `Response` @@ -152,15 +141,11 @@ static StructPtr register_struct(V v, StructPtr base_str fields.emplace_back(new StructFieldData(static_cast(v_field->get_identifier()->name), v_field->loc, i, v_field->type_node, default_value)); } - const GenericsDeclaration* genericTs = nullptr; - if (v->genericsT_list && !substitutedTs) { - genericTs = construct_genericTs(v->genericsT_list); - } - std::string name = std::move(override_name); if (name.empty()) { name = v->get_identifier()->name; } + const GenericsDeclaration* genericTs = nullptr; // at registering it's null; will be assigned after types resolving StructData* s_sym = new StructData(std::move(name), v->loc, std::move(fields), genericTs, substitutedTs, v); s_sym->base_struct_ref = base_struct_ref; // for `Container`, here is `Container` @@ -201,11 +186,6 @@ static FunctionPtr register_function(V v, FunctionPtr n_mutate_params += static_cast(v_param->declared_as_mutate); } - const GenericsDeclaration* genericTs = nullptr; - if (v->genericsT_list && !substitutedTs) { - genericTs = construct_genericTs(v->genericsT_list); - } - std::string method_name; if (v->receiver_type_node) { method_name = f_identifier; @@ -215,6 +195,7 @@ static FunctionPtr register_function(V v, FunctionPtr name = f_identifier; } + const GenericsDeclaration* genericTs = nullptr; // at registering it's null; will be assigned after types resolving FunctionBody f_body = v->get_body()->kind == ast_block_statement ? static_cast(new FunctionBodyCode) : static_cast(new FunctionBodyAsm); FunctionData* f_sym = new FunctionData(std::move(name), v->loc, std::move(method_name), v->receiver_type_node, v->return_type_node, std::move(parameters), 0, genericTs, substitutedTs, f_body, v); f_sym->base_fun_ref = base_fun_ref; // for `f`, here is `f` diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 084a48c91..915c01f72 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -143,28 +143,6 @@ static TypePtr try_parse_predefined_type(std::string_view str) { return nullptr; } -static const GenericsDeclaration* combine_genericTs_from_receiver_and_declaration(TypePtr receiver_type, V v_list) { - std::vector namesT; - receiver_type->replace_children_custom([&namesT](TypePtr child) { - if (const TypeDataGenericT* asT = child->try_as()) { - auto it_existing = std::find_if(namesT.begin(), namesT.end(), [asT](const std::string_view& prevT) { - return prevT == asT->nameT; - }); - if (it_existing == namesT.end()) { - namesT.emplace_back(asT->nameT); - } - } - return child; - }); - int n_from_receiver = static_cast(namesT.size()); - - if (v_list) { - GenericsDeclaration::append_ast_list_checking_duplicates(v_list, namesT); - } - - return new GenericsDeclaration(std::move(namesT), n_from_receiver); -} - class TypeNodesVisitorResolver { FunctionPtr cur_f; // exists if we're inside its body const GenericsDeclaration* genericTs; // `` if we're inside `f` or `f` @@ -375,6 +353,11 @@ class TypeNodesVisitorResolver { throw ParseError(alias_ref->loc, "type `" + alias_ref->name + "` circularly references itself"); } + if (auto v_genericsT_list = alias_ref->ast_root->as()->genericsT_list) { + const GenericsDeclaration* genericTs = construct_genericTs(nullptr, v_genericsT_list); + alias_ref->mutate()->assign_resolved_genericTs(genericTs); + } + called_stack.push_back(alias_ref); TypeNodesVisitorResolver visitor(nullptr, alias_ref->genericTs, alias_ref->substitutedTs, false); TypePtr underlying_type = visitor.finalize_type_node(alias_ref->underlying_type_node); @@ -394,6 +377,11 @@ class TypeNodesVisitorResolver { throw ParseError(struct_ref->loc, "struct `" + struct_ref->name + "` size is infinity due to recursive fields"); } + if (auto v_genericsT_list = struct_ref->ast_root->as()->genericsT_list) { + const GenericsDeclaration* genericTs = construct_genericTs(nullptr, v_genericsT_list); + struct_ref->mutate()->assign_resolved_genericTs(genericTs); + } + called_stack.push_back(struct_ref); TypeNodesVisitorResolver visitor(nullptr, struct_ref->genericTs, struct_ref->substitutedTs, false); for (int i = 0; i < struct_ref->get_num_fields(); ++i) { @@ -404,6 +392,44 @@ class TypeNodesVisitorResolver { struct_ref->mutate()->assign_visited_by_resolver(); called_stack.pop_back(); } + + static const GenericsDeclaration* construct_genericTs(TypePtr receiver_type, V v_list) { + std::vector itemsT; + if (receiver_type && receiver_type->has_genericT_inside()) { + receiver_type->replace_children_custom([&itemsT](TypePtr child) { + if (const TypeDataGenericT* asT = child->try_as()) { + auto it_existing = std::find_if(itemsT.begin(), itemsT.end(), [asT](const GenericsDeclaration::ItemT& prevT) { + return prevT.nameT == asT->nameT; + }); + if (it_existing == itemsT.end()) { + itemsT.emplace_back(asT->nameT, nullptr); + } + } + return child; + }); + } + int n_from_receiver = static_cast(itemsT.size()); + + if (v_list) { + TypeNodesVisitorResolver visitor(nullptr, nullptr, nullptr, false); + for (int i = 0; i < v_list->size(); ++i) { + auto v_item = v_list->get_item(i); + auto it_existing = std::find_if(itemsT.begin(), itemsT.end(), [v_item](const GenericsDeclaration::ItemT& prevT) { + return prevT.nameT == v_item->nameT; + }); + if (it_existing != itemsT.end()) { + v_item->error("duplicate generic parameter `" + static_cast(v_item->nameT) + "`"); + } + TypePtr default_type = nullptr; + if (v_list->get_item(i)->default_type_node) { + default_type = visitor.finalize_type_node(v_list->get_item(i)->default_type_node); + } + itemsT.emplace_back(v_item->nameT, default_type); + } + } + + return new GenericsDeclaration(std::move(itemsT), n_from_receiver); + } }; class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { @@ -502,18 +528,18 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { if (fun_ref->receiver_type_node) { TypeNodesVisitorResolver receiver_visitor(fun_ref, fun_ref->genericTs, fun_ref->substitutedTs, true); TypePtr receiver_type = receiver_visitor.finalize_type_node(fun_ref->receiver_type_node); - const GenericsDeclaration* genericTs = fun_ref->genericTs; - if (receiver_type->has_genericT_inside()) { - genericTs = combine_genericTs_from_receiver_and_declaration(receiver_type, v->genericsT_list); - } std::string name_prefix = receiver_type->as_human_readable(); bool embrace = receiver_type->try_as() && !receiver_type->try_as()->or_null; if (embrace) { name_prefix = "(" + name_prefix + ")"; } - fun_ref->mutate()->assign_resolved_receiver_type(receiver_type, genericTs, std::move(name_prefix)); + fun_ref->mutate()->assign_resolved_receiver_type(receiver_type, std::move(name_prefix)); G.symtable.add_function(fun_ref); } + if (v->genericsT_list || (fun_ref->receiver_type && fun_ref->receiver_type->has_genericT_inside())) { + const GenericsDeclaration* genericTs = TypeNodesVisitorResolver::construct_genericTs(fun_ref->receiver_type, v->genericsT_list); + fun_ref->mutate()->assign_resolved_genericTs(genericTs); + } type_nodes_visitor = TypeNodesVisitorResolver(fun_ref, fun_ref->genericTs, fun_ref->substitutedTs, false); diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 2e9fbabe5..1485eae75 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -66,11 +66,16 @@ bool FunctionData::does_need_codegen() const { return true; } -void FunctionData::assign_resolved_receiver_type(TypePtr receiver_type, const GenericsDeclaration* genericTs, std::string&& name_prefix) { +void FunctionData::assign_resolved_receiver_type(TypePtr receiver_type, std::string&& name_prefix) { this->receiver_type = receiver_type; if (!this->substitutedTs) { // after receiver has been resolve, update name to "receiver.method" name_prefix.erase(std::remove(name_prefix.begin(), name_prefix.end(), ' '), name_prefix.end()); this->name = name_prefix + "." + this->method_name; + } +} + +void FunctionData::assign_resolved_genericTs(const GenericsDeclaration* genericTs) { + if (this->substitutedTs == nullptr) { this->genericTs = genericTs; } } @@ -140,6 +145,12 @@ void AliasDefData::assign_visited_by_resolver() { this->flags |= flagVisitedByResolver; } +void AliasDefData::assign_resolved_genericTs(const GenericsDeclaration* genericTs) { + if (this->substitutedTs == nullptr) { + this->genericTs = genericTs; + } +} + void AliasDefData::assign_resolved_type(TypePtr underlying_type) { this->underlying_type = underlying_type; } @@ -156,6 +167,12 @@ void StructData::assign_visited_by_resolver() { this->flags |= flagVisitedByResolver; } +void StructData::assign_resolved_genericTs(const GenericsDeclaration* genericTs) { + if (this->substitutedTs == nullptr) { + this->genericTs = genericTs; + } +} + StructFieldPtr StructData::find_field(std::string_view field_name) const { for (StructFieldPtr field : fields) { if (field->name == field_name) { diff --git a/tolk/symtable.h b/tolk/symtable.h index 9c80d1992..38f090e7b 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -199,7 +199,8 @@ struct FunctionData final : Symbol { bool does_need_codegen() const; FunctionData* mutate() const { return const_cast(this); } - void assign_resolved_receiver_type(TypePtr receiver_type, const GenericsDeclaration* genericTs, std::string&& name_prefix); + void assign_resolved_receiver_type(TypePtr receiver_type, std::string&& name_prefix); + void assign_resolved_genericTs(const GenericsDeclaration* genericTs); void assign_resolved_type(TypePtr declared_return_type); void assign_inferred_type(TypePtr inferred_return_type, TypePtr inferred_full_type); void assign_is_used_as_noncall(); @@ -279,6 +280,7 @@ struct AliasDefData final : Symbol { AliasDefData* mutate() const { return const_cast(this); } void assign_visited_by_resolver(); + void assign_resolved_genericTs(const GenericsDeclaration* genericTs); void assign_resolved_type(TypePtr underlying_type); }; @@ -326,6 +328,7 @@ struct StructData final : Symbol { StructData* mutate() const { return const_cast(this); } void assign_visited_by_resolver(); + void assign_resolved_genericTs(const GenericsDeclaration* genericTs); StructData(std::string name, SrcLocation loc, std::vector&& fields, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, AnyV ast_root) : Symbol(std::move(name), loc) diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index 39431b4a3..b419a2e3f 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -870,7 +870,7 @@ bool TypeDataUnknown::can_rhs_be_assigned(TypePtr rhs) const { } bool TypeDataNever::can_rhs_be_assigned(TypePtr rhs) const { - return true; + return rhs == TypeDataNever::create(); } bool TypeDataVoid::can_rhs_be_assigned(TypePtr rhs) const { From 44705a1bcbb0b1e98ccf662e9ec6d6c778ebec3d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 29 Apr 2025 14:24:09 +0300 Subject: [PATCH 236/388] Load blocks from DB in ShardBlockRetainer, fix is_block_outdated --- tdutils/td/utils/tl_parsers.cpp | 3 ++- validator/db/archive-manager.cpp | 9 +++++++++ validator/db/archive-manager.hpp | 2 ++ validator/db/archive-slice.cpp | 18 ++++++++++++++++++ validator/db/archive-slice.hpp | 2 ++ validator/db/rootdb.cpp | 4 ++++ validator/db/rootdb.hpp | 1 + validator/interfaces/db.h | 2 ++ validator/interfaces/validator-manager.h | 3 +++ validator/manager.cpp | 4 ++++ validator/manager.hpp | 2 ++ validator/shard-block-retainer.cpp | 24 +++++++++++++++++++++++- validator/shard-block-retainer.hpp | 2 ++ validator/shard-block-verifier.cpp | 4 +++- 14 files changed, 77 insertions(+), 3 deletions(-) diff --git a/tdutils/td/utils/tl_parsers.cpp b/tdutils/td/utils/tl_parsers.cpp index eb78d3e1c..c41cfeb95 100644 --- a/tdutils/td/utils/tl_parsers.cpp +++ b/tdutils/td/utils/tl_parsers.cpp @@ -33,7 +33,8 @@ TlParser::TlParser(Slice slice) { if (data_len <= small_data_array.size() * sizeof(int32)) { buf = &small_data_array[0]; } else { - LOG(ERROR) << "Unexpected big unaligned data pointer of length " << slice.size() << " at " << slice.begin(); + LOG(DEBUG) << "Unexpected big unaligned data pointer of length " << slice.size() << " at " + << (const void *)slice.begin(); data_buf = std::make_unique(1 + data_len / sizeof(int32)); buf = data_buf.get(); } diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index 9e6ea797e..b39f4374f 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -1224,6 +1224,15 @@ void ArchiveManager::prepare_stats(td::Promise f) { + for (auto &[_, file] : temp_files_) { + if (file.deleted) { + continue; + } + td::actor::send_closure(file.file_actor_id(), &ArchiveSlice::iterate_block_handles, f); + } +} + void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise) { index_->begin_transaction().ensure(); td::MultiPromise mp; diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index 33dd1f3d6..9245aa61d 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -84,6 +84,8 @@ class ArchiveManager : public td::actor::Actor { void prepare_stats(td::Promise>> promise); + void iterate_temp_block_handles(std::function f); + static constexpr td::uint32 archive_size() { return 20000; } diff --git a/validator/db/archive-slice.cpp b/validator/db/archive-slice.cpp index 41ed50997..635df6dbb 100644 --- a/validator/db/archive-slice.cpp +++ b/validator/db/archive-slice.cpp @@ -686,6 +686,24 @@ void ArchiveSlice::close_files() { } } +void ArchiveSlice::iterate_block_handles(std::function f) { + before_query(); + td::uint32 range_start = ton_api::db_blockdb_key_value::ID; + td::uint32 range_end = ton_api::db_blockdb_key_value::ID + 1; + kv_->for_each_in_range(td::Slice{(char *)&range_start, 4}, td::Slice{(char *)&range_end, 4}, + [&](td::Slice key, td::Slice value) -> td::Status { + auto r_key = fetch_tl_object(key, true); + if (r_key.is_error()) { + return td::Status::OK(); + } + auto r_handle = create_block_handle(value); + if (r_handle.is_ok()) { + f(*r_handle.ok()); + } + return td::Status::OK(); + }); +} + void ArchiveSlice::do_close() { if (destroyed_) { return; diff --git a/validator/db/archive-slice.hpp b/validator/db/archive-slice.hpp index a027ec0ff..b955c270a 100644 --- a/validator/db/archive-slice.hpp +++ b/validator/db/archive-slice.hpp @@ -127,6 +127,8 @@ class ArchiveSlice : public td::actor::Actor { void open_files(); void close_files(); + void iterate_block_handles(std::function f); + private: void before_query(); void do_close(); diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 90546fbef..0b3730ede 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -527,6 +527,10 @@ void RootDb::get_persistent_state_descriptions(td::Promise f) { + td::actor::send_closure(archive_db_, &ArchiveManager::iterate_temp_block_handles, std::move(f)); +} + } // namespace validator } // namespace ton diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 2ccd7f942..05187fde7 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -139,6 +139,7 @@ class RootDb : public Db { void add_persistent_state_description(td::Ref desc, td::Promise promise) override; void get_persistent_state_descriptions(td::Promise>> promise) override; + void iterate_temp_block_handles(std::function f) override; private: td::actor::ActorId validator_manager_; diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index f21c6f902..c714e1e34 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -129,6 +129,8 @@ class Db : public td::actor::Actor { td::Promise promise) = 0; virtual void get_persistent_state_descriptions( td::Promise>> promise) = 0; + + virtual void iterate_temp_block_handles(std::function f) = 0; }; } // namespace validator diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 140f01ff2..c6151f1b7 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -274,6 +274,9 @@ class ValidatorManager : public ValidatorManagerInterface { promise.set_result(td::Unit()); } + virtual void iterate_temp_block_handles(std::function f) { + } + static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) { return ts / (1 << 17) != prev_ts / (1 << 17); } diff --git a/validator/manager.cpp b/validator/manager.cpp index f68cdf941..fa4c7bef9 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -3710,6 +3710,10 @@ void ValidatorManagerImpl::add_shard_block_retainer(adnl::AdnlNodeIdShort id) { "shardblockretainer", id, last_masterchain_state_, opts_, actor_id(this), adnl_, rldp_); } +void ValidatorManagerImpl::iterate_temp_block_handles(std::function f) { + td::actor::send_closure(db_, &Db::iterate_temp_block_handles, std::move(f)); +} + } // namespace validator } // namespace ton diff --git a/validator/manager.hpp b/validator/manager.hpp index fce2ceee4..31cd6feab 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -778,6 +778,8 @@ class ValidatorManagerImpl : public ValidatorManager { void add_shard_block_retainer(adnl::AdnlNodeIdShort id) override; + void iterate_temp_block_handles(std::function f) override; + std::map> validator_telemetry_; void init_validator_telemetry(); diff --git a/validator/shard-block-retainer.cpp b/validator/shard-block-retainer.cpp index 4fb96502e..c36a9bac6 100644 --- a/validator/shard-block-retainer.cpp +++ b/validator/shard-block-retainer.cpp @@ -15,6 +15,7 @@ along with TON Blockchain Library. If not, see . */ #include "shard-block-retainer.hpp" +#include namespace ton::validator { @@ -87,6 +88,18 @@ void ShardBlockRetainer::update_masterchain_state(td::Ref stat ++it; } } + if (!inited_) { + delay_action( + [SelfId = actor_id(this), manager = manager_]() { + td::actor::send_closure( + manager, &ValidatorManager::iterate_temp_block_handles, [SelfId](const BlockHandleInterface& handle) { + if (!handle.id().is_masterchain() && handle.received_state()) { + td::actor::send_closure(SelfId, &ShardBlockRetainer::got_block_from_db, handle.id()); + } + }); + }, + td::Timestamp::in(1.0)); + } inited_ = true; } @@ -171,8 +184,17 @@ void ShardBlockRetainer::confirm_block(BlockIdExt block_id) { LOG(INFO) << "Confirmed block " << block_id.to_str() << ", sending " << sent << " confirmations"; } +void ShardBlockRetainer::got_block_from_db(BlockIdExt block_id) { + if (!is_block_outdated(block_id)) { + LOG(INFO) << "Loaded confirmed block from DB: " << block_id.to_str(); + confirm_block(block_id); + } +} + bool ShardBlockRetainer::is_block_outdated(const BlockIdExt& block_id) const { - auto shard_desc = last_masterchain_state_->get_shard_from_config(block_id.shard_full(), false); + ShardIdFull shard = block_id.shard_full(); + shard.shard |= 1; + auto shard_desc = last_masterchain_state_->get_shard_from_config(shard, false); return shard_desc.not_null() && shard_desc->top_block_id().seqno() >= block_id.seqno(); } diff --git a/validator/shard-block-retainer.hpp b/validator/shard-block-retainer.hpp index 14a6cfa8a..ca8d4c877 100644 --- a/validator/shard-block-retainer.hpp +++ b/validator/shard-block-retainer.hpp @@ -62,6 +62,8 @@ class ShardBlockRetainer : public td::actor::Actor { void confirm_shard_block_description(td::Ref desc); void confirm_block(BlockIdExt block_id); + void got_block_from_db(BlockIdExt block_id); + bool is_block_outdated(const BlockIdExt& block_id) const; static constexpr double SUBSCRIPTION_TTL = 60.0; diff --git a/validator/shard-block-verifier.cpp b/validator/shard-block-verifier.cpp index 5beb1e9a3..000260a36 100644 --- a/validator/shard-block-verifier.cpp +++ b/validator/shard-block-verifier.cpp @@ -138,7 +138,9 @@ int ShardBlockVerifier::get_config_shard_idx(const ShardIdFull& shard_id) const } bool ShardBlockVerifier::is_block_outdated(const BlockIdExt& block_id) const { - auto shard_desc = last_masterchain_state_->get_shard_from_config(block_id.shard_full(), false); + ShardIdFull shard = block_id.shard_full(); + shard.shard |= 1; + auto shard_desc = last_masterchain_state_->get_shard_from_config(shard, false); return shard_desc.not_null() && shard_desc->top_block_id().seqno() >= block_id.seqno(); } From f361e78b5a7e8dcda0945b599f9beae16587f93c Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 1 May 2025 15:54:21 +0400 Subject: [PATCH 237/388] [Tolk] Optimize arg_order in advance when it's safe This partially "reverts" the behavior of pragma 'compute-asm-ltr' from FunC, which was always on in Tolk. Now, if it's safe, for asm functions with arg_order, arguments are evaluated and placed onto the stack in a desired order. When it's unsafe (the purpose of this pragma, actually), arguments are evaluated left-to-right. --- tolk-tester/tests/asm-arg-order.tolk | 117 ++++++++++++++++++++++++- tolk-tester/tests/cells-slices.tolk | 49 ++++++++--- tolk-tester/tests/constants-tests.tolk | 2 +- tolk-tester/tests/strings-tests.tolk | 2 +- tolk/analyzer.cpp | 4 + tolk/codegen.cpp | 6 +- tolk/pipe-ast-to-legacy.cpp | 84 ++++++++++++++++-- tolk/tolk.h | 5 +- 8 files changed, 248 insertions(+), 21 deletions(-) diff --git a/tolk-tester/tests/asm-arg-order.tolk b/tolk-tester/tests/asm-arg-order.tolk index 841dc9564..6d55b2da3 100644 --- a/tolk-tester/tests/asm-arg-order.tolk +++ b/tolk-tester/tests/asm-arg-order.tolk @@ -120,6 +120,67 @@ fun test_new_dot(): (tuple, tuple) { return (t, t2); } +@pure +fun asmAPlus1TimesB(a: int, b: int): int + asm(b a) "1 ADDCONST MUL"; + +@pure +fun int.plus1TimesB(self, b: int): int + asm(b self) "1 ADDCONST MUL"; + +@pure +fun get2Pure() { return 2; } +@pure +fun get10Pure() { return 10; } + +fun get2Impure() { return 2; } +fun get10Impure() { return 10; } + +global g2: int; +global g10: int; + +fun setG2(v: int) { g2 = v; return v; } + +@method_id(27) +fun test27() { + return asmAPlus1TimesB(2, 10); +} + +@method_id(28) +fun test28() { + return asmAPlus1TimesB(get2Pure(), get10Pure()); +} + +@method_id(29) +fun test29() { + return asmAPlus1TimesB(get2Impure(), get10Impure()); +} + +@method_id(30) +fun test30() { + g2 = 2; + g10 = 10; + return asmAPlus1TimesB(g2, g10); +} + +@method_id(31) +fun test31() { + g2 = 2; + g10 = 10; + return asmAPlus1TimesB(g2 += 2, g10 += g2); +} + +@method_id(32) +fun test32() { + return 2.plus1TimesB(10); +} + +@method_id(33) +fun test33(x: int) { + return ((x += 10).plus1TimesB(2), (x += 20).plus1TimesB(x), ((x /= (g2=2)).plus1TimesB(x*g2)), setG2(7).plus1TimesB(g2)); +} + + fun main() { } @@ -137,6 +198,60 @@ fun main() { @testcase | 24 | | [ 11 22 33 44 55 ] [ 220 330 440 110 550 ] @testcase | 25 | | [ 22 33 ] [ 220 330 ] @testcase | 26 | | [ 11 22 33 ] [ 220 330 110 ] +@testcase | 27 | | 30 +@testcase | 28 | | 30 +@testcase | 29 | | 30 +@testcase | 30 | | 30 +@testcase | 31 | | 70 +@testcase | 32 | | 30 +@testcase | 33 | 0 | 22 930 480 56 + +@fif_codegen +""" + test27 PROC:<{ + 10 PUSHINT + 2 PUSHINT + 1 ADDCONST MUL + }> +""" + +@fif_codegen +""" + test28 PROC:<{ + get10Pure CALLDICT + get2Pure CALLDICT + 1 ADDCONST MUL + }> +""" + +@fif_codegen +""" + test29 PROC:<{ + get2Impure CALLDICT + get10Impure CALLDICT + SWAP + 1 ADDCONST MUL + }> +""" + +@fif_codegen +""" + test30 PROC:<{ + ... + g10 GETGLOB + g2 GETGLOB + 1 ADDCONST MUL + }> +""" + +@fif_codegen +""" + test32 PROC:<{ + 10 PUSHINT + 2 PUSHINT + 1 ADDCONST MUL + }> +""" -@code_hash 93068291567112337250118419287631047120002003622184251973082208096953112184588 +@code_hash 78671986831403867804966279036762472603849672357801214378328975900111280733054 */ diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index 63a823ed9..70c40b531 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -198,6 +198,37 @@ fun test111() { return (op1, q1, op2, q2); } +global g8: int; +global g10: int; +global g16: int; +global g32: int; + +@inline_ref +fun initGlobals() { + g8 = 8; + g10 = 10; + g16 = 16; + g32 = 32; +} + +@method_id(116) +fun test16() { + initGlobals(); + var b1 = beginCell().storeUint(g8, g16).storeUint(0xFF, g32).storeUint(g8, g16 * 2); + var b2 = beginCell().storeUint(8, 16).storeUint(0xFF, 32).storeUint(8, 16 * 2); + assert(b1.bitsCount() == b2.bitsCount(), 400); + var c1 = b1.endCell().beginParse(); + var c2 = b2.endCell().beginParse(); + assert(c1.bitsEqual(c2), 400); + assert(c1.loadUint(g16) == g8, 400); + assert(c1.loadUint(g32) == 0xFF, 400); + assert(c1.loadUint(2 * g16) == 8, 400); + return b1; + // 00140008000000ff00000008 + // 00140008000000ff00000008 +} + + fun main(): int { return 0; } @@ -217,21 +248,19 @@ fun main(): int { @testcase | 114 | -1 | -1 0 -1 @testcase | 114 | 0 | 0 0 0 @testcase | 115 | | 123 456 123 456 +@testcase | 116 | | BC{00140008000000ff00000008} Note, that since 'compute-asm-ltr' became on be default, chaining methods codegen is not quite optimal. @fif_codegen """ test6 PROC:<{ - NEWC // '0 - 1 PUSHINT // '0 '1=1 - SWAP // '1=1 '0 - 32 STU // '0 - 2 PUSHINT // '0 '4=2 - SWAP // '4=2 '0 - 32 STU // '0 - 3 PUSHINT // '0 '7=3 - SWAP // '7=3 '0 - 32 STU // '0 + 3 PUSHINT // '0=3 + 2 PUSHINT // '0=3 '1=2 + 1 PUSHINT // '0=3 '1=2 '2=1 + NEWC // '0=3 '1=2 '2=1 '3 + 32 STU // '0=3 '1=2 '3 + 32 STU // '0=3 '3 + 32 STU // '3 }> """ */ diff --git a/tolk-tester/tests/constants-tests.tolk b/tolk-tester/tests/constants-tests.tolk index ba213093b..ed510c13c 100644 --- a/tolk-tester/tests/constants-tests.tolk +++ b/tolk-tester/tests/constants-tests.tolk @@ -118,5 +118,5 @@ fun main() { @testcase | 104 | | 1 1 2 @testcase | 105 | | -1 0 7 48 -@code_hash 80040709432962217077682091261201772251141677197885524779745956896218368868623 +@code_hash 49556957179018386976033482229516007597784982050169632168468608374010225644988 */ diff --git a/tolk-tester/tests/strings-tests.tolk b/tolk-tester/tests/strings-tests.tolk index 131659571..9feed3a1c 100644 --- a/tolk-tester/tests/strings-tests.tolk +++ b/tolk-tester/tests/strings-tests.tolk @@ -78,5 +78,5 @@ fun test1() { @testcase | 0 | | 0 @testcase | 101 | | [ 65 66 67 68 ] -@code_hash 38184847030631877916087987911699475358017315230885358090110033079289166112584 +@code_hash 55974318379341089957961227475446008591490555692181953973486962465702042912657 */ diff --git a/tolk/analyzer.cpp b/tolk/analyzer.cpp index 480165a16..f9721f5f1 100644 --- a/tolk/analyzer.cpp +++ b/tolk/analyzer.cpp @@ -851,6 +851,10 @@ void Op::set_impure_flag() { flags |= _Impure; } +void Op::set_arg_order_already_equals_asm_flag() { + flags |= _ArgOrderAlreadyEqualsAsm; +} + bool Op::mark_noreturn() { switch (cl) { case _Nop: diff --git a/tolk/codegen.cpp b/tolk/codegen.cpp index 6c970269e..ff6c39676 100644 --- a/tolk/codegen.cpp +++ b/tolk/codegen.cpp @@ -453,8 +453,10 @@ bool Op::generate_code_step(Stack& stack) { if (disabled()) { return true; } - // f_sym can be nullptr for Op::_CallInd (invoke a variable, not a function) - const std::vector* arg_order = f_sym ? f_sym->get_arg_order() : nullptr; + // f_sym can be nullptr for Op::_CallInd (invoke a variable, not a function); + // if f has arg_order, when it's safe, the compiler evaluates arguments in that order in advance (for fewer stack manipulations); + // when it's unsafe, arguments are evaluated left-to-right, and we need to match asm arg_order here + const std::vector* arg_order = f_sym && !arg_order_already_equals_asm() ? f_sym->get_arg_order() : nullptr; const std::vector* ret_order = f_sym ? f_sym->get_ret_order() : nullptr; tolk_assert(!arg_order || arg_order->size() == right.size()); tolk_assert(!ret_order || ret_order->size() == left.size()); diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index ecabb5c71..a45a4ccb6 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -278,6 +278,53 @@ class LValContext { } }; +// the purpose of this class is having a call `f(a1,a2,...)` when f has asm arg_order, to check +// whether it's safe to rearrange arguments (to evaluate them in arg_order right here for fewer stack manipulations) +// or it's unsafe, and we should evaluate them left-to-right; +// example: `f(1,2,3)` / `b.storeUint(2,32)` is safe +// example: `f(x,x+=5,x)` / `f(impureF1(), global_var)` is unsafe +class CheckReorderingForAsmArgOrderIsSafeVisitor final : public ASTVisitorFunctionBody { + bool has_side_effects = false; + +protected: + void visit(V v) override { + has_side_effects |= v->fun_maybe == nullptr || !v->fun_maybe->is_marked_as_pure(); + parent::visit(v); + } + + void visit(V v) override { + has_side_effects = true; + parent::visit(v); + } + + void visit(V v) override { + has_side_effects = true; + parent::visit(v); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + tolk_assert(false); + } + + static bool is_safe_to_reorder(V v) { + for (const LocalVarData& param : v->fun_maybe->parameters) { + if (param.declared_type->get_width_on_stack() != 1) { + return false; + } + } + + CheckReorderingForAsmArgOrderIsSafeVisitor visitor; + for (int i = 0; i < v->get_num_args(); ++i) { + visitor.ASTVisitorFunctionBody::visit(v->get_arg(i)->get_expr()); + } + if (v->dot_obj_is_self) { + visitor.ASTVisitorFunctionBody::visit(v->get_self_obj()); + } + return !visitor.has_side_effects; + } +}; + // given `{some_expr}!`, return some_expr static AnyExprV unwrap_not_null_operator(AnyExprV v) { while (auto v_notnull = v->try_as()) { @@ -469,12 +516,16 @@ static std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_t } static std::vector gen_op_call(CodeBlob& code, TypePtr ret_type, SrcLocation loc, - std::vector&& args_vars, FunctionPtr fun_ref, const char* debug_desc) { + std::vector&& args_vars, FunctionPtr fun_ref, const char* debug_desc, + bool arg_order_already_equals_asm = false) { std::vector rvect = code.create_tmp_var(ret_type, loc, debug_desc); Op& op = code.emplace_back(loc, Op::_Call, rvect, std::move(args_vars), fun_ref); if (!fun_ref->is_marked_as_pure()) { op.set_impure_flag(); } + if (arg_order_already_equals_asm) { + op.set_arg_order_already_equals_asm_flag(); + } return rvect; } @@ -1234,9 +1285,31 @@ static std::vector process_function_call(V v, Code for (int i = 0; i < v->get_num_args(); ++i) { args.push_back(v->get_arg(i)->get_expr()); } + // the purpose of tensor_tt ("tensor target type") is to transition `null` to `(int, int)?` and so on // the purpose of calling `pre_compile_tensor_inner` is to have 0-th IR vars to handle return self std::vector params_types = fun_ref->inferred_full_type->try_as()->params_types; + + // if fun_ref has asm arg_order, maybe it's safe to swap arguments here (to put them onto a stack in the right way); + // (if it's not safe, arguments are evaluated left-to-right, involving stack transformations later) + bool arg_order_already_equals_asm = false; + int asm_self_idx = 0; + if (!fun_ref->arg_order.empty() && CheckReorderingForAsmArgOrderIsSafeVisitor::is_safe_to_reorder(v)) { + std::vector new_args(args.size()); + std::vector new_params_types(params_types.size()); + for (int i = 0; i < static_cast(fun_ref->arg_order.size()); ++i) { + int real_i = fun_ref->arg_order[i]; + new_args[i] = args[real_i]; + new_params_types[i] = params_types[real_i]; + if (real_i == 0) { + asm_self_idx = i; + } + } + args = std::move(new_args); + params_types = std::move(new_params_types); + arg_order_already_equals_asm = true; + } + const TypeDataTensor* tensor_tt = TypeDataTensor::create(std::move(params_types))->try_as(); std::vector> vars_per_arg = pre_compile_tensor_inner(code, args, tensor_tt, nullptr); @@ -1263,20 +1336,21 @@ static std::vector process_function_call(V v, Code for (const std::vector& list : vars_per_arg) { args_vars.insert(args_vars.end(), list.cbegin(), list.cend()); } - std::vector rvect_apply = gen_op_call(code, op_call_type, v->loc, std::move(args_vars), fun_ref, "(fun-call)"); + std::vector rvect_apply = gen_op_call(code, op_call_type, v->loc, std::move(args_vars), fun_ref, "(fun-call)", arg_order_already_equals_asm); if (fun_ref->has_mutate_params()) { LValContext local_lval; std::vector left; for (int i = 0; i < delta_self + v->get_num_args(); ++i) { + int real_i = arg_order_already_equals_asm ? i == 0 && delta_self ? asm_self_idx : fun_ref->arg_order[i - delta_self] : i; if (fun_ref->parameters[i].is_mutate_parameter()) { - AnyExprV arg_i = obj_leftmost && i == 0 ? obj_leftmost : args[i]; + AnyExprV arg_i = obj_leftmost && i == 0 ? obj_leftmost : args[real_i]; tolk_assert(arg_i->is_lvalue || i == 0); if (arg_i->is_lvalue) { std::vector ith_var_idx = pre_compile_expr(arg_i, code, nullptr, &local_lval); left.insert(left.end(), ith_var_idx.begin(), ith_var_idx.end()); } else { - left.insert(left.end(), vars_per_arg[0].begin(), vars_per_arg[0].end()); + left.insert(left.end(), vars_per_arg[asm_self_idx].begin(), vars_per_arg[asm_self_idx].end()); } } } @@ -1292,7 +1366,7 @@ static std::vector process_function_call(V v, Code if (obj_leftmost->is_lvalue) { // to handle if obj is global var, potentially re-assigned inside a chain rvect_apply = pre_compile_expr(obj_leftmost, code, nullptr); } else { // temporary object, not lvalue, pre_compile_expr - rvect_apply = vars_per_arg[0]; + rvect_apply = vars_per_arg[asm_self_idx]; } } diff --git a/tolk/tolk.h b/tolk/tolk.h index 55d9d7677..ad2a5898d 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -287,7 +287,7 @@ struct Op { _SliceConst, }; OpKind cl; - enum { _Disabled = 1, _NoReturn = 4, _Impure = 24 }; + enum { _Disabled = 1, _NoReturn = 2, _Impure = 4, _ArgOrderAlreadyEqualsAsm = 8 }; int flags; std::unique_ptr next; FunctionPtr f_sym = nullptr; @@ -347,6 +347,9 @@ struct Op { bool impure() const { return flags & _Impure; } void set_impure_flag(); + bool arg_order_already_equals_asm() const { return flags & _ArgOrderAlreadyEqualsAsm; } + void set_arg_order_already_equals_asm_flag(); + void show(std::ostream& os, const std::vector& vars, std::string pfx = "", int mode = 0) const; void show_var_list(std::ostream& os, const std::vector& idx_list, const std::vector& vars) const; void show_var_list(std::ostream& os, const std::vector& list, const std::vector& vars) const; From 50b8873d47045466b7dd36683e1dba87529d32f8 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 10 Apr 2025 22:25:58 +0400 Subject: [PATCH 238/388] [Tolk] Bump version to v0.12 --- crypto/smartcont/tolk-stdlib/common.tolk | 2 +- crypto/smartcont/tolk-stdlib/gas-payments.tolk | 2 +- crypto/smartcont/tolk-stdlib/lisp-lists.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-dicts.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk | 2 +- tolk/tolk-version.h | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index f22f4695f..8ff2010bc 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -1,7 +1,7 @@ // Standard library for Tolk (LGPL licence). // It contains common functions that are available out of the box, the user doesn't have to import anything. // More specific functions are required to be imported explicitly, like "@stdlib/tvm-dicts". -tolk 0.11 +tolk 0.12 /// In Tolk v1.x there would be a type `map`. /// Currently, working with dictionaries is still low-level, with raw cells. diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk index b0cab4de8..2c35faf93 100644 --- a/crypto/smartcont/tolk-stdlib/gas-payments.tolk +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.11 +tolk 0.12 /** Gas and payment related primitives. diff --git a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk index 9dd6c1da9..ada7a72bc 100644 --- a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk +++ b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.11 +tolk 0.12 /** Lisp-style lists are nested 2-elements tuples: `[1, [2, [3, null]]]` represents list `[1, 2, 3]`. diff --git a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk index 5de989aed..90b392f61 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.11 +tolk 0.12 /** Dictionaries are represented as `cell` data type (cells can store anything, dicts in particular). diff --git a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk index 075514539..47b4a300e 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.11 +tolk 0.12 /// Usually `c3` has a continuation initialized by the whole code of the contract. It is used for function calls. /// The primitive returns the current value of `c3`. diff --git a/tolk/tolk-version.h b/tolk/tolk-version.h index b8f2893eb..dd9134f45 100644 --- a/tolk/tolk-version.h +++ b/tolk/tolk-version.h @@ -18,6 +18,6 @@ namespace tolk { -constexpr const char* TOLK_VERSION = "0.11.0"; +constexpr const char* TOLK_VERSION = "0.12.0"; } // namespace tolk From 0b024a648d8fdcae43875a82f2b918bc183a70c5 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 9 May 2025 19:02:08 +0300 Subject: [PATCH 239/388] Improve processing block candidate broadcasts, prepare to broadcast shard blocks without signatures (#1642) --- create-hardfork/create-hardfork.cpp | 7 +- tdutils/CMakeLists.txt | 1 + tdutils/td/utils/LRUCache.h | 123 ++++++++++++++++++++++ test/test-ton-collator.cpp | 7 +- validator/full-node-private-overlay.cpp | 4 +- validator/full-node-shard.cpp | 4 +- validator/full-node.cpp | 36 +++---- validator/full-node.hpp | 3 +- validator/impl/accept-block.cpp | 5 + validator/impl/top-shard-descr.hpp | 3 + validator/interfaces/shard-block.h | 1 + validator/interfaces/validator-manager.h | 2 +- validator/manager-disk.cpp | 14 +-- validator/manager-disk.hpp | 9 +- validator/manager-hardfork.hpp | 11 +- validator/manager.cpp | 124 +++++++++++++---------- validator/manager.hpp | 19 ++-- validator/validator-group.cpp | 27 +++++ validator/validator-group.hpp | 8 +- validator/validator.h | 9 +- 20 files changed, 299 insertions(+), 118 deletions(-) create mode 100644 tdutils/td/utils/LRUCache.h diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index 72bffae56..20e58f006 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -216,8 +216,9 @@ class HardforkCreator : public td::actor::Actor { std::move(msg), 0); } for (auto &topmsg : top_shard_descrs_) { - td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManager::new_shard_block, ton::BlockIdExt{}, - 0, std::move(topmsg)); + td::actor::send_closure(validator_manager_, + &ton::validator::ValidatorManager::new_shard_block_description_broadcast, + ton::BlockIdExt{}, 0, std::move(topmsg)); } class Callback : public ton::validator::ValidatorManagerInterface::Callback { private: @@ -246,7 +247,7 @@ class HardforkCreator : public td::actor::Actor { void send_shard_block_info(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::BufferSlice data) override { } void send_block_candidate(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) override { + td::BufferSlice data, int mode) override { } void send_broadcast(ton::BlockBroadcast broadcast, int mode) override { } diff --git a/tdutils/CMakeLists.txt b/tdutils/CMakeLists.txt index 716ec322a..b4a7fec21 100644 --- a/tdutils/CMakeLists.txt +++ b/tdutils/CMakeLists.txt @@ -214,6 +214,7 @@ set(TDUTILS_SOURCE td/utils/invoke.h td/utils/JsonBuilder.h td/utils/List.h + td/utils/LRUCache.h td/utils/logging.h td/utils/MemoryLog.h td/utils/misc.h diff --git a/tdutils/td/utils/LRUCache.h b/tdutils/td/utils/LRUCache.h new file mode 100644 index 000000000..5c30f6693 --- /dev/null +++ b/tdutils/td/utils/LRUCache.h @@ -0,0 +1,123 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include +#include +#include "List.h" +#include "check.h" + +namespace td { + +template +class LRUCache { + public: + explicit LRUCache(size_t max_size) : max_size_(max_size) { + CHECK(max_size_ > 0); + } + LRUCache(const LRUCache&) = delete; + LRUCache& operator=(const LRUCache&) = delete; + + V* get_if_exists(const K& key, bool update = true) { + auto it = cache_.find(key); + if (it == cache_.end()) { + return nullptr; + } + Entry* entry = it->get(); + if (update) { + entry->remove(); + lru_.put(entry); + } + return &entry->value; + } + + bool contains(const K& key) const { + return cache_.contains(key); + } + + bool put(const K& key, V value, bool update = true) { + bool added = false; + auto it = cache_.find(key); + if (it == cache_.end()) { + update = true; + it = cache_.insert(std::make_unique(key, std::move(value))).first; + added = true; + } else { + (*it)->value = std::move(value); + if (update) { + (*it)->remove(); + } + } + if (update) { + lru_.put(it->get()); + cleanup(); + } + return added; + } + + V& get(const K& key, bool update = true) { + auto it = cache_.find(key); + if (it == cache_.end()) { + update = true; + it = cache_.insert(std::make_unique(key)).first; + } else if (update) { + (*it)->remove(); + } + V& result = (*it)->value; + if (update) { + lru_.put(it->get()); + cleanup(); + } + return result; + } + + private: + struct Entry : ListNode { + explicit Entry(K key) : key(std::move(key)) { + } + Entry(K key, V value) : key(std::move(key)), value(std::move(value)) { + } + K key; + V value; + }; + struct Cmp { + using is_transparent = void; + bool operator()(const std::unique_ptr& a, const std::unique_ptr& b) const { + return a->key < b->key; + } + bool operator()(const std::unique_ptr& a, const K& b) const { + return a->key < b; + } + bool operator()(const K& a, const std::unique_ptr& b) const { + return a < b->key; + } + }; + std::set, Cmp> cache_; + ListNode lru_; + size_t max_size_; + + void cleanup() { + while (cache_.size() > max_size_) { + auto to_remove = (Entry*)lru_.get(); + CHECK(to_remove); + to_remove->remove(); + cache_.erase(cache_.find(to_remove->key)); + } + } +}; + +} // namespace td diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index 0fde1e68d..a062ddbe5 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -303,8 +303,9 @@ class TestNode : public td::actor::Actor { std::move(msg), 0); } for (auto &topmsg : top_shard_descrs_) { - td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManager::new_shard_block, ton::BlockIdExt{}, - 0, std::move(topmsg)); + td::actor::send_closure(validator_manager_, + &ton::validator::ValidatorManager::new_shard_block_description_broadcast, + ton::BlockIdExt{}, 0, std::move(topmsg)); } class Callback : public ton::validator::ValidatorManagerInterface::Callback { private: @@ -347,7 +348,7 @@ class TestNode : public td::actor::Actor { } } void send_block_candidate(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) override { + td::BufferSlice data, int mode) override { } void send_broadcast(ton::BlockBroadcast broadcast, int mode) override { } diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp index 1fb9c1507..386eba94a 100644 --- a/validator/full-node-private-overlay.cpp +++ b/validator/full-node-private-overlay.cpp @@ -49,8 +49,8 @@ void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, ton_api:: BlockIdExt block_id = create_block_id(query.block_->block_); VLOG(FULL_NODE_DEBUG) << "Received newShardBlockBroadcast in private overlay from " << src << ": " << block_id.to_str(); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block, block_id, - query.block_->cc_seqno_, std::move(query.block_->data_)); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block_description_broadcast, + block_id, query.block_->cc_seqno_, std::move(query.block_->data_)); } void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index a8ddb338a..2543431a8 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -780,8 +780,8 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ex void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query) { BlockIdExt block_id = create_block_id(query.block_->block_); VLOG(FULL_NODE_DEBUG) << "Received newShardBlockBroadcast from " << src << ": " << block_id.to_str(); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block, block_id, - query.block_->cc_seqno_, std::move(query.block_->data_)); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block_description_broadcast, + block_id, query.block_->cc_seqno_, std::move(query.block_->data_)); } void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { diff --git a/validator/full-node.cpp b/validator/full-node.cpp index e1951c365..953e0c599 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -313,18 +313,20 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s } void FullNodeImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) { - send_block_candidate_broadcast_to_custom_overlays(block_id, cc_seqno, validator_set_hash, data); - auto shard = get_shard(ShardIdFull{masterchainId, shardIdAll}); - if (shard.empty()) { - VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; - return; + td::BufferSlice data, int mode) { + if (mode & broadcast_mode_custom) { + send_block_candidate_broadcast_to_custom_overlays(block_id, cc_seqno, validator_set_hash, data); } - if (!private_block_overlays_.empty()) { + if ((mode & broadcast_mode_private_block) && !private_block_overlays_.empty()) { td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_block_candidate, block_id, cc_seqno, validator_set_hash, data.clone()); } - if (broadcast_block_candidates_in_public_overlay_) { + if (mode & broadcast_mode_public) { + auto shard = get_shard(ShardIdFull{masterchainId, shardIdAll}); + if (shard.empty()) { + VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; + return; + } td::actor::send_closure(shard, &FullNodeShard::send_block_candidate, block_id, cc_seqno, validator_set_hash, std::move(data)); } @@ -334,11 +336,6 @@ void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, int mode) { if (mode & broadcast_mode_custom) { send_block_broadcast_to_custom_overlays(broadcast); } - auto shard = get_shard(broadcast.block_id.shard_full()); - if (shard.empty()) { - VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; - return; - } if (mode & broadcast_mode_private_block) { if (!private_block_overlays_.empty()) { td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_broadcast, @@ -346,6 +343,11 @@ void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, int mode) { } } if (mode & broadcast_mode_public) { + auto shard = get_shard(broadcast.block_id.shard_full()); + if (shard.empty()) { + VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; + return; + } td::actor::send_closure(shard, &FullNodeShard::send_broadcast, std::move(broadcast)); } } @@ -556,7 +558,7 @@ void FullNodeImpl::send_validator_telemetry(PublicKeyHash key, tl_object_ptr R) { if (R.is_error()) { if (R.error().code() == ErrorCode::notready) { @@ -572,7 +574,7 @@ void FullNodeImpl::process_block_candidate_broadcast(BlockIdExt block_id, Catcha td::uint32 validator_set_hash, td::BufferSlice data) { send_block_candidate_broadcast_to_custom_overlays(block_id, cc_seqno, validator_set_hash, data); // ignore cc_seqno and validator_hash for now - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_block_candidate, block_id, + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_block_candidate_broadcast, block_id, std::move(data)); } @@ -630,9 +632,9 @@ void FullNodeImpl::start_up() { td::actor::send_closure(id_, &FullNodeImpl::send_shard_block_info, block_id, cc_seqno, std::move(data)); } void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) override { + td::BufferSlice data, int mode) override { td::actor::send_closure(id_, &FullNodeImpl::send_block_candidate, block_id, cc_seqno, validator_set_hash, - std::move(data)); + std::move(data), mode); } void send_broadcast(BlockBroadcast broadcast, int mode) override { td::actor::send_closure(id_, &FullNodeImpl::send_broadcast, std::move(broadcast), mode); diff --git a/validator/full-node.hpp b/validator/full-node.hpp index b4c79363e..e2d5c72b0 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -66,7 +66,7 @@ class FullNodeImpl : public FullNode { void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data); void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqnp, td::BufferSlice data); void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data); + td::BufferSlice data, int mode); void send_broadcast(BlockBroadcast broadcast, int mode); void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, @@ -144,7 +144,6 @@ class FullNodeImpl : public FullNode { FullNodeOptions opts_; std::map> private_block_overlays_; - bool broadcast_block_candidates_in_public_overlay_ = false; struct CustomOverlayInfo { CustomOverlayParams params_; diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index de48626d7..fa68580d4 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -951,6 +951,11 @@ void AcceptBlockQuery::applied() { // do not wait for answer td::actor::send_closure_later(manager_, &ValidatorManager::send_block_broadcast, std::move(b), send_broadcast_mode_); + // Do this for shard blocks later: + // td::actor::send_closure(manager_, &ValidatorManager::send_block_candidate_broadcast, id_, + // validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), + // std::move(b.data), send_broadcast_mode_); + finish_query(); } diff --git a/validator/impl/top-shard-descr.hpp b/validator/impl/top-shard-descr.hpp index 5c866a6e3..e1fd5778f 100644 --- a/validator/impl/top-shard-descr.hpp +++ b/validator/impl/top-shard-descr.hpp @@ -74,6 +74,9 @@ class ShardTopBlockDescrQ final : public ShardTopBlockDescrQBase { std::size_t size() const { return chain_blk_ids_.size(); } + const std::vector& get_chain_blocks() const override { + return chain_blk_ids_; + } UnixTime generated_at() const override { return gen_utime_; } diff --git a/validator/interfaces/shard-block.h b/validator/interfaces/shard-block.h index e88a970bb..6e31cc136 100644 --- a/validator/interfaces/shard-block.h +++ b/validator/interfaces/shard-block.h @@ -35,6 +35,7 @@ class ShardTopBlockDescription : public td::CntObject { virtual bool after_split() const = 0; virtual bool after_merge() const = 0; virtual CatchainSeqno catchain_seqno() const = 0; + virtual const std::vector& get_chain_blocks() const = 0; virtual UnixTime generated_at() const = 0; // if method returns false this shard block description is discarded diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 1f08c4ef5..077ce7849 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -104,7 +104,7 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::Promise promise) = 0; virtual void send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) = 0; + td::BufferSlice data, int mode) = 0; virtual void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 8e4a4d086..82081cd11 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -64,7 +64,7 @@ void ValidatorManagerImpl::validate_block(ReceivedBlock block, td::Promise promise) { +void ValidatorManagerImpl::new_block_broadcast(BlockBroadcast broadcast, td::Promise promise) { UNREACHABLE(); } @@ -276,7 +276,8 @@ void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { } } -void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { +void ValidatorManagerImpl::new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) { if (!last_masterchain_block_handle_) { shard_blocks_raw_.push_back(std::move(data)); return; @@ -528,7 +529,7 @@ void ValidatorManagerImpl::get_shard_blocks(BlockIdExt masterchain_block_id, } if (!shard_blocks_raw_.empty()) { for (auto &raw : shard_blocks_raw_) { - new_shard_block(BlockIdExt{}, 0, std::move(raw)); + new_shard_block_description_broadcast(BlockIdExt{}, 0, std::move(raw)); } shard_blocks_raw_.clear(); } @@ -786,8 +787,9 @@ void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate can } void ValidatorManagerImpl::send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, - td::uint32 validator_set_hash, td::BufferSlice data) { - callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data)); + td::uint32 validator_set_hash, td::BufferSlice data, + int mode) { + callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data), mode); } void ValidatorManagerImpl::write_handle(BlockHandle handle, td::Promise promise) { @@ -947,7 +949,7 @@ void ValidatorManagerImpl::update_shard_blocks() { } if (!shard_blocks_raw_.empty()) { for (auto &raw : shard_blocks_raw_) { - new_shard_block(BlockIdExt{}, 0, std::move(raw)); + new_shard_block_description_broadcast(BlockIdExt{}, 0, std::move(raw)); } shard_blocks_raw_.clear(); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 8526ffa4d..cdf6c1cae 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -96,7 +96,7 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } void validate_block(ReceivedBlock block, td::Promise promise) override; - void prevalidate_block(BlockBroadcast broadcast, td::Promise promise) override; + void new_block_broadcast(BlockBroadcast broadcast, td::Promise promise) override; //void create_validate_block(BlockId block, td::BufferSlice data, td::Promise promise) = 0; void sync_complete(td::Promise promise) override; @@ -133,8 +133,9 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } void new_ihr_message(td::BufferSlice data) override; - void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; - void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) override { + void new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) override; + void new_block_candidate_broadcast(BlockIdExt block_id, td::BufferSlice data) override { } void add_ext_server_id(adnl::AdnlNodeIdShort id) override { @@ -186,7 +187,7 @@ class ValidatorManagerImpl : public ValidatorManager { void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::Promise promise) override; void send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) override; + td::BufferSlice data, int mode) override; void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index b2bd9250b..fbb40901f 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -107,7 +107,7 @@ class ValidatorManagerImpl : public ValidatorManager { void validate_block(ReceivedBlock block, td::Promise promise) override { UNREACHABLE(); } - void prevalidate_block(BlockBroadcast broadcast, td::Promise promise) override { + void new_block_broadcast(BlockBroadcast broadcast, td::Promise promise) override { UNREACHABLE(); } @@ -154,10 +154,11 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } void new_ihr_message(td::BufferSlice data) override; - void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override { + void new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) override { UNREACHABLE(); } - void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) override { + void new_block_candidate_broadcast(BlockIdExt block_id, td::BufferSlice data) override { UNREACHABLE(); } @@ -228,8 +229,8 @@ class ValidatorManagerImpl : public ValidatorManager { promise.set_value(td::Unit()); } void send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) { - callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data)); + td::BufferSlice data, int mode) { + callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data), mode); } void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, diff --git a/validator/manager.cpp b/validator/manager.cpp index 85f3a30ab..a94f9c07c 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -210,7 +210,7 @@ void ValidatorManagerImpl::validate_block(ReceivedBlock block, td::Promise promise) { +void ValidatorManagerImpl::new_block_broadcast(BlockBroadcast broadcast, td::Promise promise) { if (!started_) { promise.set_error(td::Status::Error(ErrorCode::notready, "node not started")); return; @@ -487,20 +487,22 @@ void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { } } -void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { - if (!is_validator() && !cached_block_candidates_.count(block_id)) { +void ValidatorManagerImpl::new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) { + if (!last_masterchain_block_handle_ || !started_) { + VLOG(VALIDATOR_DEBUG) << "dropping shard block description broadcast: not inited"; return; } - if (!last_masterchain_block_handle_) { - VLOG(VALIDATOR_DEBUG) << "dropping top shard block broadcast: not inited"; + if (!is_validator() && !opts_->need_monitor(block_id.shard_full(), last_masterchain_state_)) { return; } - if (!started_) { + if (cached_checked_shard_block_descriptions_.contains(block_id)) { + VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block description broadcast"; return; } auto it = shard_blocks_.find(ShardTopBlockDescriptionId{block_id.shard_full(), cc_seqno}); if (it != shard_blocks_.end() && block_id.id.seqno <= it->second->block_id().id.seqno) { - VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block broadcast"; + VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block description broadcast"; return; } auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { @@ -514,25 +516,37 @@ void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc actor_id(this), td::Timestamp::in(2.0), std::move(P)); } -void ValidatorManagerImpl::new_block_candidate(BlockIdExt block_id, td::BufferSlice data) { - if (!last_masterchain_block_handle_) { +void ValidatorManagerImpl::new_block_candidate_broadcast(BlockIdExt block_id, td::BufferSlice data) { + if (!last_masterchain_block_handle_ || !started_) { VLOG(VALIDATOR_DEBUG) << "dropping top shard block broadcast: not inited"; return; } - if (!started_) { - return; - } if (!need_monitor(block_id.shard_full())) { VLOG(VALIDATOR_DEBUG) << "dropping block candidate broadcast: not monitoring shard"; return; } - add_cached_block_candidate(ReceivedBlock{block_id, std::move(data)}); + add_cached_block_data(block_id, std::move(data)); } void ValidatorManagerImpl::add_shard_block_description(td::Ref desc) { + for (const BlockIdExt &block_id : desc->get_chain_blocks()) { + if (cached_checked_shard_block_descriptions_.put(block_id, td::Unit())) { + if (cached_block_data_.contains(block_id)) { + wait_block_data_short(block_id, 0, td::Timestamp::in(60.0), [id = block_id](td::Result> R) { + if (R.is_error()) { + LOG(WARNING) << "Failed to store block data from cache for new shard block description " << id.to_str() + << " : " << R.move_as_error(); + } + }); + } + } + } if (!desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { return; } + if (!is_validator()) { + return; + } auto it = shard_blocks_.find(ShardTopBlockDescriptionId{desc->shard(), desc->catchain_seqno()}); if (it != shard_blocks_.end() && desc->block_id().id.seqno <= it->second->block_id().id.seqno) { VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block broadcast"; @@ -555,33 +569,41 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refsecond.actor_, &WaitBlockData::loaded_block_data, r_block.move_as_ok()); - } + td::BufferSlice& block_data = cached_block_data_.get(block_id); + if (!block_data.empty()) { + return; + } + block_data = std::move(data); + { + auto it = wait_block_data_.find(block_id); + if (it != wait_block_data_.end()) { + auto r_block = create_block(ReceivedBlock{block_id, block_data.clone()}); + if (r_block.is_ok()) { + td::actor::send_closure(it->second.actor_, &WaitBlockData::loaded_block_data, r_block.move_as_ok()); + } else { + LOG(WARNING) << "Failed to parse cached block " << block_id.to_str() << " : " << r_block.move_as_error(); } } - { - auto it = wait_state_.find(id); - if (it != wait_state_.end()) { - // Proof link is not ready at this point, but this will force WaitBlockState to redo send_get_proof_link_request - td::actor::send_closure(it->second.actor_, &WaitBlockState::after_get_proof_link); - } + } + { + auto it = wait_state_.find(block_id); + if (it != wait_state_.end()) { + // Proof link is not ready at this point, but this will force WaitBlockState to redo send_get_proof_link_request + td::actor::send_closure(it->second.actor_, &WaitBlockState::after_get_proof_link); } } - if (cached_block_candidates_lru_.size() > max_cached_candidates()) { - CHECK(cached_block_candidates_.erase(cached_block_candidates_lru_.front())); - cached_block_candidates_lru_.pop_front(); + + if (cached_checked_shard_block_descriptions_.contains(block_id)) { + wait_block_data_short(block_id, 0, td::Timestamp::in(60.0), [id = block_id](td::Result> R) { + if (R.is_error()) { + LOG(WARNING) << "Failed to store block data from cache for checked shard block description " << id.to_str() + << " : " << R.move_as_error(); + } + }); } } @@ -1058,9 +1080,8 @@ void ValidatorManagerImpl::get_block_candidate_from_db(PublicKey source, BlockId } void ValidatorManagerImpl::get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) { - auto it = cached_block_candidates_.find(id); - if (it != cached_block_candidates_.end()) { - promise.set_result(it->second.data.clone()); + if (auto cached = cached_block_data_.get_if_exists(id, false)) { + promise.set_result(cached->clone()); return; } td::actor::send_closure(db_, &Db::get_block_candidate_by_block_id, id, @@ -1307,14 +1328,15 @@ void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate can PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}, candidate.collated_file_hash); } if (!id.is_masterchain()) { - add_cached_block_candidate(ReceivedBlock{id, candidate.data.clone()}); + add_cached_block_data(id, candidate.data.clone()); } td::actor::send_closure(db_, &Db::store_block_candidate, std::move(candidate), std::move(promise)); } void ValidatorManagerImpl::send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, - td::uint32 validator_set_hash, td::BufferSlice data) { - callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data)); + td::uint32 validator_set_hash, td::BufferSlice data, + int mode) { + callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data), mode); } void ValidatorManagerImpl::write_handle(BlockHandle handle, td::Promise promise) { @@ -1570,13 +1592,11 @@ void ValidatorManagerImpl::get_last_liteserver_state_block( void ValidatorManagerImpl::send_get_block_request(BlockIdExt id, td::uint32 priority, td::Promise promise) { - { - auto it = cached_block_candidates_.find(id); - if (it != cached_block_candidates_.end()) { - LOG(DEBUG) << "send_get_block_request: got result from candidates cache for " << id.to_str(); - return promise.set_value(it->second.clone()); - } + if (auto cached = cached_block_data_.get_if_exists(id, false)) { + LOG(DEBUG) << "send_get_block_request: got result from block data cache for " << id.to_str(); + return promise.set_value(ReceivedBlock{id, cached->clone()}); } + LOG(ERROR) << "XXXX download " << id.id.to_str(); callback_->download_block(id, priority, td::Timestamp::in(10.0), std::move(promise)); } @@ -1600,19 +1620,19 @@ void ValidatorManagerImpl::send_get_block_proof_request(BlockIdExt block_id, td: void ValidatorManagerImpl::send_get_block_proof_link_request(BlockIdExt block_id, td::uint32 priority, td::Promise promise) { if (!block_id.is_masterchain()) { - auto it = cached_block_candidates_.find(block_id); - if (it != cached_block_candidates_.end()) { - // Proof link can be created from the cached block candidate - LOG(DEBUG) << "send_get_block_proof_link_request: creating proof link from cached caniddate for " + if (auto cached = cached_block_data_.get_if_exists(block_id, false)) { + // Proof link can be created from the cached block data + LOG(DEBUG) << "send_get_block_proof_link_request: creating proof link from cached block data for " << block_id.to_str(); - TRY_RESULT_PROMISE_PREFIX(promise, block_root, vm::std_boc_deserialize(it->second.data), + TRY_RESULT_PROMISE_PREFIX(promise, block_root, vm::std_boc_deserialize(*cached), "failed to create proof link: "); - TRY_RESULT_PROMISE_PREFIX(promise, proof_link, WaitBlockData::generate_proof_link(it->second.id, block_root), + TRY_RESULT_PROMISE_PREFIX(promise, proof_link, WaitBlockData::generate_proof_link(block_id, block_root), "failed to create proof link: "); promise.set_result(std::move(proof_link)); return; } } + LOG(ERROR) << "XXXX download link " << block_id.id.to_str(); callback_->download_block_proof_link(block_id, priority, td::Timestamp::in(10.0), std::move(promise)); } diff --git a/validator/manager.hpp b/validator/manager.hpp index c406d36b0..a321684b7 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -36,6 +36,7 @@ #include "queue-size-counter.hpp" #include "validator-telemetry.hpp" #include "impl/candidates-buffer.hpp" +#include "td/utils/LRUCache.h" #include #include @@ -234,8 +235,8 @@ class ValidatorManagerImpl : public ValidatorManager { // DATA FOR COLLATOR std::map> shard_blocks_; - std::map cached_block_candidates_; - std::list cached_block_candidates_lru_; + td::LRUCache cached_block_data_{/* max_size = */ 128}; + td::LRUCache cached_checked_shard_block_descriptions_{/* max_size = */ 1024}; struct ExtMessages { std::map, std::unique_ptr>> ext_messages_; @@ -366,7 +367,7 @@ class ValidatorManagerImpl : public ValidatorManager { void validate_block_proof_rel(BlockIdExt block_id, BlockIdExt rel_block_id, td::BufferSlice proof, td::Promise promise) override; void validate_block(ReceivedBlock block, td::Promise promise) override; - void prevalidate_block(BlockBroadcast broadcast, td::Promise promise) override; + void new_block_broadcast(BlockBroadcast broadcast, td::Promise promise) override; void validated_block_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno); //void create_validate_block(BlockId block, td::BufferSlice data, td::Promise promise) = 0; @@ -396,8 +397,9 @@ class ValidatorManagerImpl : public ValidatorManager { void check_external_message(td::BufferSlice data, td::Promise> promise) override; void new_ihr_message(td::BufferSlice data) override; - void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; - void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) override; + void new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) override; + void new_block_candidate_broadcast(BlockIdExt block_id, td::BufferSlice data) override; void add_ext_server_id(adnl::AdnlNodeIdShort id) override; void add_ext_server_port(td::uint16 port) override; @@ -445,7 +447,7 @@ class ValidatorManagerImpl : public ValidatorManager { void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::Promise promise) override; void send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) override; + td::BufferSlice data, int mode) override; void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; @@ -545,7 +547,7 @@ class ValidatorManagerImpl : public ValidatorManager { } void add_shard_block_description(td::Ref desc); - void add_cached_block_candidate(ReceivedBlock block); + void add_cached_block_data(BlockIdExt block_id, td::BufferSlice data); void register_block_handle(BlockHandle handle); @@ -696,9 +698,6 @@ class ValidatorManagerImpl : public ValidatorManager { double max_mempool_num() const { return opts_->max_mempool_num(); } - size_t max_cached_candidates() const { - return 128; - } static double max_ext_msg_per_addr_time_window() { return 10.0; } diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 110ccd813..498e5ce20 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -185,6 +185,7 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so auto block = block_data.size() > 0 ? create_block(next_block_id, std::move(block_data)).move_as_ok() : td::Ref{}; + // OLD BROADCAST BEHAVIOR: // Creator of the block sends broadcast to public overlays // Creator of the block sends broadcast to private block overlay unless candidate broadcast was sent // Any node sends broadcast to custom overlays unless candidate broadcast was sent @@ -200,6 +201,23 @@ void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo so send_broadcast_mode |= fullnode::FullNode::broadcast_mode_custom; } + // NEW BROADCAST BEHAVIOR (activate later): + // Masterchain block are broadcasted as Block Broadcast (with signatures). Shard blocks are broadcasted as Block Candidate Broadcast (only block data). + // Public and private overlays: creator sends masterchain blocks, all validators send shard blocks. + // Custom overlays: all nodes send all blocks. + // If the block was broadcasted earlier as a candidate (to private and custom overlays), the broadcast is not repeated. + /*int send_broadcast_mode = 0; + bool sent_candidate = sent_candidate_broadcasts_.contains(next_block_id); + if (!shard_.is_masterchain() || source_info.source.compute_short_id() == local_id_) { + send_broadcast_mode |= fullnode::FullNode::broadcast_mode_public; + if (!sent_candidate) { + send_broadcast_mode |= fullnode::FullNode::broadcast_mode_private_block; + } + } + if (!sent_candidate) { + send_broadcast_mode |= fullnode::FullNode::broadcast_mode_custom; + }*/ + auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this), block_id = next_block_id, prev = prev_block_ids_, promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { @@ -516,6 +534,15 @@ void ValidatorGroup::get_validator_group_info_for_litequery_cont( promise.set_result(std::move(result)); } +void ValidatorGroup::send_block_candidate_broadcast(BlockIdExt id, td::BufferSlice data) { + if (sent_candidate_broadcasts_.insert(id).second) { + td::actor::send_closure( + manager_, &ValidatorManager::send_block_candidate_broadcast, id, validator_set_->get_catchain_seqno(), + validator_set_->get_validator_set_hash(), std::move(data), + fullnode::FullNode::broadcast_mode_private_block | fullnode::FullNode::broadcast_mode_custom); + } +} + } // namespace validator } // namespace ton diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index db55614f4..fee7ee580 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -165,13 +165,7 @@ class ValidatorGroup : public td::actor::Actor { std::set sent_candidate_broadcasts_; - void send_block_candidate_broadcast(BlockIdExt id, td::BufferSlice data) { - if (sent_candidate_broadcasts_.insert(id).second) { - td::actor::send_closure(manager_, &ValidatorManager::send_block_candidate_broadcast, id, - validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), - std::move(data)); - } - } + void send_block_candidate_broadcast(BlockIdExt id, td::BufferSlice data); }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 7868bc16f..dc0f5761a 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -178,7 +178,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; virtual void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, - td::BufferSlice data) = 0; + td::BufferSlice data, int mode) = 0; virtual void send_broadcast(BlockBroadcast broadcast, int mode) = 0; virtual void download_block(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; @@ -216,7 +216,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void validate_block_proof_rel(BlockIdExt block_id, BlockIdExt rel_block_id, td::BufferSlice proof, td::Promise promise) = 0; virtual void validate_block(ReceivedBlock block, td::Promise promise) = 0; - virtual void prevalidate_block(BlockBroadcast broadcast, td::Promise promise) = 0; + virtual void new_block_broadcast(BlockBroadcast broadcast, td::Promise promise) = 0; //virtual void create_validate_block(BlockId block, td::BufferSlice data, td::Promise promise) = 0; virtual void sync_complete(td::Promise promise) = 0; @@ -252,8 +252,9 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void new_external_message(td::BufferSlice data, int priority) = 0; virtual void check_external_message(td::BufferSlice data, td::Promise> promise) = 0; virtual void new_ihr_message(td::BufferSlice data) = 0; - virtual void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; - virtual void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) = 0; + virtual void new_shard_block_description_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) = 0; + virtual void new_block_candidate_broadcast(BlockIdExt block_id, td::BufferSlice data) = 0; virtual void add_ext_server_id(adnl::AdnlNodeIdShort id) = 0; virtual void add_ext_server_port(td::uint16 port) = 0; From 5a86f0ac2d5216a64c33e14f16cb2f3a360124ff Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 9 May 2025 19:12:21 +0300 Subject: [PATCH 240/388] Fix calculating storage dict, simplify storage stat code (#1643) --- crypto/block/account-storage-stat.cpp | 47 +++++++++++++-------------- crypto/block/account-storage-stat.h | 3 +- crypto/block/transaction.cpp | 32 +++++++++++++----- 3 files changed, 47 insertions(+), 35 deletions(-) diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp index 56cc8f1e1..82359f827 100644 --- a/crypto/block/account-storage-stat.cpp +++ b/crypto/block/account-storage-stat.cpp @@ -63,7 +63,7 @@ td::Status AccountStorageStat::replace_roots(std::vector> new_root for (const Ref& root : to_add) { TRY_RESULT(info, add_cell(root)); if (check_merkle_depth && info.max_merkle_depth > MAX_MERKLE_DEPTH) { - return td::Status::Error("too big Merkle depth"); + return td::Status::Error(errorcode_limits_exceeded, "too big Merkle depth"); } } for (const Ref& root : to_del) { @@ -92,10 +92,10 @@ void AccountStorageStat::add_hint(const td::HashSet& hint) { return; } } + e.max_merkle_depth = 0; if (hint.contains(cell->get_hash())) { bool spec; vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); - e.size_bits = cs.size(); for (unsigned i = 0; i < cs.size_refs(); ++i) { dfs(cs.prefetch_ref(i), false); } @@ -108,11 +108,11 @@ void AccountStorageStat::add_hint(const td::HashSet& hint) { td::Result AccountStorageStat::add_cell(const Ref& cell) { Entry& e = get_entry(cell); - if (!e.exists_known || e.refcnt_diff < 0) { + if (!e.exists_known) { TRY_STATUS(fetch_from_dict(e)); } ++e.refcnt_diff; - if (e.exists || e.refcnt_diff > 1 || (e.refcnt && e.refcnt.value() + e.refcnt_diff != 1)) { + if (e.exists || e.refcnt_diff > 1) { if (!e.max_merkle_depth) { TRY_STATUS(fetch_from_dict(e)); if (!e.max_merkle_depth) { @@ -125,7 +125,6 @@ td::Result AccountStorageStat::add_cell(const Ref< td::uint32 max_merkle_depth = 0; bool spec; vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); - e.size_bits = cs.size(); for (unsigned i = 0; i < cs.size_refs(); ++i) { TRY_RESULT(info, add_cell(cs.prefetch_ref(i))); max_merkle_depth = std::max(max_merkle_depth, info.max_merkle_depth); @@ -137,6 +136,8 @@ td::Result AccountStorageStat::add_cell(const Ref< max_merkle_depth = std::min(max_merkle_depth, MERKLE_DEPTH_LIMIT); Entry& e2 = get_entry(cell); e2.max_merkle_depth = max_merkle_depth; + ++total_cells_; + total_bits_ += cs.size(); return CellInfo{max_merkle_depth}; } @@ -158,16 +159,28 @@ td::Status AccountStorageStat::remove_cell(const Ref& cell) { } bool spec; vm::CellSlice cs = vm::load_cell_slice_special(cell, spec); - e.size_bits = cs.size(); for (unsigned i = 0; i < cs.size_refs(); ++i) { TRY_STATUS(remove_cell(cs.prefetch_ref(i))); } + --total_cells_; + total_bits_ -= cs.size(); return td::Status::OK(); } td::Result> AccountStorageStat::get_dict_root() { if (!dict_up_to_date_) { std::vector>> values; + if (parent_ && !parent_->dict_up_to_date_) { + for (auto& [_, ep] : parent_->cache_) { + if (ep.dict_refcnt_diff == 0) { + continue; + } + Entry& e = cache_[ep.hash]; + if (!e.inited) { + e = ep; + } + } + } for (auto& [_, e] : cache_) { if (e.dict_refcnt_diff == 0) { continue; @@ -259,25 +272,9 @@ td::Status AccountStorageStat::finalize_entry(Entry& e) { e.refcnt.value() += e.refcnt_diff; e.dict_refcnt_diff += e.refcnt_diff; e.refcnt_diff = 0; - if (e.refcnt.value() == 0) { - if (!e.size_bits) { - return td::Status::Error(PSTRING() << "Failed to store entry " << e.hash.to_hex() << " : unknown cell bits"); - } - --total_cells_; - total_bits_ -= e.size_bits.value(); - e.exists = false; - } else { - if (!e.exists) { - if (!e.size_bits) { - return td::Status::Error(PSTRING() << "Failed to store entry " << e.hash.to_hex() << " : unknown cell bits"); - } - ++total_cells_; - total_bits_ += e.size_bits.value(); - } - e.exists = true; - if (!e.max_merkle_depth) { - return td::Status::Error(PSTRING() << "Failed to store entry " << e.hash.to_hex() << " : unknown merkle depth"); - } + e.exists = (e.refcnt.value() != 0); + if (e.exists && !e.max_merkle_depth) { + return td::Status::Error(PSTRING() << "Error on entry " << e.hash.to_hex() << " : unknown merkle depth"); } return td::Status::OK(); } diff --git a/crypto/block/account-storage-stat.h b/crypto/block/account-storage-stat.h index 645f0744d..e939a0ff5 100644 --- a/crypto/block/account-storage-stat.h +++ b/crypto/block/account-storage-stat.h @@ -59,6 +59,8 @@ class AccountStorageStat { void apply_child_stat(AccountStorageStat &&child); + static constexpr int errorcode_limits_exceeded = 999; + private: vm::Dictionary dict_; bool dict_up_to_date_ = true; @@ -76,7 +78,6 @@ class AccountStorageStat { struct Entry { bool inited = false; vm::CellHash hash; - td::optional size_bits; bool exists_known = false; bool exists = false; td::optional refcnt, max_merkle_depth; diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index b892b5a21..1b6c3582f 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -1930,12 +1930,17 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { ap.action_fine = td::zero_refint(); td::Ref old_code = new_code, old_data = new_data, old_library = new_library; - auto enforce_state_limits = [&]() { + // 1 - ok, 0 - limits exceeded, -1 - fatal error + auto enforce_state_limits = [&]() -> int { if (account.is_special) { - return true; + return 1; } auto S = check_state_limits(cfg.size_limits); if (S.is_error()) { + if (S.code() != AccountStorageStat::errorcode_limits_exceeded) { + LOG(ERROR) << "Account storage stat error: " << S.move_as_error(); + return -1; + } // Rollback changes to state, fail action phase LOG(INFO) << "Account state size exceeded limits: " << S.move_as_error(); new_account_storage_stat = {}; @@ -1944,9 +1949,9 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { new_library = old_library; ap.result_code = 50; ap.state_exceeds_limits = true; - return false; + return 0; } - return true; + return 1; }; int n = 0; @@ -2057,7 +2062,9 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { } LOG(DEBUG) << "invalid action " << ap.result_arg << " in action list: error code " << ap.result_code; // This is required here because changes to libraries are applied even if action phase fails - enforce_state_limits(); + if (enforce_state_limits() == -1) { + return false; + } if (cfg.action_fine_enabled) { ap.action_fine = std::min(ap.action_fine, balance.grams); ap.total_action_fees = ap.action_fine; @@ -2079,7 +2086,11 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { new_code = ap.new_code; } new_data = compute_phase->new_data; // tentative persistent data update applied - if (!enforce_state_limits()) { + int res = enforce_state_limits(); + if (res == -1) { + return false; + } + if (res == 0) { if (cfg.extra_currency_v2) { end_lt = ap.end_lt = start_lt + 1; if (cfg.action_fine_enabled) { @@ -3054,7 +3065,8 @@ static td::uint32 get_public_libraries_diff_count(const td::Ref& old_l * * @returns A `td::Status` indicating the result of the check. * - If the state limits are within the allowed range, returns OK. - * - If the state limits exceed the maximum allowed range, returns an error. + * - If the state limits exceed the maximum allowed range, returns an error with AccountStorageStat::errorcode_limits_exceeded code. + * - If an error occurred during storage stat calculation, returns other error. */ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, bool update_storage_stat) { auto cell_equal = [](const td::Ref& a, const td::Ref& b) -> bool { @@ -3079,7 +3091,8 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, if (storage_stat.get_total_cells() > size_limits.max_acc_state_cells || storage_stat.get_total_bits() > size_limits.max_acc_state_bits) { - return td::Status::Error(PSTRING() << "account state is too big: cells=" << storage_stat.get_total_cells() + return td::Status::Error(AccountStorageStat::errorcode_limits_exceeded, + PSTRING() << "account state is too big: cells=" << storage_stat.get_total_cells() << ", bits=" << storage_stat.get_total_bits() << " (max cells=" << size_limits.max_acc_state_cells << ", max bits=" << size_limits.max_acc_state_bits << ")"); @@ -3087,7 +3100,8 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, if (account.is_masterchain() && !cell_equal(account.library, new_library)) { auto libraries_count = get_public_libraries_count(new_library); if (libraries_count > size_limits.max_acc_public_libraries) { - return td::Status::Error(PSTRING() << "too many public libraries: " << libraries_count << " (max " + return td::Status::Error(AccountStorageStat::errorcode_limits_exceeded, + PSTRING() << "too many public libraries: " << libraries_count << " (max " << size_limits.max_acc_public_libraries << ")"); } } From 9e5109d56bc4f2345a00b2271c3711103841b799 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 9 May 2025 19:13:21 +0300 Subject: [PATCH 241/388] Fix type check in INMSGPARAM (#1652) --- crypto/vm/tonops.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crypto/vm/tonops.cpp b/crypto/vm/tonops.cpp index b62e5fb99..135ed7026 100644 --- a/crypto/vm/tonops.cpp +++ b/crypto/vm/tonops.cpp @@ -166,6 +166,9 @@ int exec_get_in_msg_param(VmState* st, unsigned idx, const char* name) { VM_LOG(st) << "execute " << name; } Ref t = get_param(st, inmsgparams_idx).as_tuple(); + if (t.is_null()) { + throw VmError{Excno::type_chk, "intermediate value is not a tuple"}; + } st->get_stack().push(tuple_index(t, idx)); return 0; } From a6449abf21a88c39000d4e3d9183ee7906920b61 Mon Sep 17 00:00:00 2001 From: Evgeny Kapun Date: Tue, 13 May 2025 09:18:59 +0300 Subject: [PATCH 242/388] Adnl optimizations (#1620) * Remove AdnlStaticNodesManager * Remove AdnlPeer --------- Co-authored-by: EmelyanenkoK --- adnl/CMakeLists.txt | 3 - adnl/adnl-peer-table.cpp | 155 ++++++++++++++++++++++++-------- adnl/adnl-peer-table.h | 4 +- adnl/adnl-peer-table.hpp | 31 ++++--- adnl/adnl-peer.cpp | 178 +------------------------------------ adnl/adnl-peer.h | 36 -------- adnl/adnl-peer.hpp | 59 +----------- adnl/adnl-static-nodes.cpp | 55 ------------ adnl/adnl-static-nodes.h | 43 --------- adnl/adnl-static-nodes.hpp | 42 --------- 10 files changed, 140 insertions(+), 466 deletions(-) delete mode 100644 adnl/adnl-static-nodes.cpp delete mode 100644 adnl/adnl-static-nodes.h delete mode 100644 adnl/adnl-static-nodes.hpp diff --git a/adnl/CMakeLists.txt b/adnl/CMakeLists.txt index f1d1bfbfb..3604dfb3a 100644 --- a/adnl/CMakeLists.txt +++ b/adnl/CMakeLists.txt @@ -23,8 +23,6 @@ set(ADNL_HEADERS adnl-peer.h adnl-peer.hpp adnl-query.h - adnl-static-nodes.h - adnl-static-nodes.hpp adnl-proxy-types.h adnl-proxy-types.hpp adnl.h @@ -46,7 +44,6 @@ set(ADNL_SOURCE adnl-peer.cpp adnl-query.cpp adnl-channel.cpp - adnl-static-nodes.cpp adnl-proxy-types.cpp utils.cpp ${ADNL_HEADERS} diff --git a/adnl/adnl-peer-table.cpp b/adnl/adnl-peer-table.cpp index b8aab5671..5a92ca1c1 100644 --- a/adnl/adnl-peer-table.cpp +++ b/adnl/adnl-peer-table.cpp @@ -84,6 +84,32 @@ void AdnlPeerTableImpl::receive_packet(td::IPAddress addr, AdnlCategoryMask cat_ << " (len=" << (data.size() + 32) << ")"; } +void AdnlPeerTableImpl::update_id(AdnlPeerTableImpl::PeerInfo &peer_info, AdnlNodeIdFull &&peer_id) { + if (peer_info.peer_id.empty()) { + peer_info.peer_id = std::move(peer_id); + for (auto &e: peer_info.peers) { + td::actor::send_closure(e.second, &AdnlPeerPair::update_peer_id, peer_info.peer_id); + } + } +} + +td::actor::ActorOwn &AdnlPeerTableImpl::get_peer_pair( + AdnlNodeIdShort peer_id, AdnlPeerTableImpl::PeerInfo &peer_info, + AdnlNodeIdShort local_id, AdnlPeerTableImpl::LocalIdInfo &local_id_info) { + auto it = peer_info.peers.find(local_id); + if (it == peer_info.peers.end()) { + it = peer_info.peers.emplace(local_id, + AdnlPeerPair::create(network_manager_, actor_id(this), + local_id_info.mode, local_id_info.local_id.get(), + dht_node_, local_id, peer_id)) + .first; + if (!peer_info.peer_id.empty()) { + td::actor::send_closure(it->second, &AdnlPeerPair::update_peer_id, peer_info.peer_id); + } + } + return it->second; +} + void AdnlPeerTableImpl::receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket packet, td::uint64 serialized_size) { packet.run_basic_checks().ensure(); @@ -91,8 +117,9 @@ void AdnlPeerTableImpl::receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket VLOG(ADNL_INFO) << this << ": dropping IN message [?->" << dst << "]: destination not set"; return; } + AdnlNodeIdShort src = packet.from_short(); - auto it = peers_.find(packet.from_short()); + auto it = peers_.find(src); if (it == peers_.end()) { if (!packet.inited_from()) { VLOG(ADNL_NOTICE) << this << ": dropping IN message [" << packet.from_short() << "->" << dst @@ -105,11 +132,7 @@ void AdnlPeerTableImpl::receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket return; } - it = peers_ - .emplace(packet.from_short(), - AdnlPeer::create(network_manager_, actor_id(this), dht_node_, packet.from_short())) - .first; - CHECK(it != peers_.end()); + it = peers_.try_emplace(src).first; } auto it2 = local_ids_.find(dst); @@ -118,8 +141,13 @@ void AdnlPeerTableImpl::receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket << "]: unknown dst (but how did we decrypt message?)"; return; } - td::actor::send_closure(it->second, &AdnlPeer::receive_packet, dst, it2->second.mode, it2->second.local_id.get(), - std::move(packet), serialized_size); + + if (packet.inited_from()) { + update_id(it->second, packet.from()); + } + + td::actor::send_closure(get_peer_pair(src, it->second, dst, it2->second), + &AdnlPeerPair::receive_packet, std::move(packet), serialized_size); } void AdnlPeerTableImpl::add_peer(AdnlNodeIdShort local_id, AdnlNodeIdFull id, AdnlAddressList addr_list) { @@ -129,31 +157,25 @@ void AdnlPeerTableImpl::add_peer(AdnlNodeIdShort local_id, AdnlNodeIdFull id, Ad auto it2 = local_ids_.find(local_id); CHECK(it2 != local_ids_.end()); - auto it = peers_.find(id_short); - if (it == peers_.end()) { - it = peers_.emplace(id_short, AdnlPeer::create(network_manager_, actor_id(this), dht_node_, id_short)).first; - CHECK(it != peers_.end()); - } - td::actor::send_closure(it->second, &AdnlPeer::update_id, std::move(id)); + auto &peer_info = peers_[id_short]; + update_id(peer_info, std::move(id)); if (!addr_list.empty()) { - td::actor::send_closure(it->second, &AdnlPeer::update_addr_list, local_id, it2->second.mode, - it2->second.local_id.get(), std::move(addr_list)); + td::actor::send_closure(get_peer_pair(id_short, peer_info, local_id, it2->second), + &AdnlPeerPair::update_addr_list, std::move(addr_list)); } } void AdnlPeerTableImpl::add_static_nodes_from_config(AdnlNodesList nodes) { - for (auto &it : nodes.nodes()) { - add_static_node(it); + for (auto &node : nodes.nodes()) { + auto id_short = node.compute_short_id(); + VLOG(ADNL_INFO) << "[staticnodes] adding static node " << id_short; + static_nodes_.emplace(id_short, std::move(node)); } } void AdnlPeerTableImpl::send_message_in(AdnlNodeIdShort src, AdnlNodeIdShort dst, AdnlMessage message, td::uint32 flags) { - auto it = peers_.find(dst); - - if (it == peers_.end()) { - it = peers_.emplace(dst, AdnlPeer::create(network_manager_, actor_id(this), dht_node_, dst)).first; - } + auto &peer_info = peers_[dst]; auto it2 = local_ids_.find(src); if (it2 == local_ids_.end()) { @@ -161,8 +183,10 @@ void AdnlPeerTableImpl::send_message_in(AdnlNodeIdShort src, AdnlNodeIdShort dst return; } - td::actor::send_closure(it->second, &AdnlPeer::send_one_message, src, it2->second.mode, it2->second.local_id.get(), - OutboundAdnlMessage{std::move(message), flags}); + std::vector messages; + messages.push_back(OutboundAdnlMessage{std::move(message), flags}); + td::actor::send_closure(get_peer_pair(dst, peer_info, src, it2->second), + &AdnlPeerPair::send_messages, std::move(messages)); } void AdnlPeerTableImpl::answer_query(AdnlNodeIdShort src, AdnlNodeIdShort dst, AdnlQueryId query_id, @@ -182,11 +206,7 @@ void AdnlPeerTableImpl::send_query(AdnlNodeIdShort src, AdnlNodeIdShort dst, std VLOG(ADNL_WARNING) << "DUMP: " << td::buffer_to_hex(data.as_slice().truncate(128)); return; } - auto it = peers_.find(dst); - - if (it == peers_.end()) { - it = peers_.emplace(dst, AdnlPeer::create(network_manager_, actor_id(this), dht_node_, dst)).first; - } + auto &peer_info = peers_[dst]; auto it2 = local_ids_.find(src); if (it2 == local_ids_.end()) { @@ -194,8 +214,8 @@ void AdnlPeerTableImpl::send_query(AdnlNodeIdShort src, AdnlNodeIdShort dst, std return; } - td::actor::send_closure(it->second, &AdnlPeer::send_query, src, it2->second.mode, it2->second.local_id.get(), name, - std::move(promise), timeout, std::move(data), 0); + td::actor::send_closure(get_peer_pair(dst, peer_info, src, it2->second), + &AdnlPeerPair::send_query, name, std::move(promise), timeout, std::move(data), 0); } void AdnlPeerTableImpl::add_id_ex(AdnlNodeIdFull id, AdnlAddressList addr_list, td::uint8 cat, td::uint32 mode) { @@ -247,8 +267,10 @@ void AdnlPeerTableImpl::unsubscribe(AdnlNodeIdShort dst, std::string prefix) { void AdnlPeerTableImpl::register_dht_node(td::actor::ActorId dht_node) { dht_node_ = dht_node; - for (auto &peer : peers_) { - td::actor::send_closure(peer.second, &AdnlPeer::update_dht_node, dht_node_); + for (auto &e : peers_) { + for (auto &e2 : e.second.peers) { + td::actor::send_closure(e2.second, &AdnlPeerPair::update_dht_node, dht_node_); + } } for (auto &local_id : local_ids_) { td::actor::send_closure(local_id.second.local_id, &AdnlLocalId::update_dht_node, dht_node_); @@ -332,8 +354,6 @@ void AdnlPeerTableImpl::get_addr_list_from_db(AdnlNodeIdShort local_id, AdnlNode AdnlPeerTableImpl::AdnlPeerTableImpl(std::string db_root, td::actor::ActorId keyring) { keyring_ = keyring; - static_nodes_manager_ = AdnlStaticNodesManager::create(); - if (!db_root.empty()) { db_ = AdnlDb::create(db_root + "/adnl"); } @@ -382,7 +402,64 @@ void AdnlPeerTableImpl::get_conn_ip_str(AdnlNodeIdShort l_id, AdnlNodeIdShort p_ promise.set_value("undefined"); return; } - td::actor::send_closure(it->second, &AdnlPeer::get_conn_ip_str, l_id, std::move(promise)); + auto it2 = it->second.peers.find(l_id); + if (it2 == it->second.peers.end()) { + promise.set_value("undefined"); + return; + } + td::actor::send_closure(it2->second, &AdnlPeerPair::get_conn_ip_str, std::move(promise)); +} + +void AdnlPeerTableImpl::get_stats_peer(AdnlNodeIdShort peer_id, AdnlPeerTableImpl::PeerInfo &peer_info, bool all, + td::Promise>> promise) { + class Cb : public td::actor::Actor { + public: + explicit Cb(td::Promise>> promise) + : promise_(std::move(promise)) { + } + + void got_peer_pair_stats(tl_object_ptr peer_pair) { + if (peer_pair) { + result_.push_back(std::move(peer_pair)); + } + dec_pending(); + } + + void inc_pending() { + ++pending_; + } + + void dec_pending() { + CHECK(pending_ > 0); + --pending_; + if (pending_ == 0) { + promise_.set_result(std::move(result_)); + stop(); + } + } + + private: + td::Promise>> promise_; + size_t pending_ = 1; + std::vector> result_; + }; + auto callback = td::actor::create_actor("adnlpeerstats", std::move(promise)).release(); + + for (auto &[local_id, peer_pair] : peer_info.peers) { + td::actor::send_closure(callback, &Cb::inc_pending); + td::actor::send_closure(peer_pair, &AdnlPeerPair::get_stats, all, + [local_id = local_id, peer_id = peer_id, + callback](td::Result> R) { + if (R.is_error()) { + VLOG(ADNL_NOTICE) << "failed to get stats for peer pair " << peer_id << "->" << local_id + << " : " << R.move_as_error(); + td::actor::send_closure(callback, &Cb::dec_pending); + } else { + td::actor::send_closure(callback, &Cb::got_peer_pair_stats, R.move_as_ok()); + } + }); + } + td::actor::send_closure(callback, &Cb::dec_pending); } void AdnlPeerTableImpl::get_stats(bool all, td::Promise> promise) { @@ -453,8 +530,8 @@ void AdnlPeerTableImpl::get_stats(bool all, td::Promise>> R) { if (R.is_error()) { VLOG(ADNL_NOTICE) << "failed to get stats for peer " << id << " : " << R.move_as_error(); diff --git a/adnl/adnl-peer-table.h b/adnl/adnl-peer-table.h index 055f32ac1..13b4adb53 100644 --- a/adnl/adnl-peer-table.h +++ b/adnl/adnl-peer-table.h @@ -97,9 +97,7 @@ class AdnlPeerTable : public Adnl { td::actor::ActorId channel) = 0; virtual void unregister_channel(AdnlChannelIdShort id) = 0; - virtual void add_static_node(AdnlNode node) = 0; - virtual void del_static_node(AdnlNodeIdShort id) = 0; - virtual void get_static_node(AdnlNodeIdShort id, td::Promise promise) = 0; + virtual td::Result get_static_node(AdnlNodeIdShort id) = 0; virtual void write_new_addr_list_to_db(AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id, AdnlDbItem node, td::Promise promise) = 0; diff --git a/adnl/adnl-peer-table.hpp b/adnl/adnl-peer-table.hpp index 9ad61b653..408f846d0 100644 --- a/adnl/adnl-peer-table.hpp +++ b/adnl/adnl-peer-table.hpp @@ -28,7 +28,6 @@ #include "adnl-local-id.h" #include "adnl-query.h" #include "utils.hpp" -#include "adnl-static-nodes.h" #include "adnl-ext-server.h" #include "adnl-address-list.h" @@ -86,15 +85,12 @@ class AdnlPeerTableImpl : public AdnlPeerTable { void get_addr_list_from_db(AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id, td::Promise promise) override; - void add_static_node(AdnlNode node) override { - CHECK(!static_nodes_manager_.empty()); - td::actor::send_closure(static_nodes_manager_, &AdnlStaticNodesManager::add_node, std::move(node)); - } - void del_static_node(AdnlNodeIdShort id) override { - td::actor::send_closure(static_nodes_manager_, &AdnlStaticNodesManager::del_node, id); - } - void get_static_node(AdnlNodeIdShort id, td::Promise promise) override { - td::actor::send_closure(static_nodes_manager_, &AdnlStaticNodesManager::get_node, id, std::move(promise)); + td::Result get_static_node(AdnlNodeIdShort id) override { + auto it = static_nodes_.find(id); + if (it == static_nodes_.end()) { + return td::Status::Error(ErrorCode::notready, "static node not found"); + } + return it->second; } void deliver(AdnlNodeIdShort src, AdnlNodeIdShort dst, td::BufferSlice data) override; void deliver_query(AdnlNodeIdShort src, AdnlNodeIdShort dst, td::BufferSlice data, @@ -116,20 +112,26 @@ class AdnlPeerTableImpl : public AdnlPeerTable { } private: + struct PeerInfo { + AdnlNodeIdFull peer_id; + std::map> peers; + }; + struct LocalIdInfo { td::actor::ActorOwn local_id; td::uint8 cat; td::uint32 mode; }; + td::actor::ActorId keyring_; td::actor::ActorId network_manager_; td::actor::ActorId dht_node_; - td::actor::ActorOwn static_nodes_manager_; + std::map static_nodes_; void deliver_one_message(AdnlNodeIdShort src, AdnlNodeIdShort dst, AdnlMessage message); - std::map> peers_; + std::map peers_; std::map local_ids_; std::map, td::uint8>> channels_; @@ -140,6 +142,11 @@ class AdnlPeerTableImpl : public AdnlPeerTable { AdnlNodeIdShort proxy_addr_; //std::map> out_queries_; //td::uint64 last_query_id_ = 1; + + static void update_id(PeerInfo &peer_info, AdnlNodeIdFull &&peer_id); + td::actor::ActorOwn &get_peer_pair(AdnlNodeIdShort peer_id, PeerInfo &peer_info, AdnlNodeIdShort local_id, LocalIdInfo &local_id_info); + static void get_stats_peer(AdnlNodeIdShort peer_id, PeerInfo &peer_info, bool all, + td::Promise>> promise); }; inline td::StringBuilder &operator<<(td::StringBuilder &sb, const AdnlPeerTableImpl::PrintId &id) { diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp index 88a38f7b0..4913216ee 100644 --- a/adnl/adnl-peer.cpp +++ b/adnl/adnl-peer.cpp @@ -475,13 +475,12 @@ void AdnlPeerPairImpl::alarm_query(AdnlQueryId id) { AdnlPeerPairImpl::AdnlPeerPairImpl(td::actor::ActorId network_manager, td::actor::ActorId peer_table, td::uint32 local_mode, - td::actor::ActorId local_actor, td::actor::ActorId peer, + td::actor::ActorId local_actor, td::actor::ActorId dht_node, AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id) { network_manager_ = network_manager; peer_table_ = peer_table; local_actor_ = local_actor; - peer_ = peer; dht_node_ = dht_node; mode_ = local_mode; @@ -871,19 +870,6 @@ void AdnlPeerPairImpl::get_stats(bool all, td::Promise peer, td::actor::ActorId network_manager, td::actor::ActorId adnl) { @@ -902,171 +888,13 @@ void AdnlPeerPairImpl::conn_change_state(AdnlConnectionIdShort id, bool ready) { td::actor::ActorOwn AdnlPeerPair::create( td::actor::ActorId network_manager, td::actor::ActorId peer_table, - td::uint32 local_mode, td::actor::ActorId local_actor, td::actor::ActorId peer_actor, + td::uint32 local_mode, td::actor::ActorId local_actor, td::actor::ActorId dht_node, AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id) { auto X = td::actor::create_actor("peerpair", network_manager, peer_table, local_mode, local_actor, - peer_actor, dht_node, local_id, peer_id); + dht_node, local_id, peer_id); return td::actor::ActorOwn(std::move(X)); } -td::actor::ActorOwn AdnlPeer::create(td::actor::ActorId network_manager, - td::actor::ActorId peer_table, - td::actor::ActorId dht_node, AdnlNodeIdShort peer_id) { - auto X = td::actor::create_actor("peer", network_manager, peer_table, dht_node, peer_id); - return td::actor::ActorOwn(std::move(X)); -} - -void AdnlPeerImpl::receive_packet(AdnlNodeIdShort dst, td::uint32 dst_mode, td::actor::ActorId dst_actor, - AdnlPacket packet, td::uint64 serialized_size) { - if (packet.inited_from()) { - update_id(packet.from()); - } - - auto it = peer_pairs_.find(dst); - if (it == peer_pairs_.end()) { - auto X = AdnlPeerPair::create(network_manager_, peer_table_, dst_mode, dst_actor, actor_id(this), dht_node_, dst, - peer_id_short_); - peer_pairs_.emplace(dst, std::move(X)); - it = peer_pairs_.find(dst); - CHECK(it != peer_pairs_.end()); - - if (!peer_id_.empty()) { - td::actor::send_closure(it->second.get(), &AdnlPeerPair::update_peer_id, peer_id_); - } - } - - td::actor::send_closure(it->second.get(), &AdnlPeerPair::receive_packet, std::move(packet), serialized_size); -} - -void AdnlPeerImpl::send_messages(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, - std::vector messages) { - auto it = peer_pairs_.find(src); - if (it == peer_pairs_.end()) { - auto X = AdnlPeerPair::create(network_manager_, peer_table_, src_mode, src_actor, actor_id(this), dht_node_, src, - peer_id_short_); - peer_pairs_.emplace(src, std::move(X)); - it = peer_pairs_.find(src); - CHECK(it != peer_pairs_.end()); - - if (!peer_id_.empty()) { - td::actor::send_closure(it->second.get(), &AdnlPeerPair::update_peer_id, peer_id_); - } - } - - td::actor::send_closure(it->second, &AdnlPeerPair::send_messages, std::move(messages)); -} - -void AdnlPeerImpl::send_query(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, - std::string name, td::Promise promise, td::Timestamp timeout, - td::BufferSlice data, td::uint32 flags) { - auto it = peer_pairs_.find(src); - if (it == peer_pairs_.end()) { - auto X = AdnlPeerPair::create(network_manager_, peer_table_, src_mode, src_actor, actor_id(this), dht_node_, src, - peer_id_short_); - peer_pairs_.emplace(src, std::move(X)); - it = peer_pairs_.find(src); - CHECK(it != peer_pairs_.end()); - - if (!peer_id_.empty()) { - td::actor::send_closure(it->second.get(), &AdnlPeerPair::update_peer_id, peer_id_); - } - } - - td::actor::send_closure(it->second, &AdnlPeerPair::send_query, name, std::move(promise), timeout, std::move(data), - flags); -} - -void AdnlPeerImpl::del_local_id(AdnlNodeIdShort local_id) { - peer_pairs_.erase(local_id); -} - -void AdnlPeerImpl::update_dht_node(td::actor::ActorId dht_node) { - dht_node_ = dht_node; - for (auto it = peer_pairs_.begin(); it != peer_pairs_.end(); it++) { - td::actor::send_closure(it->second, &AdnlPeerPair::update_dht_node, dht_node_); - } -} - -void AdnlPeerImpl::get_conn_ip_str(AdnlNodeIdShort l_id, td::Promise promise) { - auto it = peer_pairs_.find(l_id); - if (it == peer_pairs_.end()) { - promise.set_value("undefined"); - return; - } - - td::actor::send_closure(it->second, &AdnlPeerPair::get_conn_ip_str, std::move(promise)); -} - -void AdnlPeerImpl::update_addr_list(AdnlNodeIdShort local_id, td::uint32 local_mode, - td::actor::ActorId local_actor, AdnlAddressList addr_list) { - auto it = peer_pairs_.find(local_id); - if (it == peer_pairs_.end()) { - auto X = AdnlPeerPair::create(network_manager_, peer_table_, local_mode, local_actor, actor_id(this), dht_node_, - local_id, peer_id_short_); - peer_pairs_.emplace(local_id, std::move(X)); - it = peer_pairs_.find(local_id); - CHECK(it != peer_pairs_.end()); - - if (!peer_id_.empty()) { - td::actor::send_closure(it->second.get(), &AdnlPeerPair::update_peer_id, peer_id_); - } - } - - td::actor::send_closure(it->second, &AdnlPeerPair::update_addr_list, std::move(addr_list)); -} - -void AdnlPeerImpl::get_stats(bool all, td::Promise>> promise) { - class Cb : public td::actor::Actor { - public: - explicit Cb(td::Promise>> promise) - : promise_(std::move(promise)) { - } - - void got_peer_pair_stats(tl_object_ptr peer_pair) { - if (peer_pair) { - result_.push_back(std::move(peer_pair)); - } - dec_pending(); - } - - void inc_pending() { - ++pending_; - } - - void dec_pending() { - CHECK(pending_ > 0); - --pending_; - if (pending_ == 0) { - promise_.set_result(std::move(result_)); - stop(); - } - } - - private: - td::Promise>> promise_; - size_t pending_ = 1; - std::vector> result_; - }; - auto callback = td::actor::create_actor("adnlpeerstats", std::move(promise)).release(); - - for (auto &[local_id, peer_pair] : peer_pairs_) { - td::actor::send_closure(callback, &Cb::inc_pending); - td::actor::send_closure(peer_pair, &AdnlPeerPair::get_stats, all, - [local_id = local_id, peer_id = peer_id_short_, - callback](td::Result> R) { - if (R.is_error()) { - VLOG(ADNL_NOTICE) << "failed to get stats for peer pair " << peer_id << "->" << local_id - << " : " << R.move_as_error(); - td::actor::send_closure(callback, &Cb::dec_pending); - } else { - td::actor::send_closure(callback, &Cb::got_peer_pair_stats, R.move_as_ok()); - } - }); - } - td::actor::send_closure(callback, &Cb::dec_pending); -} - - void AdnlPeerPairImpl::got_data_from_db(td::Result R) { received_from_db_ = false; if (R.is_error()) { diff --git a/adnl/adnl-peer.h b/adnl/adnl-peer.h index 1215f71da..4d6812a89 100644 --- a/adnl/adnl-peer.h +++ b/adnl/adnl-peer.h @@ -64,46 +64,10 @@ class AdnlPeerPair : public td::actor::Actor { static td::actor::ActorOwn create(td::actor::ActorId network_manager, td::actor::ActorId peer_table, td::uint32 local_mode, td::actor::ActorId local_actor, - td::actor::ActorId peer_actor, td::actor::ActorId dht_node, AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id); }; -class AdnlPeer : public td::actor::Actor { - public: - virtual void receive_packet(AdnlNodeIdShort dst, td::uint32 dst_mode, td::actor::ActorId dst_actor, - AdnlPacket message, td::uint64 serialized_size) = 0; - virtual void send_messages(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, - std::vector messages) = 0; - virtual void send_query(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, - std::string name, td::Promise promise, td::Timestamp timeout, - td::BufferSlice data, td::uint32 flags) = 0; - void send_one_message(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, - OutboundAdnlMessage message) { - std::vector vec; - vec.push_back(std::move(message)); - send_messages(src, src_mode, src_actor, std::move(vec)); - } - - void send_message(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, - td::BufferSlice data, td::uint32 flags) { - auto M = OutboundAdnlMessage{adnlmessage::AdnlMessageCustom{std::move(data)}, flags}; - send_one_message(src, src_mode, src_actor, std::move(M)); - } - - static td::actor::ActorOwn create(td::actor::ActorId network_manager, - td::actor::ActorId peer_table, - td::actor::ActorId dht_node, AdnlNodeIdShort peer_id); - - virtual void del_local_id(AdnlNodeIdShort local_id) = 0; - virtual void update_id(AdnlNodeIdFull id) = 0; - virtual void update_addr_list(AdnlNodeIdShort local_id, td::uint32 local_mode, - td::actor::ActorId local_actor, AdnlAddressList addr_list) = 0; - virtual void update_dht_node(td::actor::ActorId dht_node) = 0; - virtual void get_conn_ip_str(AdnlNodeIdShort l_id, td::Promise promise) = 0; - virtual void get_stats(bool all, td::Promise>> promise) = 0; -}; - } // namespace adnl } // namespace ton diff --git a/adnl/adnl-peer.hpp b/adnl/adnl-peer.hpp index 243974ba1..b1d983578 100644 --- a/adnl/adnl-peer.hpp +++ b/adnl/adnl-peer.hpp @@ -60,7 +60,7 @@ class AdnlPeerPairImpl : public AdnlPeerPair { AdnlPeerPairImpl(td::actor::ActorId network_manager, td::actor::ActorId peer_table, td::uint32 local_mode, td::actor::ActorId local_actor, - td::actor::ActorId peer, td::actor::ActorId dht_node, AdnlNodeIdShort local_id, + td::actor::ActorId dht_node, AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id); void start_up() override; void alarm() override; @@ -209,7 +209,6 @@ class AdnlPeerPairImpl : public AdnlPeerPair { td::actor::ActorId network_manager_; td::actor::ActorId peer_table_; td::actor::ActorId local_actor_; - td::actor::ActorId peer_; td::actor::ActorId dht_node_; td::uint32 priority_ = 0; @@ -287,68 +286,12 @@ class AdnlPeerPairImpl : public AdnlPeerPair { void prepare_packet_stats(); }; -class AdnlPeerImpl : public AdnlPeer { - public: - void receive_packet(AdnlNodeIdShort dst, td::uint32 dst_mode, td::actor::ActorId dst_actor, - AdnlPacket packet, td::uint64 serialized_size) override; - void send_messages(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, - std::vector messages) override; - void send_query(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, std::string name, - td::Promise promise, td::Timestamp timeout, td::BufferSlice data, - td::uint32 flags) override; - - void del_local_id(AdnlNodeIdShort local_id) override; - void update_id(AdnlNodeIdFull id) override; - void update_addr_list(AdnlNodeIdShort local_id, td::uint32 local_mode, td::actor::ActorId local_actor, - AdnlAddressList addr_list) override; - void update_dht_node(td::actor::ActorId dht_node) override; - void get_conn_ip_str(AdnlNodeIdShort l_id, td::Promise promise) override; - void get_stats(bool all, td::Promise>> promise) override; - //void check_signature(td::BufferSlice data, td::BufferSlice signature, td::Promise promise) override; - - AdnlPeerImpl(td::actor::ActorId network_manager, td::actor::ActorId peer_table, - td::actor::ActorId dht_node, AdnlNodeIdShort peer_id) - : peer_id_short_(peer_id), dht_node_(dht_node), peer_table_(peer_table), network_manager_(network_manager) { - } - - struct PrintId { - AdnlNodeIdShort peer_id; - }; - - PrintId print_id() const { - return PrintId{peer_id_short_}; - } - - private: - AdnlNodeIdShort peer_id_short_; - AdnlNodeIdFull peer_id_; - std::map> peer_pairs_; - td::actor::ActorId dht_node_; - td::actor::ActorId peer_table_; - td::actor::ActorId network_manager_; -}; - } // namespace adnl } // namespace ton namespace td { -inline td::StringBuilder &operator<<(td::StringBuilder &sb, const ton::adnl::AdnlPeerImpl::PrintId &id) { - sb << "[peer " << id.peer_id << "]"; - return sb; -} - -inline td::StringBuilder &operator<<(td::StringBuilder &sb, const ton::adnl::AdnlPeerImpl &peer) { - sb << peer.print_id(); - return sb; -} - -inline td::StringBuilder &operator<<(td::StringBuilder &sb, const ton::adnl::AdnlPeerImpl *peer) { - sb << peer->print_id(); - return sb; -} - inline td::StringBuilder &operator<<(td::StringBuilder &sb, const ton::adnl::AdnlPeerPairImpl::PrintId &id) { sb << "[peerpair " << id.peer_id << "-" << id.local_id << "]"; return sb; diff --git a/adnl/adnl-static-nodes.cpp b/adnl/adnl-static-nodes.cpp deleted file mode 100644 index 251715be8..000000000 --- a/adnl/adnl-static-nodes.cpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "adnl-static-nodes.h" -#include "adnl-static-nodes.hpp" - -#include "utils.hpp" - -namespace ton { - -namespace adnl { - -void AdnlStaticNodesManagerImpl::add_node(AdnlNode node) { - auto id_short = node.compute_short_id(); - VLOG(ADNL_INFO) << "[staticnodes] adding static node " << id_short; - - nodes_.emplace(id_short, std::move(node)); -} - -void AdnlStaticNodesManagerImpl::del_node(AdnlNodeIdShort id) { - nodes_.erase(id); -} - -td::Result AdnlStaticNodesManagerImpl::get_node(AdnlNodeIdShort id) { - auto it = nodes_.find(id); - if (it == nodes_.end()) { - return td::Status::Error(ErrorCode::notready, "static node not found"); - } - - return it->second; -} - -td::actor::ActorOwn AdnlStaticNodesManager::create() { - auto X = td::actor::create_actor("staticnodesmanager"); - return td::actor::ActorOwn(std::move(X)); -} - -} // namespace adnl - -} // namespace ton diff --git a/adnl/adnl-static-nodes.h b/adnl/adnl-static-nodes.h deleted file mode 100644 index 91e3bed0c..000000000 --- a/adnl/adnl-static-nodes.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once - -#include "td/actor/actor.h" -#include "td/utils/Status.h" -#include "td/actor/PromiseFuture.h" -#include "auto/tl/ton_api.h" - -#include "adnl-peer-table.h" - -namespace ton { - -namespace adnl { - -class AdnlStaticNodesManager : public td::actor::Actor { - public: - virtual void add_node(AdnlNode node) = 0; - virtual void del_node(AdnlNodeIdShort id) = 0; - virtual td::Result get_node(AdnlNodeIdShort id) = 0; - - static td::actor::ActorOwn create(); -}; - -} // namespace adnl - -} // namespace ton diff --git a/adnl/adnl-static-nodes.hpp b/adnl/adnl-static-nodes.hpp deleted file mode 100644 index 166be5aa1..000000000 --- a/adnl/adnl-static-nodes.hpp +++ /dev/null @@ -1,42 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once - -#include -#include "adnl-static-nodes.h" - -namespace ton { - -namespace adnl { - -class AdnlStaticNodesManagerImpl : public AdnlStaticNodesManager { - public: - void add_node(AdnlNode node) override; - void del_node(AdnlNodeIdShort id) override; - td::Result get_node(AdnlNodeIdShort id) override; - AdnlStaticNodesManagerImpl() { - } - - private: - std::map nodes_; -}; - -} // namespace adnl - -} // namespace ton From b8251cb4500cb625790aaadd2460bca791daa737 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 13 May 2025 21:19:58 +0300 Subject: [PATCH 243/388] Remove debug logs --- validator/manager.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/validator/manager.cpp b/validator/manager.cpp index 2118ffb37..266079a5a 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1601,7 +1601,6 @@ void ValidatorManagerImpl::send_get_block_request(BlockIdExt id, td::uint32 prio LOG(DEBUG) << "send_get_block_request: got result from block data cache for " << id.to_str(); return promise.set_value(ReceivedBlock{id, cached->clone()}); } - LOG(ERROR) << "XXXX download " << id.id.to_str(); callback_->download_block(id, priority, td::Timestamp::in(10.0), std::move(promise)); } @@ -1637,7 +1636,6 @@ void ValidatorManagerImpl::send_get_block_proof_link_request(BlockIdExt block_id return; } } - LOG(ERROR) << "XXXX download link " << block_id.id.to_str(); callback_->download_block_proof_link(block_id, priority, td::Timestamp::in(10.0), std::move(promise)); } From 215b157177351d3c7e28e0f8644d54bc4463eb5b Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 13 May 2025 21:36:52 +0300 Subject: [PATCH 244/388] Fix downloading initial proof --- validator/manager-init.cpp | 25 ++++++++++++++++++------- validator/manager-init.hpp | 2 +- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index 3b57eb9b8..3552d9164 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -74,7 +74,7 @@ void ValidatorManagerMasterchainReiniter::got_masterchain_handle(BlockHandle han download_proof_link(); } -void ValidatorManagerMasterchainReiniter::download_proof_link() { +void ValidatorManagerMasterchainReiniter::download_proof_link(bool try_local) { if (handle_->id().id.seqno == 0) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); @@ -84,18 +84,29 @@ void ValidatorManagerMasterchainReiniter::download_proof_link() { td::Timestamp::in(3600), std::move(P)) .release(); } else { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { - LOG(WARNING) << "failed to download proof link: " << R.move_as_error(); + if (try_local) { + LOG(DEBUG) << "failed to get proof link from local import: " << R.move_as_error(); + } else { + LOG(WARNING) << "failed to download proof link: " << R.move_as_error(); + } delay_action( - [SelfId]() { td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_proof_link); }, + [SelfId]() { + td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_proof_link, false); + }, td::Timestamp::in(1.0)); } else { td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::downloaded_proof_link, R.move_as_ok()); } }); - td::actor::send_closure(manager_, &ValidatorManager::get_block_proof_link_from_import, handle_->id(), handle_->id(), - std::move(P)); + if (try_local) { + td::actor::send_closure(manager_, &ValidatorManager::get_block_proof_link_from_import, handle_->id(), + handle_->id(), std::move(P)); + } else { + td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_link_request, handle_->id(), 2, + std::move(P)); + } } } @@ -111,7 +122,7 @@ void ValidatorManagerMasterchainReiniter::downloaded_proof_link(td::BufferSlice auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), db = db_, proof](td::Result R) { if (R.is_error()) { LOG(WARNING) << "downloaded proof link failed: " << R.move_as_error(); - td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_proof_link); + td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_proof_link, false); } else { auto P = td::PromiseCreator::lambda([SelfId, handle = R.move_as_ok()](td::Result R) { R.ensure(); diff --git a/validator/manager-init.hpp b/validator/manager-init.hpp index 901b826bd..90dc5ccc9 100644 --- a/validator/manager-init.hpp +++ b/validator/manager-init.hpp @@ -44,7 +44,7 @@ class ValidatorManagerMasterchainReiniter : public td::actor::Actor { void start_up() override; void written_hardforks(); void got_masterchain_handle(BlockHandle handle); - void download_proof_link(); + void download_proof_link(bool try_local = true); void downloaded_proof_link(td::BufferSlice data); void downloaded_zero_state(); From 6d04184337bc3f578f26cff9c08e9b72d444a2c6 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 16 May 2025 16:17:48 +0300 Subject: [PATCH 245/388] Fix check in overlay-peers.cpp --- overlay/overlay-peers.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index 923f0fb12..52cc40cf6 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -196,7 +196,6 @@ void OverlayImpl::add_peer(OverlayNode node) { if (R.is_error()) { VLOG(OVERLAY_WARNING) << this << ": bad peer certificate node=" << node.adnl_id_short() << ": " << R.move_as_error(); - UNREACHABLE(); return; } } From d65e054734b216829ab5a4d189c22bfe988793b2 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 19 May 2025 17:56:59 +0300 Subject: [PATCH 246/388] Fix tracking storage stat updates --- crypto/block/transaction.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 1c4cbe35b..0dded1ff7 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -3199,6 +3199,11 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, } StorageStatCalculationContext context{is_account_stat}; StorageStatCalculationContext::Guard guard{&context}; + if (is_account_stat) { + storage_stats_updates.push_back(new_code); + storage_stats_updates.push_back(new_data); + storage_stats_updates.push_back(new_library); + } TRY_STATUS(storage_stat.replace_roots({new_code, new_data, new_library}, /* check_merkle_depth = */ true)); if (timer.elapsed() > 0.1) { LOG(INFO) << "Compute used storage (1) took " << timer.elapsed() << "s"; @@ -3224,9 +3229,6 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, if (is_account_stat) { // storage_stat will be reused in compute_state() new_account_storage_stat.value_force() = std::move(storage_stat); - storage_stats_updates.push_back(new_code); - storage_stats_updates.push_back(new_data); - storage_stats_updates.push_back(new_library); } return td::Status::OK(); } From 18a8ed8a5ab01107b98ebeedc12b9b5ad815756a Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 20 May 2025 15:11:21 +0300 Subject: [PATCH 247/388] Bugfix in AccountStorageStat::remove_cell --- crypto/block/account-storage-stat.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp index 82359f827..a6159c224 100644 --- a/crypto/block/account-storage-stat.cpp +++ b/crypto/block/account-storage-stat.cpp @@ -154,7 +154,7 @@ td::Status AccountStorageStat::remove_cell(const Ref& cell) { if (e.refcnt_diff < 0 && !e.refcnt) { TRY_STATUS(fetch_from_dict(e)); } - if (e.refcnt.value() + e.refcnt_diff != 0) { + if (e.refcnt_diff >= 0 || e.refcnt.value() + e.refcnt_diff != 0) { return td::Status::OK(); } bool spec; From ed3860f838900bb4c2780c7f03ca624ca12ee90d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 20 May 2025 15:11:54 +0300 Subject: [PATCH 248/388] Fix returning null as c4/c5 in RUNVM --- crypto/vm/vm.cpp | 19 ++++++++++++++----- doc/GlobalVersions.md | 5 ++++- test/regression-tests.ans | 2 +- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/crypto/vm/vm.cpp b/crypto/vm/vm.cpp index 272e3930e..356f0df22 100644 --- a/crypto/vm/vm.cpp +++ b/crypto/vm/vm.cpp @@ -785,11 +785,20 @@ void VmState::restore_parent_vm(int res) { cur_stack.push(std::move(child_state.stack->at(i))); } cur_stack.push_smallint(res); - if (parent->return_data) { - cur_stack.push_cell(child_state.get_committed_state().c4); - } - if (parent->return_actions) { - cur_stack.push_cell(child_state.get_committed_state().c5); + if (global_version >= 11 && !child_state.get_committed_state().committed) { + if (parent->return_data) { + cur_stack.push_null(); + } + if (parent->return_actions) { + cur_stack.push_null(); + } + } else { + if (parent->return_data) { + cur_stack.push_cell(child_state.get_committed_state().c4); + } + if (parent->return_actions) { + cur_stack.push_cell(child_state.get_committed_state().c5); + } } if (parent->return_gas) { cur_stack.push_smallint(child_state.gas.gas_consumed()); diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index 7bd9b0cc9..d4113b3f1 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -221,4 +221,7 @@ Reserve modes `+1`, `+4` and `+8` ("reserve all except", "add original balance" Along with the storage stat (cells and bits count), each account now stores the hash of the **storage dict**. **Storage dict** is the dictionary that stores refcnt for each cell in the account state. -This is required to help computing storage stats in the future, after collator-validator separation. \ No newline at end of file +This is required to help computing storage stats in the future, after collator-validator separation. + +### Other TVM changes +- Fix returning `null` as `c4` and `c5` (when VM state is not committed) in `RUNVM`. \ No newline at end of file diff --git a/test/regression-tests.ans b/test/regression-tests.ans index 22e7826ea..5831ebe74 100644 --- a/test/regression-tests.ans +++ b/test/regression-tests.ans @@ -25,7 +25,7 @@ Test_Fift_test_rist255_default f4d7558f200a656934f986145c19b1dedbe2ad029292a5a97 Test_Fift_test_secp256k1_default 3118450dace6af05fcdbd54a87d9446162ce11ac6ef6dfc57998cf113587d602 Test_Fift_test_sort2_default 9b57d47e6a10e7d1bbb565db35400debf2f963031f434742a702ec76555a5d3a Test_Fift_test_sort_default 9b57d47e6a10e7d1bbb565db35400debf2f963031f434742a702ec76555a5d3a -Test_Fift_test_tvm_runvm_default ff3d2a4031b543c18d6b555f0a1f1a891c7825e6d1e2e9beb4bf13b37441450b +Test_Fift_test_tvm_runvm_default b3e0d70c00f0e8bdf6d45c56c7c0817fa5be7a3fc540190786233a394de72e42 Test_Fift_testvm2_default 8a6e35fc0224398be9d2de39d31c86ea96965ef1eca2aa9e0af2303150ed4a7b Test_Fift_testvm3_default 3c1b77471c5fd914ed8b5f528b9faed618e278693f5030b953ff150e543864ae Test_Fift_testvm4_default 8a6e35fc0224398be9d2de39d31c86ea96965ef1eca2aa9e0af2303150ed4a7b From d8b5fad74e799c15388f297523a88312dafeb7d4 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 20 May 2025 20:10:23 +0300 Subject: [PATCH 249/388] Fix collated data calculation in Collator::process_account_storage_dict --- crypto/block/account-storage-stat.cpp | 6 +++- crypto/block/account-storage-stat.h | 3 +- crypto/block/transaction.cpp | 14 ++++++---- crypto/block/transaction.h | 4 +-- validator/impl/collator-impl.h | 1 + validator/impl/collator.cpp | 40 +++++++++++++-------------- 6 files changed, 37 insertions(+), 31 deletions(-) diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp index 82359f827..f25caadef 100644 --- a/crypto/block/account-storage-stat.cpp +++ b/crypto/block/account-storage-stat.cpp @@ -36,7 +36,8 @@ AccountStorageStat::AccountStorageStat(const AccountStorageStat* parent) CHECK(parent_->parent_ == nullptr); } -td::Status AccountStorageStat::replace_roots(std::vector> new_roots, bool check_merkle_depth) { +td::Status AccountStorageStat::replace_roots(std::vector> new_roots, bool check_merkle_depth, + td::HashSet* store_added) { std::erase_if(new_roots, [](const Ref& c) { return c.is_null(); }); if (new_roots.empty()) { roots_.clear(); @@ -73,6 +74,9 @@ td::Status AccountStorageStat::replace_roots(std::vector> new_root roots_ = std::move(new_roots); dict_up_to_date_ = false; for (auto& [_, e] : cache_) { + if (store_added && !e.exists && e.refcnt_diff > 0) { + store_added->insert(e.hash); + } TRY_STATUS(finalize_entry(e)); } return td::Status::OK(); diff --git a/crypto/block/account-storage-stat.h b/crypto/block/account-storage-stat.h index e939a0ff5..7322c991d 100644 --- a/crypto/block/account-storage-stat.h +++ b/crypto/block/account-storage-stat.h @@ -39,7 +39,8 @@ class AccountStorageStat { AccountStorageStat &operator=(const AccountStorageStat &other) = delete; AccountStorageStat &operator=(AccountStorageStat &&other) = default; - td::Status replace_roots(std::vector> new_roots, bool check_merkle_depth = false); + td::Status replace_roots(std::vector> new_roots, bool check_merkle_depth = false, + td::HashSet *store_added = nullptr); void add_hint(const td::HashSet &visited); td::uint64 get_total_cells() const { diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 0dded1ff7..b3d5c8c69 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -602,9 +602,11 @@ static td::Ref storage_without_extra_currencies(td::Ref> Account::compute_account_storage_dict() const { +td::Result> Account::compute_account_storage_dict(td::HashSet* storage_cells) const { if (storage.is_null()) { return td::Status::Error("cannot compute storage dict: empty storage"); } @@ -616,7 +618,7 @@ td::Result> Account::compute_account_storage_dict() const { if (storage_for_stat.is_null()) { return td::Status::Error("cannot compute storage dict: invalid storage"); } - TRY_STATUS(stat.replace_roots(storage_for_stat->prefetch_all_refs())); + TRY_STATUS(stat.replace_roots(storage_for_stat->prefetch_all_refs(), false, storage_cells)); // Root of AccountStorage is not counted in AccountStorageStat td::uint64 expected_cells = stat.get_total_cells() + 1; td::uint64 expected_bits = stat.get_total_bits() + storage->size(); @@ -3200,9 +3202,9 @@ td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, StorageStatCalculationContext context{is_account_stat}; StorageStatCalculationContext::Guard guard{&context}; if (is_account_stat) { - storage_stats_updates.push_back(new_code); - storage_stats_updates.push_back(new_data); - storage_stats_updates.push_back(new_library); + storage_stat_updates.push_back(new_code); + storage_stat_updates.push_back(new_data); + storage_stat_updates.push_back(new_library); } TRY_STATUS(storage_stat.replace_roots({new_code, new_data, new_library}, /* check_merkle_depth = */ true)); if (timer.elapsed() > 0.1) { @@ -3506,7 +3508,7 @@ bool Transaction::compute_state(const SerializeConfig& cfg) { AccountStorageStat& stats = new_account_storage_stat.value_force(); // Don't check Merkle depth and size here - they were checked in check_state_limits auto roots = new_storage_for_stat->prefetch_all_refs(); - storage_stats_updates.insert(storage_stats_updates.end(), roots.begin(), roots.end()); + storage_stat_updates.insert(storage_stat_updates.end(), roots.begin(), roots.end()); { StorageStatCalculationContext context{true}; StorageStatCalculationContext::Guard guard{&context}; diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index b7050afa0..4e9440e12 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -300,7 +300,7 @@ struct Account { bool set_address(ton::WorkchainId wc, td::ConstBitPtr new_addr); bool unpack(Ref account, ton::UnixTime now, bool special); bool init_new(ton::UnixTime now); - td::Result> compute_account_storage_dict() const; + td::Result> compute_account_storage_dict(td::HashSet* storage_cells = nullptr) const; td::Status init_account_storage_stat(Ref dict_root); bool deactivate(); bool recompute_tmp_addr(Ref& tmp_addr, int fixed_prefix_length, td::ConstBitPtr orig_addr_rewrite) const; @@ -397,7 +397,7 @@ struct Transaction { td::optional new_account_storage_stat; td::optional new_storage_dict_hash; bool gas_limit_overridden{false}; - std::vector> storage_stats_updates; + std::vector> storage_stat_updates; Transaction(const Account& _account, int ttype, ton::LogicalTime req_start_lt, ton::UnixTime _now, Ref _inmsg = {}); bool unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* cfg); diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 405fad334..303672f5d 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -227,6 +227,7 @@ class Collator final : public td::actor::Actor { vm::ProofStorageStat proof_stat; bool add_to_collated_data = false; std::vector> storage_stat_updates; + td::HashSet original_storage_cells; }; std::map account_storage_dicts_; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 6e7a5b87a..069abc0f9 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -2625,9 +2625,6 @@ bool Collator::init_account_storage_dict(block::Account& account) { if (!full_collated_data_ || is_masterchain() || !account.storage_dict_hash || account.storage.is_null()) { return true; } - if (account.storage_used.cells < 10) { // TODO: some other threshold? - return true; - } td::Bits256 storage_dict_hash = account.storage_dict_hash.value(); if (storage_dict_hash.is_zero()) { return true; @@ -2637,7 +2634,7 @@ bool Collator::init_account_storage_dict(block::Account& account) { dict.inited = true; // don't mark cells in account state as loaded during compute_account_storage_dict state_usage_tree_->set_ignore_loads(true); - auto res = account.compute_account_storage_dict(); + auto res = account.compute_account_storage_dict(&dict.original_storage_cells); state_usage_tree_->set_ignore_loads(false); if (res.is_error()) { return fatal_error(res.move_as_error_prefix(PSTRING() << "Failed to init account storage dict for " @@ -2740,37 +2737,38 @@ bool Collator::process_account_storage_dict(const block::Account& account) { td::HashSet visited; bool calculate_proof_size_diff = true; td::int64 proof_size_diff = 0; - std::function&, bool)> dfs = [&](const Ref& cell, bool from_old_state) { + std::function&)> dfs = [&](const Ref& cell) { if (cell.is_null() || !visited.emplace(cell->get_hash()).second) { return; } auto loaded_cell = cell->load_cell().move_as_ok(); - if (!loaded_cell.tree_node.empty()) { - from_old_state = true; - } - if (calculate_proof_size_diff && from_old_state) { - switch (collated_data_stat.get_cell_status(cell->get_hash())) { - case vm::ProofStorageStat::c_none: - proof_size_diff += vm::ProofStorageStat::estimate_serialized_size(loaded_cell.data_cell); + if (dict.original_storage_cells.contains(cell->get_hash())) { + if (calculate_proof_size_diff) { + switch (collated_data_stat.get_cell_status(cell->get_hash())) { + case vm::ProofStorageStat::c_none: + proof_size_diff += vm::ProofStorageStat::estimate_serialized_size(loaded_cell.data_cell); break; - case vm::ProofStorageStat::c_prunned: - proof_size_diff -= vm::ProofStorageStat::estimate_prunned_size(); + case vm::ProofStorageStat::c_prunned: + proof_size_diff -= vm::ProofStorageStat::estimate_prunned_size(); proof_size_diff += vm::ProofStorageStat::estimate_serialized_size(loaded_cell.data_cell); break; - case vm::ProofStorageStat::c_loaded: - break; + case vm::ProofStorageStat::c_loaded: + break; + } + } else { + collated_data_stat.add_loaded_cell(loaded_cell.data_cell, loaded_cell.virt.get_level()); } } - vm::CellSlice cs{std::move(loaded_cell)}; + vm::CellSlice cs{std::move(loaded_cell.data_cell)}; for (unsigned i = 0; i < cs.size_refs(); ++i) { - dfs(cs.prefetch_ref(i), from_old_state); + dfs(cs.prefetch_ref(i)); } }; // Visit cells that were used in storage stat computation to calculate collated data increase state_usage_tree_->set_ignore_loads(true); for (const auto& cell : dict.storage_stat_updates) { - dfs(cell, false); + dfs(cell); } state_usage_tree_->set_ignore_loads(false); @@ -2789,7 +2787,7 @@ bool Collator::process_account_storage_dict(const block::Account& account) { calculate_proof_size_diff = false; visited.clear(); for (const auto& cell : dict.storage_stat_updates) { - dfs(cell, false); + dfs(cell); } } @@ -5465,7 +5463,7 @@ void Collator::update_account_storage_dict_info(const block::transaction::Transa if (it == account_storage_dicts_.end()) { return; } - for (const Ref& cell : trans.storage_stats_updates) { + for (const Ref& cell : trans.storage_stat_updates) { if (cell.not_null()) { it->second.storage_stat_updates.push_back(cell); } From ffa872c2fb8677e7851dcca4a8dcc4d59013a7c7 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Wed, 21 May 2025 14:20:48 +0300 Subject: [PATCH 250/388] Update note on +2 RUNVM flag --- crypto/vm/contops.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/vm/contops.cpp b/crypto/vm/contops.cpp index 1ccf53daf..e4499aa02 100644 --- a/crypto/vm/contops.cpp +++ b/crypto/vm/contops.cpp @@ -214,7 +214,7 @@ int exec_ret_data(VmState* st) { // Mode: // +1 = same_c3 (set c3 to code) -// +2 = push_0 (push an implicit 0 before running the code) +// +2 = push_0 (push an implicit 0 before running the code); only works with +1 enabled // +4 = load c4 (persistent data) from stack and return its final value // +8 = load gas limit from stack and return consumed gas // +16 = load c7 (smart-contract context) From 7d9f081e38cf639604397e1007a3b0336afc7cb6 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 21 May 2025 16:15:43 +0300 Subject: [PATCH 251/388] Improve downloading shard archives --- validator/import-db-slice.cpp | 31 +++++++++++++++++++++++++------ validator/import-db-slice.hpp | 2 +- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/validator/import-db-slice.cpp b/validator/import-db-slice.cpp index 96dba7ea8..55c2cd794 100644 --- a/validator/import-db-slice.cpp +++ b/validator/import-db-slice.cpp @@ -54,7 +54,7 @@ ArchiveImporter::ArchiveImporter(std::string db_root, td::Ref void ArchiveImporter::start_up() { if (use_imported_files_) { LOG(INFO) << "Importing archive for masterchain seqno #" << start_import_seqno_ << " from disk"; - for (const std::string& path : to_import_files_) { + for (const std::string &path : to_import_files_) { LOG(INFO) << "Importing file from disk " << path; td::Status S = process_package(path, true); if (S.is_error()) { @@ -271,13 +271,13 @@ void ArchiveImporter::applied_masterchain_block(BlockHandle handle) { LOG(DEBUG) << "Applied masterchain block #" << handle->id().seqno(); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); - td::actor::send_closure(SelfId, &ArchiveImporter::got_new_materchain_state, + td::actor::send_closure(SelfId, &ArchiveImporter::got_new_masterchain_state, td::Ref(R.move_as_ok())); }); td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle, std::move(P)); } -void ArchiveImporter::got_new_materchain_state(td::Ref state) { +void ArchiveImporter::got_new_masterchain_state(td::Ref state) { last_masterchain_state_ = std::move(state); imported_any_ = true; check_masterchain_block(last_masterchain_state_->get_block_id().seqno() + 1); @@ -300,6 +300,20 @@ void ArchiveImporter::checked_all_masterchain_blocks() { }); } +static bool have_new_shards(td::Ref new_state, td::Ref old_state, + ShardIdFull shard_prefix) { + for (auto &shard : new_state->get_shards()) { + if (!shard_intersects(shard_prefix, shard->shard())) { + continue; + } + auto old_shard = old_state->get_shard_from_config(shard->shard()); + if (old_shard.is_null() || old_shard->top_block_id() != shard->top_block_id()) { + return true; + } + } + return false; +} + void ArchiveImporter::download_shard_archives(td::Ref start_state) { start_state_ = start_state; td::uint32 monitor_min_split = start_state->monitor_min_split_depth(basechainId); @@ -310,9 +324,14 @@ void ArchiveImporter::download_shard_archives(td::Ref start_st for (td::uint64 i = 0; i < (1ULL << monitor_min_split); ++i) { ShardIdFull shard_prefix{basechainId, (i * 2 + 1) << (64 - monitor_min_split - 1)}; if (opts_->need_monitor(shard_prefix, start_state)) { - ++pending_shard_archives_; - LOG(DEBUG) << "Downloading shard archive #" << start_import_seqno_ << " " << shard_prefix.to_str(); - download_shard_archive(shard_prefix); + if (have_new_shards(last_masterchain_state_, start_state_, shard_prefix)) { + ++pending_shard_archives_; + LOG(INFO) << "Downloading shard archive #" << start_import_seqno_ << " " << shard_prefix.to_str(); + download_shard_archive(shard_prefix); + } else { + LOG(INFO) << "Not downloading shard archive #" << start_import_seqno_ << " " << shard_prefix.to_str() + << " : no new shard blocks"; + } } } } else { diff --git a/validator/import-db-slice.hpp b/validator/import-db-slice.hpp index 4cf6ce40a..fc295216a 100644 --- a/validator/import-db-slice.hpp +++ b/validator/import-db-slice.hpp @@ -44,7 +44,7 @@ class ArchiveImporter : public td::actor::Actor { void check_masterchain_block(BlockSeqno seqno); void checked_masterchain_proof(BlockHandle handle, td::Ref data); void applied_masterchain_block(BlockHandle handle); - void got_new_materchain_state(td::Ref state); + void got_new_masterchain_state(td::Ref state); void checked_all_masterchain_blocks(); void download_shard_archives(td::Ref start_state); From 80a968ee201300c78c2164e727159c313b6e2e67 Mon Sep 17 00:00:00 2001 From: Evgeny Kapun Date: Thu, 22 May 2025 21:23:12 +0300 Subject: [PATCH 252/388] Remove pow-miner (#1683) --- assembly/native/build-macos-portable.sh | 4 +- assembly/native/build-macos-shared.sh | 4 +- assembly/native/build-ubuntu-appimages.sh | 4 +- assembly/native/build-ubuntu-portable.sh | 4 +- assembly/native/build-ubuntu-shared.sh | 4 +- assembly/native/build-windows-2019.bat | 4 +- assembly/native/build-windows.bat | 4 +- crypto/CMakeLists.txt | 8 - crypto/util/Miner.cpp | 129 ------------ crypto/util/Miner.h | 43 ---- crypto/util/pow-miner.cpp | 245 ---------------------- doc/TestGrams-HOWTO | 163 -------------- tonlib/CMakeLists.txt | 4 +- tonlib/tonlib/tonlib-cli.cpp | 219 ------------------- 14 files changed, 16 insertions(+), 823 deletions(-) delete mode 100644 crypto/util/Miner.cpp delete mode 100644 crypto/util/Miner.h delete mode 100644 crypto/util/pow-miner.cpp delete mode 100644 doc/TestGrams-HOWTO diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index c46ea416e..c3dec11d2 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -150,7 +150,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli blockchain-explorer \ tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ - lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server \ + lite-client validator-engine-console generate-random-id json2tlo dht-server \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \ test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont \ test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ @@ -159,7 +159,7 @@ if [ "$with_tests" = true ]; then else ninja storage-daemon storage-daemon-cli blockchain-explorer \ tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ - lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server \ + lite-client validator-engine-console generate-random-id json2tlo dht-server \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh index a32669490..c769e9579 100644 --- a/assembly/native/build-macos-shared.sh +++ b/assembly/native/build-macos-shared.sh @@ -75,7 +75,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli blockchain-explorer \ tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ - lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server \ + lite-client validator-engine-console generate-random-id json2tlo dht-server \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \ test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont \ test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ @@ -84,7 +84,7 @@ if [ "$with_tests" = true ]; then else ninja storage-daemon storage-daemon-cli blockchain-explorer \ tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ - lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server \ + lite-client validator-engine-console generate-random-id json2tlo dht-server \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi diff --git a/assembly/native/build-ubuntu-appimages.sh b/assembly/native/build-ubuntu-appimages.sh index 458ca9d0b..b21d7df8c 100644 --- a/assembly/native/build-ubuntu-appimages.sh +++ b/assembly/native/build-ubuntu-appimages.sh @@ -60,7 +60,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ - validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ + validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \ test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ @@ -69,7 +69,7 @@ ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib- test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ - validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ + validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ adnl-proxy create-state emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } diff --git a/assembly/native/build-ubuntu-portable.sh b/assembly/native/build-ubuntu-portable.sh index f208148c9..d2e733954 100644 --- a/assembly/native/build-ubuntu-portable.sh +++ b/assembly/native/build-ubuntu-portable.sh @@ -134,7 +134,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ - validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ + validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \ test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ @@ -143,7 +143,7 @@ ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib- test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ - validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ + validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ adnl-proxy create-state emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } diff --git a/assembly/native/build-ubuntu-shared.sh b/assembly/native/build-ubuntu-shared.sh index 68ab6aba6..3fc6db315 100644 --- a/assembly/native/build-ubuntu-shared.sh +++ b/assembly/native/build-ubuntu-shared.sh @@ -63,7 +63,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ - validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ + validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \ test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ @@ -72,7 +72,7 @@ ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib- test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ - validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ + validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ adnl-proxy create-state emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } diff --git a/assembly/native/build-windows-2019.bat b/assembly/native/build-windows-2019.bat index 0b96978ca..f5fd0be36 100644 --- a/assembly/native/build-windows-2019.bat +++ b/assembly/native/build-windows-2019.bat @@ -145,7 +145,7 @@ IF %errorlevel% NEQ 0 ( IF "%1"=="-t" ( ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ -tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^ +tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net ^ test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ @@ -156,7 +156,7 @@ IF %errorlevel% NEQ 0 ( ) ) else ( ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ -tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^ +tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator proxy-liteserver IF %errorlevel% NEQ 0 ( echo Can't compile TON diff --git a/assembly/native/build-windows.bat b/assembly/native/build-windows.bat index c51314078..41697eb0f 100644 --- a/assembly/native/build-windows.bat +++ b/assembly/native/build-windows.bat @@ -145,7 +145,7 @@ IF %errorlevel% NEQ 0 ( IF "%1"=="-t" ( ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ -tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^ +tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net ^ test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ @@ -156,7 +156,7 @@ IF %errorlevel% NEQ 0 ( ) ) else ( ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ -tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^ +tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator proxy-liteserver IF %errorlevel% NEQ 0 ( echo Can't compile TON diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 79d1117ea..cd2c7e47e 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -421,16 +421,8 @@ if (WINGETOPT_FOUND) target_link_libraries_system(tlbc wingetopt) endif() -add_library(pow-miner-lib util/Miner.cpp util/Miner.h) -target_include_directories(pow-miner-lib PUBLIC $) -target_link_libraries(pow-miner-lib PUBLIC ton_crypto) - -add_executable(pow-miner util/pow-miner.cpp) -target_link_libraries(pow-miner PRIVATE ton_crypto pow-miner-lib git) - if (WINGETOPT_FOUND) target_link_libraries_system(fift wingetopt) - target_link_libraries_system(pow-miner wingetopt) endif() add_executable(mintless-proof-generator util/mintless-proof-generator.cpp) diff --git a/crypto/util/Miner.cpp b/crypto/util/Miner.cpp deleted file mode 100644 index 3e26fac6b..000000000 --- a/crypto/util/Miner.cpp +++ /dev/null @@ -1,129 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "Miner.h" - -#include "td/utils/Random.h" -#include "td/utils/misc.h" -#include "td/utils/crypto.h" -#include "td/utils/port/Clocks.h" -#include - -namespace ton { -#pragma pack(push, 1) -struct HData { - unsigned char op[4]; - signed char flags = -4; - unsigned char expire[4] = {}, myaddr[32] = {}, rdata1[32] = {}, pseed[16] = {}, rdata2[32] = {}; - void inc() { - for (long i = 31; !(rdata1[i] = ++(rdata2[i])); --i) { - } - } - void set_expire(unsigned x) { - for (int i = 3; i >= 0; --i) { - expire[i] = (x & 0xff); - x >>= 8; - } - } - - td::Slice as_slice() const { - return td::Slice(reinterpret_cast(this), sizeof(*this)); - } -}; - -struct HDataEnv { - unsigned char d1 = 0, d2 = sizeof(HData) * 2; - HData body; - - td::Slice as_slice() const { - return td::Slice(reinterpret_cast(this), sizeof(*this)); - } - - void init(const block::StdAddress& my_address, td::Slice seed) { - std::memcpy(body.myaddr, my_address.addr.data(), sizeof(body.myaddr)); - body.flags = static_cast(my_address.workchain * 4 + my_address.bounceable); - CHECK(seed.size() == 16); - std::memcpy(body.pseed, seed.data(), 16); - std::memcpy(body.op, "Mine", 4); - - td::Random::secure_bytes(body.rdata1, 32); - std::memcpy(body.rdata2, body.rdata1, 32); - } -}; - -static_assert(std::is_trivially_copyable::value, "HDataEnv must be a trivial type"); -#pragma pack(pop) - -td::optional Miner::run(const Options& options) { - HDataEnv H; - H.init(options.my_address, td::Slice(options.seed.data(), options.seed.size())); - - td::Slice data = H.as_slice(); - CHECK(data.size() == 123); - - constexpr size_t prefix_size = 72; - constexpr size_t guard_pos = prefix_size - (72 - 28); - CHECK(0 <= guard_pos && guard_pos < 32); - size_t got_prefix_size = (const unsigned char*)H.body.rdata1 + guard_pos + 1 - (const unsigned char*)&H; - CHECK(prefix_size == got_prefix_size); - - auto head = data.substr(0, prefix_size); - auto tail = data.substr(prefix_size); - - SHA256_CTX shactx1, shactx2; - std::array hash; - SHA256_Init(&shactx1); - auto guard = head.back(); - - td::int64 i = 0, i0 = 0; - for (; i < options.max_iterations; i++) { - if (!(i & 0xfffff) || head.back() != guard) { - if (options.token_) { - break; - } - if (options.hashes_computed) { - *options.hashes_computed += i - i0; - } - i0 = i; - if (options.expire_at && options.expire_at.value().is_in_past(td::Timestamp::now())) { - break; - } - H.body.set_expire((unsigned)td::Clocks::system() + 900); - guard = head.back(); - SHA256_Init(&shactx1); - SHA256_Update(&shactx1, head.ubegin(), head.size()); - } - shactx2 = shactx1; - SHA256_Update(&shactx2, tail.ubegin(), tail.size()); - SHA256_Final(hash.data(), &shactx2); - - if (memcmp(hash.data(), options.complexity.data(), 32) < 0) { - // FOUND - if (options.hashes_computed) { - *options.hashes_computed += i - i0; - } - return H.body.as_slice().str(); - } - H.body.inc(); - } - if (options.hashes_computed) { - *options.hashes_computed += i - i0; - } - return {}; -} -} // namespace ton diff --git a/crypto/util/Miner.h b/crypto/util/Miner.h deleted file mode 100644 index f91a66866..000000000 --- a/crypto/util/Miner.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once - -#include "block/block.h" -#include "td/utils/CancellationToken.h" -#include "td/utils/optional.h" -#include "td/utils/Time.h" -#include -#include - -namespace ton { -class Miner { - public: - struct Options { - block::StdAddress my_address; - std::array seed; - std::array complexity; - td::optional expire_at; - td::int64 max_iterations = std::numeric_limits::max(); - std::atomic* hashes_computed{nullptr}; - td::CancellationToken token_; - }; - - static td::optional run(const Options& options); -}; -} // namespace ton diff --git a/crypto/util/pow-miner.cpp b/crypto/util/pow-miner.cpp deleted file mode 100644 index c065fdc70..000000000 --- a/crypto/util/pow-miner.cpp +++ /dev/null @@ -1,245 +0,0 @@ -/* - This file is part of TON Blockchain source code. - - TON Blockchain is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - TON Blockchain is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with TON Blockchain. If not, see . - - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement - from all source files in the program, then also delete it here. - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "common/bigint.hpp" -#include "common/refint.h" -#include "block/block.h" -#include "td/utils/benchmark.h" -#include "td/utils/filesystem.h" -#include "vm/boc.h" -#include "openssl/digest.hpp" -#include -#include -#include -#include -#include -#include "git.h" -#include "Miner.h" - -const char* progname; - -int usage() { - std::cerr - << "usage: " << progname - << " [-v][-B][-w] [-t] [ " - "] [-V]\n" - "Outputs a valid value for proof-of-work testgiver after computing at most hashes " - "or terminates with non-zero exit code\n"; - std::exit(2); -} - -td::RefInt256 parse_bigint(std::string str, int bits) { - int len = (int)str.size(); - auto num = td::make_refint(); - auto& x = num.write(); - if (len >= 3 && str[0] == '0' && str[1] == 'x') { - if (x.parse_hex(str.data() + 2, len - 2) != len - 2) { - return {}; - } - } else if (!len || x.parse_dec(str.data(), len) != len) { - return {}; - } - return x.unsigned_fits_bits(bits) ? std::move(num) : td::RefInt256{}; -} - -td::RefInt256 parse_bigint_chk(std::string str, int bits) { - auto x = parse_bigint(std::move(str), bits); - if (x.is_null()) { - std::cerr << "fatal: `" << str << "` is not an integer" << std::endl; - usage(); - } - return x; -} - -void parse_addr(std::string str, block::StdAddress& addr) { - if (!addr.parse_addr(str) || (addr.workchain != -1 && addr.workchain != 0)) { - std::cerr << "fatal: `" << str.c_str() << "` is not a valid blockchain address" << std::endl; - usage(); - } -} - -bool make_boc = false; -std::string boc_filename; -block::StdAddress miner_address; - -int verbosity = 0; -std::atomic hashes_computed{0}; -td::Timestamp start_at; - -void print_stats() { - auto passed = td::Timestamp::now().at() - start_at.at(); - if (passed < 1e-9) { - passed = 1; - } - std::cerr << "[ hashes computed: " << hashes_computed << " ]" << std::endl; - std::cerr << "[ speed: " << static_cast(hashes_computed) / passed << " hps ]" << std::endl; -} - -int found(td::Slice data) { - for (unsigned i = 0; i < data.size(); i++) { - printf("%02X", data.ubegin()[i]); - } - printf("\n"); - if (make_boc) { - vm::CellBuilder cb; - td::Ref ext_msg, body; - CHECK(cb.store_bytes_bool(data) // body - && cb.finalize_to(body) // -> body - && cb.store_long_bool(0x44, 7) // ext_message_in$10 ... - && cb.store_long_bool(miner_address.workchain, 8) // workchain - && cb.store_bytes_bool(miner_address.addr.as_slice()) // miner addr - && cb.store_long_bool(1, 6) // amount:Grams ... - && cb.store_ref_bool(std::move(body)) // body:^Cell - && cb.finalize_to(ext_msg)); - auto boc = vm::std_boc_serialize(std::move(ext_msg), 2).move_as_ok(); - std::cerr << "Saving " << boc.size() << " bytes of serialized external message into file `" << boc_filename << "`" - << std::endl; - td::write_file(boc_filename, boc).ensure(); - } - if (verbosity > 0) { - print_stats(); - } - std::exit(0); - return 0; -} - -void miner(const ton::Miner::Options& options) { - auto res = ton::Miner::run(options); - if (res) { - found(res.value()); - } -} - -class MinerBench : public td::Benchmark { - public: - std::string get_description() const override { - return "Miner"; - } - - void run(int n) override { - ton::Miner::Options options; - options.my_address.parse_addr("EQDU86V5wyPrLd4nQ0RHPcCLPZq_y1O5wFWyTsMw63vjXTOv"); - std::fill(options.seed.begin(), options.seed.end(), 0xa7); - std::fill(options.complexity.begin(), options.complexity.end(), 0); - options.max_iterations = n; - CHECK(!ton::Miner::run(options)); - } -}; - -int main(int argc, char* const argv[]) { - ton::Miner::Options options; - - progname = argv[0]; - int i, threads = 0; - bool bounce = false, benchmark = false; - while ((i = getopt(argc, argv, "bnvw:t:Bh:V")) != -1) { - switch (i) { - case 'v': - ++verbosity; - break; - case 'w': - threads = atoi(optarg); - CHECK(threads > 0 && threads <= 256); - break; - case 't': { - int timeout = atoi(optarg); - CHECK(timeout > 0); - options.expire_at = td::Timestamp::in(timeout); - break; - } - case 'B': - benchmark = true; - break; - case 'b': - bounce = true; - break; - case 'n': - bounce = false; - break; - case 'V': - std::cout << "pow-miner build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; - std::exit(0); - break; - case 'h': - return usage(); - default: - std::cerr << "unknown option" << std::endl; - return usage(); - } - } - if (benchmark && argc == optind) { - td::bench(MinerBench()); - return 0; - } - - if (argc != optind + 4 && argc != optind + 6) { - return usage(); - } - - parse_addr(argv[optind], options.my_address); - options.my_address.bounceable = bounce; - CHECK(parse_bigint_chk(argv[optind + 1], 128)->export_bytes(options.seed.data(), 16, false)); - - auto cmplx = parse_bigint_chk(argv[optind + 2], 256); - CHECK(cmplx->export_bytes(options.complexity.data(), 32, false)); - CHECK(!cmplx->unsigned_fits_bits(256 - 62)); - td::BigInt256 bigpower, hrate; - bigpower.set_pow2(256).mod_div(*cmplx, hrate); - long long hash_rate = hrate.to_long(); - options.max_iterations = parse_bigint_chk(argv[optind + 3], 50)->to_long(); - if (argc == optind + 6) { - make_boc = true; - parse_addr(argv[optind + 4], miner_address); - boc_filename = argv[optind + 5]; - } - - if (verbosity >= 2) { - std::cerr << "[ expected required hashes for success: " << hash_rate << " ]" << std::endl; - } - if (benchmark) { - td::bench(MinerBench()); - } - - start_at = td::Timestamp::now(); - - options.hashes_computed = &hashes_computed; - // may invoke several miner threads - if (threads <= 0) { - miner(options); - } else { - std::vector T; - for (int i = 0; i < threads; i++) { - T.emplace_back(miner, options); - } - for (auto& thr : T) { - thr.join(); - } - } - if (verbosity > 0) { - print_stats(); - } -} diff --git a/doc/TestGrams-HOWTO b/doc/TestGrams-HOWTO deleted file mode 100644 index 8156e9757..000000000 --- a/doc/TestGrams-HOWTO +++ /dev/null @@ -1,163 +0,0 @@ -The aim of this text is to describe how to quickly obtain a small amount of test Grams for test purposes, or a larger amount of test Grams for running a validator in the test network. We assume familiarity with the TON Blockchain LiteClient as explained in the LiteClient-HOWTO, and with the procedure required to compile the LiteClient and other software. For obtaining larger amount of test Grams required for running a validator, we also assume acquaintance with the FullNode-HOWTO and Validator-HOWTO. You will also need a dedicated server powerful enough for running a Full Node in order to obtain the larger amount of test Grams. Obtaining small amounts of test Grams does not require a dedicated server and may be done in several minutes on a home computer. - -1. Proof-of-Work TestGiver smart contracts -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In order to prevent a small number of malicious parties to collect all test Grams reserved for test purposes, a special kind of "Proof-of-Work TestGiver" smart contracts have been deployed in the masterchain of the test network. The addresses of these smart contacts are: - -Small testgivers (deliver from 10 to 100 test Grams every several minutes): - -kf-kkdY_B7p-77TLn2hUhM6QidWrrsl8FYWCIvBMpZKprBtN -kf8SYc83pm5JkGt0p3TQRkuiM58O9Cr3waUtR9OoFq716lN- -kf-FV4QTxLl-7Ct3E6MqOtMt-RGXMxi27g4I645lw6MTWraV -kf_NSzfDJI1A3rOM0GQm7xsoUXHTgmdhN5-OrGD8uwL2JMvQ -kf8gf1PQy4u2kURl-Gz4LbS29eaN4sVdrVQkPO-JL80VhOe6 -kf8kO6K6Qh6YM4ddjRYYlvVAK7IgyW8Zet-4ZvNrVsmQ4EOF -kf-P_TOdwcCh0AXHhBpICDMxStxHenWdLCDLNH5QcNpwMHJ8 -kf91o4NNTryJ-Cw3sDGt9OTiafmETdVFUMvylQdFPoOxIsLm -kf9iWhwk9GwAXjtwKG-vN7rmXT3hLIT23RBY6KhVaynRrIK7 -kf8JfFUEJhhpRW80_jqD7zzQteH6EBHOzxiOhygRhBdt4z2N - -Large testgivers (deliver 10000 test Grams at least once a day): - -kf8guqdIbY6kpMykR8WFeVGbZcP2iuBagXfnQuq0rGrxgE04 -kf9CxReRyaGj0vpSH0gRZkOAitm_yDHvgiMGtmvG-ZTirrMC -kf-WXA4CX4lqyVlN4qItlQSWPFIy00NvO2BAydgC4CTeIUme -kf8yF4oXfIj7BZgkqXM6VsmDEgCqWVSKECO1pC0LXWl399Vx -kf9nNY69S3_heBBSUtpHRhIzjjqY0ChugeqbWcQGtGj-gQxO -kf_wUXx-l1Ehw0kfQRgFtWKO07B6WhSqcUQZNyh4Jmj8R4zL -kf_6keW5RniwNQYeq3DNWGcohKOwI85p-V2MsPk4v23tyO3I -kf_NSPpF4ZQ7mrPylwk-8XQQ1qFD5evLnx5_oZVNywzOjSfh -kf-uNWj4JmTJefr7IfjBSYQhFbd3JqtQ6cxuNIsJqDQ8SiEA -kf8mO4l6ZB_eaMn1OqjLRrrkiBcSt7kYTvJC_dzJLdpEDKxn - -The first ten smart contracts enable a tester willing to obtain a small amount of test Grams to obtain some without spending too much computing power (typically, several minutes of work of a home computer should suffice). The remaining smart contracts are for obtaining larger amounts of test Grams required for running a validator in the test network; typically, a day of work of a dedicated server powerful enough to run a validator should suffice to obtain the necessary amount. - -You should randomly choose one of these "proof-of-work testgiver" smart contracts (from one of these two lists depending on your purpose) and obtain test Grams from this smart contract by a procedure similar to "mining". Essentially, you have to present an external message containing the proof of work and the address of your wallet to the chosen "proof-of-work testgiver" smart contract, and then the necessary amount will be sent to you. - -2. The "mining" process -~~~~~~~~~~~~~~~~~~~~~~~ - -In order to create an external message containing the "proof of work", you should run a special "mining" utility, compiled from the TON sources located in the GitHub repository. The utility is located in file './crypto/pow-miner' with respect to the build directory, and can be compiled by typing 'make pow-miner' in the build directory. - -However, before running "pow-miner", you need to know the actual values of "seed" and "complexity" parameters of the chosen "proof-of-work testgiver" smart contract. This can be done by invoking get-method "get_pow_params" of this smart contract. For instance, if you use testgiver smart contract kf-kkdY_B7p-77TLn2hUhM6QidWrrsl8FYWCIvBMpZKprBtN, you can simply type - - > runmethod kf-kkdY_B7p-77TLn2hUhM6QidWrrsl8FYWCIvBMpZKprBtN get_pow_params - -in the LiteClient console and obtain output like - - ... - arguments: [ 101616 ] - result: [ 229760179690128740373110445116482216837 53919893334301279589334030174039261347274288845081144962207220498432 100000000000 256 ] - remote result (not to be trusted): [ 229760179690128740373110445116482216837 53919893334301279589334030174039261347274288845081144962207220498432 100000000000 256 ] - -The two first large numbers in the "result:" line are the "seed" and the "complexity" of this smart contract. In this example, the seed is 229760179690128740373110445116482216837 and the complexity is 53919893334301279589334030174039261347274288845081144962207220498432. - -Next, you invoke the pow-miner utility as follows: - - $ crypto/pow-miner -vv -w -t - -Here is the number of CPU cores that you want to use for mining, is the maximal amount of seconds that the miner would run before admitting failure, is the address of your wallet (possibly not initialized yet), either in the masterchain or in the workchain (note that you need a masterchain wallet to control a validator), and are the most recent values obtained by running get-method 'get-pow-params', is the address of the chosen proof-of-work testgiver smartcontract, and is the filename of the output file where the external message with the proof of work will be saved in the case of success. - -For example, if your wallet address is kQBWkNKqzCAwA9vjMwRmg7aY75Rf8lByPA9zKXoqGkHi8SM7, you might run - - $ crypto/pow-miner -vv -w7 -t100 kQBWkNKqzCAwA9vjMwRmg7aY75Rf8lByPA9zKXoqGkHi8SM7 229760179690128740373110445116482216837 53919893334301279589334030174039261347274288845081144962207220498432 100000000000 kf-kkdY_B7p-77TLn2hUhM6QidWrrsl8FYWCIvBMpZKprBtN mined.boc - -The program will run for some time (at most 100 seconds in this case) and either terminate successfully (with zero exit code) and save the required proof of work into file "mined.boc", or terminate with a non-zero exit code if no proof of work was found. - -In the case of failure, you will see something like - - [ expected required hashes for success: 2147483648 ] - [ hashes computed: 1192230912 ] - -and the program will terminate with a non-zero exit code. Then you have to obtain the "seed" and "complexity" again (because they may have changed in the meantime as a result of processing requests from more successful miners) and re-run the "pow-miner" with the new parameters, repeating the process again and again until success. - -In the case of success, you will see something like - - [ expected required hashes for success: 2147483648 ] - 4D696E65005EFE49705690D2AACC203003DBE333046683B698EF945FF250723C0F73297A2A1A41E2F1A1F533B3BC4F5664D6C743C1C5C74BB3342F3A7314364B3D0DA698E6C80C1EA4ACDA33755876665780BAE9BE8A4D6385A1F533B3BC4F5664D6C743C1C5C74BB3342F3A7314364B3D0DA698E6C80C1EA4 - Saving 176 bytes of serialized external message into file `mined.boc` - [ hashes computed: 1122036095 ] - -Then you can use the LiteClient to send external message from file "mined.boc" to the proof-of-work testgiver smart contract (and you must do this as soon as possible): - -> sendfile mined.boc -... external message status is 1 - -You can wait for several seconds and check the state of your wallet: - -> last -> getaccount kQBWkNKqzCAwA9vjMwRmg7aY75Rf8lByPA9zKXoqGkHi8SM7 -... -account state is (account - addr:(addr_std - anycast:nothing workchain_id:0 address:x5690D2AACC203003DBE333046683B698EF945FF250723C0F73297A2A1A41E2F1) - storage_stat:(storage_info - used:(storage_used - cells:(var_uint len:1 value:1) - bits:(var_uint len:1 value:111) - public_cells:(var_uint len:0 value:0)) last_paid:1593722498 - due_payment:nothing) - storage:(account_storage last_trans_lt:7720869000002 - balance:(currencies - grams:(nanograms - amount:(var_uint len:5 value:100000000000)) - other:(extra_currencies - dict:hme_empty)) - state:account_uninit)) -x{C005690D2AACC203003DBE333046683B698EF945FF250723C0F73297A2A1A41E2F12025BC2F7F2341000001C169E9DCD0945D21DBA0004_} -last transaction lt = 7720869000001 hash = 83C15CDED025970FEF7521206E82D2396B462AADB962C7E1F4283D88A0FAB7D4 -account balance is 100000000000ng - -If nobody has sent a valid proof of work with this *seed* and *complexity* before you, the proof-of-work testgiver will accept your proof of work and this will be reflected in the balance of your wallet (10 or 20 seconds may elapse after sending the external message before this happens; be sure to make several attempts and type "last" each time before checking the balance of your wallet to refresh the LiteClient state). In the case of success, you will see that the balance has been increased (and even that your wallet has been created in uninitialized state if it did not exist before). In the case of failure, you will have to obtain the new "seed" and "complexity" and repeat the mining process from the very beginning. - -If you have been lucky and the balance of your wallet has been increased, you may want to initialize the wallet if it wasn't initialized before (more information on wallet creation can be found in LiteClient-HOWTO): - -> sendfile new-wallet-query.boc -... external message status is 1 -> last -> getaccount kQBWkNKqzCAwA9vjMwRmg7aY75Rf8lByPA9zKXoqGkHi8SM7 -... -account state is (account - addr:(addr_std - anycast:nothing workchain_id:0 address:x5690D2AACC203003DBE333046683B698EF945FF250723C0F73297A2A1A41E2F1) - storage_stat:(storage_info - used:(storage_used - cells:(var_uint len:1 value:3) - bits:(var_uint len:2 value:1147) - public_cells:(var_uint len:0 value:0)) last_paid:1593722691 - due_payment:nothing) - storage:(account_storage last_trans_lt:7720945000002 - balance:(currencies - grams:(nanograms - amount:(var_uint len:5 value:99995640998)) - other:(extra_currencies - dict:hme_empty)) - state:(account_active - ( - split_depth:nothing - special:nothing - code:(just - value:(raw@^Cell - x{} - x{FF0020DD2082014C97BA218201339CBAB19C71B0ED44D0D31FD70BFFE304E0A4F260810200D71820D70B1FED44D0D31FD3FFD15112BAF2A122F901541044F910F2A2F80001D31F3120D74A96D307D402FB00DED1A4C8CB1FCBFFC9ED54} - )) - data:(just - value:(raw@^Cell - x{} - x{00000001CE6A50A6E9467C32671667F8C00C5086FC8D62E5645652BED7A80DF634487715} - )) - library:hme_empty)))) -x{C005690D2AACC203003DBE333046683B698EF945FF250723C0F73297A2A1A41E2F1206811EC2F7F23A1800001C16B0BC790945D20D1929934_} - x{FF0020DD2082014C97BA218201339CBAB19C71B0ED44D0D31FD70BFFE304E0A4F260810200D71820D70B1FED44D0D31FD3FFD15112BAF2A122F901541044F910F2A2F80001D31F3120D74A96D307D402FB00DED1A4C8CB1FCBFFC9ED54} - x{00000001CE6A50A6E9467C32671667F8C00C5086FC8D62E5645652BED7A80DF634487715} -last transaction lt = 7720945000001 hash = 73353151859661AB0202EA5D92FF409747F201D10F1E52BD0CBB93E1201676BF -account balance is 99995640998ng - -Now you are a happy owner of 100 test Grams that can be used for whatever testing purposes you had in mind. Congratulations! - -3. Automating the mining process in the case of failure -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If you fail to obtain your test Grams for a long time, this may happen because too many other testers are simultaneously "mining" from the same proof-of-work testgiver smart contract. Maybe you should choose another proof-of-work testgiver smart contract from one of the lists given above. Alternatively, you can write a simple script to automatically run `pow-miner` with the correct parameters again and again until success (detected by checking the exit code of `pow-miner`) and invoke the lite-client with parameter -c 'sendfile mined.boc' to send the external message immediately after it is found. - diff --git a/tonlib/CMakeLists.txt b/tonlib/CMakeLists.txt index f86364411..f28176223 100644 --- a/tonlib/CMakeLists.txt +++ b/tonlib/CMakeLists.txt @@ -72,14 +72,14 @@ if (TONLIB_ENABLE_JNI AND NOT ANDROID) # jni is available by default on Android endif() add_executable(tonlib-cli tonlib/tonlib-cli.cpp) -target_link_libraries(tonlib-cli tonlib tdactor tdutils terminal pow-miner-lib git) +target_link_libraries(tonlib-cli tonlib tdactor tdutils terminal ton_crypto git) if (NOT CMAKE_CROSSCOMPILING) if (TONLIB_ENABLE_JNI) #FIXME #add_dependencies(tonlib tonlib_generate_java_api) endif() -endif() +endif() add_library(tonlibjson_private STATIC tonlib/ClientJson.cpp tonlib/ClientJson.h) target_include_directories(tonlibjson_private PUBLIC $ diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index 2c7100f24..45eec07de 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -52,7 +52,6 @@ #include "auto/tl/tonlib_api.hpp" -#include "crypto/util/Miner.h" #include "vm/boc.h" #include "vm/cells/CellBuilder.h" #include "lite-client/ext-client.h" @@ -329,11 +328,6 @@ class TonlibCli : public td::actor::Actor { td::TerminalIO::out() << "rwallet address \n"; td::TerminalIO::out() << "rwallet init [: ...]\n"; } - void pminer_help() { - td::TerminalIO::out() << "pminer help\n"; - td::TerminalIO::out() << "pminer start \n"; - td::TerminalIO::out() << "pminer stop\n"; - } void parse_line(td::BufferSlice line) { if (is_closing_) { @@ -393,7 +387,6 @@ class TonlibCli : public td::actor::Actor { dns_help(); pchan_help(); rwallet_help(); - pminer_help(); td::TerminalIO::out() << "blockmode auto|manual\tWith auto mode, all queries will be executed with respect to the latest block. " @@ -504,8 +497,6 @@ class TonlibCli : public td::actor::Actor { run_pchan_cmd(parser, std::move(cmd_promise)); } else if (cmd == "rwallet") { run_rwallet_cmd(parser, std::move(cmd_promise)); - } else if (cmd == "pminer") { - run_pminer_cmd(parser, std::move(cmd_promise)); } else if (cmd == "gethistory") { get_history(parser.read_word(), std::move(cmd_promise)); } else if (cmd == "guessrevision") { @@ -590,216 +581,6 @@ class TonlibCli : public td::actor::Actor { promise.set_error(td::Status::Error("Unknown command")); } - class PowMiner : public td::actor::Actor { - public: - class Callback { - public: - }; - struct Options { - Address giver_address; - Address my_address; - }; - - PowMiner(Options options, td::actor::ActorId client) - : options_(std::move(options)), client_(std::move(client)) { - } - - private: - Options options_; - td::actor::ActorId client_; - - td::optional miner_options_; - static constexpr double QUERY_EACH = 5.0; - td::Timestamp next_options_query_at_; - - bool need_run_miners_{false}; - td::CancellationTokenSource source_; - ton::Miner::Options miner_options_copy_; - std::size_t threads_alive_{0}; - std::vector threads_; - - bool close_flag_{false}; - - template - void send_query(QueryT query, td::Promise promise) { - td::actor::send_lambda(client_, - [self = client_, query = std::move(query), promise = std::move(promise)]() mutable { - self.get_actor_unsafe().make_request(std::move(query), std::move(promise)); - }); - } - - void start_up() override { - next_options_query_at_ = td::Timestamp::now(); - loop(); - } - void hangup() override { - close_flag_ = true; - source_.cancel(); - try_stop(); - } - void try_stop() { - if (threads_alive_ == 0) { - td::TerminalIO::out() << "pminer: stopped\n"; - stop(); - } - } - - void loop() override { - if (close_flag_) { - try_stop(); - return; - } - if (next_options_query_at_ && next_options_query_at_.is_in_past()) { - send_query(tonlib_api::smc_load(options_.giver_address.tonlib_api()), - promise_send_closure(td::actor::actor_id(this), &PowMiner::with_giver_state)); - next_options_query_at_ = {}; - } - - if (miner_options_ && threads_.empty() && need_run_miners_) { - td::TerminalIO::out() << "pminer: start workers\n"; - need_run_miners_ = false; - miner_options_copy_ = miner_options_.value(); - miner_options_copy_.token_ = source_.get_cancellation_token(); - auto n = td::thread::hardware_concurrency(); - threads_alive_ = n; - for (td::uint32 i = 0; i < n; i++) { - threads_.emplace_back([this, actor_id = actor_id(this)] { - auto res = ton::Miner::run(miner_options_copy_); - global_scheduler_->run_in_context_external( - [&] { send_closure(actor_id, &PowMiner::got_answer, std::move(res)); }); - }); - } - } - - alarm_timestamp().relax(next_options_query_at_); - } - - void got_answer(td::optional answer) { - source_.cancel(); - if (--threads_alive_ == 0) { - threads_.clear(); - } - if (answer) { - td::TerminalIO::out() << "pminer: got some result - sending query to the giver\n"; - vm::CellBuilder cb; - cb.store_bytes(answer.unwrap()); - send_query(tonlib_api::raw_createAndSendMessage( - options_.giver_address.tonlib_api(), "", - vm::std_boc_serialize(cb.finalize_novm()).move_as_ok().as_slice().str()), - promise_send_closure(td::actor::actor_id(this), &PowMiner::on_query_sent)); - } - loop(); - } - - void on_query_sent(td::Result> r_ok) { - LOG_IF(ERROR, r_ok.is_error()) << "pminer: " << r_ok.error(); - } - - void with_giver_state(td::Result> r_info) { - if (r_info.is_error()) { - return with_giver_info(r_info.move_as_error()); - } - send_query(tonlib_api::smc_runGetMethod(r_info.ok()->id_, - make_object("get_pow_params"), {}), - promise_send_closure(td::actor::actor_id(this), &PowMiner::with_giver_info)); - } - - void with_giver_info(td::Result> r_info) { - auto status = do_with_giver_info(std::move(r_info)); - LOG_IF(ERROR, status.is_error()) << "pminer: " << status; - next_options_query_at_ = td::Timestamp::in(QUERY_EACH); - return loop(); - } - - td::Result to_number(const tonlib_api::object_ptr& entry, - td::int32 bits) { - if (entry->get_id() != tonlib_api::tvm_stackEntryNumber::ID) { - return td::Status::Error("Expected stackEntryNumber"); - } - auto& number_str = static_cast(*entry.get()).number_->number_; - auto num = td::make_refint(); - if (num.write().parse_dec(number_str.data(), (int)number_str.size()) < (int)number_str.size()) { - return td::Status::Error("Failed to parse a number"); - } - if (!num->unsigned_fits_bits(bits)) { - return td::Status::Error(PSLICE() << "Number is too big " << num->to_dec_string() << " " << bits); - } - return num; - } - - td::Status do_with_giver_info(td::Result> r_info) { - TRY_RESULT(info, std::move(r_info)); - if (info->stack_.size() < 2) { - return td::Status::Error("Unexpected `get_pow_params` result format"); - } - TRY_RESULT(seed, to_number(info->stack_[0], 128)); - TRY_RESULT(complexity, to_number(info->stack_[1], 256)); - ton::Miner::Options options; - seed->export_bytes(options.seed.data(), 16, false); - complexity->export_bytes(options.complexity.data(), 32, false); - - TRY_RESULT(address, block::StdAddress::parse(options_.my_address.address->account_address_)); - options.my_address = std::move(address); - options.token_ = source_.get_cancellation_token(); - - if (miner_options_ && miner_options_.value().seed == options.seed) { - return td::Status::OK(); - } - - td::TerminalIO::out() << "pminer: got new options\n"; - td::BigInt256 bigpower, hrate; - bigpower.set_pow2(256).mod_div(*complexity, hrate); - long long hash_rate = hrate.to_long(); - td::TerminalIO::out() << "[ expected required hashes for success: " << hash_rate << " ]\n"; - miner_options_ = std::move(options); - need_run_miners_ = true; - source_.cancel(); - - return td::Status::OK(); - } - }; - - td::uint64 pow_miner_id_{0}; - std::map> pow_miners_; - - void pminer_start(td::ConstParser& parser, td::Promise promise) { - if (!pow_miners_.empty()) { - promise.set_error(td::Status::Error("One pminer is already running")); - } - TRY_RESULT_PROMISE_PREFIX(promise, giver_address, to_account_address(parser.read_word(), false), "giver address"); - TRY_RESULT_PROMISE_PREFIX(promise, my_address, to_account_address(parser.read_word(), false), "my address"); - - auto id = ++pow_miner_id_; - - PowMiner::Options options; - options.giver_address = std::move(giver_address); - options.my_address = std::move(my_address); - pow_miners_.emplace(id, td::actor::create_actor("PowMiner", std::move(options), client_.get())); - td::TerminalIO::out() << "Miner #" << id << " created"; - promise.set_value({}); - } - - void pminer_stop(td::ConstParser& parser, td::Promise promise) { - pow_miners_.clear(); - promise.set_value({}); - } - - void run_pminer_cmd(td::ConstParser& parser, td::Promise promise) { - auto cmd = parser.read_word(); - if (cmd == "help") { - pminer_help(); - return promise.set_value(td::Unit()); - } - - if (cmd == "start") { - return pminer_start(parser, std::move(promise)); - } - if (cmd == "stop") { - return pminer_stop(parser, std::move(promise)); - } - promise.set_error(td::Status::Error("Unknown command")); - } - void run_pchan_cmd(td::ConstParser& parser, td::Promise promise) { auto cmd = parser.read_word(); if (cmd == "help") { From d1e00f5e80e6759f295474ea63e75b83bdf0a3d2 Mon Sep 17 00:00:00 2001 From: Evgeny Kapun Date: Thu, 22 May 2025 21:24:03 +0300 Subject: [PATCH 253/388] Remove legacy Ed25519 code (#1684) --- CMakeLists.txt | 3 +- assembly/native/build-macos-portable.sh | 2 +- assembly/native/build-macos-shared.sh | 2 +- assembly/native/build-ubuntu-appimages.sh | 2 +- assembly/native/build-ubuntu-portable.sh | 2 +- assembly/native/build-ubuntu-shared.sh | 2 +- assembly/native/build-windows-2019.bat | 2 +- assembly/native/build-windows.bat | 2 +- crypto/CMakeLists.txt | 13 - crypto/Ed25519.cpp | 97 ------- crypto/ellcurve/Ed25519.cpp | 280 ------------------- crypto/ellcurve/Ed25519.h | 188 ------------- crypto/ellcurve/Fp25519.cpp | 33 --- crypto/ellcurve/Fp25519.h | 32 --- crypto/ellcurve/Montgomery.cpp | 138 ---------- crypto/ellcurve/Montgomery.h | 123 --------- crypto/ellcurve/TwEdwards.cpp | 255 ------------------ crypto/ellcurve/TwEdwards.h | 145 ---------- crypto/test/Ed25519.cpp | 1 - crypto/test/test-ed25519-crypto.cpp | 314 ---------------------- doc/Tests.md | 4 +- 21 files changed, 10 insertions(+), 1630 deletions(-) delete mode 100644 crypto/ellcurve/Ed25519.cpp delete mode 100644 crypto/ellcurve/Ed25519.h delete mode 100644 crypto/ellcurve/Fp25519.cpp delete mode 100644 crypto/ellcurve/Fp25519.h delete mode 100644 crypto/ellcurve/Montgomery.cpp delete mode 100644 crypto/ellcurve/Montgomery.h delete mode 100644 crypto/ellcurve/TwEdwards.cpp delete mode 100644 crypto/ellcurve/TwEdwards.h delete mode 100644 crypto/test/test-ed25519-crypto.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b002b2844..5cc0a32da 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -181,7 +181,7 @@ endif() if (TON_ARCH AND NOT MSVC) CHECK_CXX_COMPILER_FLAG( "-march=${TON_ARCH}" COMPILER_OPT_ARCH_SUPPORTED ) if (TON_ARCH STREQUAL "apple-m1") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=${TON_ARCH}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=${TON_ARCH}") elseif(COMPILER_OPT_ARCH_SUPPORTED) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=${TON_ARCH}") elseif(NOT TON_ARCH STREQUAL "native") @@ -534,7 +534,6 @@ endif() enable_testing() set(TEST_OPTIONS "--regression ${CMAKE_CURRENT_SOURCE_DIR}/test/regression-tests.ans --filter -Bench") separate_arguments(TEST_OPTIONS) -add_test(test-ed25519-crypto crypto/test-ed25519-crypto) add_test(test-ed25519 test-ed25519) add_test(test-bigint test-bigint) add_test(test-vm test-vm ${TEST_OPTIONS}) diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index c3dec11d2..e112ab499 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -152,7 +152,7 @@ if [ "$with_tests" = true ]; then tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ lite-client validator-engine-console generate-random-id json2tlo dht-server \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \ - test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont \ + test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont \ test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh index c769e9579..b21f21f89 100644 --- a/assembly/native/build-macos-shared.sh +++ b/assembly/native/build-macos-shared.sh @@ -77,7 +77,7 @@ if [ "$with_tests" = true ]; then tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ lite-client validator-engine-console generate-random-id json2tlo dht-server \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \ - test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont \ + test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont \ test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } diff --git a/assembly/native/build-ubuntu-appimages.sh b/assembly/native/build-ubuntu-appimages.sh index b21d7df8c..015b33cb7 100644 --- a/assembly/native/build-ubuntu-appimages.sh +++ b/assembly/native/build-ubuntu-appimages.sh @@ -62,7 +62,7 @@ if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ - adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \ + adnl-proxy create-state emulator test-ed25519 test-bigint \ test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver diff --git a/assembly/native/build-ubuntu-portable.sh b/assembly/native/build-ubuntu-portable.sh index d2e733954..3a65ddd2b 100644 --- a/assembly/native/build-ubuntu-portable.sh +++ b/assembly/native/build-ubuntu-portable.sh @@ -136,7 +136,7 @@ if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ - adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \ + adnl-proxy create-state emulator test-ed25519 test-bigint \ test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver diff --git a/assembly/native/build-ubuntu-shared.sh b/assembly/native/build-ubuntu-shared.sh index 3fc6db315..0bdf4072b 100644 --- a/assembly/native/build-ubuntu-shared.sh +++ b/assembly/native/build-ubuntu-shared.sh @@ -65,7 +65,7 @@ if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ - adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \ + adnl-proxy create-state emulator test-ed25519 test-bigint \ test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver diff --git a/assembly/native/build-windows-2019.bat b/assembly/native/build-windows-2019.bat index f5fd0be36..2c635d3a6 100644 --- a/assembly/native/build-windows-2019.bat +++ b/assembly/native/build-windows-2019.bat @@ -147,7 +147,7 @@ IF "%1"=="-t" ( ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ -test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net ^ +test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net ^ test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver IF %errorlevel% NEQ 0 ( diff --git a/assembly/native/build-windows.bat b/assembly/native/build-windows.bat index 41697eb0f..6ff7166af 100644 --- a/assembly/native/build-windows.bat +++ b/assembly/native/build-windows.bat @@ -147,7 +147,7 @@ IF "%1"=="-t" ( ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ -test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net ^ +test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net ^ test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver IF %errorlevel% NEQ 0 ( diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index cd2c7e47e..fd3c41c09 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -10,10 +10,6 @@ set(TON_CRYPTO_CORE_SOURCE common/bigexp.cpp common/bitstring.cpp common/util.cpp - ellcurve/Ed25519.cpp - ellcurve/Fp25519.cpp - ellcurve/Montgomery.cpp - ellcurve/TwEdwards.cpp openssl/bignum.cpp openssl/residue.cpp openssl/rand.cpp @@ -32,11 +28,6 @@ set(TON_CRYPTO_CORE_SOURCE common/linalloc.hpp common/promiseop.hpp - ellcurve/Ed25519.h - ellcurve/Fp25519.h - ellcurve/Montgomery.h - ellcurve/TwEdwards.h - openssl/bignum.h openssl/digest.hpp openssl/rand.hpp @@ -352,10 +343,6 @@ target_include_directories(ton_db PUBLIC $) target_link_libraries(ton_db PUBLIC tdutils tddb ton_crypto) -add_executable(test-ed25519-crypto test/test-ed25519-crypto.cpp) -target_include_directories(test-ed25519-crypto PUBLIC $) -target_link_libraries(test-ed25519-crypto PUBLIC ton_crypto) - add_library(fift-lib STATIC ${FIFT_SOURCE}) target_include_directories(fift-lib PUBLIC $) target_link_libraries(fift-lib PUBLIC ton_crypto) diff --git a/crypto/Ed25519.cpp b/crypto/Ed25519.cpp index c263a0cf1..fa5b4f2f6 100644 --- a/crypto/Ed25519.cpp +++ b/crypto/Ed25519.cpp @@ -18,12 +18,8 @@ */ #include "crypto/Ed25519.h" -#if TD_HAVE_OPENSSL - #include -#if OPENSSL_VERSION_NUMBER >= 0x10101000L && OPENSSL_VERSION_NUMBER != 0x20000000L || defined(OPENSSL_IS_BORINGSSL) - #include "td/utils/base64.h" #include "td/utils/BigNum.h" #include "td/utils/format.h" @@ -35,12 +31,6 @@ #include #include -#else - -#include "crypto/ellcurve/Ed25519.h" - -#endif - namespace td { Ed25519::PublicKey::PublicKey(SecureString octet_string) : octet_string_(std::move(octet_string)) { @@ -57,8 +47,6 @@ SecureString Ed25519::PrivateKey::as_octet_string() const { return octet_string_.copy(); } -#if OPENSSL_VERSION_NUMBER >= 0x10101000L && OPENSSL_VERSION_NUMBER != 0x20000000L || defined(OPENSSL_IS_BORINGSSL) - namespace detail { static Result X25519_key_from_PKEY(EVP_PKEY *pkey, bool is_private) { @@ -314,89 +302,4 @@ int Ed25519::version() { return OPENSSL_VERSION_NUMBER; } -#else - -Result Ed25519::generate_private_key() { - crypto::Ed25519::PrivateKey private_key; - if (!private_key.random_private_key(true)) { - return Status::Error("Can't generate random private key"); - } - SecureString private_key_buf(32); - if (!private_key.export_private_key(private_key_buf.as_mutable_slice())) { - return Status::Error("Failed to export private key"); - } - return PrivateKey(std::move(private_key_buf)); -} - -Result Ed25519::PrivateKey::get_public_key() const { - crypto::Ed25519::PrivateKey private_key; - if (!private_key.import_private_key(Slice(octet_string_).ubegin())) { - return Status::Error("Bad private key"); - } - SecureString public_key(32); - if (!private_key.get_public_key().export_public_key(public_key.as_mutable_slice())) { - return Status::Error("Failed to export public key"); - } - return PublicKey(std::move(public_key)); -} - -Result Ed25519::PrivateKey::as_pem(Slice password) const { - return Status::Error("Not supported"); -} - -Result Ed25519::PrivateKey::from_pem(Slice pem, Slice password) { - return Status::Error("Not supported"); -} - -Result Ed25519::PrivateKey::sign(Slice data) const { - crypto::Ed25519::PrivateKey private_key; - if (!private_key.import_private_key(Slice(octet_string_).ubegin())) { - return Status::Error("Bad private key"); - } - SecureString signature(crypto::Ed25519::sign_bytes, '\0'); - if (!private_key.sign_message(signature.as_mutable_slice(), data)) { - return Status::Error("Failed to sign message"); - } - return std::move(signature); -} - -Status Ed25519::PublicKey::verify_signature(Slice data, Slice signature) const { - if (signature.size() != crypto::Ed25519::sign_bytes) { - return Status::Error("Signature has invalid length"); - } - - crypto::Ed25519::PublicKey public_key; - if (!public_key.import_public_key(Slice(octet_string_).ubegin())) { - return Status::Error("Bad public key"); - } - if (public_key.check_message_signature(signature, data)) { - return Status::OK(); - } - return Status::Error("Wrong signature"); -} - -Result Ed25519::compute_shared_secret(const PublicKey &public_key, const PrivateKey &private_key) { - crypto::Ed25519::PrivateKey tmp_private_key; - if (!tmp_private_key.import_private_key(Slice(private_key.as_octet_string()).ubegin())) { - return Status::Error("Bad private key"); - } - crypto::Ed25519::PublicKey tmp_public_key; - if (!tmp_public_key.import_public_key(Slice(public_key.as_octet_string()).ubegin())) { - return Status::Error("Bad public key"); - } - SecureString shared_secret(32, '\0'); - if (!tmp_private_key.compute_shared_secret(shared_secret.as_mutable_slice(), tmp_public_key)) { - return Status::Error("Failed to compute shared secret"); - } - return std::move(shared_secret); -} - -int Ed25519::version() { - return 0; -} - -#endif - } // namespace td - -#endif diff --git a/crypto/ellcurve/Ed25519.cpp b/crypto/ellcurve/Ed25519.cpp deleted file mode 100644 index a38eb079e..000000000 --- a/crypto/ellcurve/Ed25519.cpp +++ /dev/null @@ -1,280 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "Ed25519.h" - -#include "td/utils/Random.h" - -namespace crypto { -namespace Ed25519 { - -bool all_bytes_same(const unsigned char *str, std::size_t size) { - unsigned char c = str[0]; - for (std::size_t i = 0; i < size; i++) { - if (str[i] != c) { - return false; - } - } - return true; -} - -void PublicKey::clear(void) { - if (inited != pk_empty) { - std::memset(pubkey, 0, pubkey_bytes); - PubKey.zeroize(); - PubKey_xz.zeroize(); - } - inited = pk_empty; -} - -PublicKey::PublicKey(const unsigned char pub_key[pubkey_bytes]) - : inited(pk_empty), PubKey(ellcurve::Fp25519()), PubKey_xz(ellcurve::Fp25519()) { - import_public_key(pub_key); -} - -PublicKey::PublicKey(const ellcurve::TwEdwardsCurve::SegrePoint &Pub_Key) - : inited(pk_empty), PubKey(ellcurve::Fp25519()), PubKey_xz(ellcurve::Fp25519()) { - import_public_key(Pub_Key); -} - -bool PublicKey::import_public_key(const unsigned char pub_key[pubkey_bytes]) { - clear(); - if (all_bytes_same(pub_key, pubkey_bytes)) { - return false; - } - bool ok = false; - PubKey = ellcurve::Ed25519().import_point(pub_key, ok); - if (!ok) { - clear(); - return false; - } - std::memcpy(pubkey, pub_key, pubkey_bytes); - PubKey_xz.X = PubKey.Z + PubKey.Y; - PubKey_xz.Z = PubKey.Z - PubKey.Y; - inited = pk_init; - return true; -} - -bool PublicKey::import_public_key(const ellcurve::TwEdwardsCurve::SegrePoint &Pub_Key) { - clear(); - if (!Pub_Key.is_valid()) { - return false; - } - PubKey = Pub_Key; - PubKey_xz.X = PubKey.Z + PubKey.Y; - PubKey_xz.Z = PubKey.Z - PubKey.Y; - inited = pk_init; - - if (!PubKey.export_point(pubkey)) { - clear(); - return false; - } - return true; -} - -bool PublicKey::export_public_key(unsigned char pubkey_buffer[pubkey_bytes]) const { - if (inited != pk_init) { - std::memset(pubkey_buffer, 0, pubkey_bytes); - return false; - } else { - std::memcpy(pubkey_buffer, pubkey, pubkey_bytes); - return true; - } -} - -bool PublicKey::check_message_signature(const unsigned char signature[sign_bytes], const unsigned char *message, - std::size_t msg_size) { - if (inited != pk_init) { - return false; - } - unsigned char hash[64]; - { - digest::SHA512 hasher(signature, 32); - hasher.feed(pubkey, 32); - hasher.feed(message, msg_size); - hasher.extract(hash); - } - auto &E = ellcurve::Ed25519(); - const arith::Bignum &L = E.get_ell(); - arith::Bignum H, S; - S.import_lsb(signature + 32, 32); - H.import_lsb(hash, 64); - H %= L; - H = L - H; - auto sG = E.power_gen(S); - auto hA = E.power_point(PubKey, H); - auto pR1 = E.add_points(sG, hA); - unsigned char pR1_bytes[32]; - if (!pR1.export_point(pR1_bytes)) { - return false; - } - return !std::memcmp(pR1_bytes, signature, 32); -} - -// --------------------- -class PrivateKey; - -bool PrivateKey::random_private_key(bool strong) { - inited = false; - if (!prng::rand_gen().rand_bytes(privkey, privkey_bytes, strong)) { - clear(); - return false; - } - return process_private_key(); -} - -void PrivateKey::clear(void) { - std::memset(privkey, 0, privkey_bytes); - std::memset(priv_salt, 0, sizeof(priv_salt)); - priv_exp.clear(); - PubKey.clear(); - inited = false; -} - -bool PrivateKey::import_private_key(const unsigned char pk[privkey_bytes]) { - clear(); - if (all_bytes_same(pk, privkey_bytes)) { - return false; - } - std::memcpy(privkey, pk, privkey_bytes); - return process_private_key(); -} - -bool PrivateKey::export_private_key(unsigned char pk[privkey_bytes]) const { // careful! - if (!inited) { - std::memset(pk, 0, privkey_bytes); - return false; - } else { - std::memcpy(pk, privkey, privkey_bytes); - return true; - } -} - -bool PrivateKey::process_private_key() { - unsigned char buff[64]; - digest::hash_str(buff, privkey, privkey_bytes); - std::memcpy(priv_salt, buff + 32, 32); - buff[0] = (unsigned char)(buff[0] & -8); - buff[31] = (unsigned char)((buff[31] | 0x40) & ~0x80); - priv_exp.import_lsb(buff, 32); - PubKey = ellcurve::Ed25519().power_gen(priv_exp, true); // uniform - inited = PubKey.ok(); - if (!inited) { - clear(); - } - return inited; -} - -bool PrivateKey::compute_shared_secret(unsigned char secret[shared_secret_bytes], const PublicKey &Pub) { - if (!inited || !Pub.ok()) { - std::memset(secret, 0, shared_secret_bytes); - *(long *)secret = static_cast(td::Random::fast_uint64()); - return false; - } - // uniform power! - auto P = ellcurve::Curve25519().power_xz(Pub.get_point_xz(), priv_exp); - if (P.is_infty()) { - std::memset(secret, 0, shared_secret_bytes); - *(long *)secret = static_cast(td::Random::fast_uint64()); - return false; - } - P.export_point_u(secret); - return true; -} - -bool PrivateKey::compute_temp_shared_secret(unsigned char secret[shared_secret_bytes], - const unsigned char temp_pub_key[pubkey_bytes]) { - PublicKey tempPubkey(temp_pub_key); - if (!tempPubkey.ok()) { - return false; - } - return compute_shared_secret(secret, tempPubkey); -} - -bool PrivateKey::sign_message(unsigned char signature[sign_bytes], const unsigned char *message, std::size_t msg_size) { - if (!inited) { - std::memset(signature, 0, sign_bytes); - return false; - } - unsigned char r_bytes[64]; - digest::hash_two_str(r_bytes, priv_salt, 32, message, msg_size); - const arith::Bignum &L = ellcurve::Ed25519().get_ell(); - arith::Bignum eR; - eR.import_lsb(r_bytes, 64); - eR %= L; - std::memset(r_bytes, 0, sizeof(r_bytes)); - - // uniform power - auto pR = ellcurve::Ed25519().power_gen(eR, true); - - auto ok = pR.export_point(signature, true); - (void)ok; - assert(ok); - { - digest::SHA512 hasher(signature, 32); - hasher.feed(PubKey.get_pubkey_ptr(), 32); - hasher.feed(message, msg_size); - hasher.extract(r_bytes); - } - arith::Bignum S; - S.import_lsb(r_bytes, 64); - S %= L; - S *= priv_exp; - S += eR; - S %= L; - eR.clear(); - S.export_lsb(signature + 32, 32); - return true; -} - -// --------------------------------- -class TempKeyGenerator; - -unsigned char *TempKeyGenerator::get_temp_private_key(unsigned char *to, const unsigned char *message, std::size_t size, - const unsigned char *rand, - std::size_t rand_size) { // rand may be 0 - digest::SHA256 hasher(message, size); - hasher.feed(random_salt, salt_size); - if (rand && rand_size) { - hasher.feed(rand, rand_size); - } - if (!to) { - to = buffer; - } - hasher.extract(to); - //++ *((long *)random_salt); - return to; -} - -void TempKeyGenerator::create_temp_private_key(PrivateKey &pk, const unsigned char *message, std::size_t size, - const unsigned char *rand, std::size_t rand_size) { - pk.import_private_key(get_temp_private_key(buffer, message, size, rand, rand_size)); - std::memset(buffer, 0, privkey_bytes); -} - -bool TempKeyGenerator::create_temp_shared_secret(unsigned char temp_pub_key[pubkey_bytes], - unsigned char shared_secret[shared_secret_bytes], - const PublicKey &recipientPubKey, const unsigned char *message, - std::size_t size, const unsigned char *rand, std::size_t rand_size) { - PrivateKey tmpPk; - create_temp_private_key(tmpPk, message, size, rand, rand_size); - return tmpPk.export_public_key(temp_pub_key) && tmpPk.compute_shared_secret(shared_secret, recipientPubKey); -} - -} // namespace Ed25519 -} // namespace crypto diff --git a/crypto/ellcurve/Ed25519.h b/crypto/ellcurve/Ed25519.h deleted file mode 100644 index 4b9e6348f..000000000 --- a/crypto/ellcurve/Ed25519.h +++ /dev/null @@ -1,188 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once -#include "ellcurve/Montgomery.h" -#include "ellcurve/TwEdwards.h" -#include "openssl/digest.hpp" -#include "openssl/rand.hpp" -#include -#include - -#include "td/utils/buffer.h" - -namespace crypto { -namespace Ed25519 { - -const int privkey_bytes = 32; -const int pubkey_bytes = 32; -const int sign_bytes = 64; -const int shared_secret_bytes = 32; - -class PublicKey { - enum { pk_empty, pk_xz, pk_init } inited; - unsigned char pubkey[pubkey_bytes]; - ellcurve::TwEdwardsCurve::SegrePoint PubKey; - ellcurve::MontgomeryCurve::PointXZ PubKey_xz; - - public: - PublicKey() : inited(pk_empty), PubKey(ellcurve::Fp25519()), PubKey_xz(ellcurve::Fp25519()) { - } - PublicKey(const unsigned char pub_key[pubkey_bytes]); - PublicKey(td::Slice pub_key) : PublicKey(pub_key.ubegin()) { - CHECK(pub_key.size() == pubkey_bytes); - } - PublicKey(const ellcurve::TwEdwardsCurve::SegrePoint &Pub_Key); - - bool import_public_key(const unsigned char pub_key[pubkey_bytes]); - bool import_public_key(td::Slice pub_key) { - CHECK(pub_key.size() == pubkey_bytes); - return import_public_key(pub_key.ubegin()); - } - bool import_public_key(const ellcurve::TwEdwardsCurve::SegrePoint &Pub_Key); - bool export_public_key(unsigned char pubkey_buffer[pubkey_bytes]) const; - bool export_public_key(td::MutableSlice pubk) const { - CHECK(pubk.size() == pubkey_bytes); - return export_public_key(pubk.ubegin()); - } - bool check_message_signature(const unsigned char signature[sign_bytes], const unsigned char *message, - std::size_t msg_size); - bool check_message_signature(td::Slice signature, td::Slice message) { - CHECK(signature.size() == sign_bytes); - return check_message_signature(signature.ubegin(), message.ubegin(), message.size()); - } - - void clear(); - bool ok() const { - return inited == pk_init; - } - - const unsigned char *get_pubkey_ptr() const { - return inited == pk_init ? pubkey : 0; - } - const ellcurve::TwEdwardsCurve::SegrePoint &get_point() const { - return PubKey; - } - const ellcurve::MontgomeryCurve::PointXZ &get_point_xz() const { - return PubKey_xz; - } -}; - -class PrivateKey { - public: - struct priv_key_no_copy {}; - PrivateKey() : inited(false) { - memset(privkey, 0, privkey_bytes); - } - PrivateKey(const unsigned char pk[privkey_bytes]) : inited(false) { - memset(privkey, 0, privkey_bytes); - import_private_key(pk); - } - PrivateKey(td::Slice pk) : inited(false) { - CHECK(pk.size() == privkey_bytes); - memset(privkey, 0, privkey_bytes); - import_private_key(pk.ubegin()); - } - ~PrivateKey() { - clear(); - } - bool random_private_key(bool strong = false); - bool import_private_key(const unsigned char pk[privkey_bytes]); - bool import_private_key(td::Slice pk) { - CHECK(pk.size() == privkey_bytes); - return import_private_key(pk.ubegin()); - } - bool export_private_key(unsigned char pk[privkey_bytes]) const; // careful! - bool export_private_key(td::MutableSlice pk) const { // careful! - return export_private_key(pk.ubegin()); - } - bool export_public_key(unsigned char pubk[pubkey_bytes]) const { - return PubKey.export_public_key(pubk); - } - bool export_public_key(td::MutableSlice pubk) const { - return PubKey.export_public_key(pubk); - } - void clear(); - bool ok() const { - return inited; - } - - // used for EdDSA (sign) - bool sign_message(unsigned char signature[sign_bytes], const unsigned char *message, std::size_t msg_size); - bool sign_message(td::MutableSlice signature, td::Slice message) { - CHECK(signature.size() == sign_bytes); - return sign_message(signature.ubegin(), message.ubegin(), message.size()); - } - // used for ECDH (encrypt / decrypt) - bool compute_shared_secret(unsigned char secret[shared_secret_bytes], const PublicKey &Pub); - bool compute_shared_secret(td::MutableSlice secret, const PublicKey &Pub) { - CHECK(secret.size() == shared_secret_bytes); - return compute_shared_secret(secret.ubegin(), Pub); - } - // used for EC asymmetric decryption - bool compute_temp_shared_secret(unsigned char secret[shared_secret_bytes], - const unsigned char temp_pub_key[pubkey_bytes]); - - const PublicKey &get_public_key() const { - return PubKey; - } - - private: - bool inited; - unsigned char privkey[privkey_bytes]; - unsigned char priv_salt[32]; - arith::Bignum priv_exp; - PublicKey PubKey; - - bool process_private_key(); - PrivateKey(const PrivateKey &) { - throw priv_key_no_copy(); - } - PrivateKey &operator=(const PrivateKey &) { - throw priv_key_no_copy(); - } -}; - -// use one TempKeyGenerator object a lot of times -class TempKeyGenerator { - enum { salt_size = 64 }; - unsigned char random_salt[salt_size]; - unsigned char buffer[privkey_bytes]; - - public: - TempKeyGenerator() { - prng::rand_gen().strong_rand_bytes(random_salt, salt_size); - } - ~TempKeyGenerator() { - memset(random_salt, 0, salt_size); - memset(buffer, 0, privkey_bytes); - } - - unsigned char *get_temp_private_key(unsigned char *to, const unsigned char *message, std::size_t size, - const unsigned char *rand = 0, std::size_t rand_size = 0); // rand may be 0 - void create_temp_private_key(PrivateKey &pk, const unsigned char *message, std::size_t size, - const unsigned char *rand = 0, std::size_t rand_size = 0); - - // sets temp_pub_key and shared_secret for one-time asymmetric encryption of message - bool create_temp_shared_secret(unsigned char temp_pub_key[pubkey_bytes], unsigned char secret[shared_secret_bytes], - const PublicKey &recipientPubKey, const unsigned char *message, std::size_t size, - const unsigned char *rand = 0, std::size_t rand_size = 0); -}; - -} // namespace Ed25519 -} // namespace crypto diff --git a/crypto/ellcurve/Fp25519.cpp b/crypto/ellcurve/Fp25519.cpp deleted file mode 100644 index 7b248f64a..000000000 --- a/crypto/ellcurve/Fp25519.cpp +++ /dev/null @@ -1,33 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "ellcurve/Fp25519.h" - -namespace ellcurve { -using namespace arith; - -const Bignum& P25519() { - static const Bignum P25519 = (Bignum(1) << 255) - 19; - return P25519; -} - -td::Ref Fp25519() { - static const td::Ref Fp25519(true, P25519()); - return Fp25519; -} -} // namespace ellcurve diff --git a/crypto/ellcurve/Fp25519.h b/crypto/ellcurve/Fp25519.h deleted file mode 100644 index b6b63be5e..000000000 --- a/crypto/ellcurve/Fp25519.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once -#include "common/refcnt.hpp" -#include "openssl/residue.h" - -namespace ellcurve { -using namespace arith; - -// returns 2^255-19 -const Bignum& P25519(); - -// residue ring modulo P25519 -td::Ref Fp25519(); - -} // namespace ellcurve diff --git a/crypto/ellcurve/Montgomery.cpp b/crypto/ellcurve/Montgomery.cpp deleted file mode 100644 index ed71910d5..000000000 --- a/crypto/ellcurve/Montgomery.cpp +++ /dev/null @@ -1,138 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "ellcurve/Montgomery.h" - -#include -#include - -namespace ellcurve { -using namespace arith; - -class MontgomeryCurve; - -void MontgomeryCurve::init() { - assert(!((a_short + 2) & 3) && a_short >= 0); -} - -void MontgomeryCurve::set_order_cofactor(const Bignum& order, int cof) { - assert(order > 0); - assert(cof >= 0); - assert(cof == 0 || (order % cof) == 0); - Order = order; - cofactor = cofactor_short = cof; - if (cof > 0) { - L = order / cof; - assert(is_prime(L)); - } - assert(!power_gen_xz(1).is_infty()); - assert(power_gen_xz(Order).is_infty()); -} - -// computes u(P+Q)*u(P-Q) as X/Z -MontgomeryCurve::PointXZ MontgomeryCurve::add_xz(const MontgomeryCurve::PointXZ& P, - const MontgomeryCurve::PointXZ& Q) const { - Residue u = (P.X + P.Z) * (Q.X - Q.Z); - Residue v = (P.X - P.Z) * (Q.X + Q.Z); - return MontgomeryCurve::PointXZ(sqr(u + v), sqr(u - v)); -} - -// computes u(2P) as X/Z -MontgomeryCurve::PointXZ MontgomeryCurve::double_xz(const MontgomeryCurve::PointXZ& P) const { - Residue u = sqr(P.X + P.Z); - Residue v = sqr(P.X - P.Z); - Residue w = u - v; - return PointXZ(u * v, w * (v + Residue(a_short, ring) * w)); -} - -MontgomeryCurve::PointXZ MontgomeryCurve::power_gen_xz(const Bignum& n) const { - return power_xz(Gu, n); -} - -MontgomeryCurve::PointXZ MontgomeryCurve::power_xz(const Residue& u, const Bignum& n) const { - return power_xz(PointXZ(u), n); -} - -// computes u([n]P) in form X/Z -MontgomeryCurve::PointXZ MontgomeryCurve::power_xz(const PointXZ& A, const Bignum& n) const { - assert(n >= 0); - if (n == 0) { - return PointXZ(ring); - } - - int k = n.num_bits(); - PointXZ P(A); - PointXZ Q(double_xz(P)); - for (int i = k - 2; i >= 0; --i) { - PointXZ PQ(add_xz(P, Q)); - PQ.X *= A.Z; - PQ.Z *= A.X; - if (n[i]) { - P = PQ; - Q = double_xz(Q); - } else { - Q = PQ; - P = double_xz(P); - } - } - return P; -} - -bool MontgomeryCurve::PointXZ::export_point_y(unsigned char buffer[32]) const { - if ((X + Z).is_zero()) { - std::memset(buffer, 0xff, 32); - return false; - } else { - get_y().extract().export_lsb(buffer, 32); - return true; - } -} - -bool MontgomeryCurve::PointXZ::export_point_u(unsigned char buffer[32]) const { - if (Z.is_zero()) { - std::memset(buffer, 0xff, 32); - return false; - } else { - get_u().extract().export_lsb(buffer, 32); - return true; - } -} - -MontgomeryCurve::PointXZ MontgomeryCurve::import_point_u(const unsigned char point[32]) const { - Bignum u; - u.import_lsb(point, 32); - u[255] = 0; - return PointXZ(Residue(u, ring)); -} - -MontgomeryCurve::PointXZ MontgomeryCurve::import_point_y(const unsigned char point[32]) const { - Bignum y; - y.import_lsb(point, 32); - y[255] = 0; - return PointXZ(Residue(y, ring), true); -} - -const MontgomeryCurve& Curve25519() { - static const MontgomeryCurve Curve25519 = [] { - MontgomeryCurve res(486662, 9, Fp25519()); - res.set_order_cofactor(hex_string{"80000000000000000000000000000000a6f7cef517bce6b2c09318d2e7ae9f68"}, 8); - return res; - }(); - return Curve25519; -} -} // namespace ellcurve diff --git a/crypto/ellcurve/Montgomery.h b/crypto/ellcurve/Montgomery.h deleted file mode 100644 index 94d852c06..000000000 --- a/crypto/ellcurve/Montgomery.h +++ /dev/null @@ -1,123 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once -#include -#include - -#include "openssl/bignum.h" -#include "openssl/residue.h" -#include "ellcurve/Fp25519.h" - -namespace ellcurve { -using namespace arith; - -class MontgomeryCurve { - td::Ref ring; - int A_short; // v^2 = u^2 + Au + 1 - int Gu_short; // u(G) - int a_short; // (A+2)/4 - Residue A_; - Residue Gu; - Bignum P_; - Bignum L; - Bignum Order; - Bignum cofactor; - int cofactor_short; - - void init(); - - public: - MontgomeryCurve(int _A, int _Gu, td::Ref _R) - : ring(_R) - , A_short(_A) - , Gu_short(_Gu) - , a_short((_A + 2) / 4) - , A_(_A, _R) - , Gu(_Gu, _R) - , P_(_R->get_modulus()) - , cofactor_short(0) { - init(); - } - - const Residue& get_gen_u() const { - return Gu; - } - const Bignum& get_ell() const { - return L; - } - const Bignum& get_order() const { - return Order; - } - td::Ref get_base_ring() const { - return ring; - } - const Bignum& get_p() const { - return P_; - } - - void set_order_cofactor(const Bignum& order, int cof); - - struct PointXZ { - Residue X, Z; - PointXZ(Residue x, Residue z) : X(x), Z(z) { - x.same_ring(z); - } - PointXZ(td::Ref r) : X(r->one()), Z(r->zero()) { - } - explicit PointXZ(Residue u) : X(u), Z(u.ring_of().one()) { - } - explicit PointXZ(Residue y, bool) : X(y.ring_of().one() + y), Z(y.ring_of().one() - y) { - } - PointXZ(const PointXZ& P) : X(P.X), Z(P.Z) { - } - PointXZ& operator=(const PointXZ& P) { - X = P.X; - Z = P.Z; - return *this; - } - Residue get_u() const { - return X * inverse(Z); - } - Residue get_v(bool sign_v = false) const; - bool is_infty() const { - return Z.is_zero(); - } - Residue get_y() const { - return (X - Z) * inverse(X + Z); - } - bool export_point_y(unsigned char buffer[32]) const; - bool export_point_u(unsigned char buffer[32]) const; - void zeroize() { - X = Z = Z.ring_of().zero(); - } - }; - - PointXZ power_gen_xz(const Bignum& n) const; - PointXZ power_xz(const Residue& u, const Bignum& n) const; - PointXZ power_xz(const PointXZ& P, const Bignum& n) const; - PointXZ add_xz(const PointXZ& P, const PointXZ& Q) const; - PointXZ double_xz(const PointXZ& P) const; - - PointXZ import_point_u(const unsigned char point[32]) const; - PointXZ import_point_y(const unsigned char point[32]) const; -}; - -const MontgomeryCurve& Curve25519(); - -} // namespace ellcurve diff --git a/crypto/ellcurve/TwEdwards.cpp b/crypto/ellcurve/TwEdwards.cpp deleted file mode 100644 index eb131f702..000000000 --- a/crypto/ellcurve/TwEdwards.cpp +++ /dev/null @@ -1,255 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "ellcurve/TwEdwards.h" -#include -#include - -namespace ellcurve { -using namespace arith; - -class TwEdwardsCurve; - -TwEdwardsCurve::TwEdwardsCurve(const Residue& _D, const Residue& _Gy, td::Ref _R) - : ring(_R) - , D(_D) - , D2(_D + _D) - , Gy(_Gy) - , P_(_R->get_modulus()) - , cofactor_short(0) - , G(_R) - , O(_R) - , table_lines(0) - , table() { - init(); -} - -TwEdwardsCurve::~TwEdwardsCurve() { -} - -void TwEdwardsCurve::init() { - assert(D != ring->zero() && D != ring->convert(-1)); - O.X = O.Z = ring->one(); - G = SegrePoint(*this, Gy, 0); - assert(!G.XY.is_zero()); -} - -void TwEdwardsCurve::set_order_cofactor(const Bignum& order, int cof) { - assert(order > 0); - assert(cof >= 0); - assert(cof == 0 || (order % cof) == 0); - Order = order; - cofactor = cofactor_short = cof; - if (cof > 0) { - L = order / cof; - assert(is_prime(L)); - assert(!power_gen(1).is_zero()); - assert(power_gen(L).is_zero()); - } -} - -TwEdwardsCurve::SegrePoint::SegrePoint(const TwEdwardsCurve& E, const Residue& y, bool x_sign) - : XY(y), X(E.get_base_ring()), Y(y), Z(E.get_base_ring()->one()) { - Residue x(y.ring_ref()); - if (E.recover_x(x, y, x_sign)) { - XY *= x; - X = x; - } else { - XY = Y = Z = E.get_base_ring()->zero(); - } -} - -bool TwEdwardsCurve::recover_x(Residue& x, const Residue& y, bool x_sign) const { - // recovers x from equation -x^2+y^2 = 1+d*x^2*y^2 - Residue z = inverse(ring->one() + D * sqr(y)); - if (z.is_zero()) { - return false; - } - z *= sqr(y) - ring->one(); - Residue t = sqrt(z); - if (sqr(t) == z) { - x = (t.extract().odd() == x_sign) ? t : -t; - //std::cout << "x=" << x << ", y=" << y << std::endl; - return true; - } else { - return false; - } -} - -void TwEdwardsCurve::add_points(SegrePoint& Res, const SegrePoint& P, const SegrePoint& Q) const { - Residue a((P.X + P.Y) * (Q.X + Q.Y)); - Residue b((P.X - P.Y) * (Q.X - Q.Y)); - Residue c(P.Z * Q.Z * ring->convert(2)); - Residue d(P.XY * Q.XY * D2); - Residue x_num(a - b); // 2(x1y2+x2y1) - Residue y_num(a + b); // 2(x1x2+y1y2) - Residue x_den(c + d); // 2(1+dx1x2y1y2) - Residue y_den(c - d); // 2(1-dx1x2y1y2) - Res.X = x_num * y_den; // x = x_num/x_den, y = y_num/y_den - Res.Y = y_num * x_den; - Res.XY = x_num * y_num; - Res.Z = x_den * y_den; -} - -TwEdwardsCurve::SegrePoint TwEdwardsCurve::add_points(const SegrePoint& P, const SegrePoint& Q) const { - SegrePoint Res(ring); - add_points(Res, P, Q); - return Res; -} - -void TwEdwardsCurve::double_point(SegrePoint& Res, const SegrePoint& P) const { - add_points(Res, P, P); -} - -TwEdwardsCurve::SegrePoint TwEdwardsCurve::double_point(const SegrePoint& P) const { - SegrePoint Res(ring); - double_point(Res, P); - return Res; -} - -// computes u([n]P) in form (xy,x,y,1)*Z -TwEdwardsCurve::SegrePoint TwEdwardsCurve::power_point(const SegrePoint& A, const Bignum& n, bool uniform) const { - assert(n >= 0); - if (n == 0) { - return O; - } - - int k = n.num_bits(); - SegrePoint P(A); - - if (uniform) { - SegrePoint Q(double_point(A)); - - for (int i = k - 2; i >= 0; --i) { - if (n[i]) { - add_points(P, P, Q); - double_point(Q, Q); - } else { - // we do more operations than necessary for uniformicity - add_points(Q, P, Q); - double_point(P, P); - } - } - } else { - for (int i = k - 2; i >= 0; --i) { - double_point(P, P); - if (n[i]) { - add_points(P, P, A); // may optimize further if A.z = 1 - } - } - } - return P; -} - -int TwEdwardsCurve::build_table() { - if (table.size()) { - return -1; - } - table_lines = (P_.num_bits() >> 2) + 2; - table.reserve(table_lines * 15 + 1); - table.emplace_back(get_base_point()); - for (int i = 0; i < table_lines; i++) { - for (int j = 0; j < 15; j++) { - table.emplace_back(add_points(table[15 * i + j], table[15 * i])); - } - } - return 1; -} - -int get_nibble(const Bignum& n, int idx) { - return n[idx * 4 + 3] * 8 + n[idx * 4 + 2] * 4 + n[idx * 4 + 1] * 2 + n[idx * 4]; -} - -TwEdwardsCurve::SegrePoint TwEdwardsCurve::power_gen(const Bignum& n, bool uniform) const { - if (uniform || n.num_bits() > table_lines * 4) { - return power_point(G, n, uniform); - } else if (n.is_zero()) { - return O; - } else { - int k = (n.num_bits() + 3) >> 2; - assert(k > 0 && k <= table_lines); - int x = get_nibble(n, k - 1); - assert(x > 0 && x < 16); - SegrePoint P(table[15 * (k - 1) + x - 1]); - for (int i = k - 2; i >= 0; i--) { - x = get_nibble(n, i); - assert(x >= 0 && x < 16); - if (x > 0) { - add_points(P, P, table[15 * i + x - 1]); - } - } - return P; - } -} - -bool TwEdwardsCurve::SegrePoint::export_point(unsigned char buffer[32], bool need_x) const { - if (!is_normalized()) { - if (Z.is_zero()) { - std::memset(buffer, 0xff, 32); - return false; - } - Residue f(inverse(Z)); - Bignum y((Y * f).extract()); - assert(!y[255]); - if (need_x) { - y[255] = (X * f).extract().odd(); - } - y.export_lsb(buffer, 32); - } else { - Bignum y(Y.extract()); - assert(!y[255]); - if (need_x) { - y[255] = X.extract().odd(); - } - y.export_lsb(buffer, 32); - } - return true; -} - -bool TwEdwardsCurve::SegrePoint::export_point_u(unsigned char buffer[32]) const { - if (Z == Y) { - std::memset(buffer, 0xff, 32); - return false; - } - Residue f(inverse(Z - Y)); - ((Z + Y) * f).extract().export_lsb(buffer, 32); - assert(!(buffer[31] & 0x80)); - return true; -} - -TwEdwardsCurve::SegrePoint TwEdwardsCurve::import_point(const unsigned char point[32], bool& ok) const { - Bignum y; - y.import_lsb(point, 32); - bool x_sign = y[255]; - y[255] = 0; - Residue yr(y, ring); - Residue xr(ring); - ok = recover_x(xr, yr, x_sign); - return ok ? SegrePoint(xr, yr) : SegrePoint(ring); -} - -const TwEdwardsCurve& Ed25519() { - static const TwEdwardsCurve Ed25519 = [] { - TwEdwardsCurve res(Fp25519()->frac(-121665, 121666), Fp25519()->frac(4, 5), Fp25519()); - res.set_order_cofactor(hex_string{"80000000000000000000000000000000a6f7cef517bce6b2c09318d2e7ae9f68"}, 8); - res.build_table(); - return res; - }(); - return Ed25519; -} -} // namespace ellcurve diff --git a/crypto/ellcurve/TwEdwards.h b/crypto/ellcurve/TwEdwards.h deleted file mode 100644 index c720ecca9..000000000 --- a/crypto/ellcurve/TwEdwards.h +++ /dev/null @@ -1,145 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once -#include -#include "common/refcnt.hpp" -#include "openssl/residue.h" -#include "ellcurve/Fp25519.h" - -namespace ellcurve { -using namespace arith; - -class TwEdwardsCurve { - public: - struct SegrePoint { - Residue XY, X, Y, Z; // if x=X/Z and y=Y/T, stores (xy,x,y,1)*Z*T - SegrePoint(td::Ref R) : XY(R), X(R), Y(R), Z(R) { - } - SegrePoint(const Residue& x, const Residue& y) : XY(x * y), X(x), Y(y), Z(y.ring_of().one()) { - } - SegrePoint(const TwEdwardsCurve& E, const Residue& y, bool x_sign); - SegrePoint(const SegrePoint& P) : XY(P.XY), X(P.X), Y(P.Y), Z(P.Z) { - } - SegrePoint& operator=(const SegrePoint& P) { - XY = P.XY; - X = P.X; - Y = P.Y; - Z = P.Z; - return *this; - } - bool is_zero() const { - return X.is_zero() && (Y == Z); - } - bool is_valid() const { - return (XY * Z == X * Y) && !(XY.is_zero() && X.is_zero() && Y.is_zero() && Z.is_zero()); - } - bool is_finite() const { - return !Z.is_zero(); - } - bool is_normalized() const { - return Z == Z.ring_of().one(); - } - SegrePoint& normalize() { - auto f = inverse(Z); - XY *= f; - X *= f; - Y *= f; - Z = Z.ring_of().one(); - return *this; - } - SegrePoint& zeroize() { - XY = X = Y = Z = Z.ring_of().zero(); - return *this; - } - bool export_point(unsigned char buffer[32], bool need_x = true) const; - bool export_point_y(unsigned char buffer[32]) const { - return export_point(buffer, false); - } - bool export_point_u(unsigned char buffer[32]) const; - Residue get_y() const { - return Y * inverse(Z); - } - Residue get_x() const { - return X * inverse(Z); - } - Residue get_u() const { - return (Z + Y) * inverse(Z - Y); - } - void negate() { - XY.negate(); - X.negate(); - } - }; - - private: - td::Ref ring; - Residue D; - Residue D2; - Residue Gy; - Bignum P_; - Bignum L; - Bignum Order; - Bignum cofactor; - int cofactor_short; - SegrePoint G; - SegrePoint O; - int table_lines; - std::vector table; - - void init(); - - public: - TwEdwardsCurve(const Residue& _D, const Residue& _Gy, td::Ref _R); - ~TwEdwardsCurve(); - const Residue& get_gen_y() const { - return Gy; - } - const Bignum& get_ell() const { - return L; - } - const Bignum& get_order() const { - return Order; - } - td::Ref get_base_ring() const { - return ring; - } - const Bignum& get_p() const { - return P_; - } - const SegrePoint& get_base_point() const { - return G; - } - - void set_order_cofactor(const Bignum& order, int cof); - bool recover_x(Residue& x, const Residue& y, bool x_sign) const; - - void add_points(SegrePoint& R, const SegrePoint& P, const SegrePoint& Q) const; - SegrePoint add_points(const SegrePoint& P, const SegrePoint& Q) const; - void double_point(SegrePoint& R, const SegrePoint& P) const; - SegrePoint double_point(const SegrePoint& P) const; - SegrePoint power_point(const SegrePoint& A, const Bignum& n, bool uniform = false) const; - SegrePoint power_gen(const Bignum& n, bool uniform = false) const; - int build_table(); - - SegrePoint import_point(const unsigned char point[32], bool& ok) const; -}; - -std::ostream& operator<<(std::ostream& os, const TwEdwardsCurve::SegrePoint& P); -const TwEdwardsCurve& Ed25519(); -} // namespace ellcurve diff --git a/crypto/test/Ed25519.cpp b/crypto/test/Ed25519.cpp index 131bfe92e..940b50796 100644 --- a/crypto/test/Ed25519.cpp +++ b/crypto/test/Ed25519.cpp @@ -17,7 +17,6 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "crypto/Ed25519.h" -#include "ellcurve/Ed25519.h" #include "td/utils/logging.h" #include "td/utils/misc.h" diff --git a/crypto/test/test-ed25519-crypto.cpp b/crypto/test/test-ed25519-crypto.cpp deleted file mode 100644 index 3e3dab896..000000000 --- a/crypto/test/test-ed25519-crypto.cpp +++ /dev/null @@ -1,314 +0,0 @@ -/* - This file is part of TON Blockchain source code. - - TON Blockchain is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - TON Blockchain is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with TON Blockchain. If not, see . - - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement - from all source files in the program, then also delete it here. - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include -#include -#include -#include - -#include "crypto/ellcurve/Ed25519.h" - -static void my_assert_impl(bool cond, const char* str, const char* file, int line) { - if (!cond) { - std::cerr << "Failed " << str << " in " << file << " at " << line << ".\n"; - } -} -#define my_assert(x) my_assert_impl(x, #x, __FILE__, __LINE__) - -void print_buffer(const unsigned char buffer[32]) { - for (int i = 0; i < 32; i++) { - char buff[4]; - sprintf(buff, "%02x", buffer[i]); - std::cout << buff; - } -} - -std::string buffer_to_hex(const unsigned char* buffer, std::size_t size = 32) { - const char* hex = "0123456789ABCDEF"; - std::string res(2 * size, '\0'); - for (std::size_t i = 0; i < size; i++) { - auto c = buffer[i]; - res[2 * i] = hex[c & 15]; - res[2 * i + 1] = hex[c >> 4]; - } - return res; -} - -// export of (17/12)G on twisted Edwards curve -unsigned char test_vector1[32] = {0xfc, 0xb7, 0x42, 0x1e, 0x26, 0xad, 0x1b, 0x17, 0xf6, 0xb1, 0x52, - 0x0c, 0xdb, 0x8a, 0x64, 0x7d, 0x28, 0xa7, 0x56, 0x69, 0xd4, 0xb6, - 0x0c, 0xec, 0x63, 0x72, 0x5e, 0xe6, 0x32, 0x4d, 0xf7, 0xe6}; - -unsigned char rfc7748_output[32] = { - 0x95, 0xcb, 0xde, 0x94, 0x76, 0xe8, 0x90, 0x7d, 0x7a, 0xad, 0xe4, 0x5c, 0xb4, 0xb8, 0x73, 0xf8, - 0x8b, 0x59, 0x5a, 0x68, 0x79, 0x9f, 0xa1, 0x52, 0xe6, 0xf8, 0xf7, 0x64, 0x7a, 0xac, 0x79, 0x57, -}; - -bool test_ed25519_impl(void) { - std::cout << "************** Testing Curve25519 / Ed25519 operations ************\n"; - auto& E = ellcurve::Curve25519(); - auto& Edw = ellcurve::Ed25519(); - arith::Bignum L = E.get_ell(); - my_assert(arith::is_prime(L)); - my_assert(L == Edw.get_ell()); - arith::ResidueRing Fl(L); - arith::Bignum s = Fl.frac(17, 12).extract(); - arith::Bignum t = Fl.frac(12, 17).extract(); - std::cout << "l = " << L << std::endl; - std::cout << "s = 17/12 mod l = " << s << std::endl; - std::cout << "t = 12/17 mod l = " << t << std::endl; - auto sG = E.power_gen_xz(s); - auto u_sG = sG.get_u(); - std::cout << "Curve25519 u(sG) = " << sG.get_u().extract() << std::endl; - std::cout << "Curve25519 y(sG) = " << sG.get_y().extract() << std::endl; - auto sG1 = Edw.power_gen(s); - std::cout << "Ed25519 u(sG) = " << sG1.get_u().extract() << std::endl; - std::cout << "Ed25519 y(sG) = " << sG1.get_y().extract() << std::endl; - std::cout << "Ed25519 x(sG) = " << sG1.get_x().extract() << std::endl; - my_assert(sG1.get_x().extract() != sG1.get_y().extract()); - my_assert(sG.get_u() == sG1.get_u()); - my_assert(sG.get_y() == sG1.get_y()); - - my_assert( - sG1.get_x().extract() == - arith::Bignum(arith::dec_string{"9227429025021714590777223519505276506601225973596506606120015751301699519597"})); - my_assert(sG1.get_y().extract() == - arith::Bignum( - arith::dec_string{"46572854587220149033453000581008590225032365765275643343836649812808016508924"})); - - auto sG2 = Edw.power_gen(s, true); - my_assert(sG1.get_u() == sG2.get_u()); - my_assert(sG1.get_y() == sG2.get_y()); - unsigned char buff[32]; - std::memset(buff, 0, 32); - my_assert(sG1.export_point(buff)); - std::cout << "sG export = " << buffer_to_hex(buff) << std::endl; - bool ok; - auto sG3 = Edw.import_point(buff, ok); - my_assert(ok); - my_assert(!std::memcmp(buff, test_vector1, 32)); - my_assert(sG3.get_u() == sG1.get_u()); - my_assert(sG2.get_x() == sG2.get_x()); - my_assert(sG2.get_y() == sG2.get_y()); - - auto stG = E.power_xz(u_sG, t); - std::cout << "Curve25519 u(stG) = " << stG.get_u().extract() << std::endl; - my_assert(stG.get_u().extract() == 9); - auto stG1 = Edw.power_point(sG1, t); - std::cout << "Ed25519 u(stG) = " << stG1.get_u().extract() << std::endl; - my_assert(stG1.get_u().extract() == 9); - stG1.normalize(); - my_assert(stG1.XY == Edw.get_base_point().XY); - my_assert(stG1.X == Edw.get_base_point().X); - my_assert(stG1.Y == Edw.get_base_point().Y); - my_assert(stG1.Z == Edw.get_base_point().Z); - auto stG2 = Edw.power_point(sG2, t, true); - my_assert(stG2.get_u().extract() == 9); - stG2.normalize(); - my_assert(stG2.XY == stG1.XY && stG2.X == stG1.X && stG2.Y == stG1.Y); - auto stG3 = Edw.power_point(sG3, t).normalize(); - auto stG4 = Edw.power_point(sG3, t, true).normalize(); - my_assert(stG3.XY == stG1.XY && stG3.X == stG1.X && stG3.Y == stG1.Y); - my_assert(stG4.XY == stG1.XY && stG4.X == stG1.X && stG4.Y == stG1.Y); - - // RFC7748 test vector - auto u = - arith::Bignum(arith::dec_string{"8883857351183929894090759386610649319417338800022198945255395922347792736741"}); - //u[255] = 0; - auto n = - arith::Bignum(arith::dec_string{"35156891815674817266734212754503633747128614016119564763269015315466259359304"}); - //n[255] = 0; n[254] = 1; - //n[0] = n[1] = n[2] = 0; - auto umodp = arith::Residue(u, E.get_base_ring()); - auto nP = E.power_xz(umodp, n); - std::cout << "u(P) = " << u.to_hex() << std::endl; - std::cout << "n = " << n.to_hex() << std::endl; - std::cout << "u(nP) = " << nP.get_u().extract().to_hex() << std::endl; - unsigned char buffer[32]; - std::memset(buffer, 0, 32); - nP.export_point_u(buffer); - std::cout << "u(nP) export = " << buffer_to_hex(buffer) << std::endl; - my_assert(!std::memcmp(buffer, rfc7748_output, 32)); - - std::cout << "********* ok\n\n"; - return true; -} - -unsigned char fixed_privkey[32] = "abacabadabacabaeabacabadabacaba"; -unsigned char fixed_pubkey[32] = {0x6f, 0x9e, 0x5b, 0xde, 0xce, 0x87, 0x21, 0xeb, 0x57, 0x37, 0xfb, - 0xb5, 0x92, 0x28, 0xba, 0x07, 0xf7, 0x88, 0x0f, 0x73, 0xce, 0x5b, - 0xfa, 0xa1, 0xb7, 0x15, 0x73, 0x03, 0xd4, 0x20, 0x1e, 0x74}; - -unsigned char rfc8032_secret_key1[32] = {0x9d, 0x61, 0xb1, 0x9d, 0xef, 0xfd, 0x5a, 0x60, 0xba, 0x84, 0x4a, - 0xf4, 0x92, 0xec, 0x2c, 0xc4, 0x44, 0x49, 0xc5, 0x69, 0x7b, 0x32, - 0x69, 0x19, 0x70, 0x3b, 0xac, 0x03, 0x1c, 0xae, 0x7f, 0x60}; - -unsigned char rfc8032_public_key1[32] = {0xd7, 0x5a, 0x98, 0x01, 0x82, 0xb1, 0x0a, 0xb7, 0xd5, 0x4b, 0xfe, - 0xd3, 0xc9, 0x64, 0x07, 0x3a, 0x0e, 0xe1, 0x72, 0xf3, 0xda, 0xa6, - 0x23, 0x25, 0xaf, 0x02, 0x1a, 0x68, 0xf7, 0x07, 0x51, 0x1a}; - -unsigned char rfc8032_signature1[64] = { - 0xe5, 0x56, 0x43, 0x00, 0xc3, 0x60, 0xac, 0x72, 0x90, 0x86, 0xe2, 0xcc, 0x80, 0x6e, 0x82, 0x8a, - 0x84, 0x87, 0x7f, 0x1e, 0xb8, 0xe5, 0xd9, 0x74, 0xd8, 0x73, 0xe0, 0x65, 0x22, 0x49, 0x01, 0x55, - 0x5f, 0xb8, 0x82, 0x15, 0x90, 0xa3, 0x3b, 0xac, 0xc6, 0x1e, 0x39, 0x70, 0x1c, 0xf9, 0xb4, 0x6b, - 0xd2, 0x5b, 0xf5, 0xf0, 0x59, 0x5b, 0xbe, 0x24, 0x65, 0x51, 0x41, 0x43, 0x8e, 0x7a, 0x10, 0x0b, -}; - -unsigned char rfc8032_secret_key2[32] = { - 0xc5, 0xaa, 0x8d, 0xf4, 0x3f, 0x9f, 0x83, 0x7b, 0xed, 0xb7, 0x44, 0x2f, 0x31, 0xdc, 0xb7, 0xb1, - 0x66, 0xd3, 0x85, 0x35, 0x07, 0x6f, 0x09, 0x4b, 0x85, 0xce, 0x3a, 0x2e, 0x0b, 0x44, 0x58, 0xf7, -}; - -unsigned char rfc8032_public_key2[32] = { - 0xfc, 0x51, 0xcd, 0x8e, 0x62, 0x18, 0xa1, 0xa3, 0x8d, 0xa4, 0x7e, 0xd0, 0x02, 0x30, 0xf0, 0x58, - 0x08, 0x16, 0xed, 0x13, 0xba, 0x33, 0x03, 0xac, 0x5d, 0xeb, 0x91, 0x15, 0x48, 0x90, 0x80, 0x25, -}; - -unsigned char rfc8032_message2[2] = {0xaf, 0x82}; - -unsigned char rfc8032_signature2[64] = { - 0x62, 0x91, 0xd6, 0x57, 0xde, 0xec, 0x24, 0x02, 0x48, 0x27, 0xe6, 0x9c, 0x3a, 0xbe, 0x01, 0xa3, - 0x0c, 0xe5, 0x48, 0xa2, 0x84, 0x74, 0x3a, 0x44, 0x5e, 0x36, 0x80, 0xd7, 0xdb, 0x5a, 0xc3, 0xac, - 0x18, 0xff, 0x9b, 0x53, 0x8d, 0x16, 0xf2, 0x90, 0xae, 0x67, 0xf7, 0x60, 0x98, 0x4d, 0xc6, 0x59, - 0x4a, 0x7c, 0x15, 0xe9, 0x71, 0x6e, 0xd2, 0x8d, 0xc0, 0x27, 0xbe, 0xce, 0xea, 0x1e, 0xc4, 0x0a, -}; - -bool test_ed25519_crypto() { - std::cout << "************** Testing Curve25519 / Ed25519 cryptographic primitives ************\n"; - crypto::Ed25519::PrivateKey PK1, PK2, PK3, PK4, PK5; - PK1.random_private_key(); - PK2.import_private_key(fixed_privkey); - unsigned char priv2_export[32]; - bool ok = PK1.export_private_key(priv2_export); - std::cout << "PK1 = " << ok << " " << buffer_to_hex(priv2_export) << std::endl; - my_assert(ok); - ok = PK2.export_private_key(priv2_export); - std::cout << "PK2 = " << ok << " " << buffer_to_hex(priv2_export) << std::endl; - my_assert(ok); - PK3.import_private_key(priv2_export); - std::cout << "PK3 = " << PK3.ok() << std::endl; - my_assert(PK3.ok()); - - unsigned char pub_export[32]; - ok = PK1.export_public_key(pub_export); - std::cout << "PubK1 = " << ok << " " << buffer_to_hex(pub_export) << std::endl; - my_assert(ok); - crypto::Ed25519::PublicKey PubK1(pub_export); - ok = PK2.export_public_key(pub_export); - std::cout << "PubK2 = " << ok << " " << buffer_to_hex(pub_export) << std::endl; - my_assert(ok); - my_assert(!std::memcmp(pub_export, fixed_pubkey, 32)); - crypto::Ed25519::PublicKey PubK2(pub_export); - ok = PK3.export_public_key(pub_export); - std::cout << "PubK3 = " << ok << " " << buffer_to_hex(pub_export) << std::endl; - my_assert(ok); - my_assert(!std::memcmp(pub_export, fixed_pubkey, 32)); - crypto::Ed25519::PublicKey PubK3(pub_export); - ok = PubK1.export_public_key(pub_export); - std::cout << "PubK1 = " << ok << " " << buffer_to_hex(pub_export) << std::endl; - my_assert(ok); - - unsigned char secret22[32]; - ok = PK2.compute_shared_secret(secret22, PubK3); - std::cout << "secret(PK2,PubK2)=" << ok << " " << buffer_to_hex(secret22) << std::endl; - my_assert(ok); - - unsigned char secret12[32], secret21[32]; - ok = PK1.compute_shared_secret(secret12, PubK3); - std::cout << "secret(PK1,PubK2)=" << ok << " " << buffer_to_hex(secret12) << std::endl; - my_assert(ok); - ok = PK2.compute_shared_secret(secret21, PubK1); - std::cout << "secret(PK2,PubK1)=" << ok << " " << buffer_to_hex(secret21) << std::endl; - my_assert(ok); - my_assert(!std::memcmp(secret12, secret21, 32)); - - // for (int i = 0; i < 1000; i++) { - // ok = PK1.compute_shared_secret(secret12, PubK3); - // my_assert(ok); - // ok = PK2.compute_shared_secret(secret21, PubK1); - // my_assert(ok); - // } - - unsigned char signature[64]; - ok = PK1.sign_message(signature, (const unsigned char*)"abc", 3); - std::cout << "PK1.signature=" << ok << " " << buffer_to_hex(signature, 64) << std::endl; - my_assert(ok); - - // signature[63] ^= 1; - ok = PubK1.check_message_signature(signature, (const unsigned char*)"abc", 3); - std::cout << "PubK1.check_signature=" << ok << std::endl; - my_assert(ok); - - PK4.import_private_key(rfc8032_secret_key1); - PK4.export_public_key(pub_export); - std::cout << "PK4.private_key = " << buffer_to_hex(rfc8032_secret_key1) << std::endl; - std::cout << "PK4.public_key = " << buffer_to_hex(pub_export) << std::endl; - my_assert(!std::memcmp(pub_export, rfc8032_public_key1, 32)); - ok = PK4.sign_message(signature, (const unsigned char*)"", 0); - std::cout << "PK4.signature('') = " << buffer_to_hex(signature, 64) << std::endl; - my_assert(ok); - my_assert(!std::memcmp(signature, rfc8032_signature1, 32)); - - PK5.import_private_key(rfc8032_secret_key2); - PK5.export_public_key(pub_export); - std::cout << "PK5.private_key = " << buffer_to_hex(rfc8032_secret_key2) << std::endl; - std::cout << "PK5.public_key = " << buffer_to_hex(pub_export) << std::endl; - my_assert(!std::memcmp(pub_export, rfc8032_public_key2, 32)); - ok = PK5.sign_message(signature, rfc8032_message2, 2); - std::cout << "PK5.signature('') = " << buffer_to_hex(signature, 64) << std::endl; - my_assert(ok); - my_assert(!std::memcmp(signature, rfc8032_signature2, 32)); - crypto::Ed25519::PublicKey PubK5(pub_export); - - // for (int i = 0; i < 10000; i++) { - // ok = PK5.sign_message (signature, rfc8032_message2, 2); - // my_assert (ok); - // } - // for (int i = 0; i < 10000; i++) { - // ok = PubK5.check_message_signature (signature, rfc8032_message2, 2); - // my_assert (ok); - // } - - unsigned char temp_pubkey[32]; - crypto::Ed25519::TempKeyGenerator TKG; // use one generator a lot of times - - TKG.create_temp_shared_secret(temp_pubkey, secret12, PubK1, (const unsigned char*)"abc", 3); - std::cout << "secret12=" << buffer_to_hex(secret12) << "; temp_pubkey=" << buffer_to_hex(temp_pubkey) << std::endl; - - PK1.compute_temp_shared_secret(secret21, temp_pubkey); - std::cout << "secret21=" << buffer_to_hex(secret21) << std::endl; - my_assert(!std::memcmp(secret12, secret21, 32)); - - std::cout << "********* ok\n\n"; - return true; -} - -int main(void) { - test_ed25519_impl(); - test_ed25519_crypto(); - return 0; -} diff --git a/doc/Tests.md b/doc/Tests.md index c883731a9..4ae21a75a 100644 --- a/doc/Tests.md +++ b/doc/Tests.md @@ -3,13 +3,13 @@ TON contains multiple unit-tests, that facilitate detection of erroneous blockch ## Build tests Go inside the build directory and, if you use ninja, build the tests using the following command: -```ninja test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state``` +```ninja test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state``` For more details on how to build TON artifacts, please refer to any of Github actions. For cmake use: -```cmake --build . --target test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state``` +```cmake --build . --target test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state``` ## Run tests Go inside the build directory and with ninja execute: From 611cab74ada795ad81d55a76c8d7f7e537f92a95 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 23 May 2025 17:22:54 +0300 Subject: [PATCH 254/388] More session stats --- tl/generate/scheme/ton_api.tl | 6 ++++-- tl/generate/scheme/ton_api.tlo | Bin 113468 -> 113716 bytes validator-session/validator-session-types.h | 11 ++++++++--- validator/impl/collator.cpp | 2 ++ validator/interfaces/validator-manager.h | 8 ++++++-- validator/validator-group.cpp | 1 + 6 files changed, 21 insertions(+), 7 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index c5bc4afe3..5d42796ec 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -936,6 +936,8 @@ validatorStats.blockLimitsStatus limits_log:string = validatorStats.BlockLimitsStatus; validatorStats.extMsgsStats ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int = validatorStats.ExtMsgsStats; +validatorStats.blockStats + ext_msgs:validatorStats.extMsgsStats transactions:int = validatorStats.BlockStats; validatorStats.collatedBlock block_id:tonNode.blockIdExt collated_data_hash:int256 cc_seqno:int collated_at:double @@ -943,7 +945,7 @@ validatorStats.collatedBlock self:int256 is_validator:Bool total_time:double work_time:double cpu_work_time:double time_stats:string block_limits:validatorStats.blockLimitsStatus - ext_msgs_stats:validatorStats.extMsgsStats = validatorSession.stats.CollatedBlock; + block_stats:validatorStats.blockStats = validatorSession.stats.CollatedBlock; validatorStats.validatedBlock block_id:tonNode.blockIdExt collated_data_hash:int256 validated_at:double @@ -954,7 +956,7 @@ validatorStats.validatedBlock validatorStats.newValidatorGroup.node id:int256 pubkey:PublicKey adnl_id:int256 weight:long = validatorStats.newValidatorGroup.Node; validatorStats.newValidatorGroup session_id:int256 shard:tonNode.shardId cc_seqno:int - last_key_block_seqno:int started_at:double + last_key_block_seqno:int started_at:double prev:(vector tonNode.blockIdExt) self_idx:int self:int256 nodes:(vector validatorStats.newValidatorGroup.node) = validatorStats.NewValidatorGroup; validatorStats.endValidatorGroup.node id:int256 catchain_blocks:int = validatorStats.endValidatorGroup.Node; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 744323b6de215969871ddd09ac1cfb41e3619ed6..7b0b3750ac301801cd4b6efe14317c597d86146d 100644 GIT binary patch delta 362 zcmdn5sSn#Ho7nANYKayE!LF;RN*2fH0C_HGx?ZC+!)gpu(=bD-n)Ku5;)f~+7TWTr0+XB3?-70k%R z3F6;8Cgj6EIk8E5`h*BZjqMG=jC&Y$xRXHU#TSFj204Pmw(TIuT1Kdgik#VWKw=;R zw&#R08Zaeba~0SWumc$wK=L5Fn1Dt|KfABVky=p_pIe*`HXmw*|D1|!m=z#3JS9bm zdBuszC7JnolMOr+SU`qspAgO{!ORNspu*&Z4hlR#%^(iQaL(zBk&N2YH`Fr9Ojn3z hteM=?BrrKfCWcWV9st^9f%pIb delta 181 zcmdn;gKf_@Hr_|G^{p77z;Pq5vK`9mTWWbkNQ4*h9oL(GXT#{H)3^tO(w(X$*oQiBls0BsN zY&sw@kQLh(gfbd1ZBK|`B$iPj F9snI!K6wBD diff --git a/validator-session/validator-session-types.h b/validator-session/validator-session-types.h index e613b08a5..18f6b702b 100644 --- a/validator-session/validator-session-types.h +++ b/validator-session/validator-session-types.h @@ -223,19 +223,24 @@ struct NewValidatorGroupStats { CatchainSeqno cc_seqno = 0; BlockSeqno last_key_block_seqno = 0; double started_at = -1.0; + std::vector prev; td::uint32 self_idx = 0; PublicKeyHash self = PublicKeyHash::zero(); std::vector nodes; tl_object_ptr tl() const { + std::vector> prev_arr; + for (const auto &p : prev) { + prev_arr.push_back(create_tl_block_id(p)); + } std::vector> nodes_arr; for (const auto &node : nodes) { nodes_arr.push_back(create_tl_object( node.id.bits256_value(), node.pubkey.tl(), node.adnl_id.bits256_value(), node.weight)); } - return create_tl_object(session_id, create_tl_shard_id(shard), cc_seqno, - last_key_block_seqno, started_at, self_idx, - self.bits256_value(), std::move(nodes_arr)); + return create_tl_object( + session_id, create_tl_shard_id(shard), cc_seqno, last_key_block_seqno, started_at, std::move(prev_arr), + self_idx, self.bits256_value(), std::move(nodes_arr)); } }; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 069abc0f9..93fbaa198 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -3063,6 +3063,7 @@ bool Collator::create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, t update_max_lt(acc->last_trans_end_lt_); block::MsgMetadata new_msg_metadata{0, acc->workchain, acc->addr, trans->start_lt}; register_new_msgs(*trans, std::move(new_msg_metadata)); + ++stats_.transactions; return true; } @@ -3170,6 +3171,7 @@ Ref Collator::create_ordinary_transaction(Ref msg_root, register_new_msgs(*trans, std::move(new_msg_metadata)); update_max_lt(acc->last_trans_end_lt_); value_flow_.burned += trans->blackhole_burned; + ++stats_.transactions; return trans_root; } diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index b09a8f244..d1394c942 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -69,6 +69,7 @@ struct CollationStats { td::uint32 estimated_bytes = 0, gas = 0, lt_delta = 0, estimated_collated_data_bytes = 0; int cat_bytes = 0, cat_gas = 0, cat_lt_delta = 0, cat_collated_data_bytes = 0; std::string limits_log; + td::uint32 transactions = 0; td::uint32 ext_msgs_total = 0; td::uint32 ext_msgs_filtered = 0; td::uint32 ext_msgs_accepted = 0; @@ -77,6 +78,10 @@ struct CollationStats { std::string time_stats; tl_object_ptr tl() const { + auto block_stats = create_tl_object( + create_tl_object(ext_msgs_total, ext_msgs_filtered, ext_msgs_accepted, + ext_msgs_rejected), + transactions); return create_tl_object( create_tl_block_id(block_id), collated_data_hash, cc_seqno, collated_at, actual_bytes, actual_collated_data_bytes, attempt, self.bits256_value(), is_validator, total_time, work_time, cpu_work_time, @@ -84,8 +89,7 @@ struct CollationStats { create_tl_object(estimated_bytes, gas, lt_delta, estimated_collated_data_bytes, cat_bytes, cat_gas, cat_lt_delta, cat_collated_data_bytes, limits_log), - create_tl_object(ext_msgs_total, ext_msgs_filtered, ext_msgs_accepted, - ext_msgs_rejected)); + std::move(block_stats)); } }; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 8ba4febcf..7a2fa187a 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -432,6 +432,7 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch .cc_seqno = validator_set_->get_catchain_seqno(), .last_key_block_seqno = last_key_block_seqno_, .started_at = td::Clocks::system(), + .prev = prev, .self = local_id_}; td::uint32 idx = 0; for (const auto &node : validator_set_->export_vector()) { From d4fcef4374b3e2c85751096ba1d9ac6a6a97abf9 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 26 May 2025 15:20:50 +0300 Subject: [PATCH 255/388] Version 11: fix account storage stat and null in RUNVM (#1690) * Bugfix in AccountStorageStat::remove_cell * Fix returning null as c4/c5 in RUNVM --- crypto/block/account-storage-stat.cpp | 2 +- crypto/vm/vm.cpp | 19 ++++++++++++++----- doc/GlobalVersions.md | 5 ++++- test/regression-tests.ans | 2 +- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp index 82359f827..a6159c224 100644 --- a/crypto/block/account-storage-stat.cpp +++ b/crypto/block/account-storage-stat.cpp @@ -154,7 +154,7 @@ td::Status AccountStorageStat::remove_cell(const Ref& cell) { if (e.refcnt_diff < 0 && !e.refcnt) { TRY_STATUS(fetch_from_dict(e)); } - if (e.refcnt.value() + e.refcnt_diff != 0) { + if (e.refcnt_diff >= 0 || e.refcnt.value() + e.refcnt_diff != 0) { return td::Status::OK(); } bool spec; diff --git a/crypto/vm/vm.cpp b/crypto/vm/vm.cpp index 272e3930e..356f0df22 100644 --- a/crypto/vm/vm.cpp +++ b/crypto/vm/vm.cpp @@ -785,11 +785,20 @@ void VmState::restore_parent_vm(int res) { cur_stack.push(std::move(child_state.stack->at(i))); } cur_stack.push_smallint(res); - if (parent->return_data) { - cur_stack.push_cell(child_state.get_committed_state().c4); - } - if (parent->return_actions) { - cur_stack.push_cell(child_state.get_committed_state().c5); + if (global_version >= 11 && !child_state.get_committed_state().committed) { + if (parent->return_data) { + cur_stack.push_null(); + } + if (parent->return_actions) { + cur_stack.push_null(); + } + } else { + if (parent->return_data) { + cur_stack.push_cell(child_state.get_committed_state().c4); + } + if (parent->return_actions) { + cur_stack.push_cell(child_state.get_committed_state().c5); + } } if (parent->return_gas) { cur_stack.push_smallint(child_state.gas.gas_consumed()); diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index 7bd9b0cc9..d4113b3f1 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -221,4 +221,7 @@ Reserve modes `+1`, `+4` and `+8` ("reserve all except", "add original balance" Along with the storage stat (cells and bits count), each account now stores the hash of the **storage dict**. **Storage dict** is the dictionary that stores refcnt for each cell in the account state. -This is required to help computing storage stats in the future, after collator-validator separation. \ No newline at end of file +This is required to help computing storage stats in the future, after collator-validator separation. + +### Other TVM changes +- Fix returning `null` as `c4` and `c5` (when VM state is not committed) in `RUNVM`. \ No newline at end of file diff --git a/test/regression-tests.ans b/test/regression-tests.ans index 22e7826ea..5831ebe74 100644 --- a/test/regression-tests.ans +++ b/test/regression-tests.ans @@ -25,7 +25,7 @@ Test_Fift_test_rist255_default f4d7558f200a656934f986145c19b1dedbe2ad029292a5a97 Test_Fift_test_secp256k1_default 3118450dace6af05fcdbd54a87d9446162ce11ac6ef6dfc57998cf113587d602 Test_Fift_test_sort2_default 9b57d47e6a10e7d1bbb565db35400debf2f963031f434742a702ec76555a5d3a Test_Fift_test_sort_default 9b57d47e6a10e7d1bbb565db35400debf2f963031f434742a702ec76555a5d3a -Test_Fift_test_tvm_runvm_default ff3d2a4031b543c18d6b555f0a1f1a891c7825e6d1e2e9beb4bf13b37441450b +Test_Fift_test_tvm_runvm_default b3e0d70c00f0e8bdf6d45c56c7c0817fa5be7a3fc540190786233a394de72e42 Test_Fift_testvm2_default 8a6e35fc0224398be9d2de39d31c86ea96965ef1eca2aa9e0af2303150ed4a7b Test_Fift_testvm3_default 3c1b77471c5fd914ed8b5f528b9faed618e278693f5030b953ff150e543864ae Test_Fift_testvm4_default 8a6e35fc0224398be9d2de39d31c86ea96965ef1eca2aa9e0af2303150ed4a7b From 9b2933cb8480c99f72da7d47a24fb329706d4229 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 26 May 2025 15:22:29 +0300 Subject: [PATCH 256/388] Store shard archives separately even if monitor_min_split = 0 (#1689) --- validator/db/archive-slice.cpp | 35 +++++++++++++++++----------------- validator/db/archive-slice.hpp | 1 + validator/import-db-slice.cpp | 8 ++++---- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/validator/db/archive-slice.cpp b/validator/db/archive-slice.cpp index 41ed50997..0691f6740 100644 --- a/validator/db/archive-slice.cpp +++ b/validator/db/archive-slice.cpp @@ -554,14 +554,14 @@ void ArchiveSlice::get_slice(td::uint64 archive_id, td::uint64 offset, td::uint3 before_query(); auto value = static_cast(archive_id >> 32); PackageInfo *p; - if (shard_split_depth_ == 0) { - TRY_RESULT_PROMISE_ASSIGN(promise, p, choose_package(value, ShardIdFull{masterchainId}, false)); - } else { + if (shard_separated_) { if (value >= packages_.size()) { promise.set_error(td::Status::Error(ErrorCode::notready, "no such package")); return; } p = &packages_[value]; + } else { + TRY_RESULT_PROMISE_ASSIGN(promise, p, choose_package(value, ShardIdFull{masterchainId}, false)); } promise = begin_async_query(std::move(promise)); td::actor::create_actor("readfile", p->path, offset, limit, 0, std::move(promise)).release(); @@ -574,10 +574,10 @@ void ArchiveSlice::get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shar promise.set_result(archive_id_); } else { TRY_RESULT_PROMISE(promise, p, choose_package(masterchain_seqno, shard_prefix, false)); - if (shard_split_depth_ == 0) { - promise.set_result(p->seqno * (1ull << 32) + archive_id_); - } else { + if (shard_separated_) { promise.set_result(p->idx * (1ull << 32) + archive_id_); + } else { + promise.set_result(p->seqno * (1ull << 32) + archive_id_); } } } @@ -609,8 +609,10 @@ void ArchiveSlice::before_query() { if (R2.move_as_ok() == td::KeyValue::GetStatus::Ok) { shard_split_depth_ = td::to_integer(value); CHECK(shard_split_depth_ <= 60); + shard_separated_ = true; } else { shard_split_depth_ = 0; + shard_separated_ = false; } for (td::uint32 i = 0; i < tot; i++) { R2 = kv_->get(PSTRING() << "status." << i, value); @@ -625,16 +627,16 @@ void ArchiveSlice::before_query() { } td::uint32 seqno; ShardIdFull shard_prefix; - if (shard_split_depth_ == 0) { - seqno = archive_id_ + slice_size_ * i; - shard_prefix = ShardIdFull{masterchainId}; - } else { + if (shard_separated_) { R2 = kv_->get(PSTRING() << "info." << i, value); R2.ensure(); CHECK(R2.move_as_ok() == td::KeyValue::GetStatus::Ok); unsigned long long shard; CHECK(sscanf(value.c_str(), "%u.%d:%016llx", &seqno, &shard_prefix.workchain, &shard) == 3); shard_prefix.shard = shard; + } else { + seqno = archive_id_ + slice_size_ * i; + shard_prefix = ShardIdFull{masterchainId}; } add_package(seqno, shard_prefix, len, ver); } @@ -651,10 +653,9 @@ void ArchiveSlice::before_query() { kv_->set("slice_size", td::to_string(slice_size_)).ensure(); kv_->set("status.0", "0").ensure(); kv_->set("version.0", td::to_string(default_package_version())).ensure(); - if (shard_split_depth_ > 0) { - kv_->set("info.0", package_info_to_str(archive_id_, ShardIdFull{masterchainId})).ensure(); - kv_->set("shard_split_depth", td::to_string(shard_split_depth_)).ensure(); - } + shard_separated_ = true; + kv_->set("info.0", package_info_to_str(archive_id_, ShardIdFull{masterchainId})).ensure(); + kv_->set("shard_split_depth", td::to_string(shard_split_depth_)).ensure(); kv_->commit_transaction().ensure(); add_package(archive_id_, ShardIdFull{masterchainId}, 0, default_package_version()); } else { @@ -779,7 +780,7 @@ td::Result ArchiveSlice::choose_package(BlockSeqno } masterchain_seqno -= (masterchain_seqno - archive_id_) % slice_size_; CHECK((masterchain_seqno - archive_id_) % slice_size_ == 0); - if (shard_split_depth_ == 0) { + if (!shard_separated_) { shard_prefix = ShardIdFull{masterchainId}; } else if (!shard_prefix.is_masterchain()) { shard_prefix.shard |= 1; // In case length is < split depth @@ -795,7 +796,7 @@ td::Result ArchiveSlice::choose_package(BlockSeqno kv_->set("slices", td::to_string(v + 1)).ensure(); kv_->set(PSTRING() << "status." << v, "0").ensure(); kv_->set(PSTRING() << "version." << v, td::to_string(default_package_version())).ensure(); - if (shard_split_depth_ > 0) { + if (shard_separated_) { kv_->set(PSTRING() << "info." << v, package_info_to_str(masterchain_seqno, shard_prefix)).ensure(); } commit_transaction(); @@ -1079,7 +1080,7 @@ void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle, td:: package.idx = i; kv_->set(PSTRING() << "status." << i, td::to_string(package.package->size())).ensure(); kv_->set(PSTRING() << "version." << i, td::to_string(package.version)).ensure(); - if (shard_split_depth_ > 0) { + if (shard_separated_) { kv_->set(PSTRING() << "info." << i, package_info_to_str(package.seqno, package.shard_prefix)).ensure(); } id_to_package_[{package.seqno, package.shard_prefix}] = i; diff --git a/validator/db/archive-slice.hpp b/validator/db/archive-slice.hpp index a027ec0ff..0cb2c9fbf 100644 --- a/validator/db/archive-slice.hpp +++ b/validator/db/archive-slice.hpp @@ -159,6 +159,7 @@ class ArchiveSlice : public td::actor::Actor { bool sliced_mode_{false}; td::uint32 huge_transaction_size_ = 0; td::uint32 slice_size_{100}; + bool shard_separated_{false}; td::uint32 shard_split_depth_ = 0; enum Status { diff --git a/validator/import-db-slice.cpp b/validator/import-db-slice.cpp index 3e1392bef..c0d502cb2 100644 --- a/validator/import-db-slice.cpp +++ b/validator/import-db-slice.cpp @@ -300,10 +300,10 @@ void ArchiveImporter::checked_all_masterchain_blocks() { void ArchiveImporter::download_shard_archives(td::Ref start_state) { start_state_ = start_state; td::uint32 monitor_min_split = start_state->monitor_min_split_depth(basechainId); - LOG(DEBUG) << "Monitor min split = " << monitor_min_split; - // If monitor_min_split == 0, we use the old archive format (packages are not separated by shard) + LOG(DEBUG) << "Monitor min split = " << monitor_min_split + << (have_shard_blocks_ ? ", shard blocks in the main package" : ", no shard blocks in the main package"); // If masterchain package has shard blocks then it's old archive format, don't need to download shards - if (monitor_min_split > 0 && !have_shard_blocks_ && !use_imported_files_) { + if (!have_shard_blocks_ && !use_imported_files_) { for (td::uint64 i = 0; i < (1ULL << monitor_min_split); ++i) { ShardIdFull shard_prefix{basechainId, (i * 2 + 1) << (64 - monitor_min_split - 1)}; if (opts_->need_monitor(shard_prefix, start_state)) { @@ -313,7 +313,7 @@ void ArchiveImporter::download_shard_archives(td::Ref start_st } } } else { - LOG(DEBUG) << "Skip downloading shard archives"; + LOG(INFO) << "Skip downloading shard archives"; } if (pending_shard_archives_ == 0) { check_next_shard_client_seqno(shard_client_seqno_ + 1); From 6a1dbc72a3c0205c413b08f9c4394f38d32c37e3 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 26 May 2025 15:46:57 +0300 Subject: [PATCH 257/388] Don't destroy ValidatorGroup immediately, fix flags in session stats --- validator-session/validator-session-types.h | 4 +- validator/validator-group.cpp | 44 +++++++++++++++------ validator/validator-group.hpp | 2 + 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/validator-session/validator-session-types.h b/validator-session/validator-session-types.h index 18f6b702b..5b1f5c426 100644 --- a/validator-session/validator-session-types.h +++ b/validator-session/validator-session-types.h @@ -143,7 +143,9 @@ struct ValidatorSessionStats { signers_str[i] = '0' + signers[i]; } int flags = - (block_status != status_none ? ton_api::validatorStats_stats_producer::Flags::BLOCK_ID_MASK : 0) | + (block_status != status_none || !candidate_id.is_zero() + ? ton_api::validatorStats_stats_producer::Flags::BLOCK_ID_MASK + : 0) | (collated_at >= 0.0 ? ton_api::validatorStats_stats_producer::Flags::COLLATED_AT_MASK : 0) | (!collator_node_id.is_zero() ? ton_api::validatorStats_stats_producer::Flags::COLLATOR_NODE_ID_MASK : 0) | (validated_at >= 0.0 ? ton_api::validatorStats_stats_producer::Flags::VALIDATED_AT_MASK : 0) | diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 7a2fa187a..f9a8f2732 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -38,6 +38,10 @@ static bool need_send_candidate_broadcast(const validatorsession::BlockSourceInf void ValidatorGroup::generate_block_candidate(validatorsession::BlockSourceInfo source_info, td::Promise promise) { + if (destroying_) { + promise.set_error(td::Status::Error("validator session finished")); + return; + } td::uint32 round_id = source_info.priority.round; if (round_id > last_known_round_id_) { last_known_round_id_ = round_id; @@ -99,6 +103,10 @@ void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo void ValidatorGroup::validate_block_candidate(validatorsession::BlockSourceInfo source_info, BlockCandidate block, td::Promise> promise) { + if (destroying_) { + promise.set_error(td::Status::Error("validator session finished")); + return; + } td::uint32 round_id = source_info.priority.round; if (round_id > last_known_round_id_) { last_known_round_id_ = round_id; @@ -451,6 +459,28 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch } void ValidatorGroup::destroy() { + if (destroying_) { + return; + } + destroying_ = true; + if (!session_.empty()) { + td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_end_stats, + [manager = manager_](td::Result R) { + if (R.is_error()) { + LOG(DEBUG) << "Failed to get validator session end stats: " << R.move_as_error(); + return; + } + auto stats = R.move_as_ok(); + td::actor::send_closure(manager, &ValidatorManager::log_end_validator_group_stats, + std::move(stats)); + }); + } + cancellation_token_source_.cancel(); + delay_action([SelfId = actor_id(this)]() { td::actor::send_closure(SelfId, &ValidatorGroup::destroy_cont); }, + td::Timestamp::in(10.0)); +} + +void ValidatorGroup::destroy_cont() { if (!session_.empty()) { td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_current_stats, [manager = manager_, cc_seqno = validator_set_->get_catchain_seqno(), @@ -469,21 +499,9 @@ void ValidatorGroup::destroy() { td::actor::send_closure(manager, &ValidatorManager::log_validator_session_stats, std::move(stats)); }); - td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_end_stats, - [manager = manager_](td::Result R) { - if (R.is_error()) { - LOG(DEBUG) << "Failed to get validator session end stats: " << R.move_as_error(); - return; - } - auto stats = R.move_as_ok(); - td::actor::send_closure(manager, &ValidatorManager::log_end_validator_group_stats, - std::move(stats)); - }); auto ses = session_.release(); - delay_action([ses]() mutable { td::actor::send_closure(ses, &validatorsession::ValidatorSession::destroy); }, - td::Timestamp::in(10.0)); + td::actor::send_closure(ses, &validatorsession::ValidatorSession::destroy); } - cancellation_token_source_.cancel(); stop(); } diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 971fcc17b..c7f940412 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -102,6 +102,7 @@ class ValidatorGroup : public td::actor::Actor { private: std::unique_ptr make_validator_session_callback(); + void destroy_cont(); struct PostponedAccept { RootHash root_hash; @@ -143,6 +144,7 @@ class ValidatorGroup : public td::actor::Actor { td::Ref opts_; td::uint32 last_known_round_id_ = 0; bool monitoring_shard_ = true; + bool destroying_ = false; struct CachedCollatedBlock { td::optional result; From 3fbab2c601380eba5ba68048f45d24a359bd2936 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 26 May 2025 17:27:06 +0300 Subject: [PATCH 258/388] Always set ihr_disabled in internal messages (#1691) --- crypto/block/transaction.cpp | 4 ++++ crypto/block/transaction.h | 1 + crypto/vm/tonops.cpp | 2 +- doc/GlobalVersions.md | 5 +++-- validator/impl/validate-query.cpp | 1 + 5 files changed, 10 insertions(+), 3 deletions(-) diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 1b6c3582f..8a72f8e48 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -2569,6 +2569,9 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, fwd_fee = block::tlb::t_Grams.as_integer(info.fwd_fee); ihr_fee = block::tlb::t_Grams.as_integer(info.ihr_fee); } + if (cfg.disable_ihr_flag) { + info.ihr_disabled = true; + } } // set created_at and created_lt to correct values info.created_at = now; @@ -4080,6 +4083,7 @@ td::Status FetchConfigParams::fetch_config_params( action_phase_cfg->mc_blackhole_addr = config.get_burning_config().blackhole_addr; action_phase_cfg->extra_currency_v2 = config.get_global_version() >= 10; action_phase_cfg->disable_anycast = config.get_global_version() >= 10; + action_phase_cfg->disable_ihr_flag = config.get_global_version() >= 11; } { serialize_cfg->extra_currency_v2 = config.get_global_version() >= 10; diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index fe7006006..0b3a4358d 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -174,6 +174,7 @@ struct ActionPhaseConfig { bool disable_custom_fess{false}; bool reserve_extra_enabled{false}; bool extra_currency_v2{false}; + bool disable_ihr_flag{false}; td::optional mc_blackhole_addr; bool disable_anycast{false}; const MsgPrices& fetch_msg_prices(bool is_masterchain) const { diff --git a/crypto/vm/tonops.cpp b/crypto/vm/tonops.cpp index 135ed7026..b06fb1b77 100644 --- a/crypto/vm/tonops.cpp +++ b/crypto/vm/tonops.cpp @@ -1834,7 +1834,7 @@ int exec_send_message(VmState* st) { if (!tlb::csr_unpack(msg.info, info)) { throw VmError{Excno::unknown, "invalid message"}; } - ihr_disabled = info.ihr_disabled; + ihr_disabled = info.ihr_disabled || st->get_global_version() >= 11; dest = std::move(info.dest); Ref extra; if (!block::tlb::t_CurrencyCollection.unpack_special(info.value.write(), value, extra)) { diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index d4113b3f1..ad5eba948 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -223,5 +223,6 @@ Along with the storage stat (cells and bits count), each account now stores the **Storage dict** is the dictionary that stores refcnt for each cell in the account state. This is required to help computing storage stats in the future, after collator-validator separation. -### Other TVM changes -- Fix returning `null` as `c4` and `c5` (when VM state is not committed) in `RUNVM`. \ No newline at end of file +### Other changes +- Fix returning `null` as `c4` and `c5` (when VM state is not committed) in `RUNVM`. +- In new internal messages `ihr_disabled` is automatically set to `1`, `ihr_fee` is always zero. \ No newline at end of file diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 88ae031e2..8c974b431 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -1007,6 +1007,7 @@ bool ValidateQuery::fetch_config_params() { action_phase_cfg_.mc_blackhole_addr = config_->get_burning_config().blackhole_addr; action_phase_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; action_phase_cfg_.disable_anycast = config_->get_global_version() >= 10; + action_phase_cfg_.disable_ihr_flag = config_->get_global_version() >= 11; } { serialize_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; From a213e463ee4690a6d629562b6392b1ea806502b4 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 27 May 2025 14:23:47 +0300 Subject: [PATCH 259/388] "--shard-block-retainer fullnode" option --- validator-engine/validator-engine.cpp | 33 ++++++++++++++++++--------- validator-engine/validator-engine.hpp | 4 ++++ 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 8c15b66c7..a7d28ea7f 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -2194,6 +2194,9 @@ void ValidatorEngine::start_validator() { } } + if (shard_block_retainer_adnl_id_fullnode_) { + shard_block_retainer_adnl_id_ = ton::adnl::AdnlNodeIdShort{config_.full_node}; + } if (!shard_block_retainer_adnl_id_.is_zero()) { if (config_.adnl_ids.contains(shard_block_retainer_adnl_id_.pubkey_hash())) { td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_shard_block_retainer, @@ -5575,17 +5578,25 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_sync_shards_upto, v); }); return td::Status::OK(); }); - p.add_checked_option( - '\0', "shard-block-retainer", "adnl id for shard block retainer (hex)", [&](td::Slice arg) -> td::Status { - td::Bits256 v; - if (v.from_hex(arg) != 256) { - return td::Status::Error("invalid adnl-id"); - } - acts.push_back([&x, v]() { - td::actor::send_closure(x, &ValidatorEngine::set_shard_block_retainer_adnl_id, ton::adnl::AdnlNodeIdShort{v}); - }); - return td::Status::OK(); - }); + p.add_checked_option('\0', "shard-block-retainer", + "adnl id for shard block retainer (hex), or \"fullnode\" for full node id", + [&](td::Slice arg) -> td::Status { + if (arg == "fullnode") { + acts.push_back([&x]() { + td::actor::send_closure(x, &ValidatorEngine::set_shard_block_retainer_adnl_id_fullnode); + }); + } else { + td::Bits256 v; + if (v.from_hex(arg) != 256) { + return td::Status::Error("invalid adnl-id"); + } + acts.push_back([&x, v]() { + td::actor::send_closure(x, &ValidatorEngine::set_shard_block_retainer_adnl_id, + ton::adnl::AdnlNodeIdShort{v}); + }); + } + return td::Status::OK(); + }); auto S = p.run(argc, argv); if (S.is_error()) { LOG(ERROR) << "failed to parse options: " << S.move_as_error(); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index b8ec238e9..ea6ef0da4 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -259,6 +259,7 @@ class ValidatorEngine : public td::actor::Actor { bool skip_key_sync_ = false; td::optional sync_shards_upto_; ton::adnl::AdnlNodeIdShort shard_block_retainer_adnl_id_ = ton::adnl::AdnlNodeIdShort::zero(); + bool shard_block_retainer_adnl_id_fullnode_ = false; std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; @@ -386,6 +387,9 @@ class ValidatorEngine : public td::actor::Actor { void set_shard_block_retainer_adnl_id(ton::adnl::AdnlNodeIdShort id) { shard_block_retainer_adnl_id_ = id; } + void set_shard_block_retainer_adnl_id_fullnode() { + shard_block_retainer_adnl_id_fullnode_ = true; + } void start_up() override; ValidatorEngine() { From b2028ae139cbdd426a47219aea004d62dd37b123 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Mon, 31 Mar 2025 20:55:01 +0300 Subject: [PATCH 260/388] [Tolk] Lateinit variables Allow a variable be unassigned at declaration if it is definitely assigned before the first usage --- tolk-tester/tests/assignment-tests.tolk | 38 +++++++++++++++++++ .../tests/invalid-declaration/err-1190.tolk | 9 +++++ .../tests/invalid-declaration/err-1617.tolk | 8 ++++ .../tests/invalid-semantics/err-4301.tolk | 15 ++++++++ .../tests/invalid-semantics/err-4309.tolk | 32 ++++++++++++++++ tolk/ast-from-tokens.cpp | 34 +++++++++++------ tolk/ast-replicator.h | 2 +- tolk/ast.h | 5 ++- tolk/pipe-ast-to-legacy.cpp | 1 + tolk/pipe-calc-rvalue-lvalue.cpp | 2 +- tolk/pipe-infer-types-and-calls.cpp | 22 +++++++++++ tolk/pipe-resolve-identifiers.cpp | 8 ++-- tolk/symtable.h | 2 + 13 files changed, 158 insertions(+), 20 deletions(-) create mode 100644 tolk-tester/tests/invalid-declaration/err-1190.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1617.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4301.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4309.tolk diff --git a/tolk-tester/tests/assignment-tests.tolk b/tolk-tester/tests/assignment-tests.tolk index a52b3ca50..e53020d05 100644 --- a/tolk-tester/tests/assignment-tests.tolk +++ b/tolk-tester/tests/assignment-tests.tolk @@ -212,6 +212,43 @@ fun test117() { return c; } +struct Storage { + owner: slice; +} + +@method_id(118) +fun test118(x: int) { + var i1: int; + if (10 > 3) { + i1 = 10; + } else { { (_, i1) = (5, 20); } } + + var i2: int10; + match (x) { + 1 => i2 = 1, + 2 => i2 = 2, + 3 => i2 = 3, + else => i2 = 4, + } + + var st: Storage?; + if (x > 100) { + throw 123; + } else { + if (x < -100) { throw 456; } + else { st = { owner: "" }} + } + + var i3: (int, int) | builder; + match (i1) { + int => i3 = (1, 2), + } + + var unused: int8; + + return (i1, i2, st.owner.remainingBitsCount(), i3.0); +} + fun main(value: int, ) { @@ -241,6 +278,7 @@ fun main(value: int, ) { @testcase | 115 | | [ 101 111 ] 9 9 @testcase | 116 | | 1 2 3 4 [ 1 2 3 4 ] @testcase | 117 | | [ 20 ] +@testcase | 118 | 3 | 10 3 0 1 @fif_codegen diff --git a/tolk-tester/tests/invalid-declaration/err-1190.tolk b/tolk-tester/tests/invalid-declaration/err-1190.tolk new file mode 100644 index 000000000..510e1d1cc --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1190.tolk @@ -0,0 +1,9 @@ +fun main() { + var i; +} + +/** +@compilation_should_fail +@stderr provide a type for a variable, because its default value is omitted: +@stderr var i: ; + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1617.tolk b/tolk-tester/tests/invalid-declaration/err-1617.tolk new file mode 100644 index 000000000..dbfe89719 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1617.tolk @@ -0,0 +1,8 @@ +fun main() { + val (s, i); +} + +/** +@compilation_should_fail +@stderr variables declaration must be followed by assignment + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4301.tolk b/tolk-tester/tests/invalid-semantics/err-4301.tolk new file mode 100644 index 000000000..800495da3 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4301.tolk @@ -0,0 +1,15 @@ +fun main() { + var i: int; + match (random.uint256()) { + 0 => { i = 0; debug.print(i) }, + 1 => { if (true) { i = 1 } }, + 2 => throw i = 2, + } + if (i == 0) {} +} + +/** +@compilation_should_fail +@stderr using variable `i` before it's definitely assigned +@stderr if (i == 0) + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4309.tolk b/tolk-tester/tests/invalid-semantics/err-4309.tolk new file mode 100644 index 000000000..3fa24df91 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4309.tolk @@ -0,0 +1,32 @@ +fun increment(mutate x: int) { + x += 1; +} + +fun main() { + var i: int; + if (true) { + i = 10; + } + increment(mutate i); // ok + + var j: int8 | int16; + if (random.uint256()) { + (j, _) = (10 as int8, i); + } else { + return; + } + match (j) { // ok + int8 => increment(mutate i), + int16 => increment(mutate j), + } + + var k: int; + { + increment(mutate k); + } +} + +/** +@compilation_should_fail +@stderr using variable `k` before it's definitely assigned + */ diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index ff13ed386..28f0ba182 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -400,11 +400,11 @@ static AnyV parse_type_alias_declaration(Lexer& lex, const std::vector(loc, v_ident, genericsT_list, underlying_type); } -static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable) { +static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable, bool allow_lateinit) { SrcLocation loc = lex.cur_location(); if (lex.tok() == tok_oppar) { lex.next(); - AnyExprV first = parse_var_declaration_lhs(lex, is_immutable); + AnyExprV first = parse_var_declaration_lhs(lex, is_immutable, false); if (lex.tok() == tok_clpar) { lex.next(); return first; @@ -412,17 +412,17 @@ static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable) { std::vector args(1, first); while (lex.tok() == tok_comma) { lex.next(); - args.push_back(parse_var_declaration_lhs(lex, is_immutable)); + args.push_back(parse_var_declaration_lhs(lex, is_immutable, false)); } lex.expect(tok_clpar, "`)`"); return createV(loc, std::move(args)); } if (lex.tok() == tok_opbracket) { lex.next(); - std::vector args(1, parse_var_declaration_lhs(lex, is_immutable)); + std::vector args(1, parse_var_declaration_lhs(lex, is_immutable, false)); while (lex.tok() == tok_comma) { lex.next(); - args.push_back(parse_var_declaration_lhs(lex, is_immutable)); + args.push_back(parse_var_declaration_lhs(lex, is_immutable, false)); } lex.expect(tok_clbracket, "`]`"); return createV(loc, std::move(args)); @@ -431,6 +431,7 @@ static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable) { auto v_ident = createV(loc, lex.cur_str()); AnyTypeV declared_type = nullptr; bool marked_as_redef = false; + bool is_lateinit = false; lex.next(); if (lex.tok() == tok_colon) { lex.next(); @@ -439,7 +440,13 @@ static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable) { lex.next(); marked_as_redef = true; } - return createV(loc, v_ident, declared_type, is_immutable, marked_as_redef); + if (lex.tok() == tok_semicolon && allow_lateinit) { + if (declared_type == nullptr) { + lex.error("provide a type for a variable, because its default value is omitted:\n> var " + static_cast(v_ident->name) + ": ;"); + } + is_lateinit = true; + } + return createV(loc, v_ident, declared_type, is_immutable, is_lateinit, marked_as_redef); } if (lex.tok() == tok_underscore) { AnyTypeV declared_type = nullptr; @@ -448,18 +455,21 @@ static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable) { lex.next(); declared_type = parse_type_from_tokens(lex); } - return createV(loc, createV(loc, ""), declared_type, true, false); + return createV(loc, createV(loc, ""), declared_type, true, false, false); } lex.unexpected("variable name"); } -static AnyExprV parse_local_vars_declaration_assignment(Lexer& lex) { +static AnyExprV parse_local_vars_declaration(Lexer& lex, bool allow_lateinit) { SrcLocation loc = lex.cur_location(); bool is_immutable = lex.tok() == tok_val; lex.next(); - AnyExprV lhs = createV(loc, parse_var_declaration_lhs(lex, is_immutable)); + AnyExprV lhs = parse_var_declaration_lhs(lex, is_immutable, allow_lateinit); if (lex.tok() != tok_assign) { + if (auto lhs_var = lhs->try_as(); lhs_var && lhs_var->is_lateinit) { + return lhs; // just ast_local_var_lhs inside AST tree + } lex.error("variables declaration must be followed by assignment: `var xxx = ...`"); } lex.next(); @@ -468,7 +478,7 @@ static AnyExprV parse_local_vars_declaration_assignment(Lexer& lex) { if (lex.tok() == tok_comma) { lex.error("multiple declarations are not allowed, split variables on separate lines"); } - return createV(loc, lhs, rhs); + return createV(loc, createV(loc, lhs), rhs); } // "parameters" are at function declaration: `fun f(param1: int, mutate param2: slice)` @@ -666,7 +676,7 @@ static V parse_match_expression(Lexer& lex) { lex.expect(tok_oppar, "`(`"); AnyExprV subject = lex.tok() == tok_var || lex.tok() == tok_val // `match (var x = rhs)` - ? parse_local_vars_declaration_assignment(lex) + ? parse_local_vars_declaration(lex, false) : parse_expr(lex); lex.expect(tok_clpar, "`)`"); @@ -1179,7 +1189,7 @@ AnyV parse_statement(Lexer& lex) { switch (lex.tok()) { case tok_var: // `var x = 0` is technically an expression, but can not appear in "any place", case tok_val: // only as a separate declaration - return parse_local_vars_declaration_assignment(lex); + return parse_local_vars_declaration(lex, true); case tok_opbrace: return parse_block_statement(lex); case tok_return: diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index fc1dafa22..d72513906 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -97,7 +97,7 @@ class ASTReplicator final { return createV(v->loc, clone(v->get_identifier()), v->has_instantiationTs() ? clone(v->get_instantiationTs()) : nullptr); } static V clone(V v) { - return createV(v->loc, clone(v->get_identifier()), clone(v->type_node), v->is_immutable, v->marked_as_redef); + return createV(v->loc, clone(v->get_identifier()), clone(v->type_node), v->is_immutable, v->is_lateinit, v->marked_as_redef); } static V clone(V v) { return createV(v->loc, clone(v->get_expr())); diff --git a/tolk/ast.h b/tolk/ast.h index ee550064c..0770d9b84 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -584,6 +584,7 @@ struct Vertex final : ASTExprLeaf { LocalVarPtr var_ref = nullptr; // filled on resolve identifiers; for `redef` points to declared above; for underscore, name is empty AnyTypeV type_node; // exists for `var x: int = rhs`, otherwise nullptr bool is_immutable; // declared via 'val', not 'var' + bool is_lateinit; // var st: Storage lateinit (no assignment) bool marked_as_redef; // var (existing_var redef, new_var: int) = ... V get_identifier() const { return identifier; } @@ -592,9 +593,9 @@ struct Vertex final : ASTExprLeaf { Vertex* mutate() const { return const_cast(this); } void assign_var_ref(LocalVarPtr var_ref); - Vertex(SrcLocation loc, V identifier, AnyTypeV type_node, bool is_immutable, bool marked_as_redef) + Vertex(SrcLocation loc, V identifier, AnyTypeV type_node, bool is_immutable, bool is_lateinit, bool marked_as_redef) : ASTExprLeaf(ast_local_var_lhs, loc) - , identifier(identifier), type_node(type_node), is_immutable(is_immutable), marked_as_redef(marked_as_redef) {} + , identifier(identifier), type_node(type_node), is_immutable(is_immutable), is_lateinit(is_lateinit), marked_as_redef(marked_as_redef) {} }; template<> diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index a45a4ccb6..370460d3c 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -1476,6 +1476,7 @@ static std::vector process_local_var(V v, CodeBlob static std::vector process_local_vars_declaration(V, CodeBlob&) { // it can not appear as a standalone expression // `var ... = rhs` is handled by ast_assign + // `var rhs: int lateinit` is ast_local_var_lhs tolk_assert(false); } diff --git a/tolk/pipe-calc-rvalue-lvalue.cpp b/tolk/pipe-calc-rvalue-lvalue.cpp index b6a239116..d451a7c60 100644 --- a/tolk/pipe-calc-rvalue-lvalue.cpp +++ b/tolk/pipe-calc-rvalue-lvalue.cpp @@ -246,7 +246,7 @@ class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { } void visit(V v) override { - tolk_assert(cur_state == MarkingState::LValue); + tolk_assert(cur_state == MarkingState::LValue || v->is_lateinit); mark_vertex(v); parent::visit(v); } diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index abeaffa1f..71149cd4b 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -150,6 +150,12 @@ static void fire_error_cannot_deduce_untyped_tuple_access(FunctionPtr cur_f, Src fire(cur_f, loc, "can not deduce type of `" + idx_access + "`\neither assign it to variable like `var c: int = " + idx_access + "` or cast the result like `" + idx_access + " as int`"); } +// fire an error on using lateinit variable before definite assignment +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_using_lateinit_variable_uninitialized(FunctionPtr cur_f, SrcLocation loc, std::string_view name) { + fire(cur_f, loc, "using variable `" + static_cast(name) + "` before it's definitely assigned"); +} + // helper function: given hint = `Ok | Err` and struct `Ok`, return `Ok` // example: `match (...) { Ok => ... }` we need to deduce `Ok` based on subject static TypePtr try_pick_instantiated_generic_from_hint(TypePtr hint, StructPtr lookup_ref) { @@ -406,6 +412,7 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, v->var_ref->declared_type); } else { assign_inferred_type(v, v->type_node ? v->type_node->resolved_type : TypeDataUnknown::create()); + flow.register_known_type(SinkExpression(v->var_ref), TypeDataUnknown::create()); // it's unknown before assigned } return ExprFlow(std::move(flow), used_as_condition); } @@ -842,6 +849,9 @@ class InferTypesAndCallsAndFieldsVisitor final { TypePtr declared_or_smart_casted = flow.smart_cast_if_exists(SinkExpression(var_ref)); tolk_assert(declared_or_smart_casted != nullptr); // all local vars are presented in flow assign_inferred_type(v, declared_or_smart_casted); + if (var_ref->is_lateinit() && declared_or_smart_casted == TypeDataUnknown::create() && v->is_rvalue) { + fire_error_using_lateinit_variable_uninitialized(cur_f, v->loc, v->get_name()); + } // it might be `local_var()` also, don't fill out_f_called, we have no fun_ref, it's a call of arbitrary expression } else if (GlobalConstPtr const_ref = v->sym->try_as()) { @@ -1220,6 +1230,8 @@ class InferTypesAndCallsAndFieldsVisitor final { TypeInferringUnifyStrategy branches_unifier; FlowContext arms_entry_facts = flow.clone(); FlowContext match_out_flow; + bool has_expr_arm = false; + bool has_else_arm = false; for (int i = 0; i < v->get_arms_count(); ++i) { auto v_arm = v->get_arm(i); @@ -1245,7 +1257,13 @@ class InferTypesAndCallsAndFieldsVisitor final { } } arm_flow.register_known_type(s_expr, exact_type); + + } else if (v_arm->pattern_kind == MatchArmKind::const_expression) { + has_expr_arm = true; + } else if (v_arm->pattern_kind == MatchArmKind::else_branch) { + has_else_arm = true; } + arm_flow = infer_any_expr(v_arm->get_body(), std::move(arm_flow), false, hint).out_flow; match_out_flow = i == 0 ? std::move(arm_flow) : FlowContext::merge_flow(std::move(match_out_flow), std::move(arm_flow)); branches_unifier.unify_with(v_arm->get_body()->inferred_type, hint); @@ -1253,6 +1271,10 @@ class InferTypesAndCallsAndFieldsVisitor final { if (v->get_arms_count() == 0) { match_out_flow = std::move(arms_entry_facts); } + if (has_expr_arm && !has_else_arm) { + FlowContext else_flow = process_any_statement(createV(v->loc), std::move(arms_entry_facts)); + match_out_flow = FlowContext::merge_flow(std::move(match_out_flow), std::move(else_flow)); + } if (v->is_statement()) { assign_inferred_type(v, TypeDataVoid::create()); diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index 4452ea573..ab17c5b42 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -132,15 +132,15 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { static NameAndScopeResolver current_scope; static FunctionPtr cur_f; - static LocalVarPtr create_local_var_sym(std::string_view name, SrcLocation loc, AnyTypeV declared_type_node, bool immutable) { - LocalVarData* v_sym = new LocalVarData(static_cast(name), loc, declared_type_node, immutable * LocalVarData::flagImmutable, -1); + static LocalVarPtr create_local_var_sym(std::string_view name, SrcLocation loc, AnyTypeV declared_type_node, bool immutable, bool lateinit) { + LocalVarData* v_sym = new LocalVarData(static_cast(name), loc, declared_type_node, immutable * LocalVarData::flagImmutable + lateinit * LocalVarData::flagLateInit, -1); current_scope.add_local_var(v_sym); return v_sym; } static void process_catch_variable(AnyExprV catch_var) { if (auto v_ref = catch_var->try_as()) { - LocalVarPtr var_ref = create_local_var_sym(v_ref->get_name(), catch_var->loc, nullptr, true); + LocalVarPtr var_ref = create_local_var_sym(v_ref->get_name(), catch_var->loc, nullptr, true, false); v_ref->mutate()->assign_sym(var_ref); } } @@ -158,7 +158,7 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { } v->mutate()->assign_var_ref(var_ref); } else { - LocalVarPtr var_ref = create_local_var_sym(v->get_name(), v->loc, v->type_node, v->is_immutable); + LocalVarPtr var_ref = create_local_var_sym(v->get_name(), v->loc, v->type_node, v->is_immutable, v->is_lateinit); v->mutate()->assign_var_ref(var_ref); } } diff --git a/tolk/symtable.h b/tolk/symtable.h index 38f090e7b..e6b2f1ce6 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -49,6 +49,7 @@ struct LocalVarData final : Symbol { enum { flagMutateParameter = 1, // parameter was declared with `mutate` keyword flagImmutable = 2, // variable was declared via `val` (not `var`) + flagLateInit = 4, // variable was declared via `lateinit` (not assigned at declaration) }; AnyTypeV type_node; // either at declaration `var x:int`, or if omitted, from assigned value `var x=2` @@ -74,6 +75,7 @@ struct LocalVarData final : Symbol { bool is_parameter() const { return param_idx >= 0; } bool is_immutable() const { return flags & flagImmutable; } + bool is_lateinit() const { return flags & flagLateInit; } bool is_mutate_parameter() const { return flags & flagMutateParameter; } LocalVarData* mutate() const { return const_cast(this); } From 68b67d4d7a3baebbc1f3c4fdc0e087116486cb91 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 28 May 2025 17:08:28 +0300 Subject: [PATCH 261/388] Fix loop in collation manager --- validator/collation-manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index 27af220df..e987de917 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -94,7 +94,7 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas } case CollatorsList::mode_round_robin: { size_t iters = 0; - for (size_t i = s->cur_idx; iters < s->collators.size(); (++i) %= s->collators.size()) { + for (size_t i = s->cur_idx; iters < s->collators.size(); (++i) %= s->collators.size(), ++iters) { adnl::AdnlNodeIdShort& collator = s->collators[i]; if (collators_[collator].alive) { selected_collator = collator; From 7e01156c4eafc5c72e8406e1026eca9a4ba366ac Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 29 May 2025 11:21:04 +0300 Subject: [PATCH 262/388] Use rldp2 in collator node and shard block retainer --- validator-engine/validator-engine.cpp | 2 +- validator/collation-manager.cpp | 6 +++--- validator/collation-manager.hpp | 6 +++--- validator/collator-node.cpp | 4 ++-- validator/collator-node.hpp | 5 +++-- validator/manager.cpp | 12 ++++++------ validator/manager.h | 2 ++ validator/manager.hpp | 13 +++++++++++-- validator/shard-block-retainer.cpp | 4 ++-- validator/shard-block-retainer.hpp | 6 +++--- validator/shard-block-verifier.cpp | 4 ++-- validator/shard-block-verifier.hpp | 6 +++--- 12 files changed, 41 insertions(+), 29 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index a7d28ea7f..c78e0acc3 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -2182,7 +2182,7 @@ void ValidatorEngine::start_validator() { load_collator_options(); validator_manager_ = ton::validator::ValidatorManagerFactory::create( - validator_options_, db_root_, keyring_.get(), adnl_.get(), rldp_.get(), overlay_manager_.get()); + validator_options_, db_root_, keyring_.get(), adnl_.get(), rldp_.get(), rldp2_.get(), overlay_manager_.get()); for (auto &v : config_.validators) { td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::add_permanent_key, v.first, diff --git a/validator/collation-manager.cpp b/validator/collation-manager.cpp index e987de917..853ac37ae 100644 --- a/validator/collation-manager.cpp +++ b/validator/collation-manager.cpp @@ -27,7 +27,7 @@ namespace ton::validator { void CollationManager::start_up() { - td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); + td::actor::send_closure(rldp_, &rldp2::Rldp::add_id, local_id_); update_collators_list(*opts_->get_collators_list()); } @@ -187,7 +187,7 @@ void CollationManager::collate_shard_block(ShardIdFull shard, BlockIdExt min_mas << selected_collator << ") in " << timer.elapsed() << "s"; P.set_result(std::move(candidate)); }; - td::actor::send_closure(rldp_, &rldp::Rldp::send_query_ex, local_id_, selected_collator, "collatequery", + td::actor::send_closure(rldp_, &rldp2::Rldp::send_query_ex, local_id_, selected_collator, "collatequery", std::move(P2), timeout, std::move(query), max_answer_size); } @@ -339,7 +339,7 @@ void CollationManager::alarm() { td::actor::send_closure(SelfId, &CollationManager::got_pong, id, std::move(R)); }; LOG(DEBUG) << "sending ping to " << id; - td::actor::send_closure(rldp_, &rldp::Rldp::send_query, local_id_, id, "ping", std::move(P), + td::actor::send_closure(rldp_, &rldp2::Rldp::send_query, local_id_, id, "ping", std::move(P), td::Timestamp::in(2.0), std::move(query)); } else { alarm_timestamp().relax(collator.ping_at); diff --git a/validator/collation-manager.hpp b/validator/collation-manager.hpp index 048d30812..5b8d2b7a0 100644 --- a/validator/collation-manager.hpp +++ b/validator/collation-manager.hpp @@ -17,7 +17,7 @@ #pragma once #include "interfaces/validator-manager.h" -#include "rldp/rldp.h" +#include "rldp2/rldp.h" #include namespace ton::validator { @@ -27,7 +27,7 @@ class ValidatorManager; class CollationManager : public td::actor::Actor { public: CollationManager(adnl::AdnlNodeIdShort local_id, td::Ref opts, - td::actor::ActorId manager, td::actor::ActorId rldp) + td::actor::ActorId manager, td::actor::ActorId rldp) : local_id_(local_id), opts_(opts), manager_(manager), rldp_(rldp) { } @@ -50,7 +50,7 @@ class CollationManager : public td::actor::Actor { adnl::AdnlNodeIdShort local_id_; td::Ref opts_; td::actor::ActorId manager_; - td::actor::ActorId rldp_; + td::actor::ActorId rldp_; void collate_shard_block(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, Ed25519_PublicKey creator, BlockCandidatePriority priority, diff --git a/validator/collator-node.cpp b/validator/collator-node.cpp index 238eb8921..4b0d7e140 100644 --- a/validator/collator-node.cpp +++ b/validator/collator-node.cpp @@ -29,7 +29,7 @@ namespace ton::validator { CollatorNode::CollatorNode(adnl::AdnlNodeIdShort local_id, td::Ref opts, td::actor::ActorId manager, td::actor::ActorId adnl, - td::actor::ActorId rldp) + td::actor::ActorId rldp) : local_id_(local_id) , opts_(std::move(opts)) , manager_(std::move(manager)) @@ -58,7 +58,7 @@ void CollatorNode::start_up() { td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, adnl::Adnl::int_to_bytestring(ton_api::collatorNode_ping::ID), std::make_unique(actor_id(this))); - td::actor::send_closure(rldp_, &rldp::Rldp::add_id, adnl::AdnlNodeIdShort(local_id_)); + td::actor::send_closure(rldp_, &rldp2::Rldp::add_id, adnl::AdnlNodeIdShort(local_id_)); } void CollatorNode::tear_down() { diff --git a/validator/collator-node.hpp b/validator/collator-node.hpp index 604338f0b..a4df03681 100644 --- a/validator/collator-node.hpp +++ b/validator/collator-node.hpp @@ -18,6 +18,7 @@ #include "interfaces/validator-manager.h" #include "rldp/rldp.h" +#include "rldp2/rldp.h" #include #include @@ -29,7 +30,7 @@ class CollatorNode : public td::actor::Actor { public: CollatorNode(adnl::AdnlNodeIdShort local_id, td::Ref opts, td::actor::ActorId manager, td::actor::ActorId adnl, - td::actor::ActorId rldp); + td::actor::ActorId rldp); void start_up() override; void tear_down() override; void add_shard(ShardIdFull shard); @@ -53,7 +54,7 @@ class CollatorNode : public td::actor::Actor { td::Ref opts_; td::actor::ActorId manager_; td::actor::ActorId adnl_; - td::actor::ActorId rldp_; + td::actor::ActorId rldp_; std::vector collating_shards_; std::set validator_adnl_ids_; diff --git a/validator/manager.cpp b/validator/manager.cpp index 47c243eb4..8cf85a9dc 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2643,7 +2643,7 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group td::actor::ActorId ValidatorManagerImpl::get_collation_manager(adnl::AdnlNodeIdShort adnl_id) { auto &actor = collation_managers_[adnl_id]; if (actor.empty()) { - actor = td::actor::create_actor("collation", adnl_id, opts_, actor_id(this), rldp_); + actor = td::actor::create_actor("collation", adnl_id, opts_, actor_id(this), rldp2_); } return actor.get(); } @@ -3486,7 +3486,7 @@ void ValidatorManagerImpl::add_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh auto it = collator_nodes_.find(id); if (it == collator_nodes_.end()) { it = collator_nodes_.emplace(id, Collator()).first; - it->second.actor = td::actor::create_actor("collatornode", id, opts_, actor_id(this), adnl_, rldp_); + it->second.actor = td::actor::create_actor("collatornode", id, opts_, actor_id(this), adnl_, rldp2_); if (last_masterchain_state_.not_null()) { td::actor::send_closure(it->second.actor, &CollatorNode::new_masterchain_block_notification, last_masterchain_state_); @@ -3633,10 +3633,10 @@ td::Ref ValidatorManagerImpl::get_block_persistent_s td::actor::ActorOwn ValidatorManagerFactory::create( td::Ref opts, std::string db_root, td::actor::ActorId keyring, - td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays) { return td::actor::create_actor("manager", std::move(opts), db_root, keyring, adnl, - rldp, overlays); + rldp, rldp2, overlays); } void ValidatorManagerImpl::log_collate_query_stats(CollationStats stats) { @@ -3783,7 +3783,7 @@ void ValidatorManagerImpl::init_shard_block_verifier(adnl::AdnlNodeIdShort local shard_block_verifier_ = {}; } else { shard_block_verifier_ = td::actor::create_actor( - "shardblockverifier", local_id, last_masterchain_state_, opts_, actor_id(this), adnl_, rldp_); + "shardblockverifier", local_id, last_masterchain_state_, opts_, actor_id(this), adnl_, rldp2_); } } } @@ -3799,7 +3799,7 @@ void ValidatorManagerImpl::wait_verify_shard_blocks(std::vector bloc void ValidatorManagerImpl::add_shard_block_retainer(adnl::AdnlNodeIdShort id) { shard_block_retainers_[id] = td::actor::create_actor( - "shardblockretainer", id, last_masterchain_state_, opts_, actor_id(this), adnl_, rldp_); + "shardblockretainer", id, last_masterchain_state_, opts_, actor_id(this), adnl_, rldp2_); } void ValidatorManagerImpl::iterate_temp_block_handles(std::function f) { diff --git a/validator/manager.h b/validator/manager.h index 4fe2037fc..652b77588 100644 --- a/validator/manager.h +++ b/validator/manager.h @@ -20,6 +20,7 @@ #include "validator/validator.h" #include "adnl/adnl.h" #include "rldp/rldp.h" +#include "rldp2/rldp.h" namespace ton { @@ -32,6 +33,7 @@ class ValidatorManagerFactory { td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId rldp2, td::actor::ActorId overlays); }; diff --git a/validator/manager.hpp b/validator/manager.hpp index 82986a060..ae90f651f 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -32,6 +32,7 @@ #include "manager-init.h" #include "state-serializer.hpp" #include "rldp/rldp.h" +#include "rldp2/rldp.h" #include "token-manager.h" #include "queue-size-counter.hpp" #include "validator-telemetry.hpp" @@ -595,8 +596,15 @@ class ValidatorManagerImpl : public ValidatorManager { ValidatorManagerImpl(td::Ref opts, std::string db_root, td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays) - : opts_(std::move(opts)), db_root_(db_root), keyring_(keyring), adnl_(adnl), rldp_(rldp), overlays_(overlays) { + td::actor::ActorId rldp, td::actor::ActorId rldp2, + td::actor::ActorId overlays) + : opts_(std::move(opts)) + , db_root_(db_root) + , keyring_(keyring) + , adnl_(adnl) + , rldp_(rldp) + , rldp2_(rldp2) + , overlays_(overlays) { } public: @@ -714,6 +722,7 @@ class ValidatorManagerImpl : public ValidatorManager { td::actor::ActorId keyring_; td::actor::ActorId adnl_; td::actor::ActorId rldp_; + td::actor::ActorId rldp2_; td::actor::ActorId overlays_; td::actor::ActorOwn serializer_; diff --git a/validator/shard-block-retainer.cpp b/validator/shard-block-retainer.cpp index c36a9bac6..528f6bcf7 100644 --- a/validator/shard-block-retainer.cpp +++ b/validator/shard-block-retainer.cpp @@ -41,7 +41,7 @@ void ShardBlockRetainer::start_up() { td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, adnl::Adnl::int_to_bytestring(ton_api::shardBlockVerifier_subscribe::ID), std::make_unique(actor_id(this))); - td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); + td::actor::send_closure(rldp_, &rldp2::Rldp::add_id, local_id_); } void ShardBlockRetainer::tear_down() { @@ -154,7 +154,7 @@ void ShardBlockRetainer::send_confirmations(adnl::AdnlNodeIdShort dst, std::vect for (size_t i = l; i < r; ++i) { query->blocks_.push_back(create_tl_block_id(blocks[i])); } - td::actor::send_closure(rldp_, &rldp::Rldp::send_message, local_id_, dst, serialize_tl_object(query, true)); + td::actor::send_closure(rldp_, &rldp2::Rldp::send_message, local_id_, dst, serialize_tl_object(query, true)); } } diff --git a/validator/shard-block-retainer.hpp b/validator/shard-block-retainer.hpp index ca8d4c877..366a6d506 100644 --- a/validator/shard-block-retainer.hpp +++ b/validator/shard-block-retainer.hpp @@ -16,7 +16,7 @@ */ #pragma once #include "interfaces/validator-manager.h" -#include "rldp.h" +#include "rldp2/rldp.h" #include namespace ton::validator { @@ -25,7 +25,7 @@ class ShardBlockRetainer : public td::actor::Actor { public: ShardBlockRetainer(adnl::AdnlNodeIdShort local_id, td::Ref last_masterchain_state, td::Ref opts, td::actor::ActorId manager, - td::actor::ActorId adnl, td::actor::ActorId rldp) + td::actor::ActorId adnl, td::actor::ActorId rldp) : local_id_(local_id) , last_masterchain_state_(last_masterchain_state) , opts_(std::move(opts)) @@ -49,7 +49,7 @@ class ShardBlockRetainer : public td::actor::Actor { td::Ref opts_; td::actor::ActorId manager_; td::actor::ActorId adnl_; - td::actor::ActorId rldp_; + td::actor::ActorId rldp_; bool inited_ = false; std::set validator_adnl_ids_; diff --git a/validator/shard-block-verifier.cpp b/validator/shard-block-verifier.cpp index 000260a36..ebd611144 100644 --- a/validator/shard-block-verifier.cpp +++ b/validator/shard-block-verifier.cpp @@ -41,7 +41,7 @@ void ShardBlockVerifier::start_up() { td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id_, adnl::Adnl::int_to_bytestring(ton_api::shardBlockVerifier_confirmBlocks::ID), std::make_unique(actor_id(this))); - td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); + td::actor::send_closure(rldp_, &rldp2::Rldp::add_id, local_id_); } void ShardBlockVerifier::tear_down() { @@ -107,7 +107,7 @@ void ShardBlockVerifier::alarm() { LOG(WARNING) << "Subscribe to " << node_id << " for " << shard.to_str() << " : " << R.move_as_error(); } }; - td::actor::send_closure(rldp_, &rldp::Rldp::send_query, local_id_, node_id, "subscribe", std::move(P), + td::actor::send_closure(rldp_, &rldp2::Rldp::send_query, local_id_, node_id, "subscribe", std::move(P), td::Timestamp::in(3.0), create_serialize_tl_object( create_tl_shard_id(shard_config.shard_id), 0)); diff --git a/validator/shard-block-verifier.hpp b/validator/shard-block-verifier.hpp index 82c1e3738..025996681 100644 --- a/validator/shard-block-verifier.hpp +++ b/validator/shard-block-verifier.hpp @@ -16,7 +16,7 @@ */ #pragma once #include "interfaces/validator-manager.h" -#include "rldp.h" +#include "rldp2/rldp.h" #include namespace ton::validator { @@ -25,7 +25,7 @@ class ShardBlockVerifier : public td::actor::Actor { public: ShardBlockVerifier(adnl::AdnlNodeIdShort local_id, td::Ref last_masterchain_state, td::Ref opts, td::actor::ActorId manager, - td::actor::ActorId adnl, td::actor::ActorId rldp) + td::actor::ActorId adnl, td::actor::ActorId rldp) : local_id_(local_id) , last_masterchain_state_(last_masterchain_state) , opts_(std::move(opts)) @@ -54,7 +54,7 @@ class ShardBlockVerifier : public td::actor::Actor { td::Ref opts_; td::actor::ActorId manager_; td::actor::ActorId adnl_; - td::actor::ActorId rldp_; + td::actor::ActorId rldp_; td::Ref config_; From d6db65bc630c937b513f8c900f70c68dda2bfa34 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Mon, 14 Apr 2025 16:30:19 +0300 Subject: [PATCH 263/388] [Tolk] Built-in type `address` It holds internal/external/none addresses. It's a slice at runtime, but a dedicated type from the type system point of view. Auto-serialization will easily pack/unpack it. Operators == and != work on addresses. --- crypto/smartcont/tolk-stdlib/common.tolk | 89 +++--- tolk-tester/tests/cells-slices.tolk | 2 +- tolk-tester/tests/constants-tests.tolk | 11 +- .../tests/invalid-syntax/err-3208.tolk | 8 + .../tests/invalid-syntax/err-3357.tolk | 8 + .../tests/invalid-typing/err-6710.tolk | 15 + tolk-tester/tests/parse-address.tolk | 266 +++++++++++------- tolk-tester/tests/strings-tests.tolk | 8 +- tolk/builtins.cpp | 4 +- tolk/constant-evaluator.cpp | 50 ++-- tolk/pipe-ast-to-legacy.cpp | 26 ++ tolk/pipe-check-inferred-types.cpp | 19 +- tolk/pipe-optimize-boolean-expr.cpp | 9 + tolk/pipe-resolve-types.cpp | 1 + tolk/type-system.cpp | 32 ++- tolk/type-system.h | 19 ++ 16 files changed, 398 insertions(+), 169 deletions(-) create mode 100644 tolk-tester/tests/invalid-syntax/err-3208.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3357.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6710.tolk diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 8ff2010bc..642337df5 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -133,10 +133,10 @@ fun mulDivMod(x: int, y: int, z: int): (int, int) /// Example: `contract.getCode()` and other methods. struct contract {} -/// Returns the internal address of the current smart contract as a Slice with a `MsgAddressInt`. -/// If necessary, it can be parsed further using primitives such as [parseStandardAddress]. +/// Returns the internal address of the current smart contract. +/// If necessary, it can be parsed further using [address.getWorkchain] and others. @pure -fun contract.getAddress(): slice +fun contract.getAddress(): address asm "MYADDR"; /// Returns the balance (in nanotoncoins) of the smart contract at the start of Computation Phase. @@ -407,15 +407,6 @@ fun debug.dumpStack(): void When you _preload_ some data, you just get the result without mutating the slice. */ -/// Compile-time function that converts a constant string to TL-encoded MsgAddressInt (TVM slice). -/// Example: stringAddressToSlice("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5") -/// Example: stringAddressToSlice("0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8") -/// Note: it's a compile-time function, `stringAddressToSlice(slice_var)` does not work. -/// Use `parseStandardAddress` to decode a slice at runtime into workchain and hash. -@pure -fun stringAddressToSlice(constStringAddress: slice): slice - builtin; - /// Compile-time function that converts a constant hex-encoded string to N/2 bytes. /// Example: `const v = stringHexToSlice("abcdef")` = slice with 3 bytes `[ 0xAB, 0xCD, 0xEF ]` /// Note: stringHexToSlice(slice_var) does not work! It accepts a constant string and works at compile-time. @@ -573,6 +564,11 @@ fun builder.storeUint(mutate self, x: int, len: int): self fun builder.storeSlice(mutate self, s: slice): self asm "STSLICER"; +/// Stores an address into a builder. +@pure +fun builder.storeAddress(mutate self, addr: address): self + asm "STSLICER"; + /// Stores amount of Toncoins into a builder. @pure fun builder.storeCoins(mutate self, x: coins): self @@ -694,37 +690,64 @@ fun builder.bitsCount(self): int where `u`, `x`, and `s` have the same meaning as for `addr_std`. */ -/// Loads from slice [s] the only prefix that is a valid `MsgAddress`, -/// and returns both this prefix `s'` and the remainder `s''` of [s] as slices. +/// Compile-time function that parses a valid contract address. +/// Example: address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5") +/// Example: address("0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8") +/// Returns `address`, which can be stored in a builder, compared with `==`, etc. @pure -fun slice.loadAddress(mutate self): slice - asm( -> 1 0) "LDMSGADDR"; +fun address(stdAddress: slice): address + builtin; + +/// Creates a slice representing TL addr_none$00 (two `0` bits). +@pure +fun createAddressNone(): address + asm "b{00} PUSHSLICE"; + +/// Returns if it's an empty address. +/// Don't confuse it with null! Empty address is a slice with two `0` bits. +/// In TL/B, it's addr_none$00. +@pure +fun address.isNone(self): bool + asm "b{00} SDBEGINSQ" "NIP"; -/// Decomposes slice [s] containing a valid `MsgAddress` into a `tuple t` with separate fields of this `MsgAddress`. -/// If [s] is not a valid `MsgAddress`, a cell deserialization exception is thrown. +/// Returns if it's a standard (internal) address. Such addresses contain workchain (8 bits) and hash (256 bits). +/// All contract addresses are internal, so it's the most practical use case. +/// In TL/B it's addr_std$10. +/// For internal addresses, you can call [address.getWorkchain] and [address.getWorkchainAndHash]. @pure -fun parseAddress(s: slice): tuple - asm "PARSEMSGADDR"; +fun address.isInternal(self): bool + asm "b{10} SDBEGINSQ" "NIP"; -/// Parses slice [s] containing a valid `MsgAddressInt` (usually a `msg_addr_std`), -/// applies rewriting from the anycast (if present) to the same-length prefix of the address, -/// and returns both the workchain and the 256-bit address as integers. -/// If the address is not 256-bit, or if [s] is not a valid serialization of `MsgAddressInt`, -/// throws a cell deserialization exception. +/// Returns if it's an external address, used to communication with the outside world. +/// In TL/B it's addr_extern$01. @pure -fun parseStandardAddress(s: slice): (int, int) +fun address.isExternal(self): bool + asm "b{01} SDBEGINSQ" "NIP"; + +/// Extracts workchain and hash from a standard (internal) address. +/// If the address is not internal, throws a cell deserialization exception. +@pure +fun address.getWorkchainAndHash(self): (int8, uint256) asm "REWRITESTDADDR"; -/// Creates a slice representing TL addr_none$00 (two `0` bits). +/// Extracts workchain from a standard (internal) address. +/// If the address is not internal, throws a cell deserialization exception. @pure -fun createAddressNone(): slice - asm "b{00} PUSHSLICE"; +fun address.getWorkchain(self): int8 + asm "REWRITESTDADDR" "DROP"; -/// Returns if a slice pointer contains an empty address. -/// In other words, a slice starts with two `0` bits (TL addr_none$00). +/// Checks whether two addresses are equal. Equivalent to `a == b`. +/// Deprecated! Left for smoother transition from FunC, where you used `slice` everywhere. +/// Use just `a == b` and `a != b` to compare addresses, don't use bitsEqual. @pure -fun addressIsNone(s: slice): bool - asm "2 PLDU" "0 EQINT"; +@deprecated("use `senderAddress == ownerAddress`, not `senderAddress.bitsEqual(ownerAddress)`") +fun address.bitsEqual(self, b: address): bool + asm "SDEQ"; + +/// Loads from slice [s] a valid `MsgAddress` (none/internal/external). +@pure +fun slice.loadAddress(mutate self): address + asm( -> 1 0) "LDMSGADDR"; /** diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index 70c40b531..e5b89424a 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -183,7 +183,7 @@ fun test111() { .endCell().beginParse(); var op1 = s.loadUint(32); var q1 = s.loadUint(64); - if (addressIsNone(s)) { + if ((s as address).isNone()) { s.skipBits(2); } if (s.loadBool() == false) { diff --git a/tolk-tester/tests/constants-tests.tolk b/tolk-tester/tests/constants-tests.tolk index ed510c13c..838a7dae3 100644 --- a/tolk-tester/tests/constants-tests.tolk +++ b/tolk-tester/tests/constants-tests.tolk @@ -22,6 +22,8 @@ const nibbles: int = 4; const strange_zero = (!10 as int); const strange_minus_1: MInt = (!0 as int); +const addr1 = address("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"); + const true1 = true; const true2 = !!true; const true3 = true1 && true2; @@ -88,6 +90,12 @@ fun test5() { return (intOrN == null, int32Or64 is int32, int32Or64); } +@method_id(106) +fun test6() { + __expect_type(addr1, "address"); + return (addr1 == addr1, addr1 != addr1, addr1 == createAddressNone(), addr1.getWorkchain()); +} + fun main() { var i1: int = iget1(); var i2: int = iget2(); @@ -117,6 +125,7 @@ fun main() { @testcase | 103 | | 0 0 @testcase | 104 | | 1 1 2 @testcase | 105 | | -1 0 7 48 +@testcase | 106 | | -1 0 0 -1 -@code_hash 49556957179018386976033482229516007597784982050169632168468608374010225644988 +@code_hash 85012002134196298607946339234948178079079623823648671175958399073436861460061 */ diff --git a/tolk-tester/tests/invalid-syntax/err-3208.tolk b/tolk-tester/tests/invalid-syntax/err-3208.tolk new file mode 100644 index 000000000..75292b252 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3208.tolk @@ -0,0 +1,8 @@ +fun main() { + return address("0:gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg"); +} + +/** +@compilation_should_fail +@stderr invalid standard address + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3357.tolk b/tolk-tester/tests/invalid-syntax/err-3357.tolk new file mode 100644 index 000000000..de5569da3 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3357.tolk @@ -0,0 +1,8 @@ +fun main() { + return ton("1000000000"); +} + +/** +@compilation_should_fail +@stderr argument is too big and leads to overflow + */ diff --git a/tolk-tester/tests/invalid-typing/err-6710.tolk b/tolk-tester/tests/invalid-typing/err-6710.tolk new file mode 100644 index 000000000..d0fae1d56 --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6710.tolk @@ -0,0 +1,15 @@ +fun f(s: slice) { + +} + +fun testCantPassAddressToSlice() { + var a = createAddressNone(); + f(a as slice); // ok + f(a); +} + +/** +@compilation_should_fail +@stderr can not pass `address` to `slice` +@stderr f(a); + */ diff --git a/tolk-tester/tests/parse-address.tolk b/tolk-tester/tests/parse-address.tolk index 912fb84c2..cf74af3f1 100644 --- a/tolk-tester/tests/parse-address.tolk +++ b/tolk-tester/tests/parse-address.tolk @@ -1,113 +1,171 @@ -const cc1 = stringAddressToSlice("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e"); -const cc2 = stringAddressToSlice("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"); +const cc1 = address("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e"); +const cc2 = address("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"); -fun verifyAddr(addr: slice, workchain: int, number: int) { - assert (addr.remainingBitsCount() == 3 + 8 + 256) throw 112; - addr.skipBits(3); - assert (addr.loadUint(8) == workchain) throw 111; - assert (addr.loadUint(256) == number) throw 111; +fun verifyAddr(addr: address, workchain: int, number: int) { + assert (addr == addr) throw 111; + assert (!(addr != addr)) throw 111; + assert (addr.isInternal(), 111); + assert (!addr.isExternal(), 111); + assert (!addr.isNone(), 111); + assert (addr.bitsEqual(addr), 111); + + var (wc, h) = addr.getWorkchainAndHash(); + assert (wc == workchain) throw 111; + assert (h == number) throw 111; + assert (addr.getWorkchain() == workchain) throw 111; + + var s = addr as slice; + assert (s.remainingBitsCount() == 3 + 8 + 256) throw 112; + s.skipBits(3); + assert (s.loadInt(8) == workchain) throw 111; + assert (s.loadUint(256) == number) throw 111; +} + +fun check0(addr: address) { + assert (addr.getWorkchain() == 0) throw 111; +} + +fun codegenAddrEq(a: address, b: address) { + if (a == b) { return 1; } + if (a != b) { return 2; } + return 3; } fun main() { - verifyAddr(stringAddressToSlice("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"), 255, 23158417847463239084714197001737581570653996933128112807891516801582625927987); - verifyAddr(stringAddressToSlice("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"), 0, 0); - verifyAddr(stringAddressToSlice("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"), 0, 65607996509792174074532427555986248720836864382484024657400295821210434460432); - verifyAddr(stringAddressToSlice("UQCOgxbCOjOLH_cEuQdGgS23zBM5SrQQepMFedjK-oixYbis"), 0, 64460038539088394980732229180523693489682583665805557562964506609821558550881); - verifyAddr(stringAddressToSlice("EQDa4VOnTYlLvDJ0gZjNYm5PXfSmmtL6Vs6A_CZEtXCNICq_"), 0, 99002318936150612861744867526221033858534811876886359650897405270877291973920); - verifyAddr(stringAddressToSlice("Ef8BtXO9bcTMXjg9bgivKh4lhJmZWQPP6_rb9vfjlTP5FJtM"), 255, 772910975127952880303441415761050161913031788763061162001556772893733681428); - verifyAddr(stringAddressToSlice("Ef89xh-uy860-mCcvS8zcAUs8bApmxLGygDLEKjUk5RL-311"), 255, 27941138149036269893630478666581900122707382189183906805784676408403709676539); - verifyAddr(stringAddressToSlice("Ef_vA6yRfmt2P4UHnxlrQUZFcBnKux8mL2eMqBgpeMFPorr4"), 255, 108109262375472472702582493362335418330829651067377177643099076957184687427490); - verifyAddr(stringAddressToSlice("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi"), 255, 18502444830824300068094395885436326119386947594392869497312068745716154912158); - verifyAddr(stringAddressToSlice("Ef_fvrd0hBoVJUxoi7wH173Zk8NPiyVvxh5IoYSjEYZbOhsu"), 255, 101202732337223525952216789200341489000836292542250083765062769181728788863802); - verifyAddr(stringAddressToSlice("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf"), 255, 46952625717497919357580310066854892621799390294920450816077086267929711460872); - verifyAddr(stringAddressToSlice("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA"), 255, 48545777798729612074233611768739897492467685225150339217043102685589809464695); - verifyAddr(stringAddressToSlice("Ef9LynHHKgBxY6-l-W_dWN-CtGT2_ji5rN3EzOI-p9zWEfq6"), 255, 34281152017620085319078796986198022632548048219136747083019177301186013091345); - verifyAddr(stringAddressToSlice("Ef9hMd78gzSiVsK0zz0AHtEja8x1UoB_NDZMjn-l86NQK_2Y"), 255, 43962460814164090767878334494257755557842170134382045184921495822637115592747); - verifyAddr(stringAddressToSlice("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6"), 255, 23557057702048801338698514499604413540742716310574705490458593067566768087319); - verifyAddr(stringAddressToSlice("Ef_fdIbThooPs4_r2DE_Z6ZsWycJdHLnsuKAJHTcbaZaipez"), 255, 101071650030310556115830521522496708686577365303530257137459798093298869361290); - verifyAddr(stringAddressToSlice("Ef_lva0qEiZhWrrZJl-IJxyCcTQmmTo71fIWyQ31HxJ8NurV"), 255, 103914771557158282349484109182290824591675204108148026180964788916630125182006); - verifyAddr(stringAddressToSlice("Ef8sMGKypw006AeRYqimLjmY2Ufp-SHk8C0ZJBNgVBlzw_Nr"), 255, 19987255184378161380023126214650814972824352533523055905552702178965809886147); - verifyAddr(stringAddressToSlice("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff+W72r5gqPrHF"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); - verifyAddr(stringAddressToSlice("EQCaSCHVak-jIc9ANutTAfHpZNM3YdGky7yaDzsTrg0WhFlm"), 0, 69783625181781015447914682554083924798054947959007050695795761257887453484676); - verifyAddr(stringAddressToSlice("EQBS9U3AfD15fGmOtRMXQAxcPVBwNuItfLcDni9fkbTyyNX0"), 0, 37523067738561024305547433298623118197038688994386001017161816416175242146504); - verifyAddr(stringAddressToSlice("EQBiMNL9qNWMAkJHuM0BFneYcuHL17kzS4pswpaEO-NGWrFG"), 0, 44412924025649114419413541526870954696667907029239618728289150652715284776538); - verifyAddr(stringAddressToSlice("EQAUzE-Nef80O9dLZy91HfPiOb6EEQ8YqyWKyIU-KeaYLNUi"), 0, 9407242825041766837311851458322335726136775042891143504070507665010681354284); - verifyAddr(stringAddressToSlice("EQD-nhrinjv0B4LTgr0dRHTHwH1MOsgGhKBXJZd7vESMZUf1"), 0, 115166810931401616117484448645661180241548402534908005320733783571353775148133); - verifyAddr(stringAddressToSlice("EQAVD3Fni9I6j8XeSIl-wAGBEhqhame6OtAY0GScKT0D9X6f"), 0, 9525855215156855607080079714361451576383963668563135377495902959388099150837); - verifyAddr(stringAddressToSlice("EQC6ACq3VANZjqfRBy7JMHkpLwqQ9qyYJsCIGx1mYbQgxaKw"), 0, 84130484652351964071210477536969520113177637645401392541565606610268614566085); - verifyAddr(stringAddressToSlice("EQCIJLNFIko5CvpKn9oAkrDgLocDOoD4vwmHxNx_fsG_LkwW"), 0, 61579391178099797614367237687950512448308156724136883899001108680249616482094); - verifyAddr(stringAddressToSlice("EQCe4AYIBce1pAk2qJJPSs1OzyZRlKjkfq8zuC8D7erv6DUP"), 0, 71861245445432818728925844931259040612664802586395398157190478191760507596776); - verifyAddr(stringAddressToSlice("EQCtrtTXEAoSpoERmiqOnICe9LHxn2N89N4BH9qdHlrG-U0i"), 0, 78559023162479717496981724991265882229440558807791659796411897368395464230649); - verifyAddr(stringAddressToSlice("EQBBlraAps0OZaB9Q8ePQn2wVAaL1G411A-dNppyWe3X3GIT"), 0, 29666621803903557832193058147214384979915773445007872807927344851911086823388); - verifyAddr(stringAddressToSlice("EQBiASqUqaVizrozLRbszkWC2kETbkhpO2qniDVDPPg2_0W8"), 0, 44328719889509369519441680467651025944540360433148852643949783408843779749631); - verifyAddr(stringAddressToSlice("EQBu2Q1EO8gIoNA1qoGWnHUudKfmqlKEDTQE-DxN-_4sdg14"), 0, 50137910719490808065414827264266674858051167131188257457782826342827836714102); - verifyAddr(stringAddressToSlice("EQA5bvxWd5-q2vJUVqR9AlbEIfdFysLR0PXGgVlBf8x5hWuF"), 0, 25977927117604457079092522008276392864656238504700352770597256138254994667909); - verifyAddr(stringAddressToSlice("EQBguMSHjFv5bfoOdshr3ruS9ymSZzhRKMovoNrxGxZXvmee"), 0, 43748489720571123896506696370504498290006245978262404519821633796370658121662); - verifyAddr(stringAddressToSlice("EQAxL0oF1-zNgimPKthbDnYS4xj94rHtfNRN7_Pd1r2LNNv3"), 0, 22246882279393590648219842750911786376759362211171398419754426796438233910068); - verifyAddr(stringAddressToSlice("EQANX1uRKGZfyPIwEaIXrR0ZOqadct5q10dvKxWIxx7SQqzW"), 0, 6048549475100840191738010856156544571222758030966479209409932714701987172930); - verifyAddr(stringAddressToSlice("EQBitdFDoU5DWSjfKq7AsO29RIwAnBzzvcVVSn5ekQoB9Liv"), 0, 44647902768175374073183447303109856895983123510911038181495138821771906122228); - verifyAddr(stringAddressToSlice("EQBgbux7VSjqJHP7ByRK1q4QuVZbpSCesNgvz5qad3lfXX_B"), 0, 43618018778298854282398238948198420936771670943015013768514626213000552996701); - verifyAddr(stringAddressToSlice("EQDisBd8U7M3CEOZ8gcWCdetdmJi3AI31zIT5qBwOdmUbsxY"), 0, 102533830955233207294921564956803510155400341370448300698800842506363763004526); - verifyAddr(stringAddressToSlice("EQAZpn_eynVlf7Ii2d6jP_p1URPrdF9F3S7DiudQyelkjzwE"), 0, 11602000355550451044739442929923326898313570892134000961608306166632391730319); - verifyAddr(stringAddressToSlice("EQDE0HBgfkOiqHezLtExBGTvOs8eitthHQosBjW3BmDy1y2K"), 0, 89021598108837008984355105304701054698583123510131754065320641619941010764503); - verifyAddr(stringAddressToSlice("EQDyT36zktBN9PVWvZ1joRxhIfEUgCPt4F2isa-enUA_d6CP"), 0, 109600164736599393471831241268953938618560132398692391517933933264745646800759); - verifyAddr(stringAddressToSlice("EQDSMUGwt25IQd3_yHjI03F71G8Kp2GMaMEv2TiWoTKbsyRH"), 0, 95072727086440754059372943502908629555499501854161516009430039520728770059187); - verifyAddr(stringAddressToSlice("EQAgK1EcrvEuL9sCtoj3cNhVNOuf3lo5GIPE2gn1fwZZYB3j"), 0, 14550545393206146289454646242321274637527057595221202748348667645886114191712); - verifyAddr(stringAddressToSlice("EQCDKqL5w_6MD-Z7AOButu-uR-ZJTsgNU1fu464hn9grY81U"), 0, 59328315557704100696483472039557119625141880163887490602190749720459366378339); - verifyAddr(stringAddressToSlice("EQB1aVMyFBhnlYXmQjsma0S63kvxKU7ccZKFNCFTwX7ASPv4"), 0, 53106696421104300082516512931084483581353095629408473618166869610568148238408); - verifyAddr(stringAddressToSlice("EQBbjrXHoxDyh1ZYGBdBoQgLaScxW6pZR1hEhJC8BqF-5Kgq"), 0, 41412616102566803060532874463898939692666425753852274254609049615175463829220); - verifyAddr(stringAddressToSlice("EQC-QeZ13QP0lszxNKt380fCWuaV94vwC/bfuqmrlg1/fJPA"), 0, 86055876869280374285292827775555707420719385459150221433115419095878595346300); - verifyAddr(stringAddressToSlice("EQAiUwpF27vXCngqNhf_TQ5E_06ah0G4zuSrnfU7CLLaht5H"), 0, 15525356059048115813946213102829493539706126913595626308144289257869196581510); - verifyAddr(stringAddressToSlice("EQBqiVjmhe2iVGmgOSDO1FGjSiz_AMtb1w7lLEiP4XIF_SFy"), 0, 48187833566271418625754761625661652107159264793429628379411792200127405491709); - verifyAddr(stringAddressToSlice("EQDmwvaK2d_SbaPdpOM60effPWeKsksgDVwFPEyxuftM396K"), 0, 104376425077737068747642645125299653296942252727305637929378053253273342397663); - verifyAddr(stringAddressToSlice("EQDWtPZZgF7wvIMUHZQojuD3utiuivsW7WslRJ33dgv-5yc8"), 0, 97114682311034709685427168495629428400170984047839002197324103884924936519399); - verifyAddr(stringAddressToSlice("EQAA7z0JI0JKqbN-1uENKz9JrxIO5ZRY-ehMeg9fPncx50Ck"), 0, 422697701361909095759185681783393186844038628935759044330165207027374567911); - verifyAddr(stringAddressToSlice("EQBVUHRoCq6coQYUwOAhGSoAmQ6Mpm7dFlDYon6HMgWV8Ftr"), 0, 38588743302295548905191533977469452945717219128199196974980570837505276220912); - verifyAddr(stringAddressToSlice("EQCTdvDCf0bA5dOPI1-44tB2ZfNcMGiklzvg27TovgDEqM6E"), 0, 66700138358140658950710678965721715920748906761125730971082529064117803730088); - verifyAddr(stringAddressToSlice("EQBDBKE5WGKIlnoi3OOzw7vkKKIX55eWjPvgxJWwek8AyL2J"), 0, 30313140970524770883308749215942283658935592719811899513010665548955593408712); - verifyAddr(stringAddressToSlice("EQAvCSyLCo21GrqLAifdov4WkOxuGQCjMRxgF1cXSaNzLHZe"), 0, 21274912932379789207153885262858116665851037273450532982121514600400844714796); - verifyAddr(stringAddressToSlice("EQCsLpDeHB2qpRbmsCb_0xmsYVNx1NgeYrvHGT1TDrHkDgL4"), 0, 77880084760844670718511036557364450189323549135231036747480176919181282894862); - verifyAddr(stringAddressToSlice("EQCTQ8kPwyX92r48gCIL_pLN_RcQT9ghZygnmDTYkOkuW_j5"), 0, 66609755171046741472463433430016629628609840960137226492652665879987546041947); - verifyAddr(stringAddressToSlice("EQCTrFRSHt-tfk7WxK9ZHQmqLcgxXxTK7wGfCEbqgY2W9Mcx"), 0, 66794468397542182731534559853537484892417154018190804733043974345563210356468); - verifyAddr(stringAddressToSlice("EQCv28y49GdaLncoclv0ISdDlMUY_cxDPGNWFCPT8t4GuqUJ"), 0, 79543100951881731989812212377176715376834409392116644269458867858071577560762); - verifyAddr(stringAddressToSlice("EQCVL-k6deDR56Z8pcb0Btg0lGfaivOGfdDCD1vvyRsyL9vS"), 0, 67479265933941008511790471646564661743401752930295407567346938670637286896175); - verifyAddr(stringAddressToSlice("EQD6t2dXDjZxF1DqunKF-8dEWivJdliY_0FYiCXnthuqnDCa"), 0, 113402258385556889021060606279033166272577193563727959698596277924908309916316); - verifyAddr(stringAddressToSlice("EQDE98XNzXiPq7VnbJJ2M4-Ht3tX_OWR0xUTTnDC8NObLmyU"), 0, 89091094739778473356272490822716056624384395256388481953562551087642791090990); - verifyAddr(stringAddressToSlice("EQDfeRDE1TDhwt478CDR0Q7MDwqcTUhfjqyTT59mgoAaF6f7"), 0, 101079669463449311486034260688909914923832300293253430457119371423825321269783); - verifyAddr(stringAddressToSlice("EQDijcEyUKa-QgCbeGlggQk1uBtt2ZRHyW4Y4gB4R6MN6RLW"), 0, 102473162609487797404330889623966425536887610061087715571345738626121871855081); - verifyAddr(stringAddressToSlice("EQDOtFOt41skbjBkZF89oYXpoDECjlxIzD-ShWAOYyzuxqLA"), 0, 93495056812773926196963707371665512785268729004579280701087533371037976424134); - verifyAddr(stringAddressToSlice("EQDuJKSFWU7AYqH6KLFfAbYvMuz346eWmJvG6_2NYE42_B4T"), 0, 107715199938628393100813870735031557263256555616273999363057194279168054802172); - verifyAddr(stringAddressToSlice("EQDwGu4vFv1e3wn8min_iy7OPJXegOYTFQ5bZFZ5a5ZPiBpX"), 0, 108602665568837301744601989570019709742180613578164394799026726718721456754568); - verifyAddr(stringAddressToSlice("EQC4G2ph6AS_mD_-cIv4aIYm1z5jAgCW_TTDEr72ygXOP2X-"), 0, 83274003234732023403481554420859495155084746906198543572711543697320249249343); - verifyAddr(stringAddressToSlice("EQDpUkyAa6lZ12P3ZB2PL_rmWwI1I55BU4kxw_rssFL5dswA"), 0, 105534303174146507629736518862713754948570412188900908177600861330298381728118); - verifyAddr(stringAddressToSlice("EQDoIA20MF1qEcSPtROdCu5ukGx9dVjgMeJh1oQ4A4cf_Jif"), 0, 104993214557977037193613824776415934089204193426692473563548548423424814817276); - verifyAddr(stringAddressToSlice("EQDpUkyAa6lZ12P3ZB2PL_rmWwI1I55BU4kxw_rssFL5dswA"), 0, 105534303174146507629736518862713754948570412188900908177600861330298381728118); - verifyAddr(stringAddressToSlice("EQClLO4EnZ_rTyV1GVpWy53pLgWJRki5c4ZzuM_1O_ClBkO9"), 0, 74711004027159342540251007601464500186374346239921204216319145006974068892934); - verifyAddr(stringAddressToSlice("EQDmkj65Ab_m0aZaW8IpKw4kYqIgITw_HRstYEkVQ6NIYCyW"), 0, 104290347741656803921830951060768893809692975574470790497562993373950614128736); - verifyAddr(stringAddressToSlice("EQCqNTwAYUNhPFS0RgqZoTLGJcQQxbAJ7csUo4YO3_TONLab"), 0, 76987241268612358571638783428744566580605181728938801022059780105627411729972); - verifyAddr(stringAddressToSlice("EQCL3DmCynaRK7-vsfeNmd4Jj-UxAIHPvA4qS2xwaL6UpLbF"), 0, 63260589232981964910240894899061676480139492286430589202252472895352724165796); - verifyAddr(stringAddressToSlice("EQDbU1SVEjBE73oUqgAoM9gDcShUkM5EC2PgoCjuwVUKo-Ee"), 0, 99203745911752606845646497420891218522647962685916739950275357890977532807843); - verifyAddr(stringAddressToSlice("EQD02VdcF4TDbCKLLhZJ39NQTu6aWq2LnLjp0oXqbNu_BANK"), 0, 110748343802097970709980079967961144373090790244250392237586606542170934198020); - verifyAddr(stringAddressToSlice("EQBynBO23ywHy_CgarY9NK9FTz0yDsG82PtcbSTQgGoXwiuA"), 0, 51839428943991432793039248316067731096592274748149794482308513726460953499586); - verifyAddr(stringAddressToSlice("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); - verifyAddr(stringAddressToSlice("EQAUTbQiM522Y_XJ_T98QPhPhTmb4nV--VSPiha8kC6kRfPO"), 0, 9183547432069678364603018431103042146626948674383548774683927217595824907333); - verifyAddr(stringAddressToSlice("EQBlqsm144Dq6SjbPI4jjZvA1hqTIP3CvHovbIfW_t-SCALE"), 0, 45985353862647206060987594732861817093328871106941773337270673759241903247880); - verifyAddr(stringAddressToSlice("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); - verifyAddr(stringAddressToSlice("kQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPgpP"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); - verifyAddr(stringAddressToSlice("kf-Dfdg-YQXaR2Q97gZJ4fGBtmV1DHOU1y1RPyyZZtRy_Ikh"), 255, 59475331506450494976393625198911249698879029820580340449086829444312920781564); - verifyAddr(stringAddressToSlice("0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 0, 37304138005561100291416421295333982606153966175434134130332440738068913455320); - verifyAddr(stringAddressToSlice("0:0000000000000000000000000000000000000000000000000000000000000000"), 0, 0); - verifyAddr(stringAddressToSlice("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff"), 0, 115792089237316195423570985008687907853269984665640564039457584007913129639935); - verifyAddr(stringAddressToSlice("0:zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"), 0, 23158417847463239084714197001737581570653996933128112807891516801582625927987); - verifyAddr(stringAddressToSlice("0:0000000000000000000000000000000000000000000000000000000000000000"), 0, 0); - verifyAddr(stringAddressToSlice("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 1, 37304138005561100291416421295333982606153966175434134130332440738068913455320); - verifyAddr(stringAddressToSlice("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 9, 37304138005561100291416421295333982606153966175434134130332440738068913455320); - verifyAddr(stringAddressToSlice("99:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 99, 37304138005561100291416421295333982606153966175434134130332440738068913455320); - verifyAddr(stringAddressToSlice("-1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 255, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + __expect_type(cc1, "address"); + + verifyAddr(address("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"), -1, 23158417847463239084714197001737581570653996933128112807891516801582625927987); + verifyAddr(address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"), 0, 0); + verifyAddr(address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"), 0, 65607996509792174074532427555986248720836864382484024657400295821210434460432); + verifyAddr(address("UQCOgxbCOjOLH_cEuQdGgS23zBM5SrQQepMFedjK-oixYbis"), 0, 64460038539088394980732229180523693489682583665805557562964506609821558550881); + verifyAddr(address("EQDa4VOnTYlLvDJ0gZjNYm5PXfSmmtL6Vs6A_CZEtXCNICq_"), 0, 99002318936150612861744867526221033858534811876886359650897405270877291973920); + verifyAddr(address("Ef8BtXO9bcTMXjg9bgivKh4lhJmZWQPP6_rb9vfjlTP5FJtM"), -1, 772910975127952880303441415761050161913031788763061162001556772893733681428); + verifyAddr(address("Ef89xh-uy860-mCcvS8zcAUs8bApmxLGygDLEKjUk5RL-311"), -1, 27941138149036269893630478666581900122707382189183906805784676408403709676539); + verifyAddr(address("Ef_vA6yRfmt2P4UHnxlrQUZFcBnKux8mL2eMqBgpeMFPorr4"), -1, 108109262375472472702582493362335418330829651067377177643099076957184687427490); + verifyAddr(address("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi"), -1, 18502444830824300068094395885436326119386947594392869497312068745716154912158); + verifyAddr(address("Ef_fvrd0hBoVJUxoi7wH173Zk8NPiyVvxh5IoYSjEYZbOhsu"), -1, 101202732337223525952216789200341489000836292542250083765062769181728788863802); + verifyAddr(address("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf"), -1, 46952625717497919357580310066854892621799390294920450816077086267929711460872); + verifyAddr(address("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA"), -1, 48545777798729612074233611768739897492467685225150339217043102685589809464695); + verifyAddr(address("Ef9LynHHKgBxY6-l-W_dWN-CtGT2_ji5rN3EzOI-p9zWEfq6"), -1, 34281152017620085319078796986198022632548048219136747083019177301186013091345); + verifyAddr(address("Ef9hMd78gzSiVsK0zz0AHtEja8x1UoB_NDZMjn-l86NQK_2Y"), -1, 43962460814164090767878334494257755557842170134382045184921495822637115592747); + verifyAddr(address("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6"), -1, 23557057702048801338698514499604413540742716310574705490458593067566768087319); + verifyAddr(address("Ef_fdIbThooPs4_r2DE_Z6ZsWycJdHLnsuKAJHTcbaZaipez"), -1, 101071650030310556115830521522496708686577365303530257137459798093298869361290); + verifyAddr(address("Ef_lva0qEiZhWrrZJl-IJxyCcTQmmTo71fIWyQ31HxJ8NurV"), -1, 103914771557158282349484109182290824591675204108148026180964788916630125182006); + verifyAddr(address("Ef8sMGKypw006AeRYqimLjmY2Ufp-SHk8C0ZJBNgVBlzw_Nr"), -1, 19987255184378161380023126214650814972824352533523055905552702178965809886147); + verifyAddr(address("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff+W72r5gqPrHF"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr(address("EQCaSCHVak-jIc9ANutTAfHpZNM3YdGky7yaDzsTrg0WhFlm"), 0, 69783625181781015447914682554083924798054947959007050695795761257887453484676); + verifyAddr(address("EQBS9U3AfD15fGmOtRMXQAxcPVBwNuItfLcDni9fkbTyyNX0"), 0, 37523067738561024305547433298623118197038688994386001017161816416175242146504); + verifyAddr(address("EQBiMNL9qNWMAkJHuM0BFneYcuHL17kzS4pswpaEO-NGWrFG"), 0, 44412924025649114419413541526870954696667907029239618728289150652715284776538); + verifyAddr(address("EQAUzE-Nef80O9dLZy91HfPiOb6EEQ8YqyWKyIU-KeaYLNUi"), 0, 9407242825041766837311851458322335726136775042891143504070507665010681354284); + verifyAddr(address("EQD-nhrinjv0B4LTgr0dRHTHwH1MOsgGhKBXJZd7vESMZUf1"), 0, 115166810931401616117484448645661180241548402534908005320733783571353775148133); + verifyAddr(address("EQAVD3Fni9I6j8XeSIl-wAGBEhqhame6OtAY0GScKT0D9X6f"), 0, 9525855215156855607080079714361451576383963668563135377495902959388099150837); + verifyAddr(address("EQC6ACq3VANZjqfRBy7JMHkpLwqQ9qyYJsCIGx1mYbQgxaKw"), 0, 84130484652351964071210477536969520113177637645401392541565606610268614566085); + verifyAddr(address("EQCIJLNFIko5CvpKn9oAkrDgLocDOoD4vwmHxNx_fsG_LkwW"), 0, 61579391178099797614367237687950512448308156724136883899001108680249616482094); + verifyAddr(address("EQCe4AYIBce1pAk2qJJPSs1OzyZRlKjkfq8zuC8D7erv6DUP"), 0, 71861245445432818728925844931259040612664802586395398157190478191760507596776); + verifyAddr(address("EQCtrtTXEAoSpoERmiqOnICe9LHxn2N89N4BH9qdHlrG-U0i"), 0, 78559023162479717496981724991265882229440558807791659796411897368395464230649); + verifyAddr(address("EQBBlraAps0OZaB9Q8ePQn2wVAaL1G411A-dNppyWe3X3GIT"), 0, 29666621803903557832193058147214384979915773445007872807927344851911086823388); + verifyAddr(address("EQBiASqUqaVizrozLRbszkWC2kETbkhpO2qniDVDPPg2_0W8"), 0, 44328719889509369519441680467651025944540360433148852643949783408843779749631); + verifyAddr(address("EQBu2Q1EO8gIoNA1qoGWnHUudKfmqlKEDTQE-DxN-_4sdg14"), 0, 50137910719490808065414827264266674858051167131188257457782826342827836714102); + verifyAddr(address("EQA5bvxWd5-q2vJUVqR9AlbEIfdFysLR0PXGgVlBf8x5hWuF"), 0, 25977927117604457079092522008276392864656238504700352770597256138254994667909); + verifyAddr(address("EQBguMSHjFv5bfoOdshr3ruS9ymSZzhRKMovoNrxGxZXvmee"), 0, 43748489720571123896506696370504498290006245978262404519821633796370658121662); + verifyAddr(address("EQAxL0oF1-zNgimPKthbDnYS4xj94rHtfNRN7_Pd1r2LNNv3"), 0, 22246882279393590648219842750911786376759362211171398419754426796438233910068); + verifyAddr(address("EQANX1uRKGZfyPIwEaIXrR0ZOqadct5q10dvKxWIxx7SQqzW"), 0, 6048549475100840191738010856156544571222758030966479209409932714701987172930); + verifyAddr(address("EQBitdFDoU5DWSjfKq7AsO29RIwAnBzzvcVVSn5ekQoB9Liv"), 0, 44647902768175374073183447303109856895983123510911038181495138821771906122228); + verifyAddr(address("EQBgbux7VSjqJHP7ByRK1q4QuVZbpSCesNgvz5qad3lfXX_B"), 0, 43618018778298854282398238948198420936771670943015013768514626213000552996701); + verifyAddr(address("EQDisBd8U7M3CEOZ8gcWCdetdmJi3AI31zIT5qBwOdmUbsxY"), 0, 102533830955233207294921564956803510155400341370448300698800842506363763004526); + verifyAddr(address("EQAZpn_eynVlf7Ii2d6jP_p1URPrdF9F3S7DiudQyelkjzwE"), 0, 11602000355550451044739442929923326898313570892134000961608306166632391730319); + verifyAddr(address("EQDE0HBgfkOiqHezLtExBGTvOs8eitthHQosBjW3BmDy1y2K"), 0, 89021598108837008984355105304701054698583123510131754065320641619941010764503); + verifyAddr(address("EQDyT36zktBN9PVWvZ1joRxhIfEUgCPt4F2isa-enUA_d6CP"), 0, 109600164736599393471831241268953938618560132398692391517933933264745646800759); + verifyAddr(address("EQDSMUGwt25IQd3_yHjI03F71G8Kp2GMaMEv2TiWoTKbsyRH"), 0, 95072727086440754059372943502908629555499501854161516009430039520728770059187); + verifyAddr(address("EQAgK1EcrvEuL9sCtoj3cNhVNOuf3lo5GIPE2gn1fwZZYB3j"), 0, 14550545393206146289454646242321274637527057595221202748348667645886114191712); + verifyAddr(address("EQCDKqL5w_6MD-Z7AOButu-uR-ZJTsgNU1fu464hn9grY81U"), 0, 59328315557704100696483472039557119625141880163887490602190749720459366378339); + verifyAddr(address("EQB1aVMyFBhnlYXmQjsma0S63kvxKU7ccZKFNCFTwX7ASPv4"), 0, 53106696421104300082516512931084483581353095629408473618166869610568148238408); + verifyAddr(address("EQBbjrXHoxDyh1ZYGBdBoQgLaScxW6pZR1hEhJC8BqF-5Kgq"), 0, 41412616102566803060532874463898939692666425753852274254609049615175463829220); + verifyAddr(address("EQC-QeZ13QP0lszxNKt380fCWuaV94vwC/bfuqmrlg1/fJPA"), 0, 86055876869280374285292827775555707420719385459150221433115419095878595346300); + verifyAddr(address("EQAiUwpF27vXCngqNhf_TQ5E_06ah0G4zuSrnfU7CLLaht5H"), 0, 15525356059048115813946213102829493539706126913595626308144289257869196581510); + verifyAddr(address("EQBqiVjmhe2iVGmgOSDO1FGjSiz_AMtb1w7lLEiP4XIF_SFy"), 0, 48187833566271418625754761625661652107159264793429628379411792200127405491709); + verifyAddr(address("EQDmwvaK2d_SbaPdpOM60effPWeKsksgDVwFPEyxuftM396K"), 0, 104376425077737068747642645125299653296942252727305637929378053253273342397663); + verifyAddr(address("EQDWtPZZgF7wvIMUHZQojuD3utiuivsW7WslRJ33dgv-5yc8"), 0, 97114682311034709685427168495629428400170984047839002197324103884924936519399); + verifyAddr(address("EQAA7z0JI0JKqbN-1uENKz9JrxIO5ZRY-ehMeg9fPncx50Ck"), 0, 422697701361909095759185681783393186844038628935759044330165207027374567911); + verifyAddr(address("EQBVUHRoCq6coQYUwOAhGSoAmQ6Mpm7dFlDYon6HMgWV8Ftr"), 0, 38588743302295548905191533977469452945717219128199196974980570837505276220912); + verifyAddr(address("EQCTdvDCf0bA5dOPI1-44tB2ZfNcMGiklzvg27TovgDEqM6E"), 0, 66700138358140658950710678965721715920748906761125730971082529064117803730088); + verifyAddr(address("EQBDBKE5WGKIlnoi3OOzw7vkKKIX55eWjPvgxJWwek8AyL2J"), 0, 30313140970524770883308749215942283658935592719811899513010665548955593408712); + verifyAddr(address("EQAvCSyLCo21GrqLAifdov4WkOxuGQCjMRxgF1cXSaNzLHZe"), 0, 21274912932379789207153885262858116665851037273450532982121514600400844714796); + verifyAddr(address("EQCsLpDeHB2qpRbmsCb_0xmsYVNx1NgeYrvHGT1TDrHkDgL4"), 0, 77880084760844670718511036557364450189323549135231036747480176919181282894862); + verifyAddr(address("EQCTQ8kPwyX92r48gCIL_pLN_RcQT9ghZygnmDTYkOkuW_j5"), 0, 66609755171046741472463433430016629628609840960137226492652665879987546041947); + verifyAddr(address("EQCTrFRSHt-tfk7WxK9ZHQmqLcgxXxTK7wGfCEbqgY2W9Mcx"), 0, 66794468397542182731534559853537484892417154018190804733043974345563210356468); + verifyAddr(address("EQCv28y49GdaLncoclv0ISdDlMUY_cxDPGNWFCPT8t4GuqUJ"), 0, 79543100951881731989812212377176715376834409392116644269458867858071577560762); + verifyAddr(address("EQCVL-k6deDR56Z8pcb0Btg0lGfaivOGfdDCD1vvyRsyL9vS"), 0, 67479265933941008511790471646564661743401752930295407567346938670637286896175); + verifyAddr(address("EQD6t2dXDjZxF1DqunKF-8dEWivJdliY_0FYiCXnthuqnDCa"), 0, 113402258385556889021060606279033166272577193563727959698596277924908309916316); + verifyAddr(address("EQDE98XNzXiPq7VnbJJ2M4-Ht3tX_OWR0xUTTnDC8NObLmyU"), 0, 89091094739778473356272490822716056624384395256388481953562551087642791090990); + verifyAddr(address("EQDfeRDE1TDhwt478CDR0Q7MDwqcTUhfjqyTT59mgoAaF6f7"), 0, 101079669463449311486034260688909914923832300293253430457119371423825321269783); + verifyAddr(address("EQDijcEyUKa-QgCbeGlggQk1uBtt2ZRHyW4Y4gB4R6MN6RLW"), 0, 102473162609487797404330889623966425536887610061087715571345738626121871855081); + verifyAddr(address("EQDOtFOt41skbjBkZF89oYXpoDECjlxIzD-ShWAOYyzuxqLA"), 0, 93495056812773926196963707371665512785268729004579280701087533371037976424134); + verifyAddr(address("EQDuJKSFWU7AYqH6KLFfAbYvMuz346eWmJvG6_2NYE42_B4T"), 0, 107715199938628393100813870735031557263256555616273999363057194279168054802172); + verifyAddr(address("EQDwGu4vFv1e3wn8min_iy7OPJXegOYTFQ5bZFZ5a5ZPiBpX"), 0, 108602665568837301744601989570019709742180613578164394799026726718721456754568); + verifyAddr(address("EQC4G2ph6AS_mD_-cIv4aIYm1z5jAgCW_TTDEr72ygXOP2X-"), 0, 83274003234732023403481554420859495155084746906198543572711543697320249249343); + verifyAddr(address("EQDpUkyAa6lZ12P3ZB2PL_rmWwI1I55BU4kxw_rssFL5dswA"), 0, 105534303174146507629736518862713754948570412188900908177600861330298381728118); + verifyAddr(address("EQDoIA20MF1qEcSPtROdCu5ukGx9dVjgMeJh1oQ4A4cf_Jif"), 0, 104993214557977037193613824776415934089204193426692473563548548423424814817276); + verifyAddr(address("EQDpUkyAa6lZ12P3ZB2PL_rmWwI1I55BU4kxw_rssFL5dswA"), 0, 105534303174146507629736518862713754948570412188900908177600861330298381728118); + verifyAddr(address("EQClLO4EnZ_rTyV1GVpWy53pLgWJRki5c4ZzuM_1O_ClBkO9"), 0, 74711004027159342540251007601464500186374346239921204216319145006974068892934); + verifyAddr(address("EQDmkj65Ab_m0aZaW8IpKw4kYqIgITw_HRstYEkVQ6NIYCyW"), 0, 104290347741656803921830951060768893809692975574470790497562993373950614128736); + verifyAddr(address("EQCqNTwAYUNhPFS0RgqZoTLGJcQQxbAJ7csUo4YO3_TONLab"), 0, 76987241268612358571638783428744566580605181728938801022059780105627411729972); + verifyAddr(address("EQCL3DmCynaRK7-vsfeNmd4Jj-UxAIHPvA4qS2xwaL6UpLbF"), 0, 63260589232981964910240894899061676480139492286430589202252472895352724165796); + verifyAddr(address("EQDbU1SVEjBE73oUqgAoM9gDcShUkM5EC2PgoCjuwVUKo-Ee"), 0, 99203745911752606845646497420891218522647962685916739950275357890977532807843); + verifyAddr(address("EQD02VdcF4TDbCKLLhZJ39NQTu6aWq2LnLjp0oXqbNu_BANK"), 0, 110748343802097970709980079967961144373090790244250392237586606542170934198020); + verifyAddr(address("EQBynBO23ywHy_CgarY9NK9FTz0yDsG82PtcbSTQgGoXwiuA"), 0, 51839428943991432793039248316067731096592274748149794482308513726460953499586); + verifyAddr(address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr(address("EQAUTbQiM522Y_XJ_T98QPhPhTmb4nV--VSPiha8kC6kRfPO"), 0, 9183547432069678364603018431103042146626948674383548774683927217595824907333); + verifyAddr(address("EQBlqsm144Dq6SjbPI4jjZvA1hqTIP3CvHovbIfW_t-SCALE"), 0, 45985353862647206060987594732861817093328871106941773337270673759241903247880); + verifyAddr(address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr(address("kQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPgpP"), 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr(address("kf-Dfdg-YQXaR2Q97gZJ4fGBtmV1DHOU1y1RPyyZZtRy_Ikh"), -1, 59475331506450494976393625198911249698879029820580340449086829444312920781564); + verifyAddr(address("0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 0, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr(address("0:0000000000000000000000000000000000000000000000000000000000000000"), 0, 0); + verifyAddr(address("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff"), 0, 115792089237316195423570985008687907853269984665640564039457584007913129639935); + verifyAddr(address("0:0000000000000000000000000000000000000000000000000000000000000000"), 0, 0); + verifyAddr(address("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 1, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr(address("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 9, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr(address("99:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), 99, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr(address("-1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), -1, 37304138005561100291416421295333982606153966175434134130332440738068913455320); - return cc1.bitsEqual(cc2); + return ( + cc1 == cc2, + cc2 != cc1, + (cc1 as slice).bitsEqual(((cc2 as slice) as address) as slice), + createAddressNone() == cc1, + createAddressNone() == createAddressNone(), + check0(address("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff")) + ); } /** -@testcase | 0 | | -1 +@testcase | 0 | | -1 0 -1 0 -1 + +@fif_codegen +""" + check0 PROC:<{ // addr + REWRITESTDADDR + DROP // '2 + 0 EQINT // '4 + 111 THROWIFNOT // + }> +""" + +@fif_codegen +""" + codegenAddrEq PROC:<{ // a b + 2DUP // a b a b + SDEQ // a b '2 + IFJMP:<{ // a b + 2DROP // + 1 PUSHINT // '3=1 + }> // a b + SDEQ // '4 + IFNOTJMP:<{ // + 2 PUSHINT // '5=2 + }> // + 3 PUSHINT // '6=3 + }> +""" */ diff --git a/tolk-tester/tests/strings-tests.tolk b/tolk-tester/tests/strings-tests.tolk index 9feed3a1c..9fb5e3618 100644 --- a/tolk-tester/tests/strings-tests.tolk +++ b/tolk-tester/tests/strings-tests.tolk @@ -6,8 +6,8 @@ get raw_slice(): slice { return stringHexToSlice("abcdef"); } -get addr_slice(): slice { - return stringAddressToSlice("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"); +get addr_slice(): address { + return address("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"); } get string_hex(): int { @@ -51,7 +51,7 @@ fun calcSliceBytes(slice: slice): tuple { fun main() { var s_ascii: slice = ascii_slice(); var s_raw: slice = raw_slice(); - var s_addr: slice = addr_slice(); + var s_addr: address = addr_slice(); var i_hex: int = string_hex(); var i_mini: int = string_minihash(); var i_maxi: int = string_maxihash(); @@ -59,7 +59,7 @@ fun main() { var i_crc16: int = string_crc16(); assert(sdeq(s_ascii, newc().storeUint(0x737472696E67, 12 * 4).endcs())) throw 101; assert(sdeq(s_raw, newc().storeUint(0xABCDEF, 6 * 4).endcs())) throw 102; - assert(sdeq(s_addr, newc().storeUint(4, 3).storeInt(-1, 8) + assert(sdeq(s_addr as slice, newc().storeUint(4, 3).storeInt(-1, 8) .storeUint(0x3333333333333333333333333333333333333333333333333333333333333333, 256).endcs()), 103); assert(i_hex == 0x4142434445464748494A4B4C4D4E4F505152535455565758595A303132333435) throw 104; assert(i_mini == 0x7a62e8a8) throw 105; diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 70944cd4c..7bed41a56 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1263,10 +1263,10 @@ void define_builtins() { define_builtin_func("stringToBase256", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); - define_builtin_func("stringAddressToSlice", {TypeDataUnknown::create()}, TypeDataSlice::create(), nullptr, + define_builtin_func("stringHexToSlice", {TypeDataUnknown::create()}, TypeDataSlice::create(), nullptr, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); - define_builtin_func("stringHexToSlice", {TypeDataUnknown::create()}, TypeDataSlice::create(), nullptr, + define_builtin_func("address", {TypeDataUnknown::create()}, TypeDataAddress::create(), nullptr, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp index a281cfbe3..a68de6c7c 100644 --- a/tolk/constant-evaluator.cpp +++ b/tolk/constant-evaluator.cpp @@ -73,9 +73,9 @@ static bool parse_raw_address(std::string_view acc_string, int& workchain, ton:: int x; if (c >= '0' && c <= '9') { x = c - '0'; - } else if (c >= 'a' && c <= 'z') { + } else if (c >= 'a' && c <= 'f') { x = c - 'a' + 10; - } else if (c >= 'A' && c <= 'Z') { + } else if (c >= 'A' && c <= 'F') { x = c - 'A' + 10; } else { return false; @@ -90,6 +90,23 @@ static bool parse_raw_address(std::string_view acc_string, int& workchain, ton:: return true; } +static void parse_any_std_address(std::string_view str, SrcLocation loc, unsigned char (*data)[3 + 8 + 256]) { + ton::WorkchainId workchain; + ton::StdSmcAddress addr; + bool correct = (str.size() == 48 && parse_friendly_address(str.data(), workchain, addr)) || + (str.size() != 48 && parse_raw_address(str, workchain, addr)); + if (!correct) { + throw ParseError(loc, "invalid standard address"); + } + if (workchain < -128 || workchain >= 128) { + throw ParseError(loc, "anycast addresses not supported"); + } + + td::bitstring::bits_store_long_top(*data, 0, static_cast(4) << (64 - 3), 3); + td::bitstring::bits_store_long_top(*data, 3, static_cast(workchain) << (64 - 8), 8); + td::bitstring::bits_memcpy(*data, 3 + 8, addr.bits().ptr, 0, ton::StdSmcAddress::size()); +} + // internal helper: for `ton("0.05")`, parse string literal "0.05" to 50000000 static td::RefInt256 parse_nanotons_as_floating_string(SrcLocation loc, std::string_view str) { bool is_negative = false; @@ -106,6 +123,7 @@ static td::RefInt256 parse_nanotons_as_floating_string(SrcLocation loc, std::str // parse "0.05" into integer part (before dot) and fractional (after) int64_t integer_part = 0; int64_t fractional_part = 0; + int integer_digits = 0; int fractional_digits = 0; bool seen_dot = false; @@ -119,6 +137,9 @@ static td::RefInt256 parse_nanotons_as_floating_string(SrcLocation loc, std::str } else if (c >= '0' && c <= '9') { if (!seen_dot) { integer_part = integer_part * 10 + (c - '0'); + if (++integer_digits > 9) { + throw ParseError(loc, "argument is too big and leads to overflow"); + } } else if (fractional_digits < 9) { fractional_part = fractional_part * 10 + (c - '0'); fractional_digits++; @@ -161,6 +182,12 @@ static CompileTimeFunctionResult parse_vertex_call_to_compile_time_function(Vloc, str); } + if (f_name == "address") { // previously, postfix "..."a, but it returned `slice` (now returns `address`) + unsigned char data[3 + 8 + 256]; // addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; + parse_any_std_address(str, v_arg->loc, &data); + return td::BitSlice{data, sizeof(data)}.to_hex(); + } + if (f_name == "stringCrc32") { // previously, postfix "..."c return td::make_refint(td::crc32(td::Slice{str.data(), str.size()})); } @@ -181,25 +208,6 @@ static CompileTimeFunctionResult parse_vertex_call_to_compile_time_function(Verror("invalid standard address"); - } - if (workchain < -128 || workchain >= 128) { - v_arg->error("anycast addresses not supported"); - } - - unsigned char data[3 + 8 + 256]; // addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; - td::bitstring::bits_store_long_top(data, 0, static_cast(4) << (64 - 3), 3); - td::bitstring::bits_store_long_top(data, 3, static_cast(workchain) << (64 - 8), 8); - td::bitstring::bits_memcpy(data, 3 + 8, addr.bits().ptr, 0, ton::StdSmcAddress::size()); - return td::BitSlice{data, sizeof(data)}.to_hex(); - } - if (f_name == "stringHexToSlice") { // previously, postfix "..."s unsigned char buff[128]; long bits = td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.data(), str.data() + str.size()); diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 370460d3c..6c376491e 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -889,6 +889,18 @@ static std::vector transition_expr_to_runtime_type_impl(std::vector process_binary_operator(V v, code.close_pop_cur(v->loc); return transition_to_target_type(std::move(rvect), code, target_type, v); } + if (t == tok_eq || t == tok_neq) { + if (v->get_lhs()->inferred_type->unwrap_alias() == TypeDataAddress::create() && v->get_rhs()->inferred_type->unwrap_alias() == TypeDataAddress::create()) { + FunctionPtr f_sliceEq = lookup_global_symbol("slice.bitsEqual")->try_as(); + std::vector ir_lhs_slice = pre_compile_expr(v->get_lhs(), code); + std::vector ir_rhs_slice = pre_compile_expr(v->get_rhs(), code); + std::vector rvect = code.create_tmp_var(TypeDataBool::create(), v->loc, "(addr-eq)"); + code.emplace_back(v->loc, Op::_Call, rvect, std::vector{ir_lhs_slice[0], ir_rhs_slice[0]}, f_sliceEq); + if (t == tok_neq) { + FunctionPtr not_sym = lookup_global_symbol("!b_")->try_as(); + code.emplace_back(v->loc, Op::_Call, rvect, rvect, not_sym); + } + return transition_to_target_type(std::move(rvect), code, target_type, v); + } + } throw UnexpectedASTNodeKind(v, "process_binary_operator"); } diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index a863fc592..dc17fa652 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -155,6 +155,20 @@ static bool expect_boolean(AnyExprV v_inferred) { return expect_boolean(v_inferred->inferred_type); } +static bool expect_address(TypePtr inferred_type) { + if (inferred_type == TypeDataAddress::create()) { + return true; + } + if (const TypeDataAlias* as_alias = inferred_type->try_as()) { + return expect_address(as_alias->underlying_type); + } + return false; +} + +static bool expect_address(AnyExprV v_inferred) { + return expect_address(v_inferred->inferred_type); +} + class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { FunctionPtr cur_f = nullptr; // may be nullptr if checking `const a = ...` init_value @@ -211,7 +225,10 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { bool both_int = expect_integer(lhs) && expect_integer(rhs); bool both_bool = expect_boolean(lhs) && expect_boolean(rhs); if (!both_int && !both_bool) { - if (lhs->inferred_type->equal_to(rhs->inferred_type)) { // compare slice with slice, int? with int? + bool both_address = expect_address(lhs) && expect_address(rhs); + if (both_address) { // address can be compared with ==, but it's not integer comparison, it's handled specially + v->mutate()->assign_fun_ref(nullptr); + } else if (lhs->inferred_type->equal_to(rhs->inferred_type)) { // compare slice with slice, int? with int? fire(cur_f, v->loc, "type " + to_string(lhs) + " can not be compared with `== !=`"); } else { fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); diff --git a/tolk/pipe-optimize-boolean-expr.cpp b/tolk/pipe-optimize-boolean-expr.cpp index 4b05923c2..ef6c8208c 100644 --- a/tolk/pipe-optimize-boolean-expr.cpp +++ b/tolk/pipe-optimize-boolean-expr.cpp @@ -145,6 +145,15 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { v_cond_istype->mutate()->assign_is_negated(!v_cond_istype->is_negated); v = createV(v->loc, !v->is_ifnot, v_cond_istype, v->get_if_body(), v->get_else_body()); } + // `if (addr1 != addr2)` -> ifnot(addr1 == addr2) + if (auto v_cond_neq = v->get_cond()->try_as()) { + if (v_cond_neq->tok == tok_neq && v_cond_neq->get_lhs()->inferred_type->unwrap_alias() == TypeDataAddress::create() && v_cond_neq->get_rhs()->inferred_type->unwrap_alias() == TypeDataAddress::create()) { + auto v_cond_eq = createV(v_cond_neq->loc, "==", tok_eq, v_cond_neq->get_lhs(), v_cond_neq->get_rhs()); + v_cond_eq->mutate()->assign_inferred_type(v_cond_neq->inferred_type); + v_cond_eq->mutate()->assign_rvalue_true(); + v = createV(v->loc, !v->is_ifnot, v_cond_eq, v->get_if_body(), v->get_else_body()); + } + } return v; } diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 915c01f72..78c61d896 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -107,6 +107,7 @@ static TypePtr try_parse_predefined_type(std::string_view str) { break; case 7: if (str == "builder") return TypeDataBuilder::create(); + if (str == "address") return TypeDataAddress::create(); break; case 8: if (str == "varint16") return TypeDataIntN::create(false, true, 16); diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index b419a2e3f..58376d63b 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -135,6 +135,7 @@ TypePtr TypeDataSlice::singleton; TypePtr TypeDataBuilder::singleton; TypePtr TypeDataTuple::singleton; TypePtr TypeDataContinuation::singleton; +TypePtr TypeDataAddress::singleton; TypePtr TypeDataNullLiteral::singleton; TypePtr TypeDataCoins::singleton; TypePtr TypeDataUnknown::singleton; @@ -149,6 +150,7 @@ void type_system_init() { TypeDataBuilder::singleton = new TypeDataBuilder; TypeDataTuple::singleton = new TypeDataTuple; TypeDataContinuation::singleton = new TypeDataContinuation; + TypeDataAddress::singleton = new TypeDataAddress; TypeDataNullLiteral::singleton = new TypeDataNullLiteral; TypeDataCoins::singleton = new TypeDataCoins; TypeDataUnknown::singleton = new TypeDataUnknown; @@ -691,7 +693,7 @@ bool TypeDataSlice::can_rhs_be_assigned(TypePtr rhs) const { if (const TypeDataAlias* rhs_alias = rhs->try_as()) { return can_rhs_be_assigned(rhs_alias->underlying_type); } - return rhs == TypeDataNever::create(); // note, that bytesN is NOT automatically cast to slice without `as` operator + return rhs == TypeDataNever::create(); // note, that bytesN/address is NOT automatically cast to slice without `as` operator } bool TypeDataBuilder::can_rhs_be_assigned(TypePtr rhs) const { @@ -724,6 +726,16 @@ bool TypeDataContinuation::can_rhs_be_assigned(TypePtr rhs) const { return rhs == TypeDataNever::create(); } +bool TypeDataAddress::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + if (const TypeDataAlias* rhs_alias = rhs->try_as()) { + return can_rhs_be_assigned(rhs_alias->underlying_type); + } + return rhs == TypeDataNever::create(); // note, that slice is NOT automatically cast to address without `as` operator +} + bool TypeDataNullLiteral::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; @@ -952,6 +964,9 @@ bool TypeDataSlice::can_be_casted_with_as_operator(TypePtr cast_to) const { if (cast_to->try_as()) { // `slice` to `bytes32` / `slice` to `bits8` return true; } + if (cast_to == TypeDataAddress::create()) { + return true; + } if (const TypeDataUnion* to_union = cast_to->try_as()) { return can_be_casted_to_union(this, to_union); } @@ -991,6 +1006,19 @@ bool TypeDataContinuation::can_be_casted_with_as_operator(TypePtr cast_to) const return cast_to == this; } +bool TypeDataAddress::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (cast_to == TypeDataSlice::create()) { + return true; + } + if (const TypeDataUnion* to_union = cast_to->try_as()) { + return can_be_casted_to_union(this, to_union); + } + if (const TypeDataAlias* to_alias = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_alias->underlying_type); + } + return cast_to == this; +} + bool TypeDataNullLiteral::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataUnion* to_union = cast_to->try_as()) { // `null` to `T?` / `null` to `... | null` return can_be_casted_to_union(this, to_union); @@ -1106,7 +1134,7 @@ bool TypeDataBytesN::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } - return cast_to == TypeDataSlice::create(); + return cast_to == TypeDataSlice::create() || cast_to == TypeDataAddress::create(); } bool TypeDataCoins::can_be_casted_with_as_operator(TypePtr cast_to) const { diff --git a/tolk/type-system.h b/tolk/type-system.h index 40b291015..862a00667 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -255,6 +255,25 @@ class TypeDataContinuation final : public TypeData { bool can_be_casted_with_as_operator(TypePtr cast_to) const override; }; +/* + * `address` is TypeDataAddress — TVM slice under the hood, but since it's a very common use case, + * it's extracted as a separate type (not as a struct with slice field, but just a dedicated type). + */ +class TypeDataAddress final : public TypeData { + TypeDataAddress() : TypeData(0, 1) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + int get_type_id() const override { return 8; } + std::string as_human_readable() const override { return "address"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + /* * `null` has TypeDataNullLiteral type. * It can be assigned only to nullable types (`int?`, etc.), to ensure null safety. From 768bad66daf87a33d0f0a65720e981b70b1879c0 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 22 May 2025 22:26:01 +0300 Subject: [PATCH 264/388] [Tolk] Better handle recursive structs and infinity size This will help to create snake structs like > struct Snake { > data: ...; > next: Cell; > } --- tolk-tester/tests/generics-4.tolk | 23 ++++- .../tests/invalid-declaration/err-1585.tolk | 9 +- tolk/pipe-resolve-types.cpp | 72 ++++++++++---- tolk/symtable.cpp | 8 -- tolk/symtable.h | 16 ---- tolk/type-system.cpp | 93 +++++++++++++------ tolk/type-system.h | 72 +++++++------- 7 files changed, 190 insertions(+), 103 deletions(-) diff --git a/tolk-tester/tests/generics-4.tolk b/tolk-tester/tests/generics-4.tolk index 30ce19e35..cfe5714ff 100644 --- a/tolk-tester/tests/generics-4.tolk +++ b/tolk-tester/tests/generics-4.tolk @@ -146,15 +146,36 @@ fun test7() { return (v1, v2, v3, v4); } +struct FakeGeneric8 { + alwaysInt: int; +} + +struct Snake8 { + next: FakeGeneric8; // it's not a recursive struct, it's okay + next2: FakeGeneric8?; + next3: FakeGeneric8>; +} + +@method_id(108) +fun test8() { + var sn: Snake8 = { + next: { alwaysInt: 10 }, + next2: null, + next3: { alwaysInt: 20 } + }; + return sn; +} + fun main() { } /** @testcase | 103 | | 10 20 30 777 40 40 @testcase | 104 | | (null) (null) (null) -@testcase | 105 | | -1 (null) 0 (null) 133 777 0 123 0 456 132 +@testcase | 105 | | -1 (null) 0 (null) 134 777 0 123 0 456 133 @testcase | 106 | | -1 0 -1 -1 @testcase | 107 | | 1 10 110 117 +@testcase | 108 | | 10 (null) 20 @fif_codegen DECLPROC eqUnusedT @fif_codegen DECLPROC eqUnusedU diff --git a/tolk-tester/tests/invalid-declaration/err-1585.tolk b/tolk-tester/tests/invalid-declaration/err-1585.tolk index 0c1fef332..5abf00017 100644 --- a/tolk-tester/tests/invalid-declaration/err-1585.tolk +++ b/tolk-tester/tests/invalid-declaration/err-1585.tolk @@ -3,8 +3,13 @@ struct A { b: BAlias } type BAlias = B; +fun test(a: A) { + a.b; + a.b.a; +} + /** @compilation_should_fail -@stderr struct `B` size is infinity due to recursive fields -@stderr struct B { +(error message not stable, sometimes about A, sometimes B, it's okay, they are in a hashmap) +@stderr size is infinity due to recursive fields */ diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 78c61d896..5a2aa0a4a 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -47,6 +47,9 @@ namespace tolk { * Example: `type A = Ok`, then `Ok` is not ready yet, it's left as TypeDataGenericTypeWithTs. */ +static std::unordered_map visited_structs; +static std::unordered_map visited_aliases; + GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_unknown_type_name(FunctionPtr cur_f, SrcLocation loc, std::string_view text) { if (text == "auto") { @@ -241,7 +244,7 @@ class TypeNodesVisitorResolver { if (alias_ref->is_generic_alias() && !allow_without_type_arguments) { fire_error_generic_type_used_without_T(cur_f, loc, alias_ref->as_human_readable()); } - if (!alias_ref->was_visited_by_resolver()) { + if (!visited_aliases.contains(alias_ref)) { visit_symbol(alias_ref); } return TypeDataAlias::create(alias_ref); @@ -250,7 +253,7 @@ class TypeNodesVisitorResolver { if (struct_ref->is_generic_struct() && !allow_without_type_arguments) { fire_error_generic_type_used_without_T(cur_f, loc, struct_ref->as_human_readable()); } - if (!struct_ref->was_visited_by_resolver()) { + if (!visited_structs.contains(struct_ref)) { visit_symbol(struct_ref); } return TypeDataStruct::create(struct_ref); @@ -348,7 +351,7 @@ class TypeNodesVisitorResolver { static void visit_symbol(AliasDefPtr alias_ref) { static std::vector called_stack; - // prevent recursion like `type A = B; type B = A` + // prevent recursion like `type A = B; type B = A` (we can't create TypeDataAlias without a resolved underlying type) bool contains = std::find(called_stack.begin(), called_stack.end(), alias_ref) != called_stack.end(); if (contains) { throw ParseError(alias_ref->loc, "type `" + alias_ref->name + "` circularly references itself"); @@ -363,35 +366,24 @@ class TypeNodesVisitorResolver { TypeNodesVisitorResolver visitor(nullptr, alias_ref->genericTs, alias_ref->substitutedTs, false); TypePtr underlying_type = visitor.finalize_type_node(alias_ref->underlying_type_node); alias_ref->mutate()->assign_resolved_type(underlying_type); - alias_ref->mutate()->assign_visited_by_resolver(); + visited_aliases.insert({alias_ref, 1}); called_stack.pop_back(); } static void visit_symbol(StructPtr struct_ref) { - static std::vector called_stack; - - // prevent recursion like `struct A { field: A }` - // currently, a struct is a tensor, and recursion always leads to infinite size (`A?` also, it's also on a stack) - // if there would be an annotation to store a struct in a tuple, then it has to be reconsidered - bool contains = std::find(called_stack.begin(), called_stack.end(), struct_ref) != called_stack.end(); - if (contains) { - throw ParseError(struct_ref->loc, "struct `" + struct_ref->name + "` size is infinity due to recursive fields"); - } + visited_structs.insert({struct_ref, 1}); if (auto v_genericsT_list = struct_ref->ast_root->as()->genericsT_list) { const GenericsDeclaration* genericTs = construct_genericTs(nullptr, v_genericsT_list); struct_ref->mutate()->assign_resolved_genericTs(genericTs); } - called_stack.push_back(struct_ref); TypeNodesVisitorResolver visitor(nullptr, struct_ref->genericTs, struct_ref->substitutedTs, false); for (int i = 0; i < struct_ref->get_num_fields(); ++i) { StructFieldPtr field_ref = struct_ref->get_field(i); TypePtr declared_type = visitor.finalize_type_node(field_ref->type_node); field_ref->mutate()->assign_resolved_type(declared_type); } - struct_ref->mutate()->assign_visited_by_resolver(); - called_stack.pop_back(); } static const GenericsDeclaration* construct_genericTs(TypePtr receiver_type, V v_list) { @@ -579,6 +571,46 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { } }; +// prevent recursion like `struct A { field: A }`; +// currently, a struct is a tensor, and recursion always leads to infinite size (`A?` also, it's also on a stack); +// if there is an annotation to store a struct in a tuple, then it has to be reconsidered; +// it's crucial to detect it here; otherwise, get_width_on_stack() will silently face stack overflow +class InfiniteStructSizeDetector { + static TypePtr visit_type_deeply(TypePtr type) { + return type->replace_children_custom([](TypePtr child) { + if (const TypeDataStruct* child_struct = child->try_as()) { + check_struct_for_infinite_size(child_struct->struct_ref); + } + if (const TypeDataAlias* child_alias = child->try_as()) { + return visit_type_deeply(child_alias->underlying_type); + } + return child; + }); + }; + + static void check_struct_for_infinite_size(StructPtr struct_ref) { + static std::vector called_stack; + + bool contains = std::find(called_stack.begin(), called_stack.end(), struct_ref) != called_stack.end(); + if (contains) { + throw ParseError(struct_ref->loc, "struct `" + struct_ref->name + "` size is infinity due to recursive fields"); + } + + called_stack.push_back(struct_ref); + for (StructFieldPtr field_ref : struct_ref->fields) { + visit_type_deeply(field_ref->declared_type); + } + called_stack.pop_back(); + } + +public: + static void detect_and_fire_if_any_struct_is_infinite() { + for (auto [struct_ref, _] : visited_structs) { + check_struct_for_infinite_size(struct_ref); + } + } +}; + void pipeline_resolve_types_and_aliases() { ResolveTypesInsideFunctionVisitor visitor; @@ -600,12 +632,12 @@ void pipeline_resolve_types_and_aliases() { visitor.start_visiting_constant(v_const->const_ref); } else if (auto v_alias = v->try_as()) { - if (!v_alias->alias_ref->was_visited_by_resolver()) { + if (!visited_aliases.contains(v_alias->alias_ref)) { TypeNodesVisitorResolver::visit_symbol(v_alias->alias_ref); } } else if (auto v_struct = v->try_as()) { - if (!v_struct->struct_ref->was_visited_by_resolver()) { + if (!visited_structs.contains(v_struct->struct_ref)) { TypeNodesVisitorResolver::visit_symbol(v_struct->struct_ref); } visitor.start_visiting_struct_fields(v_struct->struct_ref); @@ -613,6 +645,10 @@ void pipeline_resolve_types_and_aliases() { } } + InfiniteStructSizeDetector::detect_and_fire_if_any_struct_is_infinite(); + visited_structs.clear(); + visited_aliases.clear(); + patch_builtins_after_stdlib_loaded(); } diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 1485eae75..9e5d4ad69 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -141,10 +141,6 @@ void LocalVarData::assign_inferred_type(TypePtr inferred_type) { this->declared_type = inferred_type; } -void AliasDefData::assign_visited_by_resolver() { - this->flags |= flagVisitedByResolver; -} - void AliasDefData::assign_resolved_genericTs(const GenericsDeclaration* genericTs) { if (this->substitutedTs == nullptr) { this->genericTs = genericTs; @@ -163,10 +159,6 @@ void StructFieldData::assign_default_value(AnyExprV default_value) { this->default_value = default_value; } -void StructData::assign_visited_by_resolver() { - this->flags |= flagVisitedByResolver; -} - void StructData::assign_resolved_genericTs(const GenericsDeclaration* genericTs) { if (this->substitutedTs == nullptr) { this->genericTs = genericTs; diff --git a/tolk/symtable.h b/tolk/symtable.h index e6b2f1ce6..3b8075b64 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -252,13 +252,8 @@ struct GlobalConstData final : Symbol { }; struct AliasDefData final : Symbol { - enum { - flagVisitedByResolver = 1, - }; - AnyTypeV underlying_type_node; TypePtr underlying_type = nullptr; // = resolved underlying_type_node - int flags = 0; const GenericsDeclaration* genericTs; const GenericsSubstitutions* substitutedTs; @@ -278,10 +273,7 @@ struct AliasDefData final : Symbol { bool is_generic_alias() const { return genericTs != nullptr; } bool is_instantiation_of_generic_alias() const { return substitutedTs != nullptr; } - bool was_visited_by_resolver() const { return flags & flagVisitedByResolver; } - AliasDefData* mutate() const { return const_cast(this); } - void assign_visited_by_resolver(); void assign_resolved_genericTs(const GenericsDeclaration* genericTs); void assign_resolved_type(TypePtr underlying_type); }; @@ -307,12 +299,7 @@ struct StructFieldData final : Symbol { }; struct StructData final : Symbol { - enum { - flagVisitedByResolver = 1, - }; - std::vector fields; - int flags = 0; const GenericsDeclaration* genericTs; const GenericsSubstitutions* substitutedTs; @@ -326,10 +313,7 @@ struct StructData final : Symbol { bool is_generic_struct() const { return genericTs != nullptr; } bool is_instantiation_of_generic_struct() const { return substitutedTs != nullptr; } - bool was_visited_by_resolver() const { return flags & flagVisitedByResolver; } - StructData* mutate() const { return const_cast(this); } - void assign_visited_by_resolver(); void assign_resolved_genericTs(const GenericsDeclaration* genericTs); StructData(std::string name, SrcLocation loc, std::vector&& fields, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, AnyV ast_root) diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index 58376d63b..cd249a5c3 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -266,11 +266,7 @@ TypePtr TypeDataStruct::create(StructPtr struct_ref) { if (TypePtr existing = hash.get_existing()) { return existing; } - int width_on_stack = 0; // if a struct is generic, it will be incorrect, it's okay - for (StructFieldPtr field_ref : struct_ref->fields) { - width_on_stack += field_ref->declared_type->get_width_on_stack(); - } - return hash.register_unique(new TypeDataStruct(width_on_stack, struct_ref)); + return hash.register_unique(new TypeDataStruct(struct_ref)); } TypePtr TypeDataTensor::create(std::vector&& items) { @@ -283,11 +279,7 @@ TypePtr TypeDataTensor::create(std::vector&& items) { if (TypePtr existing = hash.get_existing()) { return existing; } - int width_on_stack = 0; - for (TypePtr item : items) { - width_on_stack += item->get_width_on_stack(); - } - return hash.register_unique(new TypeDataTensor(hash.children_flags(), width_on_stack, std::move(items))); + return hash.register_unique(new TypeDataTensor(hash.children_flags(), std::move(items))); } TypePtr TypeDataBrackets::create(std::vector&& items) { @@ -351,7 +343,7 @@ TypePtr TypeDataUnion::create(std::vector&& variants) { or_null = variants[variants[0] == TypeDataNullLiteral::create()]; } } - return hash.register_unique(new TypeDataUnion(hash.children_flags(), -999999, or_null, std::move(variants))); + return hash.register_unique(new TypeDataUnion(hash.children_flags(), or_null, std::move(variants))); } // flatten variants and remove duplicates @@ -375,24 +367,10 @@ TypePtr TypeDataUnion::create(std::vector&& variants) { } } - int width_on_stack; - if (or_null && or_null->can_hold_tvm_null_instead()) { - width_on_stack = 1; - } else { - // `T1 | T2 | ...` occupy max(W[i]) + 1 slot for UTag (stores type_id or 0 for null) - int max_child_width = 0; - for (TypePtr i : flat_variants) { - if (i != TypeDataNullLiteral::create()) { // `Empty | () | null` totally should be 1 (0 + 1 for UTag) - max_child_width = std::max(max_child_width, i->get_width_on_stack()); - } - } - width_on_stack = max_child_width + 1; - } - if (flat_variants.size() == 1) { // `int | int` return flat_variants[0]; } - return hash.register_unique(new TypeDataUnion(hash.children_flags(), width_on_stack, or_null, std::move(flat_variants))); + return hash.register_unique(new TypeDataUnion(hash.children_flags(), or_null, std::move(flat_variants))); } TypePtr TypeDataUnion::create_nullable(TypePtr nullable) { @@ -410,6 +388,69 @@ TypePtr TypeDataUnion::create_nullable(TypePtr nullable) { } +// -------------------------------------------- +// get_width_on_stack() +// +// calculate, how many stack slots the type occupies, e.g. `int`=1, `(int,int)`=2, `(int,int)?`=3 +// it's calculated dynamically (not saved at TypeData*::create) to overcome problems with +// - recursive struct mentions (to create TypeDataStruct without knowing width of children) +// - uninitialized generics (that don't make any sense upon being instantiated) +// + +int TypeDataAlias::get_width_on_stack() const { + return underlying_type->get_width_on_stack(); +} + +int TypeDataGenericT::get_width_on_stack() const { + assert(false); + return -99999; +} + +int TypeDataGenericTypeWithTs::get_width_on_stack() const { + assert(false); + return -99999; +} + +int TypeDataStruct::get_width_on_stack() const { + int width_on_stack = 0; + for (StructFieldPtr field_ref : struct_ref->fields) { + width_on_stack += field_ref->declared_type->get_width_on_stack(); + } + return width_on_stack; +} + +int TypeDataTensor::get_width_on_stack() const { + int width_on_stack = 0; + for (TypePtr item : items) { + width_on_stack += item->get_width_on_stack(); + } + return width_on_stack; +} + +int TypeDataUnion::get_width_on_stack() const { + if (or_null && or_null->can_hold_tvm_null_instead()) { + return 1; + } + + // `T1 | T2 | ...` occupy max(W[i]) + 1 slot for UTag (stores type_id or 0 for null) + int max_child_width = 0; + for (TypePtr i : variants) { + if (i != TypeDataNullLiteral::create()) { // `Empty | () | null` totally should be 1 (0 + 1 for UTag) + max_child_width = std::max(max_child_width, i->get_width_on_stack()); + } + } + return max_child_width + 1; +} + +int TypeDataNever::get_width_on_stack() const { + return 0; +} + +int TypeDataVoid::get_width_on_stack() const { + return 0; +} + + // -------------------------------------------- // get_type_id() // diff --git a/tolk/type-system.h b/tolk/type-system.h index 862a00667..c1f9cbb6d 100644 --- a/tolk/type-system.h +++ b/tolk/type-system.h @@ -41,8 +41,6 @@ namespace tolk { class TypeData { // bits of flag_mask, to store often-used properties and return them without tree traversing const int flags; - // how many slots on a stack this type occupies (calculated on creation), e.g. `int`=1, `(int,int)`=2, `(int,int)?`=3 - const int width_on_stack; friend class TypeDataHasherForUnique; @@ -53,9 +51,8 @@ class TypeData { flag_contains_type_alias_inside = 1 << 3, }; - explicit TypeData(int flags_with_children, int width_on_stack) - : flags(flags_with_children) - , width_on_stack(width_on_stack) { + explicit TypeData(int flags_with_children) + : flags(flags_with_children) { } static bool equal_to_slow_path(TypePtr lhs, TypePtr rhs); @@ -69,7 +66,10 @@ class TypeData { return dynamic_cast(this); } - int get_width_on_stack() const { return width_on_stack; } + // how many slots on a stack this type occupies, e.g. `int`=1, `(int,int)`=2, `(int,int)?`=3 + virtual int get_width_on_stack() const { + return 1; // most types occupy 1 stack slot (int, cell, slice, etc.) + } bool equal_to(TypePtr rhs) const { return this == rhs || equal_to_slow_path(this, rhs); @@ -108,7 +108,7 @@ class TypeData { */ class TypeDataAlias final : public TypeData { explicit TypeDataAlias(int children_flags, AliasDefPtr alias_ref, TypePtr underlying_type) - : TypeData(children_flags | flag_contains_type_alias_inside, underlying_type->get_width_on_stack()) + : TypeData(children_flags | flag_contains_type_alias_inside) , alias_ref(alias_ref) , underlying_type(underlying_type) {} @@ -118,6 +118,7 @@ class TypeDataAlias final : public TypeData { static TypePtr create(AliasDefPtr alias_ref); + int get_width_on_stack() const override; int get_type_id() const override; std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; @@ -129,7 +130,7 @@ class TypeDataAlias final : public TypeData { * `int` is TypeDataInt, representation of TVM int. */ class TypeDataInt final : public TypeData { - TypeDataInt() : TypeData(0, 1) {} + TypeDataInt() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -148,7 +149,7 @@ class TypeDataInt final : public TypeData { * From the type system point of view, int and bool are different, not-autocastable types. */ class TypeDataBool final : public TypeData { - TypeDataBool() : TypeData(0, 1) {} + TypeDataBool() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -166,7 +167,7 @@ class TypeDataBool final : public TypeData { * `cell` is TypeDataCell, representation of TVM cell. */ class TypeDataCell final : public TypeData { - TypeDataCell() : TypeData(0, 1) {} + TypeDataCell() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -184,7 +185,7 @@ class TypeDataCell final : public TypeData { * `slice` is TypeDataSlice, representation of TVM slice. */ class TypeDataSlice final : public TypeData { - TypeDataSlice() : TypeData(0, 1) {} + TypeDataSlice() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -202,7 +203,7 @@ class TypeDataSlice final : public TypeData { * `builder` is TypeDataBuilder, representation of TVM builder. */ class TypeDataBuilder final : public TypeData { - TypeDataBuilder() : TypeData(0, 1) {} + TypeDataBuilder() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -222,7 +223,7 @@ class TypeDataBuilder final : public TypeData { * so getting its element results in TypeDataUnknown (which must be assigned/cast explicitly). */ class TypeDataTuple final : public TypeData { - TypeDataTuple() : TypeData(0, 1) {} + TypeDataTuple() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -241,7 +242,7 @@ class TypeDataTuple final : public TypeData { * It's like "untyped callable", not compatible with other types. */ class TypeDataContinuation final : public TypeData { - TypeDataContinuation() : TypeData(0, 1) {} + TypeDataContinuation() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -260,7 +261,7 @@ class TypeDataContinuation final : public TypeData { * it's extracted as a separate type (not as a struct with slice field, but just a dedicated type). */ class TypeDataAddress final : public TypeData { - TypeDataAddress() : TypeData(0, 1) {} + TypeDataAddress() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -281,7 +282,7 @@ class TypeDataAddress final : public TypeData { * (it's much better for user to see an error here than when he passes this variable somewhere). */ class TypeDataNullLiteral final : public TypeData { - TypeDataNullLiteral() : TypeData(0, 1) {} + TypeDataNullLiteral() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -302,7 +303,7 @@ class TypeDataNullLiteral final : public TypeData { */ class TypeDataFunCallable final : public TypeData { TypeDataFunCallable(int children_flags, std::vector&& params_types, TypePtr return_type) - : TypeData(children_flags, 1) + : TypeData(children_flags) , params_types(std::move(params_types)) , return_type(return_type) {} @@ -329,7 +330,7 @@ class TypeDataFunCallable final : public TypeData { */ class TypeDataGenericT final : public TypeData { explicit TypeDataGenericT(std::string&& nameT) - : TypeData(flag_contains_genericT_inside, -999999) // width undefined until instantiated + : TypeData(flag_contains_genericT_inside) , nameT(std::move(nameT)) {} public: @@ -337,6 +338,7 @@ class TypeDataGenericT final : public TypeData { static TypePtr create(std::string&& nameT); + int get_width_on_stack() const override; int get_type_id() const override; std::string as_human_readable() const override { return nameT; } bool can_rhs_be_assigned(TypePtr rhs) const override; @@ -351,7 +353,7 @@ class TypeDataGenericT final : public TypeData { */ class TypeDataGenericTypeWithTs final : public TypeData { TypeDataGenericTypeWithTs(int children_flags, StructPtr struct_ref, AliasDefPtr alias_ref, std::vector&& type_arguments) - : TypeData(children_flags, -999999) + : TypeData(children_flags) , struct_ref(struct_ref) , alias_ref(alias_ref) , type_arguments(std::move(type_arguments)) {} @@ -365,6 +367,7 @@ class TypeDataGenericTypeWithTs final : public TypeData { int size() const { return static_cast(type_arguments.size()); } + int get_width_on_stack() const override; int get_type_id() const override; std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; @@ -380,8 +383,8 @@ class TypeDataGenericTypeWithTs final : public TypeData { * so for `Wrapper` struct_ref points to a generic struct, and for `Wrapper` to an instantiated one. */ class TypeDataStruct final : public TypeData { - TypeDataStruct(int width_on_stack, StructPtr struct_ref) - : TypeData(0, width_on_stack) + explicit TypeDataStruct(StructPtr struct_ref) + : TypeData(0) , struct_ref(struct_ref) {} public: @@ -389,6 +392,7 @@ class TypeDataStruct final : public TypeData { static TypePtr create(StructPtr struct_ref); + int get_width_on_stack() const override; int get_type_id() const override; std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; @@ -403,8 +407,8 @@ class TypeDataStruct final : public TypeData { * A tensor can be empty. */ class TypeDataTensor final : public TypeData { - TypeDataTensor(int children_flags, int width_on_stack, std::vector&& items) - : TypeData(children_flags, width_on_stack) + TypeDataTensor(int children_flags, std::vector&& items) + : TypeData(children_flags) , items(std::move(items)) {} public: @@ -414,6 +418,7 @@ class TypeDataTensor final : public TypeData { int size() const { return static_cast(items.size()); } + int get_width_on_stack() const override; int get_type_id() const override; std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; @@ -429,7 +434,7 @@ class TypeDataTensor final : public TypeData { */ class TypeDataBrackets final : public TypeData { TypeDataBrackets(int children_flags, std::vector&& items) - : TypeData(children_flags, 1) + : TypeData(children_flags) , items(std::move(items)) {} public: @@ -454,7 +459,7 @@ class TypeDataBrackets final : public TypeData { */ class TypeDataIntN final : public TypeData { TypeDataIntN(bool is_unsigned, bool is_variadic, int n_bits) - : TypeData(0, 1) + : TypeData(0) , is_unsigned(is_unsigned) , is_variadic(is_variadic) , n_bits(n_bits) {} @@ -477,7 +482,7 @@ class TypeDataIntN final : public TypeData { * Example: `var cost = ton("0.05")` has type `coins`. */ class TypeDataCoins final : public TypeData { - TypeDataCoins() : TypeData(0, 1) {} + TypeDataCoins() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -499,7 +504,7 @@ class TypeDataCoins final : public TypeData { */ class TypeDataBytesN final : public TypeData { TypeDataBytesN(bool is_bits, int n_width) - : TypeData(0, 1) + : TypeData(0) , is_bits(is_bits) , n_width(n_width) {} @@ -525,8 +530,8 @@ class TypeDataBytesN final : public TypeData { * - `T1 | T2 | ...` is a tagged union: occupy max(T_i)+1 slots (1 for type_id) */ class TypeDataUnion final : public TypeData { - TypeDataUnion(int children_flags, int width_on_stack, TypePtr or_null, std::vector&& variants) - : TypeData(children_flags, width_on_stack) + TypeDataUnion(int children_flags, TypePtr or_null, std::vector&& variants) + : TypeData(children_flags) , or_null(or_null) , variants(std::move(variants)) {} @@ -565,6 +570,7 @@ class TypeDataUnion final : public TypeData { TypePtr calculate_exact_variant_to_fit_rhs(TypePtr rhs_type) const; bool has_all_variants_of(const TypeDataUnion* rhs_type) const; + int get_width_on_stack() const override; int get_type_id() const override; std::string as_human_readable() const override; bool can_rhs_be_assigned(TypePtr rhs) const override; @@ -580,7 +586,7 @@ class TypeDataUnion final : public TypeData { * The only thing available to do with unknown is to cast it: `catch (excNo, arg) { var i = arg as int; }` */ class TypeDataUnknown final : public TypeData { - TypeDataUnknown() : TypeData(flag_contains_unknown_inside, 1) {} + TypeDataUnknown() : TypeData(flag_contains_unknown_inside) {} static TypePtr singleton; friend void type_system_init(); @@ -601,7 +607,7 @@ class TypeDataUnknown final : public TypeData { * Such variables can not be cast to any other types, all their usage will trigger type mismatch errors. */ class TypeDataNever final : public TypeData { - TypeDataNever() : TypeData(0, 0) {} + TypeDataNever() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -609,6 +615,7 @@ class TypeDataNever final : public TypeData { public: static TypePtr create() { return singleton; } + int get_width_on_stack() const override; int get_type_id() const override { return 19; } std::string as_human_readable() const override { return "never"; } bool can_rhs_be_assigned(TypePtr rhs) const override; @@ -622,7 +629,7 @@ class TypeDataNever final : public TypeData { * Empty tensor is not compatible with void, although at IR level they are similar, 0 stack slots. */ class TypeDataVoid final : public TypeData { - TypeDataVoid() : TypeData(0, 0) {} + TypeDataVoid() : TypeData(0) {} static TypePtr singleton; friend void type_system_init(); @@ -630,6 +637,7 @@ class TypeDataVoid final : public TypeData { public: static TypePtr create() { return singleton; } + int get_width_on_stack() const override; int get_type_id() const override { return 10; } std::string as_human_readable() const override { return "void"; } bool can_rhs_be_assigned(TypePtr rhs) const override; From e4bc23501162e077d11361db318e6ae111e6ed48 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 22 May 2025 23:06:14 +0300 Subject: [PATCH 265/388] [Tolk] Better error messages for unknown method When `a.method()` is unknown, show a better error: - maybe, it's a global function - maybe, it exists for another type --- .../tests/invalid-semantics/err-4520.tolk | 2 +- .../tests/invalid-symbol/err-2188.tolk | 2 +- .../tests/invalid-symbol/err-2218.tolk | 13 ++++++++++ .../tests/invalid-symbol/err-2770.tolk | 14 +++++++++++ tolk-tester/tests/type-aliases-tests.tolk | 25 +++++++++++++++++++ tolk/generics-helpers.cpp | 6 ++++- tolk/pipe-infer-types-and-calls.cpp | 14 ++++++++++- tolk/symtable.cpp | 10 ++++++++ tolk/symtable.h | 1 + 9 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 tolk-tester/tests/invalid-symbol/err-2218.tolk create mode 100644 tolk-tester/tests/invalid-symbol/err-2770.tolk diff --git a/tolk-tester/tests/invalid-semantics/err-4520.tolk b/tolk-tester/tests/invalid-semantics/err-4520.tolk index 994440163..47b1d2d55 100644 --- a/tolk-tester/tests/invalid-semantics/err-4520.tolk +++ b/tolk-tester/tests/invalid-semantics/err-4520.tolk @@ -6,5 +6,5 @@ fun main() { /** @compilation_should_fail -@stderr method `3` not found for type `int` +@stderr method `3` not found */ diff --git a/tolk-tester/tests/invalid-symbol/err-2188.tolk b/tolk-tester/tests/invalid-symbol/err-2188.tolk index 6511fd40d..4d0c66d3d 100644 --- a/tolk-tester/tests/invalid-symbol/err-2188.tolk +++ b/tolk-tester/tests/invalid-symbol/err-2188.tolk @@ -9,6 +9,6 @@ fun main() { /** @compilation_should_fail -@stderr method `storeUnexisting` not found for type `builder` +@stderr method `storeUnexisting` not found @stderr .storeUnexisting() */ diff --git a/tolk-tester/tests/invalid-symbol/err-2218.tolk b/tolk-tester/tests/invalid-symbol/err-2218.tolk new file mode 100644 index 000000000..a9a1e7887 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2218.tolk @@ -0,0 +1,13 @@ +fun demo(x: int) { + +} + +fun main() { + 10.demo(); +} + +/** +@compilation_should_fail +@stderr method `demo` not found, but there is a global function named `demo` +@stderr (a function should be called `foo(arg)`, not `arg.foo()`) + */ diff --git a/tolk-tester/tests/invalid-symbol/err-2770.tolk b/tolk-tester/tests/invalid-symbol/err-2770.tolk new file mode 100644 index 000000000..06935c93d --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2770.tolk @@ -0,0 +1,14 @@ + +fun int.demo(self) { + +} + +fun main() { + (10 as int?).demo(); +} + +/** +@compilation_should_fail +@stderr method `demo` not found for type `int?` +@stderr (but it exists for type `int`) + */ diff --git a/tolk-tester/tests/type-aliases-tests.tolk b/tolk-tester/tests/type-aliases-tests.tolk index ed0cca11d..5d725ecf5 100644 --- a/tolk-tester/tests/type-aliases-tests.tolk +++ b/tolk-tester/tests/type-aliases-tests.tolk @@ -1,3 +1,5 @@ +import "@stdlib/tvm-dicts.tolk" + type MIntN = MInt?; type MInt = int; type MInt_v2 = int; @@ -175,6 +177,27 @@ fun test11() { __expect_type(analyzeTensor2(x3), "(MInt, MInt)?"); } +fun dict.getFakeDepth(self) { return 1; } +fun cell.getFakeDepth(self) { return 2; } + +@method_id(112) +fun test12(makeNotNull: bool) { + var d = createEmptyDict(); + if (makeNotNull) { + d.iDictSet(32, 123, ""); + } + var t = createEmptyTuple(); + if (d != null) { + __expect_type(d.getFakeDepth, "(cell) -> int"); + t.push(d.getFakeDepth()); + } else { + __expect_type(d.getFakeDepth, "(dict) -> int"); + t.push(d.getFakeDepth()); + } + t.push(d.getFakeDepth()); + return t; +} + fun main(x: MInt, y: MInt?) { return y == null ? x : x + y; @@ -184,6 +207,8 @@ fun main(x: MInt, y: MInt?) { @testcase | 0 | 3 4 | 7 @testcase | 104 | 1 2 | 3 @testcase | 110 | | 41 +@testcase | 112 | 0 | [ 1 1 ] +@testcase | 112 | -1 | [ 2 1 ] @fif_codegen """ diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index 5c0cc8867..34d2f8f33 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -443,7 +443,11 @@ AliasDefPtr instantiate_generic_alias(AliasDefPtr alias_ref, GenericsSubstitutio } // find `builder.storeInt` for called_receiver = "builder" and called_name = "storeInt" -// most practical case, when a direct method for receiver exists +// most practical case, when a direct method for receiver exists; +// note, that having an alias `type WorkchainNum = int` and methods `WorkchainNum.isMasterchain()`, +// it's okay to call `-1.isMasterchain()`, because int equals to any alias; +// currently there is no chance to change this logic, say, `type AssetList = dict` to have separate methods, +// due to smart casts, types merge of control flow rejoin, etc., which immediately become `cell?` FunctionPtr match_exact_method_for_call_not_generic(TypePtr called_receiver, std::string_view called_name) { FunctionPtr exact_found = nullptr; diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 71149cd4b..de929d70f 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -156,6 +156,18 @@ static void fire_error_using_lateinit_variable_uninitialized(FunctionPtr cur_f, fire(cur_f, loc, "using variable `" + static_cast(name) + "` before it's definitely assigned"); } +// fire an error when `obj.f()`, method `f` not found, try to locate a method for another type +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_method_not_found(FunctionPtr cur_f, SrcLocation loc, TypePtr receiver_type, std::string_view method_name) { + if (std::vector other = lookup_methods_with_name(method_name); !other.empty()) { + fire(cur_f, loc, "method `" + to_string(method_name) + "` not found for type " + to_string(receiver_type) + "\n(but it exists for type " + to_string(other.front()->receiver_type) + ")"); + } + if (const Symbol* sym = lookup_global_symbol(method_name); sym && sym->try_as()) { + fire(cur_f, loc, "method `" + to_string(method_name) + "` not found, but there is a global function named `" + to_string(method_name) + "`\n(a function should be called `foo(arg)`, not `arg.foo()`)"); + } + fire(cur_f, loc, "method `" + to_string(method_name) + "` not found"); +} + // helper function: given hint = `Ok | Err` and struct `Ok`, return `Ok` // example: `match (...) { Ok => ... }` we need to deduce `Ok` based on subject static TypePtr try_pick_instantiated_generic_from_hint(TypePtr hint, StructPtr lookup_ref) { @@ -1018,7 +1030,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } } if (out_f_called) { - fire(cur_f, v_ident->loc, "method `" + to_string(v->get_field_name()) + "` not found for type " + to_string(obj_type)); + fire_error_method_not_found(cur_f, v_ident->loc, dot_obj->inferred_type, v->get_field_name()); } else { fire(cur_f, v_ident->loc, "field `" + static_cast(field_name) + "` doesn't exist in type " + to_string(dot_obj)); } diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 9e5d4ad69..7efccc75e 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -234,4 +234,14 @@ FunctionPtr lookup_function(std::string_view name) { return G.symtable.lookup(name)->try_as(); } +std::vector lookup_methods_with_name(std::string_view name) { + std::vector result; + for (FunctionPtr method_ref : G.all_methods) { + if (method_ref->method_name == name) { + result.push_back(method_ref); + } + } + return result; +} + } // namespace tolk diff --git a/tolk/symtable.h b/tolk/symtable.h index 3b8075b64..ccc8bc527 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -358,5 +358,6 @@ class GlobalSymbolTable { const Symbol* lookup_global_symbol(std::string_view name); FunctionPtr lookup_function(std::string_view name); +std::vector lookup_methods_with_name(std::string_view name); } // namespace tolk From 49729a1de9089549c30aa21c4cfee8602ef514d7 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 29 May 2025 18:34:46 +0300 Subject: [PATCH 266/388] [Tolk] Fix potential overflow on parsing `obj.index_at` --- tolk-tester/tests/invalid-semantics/err-4208.tolk | 6 ++++++ tolk/pipe-infer-types-and-calls.cpp | 12 +++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 tolk-tester/tests/invalid-semantics/err-4208.tolk diff --git a/tolk-tester/tests/invalid-semantics/err-4208.tolk b/tolk-tester/tests/invalid-semantics/err-4208.tolk new file mode 100644 index 000000000..c40dda98c --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4208.tolk @@ -0,0 +1,6 @@ +const flyp = 9999999.77777777777777; + +/** +@compilation_should_fail +@stderr invalid numeric index + */ diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index de929d70f..8e56e62b6 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -21,6 +21,7 @@ #include "generics-helpers.h" #include "type-system.h" #include "smart-casts-cfg.h" +#include /* * This is a complicated and crucial part of the pipeline. It simultaneously does the following: @@ -168,6 +169,12 @@ static void fire_error_method_not_found(FunctionPtr cur_f, SrcLocation loc, Type fire(cur_f, loc, "method `" + to_string(method_name) + "` not found"); } +// safe version of std::stoi that does not crash on long numbers +static bool try_parse_string_to_int(std::string_view str, int& out) { + auto result = std::from_chars(str.data(), str.data() + str.size(), out); + return result.ec == std::errc() && result.ptr == str.data() + str.size(); +} + // helper function: given hint = `Ok | Err` and struct `Ok`, return `Ok` // example: `match (...) { Ok => ... }` we need to deduce `Ok` based on subject static TypePtr try_pick_instantiated_generic_from_hint(TypePtr hint, StructPtr lookup_ref) { @@ -970,7 +977,10 @@ class InferTypesAndCallsAndFieldsVisitor final { // check for indexed access (`tensorVar.0` / `tupleVar.1`) if (!fun_ref && field_name[0] >= '0' && field_name[0] <= '9') { - int index_at = std::stoi(std::string(field_name)); + int index_at; + if (!try_parse_string_to_int(field_name, index_at)) { + fire(cur_f, v_ident->loc, "invalid numeric index"); + } if (const auto* t_tensor = obj_type->try_as()) { if (index_at >= t_tensor->size()) { fire(cur_f, v_ident->loc, "invalid tensor index, expected 0.." + std::to_string(t_tensor->items.size() - 1)); From 711ca2f1a47728a5b089f6b5d479fd6e39eb3f74 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sat, 24 May 2025 15:59:53 +0300 Subject: [PATCH 267/388] [Tolk] Correctly handle object literals with shuffled fields > var p: Point = { > y: getY(), > x: getX() > } Now, `getY` will be evaluated first, even though `x` is declared first in a struct. --- tolk-tester/tests/asm-arg-order.tolk | 21 +++++- tolk-tester/tests/cells-slices.tolk | 16 +++-- tolk-tester/tests/strings-tests.tolk | 2 +- tolk-tester/tests/struct-tests.tolk | 104 +++++++++++++++++++++++++++ tolk/pipe-ast-to-legacy.cpp | 83 ++++++++++++++++++++- 5 files changed, 214 insertions(+), 12 deletions(-) diff --git a/tolk-tester/tests/asm-arg-order.tolk b/tolk-tester/tests/asm-arg-order.tolk index 6d55b2da3..cb7b6f1e3 100644 --- a/tolk-tester/tests/asm-arg-order.tolk +++ b/tolk-tester/tests/asm-arg-order.tolk @@ -180,6 +180,12 @@ fun test33(x: int) { return ((x += 10).plus1TimesB(2), (x += 20).plus1TimesB(x), ((x /= (g2=2)).plus1TimesB(x*g2)), setG2(7).plus1TimesB(g2)); } +@method_id(34) +fun test34() { + var cs = stringHexToSlice("020a"); + return asmAPlus1TimesB(cs.loadUint(8), cs.loadUint(8)); +} + fun main() { } @@ -205,6 +211,7 @@ fun main() { @testcase | 31 | | 70 @testcase | 32 | | 30 @testcase | 33 | 0 | 22 930 480 56 +@testcase | 34 | | 30 @fif_codegen """ @@ -253,5 +260,17 @@ fun main() { }> """ -@code_hash 78671986831403867804966279036762472603849672357801214378328975900111280733054 +@fif_codegen +""" + test34 PROC:<{ + x{020a} PUSHSLICE + 8 LDU + 8 LDU + DROP + SWAP + 1 ADDCONST MUL + }> +""" + +@code_hash 93297578247504549901423325446957614083213743835412767087757830691482809332788 */ diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index e5b89424a..253ebfff6 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -254,13 +254,15 @@ Note, that since 'compute-asm-ltr' became on be default, chaining methods codege @fif_codegen """ test6 PROC:<{ - 3 PUSHINT // '0=3 - 2 PUSHINT // '0=3 '1=2 - 1 PUSHINT // '0=3 '1=2 '2=1 - NEWC // '0=3 '1=2 '2=1 '3 - 32 STU // '0=3 '1=2 '3 - 32 STU // '0=3 '3 - 32 STU // '3 + 1 PUSHINT // '0=1 + NEWC // '0=1 '1 + 32 STU // '1 + 2 PUSHINT // '1 '4=2 + SWAP // '4=2 '1 + 32 STU // '1 + 3 PUSHINT // '1 '7=3 + SWAP // '7=3 '1 + 32 STU // '1 }> """ */ diff --git a/tolk-tester/tests/strings-tests.tolk b/tolk-tester/tests/strings-tests.tolk index 9fb5e3618..287dc0df3 100644 --- a/tolk-tester/tests/strings-tests.tolk +++ b/tolk-tester/tests/strings-tests.tolk @@ -78,5 +78,5 @@ fun test1() { @testcase | 0 | | 0 @testcase | 101 | | [ 65 66 67 68 ] -@code_hash 55974318379341089957961227475446008591490555692181953973486962465702042912657 +@code_hash 58447050269190289721084012139099481162782788646785441106022886746601529758643 */ diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index a484475a2..4bd407f59 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -487,6 +487,81 @@ fun test34() { return o1; } +@pure fun getXPure() { return 1; } +@pure fun getYPure() { return 2; } +global t_impure: tuple; +fun getXImpure() { t_impure.push(1); return 1; } +fun getYImpure() { t_impure.push(2); return 2; } + +@method_id(135) +fun test35() { + var p: Point = { + y: getYPure(), + x: getXPure(), + }; + return p; +} + +@method_id(136) +fun test36() { + t_impure = createEmptyTuple(); + var p: Point = { + y: getYImpure(), + x: getXImpure(), + }; + return (p, t_impure); +} + +@method_id(137) +fun test37() { + var num = 0; + var p: Point = { + y: num += 5, + x: (num *= 10) - 2, + }; + return (p, num); +} + +@method_id(138) +fun test38(num: int) { + var p: Point = { + y: num += 5, + x: (num *= 10) - 2, + }; + return (p, num); +} + +struct TwoPoints { + p1: Point; + p2: Point; +} + +@method_id(139) +fun test39(): (TwoPoints, int) { + var cs = stringHexToSlice("0102030405"); + return ({ + p2: { y: cs.loadUint(8), x: cs.loadUint(8) }, + p1: { x: cs.loadUint(8), y: cs.loadUint(8) }, + }, cs.remainingBitsCount()); +} + +@method_id(140) +fun test40() { + var cs = stringHexToSlice("0102030405"); + return TwoPoints { + p1: { y: cs.loadUint(8), x: cs.loadUint(8) }, + p2: { y: cs.loadUint(8), x: cs.loadUint(8) }, + } +} + +@method_id(141) +fun test41(rev: bool) { + var cs = stringHexToSlice("0102030405"); + return TwoPoints { + p2: rev ? { y: cs.loadUint(8), x: cs.loadUint(8) } : { x: cs.loadUint(8), y: cs.loadUint(8) }, + p1: rev ? { y: cs.loadUint(8), x: cs.loadUint(8) } : { x: cs.loadUint(8), y: cs.loadUint(8) }, + } +} fun main(x: int8, y: MInt) { __expect_type(PointAlias{x,y}, "Point"); @@ -539,6 +614,14 @@ type PointAlias = Point; @testcase | 132 | | 10 20 10 0 0 20 0 0 0 0 @testcase | 133 | | -1 0 5 (null) (null) (null) 0 0 46 @testcase | 134 | | 10 20 +@testcase | 135 | | 1 2 +@testcase | 136 | | 1 2 [ 2 1 ] +@testcase | 137 | | 48 5 50 +@testcase | 138 | 3 | 78 8 80 +@testcase | 139 | | 3 4 2 1 8 +@testcase | 140 | | 2 1 4 3 +@testcase | 141 | -1 | 4 3 2 1 +@testcase | 141 | 0 | 3 4 1 2 @fif_codegen """ @@ -571,4 +654,25 @@ type PointAlias = Point; }> """ +@fif_codegen +""" + test35 PROC:<{ // + getXPure CALLDICT // '2 + getYPure CALLDICT // p.x p.y + }> +""" + +@fif_codegen +""" + test36 PROC:<{ // + NIL // '0 + t_impure SETGLOB // + getYImpure CALLDICT // '4 + getXImpure CALLDICT // p.y p.x + t_impure GETGLOB // p.y p.x g_t_impure + s1 s2 XCHG // p.x p.y g_t_impure + }> +""" + + */ diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 6c376491e..31bccca15 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -281,14 +281,15 @@ class LValContext { // the purpose of this class is having a call `f(a1,a2,...)` when f has asm arg_order, to check // whether it's safe to rearrange arguments (to evaluate them in arg_order right here for fewer stack manipulations) // or it's unsafe, and we should evaluate them left-to-right; -// example: `f(1,2,3)` / `b.storeUint(2,32)` is safe -// example: `f(x,x+=5,x)` / `f(impureF1(), global_var)` is unsafe +// example: `f(1,2,3)` / `b.storeUint(2,32)` is safe; +// example: `f(x,x+=5,x)` / `f(impureF1(), global_var)` / `f(s.loadInt(), s.loadInt())` is unsafe; +// the same rules are used to check an object literal: is it safe to convert `{y:expr, x:expr}` to declaration order {x,y} class CheckReorderingForAsmArgOrderIsSafeVisitor final : public ASTVisitorFunctionBody { bool has_side_effects = false; protected: void visit(V v) override { - has_side_effects |= v->fun_maybe == nullptr || !v->fun_maybe->is_marked_as_pure(); + has_side_effects |= v->fun_maybe == nullptr || !v->fun_maybe->is_marked_as_pure() || v->fun_maybe->has_mutate_params(); parent::visit(v); } @@ -323,6 +324,14 @@ class CheckReorderingForAsmArgOrderIsSafeVisitor final : public ASTVisitorFuncti } return !visitor.has_side_effects; } + + static bool is_safe_to_reorder(V v) { + CheckReorderingForAsmArgOrderIsSafeVisitor visitor; + for (int i = 0; i < v->get_num_fields(); ++i) { + visitor.ASTVisitorFunctionBody::visit(v->get_field(i)->get_init_val()); + } + return !visitor.has_side_effects; + } }; // given `{some_expr}!`, return some_expr @@ -1425,10 +1434,78 @@ static std::vector process_typed_tuple(V v, CodeBl return transition_to_target_type(std::move(left), code, target_type, v); } +static std::vector process_object_literal_shuffled(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + // creating an object like `Point { y: getY(), x: getX() }`, where fields order doesn't match declaration; + // as opposed to a non-shuffled version `{x:..., y:...}`, we should at first evaluate fields as they created, + // and then to place them in a correct order + std::vector tensor_items; // create a tensor of literal fields values + std::vector target_types; + tensor_items.reserve(v->get_body()->get_num_fields()); + target_types.reserve(v->get_body()->get_num_fields()); + for (int i = 0; i < v->get_body()->get_num_fields(); ++i) { + auto v_field = v->get_body()->get_field(i); + StructFieldPtr field_ref = v->struct_ref->find_field(v_field->get_field_name()); + tensor_items.push_back(v_field->get_init_val()); + target_types.push_back(field_ref->declared_type); + } + const auto* tensor_target_type = TypeDataTensor::create(std::move(target_types))->try_as(); + std::vector literal_rvect = pre_compile_tensor(code, tensor_items, lval_ctx, tensor_target_type); + + std::vector rvect = code.create_tmp_var(TypeDataStruct::create(v->struct_ref), v->loc, "(object)"); + int stack_offset = 0; + for (StructFieldPtr field_ref : v->struct_ref->fields) { + int stack_width = field_ref->declared_type->get_width_on_stack(); + std::vector field_rvect(rvect.begin() + stack_offset, rvect.begin() + stack_offset + stack_width); + stack_offset += stack_width; + + int tensor_offset = 0; + bool exists_in_literal = false; + for (int i = 0; i < v->get_body()->get_num_fields(); ++i) { + auto v_field = v->get_body()->get_field(i); + int tensor_item_width = v_field->field_ref->declared_type->get_width_on_stack(); + if (v_field->get_field_name() == field_ref->name) { + exists_in_literal = true; + std::vector literal_field_rvect(literal_rvect.begin() + tensor_offset, literal_rvect.begin() + tensor_offset + tensor_item_width); + code.emplace_back(v->loc, Op::_Let, std::move(field_rvect), std::move(literal_field_rvect)); + break; + } + tensor_offset += tensor_item_width; + } + if (exists_in_literal || field_ref->declared_type == TypeDataNever::create()) { + continue; + } + + tolk_assert(field_ref->has_default_value()); + SrcLocation last_loc = v->get_body()->empty() ? v->loc : v->get_body()->get_all_fields().back()->loc; + ASTAuxData *aux_data = new AuxData_ForceFiftLocation(last_loc); + auto v_force_loc = createV(v->loc, field_ref->default_value, aux_data, field_ref->declared_type); + std::vector def_rvect = pre_compile_expr(v_force_loc, code, field_ref->declared_type); + code.emplace_back(v->loc, Op::_Let, std::move(field_rvect), std::move(def_rvect)); + } + + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + static std::vector process_object_literal(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { // an object (an instance of a struct) is actually a tensor at low-level // for example, `struct User { id: int; name: slice; }` occupies 2 slots // fields of a tensor are placed in order of declaration (in a literal they might be shuffled) + bool are_fields_shuffled = false; + for (int i = 1; i < v->get_body()->get_num_fields(); ++i) { + StructFieldPtr field_ref = v->struct_ref->find_field(v->get_body()->get_field(i)->get_field_name()); + StructFieldPtr prev_field_ref = v->struct_ref->find_field(v->get_body()->get_field(i - 1)->get_field_name()); + are_fields_shuffled |= prev_field_ref->field_idx > field_ref->field_idx; + } + + // if fields are created {y,x} (not {x,y}), maybe, it's nevertheless safe to evaluate them as {x,y}; + // for example, if they are just constants, calls to pure non-mutating functions, etc.; + // generally, rules of "can we evaluate {x,y} instead of {y,x}" follows the same logic + // as passing of calling `f(x,y)` with asm arg_order, is it safe to avoid SWAP + if (are_fields_shuffled && !CheckReorderingForAsmArgOrderIsSafeVisitor::is_safe_to_reorder(v->get_body())) { + // okay, we have `{y: getY(), x: getX()}` / `{y: v += 1, x: v}`, evaluate them in created order + return process_object_literal_shuffled(v, code, target_type, lval_ctx); + } + SrcLocation prev_loc = v->loc; std::vector tensor_items; std::vector target_types; From 865ee8b31fed3977d1c1fd403bec287e158f8e33 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Wed, 28 May 2025 14:46:54 +0300 Subject: [PATCH 268/388] [Tolk] Auto-packing to/from cells/builders/slices A fully working auto-serialization is made within a big single commit. All types are supported. fromCell, toCell, loadAny, storeAny, etc. Typed cells, overflow detection, error messages, postfixes and opcodes, and more. --- crypto/smartcont/tolk-stdlib/common.tolk | 140 ++- .../smartcont/tolk-stdlib/gas-payments.tolk | 2 +- tolk-tester/tests/annotations-tests.tolk | 31 + .../tests/invalid-declaration/err-1795.tolk | 13 + .../tests/invalid-serialization/err-7008.tolk | 16 + .../tests/invalid-serialization/err-7114.tolk | 22 + .../tests/invalid-serialization/err-7215.tolk | 23 + .../tests/invalid-serialization/err-7419.tolk | 24 + .../tests/invalid-serialization/err-7514.tolk | 47 + .../tests/invalid-serialization/err-7563.tolk | 20 + .../tests/invalid-serialization/err-7564.tolk | 16 + .../tests/invalid-serialization/err-7810.tolk | 20 + .../tests/invalid-serialization/err-7811.tolk | 19 + .../tests/invalid-serialization/err-7812.tolk | 20 + .../tests/invalid-serialization/err-7813.tolk | 13 + .../tests/invalid-serialization/err-7911.tolk | 22 + tolk-tester/tests/pack-unpack-1.tolk | 130 +++ tolk-tester/tests/pack-unpack-2.tolk | 682 +++++++++++ tolk-tester/tests/pack-unpack-3.tolk | 284 +++++ tolk-tester/tests/pack-unpack-4.tolk | 162 +++ tolk-tester/tests/pack-unpack-5.tolk | 183 +++ tolk-tester/tests/some-tests-2.tolk | 16 + tolk/CMakeLists.txt | 3 + tolk/abscode.cpp | 11 + tolk/ast-from-tokens.cpp | 101 +- tolk/ast-replicator.h | 2 + tolk/ast-stringifier.h | 12 +- tolk/ast.cpp | 6 + tolk/ast.h | 21 +- tolk/builtins.cpp | 108 +- tolk/constant-evaluator.cpp | 4 +- tolk/pack-unpack-api.cpp | 257 ++++ tolk/pack-unpack-api.h | 35 + tolk/pack-unpack-serializers.cpp | 1031 +++++++++++++++++ tolk/pack-unpack-serializers.h | 137 +++ tolk/pipe-ast-to-legacy.cpp | 127 +- tolk/pipe-check-inferred-types.cpp | 4 + tolk/pipe-check-serialized-fields.cpp | 115 ++ tolk/pipe-constant-folding.cpp | 8 +- tolk/pipe-infer-types-and-calls.cpp | 14 +- tolk/pipe-optimize-boolean-expr.cpp | 4 +- tolk/pipe-register-symbols.cpp | 21 +- tolk/pipeline.h | 1 + tolk/symtable.cpp | 24 + tolk/symtable.h | 33 +- tolk/tolk.cpp | 1 + tolk/tolk.h | 1 + tolk/type-system.cpp | 11 + 48 files changed, 3900 insertions(+), 97 deletions(-) create mode 100644 tolk-tester/tests/annotations-tests.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1795.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7008.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7114.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7215.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7419.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7514.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7563.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7564.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7810.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7811.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7812.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7813.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7911.tolk create mode 100644 tolk-tester/tests/pack-unpack-1.tolk create mode 100644 tolk-tester/tests/pack-unpack-2.tolk create mode 100644 tolk-tester/tests/pack-unpack-3.tolk create mode 100644 tolk-tester/tests/pack-unpack-4.tolk create mode 100644 tolk-tester/tests/pack-unpack-5.tolk create mode 100644 tolk/pack-unpack-api.cpp create mode 100644 tolk/pack-unpack-api.h create mode 100644 tolk/pack-unpack-serializers.cpp create mode 100644 tolk/pack-unpack-serializers.h create mode 100644 tolk/pipe-check-serialized-fields.cpp diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 642337df5..db3dfc0ff 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -55,6 +55,11 @@ fun tuple.size(self): int fun tuple.last(self): T asm "LAST"; +/// Pops and returns the last element of a non-empty tuple. +@pure +fun tuple.pop(mutate self): T + asm "TPOP"; + /** Mathematical primitives. @@ -210,6 +215,139 @@ fun commitContractDataAndActions(): void asm "COMMIT"; +/** + Auto packing structures to/from cells. +*/ + +/// Convert anything to a cell (most likely, you'll call it for structures). +/// Example: +/// ``` +/// var st: MyStorage = { ... }; +/// contract.setData(st.toCell()); +/// ``` +/// Internally, a builder is created, all fields are serialized one by one, and a builder is flushed +/// (beginCell() + serialize fields + endCell()). +@pure +fun T.toCell(self): Cell + builtin; + +/// Parse anything from a cell (most likely, you'll call it for structures). +/// Example: +/// ``` +/// var st = MyStorage.fromCell(contract.getData()); +/// ``` +/// Internally, a cell is unpacked to a slice, and that slice is parsed +/// (packedCell.beginParse() + read from slice). +@pure +fun T.fromCell(packedCell: cell): T + builtin; + +/// Parse anything from a slice (most likely, you'll call it for structures). +/// Example: +/// ``` +/// var msg = CounterIncrement.fromSlice(cs); +/// ``` +/// All fields are read from a slice immediately. +/// If a slice is corrupted, an exception is thrown (most likely, excode 9 "cell underflow"). +/// Note, that a passed slice is NOT mutated, its internal pointer is NOT shifted. +/// If you need to mutate it, like `cs.loadInt()`, consider calling `cs.loadAny()`. +@pure +fun T.fromSlice(rawSlice: slice): T + builtin; + +/// Parse anything from a slice, shifting its internal pointer. +/// Similar to `slice.loadUint()` and others, but allows loading structures. +/// Example: +/// ``` +/// var st: MyStorage = cs.loadAny(); // or cs.loadAny() +/// ``` +/// Similar to `MyStorage.fromSlice(cs)`, but called as a slice method and mutates the slice. +@pure +fun slice.loadAny(mutate self): T + builtin; + +/// Skip anything in a slice, shifting its internal pointer. +/// Similar to `slice.skipBits()` and others, but allows skipping structures. +/// Example: +/// ``` +/// struct TwoInts { a: int32; b: int32; } +/// cs.skipAny(); // skips 64 bits +/// ``` +@pure +fun slice.skipAny(mutate self): self + builtin; + +/// Store anything to a builder. +/// Similar to `builder.storeUint()` and others, but allows storing structures. +/// Example: +/// ``` +/// var b = beginCell().storeUint(32).storeAny(msgBody).endCell(); +/// ``` +@pure +fun builder.storeAny(mutate self, v: T): self + builtin; + +/// Cell represents a typed cell reference (as opposed to untyped `cell`). +/// Example: +/// ``` +/// struct ExtraData { ... } +/// +/// struct MyStorage { +/// ... +/// extra: Cell; // TL-B `^ExtraData` +/// optional: Cell?; // TL-B `(Maybe ^ExtraData)` +/// code: cell; // TL-B `^Cell` +/// data: cell?; // TL-B `(Maybe ^Cell)` +/// } +/// ``` +/// Note, that `st = MyStorage.fromSlice(s)` does NOT deep-load any refs; `st.extra` is `Cell`, not `T`; +/// you should manually call `st.extra.load()` to get T (ExtraData in this example). +struct Cell { + tvmCell: cell; +} + +/// Parse data from already loaded cell reference. +/// Example: +/// ``` +/// struct MyStorage { ... extra: Cell; } +/// +/// var st = MyStorage.fromCell(contract.getData()); +/// // st.extra is cell; if we need to unpack it, we do +/// var extra = st.extra.load(); // it's ExtraData, unpacked from loaded ref +/// ``` +@pure +fun Cell.load(self): T + builtin; + +/// Converts a typed cell into a slice. +@pure +fun Cell.beginParse(self): slice + asm "CTOS"; + +/// RemainingBitsAndRefs is a special built-in type to get "all the rest" slice tail on reading. +/// Example: +/// ``` +/// struct JettonMessage { +/// ... some fields +/// forwardPayload: RemainingBitsAndRefs; +/// } +/// ``` +/// When you deserialize JettonMessage, forwardPayload contains "everything left after reading fields above". +type RemainingBitsAndRefs = slice; + +/// Creates a cell with zero bits and references. +/// Equivalent to `beginCell().endCell()` but cheaper. +@pure +fun createEmptyCell(): cell + asm " PUSHREF"; + +/// Creates a slice with zero remaining bits and references. +/// Equivalent to `beginCell().endCell().beginParse()` but cheaper. +@pure +fun createEmptySlice(): slice + asm "x{} PUSHSLICE"; + + /** Signature checks, hashing, cryptography. */ @@ -421,7 +559,7 @@ fun stringHexToSlice(constStringBytesHex: slice): slice fun cell.beginParse(self): slice asm "CTOS"; -/// Checks if slice is empty. If not, throws an exception. +/// Checks if slice is empty. If not, throws an exception with code 9. fun slice.assertEnd(self): void asm "ENDS"; diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk index 2c35faf93..89366cf4f 100644 --- a/crypto/smartcont/tolk-stdlib/gas-payments.tolk +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -6,7 +6,7 @@ tolk 0.12 */ /// Returns amount of gas (in gas units) consumed in current Computation Phase. -fun getGasConsumedAtTheMoment(): coins +fun getGasConsumedAtTheMoment(): int asm "GASCONSUMED"; /// This function is required to be called when you process an external message (from an outer world) diff --git a/tolk-tester/tests/annotations-tests.tolk b/tolk-tester/tests/annotations-tests.tolk new file mode 100644 index 000000000..730b0ee0b --- /dev/null +++ b/tolk-tester/tests/annotations-tests.tolk @@ -0,0 +1,31 @@ +@deprecated("anything") +@custom("anything") +global a: int; + +@deprecated +@custom +const ASDF = 1; + +@custom("props", {allowed: false, ids: [1,2,3]}) +@deprecated +fun f() {} + +@custom({ + `type`: 123, + value: 19 +}) +@custom("another", 12, "annotation") +struct TTT {} + +@deprecated +type MyMsg = int; + +@custom(1,2,3,4) +@custom({function: "main", contract: self}) +fun main() { + return 0; +} + +/** +@testcase | 0 | | 0 + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1795.tolk b/tolk-tester/tests/invalid-declaration/err-1795.tolk new file mode 100644 index 000000000..67b062cbe --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1795.tolk @@ -0,0 +1,13 @@ +@overflow1023_policy("ignorje") +struct A { + d1: bits256; + d2: bits256; + d3: bits256; + d4: bits256; + d5: bits256; +} + +/** +@compilation_should_fail +@stderr incorrect value for @overflow1023_policy + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7008.tolk b/tolk-tester/tests/invalid-serialization/err-7008.tolk new file mode 100644 index 000000000..9203ea228 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7008.tolk @@ -0,0 +1,16 @@ +struct Point { + x: int; + y: int; +} + +fun main(p: Point) { + p.toCell(); +} + +/** +@compilation_should_fail +@stderr auto-serialization via toCell() is not available for type `Point` +@stderr because field `Point.x` of type `int` can't be serialized +@stderr because type `int` is not serializable, it doesn't define binary width +@stderr hint: replace `int` with `int32` / `uint64` / `coins` / etc. + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7114.tolk b/tolk-tester/tests/invalid-serialization/err-7114.tolk new file mode 100644 index 000000000..fe228d601 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7114.tolk @@ -0,0 +1,22 @@ +type NotSerializableTensor = (int8, slice); + +struct Demo { + a: int8; + b: int8 | int16; + c: NotSerializableTensor; +} + +fun main(p: Demo?) { + p.toCell(); +} + +/** +@compilation_should_fail +@stderr auto-serialization via toCell() is not available for type `Demo?` +@stderr because field `Demo.c` of type `NotSerializableTensor` can't be serialized +@stderr because alias `NotSerializableTensor` expands to `(int8, slice)` +@stderr because element `tensor.1` of type `slice` can't be serialized +@stderr because type `slice` is not serializable, it doesn't define binary width +@stderr hint: replace `slice` with `address` if it's an address, actually +@stderr hint: replace `slice` with `bits128` and similar if it represents fixed-width data without refs + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7215.tolk b/tolk-tester/tests/invalid-serialization/err-7215.tolk new file mode 100644 index 000000000..63b46affc --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7215.tolk @@ -0,0 +1,23 @@ +type MInt = int; + +struct CantBe { + a: int8; + b: MInt?; +} + +struct Container { + item: T; +} + +fun main(s: slice) { + Container.fromSlice(s); +} + +/** +@compilation_should_fail +@stderr auto-serialization via fromSlice() is not available for type `Container` +@stderr because field `Container.item` of type `CantBe` can't be serialized +@stderr because field `CantBe.b` of type `MInt?` can't be serialized +@stderr because alias `MInt` expands to `int` +@stderr because type `int` is not serializable + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7419.tolk b/tolk-tester/tests/invalid-serialization/err-7419.tolk new file mode 100644 index 000000000..4133c80a0 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7419.tolk @@ -0,0 +1,24 @@ +struct(0x1) Ok1 {} +struct(0x2) Ok2 {} +struct(0x3) Ok3 {} +struct NoPrefix {} + +type MsgOk1 = Ok1 | Ok2 | Ok3; + +type MsgCantBe = Ok1 | Ok2 | NoPrefix | Ok3; + +fun main() { + MsgOk1.fromCell(beginCell().endCell()); + + MsgCantBe.fromCell(beginCell().endCell()); +} + +/** +@compilation_should_fail +@stderr auto-serialization via fromCell() is not available for type `MsgCantBe` +@stderr because alias `MsgCantBe` expands to `Ok1 | Ok2 | NoPrefix | Ok3` +@stderr because could not automatically generate serialization prefixes for a union +@stderr because struct `Ok3` has opcode, but `NoPrefix` does not +@stderr hint: manually specify opcodes to all structures +@stderr err-7419.tolk:13 + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7514.tolk b/tolk-tester/tests/invalid-serialization/err-7514.tolk new file mode 100644 index 000000000..11728a6c7 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7514.tolk @@ -0,0 +1,47 @@ +struct In4 { + b: builder; +} + +struct In3 { + i: In4; +} + +struct In2 { + i: In3?; +} + +struct In1 { + i: (In2, slice); +} + +struct MaybeNothing {} +struct MaybeJust { value: T } +type Maybe = MaybeNothing | MaybeJust; + +struct CantBe { + a: address; + b: address?; + i: Maybe; +} + +fun main(c: cell) { + var d: CantBe = CantBe.fromCell(c); +} + +/** +@compilation_should_fail +@stderr auto-serialization via fromCell() is not available for type `CantBe` +@stderr because field `CantBe.i` of type `Maybe` can't be serialized +@stderr because alias `Maybe` expands to `MaybeNothing | MaybeJust` +@stderr because variant #2 of type `MaybeJust` can't be serialized +@stderr because field `MaybeJust.value` of type `In1` can't be serialized +@stderr because field `In1.i` of type `(In2, slice)` can't be serialized +@stderr because element `tensor.0` of type `In2` can't be serialized +@stderr because field `In2.i` of type `In3?` can't be serialized +@stderr because field `In3.i` of type `In4` can't be serialized +@stderr because field `In4.b` of type `builder` can't be serialized +@stderr because type `builder` can not be used for reading, only for writing +@stderr hint: use `bitsN` or `RemainingBitsAndRefs` for reading +@stderr hint: using generics, you can substitute `builder` for writing and something other for reading +@stderr CantBe.fromCell(c) + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7563.tolk b/tolk-tester/tests/invalid-serialization/err-7563.tolk new file mode 100644 index 000000000..cc83a0cd5 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7563.tolk @@ -0,0 +1,20 @@ +struct A { + d1: bits256; + d2: bits256; + d3: bits256; + d4: bits256; + d5: bits256; +} + +struct B { + a: Cell; +} + +fun main(b: B) { + b.toCell(); +} + +/** +@compilation_should_fail +@stderr struct `A` can exceed 1023 bits in serialization (estimated size: 1280..1280 bits) + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7564.tolk b/tolk-tester/tests/invalid-serialization/err-7564.tolk new file mode 100644 index 000000000..36eb63d4f --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7564.tolk @@ -0,0 +1,16 @@ +struct A { + d1: bits256; + d2: bits256; + d3: bits256; + d4: bits256; + d5: bits256; +} + +fun main(o: A | int32) { + o.toCell(); +} + +/** +@compilation_should_fail +@stderr struct `A` can exceed 1023 bits in serialization (estimated size: 1280..1280 bits) + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7810.tolk b/tolk-tester/tests/invalid-serialization/err-7810.tolk new file mode 100644 index 000000000..2cf0d32e5 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7810.tolk @@ -0,0 +1,20 @@ +struct(0b01) B {} +struct(0b00) C {} + +struct A { + multiple: B | C | int32; +} + +fun main() { + var a: A = { multiple: B{} }; + return a.toCell(); +} + +/** +@compilation_should_fail +@stderr auto-serialization via toCell() is not available for type `A` +@stderr because field `A.multiple` of type `B | C | int32` can't be serialized +@stderr because could not automatically generate serialization prefixes for a union +@stderr because of mixing primitives and struct `C` with serialization prefix +@stderr hint: extract primitives to single-field structs and provide prefixes + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7811.tolk b/tolk-tester/tests/invalid-serialization/err-7811.tolk new file mode 100644 index 000000000..5dbcab7e6 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7811.tolk @@ -0,0 +1,19 @@ +struct(0b01) B {} +struct(0b00) C {} + +struct A { + multiple: B | C | null; +} + +fun main() { + var a: A = { multiple: B{} }; + return a.toCell(); +} + +/** +@compilation_should_fail +@stderr error: auto-serialization via toCell() is not available for type `A` +@stderr because field `A.multiple` of type `B | C | null` can't be serialized +@stderr because could not automatically generate serialization prefixes for a union +@stderr because of mixing primitives and struct `C` with serialization prefix + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7812.tolk b/tolk-tester/tests/invalid-serialization/err-7812.tolk new file mode 100644 index 000000000..0b6995633 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7812.tolk @@ -0,0 +1,20 @@ +struct(0b000) B {} +struct C {} + +struct A { + multiple: (int32, B | C); +} + +fun main() { + var a: A = { multiple: (5, B{}) }; + return a.toCell(); +} + +/** +@compilation_should_fail +@stderr error: auto-serialization via toCell() is not available for type `A` +@stderr because field `A.multiple` of type `(int32, B | C)` can't be serialized +@stderr because element `tensor.1` of type `B | C` can't be serialized +@stderr because could not automatically generate serialization prefixes for a union +@stderr because struct `B` has opcode, but `C` does not + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7813.tolk b/tolk-tester/tests/invalid-serialization/err-7813.tolk new file mode 100644 index 000000000..720678cff --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7813.tolk @@ -0,0 +1,13 @@ +struct (0x0F) A {} + +fun f(x: int32 | A) { + x.toCell(); +} + +/** +@compilation_should_fail +@stderr auto-serialization via toCell() is not available for type `int32 | A` +@stderr because could not automatically generate serialization prefixes for a union +@stderr because of mixing primitives and struct `A` with serialization prefix +@stderr hint: extract primitives to single-field structs and provide prefixes + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7911.tolk b/tolk-tester/tests/invalid-serialization/err-7911.tolk new file mode 100644 index 000000000..ab045dd40 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7911.tolk @@ -0,0 +1,22 @@ +struct ExtraData { + owner: address; + lastTime: int; +} + +struct Storage { + more: Cell; +} + +fun main() { + Storage.fromSlice(""); +} + +/** +@compilation_should_fail +@stderr error: auto-serialization via fromSlice() is not available for type `Storage` +@stderr because field `Storage.more` of type `Cell` can't be serialized +@stderr because type `ExtraData` can't be serialized +@stderr because field `ExtraData.lastTime` of type `int` can't be serialized +@stderr because type `int` is not serializable, it doesn't define binary width +@stderr hint: replace `int` with `int32` / `uint64` / `coins` / etc. + */ diff --git a/tolk-tester/tests/pack-unpack-1.tolk b/tolk-tester/tests/pack-unpack-1.tolk new file mode 100644 index 000000000..00767042c --- /dev/null +++ b/tolk-tester/tests/pack-unpack-1.tolk @@ -0,0 +1,130 @@ +struct JustInt32 { + value: int32; +} + +struct Point { + x: int32; + y: int32; +} + +@method_id(101) +fun test1(value: int) { + var t: JustInt32 = { value }; + var c = t.toCell(); + var t2 = c.load(); + __expect_type(c, "Cell"); + __expect_type(t2, "JustInt32"); + if (0) { + var rawCell: cell = c; // Cell is assignable to raw cell + contract.setData(t.toCell()); // (for this purpose, since we don't have overloads) + } + + var t3 = Cell.load(c); + var t4 = JustInt32.fromCell(c); + var t5 = JustInt32.fromSlice(c.beginParse()); + return (t2.value, t3.value, t4.value, t5.value); +} + +@method_id(102) +fun test2() { + var t: JustInt32 = { value: 10 }; + var c = t.toCell(); + var s = c.tvmCell.beginParse(); + __expect_type(s.skipAny(), "slice"); + s.assertEnd(); + return true; +} + +@method_id(103) +fun test3() { + var yy = 20; + var p: Point = { x: 10, y: yy }; + var c = p.toCell(); // p is constant, storing its fields are joined into a single STI + var s = c.beginParse(); + s.skipAny(); // skipping is also merged to skip 64 bits + return s.remainingBitsCount(); +} + +@method_id(104) +fun test4() { + var s = stringHexToSlice("000000010000000200000003"); + var p = Point.fromSlice(s); // does not mutate s + return (p, s.remainingBitsCount()); +} + +@method_id(105) +fun test5() { + var s = stringHexToSlice("000000010000000200000003"); + var p: Point = s.loadAny(); // mutates s + return (p, s.remainingBitsCount()); +} + +@method_id(106) +fun test6() { + var b = beginCell().storeInt(1, 32); + var p: Point = { x: 10, y: 20 }; + b.storeAny(p); + var s = b.endCell().beginParse(); + return (s.remainingBitsCount(), b.bitsCount(), Point.fromSlice(s.getLastBits(64)), s.remainingBitsCount()); +} + +// this function is used in serialization codegen, it's not exposed to stdlib; +// it constructs "x{...}" slices for SDBEGINSQ, used for opcode matching; +// here we write some unit tests and fif codegen tests for it +fun slice.tryStripPrefix(mutate self, prefix: int, prefixLen: int): bool + builtin; + +@method_id(107) +fun test7(s: slice) { + if (s.tryStripPrefix(0x10, 32)) { return (1, s.remainingBitsCount()); } + if (s.tryStripPrefix(0x40, 31)) { return (2, s.remainingBitsCount()); } + if (s.tryStripPrefix(0xFF18, 16)) { return (3, s.remainingBitsCount()); } + if (s.tryStripPrefix(0b1001, 8)) { return (4, s.remainingBitsCount()); } + if (s.tryStripPrefix(0b01, 3)) { return (5, s.remainingBitsCount()); } + return (6, s.remainingBitsCount()); +} + +fun main(c: cell) { + c as Cell; + (c as Cell) as cell; + (c as Cell) as cell?; + __expect_type((c as Cell?)!, "Cell"); + c = c as Cell; // Cell implicitly converts to cell +} + +/** +@testcase | 101 | 10 | 10 10 10 10 +@testcase | 102 | | -1 +@testcase | 103 | | 0 +@testcase | 104 | | 1 2 96 +@testcase | 105 | | 1 2 32 +@testcase | 106 | | 96 96 10 20 96 +@testcase | 107 | x{00000080} | 2 1 +@testcase | 107 | x{09332} | 4 12 +@testcase | 107 | x{2} | 5 1 +@testcase | 107 | x{0234} | 6 16 + +@fif_codegen +""" + test3 PROC:<{ + 20 PUSHINT // yy=20 + 10 PUSHINT // p.y=20 p.x=10 + NEWC // p.y=20 p.x=10 b + 32 STI // p.y=20 b + 32 STI // b + ENDC // c + CTOS // s + 32 PUSHINT // s '12=32 + SDSKIPFIRST // s + 32 PUSHINT // s '13=32 + SDSKIPFIRST // s + SBITS // '14 + }> +""" + +@fif_codegen x{00000010} SDBEGINSQ +@fif_codegen b{0000000000000000000000001000000} SDBEGINSQ +@fif_codegen x{ff18} SDBEGINSQ +@fif_codegen x{09} SDBEGINSQ +@fif_codegen b{001} SDBEGINSQ + */ diff --git a/tolk-tester/tests/pack-unpack-2.tolk b/tolk-tester/tests/pack-unpack-2.tolk new file mode 100644 index 000000000..75f20a5fd --- /dev/null +++ b/tolk-tester/tests/pack-unpack-2.tolk @@ -0,0 +1,682 @@ + +struct MaybeNothing {} +struct MaybeJust { value: T; } +type Maybe = MaybeNothing | MaybeJust; + +struct EitherLeft { value: T } +struct EitherRight { value: T } +type Either = EitherLeft | EitherRight; + +@inline +fun makeExternalAddress(hash: int, len: int): address { + return beginCell().storeUint(0b01, 2).storeUint(len, 9).storeUint(hash, len).endCell().beginParse() as address; +} + + +fun slice.assertEqDeeply(self, rhs: slice): slice { + var lhs = self; + assert(lhs.bitsEqual(rhs), 400); + assert(lhs.remainingRefsCount() == rhs.remainingRefsCount(), 400); + while (lhs.remainingRefsCount()) { + lhs.loadRef().beginParse().assertEqDeeply(rhs.loadRef().beginParse()); + } + return self; +} + +@inline +fun generateSlice_44_with_ref45(): slice { + return generateCell_44_with_ref45().beginParse(); +} + +@inline +fun generateCell_44_with_ref45(): cell { + return beginCell().storeInt(44, 32).storeRef(beginCell().storeInt(45, 32).endCell()).endCell(); +} + +fun assert_slice_is_44_and_ref45(s: slice) { + assert(s.loadInt(32) == 44, 400); + var ref = s.loadRef().beginParse(); + assert(ref.loadInt(32) == 45, 400); + ref.assertEnd(); + s.assertEnd(); +} + +@inline +fun slice.appendRef(self, refSlice: slice): slice { + return beginCell().storeSlice(self).storeRef(beginCell().storeSlice(refSlice).endCell()).endCell().beginParse(); +} + +fun run(input: TInputStruct, ans: slice) { + repeat (2) { + var s = input.toCell().beginParse(); + input = TInputStruct.fromSlice(s.assertEqDeeply(ans)); + } + input.toCell().beginParse().skipAny().assertEnd(); +} + + +/* + value:int32 + = JustInt32; +*/ + +struct JustInt32 { + value: int32; +} + +/* + value:(Maybe int32) + = JustMaybeInt32; + */ + +struct JustMaybeInt32 { + value: int32?; +} + +/* + op:int32 + amount:Grams + = TwoInts32AndCoins; +*/ + +struct TwoInts32AndCoins { + op: int32; + amount: coins; +} + +/* + op:int32 + query_id:uint64 + = TwoInts32And64; +*/ + +struct TwoInts32And64 { + op: int32; + query_id: uint64; +} + +/* + op:int32 + query_id_ref: ^[uint64] + = TwoInts32AndRef64; +*/ + +struct TwoInts32AndRef64 { + op: int32; + query_id_ref: Cell; +} + +/* + op:int32 + query_id:(Maybe uint64) + demo_bool_field:Bool + = TwoInts32AndMaybe64; +*/ + +struct TwoInts32AndMaybe64 { + op: int32; + query_id: uint64?; + demo_bool_field: bool; +} + +/* + addr:MsgAddressInt + = JustAddress; +*/ + +struct JustAddress { + addr: address; +} + +/* + op:int32 + addr:MsgAddressExt + query_id:uint64 + = TwoInts32And64SepByAddress; +*/ + +struct TwoInts32And64SepByAddress { + op: int32; + addr_e: address; + query_id: uint64; +} + +/* + op:int32 + i8or256:(Either int8 int256) + = IntAndEitherInt8Or256; +*/ + +struct IntAndEitherInt8Or256 { + op: int32; + i8or256: int8 | int256; +} + +/* + query_id_ref:uint64 = Inner1; + i64_in_ref:int64 = Inner2; + + op:int32 + i32orRef:(Either int32 ^Inner2) + query_id_maybe_ref: (Maybe ^Inner1) + = IntAndEither32OrRef64; +*/ + +struct Inner1 { + query_id_ref: uint64; +} +struct Inner2 { + i64_in_ref: int64; +} + +struct IntAndEither32OrRef64 { + op: int32; + i32orRef: int32 | Cell; + query_id_maybe_ref: Cell?; +} + +/* + value:(Either int8 (Maybe int256)) + op:int32 + = IntAndEither8OrMaybe256; +*/ + +struct IntAndEither8OrMaybe256 { + value: Either; + op: int32; +} + +/* + value:(Either (Maybe int8) int256) + op:int32 + = IntAndEitherMaybe8Or256; +*/ + +struct IntAndEitherMaybe8Or256 { + value: Either; + op: int32; +} + +/* + value:(Maybe (Maybe int8)) + op:int32 + = IntAndMaybeMaybe8; +*/ + +struct IntAndMaybeMaybe8 { + value: Maybe>; + op: int32; +} + +/* + f1:bits8 + f2:bits3 + f3:(Maybe bits20) + f4:(Either bits100 bits200) + = SomeBytesFields; +*/ + +struct SomeBytesFields { + f1: bytes1; + f2: bits3; + f3: bits20?; + f4: bits100 | bits200; +} + +/* + op:int32 + rest:Cell + = IntAndRestInlineCell; +*/ + +struct IntAndRestInlineCell { + op: int32; + rest: RemainingBitsAndRefs; +} + +/* + op:int32 + rest:^Cell + = IntAndRestRefCell; +*/ + +struct IntAndRestRefCell { + op: int32; + rest: cell; +} + +/* + op:int32 + rest:(Either Cell ^Cell) + = IntAndRestEitherCellOrRefCell; +*/ + +struct IntAndRestEitherCellOrRefCell { + op: int32; + rest: RemainingBitsAndRefs | cell; +} + +/* + op:int32 + ref1m:(Maybe ^Cell) + ref2m:(Maybe ^Cell) + ref3:^Cell + ref4m32:(Maybe ^JustInt32) + query_id:int64 + = DifferentMaybeRefs; + */ + +struct DifferentMaybeRefs { + op: int32; + ref1m: cell?; + ref2m: dict; + ref3: cell; + ref4m32: Cell?; + query_id: int64; +} + +/* + ji:JustInt32 + jmi:JustMaybeInt32 + jiMaybe:(Maybe JustInt32) + jmiMaybe:(Maybe JustMaybeInt32) + = DifferentIntsWithMaybe; +*/ + +struct DifferentIntsWithMaybe { + ji: JustInt32; + jmi: JustMaybeInt32; + jiMaybe: JustInt32?; + jmiMaybe: JustMaybeInt32?; +} + +/* + ja1:JustAddress + ja2m:(Maybe JustAddress) + imm:IntAndMaybeMaybe8 + tis:TwoInts32And64SepByAddress + = DifferentMix1; +*/ + +@overflow1023_policy("suppress") +struct DifferentMix1 { + ja1: JustAddress; + ja2m: JustAddress?; + ext_nn: address; + imm: IntAndMaybeMaybe8; + tis: TwoInts32And64SepByAddress; +} + +/* + iae:^IntAndEither32OrRef64 + tic:TwoInts32AndCoins + rest:Cell + = DifferentMix2; +*/ + +struct DifferentMix2 { + iae: Cell; + tic: TwoInts32AndCoins; + rest: RemainingBitsAndRefs; +} + +/* + bod:(Either ^TwoInts32AndCoins ^JustInt32) + tim:(Maybe TwoInts32AndCoins) + pairm:(Maybe (Both int32 int64)) + = DifferentMix3; + */ + +struct DifferentMix3 { + bod: Cell | Cell; + tim: TwoInts32AndCoins?; + pairm: (int32, int64)?; +} + +/* + Test: write with builder, read with other struct + (type `builder` is available for writing, but not for reading) + */ + +struct WriteWithBuilder { + f1: int32; + rest: builder; +} + +struct ReadWrittenWithBuilder { + f1: int32; + someInt: uint32; + someCell: cell?; +} + +struct ReadWriteRest { + f1: int32; + f2: coins; + rest: T; +} + +struct Tail224 { + ji: JustInt32, + addr: address, + ref1: Cell?, + ref2: Cell, +} + +type ReadRest_Remaining = ReadWriteRest; + +struct ReadWriteMid { + f1: int32; + mid: T; + f3: coins; +} + + + +// --------------------------------------------- + + +@method_id(200) +fun test_JustInt32_1() { + var b = beginCell(); + b.storeAny(JustInt32{ value: 123 }); + b.storeAny(JustInt32{ value: 456 }); + return b; +} + +@method_id(201) +fun test_JustInt32() { + run({ value: 255}, stringHexToSlice("000000FF")); + var s = stringHexToSlice("0000007b000001c8"); + return (s.loadAny(), s.loadAny()); +} + +@method_id(202) +fun test_JustMaybeInt32() { + run({ value: 255 }, stringHexToSlice("8000007FC_")); + run({ value: null }, stringHexToSlice("4_")); + return true; +} + +@method_id(203) +fun test_TwoInts32AndCoins() { + run({ op: 123, amount: 0 }, stringHexToSlice("0000007B0")); + run({ op: 123, amount: 1000000000 }, stringHexToSlice("0000007B43B9ACA00")); + return true; +} + +@method_id(204) +fun test_TwoInts32And64() { + run({ op: 123, query_id: 255 }, stringHexToSlice("0000007B00000000000000FF")); + return true; +} + +@method_id(205) +fun test_TwoInts32AndRef64() { + run({ op: 123, query_id_ref: { tvmCell: beginCell().storeUint(255,64).endCell() } }, stringHexToSlice("0000007B").appendRef(stringHexToSlice("00000000000000FF"))); + return true; +} + +@method_id(206) +fun test_TwoInts32AndMaybe64() { + run({ op: 123, query_id: 255, demo_bool_field: true }, stringHexToSlice("0000007B800000000000007FE_")); + run({ op: 123, query_id: null, demo_bool_field: true }, stringHexToSlice("0000007B6_")); + run({ op: 123, query_id: null, demo_bool_field: false }, stringHexToSlice("0000007B2_")); + return true; +} + +@method_id(207) +fun test_JustAddress() { + run({ addr: address("0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e") }, stringHexToSlice("80194DC6438F99D3D9DBE151944925D90B2492954BF6B9C070FBFF2DDED5F30547D_")); + return true; +} + +@method_id(208) +fun test_TwoInts32And64SepByAddress() { + run({ op: 123, addr_e: makeExternalAddress(70, 10), query_id: 255 } , stringHexToSlice("0000007B41423000000000000007FC_")); + run({ op: 123, addr_e: makeExternalAddress(666, 20), query_id: 254 } , stringHexToSlice("0000007B4280053400000000000001FD_")); + run({ op: 123, addr_e: createAddressNone(), query_id: 253 } , stringHexToSlice("0000007B000000000000003F6_")); + return true; +} + +@method_id(209) +fun test_IntAndEitherInt8Or256() { + run({ op: 123, i8or256: 80 as int8 }, stringHexToSlice("0000007B284_")); + run({ op: 123, i8or256: 65535 as int256 }, stringHexToSlice("0000007B8000000000000000000000000000000000000000000000000000000000007FFFC_")); + return true; +} + +@method_id(210) +fun test_IntAndEither32OrRef64() { + run({ op: 123, i32orRef: Inner2{ i64_in_ref: 555 }.toCell(), query_id_maybe_ref: Inner1{ query_id_ref: 888 }.toCell() }, stringHexToSlice("0000007BE_").appendRef(stringHexToSlice("000000000000022B")).appendRef(stringHexToSlice("0000000000000378"))); + run({ op: 123, i32orRef: Inner2{ i64_in_ref: 555 }.toCell(), query_id_maybe_ref: null }, stringHexToSlice("0000007BA_").appendRef(stringHexToSlice("000000000000022B"))); + run({ op: 123, i32orRef: 555, query_id_maybe_ref: Inner1{ query_id_ref: 888 }.toCell() }, stringHexToSlice("0000007B00000115E_").appendRef(stringHexToSlice("0000000000000378"))); + run({ op: 123, i32orRef: 555, query_id_maybe_ref: null }, stringHexToSlice("0000007B00000115A_")); + return IntAndEither32OrRef64.fromSlice(stringHexToSlice("0000007B00000115A_")); +} + +@method_id(211) +fun test_IntAndEither8OrMaybe256() { + run({ value: EitherLeft { value: 100 }, op: 123 }, stringHexToSlice("320000003DC_")); + run({ value: EitherRight { value: 10000 }, op: 123 }, stringHexToSlice("C0000000000000000000000000000000000000000000000000000000000009C40000001EE_")); + run({ value: EitherRight { value: null }, op: 123 }, stringHexToSlice("8000001EE_")); + return IntAndEither8OrMaybe256.fromSlice(stringHexToSlice("8000001EE_")); +} + +@method_id(212) +fun test_IntAndEitherMaybe8Or256() { + run({ value: EitherLeft { value: 100 }, op: 123 }, stringHexToSlice("590000001EE_")); + run({ value: EitherRight { value: 10000 }, op: 123 }, stringHexToSlice("80000000000000000000000000000000000000000000000000000000000013880000003DC_")); + run({ value: EitherLeft { value: null }, op: 123 }, stringHexToSlice("0000001EE_")); + return IntAndEitherMaybe8Or256.fromSlice(stringHexToSlice("0000001EE_")); +} + +@method_id(213) +fun test_SomeBytesFields() { + run({ f1: stringHexToSlice("A4") as bytes1, f2: stringHexToSlice("7_") as bits3, f3: null, f4: stringHexToSlice("BBA87684B3DAA58C0FCC75230C4302C9D156102139D631FF56") as bits200 }, stringHexToSlice("A46DDD43B4259ED52C607E63A9186218164E8AB08109CEB18FFAB4_")); + run({ f1: stringHexToSlice("E6") as bytes1, f2: stringHexToSlice("D_") as bits3, f3: stringHexToSlice("2531C") as bits20, f4: stringHexToSlice("927E88FAB2D327D9468547217") as bits100 }, stringHexToSlice("E6D2531C493F447D596993ECA342A390BC_")); + return SomeBytesFields.fromSlice(stringHexToSlice("E6D2531C493F447D596993ECA342A390BC_")).f4 is bits100; +} + +@method_id(214) +fun test_IntAndMaybeMaybe8() { + run({ value: MaybeJust { value: MaybeJust { value: 88 } }, op: 123 }, stringHexToSlice("D60000001EE_")); + run({ value: MaybeJust { value: MaybeNothing{} }, op: 123 }, stringHexToSlice("8000001EE_")); + + val t1 = IntAndMaybeMaybe8.fromSlice(stringHexToSlice("D60000001EE_")); // (88) 123 + val v1 = t1.value; + __expect_type(v1, "Maybe>"); + assert(v1 is MaybeJust && v1.value is MaybeJust && v1.value.value == 88 && t1.op == 123, 400); + __expect_type(v1, "MaybeJust>"); + __expect_type(v1.value, "MaybeJust"); + __expect_type(v1.value.value, "int8"); + val t2 = IntAndMaybeMaybe8.fromSlice(stringHexToSlice("8000001EE_")); // (()) 123 + val v2 = t2.value; + assert(v2 is MaybeJust && v2.value is MaybeNothing && t2.op == 123, 400); + __expect_type(v2, "MaybeJust>"); + __expect_type(v2.value, "MaybeNothing"); + + return true; +} + +@method_id(215) +fun test_IntAndRestInlineCell() { + run({ op: 123, rest: generateSlice_44_with_ref45() }, stringHexToSlice("0000007B0000002C").appendRef(stringHexToSlice("0000002D"))); + return true; +} + +@method_id(216) +fun test_IntAndRestRefCell() { + run({ op: 123, rest: generateCell_44_with_ref45() }, stringHexToSlice("0000007B").appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D")))); + return true; +} + +@method_id(217) +fun test_IntAndRestEitherCellOrRefCell() { + val input1: IntAndRestEitherCellOrRefCell = { op: 123, rest: generateCell_44_with_ref45() }; + var s1 = input1.toCell().beginParse(); + s1.assertEqDeeply(stringHexToSlice("0000007BC_").appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D")))); + match (val rest = IntAndRestEitherCellOrRefCell.fromSlice(s1).rest) { + cell => assert_slice_is_44_and_ref45(rest.beginParse()), + RemainingBitsAndRefs => assert_slice_is_44_and_ref45(rest) + } + + val input2: IntAndRestEitherCellOrRefCell = { op: 123, rest: generateSlice_44_with_ref45() }; + var s2 = input2.toCell().beginParse(); + s2.assertEqDeeply(stringHexToSlice("0000007B000000164_").appendRef(stringHexToSlice("0000002D"))); + match (val rest = IntAndRestEitherCellOrRefCell.fromSlice(s2).rest) { + cell => assert_slice_is_44_and_ref45(rest.beginParse()), + RemainingBitsAndRefs => assert_slice_is_44_and_ref45(rest) + } + + return true; +} + +@method_id(218) +fun test_DifferentMaybeRefs() { + run({ op: 123, ref1m: null, ref2m: null, ref3: generateCell_44_with_ref45(), ref4m32: null, query_id: 456}, stringHexToSlice("0000007B00000000000000391_").appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D")))); + run({ op: 123, ref1m: generateCell_44_with_ref45(), ref2m: null, ref3: generateCell_44_with_ref45(), ref4m32: JustInt32{value: 999}.toCell(), query_id: 456}, stringHexToSlice("0000007BA0000000000000391_").appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D"))).appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D"))).appendRef(stringHexToSlice("000003E7"))); + run({ op: 123, ref1m: null, ref2m: beginCell().endCell(), ref3: generateCell_44_with_ref45(), ref4m32: JustInt32{value: 998}.toCell(), query_id: 456}, stringHexToSlice("0000007B60000000000000391_").appendRef("").appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D"))).appendRef(stringHexToSlice("000003E6"))); + run({ op: 123, ref1m: generateCell_44_with_ref45(), ref2m: generateCell_44_with_ref45(), ref3: beginCell().endCell(), ref4m32: null, query_id: 456}, stringHexToSlice("0000007BC0000000000000391_").appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D"))).appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D"))).appendRef("")); + run({ op: 123, ref1m: generateCell_44_with_ref45(), ref2m: beginCell().endCell(), ref3: generateCell_44_with_ref45(), ref4m32: JustInt32{value: 997}.toCell(), query_id: 456}, stringHexToSlice("0000007BE0000000000000391_").appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D"))).appendRef("").appendRef(stringHexToSlice("0000002C").appendRef(stringHexToSlice("0000002D"))).appendRef(stringHexToSlice("000003E5"))); + return true; +} + +@method_id(219) +fun test_DifferentIntsWithMaybe() { + run({ ji: { value: 44 }, jmi: { value: 45 }, jiMaybe: null, jmiMaybe: null }, stringHexToSlice("0000002C800000169_")); + run({ ji: { value: 44 }, jmi: { value: null }, jiMaybe: { value: 45 }, jmiMaybe: { value: null } }, stringHexToSlice("0000002C4000000B6")); + run({ ji: { value: 44 }, jmi: { value: null }, jiMaybe: null, jmiMaybe: { value: 46 } }, stringHexToSlice("0000002C30000002E")); + return DifferentIntsWithMaybe.fromSlice(stringHexToSlice("0000002C30000002E")); +} + +@method_id(220) +fun test_DifferentMix1() { + run({ ja1: { addr: address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5") }, ja2m: { addr: address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5") }, ext_nn: makeExternalAddress(1234,30), imm: { op: 78, value: MaybeJust { value: MaybeJust { value: 78 } } }, tis: { op: 123, query_id: 889128, addr_e: makeExternalAddress(1234, 80) } }, + stringHexToSlice("80122199EC3C49BA84BA73C79F764BF1A4C1400717E303DC86E737C60A3E3B1B62180122199EC3C49BA84BA73C79F764BF1A4C1400717E303DC86E737C60A3E3B1B62087800004D2D3800000138000001ED2800000000000000000269000000000006C8944_")); + run({ ja1: { addr: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c") }, ja2m: null, ext_nn: makeExternalAddress(0,3), imm: { op: 99, value: MaybeJust { value: MaybeJust { value: 99 } } }, tis: { op: 1234, query_id: 889129, addr_e: createAddressNone() } }, + stringHexToSlice("800000000000000000000000000000000000000000000000000000000000000000040636300000063000004D2000000000003644A6_")); + + val o = DifferentMix1.fromSlice(stringHexToSlice("800000000000000000000000000000000000000000000000000000000000000000040636300000063000004D2000000000003644A6_")); + return (o.ja1.addr.getWorkchainAndHash(), o.ja2m, o.imm, o.tis.op, o.tis.addr_e.isNone() ? null : 0, o.tis.query_id, (o.ext_nn as slice).remainingBitsCount()); +} + +@method_id(221) +fun test_DifferentMix2() { + run({ iae: IntAndEither32OrRef64{ op: 777, i32orRef: 2983, query_id_maybe_ref: null }.toCell(), tic: { op: 123, amount: 829290000 }, rest: generateSlice_44_with_ref45() }, + stringHexToSlice("0000007B4316DF6100000002C").appendRef(stringHexToSlice("00000309000005D3A_")).appendRef(stringHexToSlice("0000002D"))); + val r1 = DifferentMix2.fromSlice(stringHexToSlice("0000007B4316DF6100000002C").appendRef(stringHexToSlice("00000309000005D3A_")).appendRef(stringHexToSlice("0000002D"))); + val iae1 = r1.iae.load(); + assert(iae1.i32orRef is int32, 400); + assert(iae1.i32orRef == 2983, 400); + assert(iae1.query_id_maybe_ref == null, 400); + assert_slice_is_44_and_ref45(r1.rest); + + run({ iae: { tvmCell: IntAndEither32OrRef64{ op: 778, i32orRef: { tvmCell: Inner2{ i64_in_ref: 9919992 }.toCell() }, query_id_maybe_ref: Inner1{ query_id_ref: 889477 }.toCell() }.toCell() }, tic: { op: 123, amount: 500000 }, rest: beginCell().endCell().beginParse() }, + stringHexToSlice("0000007B307A120").appendRef(stringHexToSlice("0000030AE_").appendRef(stringHexToSlice("0000000000975DF8")).appendRef(stringHexToSlice("00000000000D9285")))); + var r2 = DifferentMix2.fromSlice(stringHexToSlice("0000007B307A120").appendRef(stringHexToSlice("0000030AE_").appendRef(stringHexToSlice("0000000000975DF8")).appendRef(stringHexToSlice("00000000000D9285")))); + val iae2 = r2.iae.load(); + assert(iae2.i32orRef is Cell, 400); + __expect_type(iae2.i32orRef.load(), "Inner2"); + assert(iae2.i32orRef.load().i64_in_ref == 9919992, 400); + assert(iae2.query_id_maybe_ref != null, 400); + __expect_type(iae2.query_id_maybe_ref.load(), "Inner1"); + assert(iae2.query_id_maybe_ref.load().query_id_ref == 889477, 400); + assert(r2.tic.amount == 500000, 400); + r2.rest.assertEnd(); + + return true; +} + +@method_id(222) +fun test_DifferentMix3() { + run({ bod: TwoInts32AndCoins{op:123, amount:80000}.toCell(), tim: {op: 456, amount:0}, pairm: null }, stringHexToSlice("4000007201_").appendRef(stringHexToSlice("0000007B3013880"))); + run({ bod: TwoInts32AndCoins{op:124, amount:10}.toCell(), tim: null, pairm: (100000,100000) }, stringHexToSlice("200030D400000000000030D41_").appendRef(stringHexToSlice("0000007C10A"))); + run({ bod: JustInt32{value:255}.toCell(), tim: null, pairm: (90,90) }, stringHexToSlice("A000000B400000000000000B5_").appendRef(stringHexToSlice("000000FF"))); + run({ bod: JustInt32{value:510}.toCell(), tim: {op: 567, amount:9392843922}, pairm: (81923,81923) }, stringHexToSlice("C000008DD408BF6DB24A000280060000000000028007_").appendRef(stringHexToSlice("000001FE"))); + + val o4 = DifferentMix3.fromSlice(stringHexToSlice("C000008DD408BF6DB24A000280060000000000028007_").appendRef(stringHexToSlice("000001FE"))); + val o2 = DifferentMix3.fromSlice(stringHexToSlice("200030D400000000000030D41_").appendRef(stringHexToSlice("0000007C10A"))); + return ( + (o4.bod is Cell) ? o4.bod.load().value as int : -1, o4.tim, o4.pairm, + 777, + o2.bod is Cell, o2.bod is Cell, o2.tim, o2.pairm, + ); +} + +@method_id(223) +fun test_WriteWithBuilderReadWithOther() { + var b = beginCell().storeUint(55, 32).storeMaybeRef(null); + var w: WriteWithBuilder = { f1: 10, rest: b }; + return ReadWrittenWithBuilder.fromCell(w.toCell()); +} + +@method_id(224) +fun test_RestIsBuilderOrRemaining() { + var remainingB = beginCell() + .storeUint(5, 32) + .storeAddress(address("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8")) + .storeMaybeRef(JustInt32 { value: 123 }.toCell()) + .storeRef(JustAddress { addr: createAddressNone() }.toCell()); + var w: ReadWriteRest = { + f1: 60, + f2: ton("0.05"), + rest: remainingB + }; + var r: ReadRest_Remaining = w.toCell().beginParse().loadAny(); + __expect_type(r.rest, "RemainingBitsAndRefs"); + var rest = Tail224.fromSlice(r.rest); + return (r.f1, r.f2, rest.ji.value, rest.addr.getWorkchain(), rest.ref1!.load(), rest.ref2.load().addr.isNone()); +} + +@method_id(225) +fun test_MidIsBuilderOrBitsN() { + var bits40_b = beginCell().storeSlice(stringHexToSlice("0000FFFF00")); + var typedCell = ReadWriteMid { + f1: 5, + mid: bits40_b, + f3: ton("0.05") + }.toCell(); + __expect_type(typedCell, "Cell>"); + var r = ReadWriteMid.fromCell(typedCell); + var mid = r.mid as slice; + return (r.f1, mid.remainingBitsCount(), mid.loadAny(), mid.remainingBitsCount(), r.f3); +} + + +fun main() { + var t: JustInt32 = { value: 10 }; + var c = t.toCell(); + var t2 = c.load(); + var t3 = JustInt32.fromSlice(c.beginParse()); + return (t2.value, t3.value); +} + +/** +@testcase | 0 | | 10 10 +@testcase | 200 | | BC{00100000007b000001c8} +@testcase | 201 | | 123 456 +@testcase | 202 | | -1 +@testcase | 203 | | -1 +@testcase | 204 | | -1 +@testcase | 205 | | -1 +@testcase | 206 | | -1 +@testcase | 207 | | -1 +@testcase | 208 | | -1 +@testcase | 209 | | -1 +@testcase | 210 | | 123 555 46 (null) +@testcase | 211 | | (null) 132 123 +@testcase | 212 | | (null) 133 123 +@testcase | 213 | | -1 +@testcase | 214 | | -1 +@testcase | 215 | | -1 +@testcase | 216 | | -1 +@testcase | 217 | | -1 +@testcase | 218 | | -1 +@testcase | 219 | | 44 (null) (null) 46 143 +@testcase | 220 | | 0 0 (null) 99 136 137 99 1234 (null) 889129 14 +@testcase | 221 | | -1 +@testcase | 222 | | 510 567 9392843922 146 81923 81923 147 777 0 -1 (null) (null) 0 100000 100000 147 +@testcase | 223 | | 10 55 (null) +@testcase | 224 | | 60 50000000 5 9 123 -1 +@testcase | 225 | | 5 40 65535 8 50000000 + */ diff --git a/tolk-tester/tests/pack-unpack-3.tolk b/tolk-tester/tests/pack-unpack-3.tolk new file mode 100644 index 000000000..74fa1b8e9 --- /dev/null +++ b/tolk-tester/tests/pack-unpack-3.tolk @@ -0,0 +1,284 @@ + +struct EitherLeft { value: T } +struct EitherRight { value: T } +type Either = EitherLeft | EitherRight; + +@inline +fun makeExternalAddress(hash: int, len: int): address { + return beginCell().storeUint(0b01, 2).storeUint(len, 9).storeUint(hash, len).endCell().beginParse() as address; +} + + +fun slice.assertEqDeeply(self, rhs: slice): slice { + var lhs = self; + assert(lhs.bitsEqual(rhs), 400); + assert(lhs.remainingRefsCount() == rhs.remainingRefsCount(), 400); + while (lhs.remainingRefsCount()) { + lhs.loadRef().beginParse().assertEqDeeply(rhs.loadRef().beginParse()); + } + return self; +} + +@inline +fun slice.appendRef(self, refSlice: slice): slice { + return beginCell().storeSlice(self).storeRef(beginCell().storeSlice(refSlice).endCell()).endCell().beginParse(); +} + +@inline +fun generateSlice_44_with_ref45(): slice { + return generateCell_44_with_ref45().beginParse(); +} + +@inline +fun generateCell_44_with_ref45(): cell { + return beginCell().storeInt(44, 32).storeRef(beginCell().storeInt(45, 32).endCell()).endCell(); +} + + +fun run(input: TInputStruct, ans: slice) { + repeat (2) { + var s = input.toCell().beginParse(); + input = TInputStruct.fromSlice(s.assertEqDeeply(ans)); + } + input.toCell().beginParse().skipAny().assertEnd(); +} + + +/* +single_prefix32#87654321 + amount1:Grams + amount2:Grams + = MsgSinglePrefix32; +*/ + +struct(0x87654321) MsgSinglePrefix32 { + amount1: coins; + amount2: coins; +} + +/* +single_prefix48#876543211234 + amount:(Either Grams uint64) + = MsgSinglePrefix48; +*/ + +struct(0x876543211234) MsgSinglePrefix48 { + amount: coins | uint64; +} + +/* +counterIncrement#12345678 + counter_id:int8 + inc_by:int32 + = MsgCounter1; +*/ + +struct(0x12345678) CounterIncrement { + counter_id: int8; + inc_by: int32; +} + +/* +counterDecrement#23456789 + counter_id:int8 + dec_by:int32 + = MsgCounter1; +*/ + +struct(0x23456789) CounterDecrement { + counter_id: int8; + dec_by: int32; +} + +/* +counterReset0#34567890 + counter_id:int8 + = MsgCounter1; +*/ + +struct(0x34567890) CounterReset0 { + counter_id: int8; +} + +/* +counterResetTo#00184300 + counter_id:int8 + initial_value:int64 + = MsgCounter1; +*/ + +struct(0x00184300) CounterResetTo { + counter_id: int8; + initial_value: int64; +} + +type MsgCounter1 = + CounterIncrement | + CounterDecrement | + CounterReset0 | + CounterResetTo | +; + +/* +bodyPayload1$001 + should_forward:Bool + n_times:int32 + content:Cell + = BodyPayload; +*/ + +struct(0b001) BodyPayload1 { + should_forward: bool; + n_times: int32; + content: RemainingBitsAndRefs; +} + +/* +bodyPayload2$01 + master_id:int8 + owner_address:MsgAddressInt + = BodyPayload; +*/ + +struct(0b01) BodyPayload2 { + master_id: int8; + owner_address: address; +} + +type BodyPayload = BodyPayload1 | BodyPayload2; + +/* +sayHiAndGoodbye#89 + dest_addr:(Maybe MsgAddressInt) + body:BodyPayload + = MsgExternal1; +*/ + +struct(0x89) SayHiAndGoodbye { + dest_addr: address?; + body: BodyPayload; +} + +/* +sayStoreInChain#0013 + in_masterchain:Bool + contents:^BodyPayload + = MsgExternal1; + */ + +struct(0x0013) SayStoreInChain { + in_masterchain: bool; + contents: Cell; +} + +type MsgExternal1 = SayHiAndGoodbye | SayStoreInChain; + +/* +transferParams1#794 + dest_int:MsgAddressInt + amount:Grams + dest_ext:MsgAddressExt + = TransferParams; +*/ + +struct(0x794) TransferParams1 { + dest_int: address; + amount: coins; + dest_ext: address; +} + +/* +transferParams2#9 + intVector:(Both int32 (Both (Maybe Grams) uint64)) + needs_more:^Bit + = TransferParams; + */ + +struct(0x9) TransferParams2 { + intVector: (int32, coins?, uint64); + needs_more: Cell; +} + +type TransferParams = TransferParams1 | TransferParams2; + +/* +_#FB3701FF + params:(Either TransferParams ^TransferParams) + = MsgTransfer; + */ + +struct(0xFB3701FF) MsgTransfer { + params: Either>; +} + + + +// --------------------------------------------- + + +@method_id(201) +fun test_MsgSinglePrefix32() { + run({ amount1: 80, amount2: 800000000 }, stringHexToSlice("8765432115042FAF0800")); + + return MsgSinglePrefix32.fromSlice(stringHexToSlice("8765432115042FAF0800")); +} + +@method_id(202) +fun test_MsgSinglePrefix48() { + run({ amount: 80 as uint64 }, stringHexToSlice("87654321123480000000000000284_")); + run({ amount: 800000000 as coins }, stringHexToSlice("876543211234217D784004_")); + + var o = MsgSinglePrefix48.fromSlice(stringHexToSlice("876543211234217D784004_")); + return (o, 777, o.amount is coins, o.amount is uint64); +} + +@method_id(203) +fun test_MsgCounter1() { + run(CounterIncrement{ counter_id: 123, inc_by: 78 }, stringHexToSlice("123456787B0000004E")); + run(CounterDecrement{ counter_id: 0, dec_by: -38 }, stringHexToSlice("2345678900FFFFFFDA")); + run(CounterReset0{ counter_id: 0 }, stringHexToSlice("3456789000")); + run(CounterResetTo{ counter_id: 0, initial_value: 29874329774732 }, stringHexToSlice("001843000000001B2BA8D06A8C")); + + val o = MsgCounter1.fromSlice(stringHexToSlice("3456789000")); + return (o, 777, o is CounterReset0, o is CounterIncrement); +} + +@method_id(204) +fun test_MsgExternal1() { + run(SayHiAndGoodbye{ dest_addr: null, body: BodyPayload2 { master_id: 10, owner_address: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c") } }, stringHexToSlice("892150000000000000000000000000000000000000000000000000000000000000000002_")); + run(SayHiAndGoodbye{ dest_addr: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"), body: BodyPayload1 { should_forward: false, n_times: 85, content: generateSlice_44_with_ref45() } }, stringHexToSlice("89C0000000000000000000000000000000000000000000000000000000000000000002000000550000002C").appendRef(stringHexToSlice("0000002D"))); + run(SayHiAndGoodbye{ dest_addr: address("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi"), body: BodyPayload2 { master_id: -5, owner_address: address("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi") } }, stringHexToSlice("89CFF28E8033DB1467CACEA8B158F0F61E682DE06E8A5947504C904F1F703D2BE4D9E7EE7F9474019ED8A33E5675458AC787B0F3416F037452CA3A82648278FB81E95F26CF4_")); + run(SayStoreInChain{ in_masterchain: true, contents: { tvmCell: BodyPayload1{ should_forward: true, n_times: 20, content: generateSlice_44_with_ref45() }.toCell()} }, stringHexToSlice("0013C_").appendRef(stringHexToSlice("3000000140000002C").appendRef(stringHexToSlice("0000002D")))); + run(SayStoreInChain{ in_masterchain: false, contents: { tvmCell: BodyPayload2{ master_id: 37, owner_address: address(("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c")) }.toCell()} }, stringHexToSlice("00134_").appendRef(stringHexToSlice("4960000000000000000000000000000000000000000000000000000000000000000004_"))); + + val o = MsgExternal1.fromSlice(stringHexToSlice("00134_").appendRef(stringHexToSlice("4960000000000000000000000000000000000000000000000000000000000000000004_"))); + assert(o is SayStoreInChain, 400); + val contents = o.contents.load(); + return (contents is BodyPayload2 && contents.owner_address == address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"), contents is BodyPayload1, contents is BodyPayload2); +} + +@method_id(205) +fun test_MsgTransfer() { + run({ params: EitherLeft { value: TransferParams1 { dest_int: address("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi"), amount: 80000000, dest_ext: makeExternalAddress(1234,80) } } }, stringHexToSlice("FB3701FF3CA4FF28E8033DB1467CACEA8B158F0F61E682DE06E8A5947504C904F1F703D2BE4D9E404C4B4004A0000000000000000009A5_")); + run({ params: EitherLeft { value: TransferParams2 { intVector: (123, 1234567890123456, 1234567890123456), needs_more: {tvmCell: beginCell().storeBool(true).endCell()} } } }, stringHexToSlice("FB3701FF48000003DDC118B54F22AEB0000118B54F22AEB02_").appendRef(stringHexToSlice("C_"))); + run({ params: EitherRight { value: { tvmCell: TransferParams1{ dest_int: address("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"), amount: 80000000, dest_ext: makeExternalAddress(1234,70) }.toCell()} } }, stringHexToSlice("FB3701FFC_").appendRef(stringHexToSlice("7948000000000000000000000000000000000000000000000000000000000000000000809896800918000000000000004D2"))); + run({ params: EitherRight { value: { tvmCell: TransferParams2{ intVector: (123, null, 0), needs_more: {tvmCell: beginCell().storeBool(false).endCell()} }.toCell()} } }, stringHexToSlice("FB3701FFC_").appendRef(stringHexToSlice("90000007B00000000000000004_").appendRef(stringHexToSlice("4_")))); + + val o = MsgTransfer.fromSlice(stringHexToSlice("FB3701FFC_").appendRef(stringHexToSlice("90000007B00000000000000004_").appendRef(stringHexToSlice("4_")))); + return ( + o.params is EitherLeft, o.params is EitherRight, + o.params is EitherRight && TransferParams.fromCell(o.params.value.tvmCell) is TransferParams1, + o.params is EitherRight && TransferParams.fromCell(o.params.value.tvmCell) is TransferParams2, + ); +} + + +fun main() {} + +/** +@testcase | 201 | | 80 800000000 +@testcase | 202 | | 800000000 17 777 -1 0 +@testcase | 203 | | (null) 0 131 777 -1 0 +@testcase | 204 | | -1 0 -1 +@testcase | 205 | | 0 -1 0 -1 + */ diff --git a/tolk-tester/tests/pack-unpack-4.tolk b/tolk-tester/tests/pack-unpack-4.tolk new file mode 100644 index 000000000..4839032dc --- /dev/null +++ b/tolk-tester/tests/pack-unpack-4.tolk @@ -0,0 +1,162 @@ + +fun slice.assertEqDeeply(self, rhs: slice): slice { + var lhs = self; + assert(lhs.bitsEqual(rhs), 400); + assert(lhs.remainingRefsCount() == rhs.remainingRefsCount(), 400); + while (lhs.remainingRefsCount()) { + lhs.loadRef().beginParse().assertEqDeeply(rhs.loadRef().beginParse()); + } + return self; +} + + +fun run(input: TInputStruct, ans: slice) { + repeat (2) { + var s = input.toCell().beginParse(); + input = TInputStruct.fromSlice(s.assertEqDeeply(ans)); + } + input.toCell().beginParse().skipAny().assertEnd(); +} + + +type Union_8_16_32 = int8 | int16 | int32; + +fun ans101_8(): slice asm "b{0000001111} PUSHSLICE"; +fun ans101_16(): slice asm "b{010000000000001111} PUSHSLICE"; +fun ans101_32(): slice asm "b{1000000000000000000000000000001111} PUSHSLICE"; + +@method_id(101) +fun test1() { + run(15 as int8, ans101_8()); + run(15 as int16, ans101_16()); + run(15 as int32, ans101_32()); + + return 0; +} + + +type Union_8_16_32_n = int8 | null | int16 | int32; + +fun ans102n_8(): slice asm "b{0100001111} PUSHSLICE"; +fun ans102n_16(): slice asm "b{100000000000001111} PUSHSLICE"; +fun ans102n_32(): slice asm "b{1100000000000000000000000000001111} PUSHSLICE"; +fun ans102n_null(): slice asm "b{00} PUSHSLICE"; + +@method_id(102) +fun test2() { + run(15 as int8, ans102n_8()); + run(15 as int16, ans102n_16()); + run(15 as int32, ans102n_32()); + run(null, ans102n_null()); + + return 0; +} + + +struct Test4_8 { a: int8 } +struct Test4_16 { a: int16 } +struct Test4_32 { a: int32 } + +type UnionStructs_8_16_32 = int8 | Test4_16 | Test4_32; +type UnionStructs_8_16_32_n = Test4_8 | null | int16 | int32; + +fun ans104_8(): slice asm "b{0000001111} PUSHSLICE"; +fun ans104_16(): slice asm "b{010000000000001111} PUSHSLICE"; +fun ans104_32(): slice asm "b{1000000000000000000000000000001111} PUSHSLICE"; + +fun ans104n_8(): slice asm "b{0100001111} PUSHSLICE"; +fun ans104n_16(): slice asm "b{100000000000001111} PUSHSLICE"; +fun ans104n_32(): slice asm "b{1100000000000000000000000000001111} PUSHSLICE"; +fun ans104n_null(): slice asm "b{00} PUSHSLICE"; + +@method_id(104) +fun test4() { + // when mixing primitives and structs with no opcode, it's like mixing primitives + run(15, ans104_8()); + run(Test4_16{a:15}, ans104_16()); + run(Test4_32{a:15}, ans104_32()); + // with null — the same behavior + run({a:15}, ans104n_8()); + run(15 as int16, ans104n_16()); + run(15 as int32, ans104n_32()); + run(null, ans104n_null()); + + return 0; +} + + +type U105 = int8 | int32 | int64; // auto-prefixes 0b00 0b01 0b10 + +fun invalid105_slice(): slice asm "b{1100} PUSHSLICE"; // prefix 0b11 doesn't exist + +@method_id(105) +fun test5() { + try { + var u = U105.fromSlice(invalid105_slice()); + return (u is int8) ? 8 : -8; + } catch (excode) { + return excode; + } +} + + +type U106 = int8 | int16 | int32 | int64; // exhaustive prefixes, checked via codegen + +fun s106_int16(): slice asm "b{010000000000001111} PUSHSLICE"; + +@method_id(106) +fun test6(): int16 { + var u = U106.fromSlice(s106_int16()); + if (u is int16) { return u; } + else { return -1; } +} + + +fun main() {} + +/** +@testcase | 101 | | 0 +@testcase | 102 | | 0 +@testcase | 104 | | 0 +@testcase | 105 | | 9 +@testcase | 106 | | 15 + +@fif_codegen +""" + test6 PROC:<{ // + b{010000000000001111} PUSHSLICE // s + b{00} SDBEGINSQ // s '6 + IF:<{ // s + 8 LDI // '10 s + DROP // '10 + 42 PUSHINT // 'USlot1 'UTag=42 + }>ELSE<{ // s + b{01} SDBEGINSQ // s '6 + IF:<{ // s + 16 LDI // '15 s + DROP // '15 + 44 PUSHINT // 'USlot1 'UTag=44 + }>ELSE<{ // s + b{10} SDBEGINSQ // s '6 + IF:<{ // s + 32 LDI // '20 s + DROP // '20 + 46 PUSHINT // 'USlot1 'UTag=46 + }>ELSE<{ // s + b{11} SDBEGINSQ // s '6 + DROP // s + 64 LDI // '25 s + DROP // '25 + 48 PUSHINT // 'USlot1 'UTag=48 + }> + }> + }> // u.USlot1 u.UTag + 44 EQINT // u.USlot1 '27 + IFJMP:<{ // u.USlot1 + }> // u.USlot1 + DROP // + -1 PUSHINT // '29=-1 + }> +""" + */ + diff --git a/tolk-tester/tests/pack-unpack-5.tolk b/tolk-tester/tests/pack-unpack-5.tolk new file mode 100644 index 000000000..068d031f2 --- /dev/null +++ b/tolk-tester/tests/pack-unpack-5.tolk @@ -0,0 +1,183 @@ +struct JustInt32 { + value: int32; +} + +type MaybeInt32 = int32?; +type MaybeCell = cell?; +type Int16Or32 = int16 | int32; + +struct JustMaybeInt32 { + value: MaybeInt32; +} + + +@method_id(101) +fun test1() { + return ( + int32.estimatePackSize(), + uint64.estimatePackSize(), + int1.estimatePackSize(), + bool.estimatePackSize(), + RemainingBitsAndRefs.estimatePackSize(), + coins.estimatePackSize(), + bits6.estimatePackSize(), + bytes8.estimatePackSize(), + ); +} + +@method_id(102) +fun test2() { + return ( + JustInt32.estimatePackSize(), + JustMaybeInt32.estimatePackSize(), + MaybeInt32.estimatePackSize(), + Int16Or32.estimatePackSize(), + ); +} + +struct Test3_1 { + f1: JustInt32, + f2: JustMaybeInt32, + f3: JustMaybeInt32, + f4: JustInt32 | uint100, +} + +struct Test3_2 { + f1: Test3_1; + f2: Test3_1?; + f3: null | Test3_1 | null; +} + +@method_id(103) +fun test3() { + return (Test3_1.estimatePackSize(), Test3_2.estimatePackSize()); +} + +struct Test4_1 { + f1: address; + f2: address; +} + +struct Test4_2 { + f1: address?; + f2: address?; +} + +@overflow1023_policy("suppress") +struct Test4_3 { + f: (Test4_1, Test4_2); +} + +@method_id(104) +fun test4() { + return (Test4_1.estimatePackSize(), Test4_2.estimatePackSize(), Test4_3.estimatePackSize()); +} + +struct Test5_1 { + f1: cell; + f2: cell?; + f3: Cell; + f4: Cell?; +} + +struct Test5_2 { + f1: Cell?; + f2: Cell?; + f3: Cell>?; + f4: Cell?; + f5: Cell?; + rest: RemainingBitsAndRefs; +} + +struct Test5_3 { + f1: int8 | Cell; + f2: bytes2 | Cell; + f3: Cell | cell; + f4: Cell | coins; +} + +@method_id(105) +fun test5() { + return (Test5_1.estimatePackSize(), Test5_2.estimatePackSize(), Test5_3.estimatePackSize()); +} + +struct(0x00112233) Test6_1 { + f1: int32; +} + +struct(0b0010) Test6_2 { + f1: int32?; + f2: Cell; +} + +type Test6_or = Test6_1 | Test6_2; + +@method_id(106) +fun test6() { + return (Test6_1.estimatePackSize(), Test6_2.estimatePackSize(), Test6_or.estimatePackSize()); +} + +struct(0x1020) Test7_1 { } +struct(0x1030) Test7_2 { } +struct(0x1040) Test7_3 { } + +type Test7_or = Test7_1 | Test7_2 | Test7_3 |; + +@method_id(107) +fun test7() { + assert((Test7_1{} as Test7_or).toCell().beginParse().remainingBitsCount() == Test7_or.estimatePackSize().0, 400); + return (Test7_1.estimatePackSize(), Test7_or.estimatePackSize()); +} + +struct(0x10) Inner8_1 { + ea: address; +} +struct(0b1) CellInner8_1 { + ref: Cell; +} +struct Inner8_2 { + t: (bits32, int1, coins?); +} + +@overflow1023_policy("suppress") +struct Test8 { + f1: Inner8_1; + f2: Inner8_2; + f3: Inner8_1?; + f4: Inner8_2?; + f5: Inner8_1 | CellInner8_1; +} + +@method_id(108) +fun test8() { + return (Inner8_1.estimatePackSize(), Inner8_2.estimatePackSize(), Test8.estimatePackSize()); +} + +struct Test9_bits2 { f: bits2; } +struct Test9_bits4 { f: bits4; } + +type Test9_f1 = int32 | int64 | int128; // auto-generated 2-bit prefix +type Test9_f2 = int32 | Inner8_2; // auto-generated 1-bit prefix (Either) +type Test9_f3 = bits1 | Test9_bits2 | bits3 | bits4 | bits5; // auto-generated 3-bit prefix +type Test9_f4 = bits1 | Test9_bits2 | bits3 | Test9_bits4?; // auto-generated 3-bit prefix + +@method_id(109) +fun test9() { + return (Test9_f1.estimatePackSize(), Test9_f2.estimatePackSize(), Test9_f3.estimatePackSize(), Test9_f4.estimatePackSize()); +} + +fun main() { + __expect_type(int8.estimatePackSize(), "[int, int, int, int]"); +} + +/** +@testcase | 101 | | [ 32 32 0 0 ] [ 64 64 0 0 ] [ 1 1 0 0 ] [ 1 1 0 0 ] [ 0 0 0 0 ] [ 4 124 0 0 ] [ 6 6 0 0 ] [ 64 64 0 0 ] +@testcase | 102 | | [ 32 32 0 0 ] [ 1 33 0 0 ] [ 1 33 0 0 ] [ 17 33 0 0 ] +@testcase | 103 | | [ 67 199 0 0 ] [ 69 599 0 0 ] +@testcase | 104 | | [ 4 534 0 0 ] [ 2 536 0 0 ] [ 6 1070 0 0 ] +@testcase | 105 | | [ 2 2 2 4 ] [ 5 5 0 5 ] [ 4 152 1 4 ] +@testcase | 106 | | [ 64 64 0 0 ] [ 5 37 1 1 ] [ 5 65 0 1 ] +@testcase | 107 | | [ 16 16 0 0 ] [ 16 16 0 0 ] +@testcase | 108 | | [ 10 275 0 0 ] [ 34 158 0 0 ] [ 47 1143 0 1 ] +@testcase | 109 | | [ 34 130 0 0 ] [ 33 159 0 0 ] [ 4 8 0 0 ] [ 3 7 0 0 ] + */ diff --git a/tolk-tester/tests/some-tests-2.tolk b/tolk-tester/tests/some-tests-2.tolk index b3b756b19..592cf96ef 100644 --- a/tolk-tester/tests/some-tests-2.tolk +++ b/tolk-tester/tests/some-tests-2.tolk @@ -89,6 +89,15 @@ fun test95() { return (cur, next); } +struct Point { + x: int, y: int +} + +@method_id(96) +fun test96(p: Point) { + debug.print(p); // works ok with non-1 stack width, checked via codegen +} + /** method_id | in | out @testcase | 0 | 101 15 | 100 1 @@ -153,4 +162,11 @@ fun test95() { next GETGLOB // g_cur g_next }> """ + +@fif_codegen +""" + test96 PROC:<{ + s1 DUMP s0 DUMP 2 BLKDROP + }> +""" */ diff --git a/tolk/CMakeLists.txt b/tolk/CMakeLists.txt index c6e992c27..7aca83688 100644 --- a/tolk/CMakeLists.txt +++ b/tolk/CMakeLists.txt @@ -6,6 +6,8 @@ set(TOLK_SOURCE ast.cpp ast-from-tokens.cpp constant-evaluator.cpp + pack-unpack-api.cpp + pack-unpack-serializers.cpp pipe-discover-parse-sources.cpp pipe-register-symbols.cpp pipe-resolve-identifiers.cpp @@ -16,6 +18,7 @@ set(TOLK_SOURCE pipe-refine-lvalue-for-mutate.cpp pipe-check-rvalue-lvalue.cpp pipe-check-pure-impure.cpp + pipe-check-serialized-fields.cpp pipe-constant-folding.cpp pipe-optimize-boolean-expr.cpp pipe-ast-to-legacy.cpp diff --git a/tolk/abscode.cpp b/tolk/abscode.cpp index 02215d884..462aad7b2 100644 --- a/tolk/abscode.cpp +++ b/tolk/abscode.cpp @@ -436,4 +436,15 @@ std::vector CodeBlob::create_var(TypePtr var_type, SrcLocation loc, s return ir_idx; } +var_idx_t CodeBlob::create_int(SrcLocation loc, int64_t value, const char* desc) { + vars.emplace_back(var_cnt, TypeDataInt::create(), std::string{}, loc); +#ifdef TOLK_DEBUG + vars.back().desc = desc; +#endif + var_idx_t ir_int = var_cnt; + var_cnt++; + emplace_back(loc, Op::_IntConst, std::vector{ir_int}, td::make_refint(value)); + return ir_int; +} + } // namespace tolk diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 28f0ba182..6f4b53b79 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -226,10 +226,6 @@ static AnyTypeV parse_type_nullable(Lexer& lex) { } static AnyTypeV parse_type_expression(Lexer& lex) { - if (lex.tok() == tok_bitwise_or) { // `type T = | T1 | T2 | ...` (each per line) (like in TypeScript) - lex.next(); - } - AnyTypeV result = parse_type_nullable(lex); if (lex.tok() == tok_bitwise_or) { // `int | slice`, `Pair2 | (Pair3 | null)` @@ -237,6 +233,9 @@ static AnyTypeV parse_type_expression(Lexer& lex) { items.emplace_back(result); while (lex.tok() == tok_bitwise_or) { lex.next(); + if (lex.tok() == tok_clpar || lex.tok() == tok_clbracket || lex.tok() == tok_semicolon) { + break; // allow trailing `|` (not leading, like in TypeScript, because of tree-sitter) + } items.emplace_back(parse_type_nullable(lex)); } result = createV(result->loc, std::move(items)); @@ -336,9 +335,6 @@ static AnyV parse_parameter(Lexer& lex, AnyTypeV self_type) { } static AnyV parse_global_var_declaration(Lexer& lex, const std::vector>& annotations) { - if (!annotations.empty()) { - lex.error("@annotations are not applicable to global var declaration"); - } SrcLocation loc = lex.cur_location(); lex.expect(tok_global, "`global`"); lex.check(tok_identifier, "global variable name"); @@ -353,13 +349,21 @@ static AnyV parse_global_var_declaration(Lexer& lex, const std::vectorkind) { + case AnnotationKind::deprecated: + case AnnotationKind::custom: + break; + default: + v_annotation->error("this annotation is not applicable to global"); + } + } + return createV(loc, v_ident, declared_type); } static AnyV parse_constant_declaration(Lexer& lex, const std::vector>& annotations) { - if (!annotations.empty()) { - lex.error("@annotations are not applicable to global var declaration"); - } SrcLocation loc = lex.cur_location(); lex.expect(tok_const, "`const`"); lex.check(tok_identifier, "constant name"); @@ -376,13 +380,21 @@ static AnyV parse_constant_declaration(Lexer& lex, const std::vectorkind) { + case AnnotationKind::deprecated: + case AnnotationKind::custom: + break; + default: + v_annotation->error("this annotation is not applicable to constant"); + } + } + return createV(loc, v_ident, declared_type, init_value); } static AnyV parse_type_alias_declaration(Lexer& lex, const std::vector>& annotations) { - if (!annotations.empty()) { - lex.error("@annotations are not applicable to type alias declaration"); - } SrcLocation loc = lex.cur_location(); lex.expect(tok_type, "`type`"); lex.check(tok_identifier, "type name"); @@ -397,6 +409,17 @@ static AnyV parse_type_alias_declaration(Lexer& lex, const std::vectorkind) { + case AnnotationKind::deprecated: + case AnnotationKind::custom: + break; + default: + v_annotation->error("this annotation is not applicable to type alias"); + } + } + return createV(loc, v_ident, genericsT_list, underlying_type); } @@ -1303,6 +1326,7 @@ static V parse_annotation(Lexer& lex) { v_arg = createV(loc, {}); break; case AnnotationKind::deprecated: + case AnnotationKind::custom: // allowed with and without arguments; it's IDE-only, the compiler doesn't analyze @deprecated break; case AnnotationKind::method_id: @@ -1310,9 +1334,15 @@ static V parse_annotation(Lexer& lex) { throw ParseError(loc, "expecting `(number)` after " + static_cast(name)); } break; + case AnnotationKind::overflow1023_policy: { + if (!v_arg || v_arg->size() != 1 || v_arg->get_item(0)->kind != ast_string_const) { + throw ParseError(loc, "expecting `(\"policy_name\")` after " + static_cast(name)); + } + break; + } } - return createV(loc, kind, v_arg); + return createV(loc, name, kind, v_arg); } static AnyV parse_function_declaration(Lexer& lex, const std::vector>& annotations) { @@ -1436,7 +1466,7 @@ static AnyV parse_function_declaration(Lexer& lex, const std::vector parse_struct_body(Lexer& lex) { static AnyV parse_struct_declaration(Lexer& lex, const std::vector>& annotations) { SrcLocation loc = lex.cur_location(); - if (!annotations.empty()) { - lex.error("@annotations are not applicable to type alias declaration"); - } lex.expect(tok_struct, "`struct`"); + + AnyExprV opcode = nullptr; + if (lex.tok() == tok_oppar) { // struct(0x0012) CounterIncrement + lex.next(); + lex.check(tok_int_const, "opcode `0x...` or `0b...`"); + std::string_view opcode_str = lex.cur_str(); + if (!opcode_str.starts_with("0x") && !opcode_str.starts_with("0b")) { + lex.unexpected("opcode `0x...` or `0b...`"); + } + opcode = createV(lex.cur_location(), parse_tok_int_const(opcode_str), opcode_str); + lex.next(); + lex.expect(tok_clpar, "`)`"); + } else { + opcode = createV(lex.cur_location()); + } + lex.check(tok_identifier, "identifier"); auto v_ident = createV(lex.cur_location(), lex.cur_str()); lex.next(); @@ -1499,7 +1542,27 @@ static AnyV parse_struct_declaration(Lexer& lex, const std::vector(loc, v_ident, genericsT_list, parse_struct_body(lex)); + StructData::Overflow1023Policy overflow1023_policy = StructData::Overflow1023Policy::not_specified; + for (auto v_annotation : annotations) { + switch (v_annotation->kind) { + case AnnotationKind::deprecated: + case AnnotationKind::custom: + break; + case AnnotationKind::overflow1023_policy: { + std::string_view str = v_annotation->get_arg()->get_item(0)->as()->str_val; + if (str == "suppress") { + overflow1023_policy = StructData::Overflow1023Policy::suppress; + } else { + v_annotation->error("incorrect value for " + static_cast(v_annotation->name)); + } + break; + } + default: + v_annotation->error("this annotation is not applicable to struct"); + } + } + + return createV(loc, v_ident, genericsT_list, overflow1023_policy, opcode, parse_struct_body(lex)); } static AnyV parse_tolk_required_version(Lexer& lex) { diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index d72513906..73ada4100 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -345,6 +345,8 @@ class ASTReplicator final { v_orig->loc, new_name_ident, clone(v_orig->genericsT_list), + v_orig->overflow1023_policy, + v_orig->has_opcode() ? static_cast(clone(v_orig->get_opcode())) : createV(v_orig->loc), clone(v_orig->get_struct_body()) ); } diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h index 8ced76a0b..395f95681 100644 --- a/tolk/ast-stringifier.h +++ b/tolk/ast-stringifier.h @@ -107,16 +107,6 @@ class ASTStringifier final : public ASTVisitor { static_assert(std::size(name_pairs) == ast_tolk_file + 1, "name_pairs needs to be updated"); - constexpr static std::pair annotation_kinds[] = { - {AnnotationKind::inline_simple, "@inline"}, - {AnnotationKind::inline_ref, "@inline_ref"}, - {AnnotationKind::method_id, "@method_id"}, - {AnnotationKind::pure, "@pure"}, - {AnnotationKind::deprecated, "@deprecated"}, - }; - - static_assert(std::size(annotation_kinds) == static_cast(AnnotationKind::unknown), "annotation_kinds needs to be updated"); - template constexpr static const char* ast_node_kind_to_string() { return name_pairs[node_kind].second; @@ -205,7 +195,7 @@ class ASTStringifier final : public ASTVisitor { case ast_if_statement: return v->as()->is_ifnot ? "ifnot" : ""; case ast_annotation: - return annotation_kinds[static_cast(v->as()->kind)].second; + return static_cast(v->as()->name); case ast_parameter: return static_cast(v->as()->param_name) + ": " + ast_type_node_to_string(v->as()->type_node); case ast_function_declaration: { diff --git a/tolk/ast.cpp b/tolk/ast.cpp index 6e725b781..59a18a163 100644 --- a/tolk/ast.cpp +++ b/tolk/ast.cpp @@ -66,6 +66,12 @@ AnnotationKind Vertex::parse_kind(std::string_view name) { if (name == "@deprecated") { return AnnotationKind::deprecated; } + if (name == "@custom") { + return AnnotationKind::custom; + } + if (name == "@overflow1023_policy") { + return AnnotationKind::overflow1023_policy; + } return AnnotationKind::unknown; } diff --git a/tolk/ast.h b/tolk/ast.h index 0770d9b84..96fc96937 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -141,6 +141,8 @@ enum class AnnotationKind { method_id, pure, deprecated, + custom, + overflow1023_policy, unknown, }; @@ -1208,15 +1210,16 @@ template<> // ast_annotation is @annotation above a declaration // example: `@pure fun ...` struct Vertex final : ASTOtherVararg { + std::string_view name; AnnotationKind kind; auto get_arg() const { return children.at(0)->as(); } static AnnotationKind parse_kind(std::string_view name); - Vertex(SrcLocation loc, AnnotationKind kind, V arg_probably_empty) + Vertex(SrcLocation loc, std::string_view name, AnnotationKind kind, V arg_probably_empty) : ASTOtherVararg(ast_annotation, loc, {arg_probably_empty}) - , kind(kind) {} + , name(name), kind(kind) {} }; template<> @@ -1336,20 +1339,24 @@ struct Vertex final : ASTOtherVararg { template<> // ast_struct_declaration is declaring a struct with fields (each having declared_type), like interfaces in TypeScript // example: `struct Storage { owner: User; validUntil: int }` +// example: `struct(0x0012) CounterIncrement { byValue: int32; }` (0x0012 is opcode, len 16) // currently, Tolk doesn't have "implements" or whatever, so struct declaration contains only body struct Vertex final : ASTOtherVararg { - StructPtr struct_ref = nullptr; // filled after register + StructPtr struct_ref = nullptr; // filled after register V genericsT_list; // exists for `Wrapper`; otherwise, nullptr + StructData::Overflow1023Policy overflow1023_policy; auto get_identifier() const { return children.at(0)->as(); } - auto get_struct_body() const { return children.at(1)->as(); } + bool has_opcode() const { return children.at(1)->kind != ast_empty_expression; } + auto get_opcode() const { return children.at(1)->as(); } + auto get_struct_body() const { return children.at(2)->as(); } Vertex* mutate() const { return const_cast(this); } void assign_struct_ref(StructPtr struct_ref); - Vertex(SrcLocation loc, V name_identifier, V genericsT_list, V struct_body) - : ASTOtherVararg(ast_struct_declaration, loc, {name_identifier, struct_body}) - , genericsT_list(genericsT_list) {} + Vertex(SrcLocation loc, V name_identifier, V genericsT_list, StructData::Overflow1023Policy overflow1023_policy, AnyExprV opcode, V struct_body) + : ASTOtherVararg(ast_struct_declaration, loc, {name_identifier, opcode, struct_body}) + , genericsT_list(genericsT_list), overflow1023_policy(overflow1023_policy) {} }; template<> diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 7bed41a56..ebcd2de92 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1022,9 +1022,9 @@ static AsmOp compile_store_int(std::vector& res, std::vector auto& z = args[2]; if (z.is_int_const() && z.int_const > 0 && z.int_const <= 256) { z.unused(); - return exec_arg_op(loc, "ST"s + (sgnd ? 'I' : 'U'), z.int_const, 2, 1); + return exec_arg_op(loc, sgnd? "STI" : "STU", z.int_const, 2, 1); } - return exec_op(loc, "ST"s + (sgnd ? "IX" : "UX"), 3, 1); + return exec_op(loc, sgnd ? "STIX" : "STUX", 3, 1); } // fun slice.loadBits (mutate self, len: int): self asm(s len -> 1 0) "LDSLICEX" @@ -1043,6 +1043,22 @@ static AsmOp compile_fetch_slice(std::vector& res, std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(args.size() == 3 && res.size() == 2); + auto& prefix = args[1]; + auto& prefix_len = args[2]; + if (prefix.is_int_const() && prefix.int_const >= 0 && prefix.int_const->signed_fits_bits(50) && + prefix_len.is_int_const() && prefix_len.int_const > 0 && prefix_len.int_const->signed_fits_bits(16)) { + prefix.unused(); + prefix_len.unused(); + StructData::PackOpcode opcode(prefix.int_const->to_long(), static_cast(prefix_len.int_const->to_long())); + return AsmOp::Custom(loc, opcode.format_as_slice() + " SDBEGINSQ", 0, 1); + } + throw ParseError(loc, "slice.tryStripPrefix can be used only with constant arguments"); +} + // fun tuple.get(t: tuple, index: int): X asm "INDEXVAR"; static AsmOp compile_tuple_get(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(args.size() == 2 && res.size() == 1); @@ -1076,8 +1092,31 @@ static AsmOp compile_strdump(std::vector&, std::vector&, Src } // fun debug.print(x: T): void; -static AsmOp compile_debug_print_to_string(std::vector&, std::vector&, SrcLocation loc) { - return AsmOp::Custom(loc, "s0 DUMP DROP", 1, 1); +static AsmOp compile_debug_print_to_string(std::vector&, std::vector& args, SrcLocation loc) { + int n = static_cast(args.size()); + if (n == 1) { // most common case + return AsmOp::Custom(loc, "s0 DUMP DROP", 1, 1); + } + if (n > 15) { + throw ParseError(loc, "call overflow, exceeds 15 elements"); + } + std::string cmd; + for (int i = n - 1; i >= 0; --i) { + cmd += "s" + std::to_string(i) + " DUMP "; + } + cmd += std::to_string(n); + cmd += " BLKDROP"; + return AsmOp::Custom(loc, cmd, n, n); +} + +// fun T.__toTuple(self): void; (T can be any number of slots, it works for structs and tensors) +static AsmOp compile_any_object_to_tuple(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 1); + int n = static_cast(args.size()); + if (n > 15) { + throw ParseError(loc, "call overflow, exceeds 15 elements"); + } + return exec_op(loc, std::to_string(args.size()) + " TUPLE", n, 1); } // fun ton(amount: slice): coins; ton("0.05") replaced by 50000000 at compile-time @@ -1120,6 +1159,7 @@ void define_builtins() { TypePtr typeT = TypeDataGenericT::create("T"); const GenericsDeclaration* declGenericT = new GenericsDeclaration(std::vector{{"T", nullptr}}, 0); + const GenericsDeclaration* declReceiverT = new GenericsDeclaration(std::vector{{"T", nullptr}}, 1); std::vector ParamsInt1 = {Int}; std::vector ParamsInt2 = {Int, Int}; @@ -1129,6 +1169,7 @@ void define_builtins() { // these types are defined in stdlib, currently unknown // see patch_builtins_after_stdlib_loaded() below TypePtr debug = TypeDataUnknown::create(); + TypePtr CellT = TypeDataUnknown::create(); // builtin operators // they are internally stored as functions, because at IR level, there is no difference @@ -1247,28 +1288,28 @@ void define_builtins() { // note their parameter being `unknown`: in order to `ton(1)` pass type inferring but fire a more gentle error later define_builtin_func("ton", {TypeDataUnknown::create()}, TypeDataCoins::create(), nullptr, compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); define_builtin_func("stringCrc32", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); define_builtin_func("stringCrc16", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); define_builtin_func("stringSha256", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); define_builtin_func("stringSha256_32", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); define_builtin_func("stringToBase256", {TypeDataUnknown::create()}, TypeDataInt::create(), nullptr, compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); define_builtin_func("stringHexToSlice", {TypeDataUnknown::create()}, TypeDataSlice::create(), nullptr, compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); define_builtin_func("address", {TypeDataUnknown::create()}, TypeDataAddress::create(), nullptr, compile_time_only_function, - FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeOnly); + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal); // functions from stdlib marked as `builtin`, implemented at compiler level for optimizations // (for example, `loadInt(1)` is `1 LDI`, but `loadInt(n)` for non-constant requires it be on a stack and `LDIX`) @@ -1305,6 +1346,9 @@ void define_builtins() { define_builtin_method("slice.preloadBits", Slice, ParamsSliceInt, Slice, nullptr, std::bind(compile_fetch_slice, _1, _2, _3, false), FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); + define_builtin_method("slice.tryStripPrefix", Slice, {Slice, Int, Int}, Bool, nullptr, + compile_slice_sdbeginsq, + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf); define_builtin_method("builder.storeInt", Builder, {Builder, Int, Int}, Unit, nullptr, std::bind(compile_store_int, _1, _2, _3, true), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf, @@ -1321,7 +1365,7 @@ void define_builtins() { FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf); define_builtin_method("debug.print", debug, {typeT}, Unit, declGenericT, compile_debug_print_to_string, - 0); + FunctionData::flagAllowAnyWidthT); define_builtin_method("debug.printString", debug, {typeT}, Unit, declGenericT, compile_strdump, 0); @@ -1329,12 +1373,42 @@ void define_builtins() { compile_dumpstk, 0); + // serialization/deserialization methods to/from cells (or, more low-level, slices/builders) + // they work with structs (or, more low-level, with arbitrary types) + define_builtin_method("T.toCell", typeT, {typeT}, CellT, declReceiverT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("T.fromCell", typeT, {TypeDataCell::create()}, typeT, declReceiverT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); + define_builtin_method("T.fromSlice", typeT, {Slice}, typeT, declReceiverT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); + define_builtin_method("T.estimatePackSize", typeT, {}, TypeDataBrackets::create({TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create()}), declReceiverT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); + define_builtin_method("Cell.load", CellT, {CellT}, typeT, declReceiverT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); + define_builtin_method("slice.loadAny", Slice, {Slice}, typeT, declGenericT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + define_builtin_method("slice.skipAny", Slice, {Slice}, Slice, declGenericT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + define_builtin_method("builder.storeAny", Builder, {Builder, typeT}, Builder, declGenericT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + // functions not presented in stdlib at all // used in tolk-tester to check/expose internal compiler state // each of them is handled in a special way, search by its name define_builtin_func("__expect_type", {TypeDataUnknown::create(), Slice}, Unit, nullptr, compile_expect_type, FunctionData::flagMarkedAsPure); + define_builtin_method("T.__toTuple", typeT, {typeT}, TypeDataTuple::create(), declReceiverT, + compile_any_object_to_tuple, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); } // there are some built-in functions that operate on types declared in stdlib (like Cell) @@ -1342,12 +1416,20 @@ void define_builtins() { // after all files have been loaded, symbols have been registered, and aliases exist, // we patch that earlier registered built-in functions providing types that now exist void patch_builtins_after_stdlib_loaded() { + TypePtr typeT = TypeDataGenericT::create("T"); StructPtr struct_debug = lookup_global_symbol("debug")->try_as(); TypePtr debug = TypeDataStruct::create(struct_debug); lookup_function("debug.print")->mutate()->receiver_type = debug; lookup_function("debug.printString")->mutate()->receiver_type = debug; lookup_function("debug.dumpStack")->mutate()->receiver_type = debug; + + StructPtr struct_ref_CellT = lookup_global_symbol("Cell")->try_as(); + TypePtr CellT = TypeDataGenericTypeWithTs::create(struct_ref_CellT, nullptr, {typeT}); + + lookup_function("Cell.load")->mutate()->parameters[0].declared_type = CellT; + lookup_function("Cell.load")->mutate()->receiver_type = CellT; + lookup_function("T.toCell")->mutate()->declared_return_type = CellT; } } // namespace tolk diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp index a68de6c7c..c6e2f4331 100644 --- a/tolk/constant-evaluator.cpp +++ b/tolk/constant-evaluator.cpp @@ -252,7 +252,7 @@ struct ConstantExpressionChecker { // `const a = ton("0.05")`, we met `ton("0.05")` static void handle_function_call(V v) { - if (v->fun_maybe && v->fun_maybe->is_compile_time_only()) { + if (v->fun_maybe && v->fun_maybe->is_compile_time_const_val()) { // `ton(local_var)` is denied; it's validated not here, but when replacing its value with a calculated one return; } @@ -318,7 +318,7 @@ std::string eval_string_const_standalone(AnyExprV v_string) { CompileTimeFunctionResult eval_call_to_compile_time_function(AnyExprV v_call) { auto v = v_call->try_as(); - tolk_assert(v && v->fun_maybe->is_compile_time_only()); + tolk_assert(v && v->fun_maybe->is_compile_time_const_val()); return parse_vertex_call_to_compile_time_function(v, v->fun_maybe->name); } diff --git a/tolk/pack-unpack-api.cpp b/tolk/pack-unpack-api.cpp new file mode 100644 index 000000000..14a20d922 --- /dev/null +++ b/tolk/pack-unpack-api.cpp @@ -0,0 +1,257 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "pack-unpack-api.h" +#include "generics-helpers.h" +#include "type-system.h" +#include + +/* + * This module provides high-level (de)serialization functions to be used from outer code: + * - pack to cell/builder + * - unpack from cell/slice + * - etc. + * + * For the implementation of packing primitives, consider `pack-unpack-serializers.cpp`. + */ + +namespace tolk { + + +// -------------------------------------------- +// checking serialization availability +// +// for every call `obj.toCell()` and similar, checks are executed to ensure that `obj` can be serialized +// if it can, the compilation process continues +// if not, a detailed explanation is shown +// + + +struct CantSerializeBecause { + std::string because_msg; + + explicit CantSerializeBecause(std::string because_msg) + : because_msg(std::move(because_msg)) {} + explicit CantSerializeBecause(const std::string& because_msg, const CantSerializeBecause& why) + : because_msg(because_msg + "\n" + why.because_msg) {} +}; + +class PackUnpackAvailabilityChecker { +public: + static std::optional detect_why_cant_serialize(TypePtr any_type, bool is_pack) { + if (any_type->try_as()) { + return {}; + } + if (any_type->try_as()) { + return {}; + } + if (any_type == TypeDataCoins::create()) { + return {}; + } + if (any_type == TypeDataBool::create()) { + return {}; + } + if (any_type == TypeDataCell::create()) { + return {}; + } + if (any_type == TypeDataAddress::create()) { + return {}; + } + if (any_type == TypeDataNever::create()) { + return {}; + } + + if (const auto* t_struct = any_type->try_as()) { + StructPtr struct_ref = t_struct->struct_ref; + for (StructFieldPtr field_ref : struct_ref->fields) { + if (auto why = detect_why_cant_serialize(field_ref->declared_type, is_pack)) { + return CantSerializeBecause("because field `" + struct_ref->name + "." + field_ref->name + "` of type `" + field_ref->declared_type->as_human_readable() + "` can't be serialized", why.value()); + } + } + if (is_type_cellT(t_struct)) { + TypePtr cellT = struct_ref->substitutedTs->typeT_at(0); + if (auto why = detect_why_cant_serialize(cellT, is_pack)) { + return CantSerializeBecause("because type `" + cellT->as_human_readable() + "` can't be serialized", why.value()); + } + } + return {}; + } + + if (const auto* t_union = any_type->try_as()) { + // a union can almost always be serialized if every of its variants can: + // - `T?` is TL/B `(Maybe T)` + // - `T1 | T2` is TL/B `(Either T1 T2)` (or, if opcodes manually set, just by opcodes) + // - `T1 | T2 | ...` is either by manual opcodes, or the compiler implicitly defines them + // so, even `int32 | int64 | int128` or `A | B | C | null` are serializable + // (unless corner cases occur, like duplicated opcodes, etc.) + for (int i = 0; i < t_union->size(); ++i) { + TypePtr variant = t_union->variants[i]; + if (variant == TypeDataNullLiteral::create()) { + continue; + } + if (auto why = detect_why_cant_serialize(variant, is_pack)) { + return CantSerializeBecause("because variant #" + std::to_string(i + 1) + " of type `" + variant->as_human_readable() + "` can't be serialized", why.value()); + } + } + if (!t_union->or_null) { + std::string because_msg; + auto_generate_opcodes_for_union(t_union, because_msg); + if (!because_msg.empty()) { + return CantSerializeBecause("because could not automatically generate serialization prefixes for a union\n" + because_msg); + } + } + return {}; + } + + if (const auto* t_tensor = any_type->try_as()) { + for (int i = 0; i < t_tensor->size(); ++i) { + if (auto why = detect_why_cant_serialize(t_tensor->items[i], is_pack)) { + return CantSerializeBecause("because element `tensor." + std::to_string(i) + "` of type `" + t_tensor->items[i]->as_human_readable() + "` can't be serialized", why.value()); + } + } + return {}; + } + + if (const auto* t_alias = any_type->try_as()) { + if (t_alias->alias_ref->name == "RemainingBitsAndRefs") { // it's built-in RemainingBitsAndRefs (slice) + return {}; + } + if (auto why = detect_why_cant_serialize(t_alias->underlying_type, is_pack)) { + return CantSerializeBecause("because alias `" + t_alias->as_human_readable() + "` expands to `" + t_alias->underlying_type->as_human_readable() + "`", why.value()); + } + return {}; + } + + // `builder` can be used for writing, but not for reading + if (any_type == TypeDataBuilder::create()) { + if (is_pack) { + return {}; + } + return CantSerializeBecause("because type `builder` can not be used for reading, only for writing\nhint: use `bitsN` or `RemainingBitsAndRefs` for reading\nhint: using generics, you can substitute `builder` for writing and something other for reading"); + } + + // serialization not available + // for common types, make a detailed explanation with a hint how to fix + + if (any_type == TypeDataInt::create()) { + return CantSerializeBecause("because type `int` is not serializable, it doesn't define binary width\nhint: replace `int` with `int32` / `uint64` / `coins` / etc."); + } + if (any_type == TypeDataSlice::create()) { + return CantSerializeBecause("because type `slice` is not serializable, it doesn't define binary width\nhint: replace `slice` with `address` if it's an address, actually\nhint: replace `slice` with `bits128` and similar if it represents fixed-width data without refs"); + } + if (any_type == TypeDataNullLiteral::create()) { + return CantSerializeBecause("because type `null` is not serializable\nhint: `int32?` and other nullable types will work"); + } + if (any_type == TypeDataTuple::create() || any_type->try_as()) { + return CantSerializeBecause("because tuples are not serializable\nhint: use tensors instead of tuples, they will work"); + } + + return CantSerializeBecause("because type `" + any_type->as_human_readable() + "` is not serializable"); + } +}; + +bool check_struct_can_be_packed_or_unpacked(TypePtr any_type, bool is_pack, std::string& because_msg) { + if (auto why = PackUnpackAvailabilityChecker::detect_why_cant_serialize(any_type, is_pack)) { + because_msg = why.value().because_msg; + return false; + } + return true; +} + + +// -------------------------------------------- +// high-level API for outer code +// + + +std::vector generate_pack_struct_to_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_obj) { + FunctionPtr f_beginCell = lookup_function("beginCell"); + FunctionPtr f_endCell = lookup_function("builder.endCell"); + std::vector rvect_builder = code.create_var(TypeDataBuilder::create(), loc, "b"); + code.emplace_back(loc, Op::_Call, rvect_builder, std::vector{}, f_beginCell); + + PackContext ctx(code, loc, rvect_builder); + ctx.generate_pack_any(any_type, std::move(ir_obj)); + + std::vector rvect_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(cell)"); + code.emplace_back(loc, Op::_Call, rvect_cell, std::move(rvect_builder), f_endCell); + + return rvect_cell; +} + +std::vector generate_pack_struct_to_builder(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_builder, std::vector&& ir_obj) { + PackContext ctx(code, loc, ir_builder); // mutate this builder + ctx.generate_pack_any(any_type, std::move(ir_obj)); + + return ir_builder; // return mutated builder +} + +std::vector generate_unpack_struct_from_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, bool mutate_slice) { + if (!mutate_slice) { + std::vector slice_copy = code.create_var(TypeDataSlice::create(), loc, "s"); + code.emplace_back(loc, Op::_Let, slice_copy, std::move(ir_slice)); + ir_slice = std::move(slice_copy); + } + + UnpackContext ctx(code, loc, std::move(ir_slice)); + std::vector rvect_struct = ctx.generate_unpack_any(any_type); + tolk_assert(any_type->get_width_on_stack() == static_cast(rvect_struct.size())); + + return rvect_struct; +} + +std::vector generate_unpack_struct_from_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_cell) { + FunctionPtr f_beginParse = lookup_function("cell.beginParse"); + std::vector ir_slice = code.create_var(TypeDataSlice::create(), loc, "s"); + code.emplace_back(loc, Op::_Call, ir_slice, std::move(ir_cell), f_beginParse); + + UnpackContext ctx(code, loc, std::move(ir_slice)); + std::vector rvect_struct = ctx.generate_unpack_any(any_type); + tolk_assert(any_type->get_width_on_stack() == static_cast(rvect_struct.size())); + + return rvect_struct; +} + +std::vector generate_skip_struct_in_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice) { + UnpackContext ctx(code, loc, ir_slice); // mutate this slice + ctx.generate_skip_any(any_type); + + return ir_slice; // return mutated slice +} + +PackSize estimate_serialization_size(TypePtr any_type) { + EstimateContext ctx; + return ctx.estimate_any(any_type); +} + +std::vector generate_estimate_size_call(CodeBlob& code, SrcLocation loc, TypePtr any_type) { + EstimateContext ctx; + PackSize pack_size = ctx.estimate_any(any_type); + + std::vector ir_tensor = code.create_tmp_var(TypeDataTensor::create({TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create()}), loc, "(result-tensor)"); + code.emplace_back(loc, Op::_IntConst, std::vector{ir_tensor[0]}, td::make_refint(pack_size.min_bits)); + code.emplace_back(loc, Op::_IntConst, std::vector{ir_tensor[1]}, td::make_refint(pack_size.max_bits)); + code.emplace_back(loc, Op::_IntConst, std::vector{ir_tensor[2]}, td::make_refint(pack_size.min_refs)); + code.emplace_back(loc, Op::_IntConst, std::vector{ir_tensor[3]}, td::make_refint(pack_size.max_refs)); + + FunctionPtr f_toTuple = lookup_function("T.__toTuple"); + std::vector ir_tuple = code.create_tmp_var(TypeDataTuple::create(), loc, "(result-tuple)"); + code.emplace_back(loc, Op::_Call, ir_tuple, ir_tensor, f_toTuple); + + return ir_tuple; +} + +} // namespace tolk diff --git a/tolk/pack-unpack-api.h b/tolk/pack-unpack-api.h new file mode 100644 index 000000000..b0a4b29ef --- /dev/null +++ b/tolk/pack-unpack-api.h @@ -0,0 +1,35 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "pack-unpack-serializers.h" +#include "tolk.h" + +namespace tolk { + +bool check_struct_can_be_packed_or_unpacked(TypePtr any_type, bool is_pack, std::string& because_msg); + +std::vector generate_pack_struct_to_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_obj); +std::vector generate_pack_struct_to_builder(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_builder, std::vector&& ir_obj); +std::vector generate_unpack_struct_from_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, bool mutate_slice); +std::vector generate_unpack_struct_from_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_cell); +std::vector generate_skip_struct_in_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice); + +PackSize estimate_serialization_size(TypePtr any_type); +std::vector generate_estimate_size_call(CodeBlob& code, SrcLocation loc, TypePtr any_type); + +} // namespace tolk diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp new file mode 100644 index 000000000..8be3d9995 --- /dev/null +++ b/tolk/pack-unpack-serializers.cpp @@ -0,0 +1,1031 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "pack-unpack-serializers.h" +#include "tolk.h" +#include "type-system.h" +#include "td/utils/crypto.h" + +/* + * This module implements serializing different types to/from cells. + * For any serializable TypePtr, we detect ISerializer, which can pack/unpack/skip/estimate size. + * See `get_serializer_for_type()`. + * Example: given an object of `struct A { f: int32 }` its type is TypeDataStruct(A), its serializer is + * "custom struct", which iterates fields, for field `f` its serializer is "intN" with N=32. + * + * Serializing compound types is complicated, involving transitioning IR variables. For example, to serialize + * `int8 | A` (it's Either), we have input rvect of size = 1 + width(A), generate dynamic IF ELSE, and in each branch, + * transition rvect slots to a narrowed type. Operating with transitions and runtime type checking are implemented + * in IR generation, here we just reference those prototypes. + * + * For high-level (de)serialization API, consider `pack-unpack-api.cpp`. + */ + +namespace tolk { + + +std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, TypePtr cmp_type, const std::vector& expr_ir_idx, SrcLocation loc, const char* debug_desc); +std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc); + +constexpr int ERR_CODEGEN_ASSERT = 9; + +bool is_type_cellT(TypePtr any_type) { + if (const TypeDataStruct* t_struct = any_type->try_as()) { + StructPtr struct_ref = t_struct->struct_ref; + return struct_ref->is_instantiation_of_generic_struct() && struct_ref->base_struct_ref->name == "Cell"; + } + return false; +} + + +// -------------------------------------------- +// options, context, common helpers +// +// some of the referenced functions are built-in, some are declared in stdlib +// serialization assumes that stdlib exists and is loaded correctly +// + + +PackContext::PackContext(CodeBlob& code, SrcLocation loc, std::vector ir_builder) + : code(code) + , loc(loc) + , f_storeInt(lookup_function("builder.storeInt")) + , f_storeUint(lookup_function("builder.storeUint")) + , ir_builder(std::move(ir_builder)) + , ir_builder0(this->ir_builder[0]) { +} + +void PackContext::storeInt(var_idx_t ir_idx, int len) const { + std::vector args = { ir_builder0, ir_idx, code.create_int(loc, len, "(storeW)") }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), f_storeInt); +} + +void PackContext::storeUint(var_idx_t ir_idx, int len) const { + std::vector args = { ir_builder0, ir_idx, code.create_int(loc, len, "(storeW)") }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), f_storeUint); +} + +void PackContext::storeBool(var_idx_t ir_idx) const { + std::vector args = { ir_builder0, ir_idx }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeBool")); +} + +void PackContext::storeCoins(var_idx_t ir_idx) const { + std::vector args = { ir_builder0, ir_idx }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeCoins")); +} + +void PackContext::storeRef(var_idx_t ir_idx) const { + std::vector args = { ir_builder0, ir_idx }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeRef")); +} + +void PackContext::storeMaybeRef(var_idx_t ir_idx) const { + std::vector args = { ir_builder0, ir_idx }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeMaybeRef")); +} + +void PackContext::storeAddress(var_idx_t ir_idx) const { + std::vector args = { ir_builder0, ir_idx }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeAddress")); +} + +void PackContext::storeBuilder(var_idx_t ir_idx) const { + std::vector args = { ir_builder0, ir_idx }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeBuilder")); +} + +void PackContext::storeSlice(var_idx_t ir_idx) const { + std::vector args = { ir_builder0, ir_idx }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeSlice")); +} + +void PackContext::storeOpcode(PackOpcode opcode) const { + std::vector args = { ir_builder0, code.create_int(loc, opcode.pack_prefix, "(struct-prefix)"), code.create_int(loc, opcode.prefix_len, "(storeW)") }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), f_storeUint); +} + + +UnpackContext::UnpackContext(CodeBlob& code, SrcLocation loc, std::vector ir_slice) + : code(code) + , loc(loc) + , f_loadInt(lookup_function("slice.loadInt")) + , f_loadUint(lookup_function("slice.loadUint")) + , f_skipBits(lookup_function("slice.skipBits")) + , ir_slice(std::move(ir_slice)) + , ir_slice0(this->ir_slice[0]) { +} + +std::vector UnpackContext::loadInt(int len, const char* debug_desc) const { + std::vector args = { ir_slice0, code.create_int(loc, len, "(loadW)") }; + std::vector result = code.create_tmp_var(TypeDataInt::create(), loc, debug_desc); + code.emplace_back(loc, Op::_Call, std::vector{ir_slice0, result[0]}, std::move(args), f_loadInt); + return result; +} + +std::vector UnpackContext::loadUint(int len, const char* debug_desc) const { + std::vector args = { ir_slice0, code.create_int(loc, len, "(loadW)") }; + std::vector result = code.create_tmp_var(TypeDataInt::create(), loc, debug_desc); + code.emplace_back(loc, Op::_Call, std::vector{ir_slice0, result[0]}, std::move(args), f_loadUint); + return result; +} + +void UnpackContext::loadAndCheckOpcode(PackOpcode opcode) const { + std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(prefix-eq)"); + std::vector args = { ir_slice0, code.create_int(loc, opcode.pack_prefix, "(pack-prefix)"), code.create_int(loc, opcode.prefix_len, "(prefix-len)") }; + code.emplace_back(loc, Op::_Call, std::vector{ir_slice0, ir_prefix_eq[0]}, std::move(args), lookup_function("slice.tryStripPrefix")); + + std::vector args_assert = { code.create_int(loc, ERR_CODEGEN_ASSERT, "(excno)"), ir_prefix_eq[0], code.create_int(loc, 0, "") }; + Op& op_assert = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); + op_assert.set_impure_flag(); +} + +void UnpackContext::skipBits(int len) const { + std::vector args = { ir_slice0, code.create_int(loc, len, "(skipW)") }; + code.emplace_back(loc, Op::_Call, ir_slice, std::move(args), f_skipBits); +} + +void UnpackContext::skipBits_var(var_idx_t ir_len) const { + std::vector args = { ir_slice0, ir_len }; + code.emplace_back(loc, Op::_Call, ir_slice, std::move(args), f_skipBits); +} + + +// -------------------------------------------- +// serializers with pack/unpack/skip/estimate +// +// for every struct field, for every atomic type, a corresponding (de)serialization instruction is generated +// we generate IR code (Ops), not ASM directly; so, all later IR analysis will later take place +// some of them are straightforward, e.g., call a predefined function for intN and coins +// some are complicated, e.g., for Either we should check a union type at runtime while packing, +// and while unpacking, read a prefix, follow different branches, and construct a resulting union +// + + +struct ISerializer { + virtual ~ISerializer() = default; + + virtual void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) = 0; + virtual std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) = 0; + + virtual void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) = 0; + virtual PackSize estimate(const EstimateContext* ctx) = 0; +}; + +struct S_IntN final : ISerializer { + const int n_bits; + const bool is_unsigned; + + explicit S_IntN(int n_bits, bool is_unsigned) + : n_bits(n_bits), is_unsigned(is_unsigned) {} + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + if (is_unsigned) { + ctx->storeUint(rvect[0], n_bits); + } else { + ctx->storeInt(rvect[0], n_bits); + } + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + if (is_unsigned) { + return ctx->loadUint(n_bits, "(loaded-uint)"); + } else { + return ctx->loadInt(n_bits, "(loaded-int)"); + } + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + ctx->skipBits(n_bits); + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(n_bits); + } +}; + +struct S_BytesN final : ISerializer { + const int n_bits; + + explicit S_BytesN(int n_width, bool is_bits) + : n_bits(is_bits ? n_width : n_width * 8) {} + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + + // bitsN/bytesN aren't checked upon writing (to have requested number of bits and 0 refs), probably to expose an option + if (false) { + FunctionPtr f_assert = lookup_function("__throw_if_unless"); + FunctionPtr f_equals = lookup_function("_==_"); + FunctionPtr f_bitAnd = lookup_function("_&_"); + FunctionPtr f_getCounts = lookup_function("slice.remainingBitsAndRefsCount"); + + std::vector ir_counts = code.create_tmp_var(TypeDataTensor::create({TypeDataInt::create(), TypeDataInt::create()}), loc, "(slice-size)"); + code.emplace_back(loc, Op::_Call, ir_counts, rvect, f_getCounts); + std::vector ir_is_n = code.create_tmp_var(TypeDataInt::create(), loc, "(n-bits)"); + code.emplace_back(loc, Op::_Call, ir_is_n, std::vector{ir_counts[0], code.create_int(loc, n_bits, "(expected-bits)")}, f_equals); + std::vector ir_is_0 = code.create_tmp_var(TypeDataInt::create(), loc, "(n-refs)"); + code.emplace_back(loc, Op::_Call, ir_is_0, std::vector{ir_counts[1], code.create_int(loc, 0, "(expected-refs)")}, f_equals); + std::vector ir_both_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(both-eq)"); + code.emplace_back(loc, Op::_Call, ir_both_eq, std::vector{ir_is_n[0], ir_is_0[0]}, f_bitAnd); + std::vector args_assert = { code.create_int(loc, ERR_CODEGEN_ASSERT, "(excno)"), ir_both_eq[0], code.create_int(loc, 0, "") }; + Op& op1 = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert), f_assert); + op1.set_impure_flag(); + } + + ctx->storeSlice(rvect[0]); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_loadBits = lookup_function("slice.loadBits"); + std::vector args = { ctx->ir_slice0, code.create_int(loc, n_bits, "(loadW)") }; + std::vector ir_result = code.create_tmp_var(TypeDataSlice::create(), loc, "(loaded-slice)"); + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_result[0]}, std::move(args), f_loadBits); + return ir_result; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + ctx->skipBits(n_bits); + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(n_bits); + } +}; + +struct S_Bool final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + ctx->storeBool(rvect[0]); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + return ctx->loadInt(1, "(loaded-bool)"); + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + ctx->skipBits(1); + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(1); + } +}; + +struct S_RawTVMcell final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + ctx->storeRef(rvect[0]); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_loadRef = lookup_function("slice.loadRef"); + std::vector args = ctx->ir_slice; + std::vector ir_result = code.create_tmp_var(TypeDataCell::create(), loc, "(loaded-cell)"); + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_result[0]}, std::move(args), f_loadRef); + return ir_result; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_loadRef = lookup_function("slice.loadRef"); + std::vector args = ctx->ir_slice; + std::vector dummy_loaded = code.create_tmp_var(TypeDataCell::create(), loc, "(loaded-cell)"); + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, dummy_loaded[0]}, std::move(args), f_loadRef); + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(0, 0, 1, 1); + } +}; + +struct S_RawTVMcellOrNull final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + ctx->storeMaybeRef(rvect[0]); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_loadMaybeRef = lookup_function("slice.loadMaybeRef"); + std::vector args = ctx->ir_slice; + std::vector ir_result = code.create_tmp_var(TypeDataCell::create(), loc, "(loaded-cell)"); + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_result[0]}, std::move(args), f_loadMaybeRef); + return ir_result; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_skipMaybeRef = lookup_function("slice.skipMaybeRef"); + code.emplace_back(loc, Op::_Call, ctx->ir_slice, ctx->ir_slice, f_skipMaybeRef); + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(1, 1, 0, 1); + } +}; + +struct S_Coins final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + ctx->storeCoins(rvect[0]); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_loadCoins = lookup_function("slice.loadCoins"); + std::vector args = ctx->ir_slice; + std::vector ir_result = code.create_tmp_var(TypeDataInt::create(), loc, "(loaded-coins)"); + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_result[0]}, std::move(args), f_loadCoins); + return ir_result; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_loadCoins = lookup_function("slice.loadCoins"); + std::vector args = ctx->ir_slice; + std::vector dummy_loaded = code.create_tmp_var(TypeDataInt::create(), loc, "(loaded-coins)"); + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, dummy_loaded[0]}, std::move(args), f_loadCoins); + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(4, 124); + } +}; + +struct S_Address final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + ctx->storeAddress(rvect[0]); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_loadAddress = lookup_function("slice.loadAddress"); + std::vector ir_address = code.create_tmp_var(TypeDataSlice::create(), loc, "(loaded-addr)"); + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_address[0]}, ctx->ir_slice, f_loadAddress); + return ir_address; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + // we can't do just + // ctx->skipBits(2 + 1 + 8 + 256); + // because it may be addr_none or addr_extern; there is no "skip address" in TVM, so just load it + FunctionPtr f_loadAddress = lookup_function("slice.loadAddress"); + std::vector ir_address = code.create_tmp_var(TypeDataSlice::create(), loc, "(tmp-addr)"); + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_address[0]}, ctx->ir_slice, f_loadAddress); + } + + PackSize estimate(const EstimateContext* ctx) override { + // we can't do just + // return PackSize(2 + 1 + 8 + 256); + // because it may be addr_none or addr_extern; but since addr_extern is very-very uncommon, don't consider it + return PackSize(2, 2 + 1 + 8 + 256); + } +}; + +struct S_RemainingBitsAndRefs final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + ctx->storeSlice(rvect[0]); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + return ctx->ir_slice; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_beginCell = lookup_function("beginCell"); + FunctionPtr f_endCell = lookup_function("builder.endCell"); + FunctionPtr f_beginParse = lookup_function("cell.beginParse"); + + std::vector ir_builder = code.create_tmp_var(TypeDataBuilder::create(), loc, "(tmp-builder)"); + std::vector ir_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(tmp-cell)"); + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, f_beginCell); + code.emplace_back(loc, Op::_Call, ir_cell, ir_builder, f_endCell); + code.emplace_back(loc, Op::_Call, ctx->ir_slice, ir_cell, f_beginParse); + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(0, 0, 0, 0); + } +}; + +struct S_Builder final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + std::vector args = { ctx->ir_builder0, rvect[0] }; + code.emplace_back(loc, Op::_Call, ctx->ir_builder, std::move(args), lookup_function("builder.storeBuilder")); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + tolk_assert(false); // `builder` can only be used for writing, checked earlier + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + tolk_assert(false); // `builder` can only be used for writing, checked earlier + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(0, 0, 0, 0); + } +}; + +struct S_Null final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + // while `null` itself is not serializable, it may be contained inside a union: + // `int32 | int64 | null`, for example; + // then the compiler generates prefixes for every variant, and `null` variant does nothing + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector ir_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null)"); + code.emplace_back(loc, Op::_Call, ir_null, std::vector{}, lookup_function("__null")); + return ir_null; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(0); + } +}; + +struct S_Never final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.empty()); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + return {}; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize(0); + } +}; + +struct S_Maybe final : ISerializer { + const TypeDataUnion* t_union; + TypePtr or_null; + + explicit S_Maybe(const TypeDataUnion* t_union) + : t_union(t_union) + , or_null(t_union->or_null) { + } + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + std::vector ir_is_null = pre_compile_is_type(code, t_union, TypeDataNullLiteral::create(), rvect, loc, "(is-null)"); + Op& if_op = code.emplace_back(loc, Op::_If, ir_is_null); + { + code.push_set_cur(if_op.block0); + ctx->storeUint(code.create_int(loc, 0, "(maybeBit)"), 1); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_op.block1); + ctx->storeUint(code.create_int(loc, 1, "(maybeBit)"), 1); + rvect = transition_to_target_type(std::move(rvect), code, t_union, t_union->or_null, loc); + ctx->generate_pack_any(t_union->or_null, std::move(rvect)); + code.close_pop_cur(loc); + } + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector ir_result = code.create_tmp_var(t_union, loc, "(loaded-maybe)"); + std::vector ir_not_null = { ctx->loadUint(1, "(maybeBit)") }; + Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_not_null)); + { + code.push_set_cur(if_op.block0); + std::vector rvect_maybe = ctx->generate_unpack_any(t_union->or_null); + rvect_maybe = transition_to_target_type(std::move(rvect_maybe), code, t_union->or_null, t_union, loc); + code.emplace_back(loc, Op::_Let, ir_result, std::move(rvect_maybe)); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_op.block1); + std::vector rvect_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(maybe-null)"); + code.emplace_back(loc, Op::_Call, rvect_null, std::vector{}, lookup_function("__null")); + rvect_null = transition_to_target_type(std::move(rvect_null), code, TypeDataNullLiteral::create(), t_union, loc); + code.emplace_back(loc, Op::_Let, ir_result, std::move(rvect_null)); + code.close_pop_cur(loc); + } + return ir_result; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector ir_not_null = { ctx->loadUint(1, "(maybeBit)") }; + Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_not_null)); + { + code.push_set_cur(if_op.block0); + ctx->generate_skip_any(t_union->or_null); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_op.block1); + code.close_pop_cur(loc); + } + } + + PackSize estimate(const EstimateContext* ctx) override { + PackSize maybe_size = ctx->estimate_any(t_union->or_null); + return PackSize(1, 1 + maybe_size.max_bits, 0, maybe_size.max_refs); + } +}; + +struct S_Either final : ISerializer { + const TypeDataUnion* t_union; + TypePtr t_left; + TypePtr t_right; + + explicit S_Either(const TypeDataUnion* t_union) + : t_union(t_union) + , t_left(t_union->variants[0]) + , t_right(t_union->variants[1]) { + } + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + std::vector ir_is_right = pre_compile_is_type(code, t_union, t_right, rvect, loc, "(is-right)"); + Op& if_op = code.emplace_back(loc, Op::_If, ir_is_right); + { + code.push_set_cur(if_op.block0); + ctx->storeUint(code.create_int(loc, 1, "(eitherBit)"), 1); + std::vector rvect_right = transition_to_target_type(std::vector(rvect), code, t_union, t_right, loc); + ctx->generate_pack_any(t_right, std::move(rvect_right)); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_op.block1); + ctx->storeUint(code.create_int(loc, 0, "(eitherBit)"), 1); + std::vector rvect_left = transition_to_target_type(std::move(rvect), code, t_union, t_left, loc); + ctx->generate_pack_any(t_left, std::move(rvect_left)); + code.close_pop_cur(loc); + } + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector ir_result = code.create_tmp_var(t_union, loc, "(loaded-either)"); + std::vector ir_is_right = ctx->loadUint(1, "(eitherBit)"); + Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_is_right)); + { + code.push_set_cur(if_op.block0); + std::vector rvect_right = ctx->generate_unpack_any(t_right); + rvect_right = transition_to_target_type(std::move(rvect_right), code, t_right, t_union, loc); + code.emplace_back(loc, Op::_Let, ir_result, std::move(rvect_right)); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_op.block1); + std::vector rvect_left = ctx->generate_unpack_any(t_left); + rvect_left = transition_to_target_type(std::move(rvect_left), code, t_left, t_union, loc); + code.emplace_back(loc, Op::_Let, ir_result, std::move(rvect_left)); + code.close_pop_cur(loc); + } + return ir_result; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector ir_is_right = ctx->loadUint(1, "(eitherBit)"); + Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_is_right)); + { + code.push_set_cur(if_op.block0); + ctx->generate_skip_any(t_right); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_op.block1); + ctx->generate_skip_any(t_left); + code.close_pop_cur(loc); + } + } + + PackSize estimate(const EstimateContext* ctx) override { + PackSize either_size = EstimateContext::minmax(ctx->estimate_any(t_left), ctx->estimate_any(t_right)); + return EstimateContext::sum(PackSize(1), either_size); + } +}; + +struct S_MultipleConstructors final : ISerializer { + const TypeDataUnion* t_union; + std::vector opcodes; + + explicit S_MultipleConstructors(const TypeDataUnion* t_union, std::vector&& opcodes) + : t_union(t_union) + , opcodes(std::move(opcodes)) { + tolk_assert(this->opcodes.size() == t_union->variants.size()); + } + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + for (int i = 0; i < t_union->size() - 1; ++i) { + TypePtr variant = t_union->variants[i]; + std::vector ir_eq_ith = pre_compile_is_type(code, t_union, variant, rvect, loc, "(arm-cond-eq)"); + Op& if_op = code.emplace_back(loc, Op::_If, std::move(ir_eq_ith)); + code.push_set_cur(if_op.block0); + std::vector ith_rvect = transition_to_target_type(std::vector(rvect), code, t_union, variant, loc); + ctx->storeUint(code.create_int(loc, opcodes[i].pack_prefix, "(ith-prefix)"), opcodes[i].prefix_len); + ctx->generate_pack_any(variant, std::move(ith_rvect), PrefixWriteMode::DoNothingAlreadyWritten); + code.close_pop_cur(loc); + code.push_set_cur(if_op.block1); // open ELSE + } + + // we're inside the last ELSE + TypePtr last_variant = t_union->variants.back(); + std::vector last_rvect = transition_to_target_type(std::move(rvect), code, t_union, last_variant, loc); + ctx->storeUint(code.create_int(loc, opcodes.back().pack_prefix, "(ith-prefix)"), opcodes.back().prefix_len); + ctx->generate_pack_any(last_variant, std::move(last_rvect), PrefixWriteMode::DoNothingAlreadyWritten); + for (int i = 0; i < t_union->size() - 1; ++i) { + code.close_pop_cur(loc); // close all outer IFs + } + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + // assume that opcodes (either automatically generated or manually specified) + // form a valid prefix tree, and the order of reading does not matter; we'll definitely match the one; + FunctionPtr f_tryStripPrefix = lookup_function("slice.tryStripPrefix"); + + std::vector ir_result = code.create_tmp_var(t_union, loc, "(loaded-union)"); + std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(prefix-eq)"); + + for (int i = 0; i < t_union->size(); ++i) { + TypePtr variant = t_union->variants[i]; + std::vector args = { ctx->ir_slice0, code.create_int(loc, opcodes[i].pack_prefix, "(pack-prefix)"), code.create_int(loc, opcodes[i].prefix_len, "(prefix-len)") }; + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_prefix_eq[0]}, std::move(args), f_tryStripPrefix); + if (i != t_union->size() - 1) { + Op& if_op = code.emplace_back(loc, Op::_If, ir_prefix_eq); + code.push_set_cur(if_op.block0); + std::vector ith_rvect = ctx->generate_unpack_any(variant, PrefixReadMode::DoNothingAlreadyLoaded); + ith_rvect = transition_to_target_type(std::move(ith_rvect), code, variant, t_union, loc); + code.emplace_back(loc, Op::_Let, ir_result, std::move(ith_rvect)); + code.close_pop_cur(loc); + code.push_set_cur(if_op.block1); // open ELSE + } else { // we're inside the last ELSE + if (!are_prefixes_exhaustive()) { + std::vector args_assert = { code.create_int(loc, ERR_CODEGEN_ASSERT, "(excno)"), ir_prefix_eq[0], code.create_int(loc, 0, "") }; + Op& op_assert = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); + op_assert.set_impure_flag(); + } + std::vector last_rvect = ctx->generate_unpack_any(variant, PrefixReadMode::DoNothingAlreadyLoaded); + last_rvect = transition_to_target_type(std::move(last_rvect), code, variant, t_union, loc); + code.emplace_back(loc, Op::_Let, ir_result, std::move(last_rvect)); + for (int j = 0; j < t_union->size() - 1; ++j) { + code.close_pop_cur(loc); // close all outer IFs + } + } + } + return ir_result; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + FunctionPtr f_tryStripPrefix = lookup_function("slice.tryStripPrefix"); + std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(prefix-eq)"); + + for (int i = 0; i < t_union->size(); ++i) { + TypePtr variant = t_union->variants[i]; + std::vector args = { ctx->ir_slice0, code.create_int(loc, opcodes[i].pack_prefix, "(pack-prefix)"), code.create_int(loc, opcodes[i].prefix_len, "(prefix-len)") }; + code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_prefix_eq[0]}, std::move(args), f_tryStripPrefix); + if (i != t_union->size() - 1) { + Op& if_op = code.emplace_back(loc, Op::_If, ir_prefix_eq); + code.push_set_cur(if_op.block0); + ctx->generate_skip_any(variant, PrefixReadMode::DoNothingAlreadyLoaded); + code.close_pop_cur(loc); + code.push_set_cur(if_op.block1); // open ELSE + } else { // we're inside the last ELSE + if (!are_prefixes_exhaustive()) { + std::vector args_assert = { code.create_int(loc, ERR_CODEGEN_ASSERT, "(excno)"), ir_prefix_eq[0], code.create_int(loc, 0, "") }; + Op& op_assert = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); + op_assert.set_impure_flag(); + } + ctx->generate_skip_any(variant, PrefixReadMode::DoNothingAlreadyLoaded); + for (int j = 0; j < t_union->size() - 1; ++j) { + code.close_pop_cur(loc); // close all outer IFs + } + } + } + } + + PackSize estimate(const EstimateContext* ctx) override { + PackSize variants_size = ctx->estimate_any(t_union->variants[0], PrefixEstimateMode::DoNothingAlreadyIncluded); + PackSize prefix_size(opcodes[0].prefix_len); + + for (int i = 1; i < t_union->size(); ++i) { + variants_size = EstimateContext::minmax(variants_size, ctx->estimate_any(t_union->variants[i], PrefixEstimateMode::DoNothingAlreadyIncluded)); + prefix_size = EstimateContext::minmax(prefix_size, PackSize(opcodes[i].prefix_len)); + } + + return EstimateContext::sum(variants_size, prefix_size); + } + + bool are_prefixes_exhaustive() const { + bool all_prefix_len_eq = true; + for (PackOpcode opcode : opcodes) { + all_prefix_len_eq &= opcode.prefix_len == opcodes[0].prefix_len; + } + return all_prefix_len_eq && t_union->size() == (1 << opcodes[0].prefix_len); + } +}; + +struct S_Tensor final : ISerializer { + const TypeDataTensor* t_tensor; + + explicit S_Tensor(const TypeDataTensor* t_tensor) + : t_tensor(t_tensor) { + } + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + int stack_offset = 0; + for (TypePtr item : t_tensor->items) { + int stack_width = item->get_width_on_stack(); + std::vector item_vars(rvect.begin() + stack_offset, rvect.begin() + stack_offset + stack_width); + ctx->generate_pack_any(item, std::move(item_vars)); + stack_offset += stack_width; + } + tolk_assert(stack_offset == t_tensor->get_width_on_stack()); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + std::vector tensor_vars; + tensor_vars.reserve(t_tensor->get_width_on_stack()); + for (TypePtr item : t_tensor->items) { + std::vector item_vars = ctx->generate_unpack_any(item); + tensor_vars.insert(tensor_vars.end(), item_vars.begin(), item_vars.end()); + } + tolk_assert(static_cast(tensor_vars.size()) == t_tensor->get_width_on_stack()); + return tensor_vars; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + for (TypePtr item : t_tensor->items) { + ctx->generate_skip_any(item); + } + } + + PackSize estimate(const EstimateContext* ctx) override { + PackSize sum = PackSize(0); + for (TypePtr item : t_tensor->items) { + sum = EstimateContext::sum(sum, ctx->estimate_any(item)); + } + return sum; + } +}; + +struct S_CustomStruct final : ISerializer { + StructPtr struct_ref; + + explicit S_CustomStruct(StructPtr struct_ref) + : struct_ref(struct_ref) { + } + + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + if (struct_ref->opcode.exists() && ctx->get_prefix_mode() == PrefixWriteMode::WritePrefixOfStruct) { + ctx->storeOpcode(struct_ref->opcode); + } + + int stack_offset = 0; + for (StructFieldPtr field_ref : struct_ref->fields) { + int stack_width = field_ref->declared_type->get_width_on_stack(); + std::vector field_vars(rvect.begin() + stack_offset, rvect.begin() + stack_offset + stack_width); + ctx->generate_pack_any(field_ref->declared_type, std::move(field_vars)); + stack_offset += stack_width; + } + tolk_assert(stack_offset == TypeDataStruct::create(struct_ref)->get_width_on_stack()); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + if (struct_ref->opcode.exists() && ctx->get_prefix_mode() == PrefixReadMode::LoadAndCheck) { + ctx->loadAndCheckOpcode(struct_ref->opcode); + } + + int total_stack_w = TypeDataStruct::create(struct_ref)->get_width_on_stack(); + std::vector ir_struct; + ir_struct.reserve(total_stack_w); + for (StructFieldPtr field_ref : struct_ref->fields) { + std::vector field_vars = ctx->generate_unpack_any(field_ref->declared_type); + ir_struct.insert(ir_struct.end(), field_vars.begin(), field_vars.end()); + } + tolk_assert(static_cast(ir_struct.size()) == total_stack_w); + return ir_struct; + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + if (struct_ref->opcode.exists() && ctx->get_prefix_mode() == PrefixReadMode::LoadAndCheck) { + ctx->loadAndCheckOpcode(struct_ref->opcode); + } + + for (StructFieldPtr field_ref : struct_ref->fields) { + ctx->generate_skip_any(field_ref->declared_type); + } + } + + PackSize estimate(const EstimateContext* ctx) override { + PackSize sum(0); + + if (struct_ref->opcode.exists() && ctx->get_prefix_mode() == PrefixEstimateMode::IncludePrefixOfStruct) { + sum = EstimateContext::sum(sum, PackSize(struct_ref->opcode.prefix_len)); + } + + for (StructFieldPtr field_ref : struct_ref->fields) { + sum = EstimateContext::sum(sum, ctx->estimate_any(field_ref->declared_type)); + } + return sum; + } +}; + + +// -------------------------------------------- +// automatically generate opcodes +// +// for union types like `T1 | T2 | ...`, if prefixes for structs are not manually specified, +// the compiler generates a valid prefix tree: for `int32 | int64 | int128` it's '00' '01' '10'; +// it works both for structs (with unspecified prefixes) and primitives: `int32 | A | B` is ok; +// but if some prefixes are specified, some not — it's an error +// + + +std::vector auto_generate_opcodes_for_union(TypePtr union_type, std::string& because_msg) { + const TypeDataUnion* t_union = union_type->try_as(); + std::vector result; + result.reserve(t_union->size()); + + int n_have_opcode = 0; + bool has_null = false; + StructPtr last_struct_with_opcode = nullptr; // for error message + StructPtr last_struct_no_opcode = nullptr; + for (TypePtr variant : t_union->variants) { + if (const TypeDataStruct* variant_struct = variant->unwrap_alias()->try_as()) { + if (variant_struct->struct_ref->opcode.exists()) { + n_have_opcode++; + last_struct_with_opcode = variant_struct->struct_ref; + } else { + last_struct_no_opcode = variant_struct->struct_ref; + } + } else if (variant == TypeDataNullLiteral::create()) { + has_null = true; + } + } + + // `A | B | C`, all of them have opcodes — just use them; + // for instance, `A | B` is not either (0/1 + data), but uses manual opcodes + if (n_have_opcode == t_union->size()) { + for (TypePtr variant : t_union->variants) { + result.push_back(variant->unwrap_alias()->try_as()->struct_ref->opcode); + } + return result; + } + + // invalid: `A | B | C`, some of them have opcodes, some not; + // example: `A | B` if A has opcode, B not; + // example: `int32 | A` if A has opcode; + // example: `int32 | int64 | A` if A has opcode; + if (n_have_opcode) { + if (last_struct_with_opcode && last_struct_no_opcode) { + because_msg = "because struct `" + last_struct_with_opcode->as_human_readable() + "` has opcode, but `" + last_struct_no_opcode->as_human_readable() + "` does not\nhint: manually specify opcodes to all structures"; + } else { + because_msg = "because of mixing primitives and struct `" + last_struct_with_opcode->as_human_readable() + "` with serialization prefix\nhint: extract primitives to single-field structs and provide prefixes"; + } + return result; + } + + // okay, none of the opcodes are specified, generate a prefix tree; + // examples: `int32 | int64 | int128` / `int32 | A | null` / `A | B` / `A | B | C`; + // create prefixes `0b00 0b01 0b10` / `0b01 0b10 0b00` (use 0b00 for null if exists): + // for 3/4 variants — two bits, for 5 — three + int prefix_len = static_cast(std::ceil(std::log2(t_union->size()))); + int cur_prefix = has_null ? 1 : 0; // will use 0b00 for null, so start with 0b01 + for (TypePtr variant : t_union->variants) { + if (variant == TypeDataNullLiteral::create()) { + result.emplace_back(0, prefix_len); + } else { + result.emplace_back(cur_prefix++, prefix_len); + } + } + return result; +} + + +// -------------------------------------------- +// detect serializer by TypePtr +// +// note that at earlier compilation steps there already passed a check that any_type is serializable; +// see `check_struct_can_be_packed_or_unpacked()`, its structure reminds this function +// + + +static std::unique_ptr get_serializer_for_type(TypePtr any_type) { + if (const auto* t_intN = any_type->try_as()) { + return std::make_unique(t_intN->n_bits, t_intN->is_unsigned); + } + if (const auto* t_bytesN = any_type->try_as()) { + return std::make_unique(t_bytesN->n_width, t_bytesN->is_bits); + } + if (any_type == TypeDataCoins::create()) { + return std::make_unique(); + } + if (any_type == TypeDataBool::create()) { + return std::make_unique(); + } + if (any_type == TypeDataCell::create()) { + return std::make_unique(); + } + if (any_type == TypeDataAddress::create()) { + return std::make_unique(); + } + if (any_type == TypeDataBuilder::create()) { + return std::make_unique(); + } + if (any_type == TypeDataNullLiteral::create()) { + return std::make_unique(); + } + if (any_type == TypeDataNever::create()) { + return std::make_unique(); + } + + if (const auto* t_struct = any_type->try_as()) { + return std::make_unique(t_struct->struct_ref); + } + + if (const auto* t_union = any_type->try_as()) { + // `T?` is always `(Maybe T)`, even if T has custom opcode (opcode will follow bit '1') + if (t_union->or_null) { + TypePtr or_null = t_union->or_null->unwrap_alias(); + if (or_null == TypeDataCell::create() || is_type_cellT(or_null)) { + return std::make_unique(); + } + return std::make_unique(t_union); + } + + // `T1 | T2` is `(Either T1 T2)` (0/1 + contents) unless they both have custom prefixes + bool all_have_opcode = true; + for (TypePtr variant : t_union->variants) { + const TypeDataStruct* variant_struct = variant->unwrap_alias()->try_as(); + all_have_opcode &= variant_struct && variant_struct->struct_ref->opcode.exists(); + } + if (t_union->size() == 2 && !all_have_opcode) { + return std::make_unique(t_union); + } + // `T1 | T2 | T3`, probably nullable, probably with primitives, probably with custom opcodes; + // compiler is able to generate serialization prefixes automatically; + // and this type is valid, it was checked earlier + std::string err_msg; + std::vector opcodes = auto_generate_opcodes_for_union(t_union, err_msg); + tolk_assert(err_msg.empty()); + return std::make_unique(t_union, std::move(opcodes)); + } + + if (const auto* t_tensor = any_type->try_as()) { + return std::make_unique(t_tensor); + } + + if (const auto* t_alias = any_type->try_as()) { + if (t_alias->alias_ref->name == "RemainingBitsAndRefs") { + return std::make_unique(); + } + return get_serializer_for_type(t_alias->underlying_type); + } + + // this should not be reachable, serialization availability is checked earlier + throw Fatal("type `" + any_type->as_human_readable() + "` can not be serialized"); +} + + +void PackContext::generate_pack_any(TypePtr any_type, std::vector&& rvect, PrefixWriteMode prefix_mode) const { + PrefixWriteMode backup = this->prefix_mode; + this->prefix_mode = prefix_mode; + get_serializer_for_type(any_type)->pack(this, code, loc, std::move(rvect)); + this->prefix_mode = backup; +} + +std::vector UnpackContext::generate_unpack_any(TypePtr any_type, PrefixReadMode prefix_mode) const { + PrefixReadMode backup = this->prefix_mode; + this->prefix_mode = prefix_mode; + std::vector result = get_serializer_for_type(any_type)->unpack(this, code, loc); + this->prefix_mode = backup; + return result; +} + +void UnpackContext::generate_skip_any(TypePtr any_type, PrefixReadMode prefix_mode) const { + PrefixReadMode backup = this->prefix_mode; + this->prefix_mode = prefix_mode; + get_serializer_for_type(any_type)->skip(this, code, loc); + this->prefix_mode = backup; +} + +PackSize EstimateContext::estimate_any(TypePtr any_type, PrefixEstimateMode prefix_mode) const { + PrefixEstimateMode backup = this->prefix_mode; + this->prefix_mode = prefix_mode; + PackSize result = get_serializer_for_type(any_type)->estimate(this); + this->prefix_mode = backup; + return result; +} + +} // namespace tolk diff --git a/tolk/pack-unpack-serializers.h b/tolk/pack-unpack-serializers.h new file mode 100644 index 000000000..47629a7db --- /dev/null +++ b/tolk/pack-unpack-serializers.h @@ -0,0 +1,137 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "fwd-declarations.h" +#include "tolk.h" + +namespace tolk { + +using PackOpcode = StructData::PackOpcode; + +struct PackSize { + int min_bits; + int max_bits; + int min_refs; + int max_refs; + + explicit PackSize(int exact_bits) + : min_bits(exact_bits), max_bits(exact_bits), min_refs(0), max_refs(0) { + } + PackSize(int min_bits, int max_bits) + : min_bits(min_bits), max_bits(max_bits), min_refs(0), max_refs() { + } + PackSize(int min_bits, int max_bits, int min_refs, int max_refs) + : min_bits(min_bits), max_bits(max_bits), min_refs(min_refs), max_refs(max_refs) { + } +}; + + +enum class PrefixWriteMode { + WritePrefixOfStruct, + DoNothingAlreadyWritten, +}; + +class PackContext { + CodeBlob& code; + SrcLocation loc; + const FunctionPtr f_storeInt; + const FunctionPtr f_storeUint; + mutable PrefixWriteMode prefix_mode = PrefixWriteMode::WritePrefixOfStruct; + +public: + const std::vector ir_builder; + const var_idx_t ir_builder0; + + PackContext(CodeBlob& code, SrcLocation loc, std::vector ir_builder); + + PrefixWriteMode get_prefix_mode() const { return prefix_mode; } + + void storeInt(var_idx_t ir_idx, int len) const; + void storeUint(var_idx_t ir_idx, int len) const; + void storeBool(var_idx_t ir_idx) const; + void storeCoins(var_idx_t ir_idx) const; + void storeRef(var_idx_t ir_idx) const; + void storeMaybeRef(var_idx_t ir_idx) const; + void storeAddress(var_idx_t ir_idx) const; + void storeBuilder(var_idx_t ir_idx) const; + void storeSlice(var_idx_t ir_idx) const; + void storeOpcode(PackOpcode opcode) const; + + void generate_pack_any(TypePtr any_type, std::vector&& rvect, PrefixWriteMode prefix_mode = PrefixWriteMode::WritePrefixOfStruct) const; +}; + + +enum class PrefixReadMode { + LoadAndCheck, + DoNothingAlreadyLoaded, +}; + +class UnpackContext { + CodeBlob& code; + SrcLocation loc; + const FunctionPtr f_loadInt; + const FunctionPtr f_loadUint; + const FunctionPtr f_skipBits; + mutable PrefixReadMode prefix_mode = PrefixReadMode::LoadAndCheck; + +public: + const std::vector ir_slice; + const var_idx_t ir_slice0; + + UnpackContext(CodeBlob& code, SrcLocation loc, std::vector ir_slice); + + PrefixReadMode get_prefix_mode() const { return prefix_mode; } + + std::vector loadInt(int len, const char* debug_desc) const; + std::vector loadUint(int len, const char* debug_desc) const; + void loadAndCheckOpcode(PackOpcode opcode) const; + void skipBits(int len) const; + void skipBits_var(var_idx_t ir_len) const; + + std::vector generate_unpack_any(TypePtr any_type, PrefixReadMode prefix_mode = PrefixReadMode::LoadAndCheck) const; + void generate_skip_any(TypePtr any_type, PrefixReadMode prefix_mode = PrefixReadMode::LoadAndCheck) const; +}; + + +enum class PrefixEstimateMode { + IncludePrefixOfStruct, + DoNothingAlreadyIncluded, +}; + +class EstimateContext { + mutable PrefixEstimateMode prefix_mode = PrefixEstimateMode::IncludePrefixOfStruct; + +public: + + PrefixEstimateMode get_prefix_mode() const { return prefix_mode; } + + static PackSize minmax(PackSize a, PackSize b) { + return PackSize(std::min(a.min_bits, b.min_bits), std::max(a.max_bits, b.max_bits), std::min(a.min_refs, b.min_refs), std::max(a.max_refs, b.max_refs)); + } + static PackSize sum(PackSize a, PackSize b) { + return PackSize(a.min_bits + b.min_bits, a.max_bits + b.max_bits, a.min_refs + b.min_refs, a.max_refs + b.max_refs); + } + + PackSize estimate_any(TypePtr any_type, PrefixEstimateMode prefix_mode = PrefixEstimateMode::IncludePrefixOfStruct) const; +}; + + +bool is_type_cellT(TypePtr any_type); +std::vector auto_generate_opcodes_for_union(TypePtr union_type, std::string& because_msg); + +} // namespace tolk diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 31bccca15..9d58a7faa 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -22,6 +22,8 @@ #include "type-system.h" #include "common/refint.h" #include "smart-casts-cfg.h" +#include "pack-unpack-api.h" +#include "generics-helpers.h" #include /* @@ -205,7 +207,7 @@ class LValContext { code.emplace_back(loc, Op::_IntConst, index_ir_idx, td::make_refint(index_at)); vars_modification_watcher.trigger_callbacks(tuple_ir_idx, loc); - FunctionPtr builtin_sym = lookup_global_symbol("tuple.set")->try_as(); + FunctionPtr builtin_sym = lookup_function("tuple.set"); code.emplace_back(loc, Op::_Call, std::vector{tuple_ir_idx}, std::vector{tuple_ir_idx[0], lval_ir_idx[0], index_ir_idx[0]}, builtin_sym); local_lval.after_let(std::move(tuple_ir_idx), code, loc); } @@ -491,10 +493,10 @@ static std::vector pre_compile_let(CodeBlob& code, AnyExprV lhs, AnyE return right; } -static std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, TypePtr cmp_type, const std::vector& expr_ir_idx, SrcLocation loc, const char* debug_desc) { - FunctionPtr eq_sym = lookup_global_symbol("_==_")->try_as(); - FunctionPtr isnull_sym = lookup_global_symbol("__isNull")->try_as(); - FunctionPtr not_sym = lookup_global_symbol("!b_")->try_as(); +std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, TypePtr cmp_type, const std::vector& expr_ir_idx, SrcLocation loc, const char* debug_desc) { + FunctionPtr eq_sym = lookup_function("_==_"); + FunctionPtr isnull_sym = lookup_function("__isNull"); + FunctionPtr not_sym = lookup_function("!b_"); std::vector result_ir_idx = code.create_tmp_var(TypeDataBool::create(), loc, debug_desc); const TypeDataUnion* lhs_union = expr_type->try_as(); @@ -538,6 +540,66 @@ static std::vector gen_op_call(CodeBlob& code, TypePtr ret_type, SrcL return rvect; } +static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob& code, SrcLocation loc, std::vector&& rvect, FunctionPtr called_f) { + if (called_f->is_instantiation_of_generic_function()) { + std::string_view f_name = called_f->base_fun_ref->name; + TypePtr typeT = called_f->substitutedTs->typeT_at(0); + int n_rvect = static_cast(rvect.size()); + + if (f_name == "T.toCell") { + // in: object T, out: Cell (just a cell, wrapped) + tolk_assert(n_rvect == typeT->get_width_on_stack()); + std::vector ir_obj = std::move(rvect); + return generate_pack_struct_to_cell(code, loc, typeT, std::move(ir_obj)); + } + if (f_name == "T.fromCell") { + // in: cell, out: object T + tolk_assert(n_rvect == 1); + std::vector ir_cell = std::move(rvect); + return generate_unpack_struct_from_cell(code, loc, typeT, std::move(ir_cell)); + } + if (f_name == "T.fromSlice") { + // in: slice, out: object T, input slice NOT mutated + tolk_assert(n_rvect == 1); + std::vector ir_slice = std::move(rvect); + return generate_unpack_struct_from_slice(code, loc, typeT, std::move(ir_slice), false); + } + if (f_name == "Cell.load") { + // in: cell, out: object T + tolk_assert(n_rvect == 1); + std::vector ir_cell = std::move(rvect); + return generate_unpack_struct_from_cell(code, loc, typeT, std::move(ir_cell)); + } + if (f_name == "slice.loadAny") { + // in: slice, out: object T, input slice is mutated, so prepend self before an object + var_idx_t ir_self = rvect[0]; + std::vector ir_obj = generate_unpack_struct_from_slice(code, loc, typeT, std::move(rvect), true); + std::vector ir_result = {ir_self}; + ir_result.insert(ir_result.end(), ir_obj.begin(), ir_obj.end()); + return ir_result; + } + if (f_name == "slice.skipAny") { + // in: slice, out: the same slice, with a shifted pointer + tolk_assert(n_rvect == 1); + std::vector ir_slice = std::move(rvect); + return generate_skip_struct_in_slice(code, loc, typeT, std::move(ir_slice)); + } + if (f_name == "builder.storeAny") { + // in: builder and object T, out: mutated builder + tolk_assert(n_rvect == 1 + typeT->get_width_on_stack()); + std::vector ir_builder = {rvect[0]}; + std::vector ir_obj(rvect.begin() + 1, rvect.end()); + return generate_pack_struct_to_builder(code, loc, typeT, std::move(ir_builder), std::vector(ir_obj)); + } + if (f_name == "T.estimatePackSize") { + tolk_assert(rvect.empty()); + return generate_estimate_size_call(code, loc, typeT); + } + } + + tolk_assert(false); +} + // "Transition to target (runtime) type" is the following process. // Imagine `fun analyze(t: (int,int)?)` and a call `analyze((1,2))`. // `(1,2)` (inferred_type) is 2 stack slots, but `t` (target_type) is 3 (one for null-flag). @@ -615,7 +677,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vector 1 && original_type == TypeDataNullLiteral::create()) { tolk_assert(t_union->has_null()); - FunctionPtr null_sym = lookup_global_symbol("__null")->try_as(); + FunctionPtr null_sym = lookup_function("__null"); rvect.reserve(target_w); // keep rvect[0], it's already null for (int i = 1; i < target_w - 1; ++i) { std::vector ith_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); @@ -657,7 +719,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectorhas_null()); - FunctionPtr null_sym = lookup_global_symbol("__null")->try_as(); + FunctionPtr null_sym = lookup_function("__null"); std::vector new_rvect = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); code.emplace_back(loc, Op::_Call, new_rvect, std::vector{}, null_sym); return new_rvect; @@ -674,7 +736,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vectortry_as(); + FunctionPtr null_sym = lookup_function("__null"); std::vector new_rvect; new_rvect.resize(target_w); for (int i = 0; i < target_w - 2; ++i) { // N-1 nulls @@ -686,7 +748,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vector eq_null_cond = code.create_tmp_var(TypeDataBool::create(), loc, "(value-is-null)"); - FunctionPtr isnull_sym = lookup_global_symbol("__isNull")->try_as(); + FunctionPtr isnull_sym = lookup_function("__isNull"); code.emplace_back(loc, Op::_Call, eq_null_cond, rvect, isnull_sym); Op& if_op = code.emplace_back(loc, Op::_If, eq_null_cond); code.push_set_cur(if_op.block0); @@ -712,7 +774,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vector prepend_nulls; prepend_nulls.reserve(target_w - t_subtype->get_width_on_stack() - 1); for (int i = 0; i < target_w - t_subtype->get_width_on_stack() - 1; ++i) { - FunctionPtr null_sym = lookup_global_symbol("__null")->try_as(); + FunctionPtr null_sym = lookup_function("__null"); std::vector ith_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(UVar.null)"); prepend_nulls.push_back(ith_null[0]); code.emplace_back(loc, Op::_Call, std::move(ith_null), std::vector{}, null_sym); @@ -747,7 +809,7 @@ static std::vector transition_expr_to_runtime_type_impl(std::vector prepend_nulls; prepend_nulls.reserve(target_w - orig_w); for (int i = 0; i < target_w - orig_w; ++i) { - FunctionPtr null_sym = lookup_global_symbol("__null")->try_as(); + FunctionPtr null_sym = lookup_function("__null"); std::vector ith_null_ir_idx = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(UVar.null)"); prepend_nulls.push_back(ith_null_ir_idx[0]); code.emplace_back(loc, Op::_Call, std::move(ith_null_ir_idx), std::vector{}, null_sym); @@ -910,6 +972,17 @@ static std::vector transition_expr_to_runtime_type_impl(std::vector` to `cell`, e.g. `setContractData(obj.toCell())` + if (target_type == TypeDataCell::create() && original_type->try_as()) { + tolk_assert(orig_w == 1 && original_type->try_as()->struct_ref->is_instantiation_of_generic_struct()); + return rvect; + } + // and vice versa, `cell as Cell` + if (original_type == TypeDataCell::create() && target_type->try_as()) { + tolk_assert(target_w == 1 && target_type->try_as()->struct_ref->is_instantiation_of_generic_struct()); + return rvect; + } + throw Fatal("unhandled transition_expr_to_runtime_type_impl() combination"); } @@ -929,7 +1002,7 @@ static std::vector transition_to_target_type(std::vector&& #ifndef TOLK_DEBUG GNU_ATTRIBUTE_ALWAYS_INLINE #endif -static std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc) { +std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc) { if (target_type != original_type) { rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type, target_type, loc); } @@ -1037,7 +1110,7 @@ static std::vector process_binary_operator(V v, v_1->mutate()->assign_inferred_type(TypeDataInt::create()); auto v_b_ne_0 = createV(v->loc, "!=", tok_neq, v->get_rhs(), v_0); v_b_ne_0->mutate()->assign_inferred_type(TypeDataInt::create()); - v_b_ne_0->mutate()->assign_fun_ref(lookup_global_symbol("_!=_")->try_as()); + v_b_ne_0->mutate()->assign_fun_ref(lookup_function("_!=_")); std::vector cond = pre_compile_expr(v->get_lhs(), code, nullptr); tolk_assert(cond.size() == 1); std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(ternary)"); @@ -1052,13 +1125,13 @@ static std::vector process_binary_operator(V v, } if (t == tok_eq || t == tok_neq) { if (v->get_lhs()->inferred_type->unwrap_alias() == TypeDataAddress::create() && v->get_rhs()->inferred_type->unwrap_alias() == TypeDataAddress::create()) { - FunctionPtr f_sliceEq = lookup_global_symbol("slice.bitsEqual")->try_as(); + FunctionPtr f_sliceEq = lookup_function("slice.bitsEqual"); std::vector ir_lhs_slice = pre_compile_expr(v->get_lhs(), code); std::vector ir_rhs_slice = pre_compile_expr(v->get_rhs(), code); std::vector rvect = code.create_tmp_var(TypeDataBool::create(), v->loc, "(addr-eq)"); code.emplace_back(v->loc, Op::_Call, rvect, std::vector{ir_lhs_slice[0], ir_rhs_slice[0]}, f_sliceEq); if (t == tok_neq) { - FunctionPtr not_sym = lookup_global_symbol("!b_")->try_as(); + FunctionPtr not_sym = lookup_function("!b_"); code.emplace_back(v->loc, Op::_Call, rvect, rvect, not_sym); } return transition_to_target_type(std::move(rvect), code, target_type, v); @@ -1112,7 +1185,7 @@ static std::vector process_is_type_operator(V v std::vector result_ir_idx = pre_compile_is_type(code, lhs_type, cmp_type, expr_ir_idx, v->loc, is_null_check ? "(is-null)" : "(is-type)"); if (v->is_negated) { - FunctionPtr not_sym = lookup_global_symbol("!b_")->try_as(); + FunctionPtr not_sym = lookup_function("!b_"); code.emplace_back(v->loc, Op::_Call, result_ir_idx, result_ir_idx, not_sym); } return transition_to_target_type(std::move(result_ir_idx), code, target_type, v); @@ -1152,7 +1225,7 @@ static std::vector process_match_expression(V v // construct nested IFs: IF is int { ... } ELSE { IF is slice { ... } ELSE { ... } } // example 3 (not exhaustive): `match (v) { -1 => ... 0 => ... 1 => ... }` // construct nested IFs: IF == -1 { ... } ELSE { IF == 0 { ... } ELSE { IF == 1 { ... } } } - FunctionPtr eq_sym = lookup_global_symbol("_==_")->try_as(); + FunctionPtr eq_sym = lookup_function("_==_"); for (int i = 0; i < n_arms - is_exhaustive; ++i) { auto v_ith_arm = v->get_arm(i); std::vector eq_ith_ir_idx; @@ -1259,7 +1332,7 @@ static std::vector process_dot_access(V v, CodeBlob& code.emplace_back(v->loc, Op::_IntConst, index_ir_idx, td::make_refint(index_at)); std::vector field_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(tuple-field)"); tolk_assert(tuple_ir_idx.size() == 1 && field_ir_idx.size() == 1); // tuples contain only 1-slot values - FunctionPtr builtin_sym = lookup_global_symbol("tuple.get")->try_as(); + FunctionPtr builtin_sym = lookup_function("tuple.get"); code.emplace_back(v->loc, Op::_Call, field_ir_idx, std::vector{tuple_ir_idx[0], index_ir_idx[0]}, builtin_sym); if (lval_ctx && calc_sink_leftmost_obj(v)) { // `tupleVar.0.1 = rhs`, then `tupleVar.0` is rval inside lval lval_ctx->capture_tuple_index_modification(v->get_obj(), index_at, field_ir_idx); @@ -1371,7 +1444,9 @@ static std::vector process_function_call(V v, Code for (const std::vector& list : vars_per_arg) { args_vars.insert(args_vars.end(), list.cbegin(), list.cend()); } - std::vector rvect_apply = gen_op_call(code, op_call_type, v->loc, std::move(args_vars), fun_ref, "(fun-call)", arg_order_already_equals_asm); + std::vector rvect_apply = fun_ref->is_compile_time_special_gen() + ? gen_compile_time_code_instead_of_fun_call(code, v->loc, std::move(args_vars), fun_ref) + : gen_op_call(code, op_call_type, v->loc, std::move(args_vars), fun_ref, "(fun-call)", arg_order_already_equals_asm); if (fun_ref->has_mutate_params()) { LValContext local_lval; @@ -1553,13 +1628,13 @@ static std::vector process_string_const(V v, CodeBl } static std::vector process_bool_const(V v, CodeBlob& code, TypePtr target_type) { - FunctionPtr builtin_sym = lookup_global_symbol(v->bool_val ? "__true" : "__false")->try_as(); + FunctionPtr builtin_sym = lookup_function(v->bool_val ? "__true" : "__false"); std::vector rvect = gen_op_call(code, v->inferred_type, v->loc, {}, builtin_sym, "(bool-const)"); return transition_to_target_type(std::move(rvect), code, target_type, v); } static std::vector process_null_keyword(V v, CodeBlob& code, TypePtr target_type) { - FunctionPtr builtin_sym = lookup_global_symbol("__null")->try_as(); + FunctionPtr builtin_sym = lookup_function("__null"); std::vector rvect = gen_op_call(code, v->inferred_type, v->loc, {}, builtin_sym, "(null-literal)"); return transition_to_target_type(std::move(rvect), code, target_type, v); } @@ -1685,7 +1760,7 @@ static void process_assert_statement(V v, CodeBlob& code) args[2]->mutate()->assign_inferred_type(TypeDataInt::create()); } - FunctionPtr builtin_sym = lookup_global_symbol("__throw_if_unless")->try_as(); + FunctionPtr builtin_sym = lookup_function("__throw_if_unless"); std::vector args_vars = pre_compile_tensor(code, args); gen_op_call(code, TypeDataVoid::create(), v->loc, std::move(args_vars), builtin_sym, "(throw-call)"); } @@ -1781,9 +1856,9 @@ static void process_do_while_statement(V v, CodeBlob& co } until_cond->mutate()->assign_inferred_type(TypeDataInt::create()); if (auto v_bin = until_cond->try_as(); v_bin && !v_bin->fun_ref) { - v_bin->mutate()->assign_fun_ref(lookup_global_symbol("_" + static_cast(v_bin->operator_name) + "_")->try_as()); + v_bin->mutate()->assign_fun_ref(lookup_function("_" + static_cast(v_bin->operator_name) + "_")); } else if (auto v_un = until_cond->try_as(); v_un && !v_un->fun_ref) { - v_un->mutate()->assign_fun_ref(lookup_global_symbol(static_cast(v_un->operator_name) + "_")->try_as()); + v_un->mutate()->assign_fun_ref(lookup_function(static_cast(v_un->operator_name) + "_")); } until_op.left = pre_compile_expr(until_cond, code, nullptr); @@ -1804,11 +1879,11 @@ static void process_while_statement(V v, CodeBlob& code) { static void process_throw_statement(V v, CodeBlob& code) { if (v->has_thrown_arg()) { - FunctionPtr builtin_sym = lookup_global_symbol("__throw_arg")->try_as(); + FunctionPtr builtin_sym = lookup_function("__throw_arg"); std::vector args_vars = pre_compile_tensor(code, {v->get_thrown_arg(), v->get_thrown_code()}); gen_op_call(code, TypeDataVoid::create(), v->loc, std::move(args_vars), builtin_sym, "(throw-call)"); } else { - FunctionPtr builtin_sym = lookup_global_symbol("__throw")->try_as(); + FunctionPtr builtin_sym = lookup_function("__throw"); std::vector args_vars = pre_compile_tensor(code, {v->get_thrown_code()}); gen_op_call(code, TypeDataVoid::create(), v->loc, std::move(args_vars), builtin_sym, "(throw-call)"); } diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index dc17fa652..8500ec59a 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -65,6 +65,10 @@ static void fire_error_cannot_apply_operator(FunctionPtr cur_f, SrcLocation loc, GNU_ATTRIBUTE_NOINLINE static void warning_condition_always_true_or_false(FunctionPtr cur_f, SrcLocation loc, AnyExprV cond, const char* operator_name) { + bool no_warning = cond->kind == ast_bool_const || cond->kind == ast_int_const; + if (no_warning) { // allow `while(true)` without a warning + return; + } loc.show_warning("condition of " + static_cast(operator_name) + " is always " + (cond->is_always_true ? "true" : "false")); } diff --git a/tolk/pipe-check-serialized-fields.cpp b/tolk/pipe-check-serialized-fields.cpp new file mode 100644 index 000000000..c49b3dc34 --- /dev/null +++ b/tolk/pipe-check-serialized-fields.cpp @@ -0,0 +1,115 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-visitor.h" +#include "pack-unpack-api.h" +#include "generics-helpers.h" +#include "type-system.h" + +namespace tolk { + +// fire an error on overflow 1023 bits +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_theoretical_overflow_1023(StructPtr struct_ref, PackSize size) { + throw ParseError(struct_ref->ast_root->loc, + "struct `" + struct_ref->as_human_readable() + "` can exceed 1023 bits in serialization (estimated size: " + std::to_string(size.min_bits) + ".." + std::to_string(size.max_bits) + " bits)\n\n" + "1) either suppress it by adding an annotation:\n" + "> @overflow1023_policy(\"suppress\")\n" + "> struct " + struct_ref->name + " {\n" + "> ...\n" + "> }\n" + " then, if limit exceeds, it will fail at runtime: you've manually agreed to ignore this\n\n" + "2) or place some fields into a separate struct (e.g. ExtraFields), and create a ref:\n" + "> struct " + struct_ref->name + " {\n" + "> ...\n" + "> more: Cell;\n" + "> }\n" + ); +} + + +class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody { + FunctionPtr cur_f = nullptr; + + static void check_type_fits_cell_or_has_policy(TypePtr serialized_type) { + if (const TypeDataStruct* s_struct = serialized_type->unwrap_alias()->try_as()) { + check_struct_fits_cell_or_has_policy(s_struct); + } else if (const TypeDataUnion* s_union = serialized_type->unwrap_alias()->try_as()) { + for (TypePtr variant : s_union->variants) { + check_type_fits_cell_or_has_policy(variant); + } + } + } + + static void check_struct_fits_cell_or_has_policy(const TypeDataStruct* t_struct) { + StructPtr struct_ref = t_struct->struct_ref; + PackSize size = estimate_serialization_size(t_struct); + if (size.max_bits > 1023) { + if (struct_ref->overflow1023_policy == StructData::Overflow1023Policy::not_specified) { + fire_error_theoretical_overflow_1023(struct_ref, size); + } + } + for (StructFieldPtr field_ref : struct_ref->fields) { + if (is_type_cellT(field_ref->declared_type)) { + const TypeDataStruct* f_struct = field_ref->declared_type->try_as(); + check_type_fits_cell_or_has_policy(f_struct->struct_ref->substitutedTs->typeT_at(0)); + } + } + } + + void visit(V v) override { + FunctionPtr fun_ref = v->fun_maybe; + if (!fun_ref || !fun_ref->is_compile_time_special_gen() || !fun_ref->is_instantiation_of_generic_function()) { + return; + } + + std::string_view f_name = fun_ref->base_fun_ref->name; + TypePtr serialized_type = nullptr; + bool is_pack = false; + if (f_name == "Cell.load" || f_name == "T.fromSlice" || f_name == "T.fromCell" || f_name == "T.toCell" || + f_name == "T.loadAny" || f_name == "slice.skipAny" || f_name == "slice.storeAny" || f_name == "T.estimatePackSize") { + serialized_type = fun_ref->substitutedTs->typeT_at(0); + is_pack = f_name == "T.toCell" || f_name == "slice.storeAny"; + } else { + return; // not a serialization function + } + + std::string because_msg; + if (!check_struct_can_be_packed_or_unpacked(serialized_type, is_pack, because_msg)) { + fire(cur_f, v->loc, "auto-serialization via " + fun_ref->method_name + "() is not available for type `" + serialized_type->as_human_readable() + "`\n" + because_msg); + } + + check_type_fits_cell_or_has_policy(serialized_type); + } + + public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && !fun_ref->is_generic_function(); + } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + parent::visit(v_function->get_body()); + } +}; + +void pipeline_check_serialized_fields() { + visit_ast_of_all_functions(); +} + +} // namespace tolk diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp index 32b7c28c2..f16bf90a5 100644 --- a/tolk/pipe-constant-folding.cpp +++ b/tolk/pipe-constant-folding.cpp @@ -58,9 +58,9 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { return inner; } - static V create_string_const(SrcLocation loc, std::string&& literal_value) { + static V create_string_const(SrcLocation loc, std::string&& literal_value, TypePtr inferred_type) { auto v_string = createV(loc, literal_value); - v_string->assign_inferred_type(TypeDataSlice::create()); + v_string->assign_inferred_type(inferred_type); v_string->assign_literal_value(std::move(literal_value)); v_string->assign_rvalue_true(); return v_string; @@ -112,12 +112,12 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { parent::replace(v); // replace `ton("0.05")` with 50000000 / `stringCrc32("some_str")` with calculated value / etc. - if (v->fun_maybe && v->fun_maybe->is_compile_time_only()) { + if (v->fun_maybe && v->fun_maybe->is_compile_time_const_val()) { CompileTimeFunctionResult value = eval_call_to_compile_time_function(v); if (std::holds_alternative(value)) { return create_int_const(v->loc, std::move(std::get(value))); } else { - return create_string_const(v->loc, std::move(std::get(value))); + return create_string_const(v->loc, std::move(std::get(value)), v->fun_maybe->declared_return_type); } } diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index 8e56e62b6..dd83a8dc2 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -577,7 +577,7 @@ class InferTypesAndCallsAndFieldsVisitor final { assign_inferred_type(v, lhs); - FunctionPtr builtin_sym = lookup_global_symbol("_" + static_cast(builtin_func) + "_")->try_as(); + FunctionPtr builtin_sym = lookup_function("_" + static_cast(builtin_func) + "_"); v->mutate()->assign_fun_ref(builtin_sym); return ExprFlow(std::move(after_rhs.out_flow), used_as_condition); @@ -607,7 +607,7 @@ class InferTypesAndCallsAndFieldsVisitor final { tolk_assert(false); } - FunctionPtr builtin_sym = lookup_global_symbol(static_cast(builtin_func) + "_")->try_as(); + FunctionPtr builtin_sym = lookup_function(static_cast(builtin_func) + "_"); v->mutate()->assign_fun_ref(builtin_sym); return after_rhs; @@ -685,7 +685,7 @@ class InferTypesAndCallsAndFieldsVisitor final { } if (!builtin_func.empty()) { - FunctionPtr builtin_sym = lookup_global_symbol("_" + static_cast(builtin_func) + "_")->try_as(); + FunctionPtr builtin_sym = lookup_function("_" + static_cast(builtin_func) + "_"); v->mutate()->assign_fun_ref(builtin_sym); } @@ -849,7 +849,7 @@ class InferTypesAndCallsAndFieldsVisitor final { // T for asm function must be a TVM primitive (width 1), otherwise, asm would act incorrectly if (fun_ref->is_asm_function() || fun_ref->is_builtin_function()) { for (int i = 0; i < substitutedTs.size(); ++i) { - if (substitutedTs.typeT_at(i)->get_width_on_stack() != 1) { + if (substitutedTs.typeT_at(i)->get_width_on_stack() != 1 && !fun_ref->is_variadic_width_T_allowed()) { fire_error_calling_asm_function_with_non1_stack_width_arg(cur_f, loc, fun_ref, substitutedTs, i); } } @@ -908,7 +908,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (out_f_called) { // so, it's `globalF()` / `genericFn()` / `genericFn()` *out_f_called = fun_ref; // (it's still may be a generic one, then Ts will be deduced from arguments) } else { // so, it's `globalF` / `genericFn` as a reference - if (fun_ref->is_compile_time_only()) { + if (fun_ref->is_compile_time_const_val() || fun_ref->is_compile_time_special_gen()) { fire(cur_f, v->loc, "can not get reference to this function, it's compile-time only"); } fun_ref->mutate()->assign_is_used_as_noncall(); @@ -1071,7 +1071,7 @@ class InferTypesAndCallsAndFieldsVisitor final { *out_f_called = fun_ref; // (it's still may be a generic one, then Ts will be deduced from arguments) *out_dot_obj = dot_obj; } else { // so, it's `user.method` / `t.tupleAt` as a reference - if (fun_ref->is_compile_time_only()) { + if (fun_ref->is_compile_time_const_val() || fun_ref->is_compile_time_special_gen()) { fire(cur_f, v->get_identifier()->loc, "can not get reference to this method, it's compile-time only"); } fun_ref->mutate()->assign_is_used_as_noncall(); @@ -1186,7 +1186,7 @@ class InferTypesAndCallsAndFieldsVisitor final { if (fun_ref->is_generic_function()) { // if `f(args)` was called, Ts were inferred; check that all of them are known std::string_view nameT_unknown = deducingTs.get_first_not_deduced_nameT(); - if (!nameT_unknown.empty() && hint && fun_ref->declared_return_type) { + if (!nameT_unknown.empty() && hint && !hint->has_genericT_inside() && fun_ref->declared_return_type) { // example: `t.tupleFirst()`, T doesn't depend on arguments, but is determined by return type // if used like `var x: int = t.tupleFirst()` / `t.tupleFirst() as int` / etc., use hint deducingTs.auto_deduce_from_argument(cur_f, v->loc, fun_ref->declared_return_type, hint); diff --git a/tolk/pipe-optimize-boolean-expr.cpp b/tolk/pipe-optimize-boolean-expr.cpp index ef6c8208c..6b053915f 100644 --- a/tolk/pipe-optimize-boolean-expr.cpp +++ b/tolk/pipe-optimize-boolean-expr.cpp @@ -53,7 +53,7 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { auto v_not = createV(loc, "!", tok_logical_not, rhs); v_not->assign_inferred_type(TypeDataBool::create()); v_not->assign_rvalue_true(); - v_not->assign_fun_ref(lookup_global_symbol("!b_")->try_as()); + v_not->assign_fun_ref(lookup_function("!b_")); return v_not; } @@ -98,7 +98,7 @@ struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { auto v_neq = createV(v->loc, "!=", tok_neq, cond_not_not, v_zero); v_neq->mutate()->assign_rvalue_true(); v_neq->mutate()->assign_inferred_type(TypeDataBool::create()); - v_neq->mutate()->assign_fun_ref(lookup_global_symbol("_!=_")->try_as()); + v_neq->mutate()->assign_fun_ref(lookup_function("_!=_")); return v_neq; } } diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index bbb105e42..71b2d3bed 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -20,6 +20,7 @@ #include "ast.h" #include "compiler-state.h" #include "generics-helpers.h" +#include "pack-unpack-serializers.h" #include "td/utils/crypto.h" #include @@ -141,12 +142,30 @@ static StructPtr register_struct(V v, StructPtr base_str fields.emplace_back(new StructFieldData(static_cast(v_field->get_identifier()->name), v_field->loc, i, v_field->type_node, default_value)); } + PackOpcode opcode(0, 0); + if (v->has_opcode()) { + auto v_opcode = v->get_opcode()->as(); + if (v_opcode->intval < 0 || v_opcode->intval > (1ULL << 48)) { + v->error("opcode must not exceed 2^48"); + } + opcode.pack_prefix = v_opcode->intval->to_long(); + + std::string_view prefix_str = v_opcode->orig_str; + if (prefix_str.starts_with("0x")) { + opcode.prefix_len = static_cast(prefix_str.size() - 2) * 4; + } else if (prefix_str.starts_with("0b")) { + opcode.prefix_len = static_cast(prefix_str.size() - 2); + } else { + tolk_assert(false); + } + } + std::string name = std::move(override_name); if (name.empty()) { name = v->get_identifier()->name; } const GenericsDeclaration* genericTs = nullptr; // at registering it's null; will be assigned after types resolving - StructData* s_sym = new StructData(std::move(name), v->loc, std::move(fields), genericTs, substitutedTs, v); + StructData* s_sym = new StructData(std::move(name), v->loc, std::move(fields), opcode, v->overflow1023_policy, genericTs, substitutedTs, v); s_sym->base_struct_ref = base_struct_ref; // for `Container`, here is `Container` G.symtable.add_struct(s_sym); diff --git a/tolk/pipeline.h b/tolk/pipeline.h index a46e3e037..4e5beca20 100644 --- a/tolk/pipeline.h +++ b/tolk/pipeline.h @@ -41,6 +41,7 @@ void pipeline_check_inferred_types(); void pipeline_refine_lvalue_for_mutate_arguments(); void pipeline_check_rvalue_lvalue(); void pipeline_check_pure_impure_operations(); +void pipeline_check_serialized_fields(); void pipeline_constant_folding(); void pipeline_optimize_boolean_expressions(); void pipeline_convert_ast_to_legacy_Expr_Op(); diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index 7efccc75e..a44191934 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -174,6 +174,24 @@ StructFieldPtr StructData::find_field(std::string_view field_name) const { return nullptr; } +// formats opcode as "x{...}" or "b{...}" +std::string StructData::PackOpcode::format_as_slice() const { + const int base = prefix_len % 4 == 0 ? 16 : 2; + const int s_len = base == 16 ? prefix_len / 4 : prefix_len; + const char* digits = "0123456789abcdef"; + + std::string result(s_len + 3, '0'); + result[0] = base == 16 ? 'x' : 'b'; + result[1] = '{'; + result[s_len + 3 - 1] = '}'; + int64_t opcode = pack_prefix; + for (int i = s_len - 1; i >= 0 && opcode != 0; --i) { + result[2 + i] = digits[opcode % base]; + opcode /= base; + } + return result; +} + GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD static void fire_error_redefinition_of_symbol(SrcLocation loc, const Symbol* previous) { SrcLocation prev_loc = previous->loc; @@ -226,6 +244,12 @@ void GlobalSymbolTable::add_struct(StructPtr s_sym) { } } +void GlobalSymbolTable::replace_function(FunctionPtr f_sym) { + auto key = key_hash(f_sym->name); + assert(entries.contains(key)); + entries[key] = f_sym; +} + const Symbol* lookup_global_symbol(std::string_view name) { return G.symtable.lookup(name); } diff --git a/tolk/symtable.h b/tolk/symtable.h index ccc8bc527..6bcba2aa7 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -111,7 +111,9 @@ struct FunctionData final : Symbol { flagAcceptsSelf = 512, // is a member function (has `self` first parameter) flagReturnsSelf = 1024, // return type is `self` (returns the mutated 1st argument), calls can be chainable flagReallyUsed = 2048, // calculated via dfs from used functions; declared but unused functions are not codegenerated - flagCompileTimeOnly = 4096, // calculated only at compile-time for constant arguments: `ton("0.05")`, `stringCrc32`, and others + flagCompileTimeVal = 4096, // calculated only at compile-time for constant arguments: `ton("0.05")`, `stringCrc32`, and others + flagCompileTimeGen = 8192, // at compile-time it's handled specially, not as a regular function: `T.toCell`, etc. + flagAllowAnyWidthT = 16384, // for built-in generic functions that is not restricted to be 1-slot type }; int tvm_method_id = EMPTY_TVM_METHOD_ID; @@ -196,7 +198,9 @@ struct FunctionData final : Symbol { bool does_return_self() const { return flags & flagReturnsSelf; } bool does_mutate_self() const { return (flags & flagAcceptsSelf) && parameters[0].is_mutate_parameter(); } bool is_really_used() const { return flags & flagReallyUsed; } - bool is_compile_time_only() const { return flags & flagCompileTimeOnly; } + bool is_compile_time_const_val() const { return flags & flagCompileTimeVal; } + bool is_compile_time_special_gen() const { return flags & flagCompileTimeGen; } + bool is_variadic_width_T_allowed() const { return flags & flagAllowAnyWidthT; } bool does_need_codegen() const; @@ -299,7 +303,26 @@ struct StructFieldData final : Symbol { }; struct StructData final : Symbol { + enum class Overflow1023Policy { // annotation @overflow1023_policy above a struct + not_specified, + suppress, + }; + + struct PackOpcode { + int64_t pack_prefix; + int prefix_len; + + PackOpcode(int64_t pack_prefix, int prefix_len) + : pack_prefix(pack_prefix), prefix_len(prefix_len) {} + + bool exists() const { return prefix_len != 0; } + + std::string format_as_slice() const; // "x{...}" (or "b{...}") + }; + std::vector fields; + PackOpcode opcode; + Overflow1023Policy overflow1023_policy; const GenericsDeclaration* genericTs; const GenericsSubstitutions* substitutedTs; @@ -316,9 +339,11 @@ struct StructData final : Symbol { StructData* mutate() const { return const_cast(this); } void assign_resolved_genericTs(const GenericsDeclaration* genericTs); - StructData(std::string name, SrcLocation loc, std::vector&& fields, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, AnyV ast_root) + StructData(std::string name, SrcLocation loc, std::vector&& fields, PackOpcode opcode, Overflow1023Policy overflow1023_policy, const GenericsDeclaration* genericTs, const GenericsSubstitutions* substitutedTs, AnyV ast_root) : Symbol(std::move(name), loc) , fields(std::move(fields)) + , opcode(opcode) + , overflow1023_policy(overflow1023_policy) , genericTs(genericTs) , substitutedTs(substitutedTs) , ast_root(ast_root) { @@ -350,6 +375,8 @@ class GlobalSymbolTable { void add_type_alias(AliasDefPtr a_sym); void add_struct(StructPtr s_sym); + void replace_function(FunctionPtr f_sym); + const Symbol* lookup(std::string_view name) const { const auto it = entries.find(key_hash(name)); return it == entries.end() ? nullptr : it->second; diff --git a/tolk/tolk.cpp b/tolk/tolk.cpp index fc38571bc..6ead2b149 100644 --- a/tolk/tolk.cpp +++ b/tolk/tolk.cpp @@ -64,6 +64,7 @@ int tolk_proceed(const std::string &entrypoint_filename) { pipeline_refine_lvalue_for_mutate_arguments(); pipeline_check_rvalue_lvalue(); pipeline_check_pure_impure_operations(); + pipeline_check_serialized_fields(); pipeline_constant_folding(); pipeline_optimize_boolean_expressions(); pipeline_convert_ast_to_legacy_Expr_Op(); diff --git a/tolk/tolk.h b/tolk/tolk.h index ad2a5898d..6da5c1ff8 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -1100,6 +1100,7 @@ struct CodeBlob { #endif return ir_idx; } + var_idx_t create_int(SrcLocation loc, int64_t value, const char* desc); bool compute_used_code_vars(); bool compute_used_code_vars(std::unique_ptr& ops, const VarDescrList& var_info, bool edit) const; void print(std::ostream& os, int flags = 0) const; diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp index cd249a5c3..d37a65778 100644 --- a/tolk/type-system.cpp +++ b/tolk/type-system.cpp @@ -721,6 +721,11 @@ bool TypeDataCell::can_rhs_be_assigned(TypePtr rhs) const { if (rhs == this) { return true; } + if (const TypeDataStruct* rhs_struct = rhs->try_as()) { + if (rhs_struct->struct_ref->is_instantiation_of_generic_struct() && rhs_struct->struct_ref->base_struct_ref->name == "Cell") { + return true; // Cell to cell, e.g. `contract.setData(obj.toCell())` + } + } if (const TypeDataAlias* rhs_alias = rhs->try_as()) { return can_rhs_be_assigned(rhs_alias->underlying_type); } @@ -995,6 +1000,9 @@ bool TypeDataCell::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataUnion* to_union = cast_to->try_as()) { return can_be_casted_to_union(this, to_union); } + if (const TypeDataStruct* to_struct = cast_to->try_as()) { // cell as Cell + return to_struct->struct_ref->is_instantiation_of_generic_struct() && to_struct->struct_ref->base_struct_ref->name == "Cell"; + } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } @@ -1107,6 +1115,9 @@ bool TypeDataStruct::can_be_casted_with_as_operator(TypePtr cast_to) const { if (const TypeDataUnion* to_union = cast_to->try_as()) { return can_be_casted_to_union(this, to_union); } + if (cast_to == TypeDataCell::create()) { // Cell as cell + return struct_ref->is_instantiation_of_generic_struct() && struct_ref->base_struct_ref->name == "Cell"; + } if (const TypeDataAlias* to_alias = cast_to->try_as()) { return can_be_casted_with_as_operator(to_alias->underlying_type); } From 89fcd7b845968555700b83daaeb64cdb21fa09e8 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sun, 25 May 2025 21:26:26 +0300 Subject: [PATCH 269/388] [Tolk] Compile-time `sizeof()` function for any variable --- crypto/smartcont/tolk-stdlib/common.tolk | 8 ++++++++ tolk-tester/tests/indexed-access.tolk | 2 +- tolk-tester/tests/invalid-semantics/err-4320.tolk | 13 +++++++++++++ tolk-tester/tests/struct-tests.tolk | 1 + tolk/builtins.cpp | 14 ++++++++++++++ 5 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 tolk-tester/tests/invalid-semantics/err-4320.tolk diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index db3dfc0ff..29b298cd9 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -516,6 +516,14 @@ fun slice.depth(self): int fun builder.depth(self): int asm "BDEPTH"; +/// Returns the number of stack slots anyVariable occupies (works at compile-time). +/// Example: sizeof(nullableInt) = 1, because `int?` is 1 TVM slot holding either NULL or a value. +/// Example: sizeof(somePoint) = 2 for `struct Point { x:int, y: int }`: two fields one slot per each. +/// Useful for debugging or when preparing stack contents for RUNVM. +@pure +fun sizeof(anyVariable: T): int + builtin; + /** Debug primitives. diff --git a/tolk-tester/tests/indexed-access.tolk b/tolk-tester/tests/indexed-access.tolk index f84d2d93e..113124dd5 100644 --- a/tolk-tester/tests/indexed-access.tolk +++ b/tolk-tester/tests/indexed-access.tolk @@ -239,7 +239,7 @@ fun getT() { return (1, 2); } @method_id(120) fun test120() { - return (getT().0 = 3, getT().0 = 4, [getT().0 = 5, getT().0 = 6]); + return (getT().0 = 3, getT().0 = sizeof(getT()) * 2, [getT().0 = 5, getT().0 = 6]); } @method_id(121) diff --git a/tolk-tester/tests/invalid-semantics/err-4320.tolk b/tolk-tester/tests/invalid-semantics/err-4320.tolk new file mode 100644 index 000000000..f57df590e --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4320.tolk @@ -0,0 +1,13 @@ +struct Point { + x: int; + y: int; +} + +fun main() { + return sizeof(Point); +} + +/** +@compilation_should_fail +@stderr `Point` only refers to a type, but is being used as a value here + */ diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index 4bd407f59..46da0207c 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -98,6 +98,7 @@ fun test3() { @method_id(104) fun test4() { var p: Point = { x: 10, y: 20 }; + assert(sizeof(p) == 2, 100); return (p == null, p, p = {x:30,y:40}, p != null); } diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index ebcd2de92..6cee0985d 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1119,6 +1119,17 @@ static AsmOp compile_any_object_to_tuple(std::vector& res, std::vector return exec_op(loc, std::to_string(args.size()) + " TUPLE", n, 1); } +// fun sizeof(anything: T): int; // (returns the number of stack elements) +static AsmOp compile_any_object_sizeof(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(res.size() == 1); + int n = static_cast(args.size()); + res[0].set_const(n); + for (int i = 0; i < n; ++i) { + args[i].unused(); + } + return AsmOp::IntConst(loc, td::make_refint(n)); +} + // fun ton(amount: slice): coins; ton("0.05") replaced by 50000000 at compile-time // same for stringCrc32(constString: slice) and others AsmOp compile_time_only_function(std::vector&, std::vector&, SrcLocation loc) { @@ -1372,6 +1383,9 @@ void define_builtins() { define_builtin_method("debug.dumpStack", debug, {}, Unit, nullptr, compile_dumpstk, 0); + define_builtin_func("sizeof", {typeT}, TypeDataInt::create(), declGenericT, + compile_any_object_sizeof, + FunctionData::flagMarkedAsPure | FunctionData::flagAllowAnyWidthT); // serialization/deserialization methods to/from cells (or, more low-level, slices/builders) // they work with structs (or, more low-level, with arbitrary types) From 5ae66704d8b797ef8dc1d4c691cc4b16b9f9299b Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Wed, 21 May 2025 17:39:19 +0300 Subject: [PATCH 270/388] [Tolk] Support default values for function parameters Now this works: > fun inc(x: int, by: int = 1) Will be used for serialization options. --- tolk-tester/tests/calls-tests.tolk | 164 ++++++++++++++++++ .../tests/invalid-declaration/err-1055.tolk | 19 ++ .../tests/invalid-declaration/err-1665.tolk | 8 + .../tests/invalid-declaration/err-1764.tolk | 9 + .../tests/invalid-semantics/err-4509.tolk | 14 ++ .../tests/invalid-typing/err-6179.tolk | 7 + .../tests/invalid-typing/err-6928.tolk | 7 + tolk-tester/tests/var-apply-tests.tolk | 12 ++ tolk/ast-from-tokens.cpp | 14 +- tolk/ast-replacer.h | 2 +- tolk/ast-replicator.h | 4 +- tolk/ast.h | 13 +- tolk/builtins.cpp | 2 +- tolk/constant-evaluator.cpp | 10 ++ tolk/generics-helpers.cpp | 2 +- tolk/pipe-ast-to-legacy.cpp | 11 +- tolk/pipe-check-inferred-types.cpp | 12 ++ tolk/pipe-constant-folding.cpp | 13 ++ tolk/pipe-infer-types-and-calls.cpp | 46 +++-- tolk/pipe-refine-lvalue-for-mutate.cpp | 2 +- tolk/pipe-register-symbols.cpp | 7 +- tolk/pipe-resolve-identifiers.cpp | 8 +- tolk/pipe-resolve-types.cpp | 9 +- tolk/symtable.cpp | 4 + tolk/symtable.h | 9 +- 25 files changed, 369 insertions(+), 39 deletions(-) create mode 100644 tolk-tester/tests/calls-tests.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1055.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1665.tolk create mode 100644 tolk-tester/tests/invalid-declaration/err-1764.tolk create mode 100644 tolk-tester/tests/invalid-semantics/err-4509.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6179.tolk create mode 100644 tolk-tester/tests/invalid-typing/err-6928.tolk diff --git a/tolk-tester/tests/calls-tests.tolk b/tolk-tester/tests/calls-tests.tolk new file mode 100644 index 000000000..de508a2d5 --- /dev/null +++ b/tolk-tester/tests/calls-tests.tolk @@ -0,0 +1,164 @@ +const ZERO = 0; + +fun sumABDef0(a: int, b: int = 0) { + return a + b; +} + +fun sumADef1BDef2(a: int = ((1)), b: int = 2+ZERO) { + return a + b; +} + +fun getAOrB(a: T, b: T, getA: bool = false): T { + return getA ? a : b; +} + +fun pushToTuple(mutate t: tuple, v: T? = null) { + t.push(v); +} + +struct Point { + x: int; + y: int; +} + +fun Point.create(x: int = 0, y: int = 0): Point { + return {x,y}; +} + +fun Point.incrementX(mutate self, deltaX: int = 1) { + self.x += deltaX; +} + +fun makeCost(cost: coins = ton("0.05")) { + return cost + ton("0.05"); +} + +fun makeUnion(v: int | slice = 0) { + return v; +} + +struct MyOptions { + negate: bool = false, + mulBy: int = 0, +} + +global t105: tuple; + +fun log105(a: int, options: MyOptions = {}) { + if (options.negate) { + a = -a; + } + if (options.mulBy) { + a *= options.mulBy; + } + t105.push(a); +} + +struct AnotherOptions { + c1: int; + leaveSign: bool = true; +} + +fun helper106(a: int, options: AnotherOptions = { c1: 1 }) { + a *= options.c1; + return options.leaveSign ? +a : -a; +} + + +@method_id(101) +fun test1(a: int) { + return ( + sumABDef0(a), + sumABDef0(a, 5), + sumABDef0(100), + sumADef1BDef2(a), + sumADef1BDef2(), + sumADef1BDef2(a, 100), + sumADef1BDef2(200, 100), + ) +} + +@method_id(102) +fun test2(a: int) { + var t = createEmptyTuple(); + pushToTuple(mutate t, getAOrB(sumADef1BDef2(a), sumADef1BDef2(a, a))); + pushToTuple(mutate t, true); + pushToTuple(mutate t, null); + pushToTuple(mutate t, null); + pushToTuple(mutate t, null); + return t; +} + +@method_id(103) +fun test3() { + var p = Point.create(); + p.incrementX(); + var p2 = Point.create(8); + p2.incrementX(p.x += 1); + return (p, p2); +} + +@method_id(104) +fun test4() { + return ( + makeCost(), + makeCost(ton("0.1")), + makeUnion() is int, + makeUnion(), + makeUnion(makeCost()), + ) +} + +@method_id(105) +fun test5() { + t105 = createEmptyTuple(); + log105(1); + log105(1, { negate: true }); + log105(1, { mulBy: 100 }); + log105(1, { mulBy: 100, negate: true }); + log105(1, {}); + return t105; +} + +@method_id(106) +fun test6(l4: bool) { + return ( + helper106(5), + helper106(5, {leaveSign: false, c1: 1}), + helper106(5, {c1: 10}), + helper106(5, {leaveSign: l4, c1: 10}), + helper106(5, {c1: 0}), + ); +} + +fun int.plus(self, v: int = 1) { + return self + v; +} + +@method_id(107) +fun test7() { + return (10.plus(5.plus()), int.plus(4)); +} + +fun createTFrom(v: (int, (U, V)) = (1, (2, 3))) { + return [v.0, v.1.0, v.1.1]; +} + +@method_id(108) +fun test8() { + __expect_type(createTFrom, "((int, (coins, int8))) -> [int, coins, int8]"); + return (createTFrom(), createTFrom((5, (8, createTFrom().0)))); +} + +fun main() {} + +/** +@testcase | 101 | 10 | 10 15 100 12 3 110 300 +@testcase | 102 | 10 | [ 20 -1 (null) (null) (null) ] +@testcase | 103 | | 2 0 10 0 +@testcase | 104 | | 100000000 150000000 -1 0 1 100000000 1 +@testcase | 105 | | [ 1 -1 100 -100 1 ] +@testcase | 106 | 0 | 5 -5 50 -50 0 +@testcase | 107 | | 16 5 +@testcase | 108 | | [ 1 2 3 ] [ 5 8 1 ] + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1055.tolk b/tolk-tester/tests/invalid-declaration/err-1055.tolk new file mode 100644 index 000000000..3b89f53e0 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1055.tolk @@ -0,0 +1,19 @@ + +struct Options { + o1: bool; + o2: bool; +} + +fun getBool() { return true; } + +fun f(x: Options = { + o1: true, + o2: getBool(), +}) { +} + +/** +@compilation_should_fail +@stderr not a constant expression +@stderr o2: getBool() + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1665.tolk b/tolk-tester/tests/invalid-declaration/err-1665.tolk new file mode 100644 index 000000000..4ba5297b9 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1665.tolk @@ -0,0 +1,8 @@ +fun f(a: int, b: int = a) { + +} + +/** +@compilation_should_fail +@stderr symbol `a` is not a constant + */ diff --git a/tolk-tester/tests/invalid-declaration/err-1764.tolk b/tolk-tester/tests/invalid-declaration/err-1764.tolk new file mode 100644 index 000000000..6afaa1849 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration/err-1764.tolk @@ -0,0 +1,9 @@ + +fun increment(mutate x: int = 0) { + x += 1; +} + +/** +@compilation_should_fail +@stderr `mutate` parameter can't have a default value + */ diff --git a/tolk-tester/tests/invalid-semantics/err-4509.tolk b/tolk-tester/tests/invalid-semantics/err-4509.tolk new file mode 100644 index 000000000..d168ddb96 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4509.tolk @@ -0,0 +1,14 @@ +fun f(a: int = 0, b: int) { + +} + +fun main() { + f(1, 2); // ok + return f(); +} + +/** +@compilation_should_fail +@stderr too few arguments in call to `f`, expected 2, have 0 +@stderr f(); + */ diff --git a/tolk-tester/tests/invalid-typing/err-6179.tolk b/tolk-tester/tests/invalid-typing/err-6179.tolk new file mode 100644 index 000000000..73148590b --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6179.tolk @@ -0,0 +1,7 @@ +fun f(x: slice = ton("0.04")) { +} + +/** +@compilation_should_fail +@stderr can not assign `coins` to `slice` + */ diff --git a/tolk-tester/tests/invalid-typing/err-6928.tolk b/tolk-tester/tests/invalid-typing/err-6928.tolk new file mode 100644 index 000000000..a335647ac --- /dev/null +++ b/tolk-tester/tests/invalid-typing/err-6928.tolk @@ -0,0 +1,7 @@ +fun int.f(self, v: (int, int) = (0, 1+"")) { +} + +/** +@compilation_should_fail +@stderr can not apply operator `+` to `int` and `slice` + */ diff --git a/tolk-tester/tests/var-apply-tests.tolk b/tolk-tester/tests/var-apply-tests.tolk index 1c0f164cb..825b48b96 100644 --- a/tolk-tester/tests/var-apply-tests.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -314,6 +314,17 @@ fun testMethodsOfGenericStruct() { return (eq(w1, w2), w2.value = w1.value as int, eq(w1, w2), creator3()); } +fun add3WithDefaults(a: int, b: int = 0, c: int = 0) { + return a + b + c; +} + +@method_id(119) +fun testSavingFunWithDefaults() { + var cc = add3WithDefaults; + __expect_type(cc, "(int, int, int) -> int"); + return cc(1, 2, 3); +} + fun main() {} @@ -343,4 +354,5 @@ fun main() {} @testcase | 116 | | 7 11 80 @testcase | 117 | | 6 8 @testcase | 118 | | 0 10 -1 (null) +@testcase | 119 | | 6 */ diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 6f4b53b79..0ca8b3aeb 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -331,7 +331,17 @@ static AnyV parse_parameter(Lexer& lex, AnyTypeV self_type) { lex.error("`self` parameter should not have a type"); } - return createV(loc, param_name, param_type, declared_as_mutate); + // optional default value + AnyExprV default_value = nullptr; + if (lex.tok() == tok_assign && !is_self) { // `a: int = 0` + if (declared_as_mutate) { + lex.error("`mutate` parameter can't have a default value"); + } + lex.next(); + default_value = parse_expr(lex); + } + + return createV(loc, param_name, param_type, default_value, declared_as_mutate); } static AnyV parse_global_var_declaration(Lexer& lex, const std::vector>& annotations) { @@ -1489,8 +1499,6 @@ static AnyV parse_struct_field(Lexer& lex) { if (lex.tok() == tok_assign) { // `id: int = 3` lex.next(); default_value = parse_expr(lex); - } else { - default_value = createV(lex.cur_location()); } return createV(loc, v_ident, default_value, declared_type); diff --git a/tolk/ast-replacer.h b/tolk/ast-replacer.h index 382f2bd71..03e7d4320 100644 --- a/tolk/ast-replacer.h +++ b/tolk/ast-replacer.h @@ -200,7 +200,7 @@ class ASTReplacerInFunctionBody : public ASTReplacer { public: virtual bool should_visit_function(FunctionPtr fun_ref) = 0; - void start_replacing_in_function(FunctionPtr fun_ref, V v_function) { + virtual void start_replacing_in_function(FunctionPtr fun_ref, V v_function) { replace(v_function->get_body()); } }; diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h index 73ada4100..03a9b0a53 100644 --- a/tolk/ast-replicator.h +++ b/tolk/ast-replicator.h @@ -223,13 +223,13 @@ class ASTReplicator final { return createV(v->loc, clone(v->get_items())); } static V clone(V v) { - return createV(v->loc, v->param_name, clone(v->type_node), v->declared_as_mutate); + return createV(v->loc, v->param_name, clone(v->type_node), v->default_value ? clone(v->default_value) : nullptr, v->declared_as_mutate); } static V clone(V v) { return createV(v->loc, clone(v->get_params())); } static V clone(V v) { - return createV(v->loc, clone(v->get_identifier()), clone(v->get_default_value()), clone(v->type_node)); + return createV(v->loc, clone(v->get_identifier()), v->default_value ? clone(v->default_value) : nullptr, clone(v->type_node)); } static V clone(V v) { return createV(v->loc, clone(v->get_all_fields())); diff --git a/tolk/ast.h b/tolk/ast.h index 96fc96937..cf82a0a17 100644 --- a/tolk/ast.h +++ b/tolk/ast.h @@ -1179,16 +1179,18 @@ struct Vertex final : ASTOtherVararg { template<> // ast_parameter is a parameter of a function in its declaration // example: `fun f(a: int, mutate b: slice)` has 2 parameters +// example: `fun f(a: int = 0)` has 1 parameter with default value struct Vertex final : ASTOtherLeaf { std::string_view param_name; AnyTypeV type_node; // always exists, typing parameters is mandatory + AnyExprV default_value; // default value of the parameter or nullptr bool declared_as_mutate; // declared as `mutate param_name` bool is_underscore() const { return param_name.empty(); } - Vertex(SrcLocation loc, std::string_view param_name, AnyTypeV type_node, bool declared_as_mutate) + Vertex(SrcLocation loc, std::string_view param_name, AnyTypeV type_node, AnyExprV default_value, bool declared_as_mutate) : ASTOtherLeaf(ast_parameter, loc) - , param_name(param_name), type_node(type_node), declared_as_mutate(declared_as_mutate) {} + , param_name(param_name), type_node(type_node), default_value(default_value), declared_as_mutate(declared_as_mutate) {} }; template<> @@ -1314,14 +1316,13 @@ template<> // example: `struct Point { x: int, y: int }` is struct declaration, its body contains 2 fields struct Vertex final : ASTOtherVararg { AnyTypeV type_node; // always exists, typing struct fields is mandatory + AnyExprV default_value; // nullptr if no default auto get_identifier() const { return children.at(0)->as(); } - bool has_default_value() const { return children.at(1)->kind != ast_empty_expression; } - auto get_default_value() const { return child_as_expr(1); } Vertex(SrcLocation loc, V name_identifier, AnyExprV default_value, AnyTypeV type_node) - : ASTOtherVararg(ast_struct_field, loc, {name_identifier, default_value}) - , type_node(type_node) {} + : ASTOtherVararg(ast_struct_field, loc, {name_identifier}) + , type_node(type_node), default_value(default_value) {} }; template<> diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 6cee0985d..1f205f2ca 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -32,7 +32,7 @@ static std::vector define_builtin_parameters(const std::vector(params_types.size()); ++i) { - LocalVarData p_sym("", {}, params_types[i], (i == 0 && is_mutate_self) * LocalVarData::flagMutateParameter, i); + LocalVarData p_sym("", {}, params_types[i], nullptr, (i == 0 && is_mutate_self) * LocalVarData::flagMutateParameter, i); parameters.push_back(std::move(p_sym)); } diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp index c6e2f4331..953cc317c 100644 --- a/tolk/constant-evaluator.cpp +++ b/tolk/constant-evaluator.cpp @@ -266,6 +266,13 @@ struct ConstantExpressionChecker { } } + // `a: Options = {}`, object literal may occur as a default value for parameters + static void handle_object_literal(V v) { + for (int i = 0; i < v->get_num_fields(); ++i) { + visit(v->get_field(i)->get_init_val()); + } + } + static void visit(AnyExprV v) { if (v->try_as() || v->try_as() || v->try_as() || v->try_as()) { return; @@ -285,6 +292,9 @@ struct ConstantExpressionChecker { if (auto v_tensor = v->try_as()) { return handle_tensor(v_tensor); } + if (auto v_obj = v->try_as()) { + return handle_object_literal(v_obj->get_body()); + } if (auto v_par = v->try_as()) { return visit(v_par->get_expr()); } diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp index 34d2f8f33..c2c7a9b54 100644 --- a/tolk/generics-helpers.cpp +++ b/tolk/generics-helpers.cpp @@ -366,7 +366,7 @@ FunctionPtr instantiate_generic_function(FunctionPtr fun_ref, GenericsSubstituti new_parameters.reserve(fun_ref->get_num_params()); for (const LocalVarData& orig_p : fun_ref->parameters) { TypePtr new_param_type = replace_genericT_with_deduced(orig_p.declared_type, allocatedTs); - new_parameters.emplace_back(orig_p.name, orig_p.loc, new_param_type, orig_p.flags, orig_p.param_idx); + new_parameters.emplace_back(orig_p.name, orig_p.loc, new_param_type, orig_p.default_value, orig_p.flags, orig_p.param_idx); } TypePtr new_return_type = replace_genericT_with_deduced(fun_ref->declared_return_type, allocatedTs); TypePtr new_receiver_type = replace_genericT_with_deduced(fun_ref->receiver_type, allocatedTs); diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 9d58a7faa..773d67ecc 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -1380,10 +1380,11 @@ static std::vector process_function_call(V v, Code return transition_to_target_type(std::move(rvect), code, target_type, v); } + // fill args for evaluation: dot object + passed arguments + parameters defaults if not all passed AnyExprV obj_leftmost = v->get_self_obj(); int delta_self = obj_leftmost != nullptr; std::vector args; - args.reserve(delta_self + v->get_num_args()); + args.reserve(fun_ref->get_num_params()); if (delta_self) { args.push_back(obj_leftmost); while (obj_leftmost->kind == ast_function_call && obj_leftmost->as()->get_self_obj() && obj_leftmost->as()->fun_maybe && obj_leftmost->as()->fun_maybe->does_return_self()) { @@ -1393,6 +1394,14 @@ static std::vector process_function_call(V v, Code for (int i = 0; i < v->get_num_args(); ++i) { args.push_back(v->get_arg(i)->get_expr()); } + for (int i = delta_self + v->get_num_args(); i < fun_ref->get_num_params(); ++i) { + LocalVarPtr param_ref = &fun_ref->get_param(i); + tolk_assert(param_ref->has_default_value()); + SrcLocation last_loc = args.empty() ? v->loc : args.back()->loc; + ASTAuxData *aux_data = new AuxData_ForceFiftLocation(last_loc); + auto v_force_loc = createV(last_loc, param_ref->default_value, aux_data, param_ref->declared_type); + args.push_back(v_force_loc); + } // the purpose of tensor_tt ("tensor target type") is to transition `null` to `(int, int)?` and so on // the purpose of calling `pre_compile_tensor_inner` is to have 0-th IR vars to handle return self diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp index 8500ec59a..ba1582559 100644 --- a/tolk/pipe-check-inferred-types.cpp +++ b/tolk/pipe-check-inferred-types.cpp @@ -730,6 +730,18 @@ class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { fire(fun_ref, v_function->get_body()->as()->loc_end, "missing return"); } } + + // visit default values of parameters + for (int i = 0; i < fun_ref->get_num_params(); ++i) { + if (LocalVarPtr param_ref = &fun_ref->get_param(i); param_ref->has_default_value()) { + parent::visit(param_ref->default_value); + + TypePtr inferred_type = param_ref->default_value->inferred_type; + if (!param_ref->declared_type->can_rhs_be_assigned(inferred_type)) { + throw ParseError(param_ref->loc, "can not assign " + to_string(inferred_type) + " to " + to_string(param_ref->declared_type)); + } + } + } } // given `const a = 2 + 3` check types within its init_value diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp index f16bf90a5..74fbb3453 100644 --- a/tolk/pipe-constant-folding.cpp +++ b/tolk/pipe-constant-folding.cpp @@ -150,6 +150,19 @@ class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { return fun_ref->is_code_function() && !fun_ref->is_generic_function(); } + void start_replacing_in_function(FunctionPtr fun_ref, V v_function) override { + // visit default values of parameters + for (int i = 0; i < fun_ref->get_num_params(); ++i) { + if (LocalVarPtr param_ref = &fun_ref->get_param(i); param_ref->has_default_value()) { + check_expression_is_constant(param_ref->default_value); + AnyExprV replaced = replace_in_expression(param_ref->default_value); + param_ref->mutate()->assign_default_value(replaced); + } + } + + parent::replace(v_function->get_body()); + } + // used to replace `ton("0.05")` and other compile-time functions inside fields defaults, etc. AnyExprV replace_in_expression(AnyExprV init_value) { return parent::replace(init_value); diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp index dd83a8dc2..0345ac0aa 100644 --- a/tolk/pipe-infer-types-and-calls.cpp +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -254,6 +254,28 @@ static MethodCallCandidate choose_only_method_to_call(FunctionPtr cur_f, SrcLoca fire(cur_f, loc, msg.str()); } +// given fun `f` and a call `f(a,b,c)`, check that argument count is expected; +// (parameters may have default values, so it's not as trivial as to compare params and args size) +void check_arguments_count_at_fun_call(FunctionPtr cur_f, V v, FunctionPtr called_f, AnyExprV self_obj) { + int delta_self = self_obj != nullptr; + int n_arguments = v->get_num_args() + delta_self; + int n_max_params = called_f->get_num_params(); + int n_min_params = n_max_params; + while (n_min_params && called_f->get_param(n_min_params - 1).has_default_value()) { + n_min_params--; + } + + if (!called_f->does_accept_self() && self_obj) { // static method `Point.create(...)` called as `p.create()` + fire(cur_f, v->loc, "method " + to_string(called_f) + " can not be called via dot\n(it's a static method, it does not accept `self`)"); + } + if (n_max_params < n_arguments) { + fire(cur_f, v->loc, "too many arguments in call to " + to_string(called_f) + ", expected " + std::to_string(n_max_params - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); + } + if (n_arguments < n_min_params) { + fire(cur_f, v->loc, "too few arguments in call to " + to_string(called_f) + ", expected " + std::to_string(n_min_params - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); + } +} + /* * This class handles all types of AST vertices and traverses them, filling all AnyExprV::inferred_type. * Note, that it isn't derived from ASTVisitor, it has manual `switch` over all existing vertex types. @@ -1119,19 +1141,9 @@ class InferTypesAndCallsAndFieldsVisitor final { // so, we have a call `f(args)` or `obj.f(args)`, f is fun_ref (function / method) (code / asm / builtin) // we're going to iterate over passed arguments, and (if generic) infer substitutedTs - // at first, check arguments count (Tolk doesn't have optional parameters, so just compare counts) + // at first, check argument count int delta_self = self_obj != nullptr; - int n_arguments = v->get_num_args() + delta_self; - int n_parameters = fun_ref->get_num_params(); - if (!fun_ref->does_accept_self() && self_obj) { // static method `Point.create(...)` called as `p.create()` - fire(cur_f, v->loc, "method " + to_string(fun_ref) + " can not be called via dot\n(it's a static method, it does not accept `self`)"); - } - if (n_parameters < n_arguments) { - fire(cur_f, v->loc, "too many arguments in call to " + to_string(fun_ref) + ", expected " + std::to_string(n_parameters - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); - } - if (n_arguments < n_parameters) { - fire(cur_f, v->loc, "too few arguments in call to " + to_string(fun_ref) + ", expected " + std::to_string(n_parameters - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); - } + check_arguments_count_at_fun_call(cur_f, v, fun_ref, self_obj); // for every passed argument, we need to infer its type // for generic functions, we need to infer type arguments (substitutedTs) on the fly @@ -1638,6 +1650,16 @@ class InferTypesAndCallsAndFieldsVisitor final { tolk_assert(fun_ref->declared_return_type); } + // visit default values of parameters; to correctly track symbols in `fun f(a: int, b: int = a)`, use flow context + FlowContext params_flow; + for (int i = 0; i < fun_ref->get_num_params(); ++i) { + LocalVarPtr param_ref = &fun_ref->get_param(i); + if (param_ref->has_default_value()) { + params_flow = infer_any_expr(param_ref->default_value, std::move(params_flow), false, param_ref->declared_type).out_flow; + } + params_flow.register_known_type(SinkExpression(param_ref), param_ref->declared_type); + } + assign_fun_full_type(fun_ref, inferred_return_type); fun_ref->mutate()->assign_is_type_inferring_done(); } diff --git a/tolk/pipe-refine-lvalue-for-mutate.cpp b/tolk/pipe-refine-lvalue-for-mutate.cpp index 7692b0a2b..3e02cf773 100644 --- a/tolk/pipe-refine-lvalue-for-mutate.cpp +++ b/tolk/pipe-refine-lvalue-for-mutate.cpp @@ -65,7 +65,7 @@ class RefineLvalueForMutateArgumentsVisitor final : public ASTVisitorFunctionBod } int delta_self = v->get_self_obj() != nullptr; - tolk_assert(fun_ref->get_num_params() == delta_self + v->get_num_args()); + tolk_assert(fun_ref->get_num_params() >= delta_self + v->get_num_args()); if (delta_self && fun_ref->does_mutate_self()) { // for `b.storeInt()`, `b` should become lvalue, since `storeInt` is a method mutating self diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp index 71b2d3bed..7eaa6ac91 100644 --- a/tolk/pipe-register-symbols.cpp +++ b/tolk/pipe-register-symbols.cpp @@ -132,14 +132,13 @@ static StructPtr register_struct(V v, StructPtr base_str for (int i = 0; i < v_body->get_num_fields(); ++i) { auto v_field = v_body->get_field(i); std::string field_name = static_cast(v_field->get_identifier()->name); - AnyExprV default_value = v_field->has_default_value() ? v_field->get_default_value() : nullptr; for (StructFieldPtr prev : fields) { if (UNLIKELY(prev->name == field_name)) { v_field->error("redeclaration of field `" + field_name + "`"); } } - fields.emplace_back(new StructFieldData(static_cast(v_field->get_identifier()->name), v_field->loc, i, v_field->type_node, default_value)); + fields.emplace_back(new StructFieldData(field_name, v_field->loc, i, v_field->type_node, v_field->default_value)); } PackOpcode opcode(0, 0); @@ -176,7 +175,7 @@ static StructPtr register_struct(V v, StructPtr base_str static LocalVarData register_parameter(V v, int idx) { if (v->is_underscore()) { - return LocalVarData{"", v->loc, v->type_node, 0, idx}; + return LocalVarData{"", v->loc, v->type_node, v->default_value, 0, idx}; } int flags = 0; @@ -186,7 +185,7 @@ static LocalVarData register_parameter(V v, int idx) { if (!v->declared_as_mutate && idx == 0 && v->param_name == "self") { flags |= LocalVarData::flagImmutable; } - return LocalVarData(static_cast(v->param_name), v->loc, v->type_node, flags, idx); + return LocalVarData(static_cast(v->param_name), v->loc, v->type_node, v->default_value, flags, idx); } static FunctionPtr register_function(V v, FunctionPtr base_fun_ref = nullptr, std::string override_name = {}, const GenericsSubstitutions* substitutedTs = nullptr) { diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp index ab17c5b42..2f6694469 100644 --- a/tolk/pipe-resolve-identifiers.cpp +++ b/tolk/pipe-resolve-identifiers.cpp @@ -133,7 +133,7 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { static FunctionPtr cur_f; static LocalVarPtr create_local_var_sym(std::string_view name, SrcLocation loc, AnyTypeV declared_type_node, bool immutable, bool lateinit) { - LocalVarData* v_sym = new LocalVarData(static_cast(name), loc, declared_type_node, immutable * LocalVarData::flagImmutable + lateinit * LocalVarData::flagLateInit, -1); + LocalVarData* v_sym = new LocalVarData(static_cast(name), loc, declared_type_node, nullptr, immutable * LocalVarData::flagImmutable + lateinit * LocalVarData::flagLateInit, -1); current_scope.add_local_var(v_sym); return v_sym; } @@ -274,7 +274,11 @@ class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { auto v_block = v->get_body()->as(); current_scope.open_scope(v->loc); for (int i = 0; i < fun_ref->get_num_params(); ++i) { - current_scope.add_local_var(&fun_ref->parameters[i]); + LocalVarPtr param_ref = &fun_ref->parameters[i]; + current_scope.add_local_var(param_ref); + if (param_ref->has_default_value()) { + parent::visit(param_ref->default_value); + } } parent::visit(v_block); current_scope.close_scope(v_block->loc_end); diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 5a2aa0a4a..25d98bb46 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -537,9 +537,12 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { type_nodes_visitor = TypeNodesVisitorResolver(fun_ref, fun_ref->genericTs, fun_ref->substitutedTs, false); for (int i = 0; i < fun_ref->get_num_params(); ++i) { - const LocalVarData& param_var = fun_ref->parameters[i]; - TypePtr declared_type = finalize_type_node(param_var.type_node); - param_var.mutate()->assign_resolved_type(declared_type); + LocalVarPtr param_ref = &fun_ref->parameters[i]; + TypePtr declared_type = finalize_type_node(param_ref->type_node); + param_ref->mutate()->assign_resolved_type(declared_type); + if (param_ref->has_default_value()) { + parent::visit(param_ref->default_value); + } } if (fun_ref->return_type_node) { TypePtr declared_return_type = finalize_type_node(fun_ref->return_type_node); diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp index a44191934..46d394938 100644 --- a/tolk/symtable.cpp +++ b/tolk/symtable.cpp @@ -141,6 +141,10 @@ void LocalVarData::assign_inferred_type(TypePtr inferred_type) { this->declared_type = inferred_type; } +void LocalVarData::assign_default_value(AnyExprV default_value) { + this->default_value = default_value; +} + void AliasDefData::assign_resolved_genericTs(const GenericsDeclaration* genericTs) { if (this->substitutedTs == nullptr) { this->genericTs = genericTs; diff --git a/tolk/symtable.h b/tolk/symtable.h index 6bcba2aa7..15a52d148 100644 --- a/tolk/symtable.h +++ b/tolk/symtable.h @@ -54,20 +54,23 @@ struct LocalVarData final : Symbol { AnyTypeV type_node; // either at declaration `var x:int`, or if omitted, from assigned value `var x=2` TypePtr declared_type = nullptr; // = resolved type_node + AnyExprV default_value = nullptr; // for function parameters, if it has a default value int flags; int param_idx; // 0...N for function parameters, -1 for local vars std::vector ir_idx; - LocalVarData(std::string name, SrcLocation loc, AnyTypeV type_node, int flags, int param_idx) + LocalVarData(std::string name, SrcLocation loc, AnyTypeV type_node, AnyExprV default_value, int flags, int param_idx) : Symbol(std::move(name), loc) , type_node(type_node) + , default_value(default_value) , flags(flags) , param_idx(param_idx) { } - LocalVarData(std::string name, SrcLocation loc, TypePtr declared_type, int flags, int param_idx) + LocalVarData(std::string name, SrcLocation loc, TypePtr declared_type, AnyExprV default_value, int flags, int param_idx) : Symbol(std::move(name), loc) , type_node(nullptr) // for built-in functions (their parameters) , declared_type(declared_type) + , default_value(default_value) , flags(flags) , param_idx(param_idx) { } @@ -77,11 +80,13 @@ struct LocalVarData final : Symbol { bool is_immutable() const { return flags & flagImmutable; } bool is_lateinit() const { return flags & flagLateInit; } bool is_mutate_parameter() const { return flags & flagMutateParameter; } + bool has_default_value() const { return default_value != nullptr; } LocalVarData* mutate() const { return const_cast(this); } void assign_ir_idx(std::vector&& ir_idx); void assign_resolved_type(TypePtr declared_type); void assign_inferred_type(TypePtr inferred_type); + void assign_default_value(AnyExprV default_value); }; struct FunctionBodyCode; From b13bcb4bbc98ba4a82d98dbba89750d5082b8baa Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Wed, 28 May 2025 14:48:17 +0300 Subject: [PATCH 271/388] [Tolk] Struct PackOptions and UnpackOptions Allow to control fromCell/toCell behavior --- crypto/smartcont/tolk-stdlib/common.tolk | 54 ++++- .../tests/invalid-symbol/err-2833.tolk | 13 + tolk-tester/tests/pack-unpack-1.tolk | 8 +- tolk-tester/tests/pack-unpack-4.tolk | 42 ++-- tolk-tester/tests/pack-unpack-5.tolk | 24 +- tolk-tester/tests/pack-unpack-6.tolk | 223 ++++++++++++++++++ tolk-tester/tests/try-catch-tests.tolk | 21 +- tolk/builtins.cpp | 56 ++++- tolk/constant-evaluator.cpp | 16 +- tolk/optimize.cpp | 30 +++ tolk/pack-unpack-api.cpp | 31 ++- tolk/pack-unpack-api.h | 10 +- tolk/pack-unpack-serializers.cpp | 140 ++++++----- tolk/pack-unpack-serializers.h | 18 +- tolk/pipe-ast-to-legacy.cpp | 43 ++-- tolk/pipe-check-serialized-fields.cpp | 4 +- tolk/tolk.h | 3 + 17 files changed, 575 insertions(+), 161 deletions(-) create mode 100644 tolk-tester/tests/invalid-symbol/err-2833.tolk create mode 100644 tolk-tester/tests/pack-unpack-6.tolk diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 29b298cd9..366345385 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -219,6 +219,32 @@ fun commitContractDataAndActions(): void Auto packing structures to/from cells. */ +/// PackOptions allows you to control behavior of `obj.toCell()` and similar functions. +struct PackOptions { + /// when a struct has a field of type `bits128` and similar (it's a slice under the hood), + /// by default, compiler inserts runtime checks (get bits/refs count + compare with 128 + compare with 0); + /// these checks ensure that serialized binary data will be correct, but they cost gas; + /// however, if you guarantee that a slice is valid (for example, it comes from trusted sources), + /// set this option to true to disable runtime checks; + /// note: `int32` and other are always validated for overflow without any extra gas, + /// so this flag controls only rarely used `bytesN` / `bitsN` types + skipBitsNFieldsValidation: bool = false, +} + +/// UnpackOptions allows you to control behavior of `MyStruct.fromCell(c)` and similar functions. +struct UnpackOptions { + // after finished reading all fields from a cell/slice, call [slice.assertEnd] to ensure no remaining data left; + // it's the default behavior, it ensures that you've fully described data you're reading with a struct; + // example: `struct Point { x: int8; y: int8 }`, input "0102" is ok, "0102FF" will throw excno 9; + // note: setting this to false does not decrease gas (DROP from a stack and ENDS cost the same); + // note: this option controls [T.fromCell] and [T.fromSlice], but is ignored by [slice.loadAny] + assertEndAfterReading: bool = true, + + /// this excNo is thrown if a prefix doesn't match, e.g. for `struct (0x01) A` given input "88..."; + /// similarly, for a union type, this is thrown when none of the opcodes match + throwIfOpcodeDoesNotMatch: int = 63, +} + /// Convert anything to a cell (most likely, you'll call it for structures). /// Example: /// ``` @@ -228,7 +254,7 @@ fun commitContractDataAndActions(): void /// Internally, a builder is created, all fields are serialized one by one, and a builder is flushed /// (beginCell() + serialize fields + endCell()). @pure -fun T.toCell(self): Cell +fun T.toCell(self, options: PackOptions = {}): Cell builtin; /// Parse anything from a cell (most likely, you'll call it for structures). @@ -239,7 +265,7 @@ fun T.toCell(self): Cell /// Internally, a cell is unpacked to a slice, and that slice is parsed /// (packedCell.beginParse() + read from slice). @pure -fun T.fromCell(packedCell: cell): T +fun T.fromCell(packedCell: cell, options: UnpackOptions = {}): T builtin; /// Parse anything from a slice (most likely, you'll call it for structures). @@ -252,7 +278,7 @@ fun T.fromCell(packedCell: cell): T /// Note, that a passed slice is NOT mutated, its internal pointer is NOT shifted. /// If you need to mutate it, like `cs.loadInt()`, consider calling `cs.loadAny()`. @pure -fun T.fromSlice(rawSlice: slice): T +fun T.fromSlice(rawSlice: slice, options: UnpackOptions = {}): T builtin; /// Parse anything from a slice, shifting its internal pointer. @@ -262,8 +288,10 @@ fun T.fromSlice(rawSlice: slice): T /// var st: MyStorage = cs.loadAny(); // or cs.loadAny() /// ``` /// Similar to `MyStorage.fromSlice(cs)`, but called as a slice method and mutates the slice. +/// Note: [options.assertEndAfterReading] is ignored by this function, because it's actually intended +/// to read data from the middle. @pure -fun slice.loadAny(mutate self): T +fun slice.loadAny(mutate self, options: UnpackOptions = {}): T builtin; /// Skip anything in a slice, shifting its internal pointer. @@ -274,7 +302,7 @@ fun slice.loadAny(mutate self): T /// cs.skipAny(); // skips 64 bits /// ``` @pure -fun slice.skipAny(mutate self): self +fun slice.skipAny(mutate self, options: UnpackOptions = {}): self builtin; /// Store anything to a builder. @@ -284,7 +312,19 @@ fun slice.skipAny(mutate self): self /// var b = beginCell().storeUint(32).storeAny(msgBody).endCell(); /// ``` @pure -fun builder.storeAny(mutate self, v: T): self +fun builder.storeAny(mutate self, v: T, options: PackOptions = {}): self + builtin; + +/// Returns serialization prefix of a struct. Works at compile-time. +/// Example: for `struct (0xF0) AssetRegular { ... }` will return `240`. +@pure +fun T.getDeclaredPackPrefix(): int + builtin; + +/// Returns serialization prefix length of a struct. Works at compile-time. +/// Example: for `struct (0xF0) AssetRegular { ... }` will return `16`. +@pure +fun T.getDeclaredPackPrefixLen(): int builtin; /// Cell represents a typed cell reference (as opposed to untyped `cell`). @@ -316,7 +356,7 @@ struct Cell { /// var extra = st.extra.load(); // it's ExtraData, unpacked from loaded ref /// ``` @pure -fun Cell.load(self): T +fun Cell.load(self, options: UnpackOptions = {}): T builtin; /// Converts a typed cell into a slice. diff --git a/tolk-tester/tests/invalid-symbol/err-2833.tolk b/tolk-tester/tests/invalid-symbol/err-2833.tolk new file mode 100644 index 000000000..8a7d249ff --- /dev/null +++ b/tolk-tester/tests/invalid-symbol/err-2833.tolk @@ -0,0 +1,13 @@ +struct Point { + x: int8; + y: int8; +} + +fun main() { + return Point.getDeclaredPackPrefix(); +} + +/** +@compilation_should_fail +@stderr type `Point` does not have a serialization prefix + */ diff --git a/tolk-tester/tests/pack-unpack-1.tolk b/tolk-tester/tests/pack-unpack-1.tolk index 00767042c..002df87dd 100644 --- a/tolk-tester/tests/pack-unpack-1.tolk +++ b/tolk-tester/tests/pack-unpack-1.tolk @@ -48,7 +48,7 @@ fun test3() { @method_id(104) fun test4() { var s = stringHexToSlice("000000010000000200000003"); - var p = Point.fromSlice(s); // does not mutate s + var p = Point.fromSlice(s, {assertEndAfterReading: false}); // does not mutate s return (p, s.remainingBitsCount()); } @@ -114,11 +114,11 @@ fun main(c: cell) { 32 STI // b ENDC // c CTOS // s - 32 PUSHINT // s '12=32 + 32 PUSHINT // s '15=32 SDSKIPFIRST // s - 32 PUSHINT // s '13=32 + 32 PUSHINT // s '16=32 SDSKIPFIRST // s - SBITS // '14 + SBITS // '17 }> """ diff --git a/tolk-tester/tests/pack-unpack-4.tolk b/tolk-tester/tests/pack-unpack-4.tolk index 4839032dc..992a0664a 100644 --- a/tolk-tester/tests/pack-unpack-4.tolk +++ b/tolk-tester/tests/pack-unpack-4.tolk @@ -92,7 +92,7 @@ fun invalid105_slice(): slice asm "b{1100} PUSHSLICE"; // prefix 0b11 doesn't e @method_id(105) fun test5() { try { - var u = U105.fromSlice(invalid105_slice()); + var u = U105.fromSlice(invalid105_slice(), {throwIfOpcodeDoesNotMatch: 9}); return (u is int8) ? 8 : -8; } catch (excode) { return excode; @@ -125,37 +125,37 @@ fun main() {} """ test6 PROC:<{ // b{010000000000001111} PUSHSLICE // s - b{00} SDBEGINSQ // s '6 + b{00} SDBEGINSQ // s '8 IF:<{ // s - 8 LDI // '10 s - DROP // '10 - 42 PUSHINT // 'USlot1 'UTag=42 + 8 LDI // '12 s + 42 PUSHINT // 'USlot1 s 'UTag=42 }>ELSE<{ // s - b{01} SDBEGINSQ // s '6 + b{01} SDBEGINSQ // s '8 IF:<{ // s - 16 LDI // '15 s - DROP // '15 - 44 PUSHINT // 'USlot1 'UTag=44 + 16 LDI // '17 s + 44 PUSHINT // 'USlot1 s 'UTag=44 }>ELSE<{ // s - b{10} SDBEGINSQ // s '6 + b{10} SDBEGINSQ // s '8 IF:<{ // s - 32 LDI // '20 s - DROP // '20 - 46 PUSHINT // 'USlot1 'UTag=46 + 32 LDI // '22 s + 46 PUSHINT // 'USlot1 s 'UTag=46 }>ELSE<{ // s - b{11} SDBEGINSQ // s '6 - DROP // s - 64 LDI // '25 s - DROP // '25 - 48 PUSHINT // 'USlot1 'UTag=48 + b{11} SDBEGINSQ // s '8 + IFNOTJMP:<{ // s + 63 THROW + }> + 64 LDI // '27 s + 48 PUSHINT // 'USlot1 s 'UTag=48 }> }> - }> // u.USlot1 u.UTag - 44 EQINT // u.USlot1 '27 + }> + SWAP // 'USlot1 'UTag s + ENDS // u.USlot1 u.UTag + 44 EQINT // u.USlot1 '29 IFJMP:<{ // u.USlot1 }> // u.USlot1 DROP // - -1 PUSHINT // '29=-1 + -1 PUSHINT // '31=-1 }> """ */ diff --git a/tolk-tester/tests/pack-unpack-5.tolk b/tolk-tester/tests/pack-unpack-5.tolk index 068d031f2..9644d7331 100644 --- a/tolk-tester/tests/pack-unpack-5.tolk +++ b/tolk-tester/tests/pack-unpack-5.tolk @@ -166,18 +166,38 @@ fun test9() { return (Test9_f1.estimatePackSize(), Test9_f2.estimatePackSize(), Test9_f3.estimatePackSize(), Test9_f4.estimatePackSize()); } +struct Test10_1 { + a: int32; + b: builder; // unpredictable +} + +type Test10_2 = (Test10_1, bool?, RemainingBitsAndRefs); + +@method_id(110) +fun test10() { + return (Test10_1.estimatePackSize(), Test10_2.estimatePackSize()); +} + +@method_id(120) +fun test20() { + return (Test7_1 .getDeclaredPackPrefixLen(), Test7_1 .getDeclaredPackPrefix(), + CellInner8_1.getDeclaredPackPrefixLen(), CellInner8_1.getDeclaredPackPrefix()); +} + fun main() { __expect_type(int8.estimatePackSize(), "[int, int, int, int]"); } /** -@testcase | 101 | | [ 32 32 0 0 ] [ 64 64 0 0 ] [ 1 1 0 0 ] [ 1 1 0 0 ] [ 0 0 0 0 ] [ 4 124 0 0 ] [ 6 6 0 0 ] [ 64 64 0 0 ] +@testcase | 101 | | [ 32 32 0 0 ] [ 64 64 0 0 ] [ 1 1 0 0 ] [ 1 1 0 0 ] [ 0 9999 0 4 ] [ 4 124 0 0 ] [ 6 6 0 0 ] [ 64 64 0 0 ] @testcase | 102 | | [ 32 32 0 0 ] [ 1 33 0 0 ] [ 1 33 0 0 ] [ 17 33 0 0 ] @testcase | 103 | | [ 67 199 0 0 ] [ 69 599 0 0 ] @testcase | 104 | | [ 4 534 0 0 ] [ 2 536 0 0 ] [ 6 1070 0 0 ] -@testcase | 105 | | [ 2 2 2 4 ] [ 5 5 0 5 ] [ 4 152 1 4 ] +@testcase | 105 | | [ 2 2 2 4 ] [ 5 9999 0 9 ] [ 4 152 1 4 ] @testcase | 106 | | [ 64 64 0 0 ] [ 5 37 1 1 ] [ 5 65 0 1 ] @testcase | 107 | | [ 16 16 0 0 ] [ 16 16 0 0 ] @testcase | 108 | | [ 10 275 0 0 ] [ 34 158 0 0 ] [ 47 1143 0 1 ] @testcase | 109 | | [ 34 130 0 0 ] [ 33 159 0 0 ] [ 4 8 0 0 ] [ 3 7 0 0 ] +@testcase | 110 | | [ 32 9999 0 4 ] [ 33 9999 0 8 ] +@testcase | 120 | | 16 4128 1 1 */ diff --git a/tolk-tester/tests/pack-unpack-6.tolk b/tolk-tester/tests/pack-unpack-6.tolk new file mode 100644 index 000000000..049109ad7 --- /dev/null +++ b/tolk-tester/tests/pack-unpack-6.tolk @@ -0,0 +1,223 @@ +struct (0x01) CounterIncrement { byValue: int8; } +struct (0x03) CounterDecrementBy1 {} +type CounterMsg = CounterIncrement | CounterDecrementBy1; + +struct SomeBytesFields { + f1: bytes1; +} + +@method_id(101) +fun test1() { + try { + return CounterIncrement.fromSlice(stringHexToSlice("880f"), {throwIfOpcodeDoesNotMatch: 101}).byValue as int; + } catch (excno) { + return excno; + } +} + +@method_id(102) +fun test2() { + try { + return CounterIncrement.fromSlice(stringHexToSlice("890f")).byValue as int; + } catch (excno) { + return excno; + } +} + +@method_id(103) +fun test3() { + var cc: Cell = { + tvmCell: beginCell().storeSlice(stringHexToSlice("0109ab")).endCell() + }; + return cc.load({assertEndAfterReading: false}); +} + +@method_id(104) +fun test4() { + try { + var msg = CounterMsg.fromSlice(stringHexToSlice("88"), {throwIfOpcodeDoesNotMatch: 104, assertEndAfterReading: false}); + __expect_type(msg, "CounterMsg"); + return (-1, msg is CounterIncrement); + } catch (excno) { + return (excno, null); + } +} + +@method_id(105) +fun test5() { + return SomeBytesFields { f1: stringHexToSlice("11") as bytes1 }.toCell().hash() & 0xFFFF; +} + +@method_id(106) +fun test6() { + return SomeBytesFields { f1: stringHexToSlice("11") as bytes1 }.toCell({skipBitsNFieldsValidation: true}).hash() & 0xFFFF; +} + +@method_id(107) +fun test7(believe: bool) { + try { + return SomeBytesFields { f1: stringHexToSlice("ffff") as bytes1 }.toCell({skipBitsNFieldsValidation: believe}).hash() & 0xFFFF; + } catch (excno) { + return excno; + } +} + +@method_id(108) +fun test8() { + return CounterIncrement.fromSlice(stringHexToSlice("010f"), { + throwIfOpcodeDoesNotMatch: 0xFFFF + }).byValue; +} + +@method_id(109) +fun test9() { + return CounterMsg.fromSlice(stringHexToSlice("010f"), { + throwIfOpcodeDoesNotMatch: 0xFFFF, + assertEndAfterReading: false, + }) is CounterIncrement; +} + +@method_id(110) +fun test10() { + var c = beginCell().storeUint(123, 64).endCell() as Cell; + try { + var b = c.load(); + return -(b.f1 as slice).remainingBitsCount(); + } catch (excno) { + return excno; + } +} + +@method_id(111) +fun test11() { + var c = beginCell().storeUint(123, 64).endCell() as Cell; + var b = c.load({assertEndAfterReading: false}); + return -(b.f1 as slice).remainingBitsCount(); +} + +fun main(){} + +/** +@testcase | 101 | | 101 +@testcase | 102 | | 63 +@testcase | 103 | | 9 +@testcase | 104 | | 104 (null) +@testcase | 105 | | 36896 +@testcase | 106 | | 36896 +@testcase | 107 | -1 | 2142 +@testcase | 107 | 0 | 9 +@testcase | 108 | | 15 +@testcase | 109 | | -1 +@testcase | 110 | | 9 +@testcase | 111 | | -8 + +@fif_codegen +""" +x{880f} PUSHSLICE +x{01} SDBEGINSQ +101 THROWIFNOT +""" + +@fif_codegen +""" +x{890f} PUSHSLICE +x{01} SDBEGINSQ +63 THROWIFNOT +8 LDI +ENDS +RETALT +""" + +@fif_codegen +""" + test3 PROC:<{ + NEWC + x{0109ab} PUSHSLICE + STSLICER + ENDC + CTOS + x{01} SDBEGINSQ + 63 THROWIFNOT + 8 LDI + DROP + }> +""" + +@fif_codegen +""" +IF:<{ + DROP + 129 PUSHINT +}>ELSE<{ + x{03} SDBEGINSQ + NIP + IFNOTJMP:<{ + 104 THROW + }> + 130 PUSHINT +}> +""" + +@fif_codegen +""" + test5 PROC:<{ + x{11} PUSHSLICE + NEWC + OVER + SBITREFS + 9 THROWIF + 8 EQINT + 9 THROWIFNOT + SWAP + STSLICER +""" + +@fif_codegen +""" + test6 PROC:<{ + x{11} PUSHSLICE + NEWC + SWAP + STSLICER + ENDC + HASHCU +""" + +@fif_codegen +""" + test8 PROC:<{ // + x{010f} PUSHSLICE // '0 + 16 PUSHPOW2DEC // s '2=65535 + SWAP // '2=65535 s + x{01} SDBEGINSQ // '2=65535 s '4 + s1 s2 XCHG // s '2=65535 '4 + THROWANYIFNOT // s + 8 LDI // '9 s + ENDS // '9 + }> +""" + +@fif_codegen +""" + test9 PROC:<{ // + x{010f} PUSHSLICE // s + x{01} SDBEGINSQ // s '6 + IF:<{ // s + DROP // + 129 PUSHINT // 'UTag=129 + }>ELSE<{ // s + x{03} SDBEGINSQ // s '6 + NIP // '6 + IFNOTJMP:<{ // + 16 PUSHPOW2DEC + THROWANY + }> + 130 PUSHINT // 'UTag=130 + }> + 129 PUSHINT // 'UTag '17=129 + SWAP // '17=129 'UTag + EQUAL // '16 + }> +""" + + */ diff --git a/tolk-tester/tests/try-catch-tests.tolk b/tolk-tester/tests/try-catch-tests.tolk index b0a50aacb..be0362e39 100644 --- a/tolk-tester/tests/try-catch-tests.tolk +++ b/tolk-tester/tests/try-catch-tests.tolk @@ -236,6 +236,16 @@ fun testCodegen3(numberId: int, paramVal: cell) { paramVal.beginParse(); } +@method_id(113) +fun testBigExcno() { + try { + throw 2048; + return 10; + } catch (excno) { + return excno; + } +} + fun main() { } @@ -264,8 +274,9 @@ fun main() { @testcase | 110 | 0 | 5 @testcase | 111 | -1 | 123 @testcase | 111 | 0 | 456 +@testcase | 113 | | 2048 -@code_hash 57361460846265694653029920796509802052573595128418810728101968091567195330515 +@code_hash 26411074751358281917876479601714698321674833208179417883888954015536273820035 @fif_codegen """ @@ -317,4 +328,12 @@ fun main() { DROP // }> """ + +@fif_codegen +""" +<{ + 11 PUSHPOW2 + THROWANY +}>CONT +""" */ diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 1f205f2ca..39075452d 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -18,6 +18,7 @@ #include "compiler-state.h" #include "type-system.h" #include "generics-helpers.h" +#include "ast.h" namespace tolk { using namespace std::literals::string_literals; @@ -938,7 +939,9 @@ static AsmOp compile_cmp_int(std::vector& res, std::vector& static AsmOp compile_throw(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(res.empty() && args.size() == 1); VarDescr& x = args[0]; - if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { + if (x.is_int_const() && x.int_const >= 0) { + // in Fift assembler, "N THROW" is valid if N < 2048; for big N (particularly, widely used 0xFFFF) + // we now still generate "N THROW", and later, in optimizer, transform it to "PUSHINT" + "THROWANY" x.unused(); return exec_arg_op(loc, "THROW", x.int_const, 0, 0); } else { @@ -1181,6 +1184,8 @@ void define_builtins() { // see patch_builtins_after_stdlib_loaded() below TypePtr debug = TypeDataUnknown::create(); TypePtr CellT = TypeDataUnknown::create(); + TypePtr PackOptions = TypeDataUnknown::create(); + TypePtr UnpackOptions = TypeDataUnknown::create(); // builtin operators // they are internally stored as functions, because at IR level, there is no difference @@ -1389,28 +1394,34 @@ void define_builtins() { // serialization/deserialization methods to/from cells (or, more low-level, slices/builders) // they work with structs (or, more low-level, with arbitrary types) - define_builtin_method("T.toCell", typeT, {typeT}, CellT, declReceiverT, + define_builtin_method("T.toCell", typeT, {typeT, PackOptions}, CellT, declReceiverT, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); - define_builtin_method("T.fromCell", typeT, {TypeDataCell::create()}, typeT, declReceiverT, + define_builtin_method("T.fromCell", typeT, {TypeDataCell::create(), UnpackOptions}, typeT, declReceiverT, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); - define_builtin_method("T.fromSlice", typeT, {Slice}, typeT, declReceiverT, + define_builtin_method("T.fromSlice", typeT, {Slice, UnpackOptions}, typeT, declReceiverT, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); define_builtin_method("T.estimatePackSize", typeT, {}, TypeDataBrackets::create({TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create(), TypeDataInt::create()}), declReceiverT, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); - define_builtin_method("Cell.load", CellT, {CellT}, typeT, declReceiverT, + define_builtin_method("T.getDeclaredPackPrefix", typeT, {}, Int, declReceiverT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal | FunctionData::flagAllowAnyWidthT); + define_builtin_method("T.getDeclaredPackPrefixLen", typeT, {}, Int, declReceiverT, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeVal | FunctionData::flagAllowAnyWidthT); + define_builtin_method("Cell.load", CellT, {CellT, UnpackOptions}, typeT, declReceiverT, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagAllowAnyWidthT); - define_builtin_method("slice.loadAny", Slice, {Slice}, typeT, declGenericT, + define_builtin_method("slice.loadAny", Slice, {Slice, UnpackOptions}, typeT, declGenericT, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); - define_builtin_method("slice.skipAny", Slice, {Slice}, Slice, declGenericT, + define_builtin_method("slice.skipAny", Slice, {Slice, UnpackOptions}, Slice, declGenericT, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); - define_builtin_method("builder.storeAny", Builder, {Builder, typeT}, Builder, declGenericT, + define_builtin_method("builder.storeAny", Builder, {Builder, typeT, PackOptions}, Builder, declGenericT, compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); @@ -1439,11 +1450,38 @@ void patch_builtins_after_stdlib_loaded() { lookup_function("debug.dumpStack")->mutate()->receiver_type = debug; StructPtr struct_ref_CellT = lookup_global_symbol("Cell")->try_as(); + StructPtr struct_ref_PackOptions = lookup_global_symbol("PackOptions")->try_as(); + StructPtr struct_ref_UnpackOptions = lookup_global_symbol("UnpackOptions")->try_as(); TypePtr CellT = TypeDataGenericTypeWithTs::create(struct_ref_CellT, nullptr, {typeT}); + TypePtr PackOptions = TypeDataStruct::create(struct_ref_PackOptions); + TypePtr UnpackOptions = TypeDataStruct::create(struct_ref_UnpackOptions); + + // in stdlib, there is a default parameter `options = {}`; since default parameters are evaluated with AST, + // emulate its presence in built-in functions; it looks ugly, but currently I don't have a better solution + auto v_empty_PackOptions = createV({}, nullptr, createV({}, {})); + v_empty_PackOptions->assign_struct_ref(struct_ref_PackOptions); + v_empty_PackOptions->assign_inferred_type(PackOptions); + auto v_empty_UnpackOptions = createV({}, nullptr, createV({}, {})); + v_empty_UnpackOptions->assign_struct_ref(struct_ref_UnpackOptions); + v_empty_UnpackOptions->assign_inferred_type(UnpackOptions); + lookup_function("T.toCell")->mutate()->declared_return_type = CellT; + lookup_function("T.toCell")->mutate()->parameters[1].declared_type = PackOptions; + lookup_function("T.toCell")->mutate()->parameters[1].default_value = v_empty_PackOptions; + lookup_function("T.fromCell")->mutate()->parameters[1].declared_type = UnpackOptions; + lookup_function("T.fromCell")->mutate()->parameters[1].default_value = v_empty_UnpackOptions; + lookup_function("T.fromSlice")->mutate()->parameters[1].declared_type = UnpackOptions; + lookup_function("T.fromSlice")->mutate()->parameters[1].default_value = v_empty_UnpackOptions; lookup_function("Cell.load")->mutate()->parameters[0].declared_type = CellT; + lookup_function("Cell.load")->mutate()->parameters[1].declared_type = UnpackOptions; + lookup_function("Cell.load")->mutate()->parameters[1].default_value = v_empty_UnpackOptions; lookup_function("Cell.load")->mutate()->receiver_type = CellT; - lookup_function("T.toCell")->mutate()->declared_return_type = CellT; + lookup_function("slice.loadAny")->mutate()->parameters[1].declared_type = UnpackOptions; + lookup_function("slice.loadAny")->mutate()->parameters[1].default_value = v_empty_UnpackOptions; + lookup_function("slice.skipAny")->mutate()->parameters[1].declared_type = UnpackOptions; + lookup_function("slice.skipAny")->mutate()->parameters[1].default_value = v_empty_UnpackOptions; + lookup_function("builder.storeAny")->mutate()->parameters[2].declared_type = PackOptions; + lookup_function("builder.storeAny")->mutate()->parameters[2].default_value = v_empty_PackOptions; } } // namespace tolk diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp index 953cc317c..20c316fdb 100644 --- a/tolk/constant-evaluator.cpp +++ b/tolk/constant-evaluator.cpp @@ -17,6 +17,7 @@ #include "constant-evaluator.h" #include "ast.h" #include "tolk.h" +#include "type-system.h" #include "openssl/digest.hpp" #include "crypto/common/util.h" #include "td/utils/crypto.h" @@ -161,8 +162,21 @@ static td::RefInt256 parse_nanotons_as_floating_string(SrcLocation loc, std::str // given `ton("0.05")` evaluate it to 50000000 // given `stringCrc32("some_str")` evaluate it // etc. -// currently, all compile-time functions accept 1 argument, a literal string static CompileTimeFunctionResult parse_vertex_call_to_compile_time_function(V v, std::string_view f_name) { + // most functions accept 1 argument, but static compile-time methods like `MyStruct.getDeclaredPackPrefix()` have 0 args + if (v->get_num_args() == 0) { + TypePtr receiver = v->fun_maybe->receiver_type; + f_name = v->fun_maybe->method_name; + + if (f_name == "getDeclaredPackPrefix" || f_name == "getDeclaredPackPrefixLen") { + const TypeDataStruct* t_struct = receiver->try_as(); + if (!t_struct || !t_struct->struct_ref->opcode.exists()) { + throw ParseError(v->loc, "type `" + receiver->as_human_readable() + "` does not have a serialization prefix"); + } + return td::make_refint(f_name.ends_with('x') ? t_struct->struct_ref->opcode.pack_prefix : t_struct->struct_ref->opcode.prefix_len); + } + } + tolk_assert(v->get_num_args() == 1); // checked by type inferring AnyExprV v_arg = v->get_arg(0)->get_expr(); diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp index 332069c03..e6d6b070d 100644 --- a/tolk/optimize.cpp +++ b/tolk/optimize.cpp @@ -135,6 +135,35 @@ bool Optimizer::find_const_op(int* op_idx, int cst) { return false; } +// purpose: transform `65535 THROW` to `PUSHINT` + `THROWANY`; +// such a technique allows pushing a number onto a stack just before THROW, even if a variable is created in advance; +// used for `T.fromSlice(s, {code:0xFFFF})`, where `tmp = 0xFFFF` + serialization match + `else throw tmp` is generated; +// but since it's constant, it transforms to (unused 0xFFFF) + ... + else "65535 THROW", unwrapped here +bool Optimizer::detect_rewrite_big_THROW() { + bool is_throw = op_[0]->is_custom() && op_[0]->op.ends_with(" THROW"); + if (!is_throw) { + return false; + } + + std::string_view s_num_throw = op_[0]->op; + size_t sp = s_num_throw.find(' '); + if (sp != s_num_throw.rfind(' ') || s_num_throw[0] < '1' || s_num_throw[0] > '9') { + return false; + } + + std::string s_number(s_num_throw.substr(0, sp)); + uint64_t excno = std::stoul(s_number); + if (excno < 2048) { // "9 THROW" left as is, but "N THROW" where N>=2^11 is invalid for Fift + return false; + } + + p_ = 1; + q_ = 2; + oq_[0] = std::make_unique(AsmOp::IntConst(op_[0]->loc, td::make_refint(excno))); + oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->loc, "THROWANY", 1, 0)); + return true; +} + bool Optimizer::is_push_const(int* i, int* c) const { return pb_ >= 3 && pb_ <= l2_ && tr_[pb_ - 1].is_push_const(i, c); } @@ -553,6 +582,7 @@ bool Optimizer::find_at_least(int pb) { (is_xchg(&i, &j) && rewrite(AsmOp::Xchg(loc, i, j))) || (is_push(&i) && rewrite(AsmOp::Push(loc, i))) || (is_pop(&i) && rewrite(AsmOp::Pop(loc, i))) || (is_pop_pop(&i, &j) && rewrite(AsmOp::Pop(loc, i), AsmOp::Pop(loc, j))) || (is_xchg_xchg(&i, &j, &k, &l) && rewrite(AsmOp::Xchg(loc, i, j), AsmOp::Xchg(loc, k, l))) || + detect_rewrite_big_THROW() || (!(mode_ & 1) && ((is_rot() && rewrite(AsmOp::Custom(loc, "ROT", 3, 3))) || (is_rotrev() && rewrite(AsmOp::Custom(loc, "-ROT", 3, 3))) || (is_2dup() && rewrite(AsmOp::Custom(loc, "2DUP", 2, 4))) || diff --git a/tolk/pack-unpack-api.cpp b/tolk/pack-unpack-api.cpp index 14a20d922..6af6650f4 100644 --- a/tolk/pack-unpack-api.cpp +++ b/tolk/pack-unpack-api.cpp @@ -177,13 +177,14 @@ bool check_struct_can_be_packed_or_unpacked(TypePtr any_type, bool is_pack, std: // -std::vector generate_pack_struct_to_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_obj) { +std::vector generate_pack_struct_to_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_obj, const std::vector& ir_options) { FunctionPtr f_beginCell = lookup_function("beginCell"); FunctionPtr f_endCell = lookup_function("builder.endCell"); std::vector rvect_builder = code.create_var(TypeDataBuilder::create(), loc, "b"); code.emplace_back(loc, Op::_Call, rvect_builder, std::vector{}, f_beginCell); - PackContext ctx(code, loc, rvect_builder); + tolk_assert(ir_options.size() == 1); // struct PackOptions + PackContext ctx(code, loc, rvect_builder, ir_options); ctx.generate_pack_any(any_type, std::move(ir_obj)); std::vector rvect_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(cell)"); @@ -192,41 +193,51 @@ std::vector generate_pack_struct_to_cell(CodeBlob& code, SrcLocation return rvect_cell; } -std::vector generate_pack_struct_to_builder(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_builder, std::vector&& ir_obj) { - PackContext ctx(code, loc, ir_builder); // mutate this builder +std::vector generate_pack_struct_to_builder(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_builder, std::vector&& ir_obj, const std::vector& ir_options) { + PackContext ctx(code, loc, ir_builder, ir_options); // mutate this builder ctx.generate_pack_any(any_type, std::move(ir_obj)); return ir_builder; // return mutated builder } -std::vector generate_unpack_struct_from_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, bool mutate_slice) { +std::vector generate_unpack_struct_from_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, bool mutate_slice, const std::vector& ir_options) { if (!mutate_slice) { std::vector slice_copy = code.create_var(TypeDataSlice::create(), loc, "s"); code.emplace_back(loc, Op::_Let, slice_copy, std::move(ir_slice)); ir_slice = std::move(slice_copy); } - UnpackContext ctx(code, loc, std::move(ir_slice)); + tolk_assert(ir_options.size() == 2); // struct UnpackOptions + UnpackContext ctx(code, loc, std::move(ir_slice), ir_options); std::vector rvect_struct = ctx.generate_unpack_any(any_type); tolk_assert(any_type->get_width_on_stack() == static_cast(rvect_struct.size())); + // slice.loadAny() ignores options.assertEndAfterReading, because it's intended to read data in the middle + if (!mutate_slice && !estimate_serialization_size(any_type).is_unpredictable_infinity()) { + ctx.assertEndIfOption(); + } return rvect_struct; } -std::vector generate_unpack_struct_from_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_cell) { +std::vector generate_unpack_struct_from_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_cell, const std::vector& ir_options) { FunctionPtr f_beginParse = lookup_function("cell.beginParse"); std::vector ir_slice = code.create_var(TypeDataSlice::create(), loc, "s"); code.emplace_back(loc, Op::_Call, ir_slice, std::move(ir_cell), f_beginParse); - UnpackContext ctx(code, loc, std::move(ir_slice)); + tolk_assert(ir_options.size() == 2); // struct UnpackOptions + UnpackContext ctx(code, loc, std::move(ir_slice), ir_options); std::vector rvect_struct = ctx.generate_unpack_any(any_type); tolk_assert(any_type->get_width_on_stack() == static_cast(rvect_struct.size())); + // if a struct has RemainingBitsAndRefs, don't test it for assertEnd + if (!estimate_serialization_size(any_type).is_unpredictable_infinity()) { + ctx.assertEndIfOption(); + } return rvect_struct; } -std::vector generate_skip_struct_in_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice) { - UnpackContext ctx(code, loc, ir_slice); // mutate this slice +std::vector generate_skip_struct_in_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, const std::vector& ir_options) { + UnpackContext ctx(code, loc, ir_slice, ir_options); // mutate this slice ctx.generate_skip_any(any_type); return ir_slice; // return mutated slice diff --git a/tolk/pack-unpack-api.h b/tolk/pack-unpack-api.h index b0a4b29ef..7737125d4 100644 --- a/tolk/pack-unpack-api.h +++ b/tolk/pack-unpack-api.h @@ -23,11 +23,11 @@ namespace tolk { bool check_struct_can_be_packed_or_unpacked(TypePtr any_type, bool is_pack, std::string& because_msg); -std::vector generate_pack_struct_to_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_obj); -std::vector generate_pack_struct_to_builder(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_builder, std::vector&& ir_obj); -std::vector generate_unpack_struct_from_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, bool mutate_slice); -std::vector generate_unpack_struct_from_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_cell); -std::vector generate_skip_struct_in_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice); +std::vector generate_pack_struct_to_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_obj, const std::vector& ir_options); +std::vector generate_pack_struct_to_builder(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_builder, std::vector&& ir_obj, const std::vector& ir_options); +std::vector generate_unpack_struct_from_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, bool mutate_slice, const std::vector& ir_options); +std::vector generate_unpack_struct_from_cell(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_cell, const std::vector& ir_options); +std::vector generate_skip_struct_in_slice(CodeBlob& code, SrcLocation loc, TypePtr any_type, std::vector&& ir_slice, const std::vector& ir_options); PackSize estimate_serialization_size(TypePtr any_type); std::vector generate_estimate_size_call(CodeBlob& code, SrcLocation loc, TypePtr any_type); diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index 8be3d9995..c0490c5c7 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -40,8 +40,6 @@ namespace tolk { std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, TypePtr cmp_type, const std::vector& expr_ir_idx, SrcLocation loc, const char* debug_desc); std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc); -constexpr int ERR_CODEGEN_ASSERT = 9; - bool is_type_cellT(TypePtr any_type) { if (const TypeDataStruct* t_struct = any_type->try_as()) { StructPtr struct_ref = t_struct->struct_ref; @@ -59,13 +57,14 @@ bool is_type_cellT(TypePtr any_type) { // -PackContext::PackContext(CodeBlob& code, SrcLocation loc, std::vector ir_builder) +PackContext::PackContext(CodeBlob& code, SrcLocation loc, std::vector ir_builder, const std::vector& ir_options) : code(code) , loc(loc) , f_storeInt(lookup_function("builder.storeInt")) , f_storeUint(lookup_function("builder.storeUint")) , ir_builder(std::move(ir_builder)) - , ir_builder0(this->ir_builder[0]) { + , ir_builder0(this->ir_builder[0]) + , option_skipBitsNFieldsValidation(ir_options[0]) { } void PackContext::storeInt(var_idx_t ir_idx, int len) const { @@ -119,14 +118,16 @@ void PackContext::storeOpcode(PackOpcode opcode) const { } -UnpackContext::UnpackContext(CodeBlob& code, SrcLocation loc, std::vector ir_slice) +UnpackContext::UnpackContext(CodeBlob& code, SrcLocation loc, std::vector ir_slice, const std::vector& ir_options) : code(code) , loc(loc) , f_loadInt(lookup_function("slice.loadInt")) , f_loadUint(lookup_function("slice.loadUint")) , f_skipBits(lookup_function("slice.skipBits")) , ir_slice(std::move(ir_slice)) - , ir_slice0(this->ir_slice[0]) { + , ir_slice0(this->ir_slice[0]) + , option_assertEndAfterReading(ir_options[0]) + , option_throwIfOpcodeDoesNotMatch(ir_options[1]) { } std::vector UnpackContext::loadInt(int len, const char* debug_desc) const { @@ -147,8 +148,7 @@ void UnpackContext::loadAndCheckOpcode(PackOpcode opcode) const { std::vector ir_prefix_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(prefix-eq)"); std::vector args = { ir_slice0, code.create_int(loc, opcode.pack_prefix, "(pack-prefix)"), code.create_int(loc, opcode.prefix_len, "(prefix-len)") }; code.emplace_back(loc, Op::_Call, std::vector{ir_slice0, ir_prefix_eq[0]}, std::move(args), lookup_function("slice.tryStripPrefix")); - - std::vector args_assert = { code.create_int(loc, ERR_CODEGEN_ASSERT, "(excno)"), ir_prefix_eq[0], code.create_int(loc, 0, "") }; + std::vector args_assert = { option_throwIfOpcodeDoesNotMatch, ir_prefix_eq[0], code.create_int(loc, 0, "") }; Op& op_assert = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); op_assert.set_impure_flag(); } @@ -163,6 +163,19 @@ void UnpackContext::skipBits_var(var_idx_t ir_len) const { code.emplace_back(loc, Op::_Call, ir_slice, std::move(args), f_skipBits); } +void UnpackContext::assertEndIfOption() const { + Op& if_assertEnd = code.emplace_back(loc, Op::_If, std::vector{option_assertEndAfterReading}); + { + code.push_set_cur(if_assertEnd.block0); + Op& op_ends = code.emplace_back(loc, Op::_Call, std::vector{}, ir_slice, lookup_function("slice.assertEnd")); + op_ends.set_impure_flag(); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_assertEnd.block1); + code.close_pop_cur(loc); + } +} // -------------------------------------------- // serializers with pack/unpack/skip/estimate @@ -226,24 +239,27 @@ struct S_BytesN final : ISerializer { void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { tolk_assert(rvect.size() == 1); - // bitsN/bytesN aren't checked upon writing (to have requested number of bits and 0 refs), probably to expose an option - if (false) { + Op& if_disabled_by_user = code.emplace_back(loc, Op::_If, std::vector{ctx->option_skipBitsNFieldsValidation}); + { + code.push_set_cur(if_disabled_by_user.block0); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_disabled_by_user.block1); FunctionPtr f_assert = lookup_function("__throw_if_unless"); - FunctionPtr f_equals = lookup_function("_==_"); - FunctionPtr f_bitAnd = lookup_function("_&_"); - FunctionPtr f_getCounts = lookup_function("slice.remainingBitsAndRefsCount"); + constexpr int EXCNO = 9; std::vector ir_counts = code.create_tmp_var(TypeDataTensor::create({TypeDataInt::create(), TypeDataInt::create()}), loc, "(slice-size)"); - code.emplace_back(loc, Op::_Call, ir_counts, rvect, f_getCounts); - std::vector ir_is_n = code.create_tmp_var(TypeDataInt::create(), loc, "(n-bits)"); - code.emplace_back(loc, Op::_Call, ir_is_n, std::vector{ir_counts[0], code.create_int(loc, n_bits, "(expected-bits)")}, f_equals); - std::vector ir_is_0 = code.create_tmp_var(TypeDataInt::create(), loc, "(n-refs)"); - code.emplace_back(loc, Op::_Call, ir_is_0, std::vector{ir_counts[1], code.create_int(loc, 0, "(expected-refs)")}, f_equals); - std::vector ir_both_eq = code.create_tmp_var(TypeDataInt::create(), loc, "(both-eq)"); - code.emplace_back(loc, Op::_Call, ir_both_eq, std::vector{ir_is_n[0], ir_is_0[0]}, f_bitAnd); - std::vector args_assert = { code.create_int(loc, ERR_CODEGEN_ASSERT, "(excno)"), ir_both_eq[0], code.create_int(loc, 0, "") }; - Op& op1 = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert), f_assert); - op1.set_impure_flag(); + code.emplace_back(loc, Op::_Call, ir_counts, rvect, lookup_function("slice.remainingBitsAndRefsCount")); + std::vector args_assert0 = { code.create_int(loc, EXCNO, "(excno)"), ir_counts[1], code.create_int(loc, 1, "") }; + Op& op_assert0 = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert0), f_assert); + op_assert0.set_impure_flag(); + std::vector ir_eq_n = code.create_tmp_var(TypeDataInt::create(), loc, "(eq-n)"); + code.emplace_back(loc, Op::_Call, ir_eq_n, std::vector{ir_counts[0], code.create_int(loc, n_bits, "(n-bits)")}, lookup_function("_==_")); + std::vector args_assertN = { code.create_int(loc, EXCNO, "(excno)"), ir_eq_n[0], code.create_int(loc, 0, "") }; + Op& op_assertN = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assertN), f_assert); + op_assertN.set_impure_flag(); + code.close_pop_cur(loc); } ctx->storeSlice(rvect[0]); @@ -414,7 +430,7 @@ struct S_RemainingBitsAndRefs final : ISerializer { } PackSize estimate(const EstimateContext* ctx) override { - return PackSize(0, 0, 0, 0); + return PackSize::unpredictable_infinity(); } }; @@ -434,7 +450,7 @@ struct S_Builder final : ISerializer { } PackSize estimate(const EstimateContext* ctx) override { - return PackSize(0, 0, 0, 0); + return PackSize::unpredictable_infinity(); } }; @@ -661,27 +677,21 @@ struct S_MultipleConstructors final : ISerializer { TypePtr variant = t_union->variants[i]; std::vector args = { ctx->ir_slice0, code.create_int(loc, opcodes[i].pack_prefix, "(pack-prefix)"), code.create_int(loc, opcodes[i].prefix_len, "(prefix-len)") }; code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_prefix_eq[0]}, std::move(args), f_tryStripPrefix); - if (i != t_union->size() - 1) { - Op& if_op = code.emplace_back(loc, Op::_If, ir_prefix_eq); - code.push_set_cur(if_op.block0); - std::vector ith_rvect = ctx->generate_unpack_any(variant, PrefixReadMode::DoNothingAlreadyLoaded); - ith_rvect = transition_to_target_type(std::move(ith_rvect), code, variant, t_union, loc); - code.emplace_back(loc, Op::_Let, ir_result, std::move(ith_rvect)); - code.close_pop_cur(loc); - code.push_set_cur(if_op.block1); // open ELSE - } else { // we're inside the last ELSE - if (!are_prefixes_exhaustive()) { - std::vector args_assert = { code.create_int(loc, ERR_CODEGEN_ASSERT, "(excno)"), ir_prefix_eq[0], code.create_int(loc, 0, "") }; - Op& op_assert = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); - op_assert.set_impure_flag(); - } - std::vector last_rvect = ctx->generate_unpack_any(variant, PrefixReadMode::DoNothingAlreadyLoaded); - last_rvect = transition_to_target_type(std::move(last_rvect), code, variant, t_union, loc); - code.emplace_back(loc, Op::_Let, ir_result, std::move(last_rvect)); - for (int j = 0; j < t_union->size() - 1; ++j) { - code.close_pop_cur(loc); // close all outer IFs - } - } + Op& if_prefix_eq = code.emplace_back(loc, Op::_If, ir_prefix_eq); + code.push_set_cur(if_prefix_eq.block0); + std::vector ith_rvect = ctx->generate_unpack_any(variant, PrefixReadMode::DoNothingAlreadyLoaded); + ith_rvect = transition_to_target_type(std::move(ith_rvect), code, variant, t_union, loc); + code.emplace_back(loc, Op::_Let, ir_result, std::move(ith_rvect)); + code.close_pop_cur(loc); + code.push_set_cur(if_prefix_eq.block1); // open ELSE + } + + // we're inside last ELSE + std::vector args_throw = { ctx->option_throwIfOpcodeDoesNotMatch }; + Op& op_throw = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_throw), lookup_function("__throw")); + op_throw.set_impure_flag(); + for (int j = 0; j < t_union->size(); ++j) { + code.close_pop_cur(loc); // close all outer IFs } return ir_result; } @@ -694,23 +704,19 @@ struct S_MultipleConstructors final : ISerializer { TypePtr variant = t_union->variants[i]; std::vector args = { ctx->ir_slice0, code.create_int(loc, opcodes[i].pack_prefix, "(pack-prefix)"), code.create_int(loc, opcodes[i].prefix_len, "(prefix-len)") }; code.emplace_back(loc, Op::_Call, std::vector{ctx->ir_slice0, ir_prefix_eq[0]}, std::move(args), f_tryStripPrefix); - if (i != t_union->size() - 1) { - Op& if_op = code.emplace_back(loc, Op::_If, ir_prefix_eq); - code.push_set_cur(if_op.block0); - ctx->generate_skip_any(variant, PrefixReadMode::DoNothingAlreadyLoaded); - code.close_pop_cur(loc); - code.push_set_cur(if_op.block1); // open ELSE - } else { // we're inside the last ELSE - if (!are_prefixes_exhaustive()) { - std::vector args_assert = { code.create_int(loc, ERR_CODEGEN_ASSERT, "(excno)"), ir_prefix_eq[0], code.create_int(loc, 0, "") }; - Op& op_assert = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_assert), lookup_function("__throw_if_unless")); - op_assert.set_impure_flag(); - } - ctx->generate_skip_any(variant, PrefixReadMode::DoNothingAlreadyLoaded); - for (int j = 0; j < t_union->size() - 1; ++j) { - code.close_pop_cur(loc); // close all outer IFs - } - } + Op& if_prefix_eq = code.emplace_back(loc, Op::_If, ir_prefix_eq); + code.push_set_cur(if_prefix_eq.block0); + ctx->generate_skip_any(variant, PrefixReadMode::DoNothingAlreadyLoaded); + code.close_pop_cur(loc); + code.push_set_cur(if_prefix_eq.block1); // open ELSE + } + + // we're inside last ELSE + std::vector args_throw = { ctx->option_throwIfOpcodeDoesNotMatch }; + Op& op_throw = code.emplace_back(loc, Op::_Call, std::vector{}, std::move(args_throw), lookup_function("__throw")); + op_throw.set_impure_flag(); + for (int j = 0; j < t_union->size(); ++j) { + code.close_pop_cur(loc); // close all outer IFs } } @@ -725,14 +731,6 @@ struct S_MultipleConstructors final : ISerializer { return EstimateContext::sum(variants_size, prefix_size); } - - bool are_prefixes_exhaustive() const { - bool all_prefix_len_eq = true; - for (PackOpcode opcode : opcodes) { - all_prefix_len_eq &= opcode.prefix_len == opcodes[0].prefix_len; - } - return all_prefix_len_eq && t_union->size() == (1 << opcodes[0].prefix_len); - } }; struct S_Tensor final : ISerializer { diff --git a/tolk/pack-unpack-serializers.h b/tolk/pack-unpack-serializers.h index 47629a7db..c2c99ea2a 100644 --- a/tolk/pack-unpack-serializers.h +++ b/tolk/pack-unpack-serializers.h @@ -29,6 +29,10 @@ struct PackSize { int min_refs; int max_refs; + bool is_unpredictable_infinity() const { + return max_bits >= 9999; + } + explicit PackSize(int exact_bits) : min_bits(exact_bits), max_bits(exact_bits), min_refs(0), max_refs(0) { } @@ -38,6 +42,10 @@ struct PackSize { PackSize(int min_bits, int max_bits, int min_refs, int max_refs) : min_bits(min_bits), max_bits(max_bits), min_refs(min_refs), max_refs(max_refs) { } + + static PackSize unpredictable_infinity() { + return PackSize(0, 9999, 0, 4); + } }; @@ -56,8 +64,9 @@ class PackContext { public: const std::vector ir_builder; const var_idx_t ir_builder0; + const var_idx_t option_skipBitsNFieldsValidation; - PackContext(CodeBlob& code, SrcLocation loc, std::vector ir_builder); + PackContext(CodeBlob& code, SrcLocation loc, std::vector ir_builder, const std::vector& ir_options); PrefixWriteMode get_prefix_mode() const { return prefix_mode; } @@ -92,8 +101,10 @@ class UnpackContext { public: const std::vector ir_slice; const var_idx_t ir_slice0; + const var_idx_t option_assertEndAfterReading; + const var_idx_t option_throwIfOpcodeDoesNotMatch; - UnpackContext(CodeBlob& code, SrcLocation loc, std::vector ir_slice); + UnpackContext(CodeBlob& code, SrcLocation loc, std::vector ir_slice, const std::vector& ir_options); PrefixReadMode get_prefix_mode() const { return prefix_mode; } @@ -102,6 +113,7 @@ class UnpackContext { void loadAndCheckOpcode(PackOpcode opcode) const; void skipBits(int len) const; void skipBits_var(var_idx_t ir_len) const; + void assertEndIfOption() const; std::vector generate_unpack_any(TypePtr any_type, PrefixReadMode prefix_mode = PrefixReadMode::LoadAndCheck) const; void generate_skip_any(TypePtr any_type, PrefixReadMode prefix_mode = PrefixReadMode::LoadAndCheck) const; @@ -124,7 +136,7 @@ class EstimateContext { return PackSize(std::min(a.min_bits, b.min_bits), std::max(a.max_bits, b.max_bits), std::min(a.min_refs, b.min_refs), std::max(a.max_refs, b.max_refs)); } static PackSize sum(PackSize a, PackSize b) { - return PackSize(a.min_bits + b.min_bits, a.max_bits + b.max_bits, a.min_refs + b.min_refs, a.max_refs + b.max_refs); + return PackSize(a.min_bits + b.min_bits, std::min(9999, a.max_bits + b.max_bits), a.min_refs + b.min_refs, a.max_refs + b.max_refs); } PackSize estimate_any(TypePtr any_type, PrefixEstimateMode prefix_mode = PrefixEstimateMode::IncludePrefixOfStruct) const; diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 773d67ecc..91891f8c1 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -540,59 +540,52 @@ static std::vector gen_op_call(CodeBlob& code, TypePtr ret_type, SrcL return rvect; } -static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob& code, SrcLocation loc, std::vector&& rvect, FunctionPtr called_f) { +static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob& code, SrcLocation loc, const std::vector>& vars_per_arg, FunctionPtr called_f) { if (called_f->is_instantiation_of_generic_function()) { std::string_view f_name = called_f->base_fun_ref->name; TypePtr typeT = called_f->substitutedTs->typeT_at(0); - int n_rvect = static_cast(rvect.size()); if (f_name == "T.toCell") { // in: object T, out: Cell (just a cell, wrapped) - tolk_assert(n_rvect == typeT->get_width_on_stack()); - std::vector ir_obj = std::move(rvect); - return generate_pack_struct_to_cell(code, loc, typeT, std::move(ir_obj)); + std::vector ir_obj = vars_per_arg[0]; + return generate_pack_struct_to_cell(code, loc, typeT, std::move(ir_obj), vars_per_arg[1]); } if (f_name == "T.fromCell") { // in: cell, out: object T - tolk_assert(n_rvect == 1); - std::vector ir_cell = std::move(rvect); - return generate_unpack_struct_from_cell(code, loc, typeT, std::move(ir_cell)); + std::vector ir_cell = vars_per_arg[0]; + return generate_unpack_struct_from_cell(code, loc, typeT, std::move(ir_cell), vars_per_arg[1]); } if (f_name == "T.fromSlice") { // in: slice, out: object T, input slice NOT mutated - tolk_assert(n_rvect == 1); - std::vector ir_slice = std::move(rvect); - return generate_unpack_struct_from_slice(code, loc, typeT, std::move(ir_slice), false); + std::vector ir_slice = vars_per_arg[0]; + return generate_unpack_struct_from_slice(code, loc, typeT, std::move(ir_slice), false, vars_per_arg[1]); } if (f_name == "Cell.load") { // in: cell, out: object T - tolk_assert(n_rvect == 1); - std::vector ir_cell = std::move(rvect); - return generate_unpack_struct_from_cell(code, loc, typeT, std::move(ir_cell)); + std::vector ir_cell = vars_per_arg[0]; + return generate_unpack_struct_from_cell(code, loc, typeT, std::move(ir_cell), vars_per_arg[1]); } if (f_name == "slice.loadAny") { // in: slice, out: object T, input slice is mutated, so prepend self before an object - var_idx_t ir_self = rvect[0]; - std::vector ir_obj = generate_unpack_struct_from_slice(code, loc, typeT, std::move(rvect), true); + var_idx_t ir_self = vars_per_arg[0][0]; + std::vector ir_slice = vars_per_arg[0]; + std::vector ir_obj = generate_unpack_struct_from_slice(code, loc, typeT, std::move(ir_slice), true, vars_per_arg[1]); std::vector ir_result = {ir_self}; ir_result.insert(ir_result.end(), ir_obj.begin(), ir_obj.end()); return ir_result; } if (f_name == "slice.skipAny") { // in: slice, out: the same slice, with a shifted pointer - tolk_assert(n_rvect == 1); - std::vector ir_slice = std::move(rvect); - return generate_skip_struct_in_slice(code, loc, typeT, std::move(ir_slice)); + std::vector ir_slice = vars_per_arg[0]; + return generate_skip_struct_in_slice(code, loc, typeT, std::move(ir_slice), vars_per_arg[1]); } if (f_name == "builder.storeAny") { // in: builder and object T, out: mutated builder - tolk_assert(n_rvect == 1 + typeT->get_width_on_stack()); - std::vector ir_builder = {rvect[0]}; - std::vector ir_obj(rvect.begin() + 1, rvect.end()); - return generate_pack_struct_to_builder(code, loc, typeT, std::move(ir_builder), std::vector(ir_obj)); + std::vector ir_builder = vars_per_arg[0]; + std::vector ir_obj = vars_per_arg[1]; + return generate_pack_struct_to_builder(code, loc, typeT, std::move(ir_builder), std::move(ir_obj), vars_per_arg[2]); } if (f_name == "T.estimatePackSize") { - tolk_assert(rvect.empty()); return generate_estimate_size_call(code, loc, typeT); } } @@ -1454,7 +1447,7 @@ static std::vector process_function_call(V v, Code args_vars.insert(args_vars.end(), list.cbegin(), list.cend()); } std::vector rvect_apply = fun_ref->is_compile_time_special_gen() - ? gen_compile_time_code_instead_of_fun_call(code, v->loc, std::move(args_vars), fun_ref) + ? gen_compile_time_code_instead_of_fun_call(code, v->loc, vars_per_arg, fun_ref) : gen_op_call(code, op_call_type, v->loc, std::move(args_vars), fun_ref, "(fun-call)", arg_order_already_equals_asm); if (fun_ref->has_mutate_params()) { diff --git a/tolk/pipe-check-serialized-fields.cpp b/tolk/pipe-check-serialized-fields.cpp index c49b3dc34..3ed8fc33a 100644 --- a/tolk/pipe-check-serialized-fields.cpp +++ b/tolk/pipe-check-serialized-fields.cpp @@ -59,7 +59,7 @@ class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody static void check_struct_fits_cell_or_has_policy(const TypeDataStruct* t_struct) { StructPtr struct_ref = t_struct->struct_ref; PackSize size = estimate_serialization_size(t_struct); - if (size.max_bits > 1023) { + if (size.max_bits > 1023 && !size.is_unpredictable_infinity()) { if (struct_ref->overflow1023_policy == StructData::Overflow1023Policy::not_specified) { fire_error_theoretical_overflow_1023(struct_ref, size); } @@ -84,7 +84,7 @@ class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody if (f_name == "Cell.load" || f_name == "T.fromSlice" || f_name == "T.fromCell" || f_name == "T.toCell" || f_name == "T.loadAny" || f_name == "slice.skipAny" || f_name == "slice.storeAny" || f_name == "T.estimatePackSize") { serialized_type = fun_ref->substitutedTs->typeT_at(0); - is_pack = f_name == "T.toCell" || f_name == "slice.storeAny"; + is_pack = f_name == "T.toCell" || f_name == "slice.storeAny" || f_name == "T.estimatePackSize"; } else { return; // not a serialization function } diff --git a/tolk/tolk.h b/tolk/tolk.h index 6da5c1ff8..6e5af87ca 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -924,6 +924,9 @@ struct Optimizer { bool is_nip_seq(int* i, int* j); bool is_pop_blkdrop(int* i, int* k); bool is_2pop_blkdrop(int* i, int* j, int* k); + + bool detect_rewrite_big_THROW(); + AsmOpConsList extract_code(); }; From e4d0ab7157486a1fd9720cfceb0556ad0e67a6e2 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Wed, 28 May 2025 18:22:51 +0300 Subject: [PATCH 272/388] [Tolk] Bump version to v0.13 It's considered as "release candidate" (although there's still a lot of work ahead) --- crypto/smartcont/tolk-stdlib/common.tolk | 2 +- crypto/smartcont/tolk-stdlib/gas-payments.tolk | 2 +- crypto/smartcont/tolk-stdlib/lisp-lists.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-dicts.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk | 2 +- tolk/tolk-version.h | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 366345385..c9e116860 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -1,7 +1,7 @@ // Standard library for Tolk (LGPL licence). // It contains common functions that are available out of the box, the user doesn't have to import anything. // More specific functions are required to be imported explicitly, like "@stdlib/tvm-dicts". -tolk 0.12 +tolk 0.13 /// In Tolk v1.x there would be a type `map`. /// Currently, working with dictionaries is still low-level, with raw cells. diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk index 89366cf4f..8fde5f3c3 100644 --- a/crypto/smartcont/tolk-stdlib/gas-payments.tolk +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.12 +tolk 0.13 /** Gas and payment related primitives. diff --git a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk index ada7a72bc..14d8f835b 100644 --- a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk +++ b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.12 +tolk 0.13 /** Lisp-style lists are nested 2-elements tuples: `[1, [2, [3, null]]]` represents list `[1, 2, 3]`. diff --git a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk index 90b392f61..9d678e83d 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.12 +tolk 0.13 /** Dictionaries are represented as `cell` data type (cells can store anything, dicts in particular). diff --git a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk index 47b4a300e..7281038d6 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.12 +tolk 0.13 /// Usually `c3` has a continuation initialized by the whole code of the contract. It is used for function calls. /// The primitive returns the current value of `c3`. diff --git a/tolk/tolk-version.h b/tolk/tolk-version.h index dd9134f45..5da1582d2 100644 --- a/tolk/tolk-version.h +++ b/tolk/tolk-version.h @@ -18,6 +18,6 @@ namespace tolk { -constexpr const char* TOLK_VERSION = "0.12.0"; +constexpr const char* TOLK_VERSION = "0.13.0"; } // namespace tolk From a13be2ed61d8d06dff0f3611a258c0a164c7a17e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 30 May 2025 10:47:33 +0300 Subject: [PATCH 273/388] Session stats: "got_block_by", "shard_configuration" --- tl/generate/scheme/ton_api.tl | 4 ++-- tl/generate/scheme/ton_api.tlo | Bin 113716 -> 113828 bytes validator-session/validator-session-types.h | 6 ++++-- validator-session/validator-session.cpp | 18 ++++++++++++++---- validator-session/validator-session.hpp | 4 ++-- validator/impl/collator.cpp | 6 ++++++ validator/interfaces/validator-manager.h | 7 ++++++- 7 files changed, 34 insertions(+), 11 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 5d42796ec..e441551ee 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -913,7 +913,7 @@ validatorStats.stats.producer flags:# validator_id:int256 block_status:int candidate_id:flags.0?int256 block_id:flags.0?tonNode.blockIdExt collated_data_hash:flags.0?int256 is_accepted:flags.0?Bool is_ours:flags.0?Bool - got_block_at:flags.0?double got_submit_at:flags.0?double gen_utime:flags.0?int comment:flags.0?string + got_block_at:flags.0?double got_block_by:flags.0?int got_submit_at:flags.0?double gen_utime:flags.0?int comment:flags.0?string collation_time:flags.1?double collated_at:flags.1?double collation_cached:flags.1?Bool self_collated:flags.1?Bool collator_node_id:flags.2?int256 validation_time:flags.3?double validated_at:flags.3?double validation_cached:flags.3?Bool approved_weight:flags.0?long approved_33pct_at:flags.0?double approved_66pct_at:flags.0?double approvers:flags.0?string @@ -937,7 +937,7 @@ validatorStats.blockLimitsStatus validatorStats.extMsgsStats ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int = validatorStats.ExtMsgsStats; validatorStats.blockStats - ext_msgs:validatorStats.extMsgsStats transactions:int = validatorStats.BlockStats; + ext_msgs:validatorStats.extMsgsStats transactions:int shard_configuration:(vector tonNode.blockIdExt) = validatorStats.BlockStats; validatorStats.collatedBlock block_id:tonNode.blockIdExt collated_data_hash:int256 cc_seqno:int collated_at:double diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 7b0b3750ac301801cd4b6efe14317c597d86146d..5c7285d51655490215647cf2954abcf83c0c7ef0 100644 GIT binary patch delta 203 zcmdn;gKf!AwhbTbSSBdR9oziHPKAl3w@59&S;ld@j3Z-?2v~f3Pbi}clO{-sWLaWP zW=dj7eo=5qVo9-HQciwyHkir40Fu9XOvs0M`p63kO2zY6O$SH8Tok9^Go8vHpC}YGE9D# as3ZnuGM2ko!V+WYEeL*JUlpp{bH%fQ_ delta 159 zcmZ4TlWofnwhbTbSZ*)5(!KeMoeC2RFPqBVW*NurGLDQnB4F|DJ)w*)Oqw7il4XfG znJI}S`9;Aci6zB)Njdq+*( flags, validator_id.bits256_value(), block_status, candidate_id, create_tl_block_id(block_id), - collated_data_hash, is_accepted, is_ours, got_block_at, got_submit_at, gen_utime, comment, collation_time, - collated_at, collation_cached, self_collated, collator_node_id, validation_time, validated_at, + collated_data_hash, is_accepted, is_ours, got_block_at, got_block_by, got_submit_at, gen_utime, comment, + collation_time, collated_at, collation_cached, self_collated, collator_node_id, validation_time, validated_at, validation_cached, approved_weight, approved_33pct_at, approved_66pct_at, std::move(approvers_str), signed_weight, signed_33pct_at, signed_66pct_at, std::move(signers_str), serialize_time, deserialize_time, serialized_size); diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 79233d699..9ce7e5404 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -222,7 +222,7 @@ bool ValidatorSessionImpl::ensure_candidate_unique(td::uint32 src_idx, td::uint3 void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice data, td::optional expected_id, - bool is_overlay_broadcast) { + bool is_overlay_broadcast, bool is_startup) { // Note: src is not necessarily equal to the sender of this message: // If requested using get_broadcast_p2p, src is the creator of the block, sender possibly is some other node. auto src_idx = description().get_source_idx(src); @@ -269,6 +269,13 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice } if (stat->got_block_at <= 0.0) { stat->got_block_at = td::Clocks::system(); + if (is_overlay_broadcast) { + stat->got_block_by = ValidatorSessionStats::recv_broadcast; + } else if (is_startup) { + stat->got_block_by = ValidatorSessionStats::recv_startup; + } else { + stat->got_block_by = ValidatorSessionStats::recv_query; + } } stat->deserialize_time = deserialize_time; stat->serialized_size = data.size(); @@ -470,6 +477,7 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, GeneratedCandidate stat->collation_time = collation_time; stat->collated_at = td::Clocks::system(); stat->got_block_at = td::Clocks::system(); + stat->got_block_by = ValidatorSessionStats::recv_collated; stat->collation_cached = c.is_cached; stat->self_collated = c.self_collated; stat->collator_node_id = c.collator_node_id; @@ -610,6 +618,7 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { } if (stat->got_block_at <= 0.0) { stat->got_block_at = td::Clocks::system(); + stat->got_block_by = ValidatorSessionStats::recv_cached; } stat->block_id.root_hash = B->root_hash_; stat->block_id.file_hash = td::sha256_bits256(B->data_); @@ -656,8 +665,9 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { VLOG(VALIDATOR_SESSION_WARNING) << print_id << ": failed to get candidate " << hash << " from " << id << ": " << R.move_as_error(); } else { + LOG(ERROR) << "QQQQQ Got block " << R.ok().size(); td::actor::send_closure(SelfId, &ValidatorSessionImpl::process_broadcast, src_id, R.move_as_ok(), - candidate_id, false); + candidate_id, false, false); } }); @@ -971,8 +981,8 @@ void ValidatorSessionImpl::on_catchain_started() { auto broadcast = create_tl_object( src.tl(), round, root_hash, std::move(B.data), std::move(B.collated_data)); td::actor::send_closure(SelfId, &ValidatorSessionImpl::process_broadcast, src, - serialize_candidate(broadcast, compress).move_as_ok(), td::optional(), - false); + serialize_candidate(broadcast, compress).move_as_ok(), + td::optional(), false, true); } }); callback_->get_approved_candidate(description().get_source_public_key(x->get_src_idx()), x->get_root_hash(), diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 2d996101a..04fdc875c 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -116,7 +116,7 @@ class ValidatorSessionImpl : public ValidatorSession { } void process_broadcast(const PublicKeyHash &src, td::BufferSlice data) override { td::actor::send_closure(id_, &ValidatorSessionImpl::process_broadcast, src, std::move(data), - td::optional(), true); + td::optional(), true, false); } void process_message(const PublicKeyHash &src, td::BufferSlice data) override { td::actor::send_closure(id_, &ValidatorSessionImpl::process_message, src, std::move(data)); @@ -204,7 +204,7 @@ class ValidatorSessionImpl : public ValidatorSession { void preprocess_block(catchain::CatChainBlock *block); bool ensure_candidate_unique(td::uint32 src_idx, td::uint32 round, ValidatorSessionCandidateId block_id); void process_broadcast(PublicKeyHash src, td::BufferSlice data, td::optional expected_id, - bool is_overlay_broadcast); + bool is_overlay_broadcast, bool is_startup); void process_message(PublicKeyHash src, td::BufferSlice data); void process_query(PublicKeyHash src, td::BufferSlice data, td::Promise promise); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 93fbaa198..33719c2f9 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -6374,6 +6374,12 @@ void Collator::finalize_stats() { stats_.work_time = work_time; stats_.cpu_work_time = cpu_work_time; stats_.time_stats = (PSTRING() << perf_log_); + if (is_masterchain() && shard_conf_) { + shard_conf_->process_shard_hashes([&](const block::McShardHash& shard) { + stats_.shard_configuration.push_back(shard.top_block_id()); + return 0; + }); + } } /** diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index d1394c942..534e5ea92 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -70,6 +70,7 @@ struct CollationStats { int cat_bytes = 0, cat_gas = 0, cat_lt_delta = 0, cat_collated_data_bytes = 0; std::string limits_log; td::uint32 transactions = 0; + std::vector shard_configuration; td::uint32 ext_msgs_total = 0; td::uint32 ext_msgs_filtered = 0; td::uint32 ext_msgs_accepted = 0; @@ -78,10 +79,14 @@ struct CollationStats { std::string time_stats; tl_object_ptr tl() const { + std::vector> shards; + for (const BlockIdExt& block_id : shard_configuration) { + shards.push_back(create_tl_block_id(block_id)); + } auto block_stats = create_tl_object( create_tl_object(ext_msgs_total, ext_msgs_filtered, ext_msgs_accepted, ext_msgs_rejected), - transactions); + transactions, std::move(shards)); return create_tl_object( create_tl_block_id(block_id), collated_data_hash, cc_seqno, collated_at, actual_bytes, actual_collated_data_bytes, attempt, self.bits256_value(), is_validator, total_time, work_time, cpu_work_time, From c89081d4bdf279af4d6cd4ab11f7d8f6363cc786 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Sun, 1 Jun 2025 15:52:52 +0300 Subject: [PATCH 274/388] Fix start_lt of tick transactions --- validator/impl/collator.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 805240b3e..cc9ee709c 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -3007,12 +3007,12 @@ bool Collator::update_last_proc_int_msg(const std::pairtick * 2 + found->tock : config_->get_smc_tick_tock(smc_addr.cbits())); From 220890f55c07bb88db501ceb9ed734e0e8f8b038 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 2 Jun 2025 13:13:46 +0300 Subject: [PATCH 275/388] Add msg import info to collatedBlock session stats --- crypto/block/block.cpp | 9 ++++ crypto/block/block.h | 4 ++ crypto/smartcont/CreateState.fif | 8 +++ tl/generate/scheme/ton_api.tl | 11 +++- tl/generate/scheme/ton_api.tlo | Bin 113828 -> 114688 bytes validator/impl/collator-impl.h | 3 +- validator/impl/collator.cpp | 60 +++++++++++++++------ validator/impl/out-msg-queue-proof.cpp | 6 +-- validator/interfaces/out-msg-queue-proof.h | 4 +- validator/interfaces/validator-manager.h | 52 ++++++++++++++---- 10 files changed, 124 insertions(+), 33 deletions(-) diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp index d06e7a40c..0893d630f 100644 --- a/crypto/block/block.cpp +++ b/crypto/block/block.cpp @@ -768,6 +768,15 @@ bool BlockLimitStatus::would_fit(unsigned cls, ton::LogicalTime end_lt, td::uint limits.collated_data.fits(cls, collated_data_size_estimate)); } +double BlockLimitStatus::load_fraction(unsigned cls) const { + if (cls >= ParamLimits::limits_cnt) { + return 0.0; + } + return std::max({(double)estimate_block_size() / (double)limits.bytes.limit(cls), + (double)gas_used / (double)limits.gas.limit(cls), + (double)collated_data_size_estimate / (double)limits.collated_data.limit(cls)}); +} + // SETS: account_dict, shard_libraries_, mc_state_extra // total_balance{,_extra}, total_validator_fees // SETS: out_msg_queue, processed_upto_, ihr_pending (via unpack_out_msg_queue_info) diff --git a/crypto/block/block.h b/crypto/block/block.h index 65834d79e..c81dfdefa 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -242,6 +242,9 @@ struct ParamLimits { td::uint32 hard() const { return limits_[3]; } + td::uint32 limit(unsigned cls) const { + return limits_[cls]; + } bool compute_medium_limit() { limits_[2] = soft() + ((hard() - soft()) >> 1); return true; @@ -299,6 +302,7 @@ struct BlockLimitStatus { bool fits(unsigned cls) const; bool would_fit(unsigned cls, ton::LogicalTime end_lt, td::uint64 more_gas, const vm::NewCellStorageStat::Stat* extra = nullptr) const; + double load_fraction(unsigned cls) const; bool add_cell(Ref cell) { st_stat.add_cell(std::move(cell)); return true; diff --git a/crypto/smartcont/CreateState.fif b/crypto/smartcont/CreateState.fif index 6cb9dbcb5..25873eefe 100644 --- a/crypto/smartcont/CreateState.fif +++ b/crypto/smartcont/CreateState.fif @@ -179,8 +179,16 @@ variable special-dict // bytes-limits gas-limits lt-limits -- c { } : make-block-limits +// bytes-limits gas-limits lt-limits collated-data-limits imported_queue_bytes imported_queue_msgs -- c +{ +} : make-block-limits-v2 + { make-block-limits 22 config! } : config.mc_block_limits! { make-block-limits 23 config! } : config.block_limits! +{ make-block-limits-v2 22 config! } : config.mc_block_limits_v2! +{ make-block-limits-v2 23 config! } : config.block_limits_v2! // mc-block-create-fee bc-block-create-fee { } : make-block-create-fees diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index e441551ee..56f3145b8 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -933,11 +933,18 @@ validatorStats.stats flags:# validatorStats.blockLimitsStatus bytes:int gas:int lt_delta:int collated_data_bytes:int cat_bytes:int cat_gas:int cat_lt_delta:int cat_collated_data_bytes:int + load_fraction_queue_cleanup:double load_fraction_dispatch:double + load_fraction_internals:double load_fraction_externals:double + load_fraction_new_msgs:double limits_log:string = validatorStats.BlockLimitsStatus; -validatorStats.extMsgsStats +validatorStats.blockStats.extMsgsStats ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int = validatorStats.ExtMsgsStats; +validatorStats.blockStats.neighborStats shard:tonNode.shardId is_trivial:Bool is_local:Bool msg_limit:int + processed_msgs:int skipped_msgs:int limit_reached:Bool = validatorStats.blockStats.NeighborStats; validatorStats.blockStats - ext_msgs:validatorStats.extMsgsStats transactions:int shard_configuration:(vector tonNode.blockIdExt) = validatorStats.BlockStats; + ext_msgs:validatorStats.blockStats.extMsgsStats transactions:int shard_configuration:(vector tonNode.blockIdExt) + old_out_msg_queue_size:long new_out_msg_queue_size:long msg_queue_cleaned:int + neighbors:(vector validatorStats.blockStats.neighborStats) = validatorStats.BlockStats; validatorStats.collatedBlock block_id:tonNode.blockIdExt collated_data_hash:int256 cc_seqno:int collated_at:double diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 5c7285d51655490215647cf2954abcf83c0c7ef0..76d27f04a2f05c3b7cd3bb76a2c4c1b57c83158c 100644 GIT binary patch delta 749 zcmZ4TldYkFo%hjfeJchiaNfwf*p8)m^Nlx?x7gXRJV;ARoBYLY3k%QH--{=&v45h& z@FMSwdRbylW=dj7eo=5qVo9-HQciwyHiYSynwg%F1eKV$L1OX*haZd=n>!u1cRDiO z76zL0a$5HXuDMiowwy*G~u`S4Bu8_cM_eL@%`2cwL1PJUuad|FXr za!F=>UVLF`YH4bGa!zVuUTMMf1+I*eVyFsIGK&imOOi7t=jTaEh@;A7=9Q!t* is_special = nullptr); bool process_inbound_internal_messages(); bool precheck_inbound_message(Ref msg, ton::LogicalTime lt); - bool process_inbound_message(Ref msg, ton::LogicalTime lt, td::ConstBitPtr key, - const block::McShardDescr& src_nb); + bool process_inbound_message(Ref msg, ton::LogicalTime lt, td::ConstBitPtr key, int src_nb_idx); bool process_inbound_external_messages(); int process_external_message(Ref msg); bool process_dispatch_queue(); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 33719c2f9..1d292fd7b 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -919,6 +919,7 @@ void Collator::got_neighbor_msg_queues(td::Result res) { if (res->block_state_proof_.not_null() && !block_id.is_masterchain()) { block_state_proofs_.emplace(block_id.root_hash, res->block_state_proof_); } + + auto &neighbor_stats = stats_.neighbors.at(i); + neighbor_stats.shard = block_id.shard_full(); + neighbor_stats.is_trivial = shard_intersects(block_id.shard_full(), shard_); + neighbor_stats.is_local = res->is_local_; + neighbor_stats.msg_limit = res->msg_count_; + Ref state_root; if (block_id.is_masterchain()) { state_root = res->state_root_; @@ -1000,18 +1008,16 @@ void Collator::got_neighbor_msg_queue(unsigned i, Ref res) { return; } outq_descr.clear(); - do { - // require masterchain blocks referred to in ProcessedUpto - // TODO: perform this only if there are messages for this shard in our output queue - // .. (have to check the above condition and perform a `break` here) .. - // .. - for (const auto& entry : descr.processed_upto->list) { - Ref state; - if (!request_aux_mc_state(entry.mc_seqno, state)) { - return; - } + // require masterchain blocks referred to in ProcessedUpto + // TODO: perform this only if there are messages for this shard in our output queue + // .. (have to check the above condition and perform a `break` here) .. + // .. + for (const auto& entry : descr.processed_upto->list) { + Ref state; + if (!request_aux_mc_state(entry.mc_seqno, state)) { + return; } - } while (false); + } } /** @@ -1940,7 +1946,7 @@ bool Collator::try_collate() { last_proc_int_msg_.second.set_zero(); first_unproc_int_msg_.first = ~0ULL; first_unproc_int_msg_.second.set_ones(); - old_out_msg_queue_size_ = out_msg_queue_size_; + stats_.old_out_msg_queue_size = old_out_msg_queue_size_ = out_msg_queue_size_; if (is_masterchain()) { LOG(DEBUG) << "getting the list of special smart contracts"; auto res = config_->get_special_smartcontracts(); @@ -2407,6 +2413,9 @@ bool Collator::dequeue_message(Ref msg_envelope, ton::LogicalTime deli * @returns True if the cleanup operation was successful, false otherwise. */ bool Collator::out_msg_queue_cleanup() { + SCOPE_EXIT { + stats_.load_fraction_queue_cleanup = block_limit_status_->load_fraction(block::ParamLimits::cl_normal); + }; LOG(INFO) << "cleaning outbound queue from messages already imported by neighbors"; if (verbosity >= 2) { FLOG(INFO) { @@ -2492,6 +2501,7 @@ bool Collator::out_msg_queue_cleanup() { } return true; }); + stats_.msg_queue_cleaned = deleted; LOG(WARNING) << "deleted " << deleted << " messages from out_msg_queue after merge, remaining queue size is " << out_msg_queue_size_; if (!ok) { @@ -2572,6 +2582,7 @@ bool Collator::out_msg_queue_cleanup() { std::swap(queue_parts[i], queue_parts.back()); queue_parts.pop_back(); } + stats_.msg_queue_cleaned = deleted; LOG(WARNING) << "deleted " << deleted << " messages from out_msg_queue, remaining queue size is " << out_msg_queue_size_; } @@ -3712,12 +3723,13 @@ bool Collator::precheck_inbound_message(Ref enq_msg, ton::Logical * @param enq_msg The inbound message serialized using EnqueuedMsg TLB-scheme. * @param lt The logical time of the message. * @param key The 32+64+256-bit key of the message. - * @param src_nb The description of the source neighbor shard. + * @param src_nb_idx The index of the source neighbor shard. * * @returns True if the message was processed successfully, false otherwise. */ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalTime lt, td::ConstBitPtr key, - const block::McShardDescr& src_nb) { + int src_nb_idx) { + const auto& src_nb = neighbors_.at(src_nb_idx); ton::LogicalTime enqueued_lt = enq_msg->prefetch_ulong(64); auto msg_env = enq_msg->prefetch_ref(); // 1. unpack MsgEnvelope @@ -3815,6 +3827,8 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT << " enqueued_lt=" << enq_msg_descr.enqueued_lt_ << " has been already processed by us before, skipping"; // should we dequeue the message if it is ours (after a merge?) // (it should have been dequeued by out_msg_queue_cleanup() before) + auto &neighbor_stats = stats_.neighbors.at(src_nb_idx); + ++neighbor_stats.skipped_msgs; return true; } // 6.1. check whether we have already processed this message by IHR @@ -3901,12 +3915,17 @@ static std::string block_full_comment(const block::BlockLimitStatus& block_limit * @returns True if the processing was successful, false otherwise. */ bool Collator::process_inbound_internal_messages() { + SCOPE_EXIT { + stats_.load_fraction_internals = block_limit_status_->load_fraction(block::ParamLimits::cl_normal); + }; while (!nb_out_msgs_->is_eof()) { block_full_ = !block_limit_status_->fits(block::ParamLimits::cl_normal); auto kv = nb_out_msgs_->extract_cur(); CHECK(kv && kv->msg.not_null()); + auto &neighbor_stats = stats_.neighbors.at(kv->source); if (kv->limit_exceeded) { LOG(INFO) << "limit for imported messages is reached, stop processing inbound internal messages"; + neighbor_stats.limit_reached = true; block::EnqueuedMsgDescr enq; enq.unpack(kv->msg.write()); // Visit cells to include it in proof break; @@ -3941,13 +3960,14 @@ bool Collator::process_inbound_internal_messages() { } LOG(DEBUG) << "processing inbound message with (lt,hash)=(" << kv->lt << "," << kv->key.to_hex() << ") from neighbor #" << kv->source; + ++neighbor_stats.processed_msgs; if (verbosity > 2) { FLOG(INFO) { sb << "inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() << " msg="; block::gen::t_EnqueuedMsg.print(sb, kv->msg); }; } - if (!process_inbound_message(kv->msg, kv->lt, kv->key.cbits(), neighbors_.at(kv->source))) { + if (!process_inbound_message(kv->msg, kv->lt, kv->key.cbits(), kv->source)) { if (verbosity > 1) { FLOG(INFO) { sb << "invalid inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() @@ -3970,6 +3990,9 @@ bool Collator::process_inbound_internal_messages() { * @returns True if the processing was successful, false otherwise. */ bool Collator::process_inbound_external_messages() { + SCOPE_EXIT { + stats_.load_fraction_externals = block_limit_status_->load_fraction(block::ParamLimits::cl_soft); + }; if (skip_extmsg_) { LOG(INFO) << "skipping processing of inbound external messages"; return true; @@ -4089,6 +4112,9 @@ int Collator::process_external_message(Ref msg) { * @returns True if the processing was successful, false otherwise. */ bool Collator::process_dispatch_queue() { + SCOPE_EXIT { + stats_.load_fraction_dispatch = block_limit_status_->load_fraction(block::ParamLimits::cl_normal); + }; if (out_msg_queue_size_ > defer_out_queue_size_limit_ && old_out_msg_queue_size_ > hard_defer_out_queue_size_limit_) { return true; } @@ -4548,6 +4574,9 @@ bool Collator::enqueue_message(block::NewOutMsg msg, td::RefInt256 fwd_fees_rema * @returns True if all new messages were processed successfully, false otherwise. */ bool Collator::process_new_messages(bool enqueue_only) { + SCOPE_EXIT { + stats_.load_fraction_new_msgs = block_limit_status_->load_fraction(block::ParamLimits::cl_normal); + }; while (!new_msgs.empty()) { block::NewOutMsg msg = new_msgs.top(); new_msgs.pop(); @@ -6380,6 +6409,7 @@ void Collator::finalize_stats() { return 0; }); } + stats_.new_out_msg_queue_size = out_msg_queue_size_; } /** diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index edd20a8d2..d5183d59c 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -196,7 +196,7 @@ td::Result>> OutMsgQueueProof::fetch(Shard if (state_root->get_hash().as_slice() != state_root_hash.as_slice()) { return td::Status::Error("state root hash mismatch"); } - res.emplace_back(true, blocks[i], state_root, block_state_proof, f.msg_counts_[i]); + res.emplace_back(true, blocks[i], state_root, block_state_proof, false, f.msg_counts_[i]); data[i].first = blocks[i]; TRY_RESULT(state, ShardStateQ::fetch(blocks[i], {}, state_root)); @@ -346,7 +346,7 @@ void OutMsgQueueImporter::get_proof_local(std::shared_ptr entry, Blo auto state = R.move_as_ok(); if (block.seqno() == 0) { std::vector> proof = { - td::Ref(true, block, state->root_cell(), td::Ref{})}; + td::Ref(true, block, state->root_cell(), td::Ref{}, true)}; td::actor::send_closure(SelfId, &OutMsgQueueImporter::got_proof, entry, std::move(proof), ProofSource::Local); return; } @@ -362,7 +362,7 @@ void OutMsgQueueImporter::get_proof_local(std::shared_ptr entry, Blo } Ref block_state_proof = create_block_state_proof(R.ok()->root_cell()).move_as_ok(); std::vector> proof = { - td::Ref(true, block, state->root_cell(), std::move(block_state_proof))}; + td::Ref(true, block, state->root_cell(), std::move(block_state_proof), true)}; td::actor::send_closure(SelfId, &OutMsgQueueImporter::got_proof, entry, std::move(proof), ProofSource::Local); }); diff --git a/validator/interfaces/out-msg-queue-proof.h b/validator/interfaces/out-msg-queue-proof.h index c0aa56106..564b040cf 100644 --- a/validator/interfaces/out-msg-queue-proof.h +++ b/validator/interfaces/out-msg-queue-proof.h @@ -26,17 +26,19 @@ namespace validator { using td::Ref; struct OutMsgQueueProof : public td::CntObject { - OutMsgQueueProof(BlockIdExt block_id, Ref state_root, Ref block_state_proof, + OutMsgQueueProof(BlockIdExt block_id, Ref state_root, Ref block_state_proof, bool is_local, td::int32 msg_count = -1) : block_id_(block_id) , state_root_(std::move(state_root)) , block_state_proof_(std::move(block_state_proof)) + , is_local_(is_local) , msg_count_(msg_count) { } BlockIdExt block_id_; Ref state_root_; Ref block_state_proof_; + bool is_local_ = false; td::int32 msg_count_; // -1 - no limit static td::Result>> fetch(ShardIdFull dst_shard, std::vector blocks, diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 534e5ea92..995afa186 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -69,31 +69,63 @@ struct CollationStats { td::uint32 estimated_bytes = 0, gas = 0, lt_delta = 0, estimated_collated_data_bytes = 0; int cat_bytes = 0, cat_gas = 0, cat_lt_delta = 0, cat_collated_data_bytes = 0; std::string limits_log; + double total_time = 0.0, work_time = 0.0, cpu_work_time = 0.0; + std::string time_stats; + td::uint32 transactions = 0; std::vector shard_configuration; td::uint32 ext_msgs_total = 0; td::uint32 ext_msgs_filtered = 0; td::uint32 ext_msgs_accepted = 0; td::uint32 ext_msgs_rejected = 0; - double total_time = 0.0, work_time = 0.0, cpu_work_time = 0.0; - std::string time_stats; + + td::uint64 old_out_msg_queue_size = 0; + td::uint64 new_out_msg_queue_size = 0; + td::uint32 msg_queue_cleaned = 0; + struct NeighborStats { + ShardIdFull shard; + bool is_trivial = false; + bool is_local = false; + int msg_limit = -1; + td::uint32 processed_msgs = 0; + td::uint32 skipped_msgs = 0; + bool limit_reached = false; + + tl_object_ptr tl() const { + return create_tl_object( + create_tl_shard_id(shard), is_trivial, is_local, msg_limit, processed_msgs, skipped_msgs, limit_reached); + } + }; + std::vector neighbors; + + double load_fraction_queue_cleanup = -1.0; + double load_fraction_dispatch = -1.0; + double load_fraction_internals = -1.0; + double load_fraction_externals = -1.0; + double load_fraction_new_msgs = -1.0; tl_object_ptr tl() const { - std::vector> shards; + std::vector> shards_obj; for (const BlockIdExt& block_id : shard_configuration) { - shards.push_back(create_tl_block_id(block_id)); + shards_obj.push_back(create_tl_block_id(block_id)); + } + std::vector> neighbors_obj; + for (const NeighborStats& neighbor : neighbors) { + neighbors_obj.push_back(neighbor.tl()); } auto block_stats = create_tl_object( - create_tl_object(ext_msgs_total, ext_msgs_filtered, ext_msgs_accepted, - ext_msgs_rejected), - transactions, std::move(shards)); + create_tl_object(ext_msgs_total, ext_msgs_filtered, + ext_msgs_accepted, ext_msgs_rejected), + transactions, std::move(shards_obj), old_out_msg_queue_size, new_out_msg_queue_size, msg_queue_cleaned, + std::move(neighbors_obj)); return create_tl_object( create_tl_block_id(block_id), collated_data_hash, cc_seqno, collated_at, actual_bytes, actual_collated_data_bytes, attempt, self.bits256_value(), is_validator, total_time, work_time, cpu_work_time, time_stats, - create_tl_object(estimated_bytes, gas, lt_delta, - estimated_collated_data_bytes, cat_bytes, cat_gas, - cat_lt_delta, cat_collated_data_bytes, limits_log), + create_tl_object( + estimated_bytes, gas, lt_delta, estimated_collated_data_bytes, cat_bytes, cat_gas, cat_lt_delta, + cat_collated_data_bytes, load_fraction_queue_cleanup, load_fraction_dispatch, load_fraction_internals, + load_fraction_externals, load_fraction_new_msgs, limits_log), std::move(block_stats)); } }; From 4e02ba69a2763c50ef851b2b0a7e6d3c632da356 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 3 Jun 2025 12:02:51 +0300 Subject: [PATCH 276/388] Check limits from config in BuildOutMsgQueueProof --- validator/full-node-shard.cpp | 8 ------ validator/impl/out-msg-queue-proof.cpp | 37 ++++++++++++++++++++++---- validator/impl/out-msg-queue-proof.hpp | 1 + 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index fd3e587f4..60125a1c8 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -706,14 +706,6 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod return; } block::ImportedMsgQueueLimits limits{(td::uint32)query.limits_->max_bytes_, (td::uint32)query.limits_->max_msgs_}; - if (limits.max_msgs > 512) { - promise.set_error(td::Status::Error("max_msgs is too big")); - return; - } - if (limits.max_bytes > (1 << 21)) { - promise.set_error(td::Status::Error("max_bytes is too big")); - return; - } FLOG(DEBUG) { sb << "Got query getOutMsgQueueProof to shard " << dst_shard.to_str() << " from blocks"; for (const BlockIdExt &id : blocks) { diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp index d5183d59c..e3330ea42 100644 --- a/validator/impl/out-msg-queue-proof.cpp +++ b/validator/impl/out-msg-queue-proof.cpp @@ -528,11 +528,11 @@ void OutMsgQueueImporter::alarm() { td::remove_if(it->second.pending_entries, [](const std::shared_ptr& entry) { return entry->done || entry->promises.empty(); }); if (it->second.timeout.is_in_past()) { - if (it->second.pending_entries.empty()) { - it = small_cache_.erase(it); - } else { - ++it; - } + if (it->second.pending_entries.empty()) { + it = small_cache_.erase(it); + } else { + ++it; + } } else { alarm_timestamp().relax(it->second.timeout); ++it; @@ -571,6 +571,33 @@ void BuildOutMsgQueueProof::abort_query(td::Status reason) { } void BuildOutMsgQueueProof::start_up() { + if (blocks_.size() > 16) { + abort_query(td::Status::Error("too many blocks")); + return; + } + td::actor::send_closure(manager_, &ValidatorManagerInterface::get_top_masterchain_state, + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::abort_query, + R.move_as_error_prefix("failed to get masterchain state: ")); + } else { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_masterchain_state, + R.move_as_ok()); + } + }); +} + +void BuildOutMsgQueueProof::got_masterchain_state(Ref mc_state) { + auto config_limits = mc_state->get_imported_msg_queue_limits(dst_shard_.is_masterchain()); + if ((td::uint64)config_limits.max_msgs * blocks_.size() < limits_.max_msgs) { + abort_query(td::Status::Error("too big max_msgs")); + return; + } + if ((td::uint64)config_limits.max_bytes * blocks_.size() < limits_.max_bytes) { + abort_query(td::Status::Error("too big max_bytes")); + return; + } + for (size_t i = 0; i < blocks_.size(); ++i) { BlockIdExt id = blocks_[i].id; ++pending; diff --git a/validator/impl/out-msg-queue-proof.hpp b/validator/impl/out-msg-queue-proof.hpp index 6f4c08d1d..b537f31a9 100644 --- a/validator/impl/out-msg-queue-proof.hpp +++ b/validator/impl/out-msg-queue-proof.hpp @@ -105,6 +105,7 @@ class BuildOutMsgQueueProof : public td::actor::Actor { void abort_query(td::Status reason); void start_up() override; + void got_masterchain_state(Ref mc_state); void got_state_root(size_t i, Ref root); void got_block_root(size_t i, Ref root); void build_proof(); From 99ac9ad1a173b42f5c8ca9378eaae4805adadba9 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 3 Jun 2025 12:05:07 +0300 Subject: [PATCH 277/388] Prepare to use rldp2 in validator group --- validator/manager.cpp | 4 ++-- validator/validator-group.cpp | 1 + validator/validator-group.hpp | 7 +++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/validator/manager.cpp b/validator/manager.cpp index 8cf85a9dc..db1a49efc 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2633,8 +2633,8 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group descr->addr.is_zero() ? ValidatorFullId{descr->key}.compute_short_id().bits256_value() : descr->addr}; auto G = td::actor::create_actor( PSTRING() << "valgroup" << shard.to_str(), shard, validator_id, session_id, validator_set, key_seqno, opts, - keyring_, adnl_, rldp_, overlays_, db_root_, actor_id(this), get_collation_manager(adnl_id), init_session, - opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_, + keyring_, adnl_, rldp_, rldp2_, overlays_, db_root_, actor_id(this), get_collation_manager(adnl_id), + init_session, opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_, opts_->need_monitor(shard, last_masterchain_state_)); return G; } diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index f9a8f2732..d2db0ae99 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -409,6 +409,7 @@ void ValidatorGroup::create_session() { } td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_adnl_id_); + td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_adnl_id_); } void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterchain_block_id) { diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index c7f940412..a7456cf9f 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -24,6 +24,7 @@ #include "validator-session/validator-session.h" #include "rldp/rldp.h" +#include "rldp2/rldp.h" #include @@ -77,8 +78,8 @@ class ValidatorGroup : public td::actor::Actor { td::Ref validator_set, BlockSeqno last_key_block_seqno, validatorsession::ValidatorSessionOptions config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, - td::actor::ActorId overlays, std::string db_root, - td::actor::ActorId validator_manager, + td::actor::ActorId rldp2, td::actor::ActorId overlays, + std::string db_root, td::actor::ActorId validator_manager, td::actor::ActorId collation_manager, bool create_session, bool allow_unsafe_self_blocks_resync, td::Ref opts, bool monitoring_shard) : shard_(shard) @@ -90,6 +91,7 @@ class ValidatorGroup : public td::actor::Actor { , keyring_(keyring) , adnl_(adnl) , rldp_(rldp) + , rldp2_(rldp2) , overlays_(overlays) , db_root_(std::move(db_root)) , manager_(validator_manager) @@ -131,6 +133,7 @@ class ValidatorGroup : public td::actor::Actor { td::actor::ActorId keyring_; td::actor::ActorId adnl_; td::actor::ActorId rldp_; + td::actor::ActorId rldp2_; td::actor::ActorId overlays_; std::string db_root_; td::actor::ActorId manager_; From 75c8ddd9bc4c88305cdbfed8b3187a3422d59253 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 4 Jun 2025 11:34:57 +0300 Subject: [PATCH 278/388] Increase default imported msg queue limit --- crypto/block/block.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/block/block.h b/crypto/block/block.h index c81dfdefa..c83fb6701 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -219,7 +219,7 @@ static inline std::ostream& operator<<(std::ostream& os, const MsgProcessedUptoC struct ImportedMsgQueueLimits { // Default values td::uint32 max_bytes = 1 << 16; - td::uint32 max_msgs = 30; + td::uint32 max_msgs = 100; bool deserialize(vm::CellSlice& cs); ImportedMsgQueueLimits operator*(td::uint32 x) const { return {max_bytes * x, max_msgs * x}; From faff9865e9da58570ad0a1dcfb55ab3d962ea7eb Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 4 Jun 2025 17:13:02 +0300 Subject: [PATCH 279/388] Fix error processing in CellDbIn::store_block_state_permanent_bulk --- validator/db/celldb.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 675859bd1..f4a3a084b 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -561,10 +561,13 @@ void CellDbIn::store_block_state_permanent_bulk(std::vector> new_blocks, async_executor, [this, SelfId = actor_id(this), timer = std::move(timer), timer_prepare = std::move(timer_prepare), promise = std::move(promise)](td::Result> R) mutable { - TRY_RESULT_PROMISE(promise, updates, std::move(R)); td::actor::send_lambda_later( - SelfId, [=, this, timer = std::move(timer), timer_prepare = std::move(timer_prepare), - updates = std::move(updates), promise = std::move(promise)]() mutable { + SelfId, [=, this, timer = std::move(timer), timer_prepare = std::move(timer_prepare), R = std::move(R), + promise = std::move(promise)]() mutable { + SCOPE_EXIT { + release_db(); + }; + TRY_RESULT_PROMISE(promise, updates, std::move(R)); TD_PERF_COUNTER(celldb_store_cell_multi); timer_prepare.pause(); td::Timer timer_write; @@ -608,7 +611,6 @@ void CellDbIn::store_block_state_permanent_bulk(std::vector> cell_db_statistics_.store_cell_bulk_queries_++; cell_db_statistics_.store_cell_bulk_total_blocks_ += updates.size(); } - release_db(); promise.set_result(td::Unit()); }); }); From dcde63acc9caae32a8ab3543cc715199641c384b Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 6 Jun 2025 14:08:29 +0300 Subject: [PATCH 280/388] Fix parsing shard block verifier config --- validator/validator-options.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/validator/validator-options.cpp b/validator/validator-options.cpp index 4768d64bf..a4a61373c 100644 --- a/validator/validator-options.cpp +++ b/validator/validator-options.cpp @@ -84,14 +84,14 @@ td::Status ShardBlockVerifierConfig::unpack(const ton_api::engine_validator_shar return td::Status::Error(PSTRING() << "duplicate node " << node_id); } shard.trusted_nodes.push_back(node_id); - if (shard_obj->required_confirms_ < 0 || shard_obj->required_confirms_ > (int)shard.trusted_nodes.size()) { - return td::Status::Error(PSTRING() - << "invalid required_confirms " << shard_obj->required_confirms_ << " for shard " - << shard.shard_id.to_str() << " (nodes: " << shard.trusted_nodes.size() << ")"); - } - shard.required_confirms = shard_obj->required_confirms_; - shards.push_back(std::move(shard)); } + if (shard_obj->required_confirms_ < 0 || shard_obj->required_confirms_ > (int)shard.trusted_nodes.size()) { + return td::Status::Error(PSTRING() + << "invalid required_confirms " << shard_obj->required_confirms_ << " for shard " + << shard.shard_id.to_str() << " (nodes: " << shard.trusted_nodes.size() << ")"); + } + shard.required_confirms = shard_obj->required_confirms_; + shards.push_back(std::move(shard)); } return td::Status::OK(); } From b4cf0ff2953a7a50f5759757403a8ba17b54ae51 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 6 Jun 2025 17:32:57 +0300 Subject: [PATCH 281/388] Bugfix in ext-client.cpp --- lite-client/ext-client.cpp | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/lite-client/ext-client.cpp b/lite-client/ext-client.cpp index a0e48e64a..fd624ff8c 100644 --- a/lite-client/ext-client.cpp +++ b/lite-client/ext-client.cpp @@ -99,7 +99,7 @@ class ExtClientImpl : public ExtClient { promise = std::move(promise)](td::Result R) mutable { if (R.is_error() && (R.error().code() == ton::ErrorCode::timeout || R.error().code() == ton::ErrorCode::cancelled)) { - td::actor::send_closure(SelfId, &ExtClientImpl::on_server_error, server_idx); + td::actor::send_closure(SelfId, &ExtClientImpl::on_server_status, server_idx, false); } promise.set_result(std::move(R)); }; @@ -163,9 +163,10 @@ class ExtClientImpl : public ExtClient { explicit Callback(td::actor::ActorId parent, size_t idx) : parent_(std::move(parent)), idx_(idx) { } void on_ready() override { + td::actor::send_closure(parent_, &ExtClientImpl::on_server_status, idx_, true); } void on_stop_ready() override { - td::actor::send_closure(parent_, &ExtClientImpl::on_server_error, idx_); + td::actor::send_closure(parent_, &ExtClientImpl::on_server_status, idx_, false); } private: @@ -199,19 +200,32 @@ class ExtClientImpl : public ExtClient { return; } for (Server& server : servers_) { - if (server.timeout && server.timeout.is_in_past()) { + if (!server.timeout) { + continue; + } + if (server.timeout.is_in_past()) { LOG(INFO) << "Closing connection to liteserver #" << server.idx << " (" << server.config.addr.get_ip_str() << ":" << server.config.addr.get_port() << ")"; server.client.reset(); server.alive = false; server.ignore_until = {}; + server.timeout = {}; + } else { + alarm_timestamp().relax(server.timeout); } } } - void on_server_error(size_t idx) { - servers_[idx].alive = false; - servers_[idx].ignore_until = td::Timestamp::in(BAD_SERVER_TIMEOUT); + void on_server_status(size_t idx, bool ok) { + if (ok) { + if (connect_to_all_) { + servers_[idx].alive = true; + servers_[idx].ignore_until = td::Timestamp::never(); + } + } else { + servers_[idx].alive = false; + servers_[idx].ignore_until = td::Timestamp::in(BAD_SERVER_TIMEOUT); + } } }; From b788322dd4a0df61eb6a635675437c2cc71cf211 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 10 Jun 2025 11:12:15 +0300 Subject: [PATCH 282/388] Fix setting block hashes in session stats --- validator-session/validator-session.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 9ce7e5404..e63b0e569 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -1240,6 +1240,9 @@ void ValidatorSessionImpl::stats_process_action(td::uint32 node_id, ton_api::val obj.round_, description().get_source_id(node_id), candidate_id); if (stat && stat->got_submit_at <= 0.0) { stat->got_submit_at = td::Clocks::system(); + stat->block_id.root_hash = obj.root_hash_; + stat->block_id.file_hash = obj.file_hash_; + stat->collated_data_hash = obj.collated_data_file_hash_; } }, [&](const ton_api::validatorSession_message_approvedBlock &obj) { From 52bcce61a2e71de4e81f934c6737d23546db3a8c Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sun, 30 Mar 2025 10:44:56 +0300 Subject: [PATCH 283/388] [Tolk] Optimize storeUint: merge consecutive constant stores Before: b.storeUint(0x18, 6) Now: b.storeUint(0,1).storeUint(1,1)... The compiler is able to join consecutive stores of const values. It works at the IR level, not at AST. That's why it works after constant folding: > x = 0; > b.storeUint(x, 10); > x = 1; > if (x==1) b.storeUint(x, 10); Merged to "store 1, 20." Same for skip bits, also merged. --- crypto/smartcont/tolk-stdlib/common.tolk | 6 +- tolk-tester/tests/cells-slices.tolk | 196 +++++++++++++++++++++-- tolk-tester/tests/pack-unpack-1.tolk | 12 +- tolk-tester/tests/strings-tests.tolk | 2 +- tolk/analyzer.cpp | 16 ++ tolk/builtins.cpp | 70 ++++++++ tolk/codegen.cpp | 6 + tolk/optimize.cpp | 97 +++++++++++ tolk/stack-transform.cpp | 14 +- tolk/tolk.h | 31 +--- 10 files changed, 387 insertions(+), 63 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index c9e116860..76305743a 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -664,7 +664,7 @@ fun slice.loadBool(mutate self): bool /// Shifts a slice pointer to [len] bits forward, mutating the slice. @pure fun slice.skipBits(mutate self, len: int): self - asm "SDSKIPFIRST"; + builtin; /// Returns the first `0 ≤ len ≤ 1023` bits of a slice. @pure @@ -758,13 +758,13 @@ fun builder.storeAddress(mutate self, addr: address): self /// Stores amount of Toncoins into a builder. @pure fun builder.storeCoins(mutate self, x: coins): self - asm "STGRAMS"; + builtin; /// Stores bool (-1 or 0) into a builder. /// Attention: true value is `-1`, not 1! If you pass `1` here, TVM will throw an exception. @pure fun builder.storeBool(mutate self, x: bool): self - asm(x self) "1 STI"; + builtin; /// Stores dictionary (represented by TVM `cell` or `null`) into a builder. /// In other words, stores a `1`-bit and a reference to [c] if [c] is not `null` and `0`-bit otherwise. diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index 253ebfff6..6a14578fd 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -77,7 +77,8 @@ fun test5(): [int,int] { @method_id(106) fun test6() { - return beginCell().storeUint(1, 32).storeUint(2, 32).storeUint(3, 32); + var st = beginCell().storeUint(1, 32).storeUint(2, 32).storeUint(3, 32).endCell().beginParse(); + return st.loadUint(96) == ((1 << 64) + (2 << 32) + 3); } @method_id(107) @@ -229,6 +230,96 @@ fun test16() { } +@method_id(117) +fun test17() { + var b = beginCell().storeUint(1, 4).storeCoins(0).storeInt(123, 8); + var s = b.endCell().beginParse(); + return (s.loadUint(4), s.loadCoins(), s.loadUint(8)); +} + +@method_id(118) +fun test18() { + var x = 0; + var b = beginCell(); + b = b.storeUint(x, 14); + x += 12; + if (10 > 3) { x += x; } + if (true) { + b.storeInt(x + 2, 8).storeUint(x = match (x) { 24 => 5, else => 0 }, 4); + } + var s = b.endCell().beginParse(); + return (s.loadUint(14), s.loadInt(8), s.loadUint(4)); +} + +fun test19() { + // numbers with potential overflow for STU are not joined, check via codegen + var b = beginCell(); + b.storeInt(123, 4).storeUint(0xFF, 8).storeUint(0xFF, 8).storeInt(-1, 8); + return b; +} + +@method_id(120) +fun test20() { + var x = false; + var n = 4; + var b = true ? beginCell() : null; + b.storeBool(true).storeBool(x); + b = b.storeBool(true).storeUint(0, n *= 2).storeBool(!!true).storeCoins(0); + var s = b.endCell().beginParse(); + return (s.loadBool(), s.loadBool(), s.loadBool(), s.loadUint(8), s.loadBool(), s.loadCoins()); +} + +fun test21(s: slice) { + // successive skipBits are also joined + var x = 8; + s.skipBits(x); + x -= 4; + s = s.skipBits(x).skipBits(2); + x *= 0; + s = s.skipBits(x); + return s; +} + +@method_id(122) +fun test22() { + // different builders aren't mixed, store inside them are joined independently + var (b1, b2) = (beginCell(), beginCell()); + b1.storeUint(8, 16).storeUint(8, 8); + b2.storeUint(8, 32).storeUint(1<<88, 100); + return ( + b1.endCell().beginParse().remainingBitsCount(), + b2.endCell().beginParse().skipBits(32).loadUint(100), + ); +} + +@method_id(123) +fun test23(uns: bool) { + // corner values, signed/unsigned 255/256 + var b = beginCell(); + if (uns) { + b.storeUint(1, 100).storeUint(2, 100).storeInt(3, 55).storeInt(0, 1); + b.storeUint(8, 256); + } else { + b.storeInt(1, 10).storeUint(2, 190).storeInt(3, 54).storeUint(1, 1); + } + return b.bitsCount(); +} + +@method_id(124) +fun test24(uns: bool) { + // doesn't fit into a single STI/STU instruction, is splitted + var b = beginCell(); + if (uns) { + b.storeUint(1, 100).storeUint(2, 100) + .storeInt(3, 100).storeInt(8, 19); + return b.endCell().beginParse().skipBits(200+100).loadInt(19); + } else { + b.storeInt(1, 20).storeUint(2, 200).storeInt(3, 35) + .storeUint(1, 1).storeUint(5, 5).storeUint(10, 10); + return b.endCell().beginParse().skipBits(255+6).loadUint(10); + } +} + fun main(): int { return 0; } @@ -239,6 +330,7 @@ fun main(): int { @testcase | 103 | 103 | 103 @testcase | 104 | | [ 1 3 ] @testcase | 105 | | [ 210 1 ] +@testcase | 106 | | -1 @testcase | 107 | | 72 40 72 @testcase | 108 | | 0 40 32 @testcase | 110 | | 64 3 0 0 -1 0 100 -1 @@ -249,20 +341,102 @@ fun main(): int { @testcase | 114 | 0 | 0 0 0 @testcase | 115 | | 123 456 123 456 @testcase | 116 | | BC{00140008000000ff00000008} +@testcase | 117 | | 1 0 123 +@testcase | 118 | | 0 26 5 +@testcase | 120 | | -1 0 -1 0 -1 0 +@testcase | 122 | | 24 309485009821345068724781056 +@testcase | 123 | -1 | 512 +@testcase | 123 | 0 | 255 +@testcase | 124 | -1 | 8 +@testcase | 124 | 0 | 10 + +We test that consequtive storeInt/storeUint with constants are joined into a single number -Note, that since 'compute-asm-ltr' became on be default, chaining methods codegen is not quite optimal. @fif_codegen """ test6 PROC:<{ - 1 PUSHINT // '0=1 - NEWC // '0=1 '1 - 32 STU // '1 - 2 PUSHINT // '1 '4=2 - SWAP // '4=2 '1 - 32 STU // '1 - 3 PUSHINT // '1 '7=3 - SWAP // '7=3 '1 - 32 STU // '1 + 18446744082299486211 PUSHINT + NEWC + 96 STU // '2 + ENDC // '11 +""" + +@fif_codegen +""" + test17 PROC:<{ + 4219 PUSHINT + NEWC + 16 STU +""" + +@fif_codegen +""" + test18 PROC:<{ + 421 PUSHINT + NEWC + 26 STU +""" + +@fif_codegen +""" + test19 PROC:<{ + NEWC + 123 PUSHINT + SWAP + 4 STI + 16 PUSHPOW2DEC + 16 STUR + -1 PUSHINT + SWAP + 8 STI }> """ + +@fif_codegen +""" + test20 PROC:<{ + 40976 PUSHINT + NEWC + 16 STU +""" + +@fif_codegen +""" + test21 PROC:<{ + 14 PUSHINT + SDSKIPFIRST + }> +""" + +@fif_codegen +""" + test22 PROC:<{ + NEWC // '2 + NEWC // b1 b2 + SWAP // b2 b1 + 2056 PUSHINT + 24 STUR // b2 b1 + SWAP // b1 b2 + 10141514286835656557042350424064 PUSHINTX + 132 STUR // b1 b2 +""" + +@fif_codegen +""" + test23 PROC:<{ + NEWC + SWAP + IF:<{ + 91343852333181432387730302044911803916571639814 PUSHINT + 256 STUR + 8 PUSHINT + 256 STUR + }>ELSE<{ + 56539106072908298546665520023773392506479484700019806659963456035401760775 PUSHINT + 255 STIR + }> + BBITS + }> +""" + */ diff --git a/tolk-tester/tests/pack-unpack-1.tolk b/tolk-tester/tests/pack-unpack-1.tolk index 002df87dd..379055c46 100644 --- a/tolk-tester/tests/pack-unpack-1.tolk +++ b/tolk-tester/tests/pack-unpack-1.tolk @@ -107,16 +107,12 @@ fun main(c: cell) { @fif_codegen """ test3 PROC:<{ - 20 PUSHINT // yy=20 - 10 PUSHINT // p.y=20 p.x=10 - NEWC // p.y=20 p.x=10 b - 32 STI // p.y=20 b - 32 STI // b + 42949672980 PUSHINT + NEWC + 64 STI // b ENDC // c CTOS // s - 32 PUSHINT // s '15=32 - SDSKIPFIRST // s - 32 PUSHINT // s '16=32 + 64 PUSHINT SDSKIPFIRST // s SBITS // '17 }> diff --git a/tolk-tester/tests/strings-tests.tolk b/tolk-tester/tests/strings-tests.tolk index 287dc0df3..bbe458326 100644 --- a/tolk-tester/tests/strings-tests.tolk +++ b/tolk-tester/tests/strings-tests.tolk @@ -78,5 +78,5 @@ fun test1() { @testcase | 0 | | 0 @testcase | 101 | | [ 65 66 67 68 ] -@code_hash 58447050269190289721084012139099481162782788646785441106022886746601529758643 +@code_hash 61963905046482786665931036224265588524206004864815957521622605110240907698574 */ diff --git a/tolk/analyzer.cpp b/tolk/analyzer.cpp index f9721f5f1..8e9fe914e 100644 --- a/tolk/analyzer.cpp +++ b/tolk/analyzer.cpp @@ -678,6 +678,16 @@ void Op::prepare_args(VarDescrList values) { } } +void Op::maybe_swap_builtin_args_to_compile() { + // in builtins.cpp, where optimizing constants are done, implementations assume that args are passed ltr (as declared); + // if a function has arg_order, called arguments might have been put on a stack not ltr, but in asm order; + // here we swap them back before calling FunctionBodyBuiltin compile, and also swap after + tolk_assert(arg_order_already_equals_asm()); + if (f_sym->method_name == "storeUint" || f_sym->method_name == "storeInt" || f_sym->method_name == "storeBool") { + std::swap(args[0], args[1]); + } +} + VarDescrList Op::fwd_analyze(VarDescrList values) { var_info.import_values(values); switch (cl) { @@ -705,7 +715,13 @@ VarDescrList Op::fwd_analyze(VarDescrList values) { } AsmOpList tmp; if (!f_sym->is_asm_function()) { + if (arg_order_already_equals_asm()) { + maybe_swap_builtin_args_to_compile(); + } std::get(f_sym->body)->compile(tmp, res, args, loc); + if (arg_order_already_equals_asm()) { + maybe_swap_builtin_args_to_compile(); + } } int j = 0; for (var_idx_t i : left) { diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 39075452d..6a82b432f 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1022,7 +1022,22 @@ static AsmOp compile_fetch_int(std::vector& res, std::vector // fun builder.storeUint (mutate self, x: int, len: int): self asm(x b len) "STUX"; static AsmOp compile_store_int(std::vector& res, std::vector& args, SrcLocation loc, bool sgnd) { tolk_assert(args.size() == 3 && res.size() == 1); + auto& x = args[1]; auto& z = args[2]; + // purpose: to merge consecutive `b.storeUint(0, 1).storeUint(1, 1)` into one "1 PUSHINT + 2 STU", + // when constant arguments are passed, keep them as a separate (fake) instruction, to be handled by optimizer later + bool value_and_len_is_const = z.is_int_const() && x.is_int_const(); + if (value_and_len_is_const && G.settings.optimization_level >= 2) { + // don't handle negative numbers or potential overflow, merging them is incorrect + bool value_is_safe = sgnd + ? x.int_const >= 0 && z.int_const < 64 && x.int_const < (1ULL << (z.int_const->to_long() - 1)) + : x.int_const >= 0; + if (value_is_safe && z.int_const > 0 && z.int_const <= (255 + !sgnd)) { + z.unused(); + x.unused(); + return AsmOp::Custom(loc, "MY_store_int"s + (sgnd ? "I " : "U ") + x.int_const->to_dec_string() + " " + z.int_const->to_dec_string(), 1); + } + } if (z.is_int_const() && z.int_const > 0 && z.int_const <= 256) { z.unused(); return exec_arg_op(loc, sgnd? "STI" : "STU", z.int_const, 2, 1); @@ -1030,6 +1045,36 @@ static AsmOp compile_store_int(std::vector& res, std::vector return exec_op(loc, sgnd ? "STIX" : "STUX", 3, 1); } +// fun builder.storeBool(mutate self, value: bool): self asm( -> 1 0) "1 STI"; +static AsmOp compile_store_bool(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(args.size() == 2 && res.size() == 1); + auto& v = args[1]; + // same purpose as for storeInt/storeUint above + // (particularly, `b.storeUint(const_int,32).storeBool(const_bool)` will be joined) + if (v.is_int_const() && v.int_const == 0 && G.settings.optimization_level >= 2) { + v.unused(); + return AsmOp::Custom(loc, "MY_store_intU 0 1", 1); + } + if (v.is_int_const() && v.int_const == -1 && G.settings.optimization_level >= 2) { + v.unused(); + return AsmOp::Custom(loc, "MY_store_intU 1 1", 1); + } + return exec_op(loc, "1 STI", 2, 1); +} + +// fun builder.storeCoins(mutate self, value: coins): self asm "STGRAMS"; +static AsmOp compile_store_coins(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(args.size() == 2 && res.size() == 1); + auto& v = args[1]; + // same purpose as for storeInt/storeUint above + // (particularly, `b.storeUint(const_int,32).storeCoins(const_zero)` will be joined) + if (v.is_int_const() && v.int_const == 0 && G.settings.optimization_level >= 2) { + v.unused(); + return AsmOp::Custom(loc, "MY_store_intU 0 4", 1); + } + return exec_op(loc, "STGRAMS", 2, 1); +} + // fun slice.loadBits (mutate self, len: int): self asm(s len -> 1 0) "LDSLICEX" // fun slice.preloadBits(self, len: int): slice asm(s len -> 1 0) "PLDSLICEX" static AsmOp compile_fetch_slice(std::vector& res, std::vector& args, SrcLocation loc, bool fetch) { @@ -1062,6 +1107,21 @@ AsmOp compile_slice_sdbeginsq(std::vector& res, std::vector& throw ParseError(loc, "slice.tryStripPrefix can be used only with constant arguments"); } +// fun slice.skipBits(mutate self, len: int): self "SDSKIPFIRST" +AsmOp compile_skip_bits_in_slice(std::vector& res, std::vector& args, SrcLocation loc) { + tolk_assert(args.size() == 2 && res.size() == 1); + auto& len = args[1]; + // same technique as for storeUint: + // consecutive `s.skipBits(8).skipBits(const_var_16)` will be joined into a single 24 + // to track this, represent it as a separate fake instruction to be detected by optimizer later + if (len.is_int_const() && len.int_const >= 0 && G.settings.optimization_level >= 2) { + len.unused(); + return AsmOp::Custom(loc, "MY_skip_bits " + len.int_const->to_dec_string(), 1); + } + return exec_op(loc, "SDSKIPFIRST", 2, 1); +} + + // fun tuple.get(t: tuple, index: int): X asm "INDEXVAR"; static AsmOp compile_tuple_get(std::vector& res, std::vector& args, SrcLocation loc) { tolk_assert(args.size() == 2 && res.size() == 1); @@ -1353,6 +1413,9 @@ void define_builtins() { std::bind(compile_fetch_slice, _1, _2, _3, true), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, {}, {1, 0}); + define_builtin_method("slice.skipBits", Slice, ParamsSliceInt, Slice, nullptr, + compile_skip_bits_in_slice, + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf); define_builtin_method("slice.preloadInt", Slice, ParamsSliceInt, Int, nullptr, std::bind(compile_fetch_int, _1, _2, _3, false, true), FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); @@ -1373,6 +1436,13 @@ void define_builtins() { std::bind(compile_store_int, _1, _2, _3, false), FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf, {1, 0, 2}, {}); + define_builtin_method("builder.storeBool", Builder, {Builder, Bool}, Unit, nullptr, + compile_store_bool, + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf, + {1, 0}, {}); + define_builtin_method("builder.storeCoins", Builder, {Builder, TypeDataCoins::create()}, Unit, nullptr, + compile_store_coins, + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf); define_builtin_method("tuple.get", Tuple, {Tuple, Int}, typeT, declGenericT, compile_tuple_get, FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); diff --git a/tolk/codegen.cpp b/tolk/codegen.cpp index ff6c39676..a7c84d27d 100644 --- a/tolk/codegen.cpp +++ b/tolk/codegen.cpp @@ -508,7 +508,13 @@ bool Op::generate_code_step(Stack& stack) { if (f_sym->is_asm_function()) { std::get(f_sym->body)->compile(stack.o, loc); // compile res := f (args) } else { + if (arg_order_already_equals_asm()) { + maybe_swap_builtin_args_to_compile(); + } std::get(f_sym->body)->compile(stack.o, res, args, loc); // compile res := f (args) + if (arg_order_already_equals_asm()) { + maybe_swap_builtin_args_to_compile(); + } } } else { if (f_sym->is_inline() || f_sym->is_inline_ref()) { diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp index e6d6b070d..c8fd8fc96 100644 --- a/tolk/optimize.cpp +++ b/tolk/optimize.cpp @@ -164,6 +164,102 @@ bool Optimizer::detect_rewrite_big_THROW() { return true; } +// purpose 1: for one constant b.storeInt(123, 32) generate not "123 PUSHINT; SWAP; STI", but "123 PUSHINT; STIR" +// purpose 2: consecutive b.storeUint(ff, 16).storeUint(ff, 16) generate one "00ff00ff" STU +// (since it works at IR level, it also works for const variables and auto-serialization) +bool Optimizer::detect_rewrite_MY_store_int() { + bool first_my_store = op_[0]->is_custom() && op_[0]->op.starts_with("MY_store_int"); + if (!first_my_store) { + return false; + } + bool first_unsigned = op_[0]->op[12] == 'U'; + + int n_merged = 0; + td::RefInt256 total_number = td::make_refint(0); + int total_len = 0; + for (int i = 0; i < pb_; ++i) { + std::string_view s_op_number_len = op_[i]->op; // "MY_store_intU 123 32" + if (!s_op_number_len.starts_with("MY_store_int")) { + break; + } + + size_t sp = s_op_number_len.rfind(' '); + std::string s_number(s_op_number_len.substr(13 + 1, sp - 13 - 1)); + int len = std::stoi(std::string(s_op_number_len.substr(sp + 1))); + + if (total_len + len > (255 + first_unsigned)) { + break; + } + if (total_number != 0) { + total_number <<= len; + } + total_number += td::string_to_int256(s_number); + total_len += len; + n_merged++; + } + + p_ = n_merged; + q_ = 2; + oq_[0] = std::make_unique(AsmOp::IntConst(op_[0]->loc, total_number)); + if (total_number == 0 && total_len == 4 && first_unsigned) { // "STGRAMS" stores four 0-bits cheaper than "4 STUR" + oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->loc, "STGRAMS", 1, 1)); + } else { + oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->loc, std::to_string(total_len) + (first_unsigned ? " STUR" : " STIR"), 1, 1)); + } + return true; +} + +// purpose: consecutive `s.skipBits(8).skipBits(const_var_16)` will be joined into a single 24 +bool Optimizer::detect_rewrite_MY_skip_bits() { + bool first_my_skip = op_[0]->is_custom() && op_[0]->op.starts_with("MY_skip_bits"); + if (!first_my_skip) { + return false; + } + + int n_merged = 0; + int total_skip_bits = 0; + for (int i = 0; i < pb_; ++i) { + std::string_view s_op_len = op_[i]->op; // "MY_skip_bits 32" + if (!s_op_len.starts_with("MY_skip_bits")) { + break; + } + + std::string s_number(s_op_len.substr(s_op_len.find(' ') + 1)); + total_skip_bits += std::stoi(s_number); + n_merged++; + } + + p_ = n_merged; + q_ = 2; + oq_[0] = std::make_unique(AsmOp::IntConst(op_[0]->loc, td::make_refint(total_skip_bits))); + oq_[1] = std::make_unique(AsmOp::Custom(op_[0]->loc, "SDSKIPFIRST")); + return true; +} + +// pattern `NEWC` + `xxx PUSHINT` + `32 STUR` -> `xxx PUSHINT` + `NEWC` + `32 STU`, it's a bit cheaper +bool Optimizer::detect_rewrite_NEWC_PUSH_STUR() { + bool first_newc = op_[0]->is_custom() && op_[0]->op == "NEWC"; + if (!first_newc || pb_ < 3) { + return false; + } + bool next_push = op_[1]->is_const() && op_[1]->op.ends_with(" PUSHINT"); // actually there can be PUSHPOWDEC2, but ok + if (!next_push) { + return false; + } + bool next_stu_r = op_[2]->is_custom() && (op_[2]->op.ends_with(" STUR") || op_[2]->op.ends_with(" STIR")); + if (!next_stu_r) { + return false; + } + + p_ = 3; + q_ = 3; + oq_[0] = std::move(op_[1]); + oq_[1] = std::move(op_[0]); + oq_[2] = std::make_unique(AsmOp::Custom(oq_[0]->loc, op_[2]->op.substr(0, op_[2]->op.size() - 1), 1, 1)); + return true; +} + + bool Optimizer::is_push_const(int* i, int* c) const { return pb_ >= 3 && pb_ <= l2_ && tr_[pb_ - 1].is_push_const(i, c); } @@ -583,6 +679,7 @@ bool Optimizer::find_at_least(int pb) { (is_pop(&i) && rewrite(AsmOp::Pop(loc, i))) || (is_pop_pop(&i, &j) && rewrite(AsmOp::Pop(loc, i), AsmOp::Pop(loc, j))) || (is_xchg_xchg(&i, &j, &k, &l) && rewrite(AsmOp::Xchg(loc, i, j), AsmOp::Xchg(loc, k, l))) || detect_rewrite_big_THROW() || + detect_rewrite_MY_store_int() || detect_rewrite_MY_skip_bits() || detect_rewrite_NEWC_PUSH_STUR() || (!(mode_ & 1) && ((is_rot() && rewrite(AsmOp::Custom(loc, "ROT", 3, 3))) || (is_rotrev() && rewrite(AsmOp::Custom(loc, "-ROT", 3, 3))) || (is_2dup() && rewrite(AsmOp::Custom(loc, "2DUP", 2, 4))) || diff --git a/tolk/stack-transform.cpp b/tolk/stack-transform.cpp index fe5735e5c..d86afaec6 100644 --- a/tolk/stack-transform.cpp +++ b/tolk/stack-transform.cpp @@ -254,7 +254,7 @@ bool StackTransform::compose(const StackTransform &a, const StackTransform &b, S x1 = a.try_load(i); } int y = b.A[j - 1].second; - if (!c.try_store(x2, a(y))) { + if (!c.try_store(x2, a.get(y))) { return false; } x2 = b.try_load(j, a.d); @@ -360,18 +360,6 @@ StackTransform StackTransform::Xchg(int i, int j, bool relaxed) { return t; } -StackTransform StackTransform::Push(int i) { - StackTransform t; - t.apply_push(i); - return t; -} - -StackTransform StackTransform::Pop(int i) { - StackTransform t; - t.apply_pop(i); - return t; -} - bool StackTransform::is_xchg(int i, int j) const { if (i == j) { return is_id(); diff --git a/tolk/tolk.h b/tolk/tolk.h index 6e5af87ca..a329b1d55 100644 --- a/tolk/tolk.h +++ b/tolk/tolk.h @@ -362,6 +362,7 @@ struct Op { bool set_var_info_except(const VarDescrList& new_var_info, const std::vector& var_list); bool set_var_info_except(VarDescrList&& new_var_info, const std::vector& var_list); void prepare_args(VarDescrList values); + void maybe_swap_builtin_args_to_compile(); VarDescrList fwd_analyze(VarDescrList values); bool mark_noreturn(); bool is_empty() const { @@ -707,9 +708,6 @@ struct StackTransform { bool almost_equal(const StackTransform& other) const { return equal(other, true); } - bool operator==(const StackTransform& other) const { - return dp == other.dp && almost_equal(other); - } bool operator<=(const StackTransform& other) const { return dp <= other.dp && almost_equal(other); } @@ -724,28 +722,6 @@ struct StackTransform { return get(i); } bool set(int i, int v, bool relaxed = false); - int operator()(int i) const { - return get(i); - } - class Pos { - StackTransform& t_; - int p_; - - public: - Pos(StackTransform& t, int p) : t_(t), p_(p) { - } - Pos& operator=(const Pos& other) = delete; - operator int() const { - return t_.get(p_); - } - const Pos& operator=(int v) const { - t_.set(p_, v); - return *this; - } - }; - Pos operator[](int i) { - return Pos(*this, i); - } static const StackTransform rot; static const StackTransform rot_rev; bool is_id() const { @@ -814,8 +790,6 @@ struct StackTransform { void show(std::ostream& os, int mode = 0) const; static StackTransform Xchg(int i, int j, bool relaxed = false); - static StackTransform Push(int i); - static StackTransform Pop(int i); private: int try_load(int& i, int offs = 0) const; // returns A[i++].first + offs or inf_x @@ -926,6 +900,9 @@ struct Optimizer { bool is_2pop_blkdrop(int* i, int* j, int* k); bool detect_rewrite_big_THROW(); + bool detect_rewrite_MY_store_int(); + bool detect_rewrite_MY_skip_bits(); + bool detect_rewrite_NEWC_PUSH_STUR(); AsmOpConsList extract_code(); }; From 3f4eca40651ecebed6e6ef55cd0688cbfb4f2c90 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sun, 1 Jun 2025 11:48:49 +0300 Subject: [PATCH 284/388] [Tolk] Better resolve receivers for conflicting names Allow `fun address.staticMethod`. Before, it was not allowed due to a global `address` function. Now, the type has more precedence. --- tolk-tester/tests/calls-tests.tolk | 10 ++++++++++ tolk-tester/tests/parse-address.tolk | 6 +++++- tolk/pipe-resolve-types.cpp | 3 ++- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/tolk-tester/tests/calls-tests.tolk b/tolk-tester/tests/calls-tests.tolk index de508a2d5..a03cd5a30 100644 --- a/tolk-tester/tests/calls-tests.tolk +++ b/tolk-tester/tests/calls-tests.tolk @@ -150,6 +150,15 @@ fun test8() { return (createTFrom(), createTFrom((5, (8, createTFrom().0)))); } +fun int(initial: int) { return initial } +fun int.create0() { return 0 } +fun int.plus1(self) { return self + 1 } + +@method_id(109) +fun test9() { + return int(10) + int(5).plus1() + int.create0() + int.create0().plus1(); +} + fun main() {} /** @@ -161,4 +170,5 @@ fun main() {} @testcase | 106 | 0 | 5 -5 50 -50 0 @testcase | 107 | | 16 5 @testcase | 108 | | [ 1 2 3 ] [ 5 8 1 ] +@testcase | 109 | | 17 */ diff --git a/tolk-tester/tests/parse-address.tolk b/tolk-tester/tests/parse-address.tolk index cf74af3f1..3f1151f7a 100644 --- a/tolk-tester/tests/parse-address.tolk +++ b/tolk-tester/tests/parse-address.tolk @@ -21,6 +21,10 @@ fun verifyAddr(addr: address, workchain: int, number: int) { assert (s.loadUint(256) == number) throw 111; } +fun address.createNone() { + return createAddressNone() +} + fun check0(addr: address) { assert (addr.getWorkchain() == 0) throw 111; } @@ -134,7 +138,7 @@ fun main() { cc2 != cc1, (cc1 as slice).bitsEqual(((cc2 as slice) as address) as slice), createAddressNone() == cc1, - createAddressNone() == createAddressNone(), + createAddressNone() == address.createNone(), check0(address("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff")) ); } diff --git a/tolk/pipe-resolve-types.cpp b/tolk/pipe-resolve-types.cpp index 25d98bb46..194cc4998 100644 --- a/tolk/pipe-resolve-types.cpp +++ b/tolk/pipe-resolve-types.cpp @@ -467,7 +467,8 @@ class ResolveTypesInsideFunctionVisitor final : public ASTVisitorFunctionBody { // for static method calls, like "int.zero()" or "Point.create()", dot obj symbol is unresolved for now // so, resolve it as a type and store as a "type reference symbol" if (auto obj_ref = v->get_obj()->try_as()) { - if (obj_ref->sym == nullptr) { + // also, `someFn.prop` doesn't make any sense, show "unknown type"; it also forces `address.staticMethod()` to work + if (obj_ref->sym == nullptr || obj_ref->sym->try_as()) { std::string_view obj_type_name = obj_ref->get_identifier()->name; AnyTypeV obj_type_node = createV(obj_ref->loc, obj_type_name); if (obj_ref->has_instantiationTs()) { // Container.create From ba0f9835f6e614b00ec547d764c458873826c6d1 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 5 Jun 2025 14:18:05 +0700 Subject: [PATCH 285/388] [Tolk] Optimization: generate `IFJMP` inside `match` arms Then `match` is the last statement, or is immediately followed by `return`, generate not `IF...ELSE` to Fift output, but `IFJMP...`. Same for `if/else` statement. --- tolk-tester/tests/if-else-tests.tolk | 171 +++++++++++++++++++++++++++ tolk/pipe-ast-to-legacy.cpp | 42 ++++++- 2 files changed, 208 insertions(+), 5 deletions(-) diff --git a/tolk-tester/tests/if-else-tests.tolk b/tolk-tester/tests/if-else-tests.tolk index 159a86bd5..1656519af 100644 --- a/tolk-tester/tests/if-else-tests.tolk +++ b/tolk-tester/tests/if-else-tests.tolk @@ -1,3 +1,5 @@ +global t: tuple; + @method_id(101) fun test1(x: int): int { if (x > 200) { @@ -100,6 +102,86 @@ fun test8(op: int) { return (withNoElse(op), withElse(op), withMatch(op)); } +fun demo9_1(x: int) { + if (x > 0) { t.push(123); } + else { t.push(456) } +} + +fun demo9_2(x: int) { + if (x > 0) { t.push(123); } +} + +fun demo9_3(x: int) { + if (x > 0) { t.push(123); } + return; +} + +fun demo9_4(x: int) { + if (x != -100) { + if (x > 0) { t.push(123); } + return; + } +} + +@method_id(109) +fun test9(x: int) { + t = createEmptyTuple(); + demo9_1(x); demo9_2(x); demo9_3(x); demo9_4(x); + return t; +} + +fun demo10_1(x: int) { + match(x) { + 0 => t.push(123), + 1 => t.push(456), + } +} + +fun demo10_2(x: int) { + if (x != -100) { + match (x) { + 0 => t.push(123), + else => t.push(456) + } + return; + } + t.push(789) +} + +@method_id(110) +fun test10(x: int) { + t = createEmptyTuple(); + demo10_1(x); demo10_2(x); + return t; +} + +fun demo_neg_11_1(mutate x: int) { + match (x) { + -1 => { x = 0; } + else => { x = 1; } + } +} + +fun justVoidPush() { + t.push(100); +} + +fun demo_neg_11_2(x: int) { + match (x) { + -1 => t.push(123), + else => t.push(456) + } + return justVoidPush(); +} + +@method_id(111) +fun test11(x: int) { + t = createEmptyTuple(); + demo_neg_11_1(mutate x); + demo_neg_11_2(x); + return (x, t); +} + fun main() { } @@ -125,6 +207,11 @@ fun main() { @testcase | 106 | 5 | 5000 @testcase | 108 | 123 | 100 100 100 @testcase | 108 | 345 | 300 300 300 +@testcase | 109 | 10 | [ 123 123 123 123 ] +@testcase | 109 | -10 | [ 456 ] +@testcase | 110 | 0 | [ 123 123 ] +@testcase | 110 | 5 | [ 456 ] +@testcase | 111 | 5 | 1 [ 456 100 ] @fif_codegen """ @@ -214,4 +301,88 @@ fun main() { 255 THROW }> """ + +@fif_codegen +""" + demo9_1 PROC:<{ + 0 GTINT + IFJMP:<{ + t GETGLOB + 123 PUSHINT + TPUSH + t SETGLOB + }> + t GETGLOB + 456 PUSHINT + TPUSH + t SETGLOB + }> +""" + +@fif_codegen +""" + demo9_2 PROC:<{ + 0 GTINT + IFJMP:<{ + t GETGLOB + 123 PUSHINT + TPUSH + t SETGLOB + }> + }> +""" + +@fif_codegen +""" + demo9_3 PROC:<{ + 0 GTINT + IFJMP:<{ + t GETGLOB + 123 PUSHINT + TPUSH + t SETGLOB + }> + }> +""" + +@fif_codegen +""" + demo9_4 PROC:<{ + DUP + -100 NEQINT + IFJMP:<{ + 0 GTINT + IFJMP:<{ + t GETGLOB + 123 PUSHINT + TPUSH + t SETGLOB + }> + }> + DROP + }> +""" + +@fif_codegen +""" + demo10_1 PROC:<{ + DUP + 0 EQINT + IFJMP:<{ + DROP + t GETGLOB + 123 PUSHINT + TPUSH + t SETGLOB + }> + 1 EQINT + IFJMP:<{ + t GETGLOB + 456 PUSHINT + TPUSH + t SETGLOB + }> + }> +""" + */ diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 91891f8c1..1f5025918 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -54,6 +54,7 @@ std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr targ std::vector pre_compile_symbol(SrcLocation loc, const Symbol* sym, CodeBlob& code, LValContext* lval_ctx); void process_any_statement(AnyV v, CodeBlob& code); +static AnyV stmt_before_immediate_return = nullptr; // The goal of VarsModificationWatcher is to detect such cases: `return (x, x += y, x)`. // Without any changes, ops will be { _Call $2 = +($0_x, $1_y); _Return $0_x, $2, $0_x } - incorrect @@ -1236,6 +1237,9 @@ static std::vector process_match_expression(V v code.push_set_cur(if_op.block0); if (v->is_statement()) { pre_compile_expr(v_ith_arm->get_body(), code); + if (v == stmt_before_immediate_return) { + code.emplace_back(v_ith_arm->loc, Op::_Return); + } } else { std::vector arm_ir_idx = pre_compile_expr(v_ith_arm->get_body(), code, v->inferred_type); code.emplace_back(v->loc, Op::_Let, result_ir_idx, std::move(arm_ir_idx)); @@ -1249,6 +1253,9 @@ static std::vector process_match_expression(V v auto v_last_arm = v->get_arm(n_arms - 1); if (v->is_statement()) { pre_compile_expr(v_last_arm->get_body(), code); + if (v == stmt_before_immediate_return) { + code.emplace_back(v_last_arm->loc, Op::_Return); + } } else { std::vector arm_ir_idx = pre_compile_expr(v_last_arm->get_body(), code, v->inferred_type); code.emplace_back(v->loc, Op::_Let, result_ir_idx, std::move(arm_ir_idx)); @@ -1743,9 +1750,30 @@ std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr targ static void process_block_statement(V v, CodeBlob& code) { - for (AnyV item : v->get_items()) { - process_any_statement(item, code); + if (v->empty()) { + return; } + + FunctionPtr cur_f = code.fun_ref; + bool does_f_return_nothing = cur_f->inferred_return_type == TypeDataVoid::create() && !cur_f->does_return_self() && !cur_f->has_mutate_params(); + bool is_toplevel_block = v == cur_f->ast_root->as()->get_body(); + + // we want to optimize `match` and `if/else`: if it's the last statement, implicitly add "return" to every branch + // (to generate IFJMP instead of nested IF ELSE); + // a competent way is to do it at the IR level (building CST, etc.), it's impossible to tweak Ops for now; + // so, for every `f() { here }` of `... here; return;`, save it into a global, and handle within match/if + AnyV backup = stmt_before_immediate_return; + for (int i = 0; i < v->size() - 1; ++i) { + AnyV stmt = v->get_item(i); + AnyV next_stmt = v->get_item(i + 1); + bool next_is_empty_return = next_stmt->kind == ast_return_statement && !next_stmt->as()->has_return_value(); + stmt_before_immediate_return = next_is_empty_return && does_f_return_nothing ? stmt : nullptr; + process_any_statement(stmt, code); + } + AnyV last_stmt = v->get_item(v->size() - 1); + stmt_before_immediate_return = is_toplevel_block && does_f_return_nothing ? last_stmt : nullptr; + process_any_statement(last_stmt, code); + stmt_before_immediate_return = backup; } static void process_assert_statement(V v, CodeBlob& code) { @@ -1817,9 +1845,15 @@ static void process_if_statement(V v, CodeBlob& code) { Op& if_op = code.emplace_back(v->loc, Op::_If, std::move(cond)); code.push_set_cur(if_op.block0); process_any_statement(v->get_if_body(), code); + if (v == stmt_before_immediate_return) { + code.emplace_back(v->get_if_body()->loc_end, Op::_Return); + } code.close_pop_cur(v->get_if_body()->loc_end); code.push_set_cur(if_op.block1); process_any_statement(v->get_else_body(), code); + if (v == stmt_before_immediate_return) { + code.emplace_back(v->get_else_body()->loc_end, Op::_Return); + } code.close_pop_cur(v->get_else_body()->loc_end); if (v->is_ifnot) { std::swap(if_op.block0, if_op.block1); @@ -1978,9 +2012,7 @@ static void convert_function_body_to_CodeBlob(FunctionPtr fun_ref, FunctionBodyC blob->in_var_cnt = blob->var_cnt; tolk_assert(blob->var_cnt == total_arg_width); - for (AnyV item : v_body->get_items()) { - process_any_statement(item, *blob); - } + process_block_statement(v_body, *blob); append_implicit_return_statement(v_body->loc_end, *blob); blob->close_blk(v_body->loc_end); From 3feba11dfd47e04e9c5097b7f676ec39ebc6ac6c Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Tue, 10 Jun 2025 17:48:43 +0700 Subject: [PATCH 286/388] [Tolk] Universal createMessage and createExternalLogMessage Features: - extra currencies - stateInit (code+data) with automatic address computation - different workchains - sharding (formerly splitDepth) - auto-serialized body - auto-detect "body ref or not" - more efficient than handwritten code --- crypto/smartcont/tolk-stdlib/common.tolk | 373 +++++++-- tolk-tester/tests/cells-slices.tolk | 5 +- tolk-tester/tests/generics-2.tolk | 12 +- tolk-tester/tests/generics-3.tolk | 2 +- tolk-tester/tests/generics-4.tolk | 2 +- tolk-tester/tests/intN-tests.tolk | 2 +- .../tests/invalid-serialization/err-7114.tolk | 8 +- .../tests/invalid-serialization/err-7604.tolk | 21 + .../tests/invalid-serialization/err-7605.tolk | 19 + tolk-tester/tests/nullable-tensors.tolk | 42 +- tolk-tester/tests/pack-unpack-2.tolk | 17 +- tolk-tester/tests/pack-unpack-3.tolk | 2 +- tolk-tester/tests/pack-unpack-6.tolk | 32 +- tolk-tester/tests/parse-address.tolk | 94 ++- tolk-tester/tests/send-msg-1.tolk | 733 ++++++++++++++++++ tolk-tester/tests/send-msg-2.tolk | 337 ++++++++ tolk-tester/tests/send-msg-3.tolk | 233 ++++++ tolk-tester/tests/smart-cast-tests.tolk | 8 +- tolk-tester/tests/struct-tests.tolk | 26 +- tolk-tester/tests/union-types-tests.tolk | 38 +- tolk-tester/tests/var-apply-tests.tolk | 4 +- tolk-tester/tolk-tester.js | 13 +- tolk-tester/tolk-tester.py | 12 +- tolk/CMakeLists.txt | 1 + tolk/builtins.cpp | 33 + tolk/pack-unpack-api.cpp | 11 +- tolk/pack-unpack-serializers.cpp | 27 + tolk/pack-unpack-serializers.h | 1 + tolk/pipe-ast-to-legacy.cpp | 17 +- tolk/pipe-check-serialized-fields.cpp | 13 +- tolk/send-message-api.cpp | 496 ++++++++++++ tolk/send-message-api.h | 28 + 32 files changed, 2489 insertions(+), 173 deletions(-) create mode 100644 tolk-tester/tests/invalid-serialization/err-7604.tolk create mode 100644 tolk-tester/tests/invalid-serialization/err-7605.tolk create mode 100644 tolk-tester/tests/send-msg-1.tolk create mode 100644 tolk-tester/tests/send-msg-2.tolk create mode 100644 tolk-tester/tests/send-msg-3.tolk create mode 100644 tolk/send-message-api.cpp create mode 100644 tolk/send-message-api.h diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index 76305743a..e522bf2c0 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -364,6 +364,11 @@ fun Cell.load(self, options: UnpackOptions = {}): T fun Cell.beginParse(self): slice asm "CTOS"; +/// Returns hash of a typed cell, same as [cell.hash]. +@pure +fun Cell.hash(self): slice + asm "HASHCU"; + /// RemainingBitsAndRefs is a special built-in type to get "all the rest" slice tail on reading. /// Example: /// ``` @@ -681,6 +686,12 @@ fun slice.removeLastBits(mutate self, len: int): self fun slice.getLastBits(self, len: int): slice asm "SDCUTLAST"; +/// Returns `0 ≤ len ≤ 1023` bits of a slice starting from `0 ≤ offset ≤ 1023`. +/// (in other words, extracts a bit substring `[offset, len)` out of the slice) +@pure +fun slice.getMiddleBits(self, offset: int, len: int): slice + asm "SDSUBSTR"; + /// Loads a dictionary (TL HashMapE structure, represented as TVM cell) from a slice. /// Returns `null` if `nothing` constructor is used. @pure @@ -884,7 +895,7 @@ fun builder.bitsCount(self): int fun address(stdAddress: slice): address builtin; -/// Creates a slice representing TL addr_none$00 (two `0` bits). +/// Creates a slice representing "none address" (TL addr_none$00 — two zero bits). @pure fun createAddressNone(): address asm "b{00} PUSHSLICE"; @@ -965,62 +976,14 @@ fun reserveExtraCurrenciesOnBalance(nanoTonCoins: coins, extraAmount: dict, rese /** - Messages sending and parsing primitives. - Working with messages is low-level right now, but still, every contract should do that. - - `Message` structure, its header and so on are specified in TL-B scheme, particularly: - int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool ... = CommonMsgInfo; + Creating and sending messages. + Basic scenario: + ``` + val outMsg = createMessage({ ... options }); // you get OutMessage + outMsg.send(mode); + ``` */ -/// 0b011000 tag - 0, ihr_disabled - 1, bounce - 1, bounced - 0, src = adr_none$00 -const BOUNCEABLE = 0x18; -/// 0b010000 tag - 0, ihr_disabled - 1, bounce - 0, bounced - 0, src = adr_none$00 -const NON_BOUNCEABLE = 0x10; - -/// Load msgFlags from incoming message body (4 bits). -@pure -fun slice.loadMessageFlags(mutate self): int - asm( -> 1 0) "4 LDU"; - -/// Having msgFlags (4 bits), check that a message is bounced. -/// Effectively, it's `msgFlags & 1` (the lowest bit present). -@pure -fun isMessageBounced(msgFlags: int): bool - asm "2 PUSHINT" "MODR"; - -/// Skip 0xFFFFFFFF prefix (when a message is bounced). -@pure -fun slice.skipBouncedPrefix(mutate self): self - asm "32 PUSHINT" "SDSKIPFIRST"; - -/// The guideline recommends to start the body of an internal message with uint32 `op` and uint64 `queryId`. -@pure -fun slice.loadMessageOp(mutate self): int - asm( -> 1 0) "32 LDU"; - -@pure -fun slice.skipMessageOp(mutate self): self - asm "32 PUSHINT" "SDSKIPFIRST"; - -@pure -fun builder.storeMessageOp(mutate self, op: int): self - asm(op self) "32 STU"; - -/// The guideline recommends that uint64 `queryId` should follow uint32 `op`. -@pure -fun slice.loadMessageQueryId(mutate self): int - asm( -> 1 0) "64 LDU"; - -@pure -fun slice.skipMessageQueryId(mutate self): self - asm "64 PUSHINT" "SDSKIPFIRST"; - -@pure -fun builder.storeMessageQueryId(mutate self, queryId: int): self - asm(queryId self) "64 STU"; - -/// SEND MODES - https://docs.ton.org/tvm.pdf page 137, SENDRAWMSG - /// mode = 0 is used for ordinary messages; the gas fees are deducted from the senging amount; action phaes should NOT be ignored. const SEND_MODE_REGULAR = 0; /// +1 means that the sender wants to pay transfer fees separately. @@ -1041,13 +1004,301 @@ const SEND_MODE_ESTIMATE_FEE_ONLY = 1024; /// +64 substitutes the entire balance of the incoming message as an outcoming value (slightly inaccurate, gas expenses that cannot be estimated before the computation is completed are not taken into account). /// +128 substitutes the value of the entire balance of the contract before the start of the computation phase (slightly inaccurate, since gas expenses that cannot be estimated before the completion of the computation phase are not taken into account). +type ExtraCurrenciesDict = dict; + +/// ContractState is "code + data" of a contract. +/// Used in outgoing messages (StateInit) to initialize a destination contract. +struct ContractState { + code: cell; + data: cell; +} + +/// AddressShardingOptions provides settings to calculate an address in another shard. +/// Consider [createMessage] and [address.buildSameAddressInAnotherShard] for usage. +struct AddressShardingOptions { + fixedPrefixLength: uint5; // shard depth, formerly splitDepth + closeTo: address; +} + +/// AutoDeployAddress is a destination that initializes a receiver contract if it does not exist yet. +/// In order to do this, it contains StateInit — and calculates an address by a hash of StateInit. +/// Example: +/// ``` +/// createMessage({ +/// dest: { +/// workchain: 0, +/// stateInit: { +/// code: jettonWalletCode, +/// data: jettonWalletEmptyStorage.toCell() +/// } +/// }, +/// ... +/// ``` +/// It's often called "deployment", but it's inaccurate. TON Blockchain doesn't have a dedicated +/// deployment mechanism. You just send a message to some address — and if it doesn't exist, but you've +/// attached a way to initialize it (code+data) — it's initialized immediately, and accepts your message. +/// You just provide code+data, and the compiler automatically calculates the destination address, +/// because in TON, the address of a contract, by definition, is a hash of its initial state. +struct AutoDeployAddress { + workchain: int8 = BASECHAIN; + stateInit: ContractState | cell; + toShard: AddressShardingOptions? = null; +} + +/// Options for creating an outgoing message. +/// Consider [createMessage] for examples. +struct CreateMessageOptions { + /// whether a message will bounce back on error + bounce: bool; + /// message value: attached tons (or tons + extra currencies) + value: coins | (coins, ExtraCurrenciesDict); + /// destination is either a provided address, or is auto-calculated by stateInit + dest: address | // either just send a message to some address + builder | // ... or a manually constructed builder with a valid address + (int8, uint256) | // ... or to workchain + hash (also known as accountID) + AutoDeployAddress; // ... or "send to stateInit" aka deploy (address auto-calculated) + /// body is any serializable object (or just miss this field for empty body) + body: TBody; +} + +/// Creates a message (`OutMessage`) — a well-formatted message cell. +/// Typically, you just send it. In advanced scenarios, you can estimate fees or even postpone sending. +/// Example: +/// ``` +/// val reply = createMessage({ +/// bounce: false, +/// value: ton("0.05"), +/// dest: senderAddress, +/// body: RequestedInfo { ... } // note: no `toCell`! just pass an object +/// }); +/// reply.send(SEND_MODE_REGULAR); +/// ``` +/// Hint: don't call `body.toCell()`, pass `body` directly! +/// (if body is small, it will be inlined without an expensive cell creation) +/// (if body is large, the compiler will automatically wrap it into a cell) +/// If you need an empty body, just miss the field `body`, fill other 3 fields. +@pure +fun createMessage(options: CreateMessageOptions): OutMessage + builtin; + + +/// ExtOutLogBucket is a variant of a custom external address for emitting logs "to the outer world". +/// It includes some "topic" (arbitrary number), that determines the format of the message body. +/// For example, you emit "deposit event" (reserving topic "deposit" = 123): +/// > `dest: ExtOutLogBucket { topic: 123 }, body: DepositData { ... }` +/// and external indexers can index your emitted logs by destination address without parsing body. +/// Currently, external messages are used only for emitting logs (for viewing them in indexers). +/// In the future, there might be +/// > 0x01 ExtOutOffchainContract { adnl: uint256 | bits256 } +/// Serialization details: '01' (addr_extern) + 256 (len) + 0x00 (prefix) + 248 bits = 267 in total +struct (0x00) ExtOutLogBucket { + topic: uint248 | bits248; +} + +/// Options for creating an external outgoing message. +/// Consider [createExternalLogMessage] for examples. +struct createExternalLogMessageOptions { + /// destination is either an external address or a pattern to calculate it + dest: address | // either some valid external/none address (not internal!) + builder | // ... or a manually constructed builder with a valid external address + ExtOutLogBucket; // ... or encode topic/eventID in destination + /// body is any serializable object (or just miss this field for empty body) + body: TBody; +} + +/// Creates an external message (`OutMessage`) — a well-formatted message cell. +/// Typically, you just send it. In advanced scenarios, you can estimate fees or even postpone sending. +/// Example: +/// ``` +/// val emitMsg = createExternalLogMessage({ +/// dest: createAddressNone(), +/// body: DepositEvent { ... } // note: no `toCell`! just pass an object +/// }); +/// emitMsg.send(SEND_MODE_REGULAR); +/// ``` +/// Note: `createMessage` also returns `OutMessage`, it's okay: a composed message cell is universal. +/// Hint: don't call `body.toCell()`, pass `body` directly! +/// (if body is small, it will be inlined without an expensive cell creation) +/// (if body is large, the compiler will automatically wrap it into a cell) +@pure +fun createExternalLogMessage(options: createExternalLogMessageOptions): OutMessage + builtin; + +/// UnsafeBodyNoRef is used to prevent default behavior: when message body is potentially large, +/// it's packed into a separate ref. Wrapping body with this struct tells the compiler: +/// "inline message body in-place, I guarantee it will fit". +/// Example: +/// ``` +/// struct ProbablyLarge { a: (coins, coins, coins, coins, coins) } // max 620 bits +/// +/// val contents: ProbablyLarge = { ... }; // you are sure: values are small +/// createMessage({ +/// // body: contents, // by default, it will be stored a ref (it's large) +/// body: UnsafeBodyNoRef { +/// bodyForceNoRef: contents, // but wrapping forces the compiler to inline it +/// } +/// ``` +/// Another example: your body contains `builder` or `RemainingBitsAndRefs`: unpredictable size. +/// If you guarantee it's small and refs won't clash with code/data, avoid creating a cell. +struct UnsafeBodyNoRef { + bodyForceNoRef: T; +} + +/// OutMessage is a result of [createMessage]. +/// Essentially, it's a composed message cell, ready to be sent. +struct OutMessage { + messageCell: cell; +} + +/// Sends a ready message cell. +/// For `sendMode`, see the constants above (SEND_MODE_*). +fun OutMessage.send(self, sendMode: int): void + asm "SENDRAWMSG"; + +fun OutMessage.sendAndEstimateFee(self, sendMode: int): coins + asm "SENDMSG"; + +@pure +fun OutMessage.estimateFeeWithoutSending(self, sendMode: int): coins + asm "10 PUSHPOW2" "OR" "SENDMSG"; + +@pure +fun OutMessage.hash(self): uint256 + asm "HASHCU"; + + +/// StateInit is a "canonical TL/B representation from block.tlb" of a contract initial state. +/// But for everyday tasks, it's too complicated. It is not used in practice. +/// To represent code+data, consider [ContractState]. +/// To represent sharding (fixedPrefixLength, formerly splitDepth), consider [AutoDeployAddress]. +struct StateInit { + fixedPrefixLength: uint5?; + special: (bool, bool)?; + code: cell?; + data: cell?; + library: cell?; +} + +/// Calculates a hash of StateInit if only code+data are set. +@pure +fun StateInit.calcHashCodeData(code: cell, data: cell): uint256 asm +""" // code data + DUP2 // code data code data + HASHCU + SWAP + HASHCU // code data dataHash codeHash + SWAP2 // dataHash codeHash code data + CDEPTH + SWAP + CDEPTH // dataHash codeHash dataDepth codeDepth + + NEWC + x{020134} STSLICECONST // store refs_descriptor | bits_descriptor | data + 16 STU // store codeDepth + 16 STU // store dataDepth + 256 STU // store codeHash + 256 STU // store dataHash + + ONE HASHEXT_SHA256 +"""; + +/// Calculates a hash of StateInit if fixedPrefixLength+code+data are set. +@pure +fun StateInit.calcHashPrefixCodeData(fixedPrefixLength: uint5, code: cell, data: cell): uint256 asm +""" // pDepth code data + DUP2 // pDepth code data code data + HASHCU + SWAP + HASHCU // pDepth code data dataHash codeHash + SWAP2 // pDepth dataHash codeHash code data + CDEPTH + SWAP + CDEPTH // pDepth dataHash codeHash dataDepth codeDepth + + s4 PUSH // pDepth dataHash codeHash dataDepth codeDepth pDepth + 10 LSHIFT# + 0x020381A0 PUSHINT + ADD // calc refs_descriptor | bits_descriptor | data + + NEWC + 32 STU + 16 STU + 16 STU + 256 STU + 256 STU + + ONE HASHEXT_SHA256 // pDepth hash + NIP +"""; + + +/// Given an internal address A="aaaa...a" returns "bbaa...a" (D bits from address B, 256-D from A). +/// Example for fixedPrefixLength (shard depth) = 8: +/// | self (A) | aaaaaaaaaaa...aaa | +/// | closeTo (B) | 01010101bbb...bbb | shardPrefix = 01010101 (depth 8) +/// | result | 01010101aaa...aaa | address of A in same shard as B +/// More precisely, self (input) is 267 bits: '100' (std addr no anycast) + workchainA + "aaaa...a". +/// The result is also 267 bits: '100' + workchainB + "bb" (D bits) + "aa...a" (256-D bits). +/// Note: returns `builder`, not `address`! Because builder is cheap, and you can send a message to it: +/// ``` +/// createMessage({ +/// dest: resBuilder, // a builder containing a valid address can be passed to createMessage() +/// ... +/// ``` +/// If you really need `address`, use `address.fromValidBuilder(resBuilder)`. +@pure +fun address.buildSameAddressInAnotherShard(self, options: AddressShardingOptions): builder + builtin; + +/// Converts a builder containing a valid address (internal/external/none) to `address` (slice under the hood). +/// Gas-expensive! Because or a cell creation: essentially, it's `b.endCell().beginParse()`. +/// Example: you've manually built a 267-bit address: '100' (std addr no anycast) + workchain + hash +/// and you want to return `address` (TVM slice) from a contract getter: +/// ``` +/// get calcWalletAddress(ownerAddress: address): address { +/// val b = calculate(...); // manually calculate into builder +/// return address.fromValidBuilder(b) // but really need address +/// } +/// ``` +/// Hint: if you want just to send a message, don't call this function, use a builder directly: +/// ``` +/// createMessage({ +/// dest: someBuilder, // a builder containing a valid address can be passed to createMessage() +/// ... // so, don't call this expensive function in this case +/// ``` +@pure +fun address.fromValidBuilder(b: builder): address + asm "ENDC" "CTOS"; + + /// Sends a raw message — a correctly serialized TL object `Message X`. -/// For `mode`, see constants above (except SEND_MODE_ESTIMATE_FEE_ONLY). +/// In practice, you'll use a high-level wrapper [createMessage]. fun sendRawMessage(msg: cell, mode: int): void asm "SENDRAWMSG"; -/// Creates an output action and returns a fee for creating a message. -/// Mode has the same effect as in the case of SENDRAWMSG. -/// For mode including SEND_MODE_ESTIMATE_FEE_ONLY it just returns estimated fee without sending a message. -fun sendMessage(msg: cell, mode: int): int - asm "SENDMSG"; + + +/** + Receiving and handling messages. +*/ + +/// Load msgFlags from incoming message body (4 bits). +@pure +fun slice.loadMessageFlags(mutate self): int + asm( -> 1 0) "4 LDU"; + +/// Skip 0xFFFFFFFF prefix (when a message is bounced). +@pure +fun slice.skipBouncedPrefix(mutate self): self + asm "32 PUSHINT" "SDSKIPFIRST"; + +/// The guideline recommends to start the body of an internal message with uint32 `op` and uint64 `queryId`. +@pure +fun slice.loadMessageOp(mutate self): int + asm( -> 1 0) "32 LDU"; + +/// The guideline recommends that uint64 `queryId` should follow uint32 `op`. +@pure +fun slice.loadMessageQueryId(mutate self): int + asm( -> 1 0) "64 LDU"; + diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk index 6a14578fd..2cbb78647 100644 --- a/tolk-tester/tests/cells-slices.tolk +++ b/tolk-tester/tests/cells-slices.tolk @@ -1,6 +1,10 @@ fun builder.store_u32(mutate self, value: int): self { return self.storeUint(value, 32); } +fun builder.storeMessageOp(mutate self, op: int): self + asm(op self) "32 STU"; +fun builder.storeMessageQueryId(mutate self, queryId: int): self + asm(queryId self) "64 STU"; fun slice.load_u32(mutate self): int { return self.loadUint(32); @@ -195,7 +199,6 @@ fun test111() { var q2 = s.loadMessageQueryId(); s.skipBits(64); s.assertEnd(); - assert(isMessageBounced(0x001) && !isMessageBounced(0x002)) throw 444; return (op1, q1, op2, q2); } diff --git a/tolk-tester/tests/generics-2.tolk b/tolk-tester/tests/generics-2.tolk index 8e95d884b..2ee2851ae 100644 --- a/tolk-tester/tests/generics-2.tolk +++ b/tolk-tester/tests/generics-2.tolk @@ -451,15 +451,15 @@ fun main(c: Wrapper, d: WrappedInt) { @testcase | 104 | | 11 11 13 13 13 2 @testcase | 105 | 0 | 80 @testcase | 105 | -1 | (null) -@testcase | 106 | | (null) (null) 0 777 1 2 142 +@testcase | 106 | | (null) (null) 0 777 1 2 typeid-14 @testcase | 107 | 5 | 20 20 5 5 20 777 (null) (null) (null) 0 -@testcase | 108 | 0 | 777 143 777 143 +@testcase | 108 | 0 | 777 typeid-15 777 typeid-15 @testcase | 108 | -1 | 777 0 777 0 @testcase | 109 | | 40 40 70 @testcase | 110 | | 20 1 20 42 -@testcase | 111 | | 5 1 132 5 1 132 5 1 132 5 1 132 -@testcase | 112 | -1 | 10 1 777 10 1 133 -@testcase | 112 | 0 | 20 1 777 (null) 0 134 +@testcase | 111 | | 5 1 typeid-4 5 1 typeid-4 5 1 typeid-4 5 1 typeid-4 +@testcase | 112 | -1 | 10 1 777 10 1 typeid-5 +@testcase | 112 | 0 | 20 1 777 (null) 0 typeid-6 @testcase | 113 | | 30 -1 @testcase | 114 | | 999 (null) 2 @testcase | 115 | | 10 0 200 1 -1 200 1 -1 @@ -467,7 +467,7 @@ fun main(c: Wrapper, d: WrappedInt) { @testcase | 117 | | 100 123 @testcase | 118 | | 123 1 777 123 1 -1 2 @testcase | 119 | | 40 40 -1 -1 -1 -1 -@testcase | 120 | | (null) 137 777 (null) (null) 0 139 -1 -1 +@testcase | 120 | | (null) typeid-9 777 (null) (null) 0 typeid-11 -1 -1 @testcase | 123 | | 10 10 10 10 10 diff --git a/tolk-tester/tests/generics-3.tolk b/tolk-tester/tests/generics-3.tolk index 4539d6144..a6fc74885 100644 --- a/tolk-tester/tests/generics-3.tolk +++ b/tolk-tester/tests/generics-3.tolk @@ -130,7 +130,7 @@ fun main() { @testcase | 101 | -1 | 10 @testcase | 101 | 0 | -1 @testcase | 103 | | 10 -1 0 -@testcase | 104 | | 12 129 132 -1 -1 -1 0 0 -1 0 0 +@testcase | 104 | | 12 typeid-1 typeid-4 -1 -1 -1 0 0 -1 0 0 @testcase | 105 | | -1 -1 -1 -1 0 0 @testcase | 108 | | 50000000 (null) (null) 100000000 12 12 24 diff --git a/tolk-tester/tests/generics-4.tolk b/tolk-tester/tests/generics-4.tolk index cfe5714ff..70c70818d 100644 --- a/tolk-tester/tests/generics-4.tolk +++ b/tolk-tester/tests/generics-4.tolk @@ -172,7 +172,7 @@ fun main() { /** @testcase | 103 | | 10 20 30 777 40 40 @testcase | 104 | | (null) (null) (null) -@testcase | 105 | | -1 (null) 0 (null) 134 777 0 123 0 456 133 +@testcase | 105 | | -1 (null) 0 (null) typeid-6 777 0 123 0 456 typeid-5 @testcase | 106 | | -1 0 -1 -1 @testcase | 107 | | 1 10 110 117 @testcase | 108 | | 10 (null) 20 diff --git a/tolk-tester/tests/intN-tests.tolk b/tolk-tester/tests/intN-tests.tolk index 6f44ed030..5b5d8563f 100644 --- a/tolk-tester/tests/intN-tests.tolk +++ b/tolk-tester/tests/intN-tests.tolk @@ -246,7 +246,7 @@ fun main() { @testcase | 110 | | 50000000 50000100 1234000000 51000000 @testcase | 111 | | [ 1000000000 1000000000 1000000000 -321123456789 321123456789 1100000000 ] @testcase | 114 | 5 | (null) (null) 0 -@testcase | 114 | 15 | 15 2 130 +@testcase | 114 | 15 | 15 2 typeid-2 @fif_codegen DECLPROC assign0 @fif_codegen DECLPROC assign0 diff --git a/tolk-tester/tests/invalid-serialization/err-7114.tolk b/tolk-tester/tests/invalid-serialization/err-7114.tolk index fe228d601..2507e81f2 100644 --- a/tolk-tester/tests/invalid-serialization/err-7114.tolk +++ b/tolk-tester/tests/invalid-serialization/err-7114.tolk @@ -6,17 +6,17 @@ struct Demo { c: NotSerializableTensor; } -fun main(p: Demo?) { - p.toCell(); +fun main() { + Demo.fromSlice(""); } /** @compilation_should_fail -@stderr auto-serialization via toCell() is not available for type `Demo?` +@stderr auto-serialization via fromSlice() is not available for type `Demo` @stderr because field `Demo.c` of type `NotSerializableTensor` can't be serialized @stderr because alias `NotSerializableTensor` expands to `(int8, slice)` @stderr because element `tensor.1` of type `slice` can't be serialized -@stderr because type `slice` is not serializable, it doesn't define binary width +@stderr because type `slice` can not be used for reading, it doesn't define binary width @stderr hint: replace `slice` with `address` if it's an address, actually @stderr hint: replace `slice` with `bits128` and similar if it represents fixed-width data without refs */ diff --git a/tolk-tester/tests/invalid-serialization/err-7604.tolk b/tolk-tester/tests/invalid-serialization/err-7604.tolk new file mode 100644 index 000000000..ea6edfe7a --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7604.tolk @@ -0,0 +1,21 @@ +struct CCC { + v: int; +} + +fun main() { + var options = CreateMessageOptions { + bounce: true, + dest: createAddressNone(), + value: 0, + body: CCC { v: 10 } + }; + createMessage(options); +} + +/** +@compilation_should_fail +@stderr error: auto-serialization via createMessage() is not available for type `CCC` +@stderr because field `CCC.v` of type `int` can't be serialized +@stderr because type `int` is not serializable, it doesn't define binary width +@stderr hint: replace `int` with `int32` / `uint64` / `coins` / etc. + */ diff --git a/tolk-tester/tests/invalid-serialization/err-7605.tolk b/tolk-tester/tests/invalid-serialization/err-7605.tolk new file mode 100644 index 000000000..087e56457 --- /dev/null +++ b/tolk-tester/tests/invalid-serialization/err-7605.tolk @@ -0,0 +1,19 @@ +struct BigBodyMessage { + v1: int256; + v2: int256; + v3: int256; + a1: address; + a2: address; +} + +fun testFail(body: BigBodyMessage) { + createExternalLogMessage({ + dest: createAddressNone(), + body, + }); +} + +/** +@compilation_should_fail +@stderr struct `BigBodyMessage` can exceed 1023 bits in serialization (estimated size: 772..1302 bits) + */ diff --git a/tolk-tester/tests/nullable-tensors.tolk b/tolk-tester/tests/nullable-tensors.tolk index 4811c4144..d6934324e 100644 --- a/tolk-tester/tests/nullable-tensors.tolk +++ b/tolk-tester/tests/nullable-tensors.tolk @@ -494,59 +494,59 @@ fun test143(setBNull: bool, setANullMid: bool) { fun main(){} /** -@testcase | 101 | | 1 2 129 -@testcase | 102 | | 1 2 129 (null) (null) 0 +@testcase | 101 | | 1 2 typeid-1 +@testcase | 102 | | 1 2 typeid-1 (null) (null) 0 @testcase | 103 | 1 2 | 3 3 0 1 2 @testcase | 104 | | 1 2 (null) (null) 0 -@testcase | 105 | | (null) (null) (null) 0 1 2 3 131 +@testcase | 105 | | (null) (null) (null) 0 1 2 3 typeid-3 @testcase | 106 | | 1 2 @testcase | 107 | | 0 0 -1 0 0 -1 -@testcase | 108 | 5 6 | 7 8 10 11 129 (null) (null) 0 +@testcase | 108 | 5 6 | 7 8 10 11 typeid-1 (null) (null) 0 @testcase | 109 | | 0 0 -1 0 -1 0 0 -1 -1 -@testcase | 110 | | 3 4 (null) (null) 0 6 7 129 +@testcase | 110 | | 3 4 (null) (null) 0 6 7 typeid-1 @testcase | 111 | | 50 30 70 90 100 @testcase | 112 | | 12 22 @testcase | 113 | | -1 @testcase | 114 | | (null) (null) (null) 0 (null) (null) (null) 0 -@testcase | 115 | | 2 3 7 (null) (null) 0 5 0 129 0 +@testcase | 115 | | 2 3 7 (null) (null) 0 5 0 typeid-1 0 @testcase | 116 | -1 | (null) (null) 0 (null) (null) 0 -@testcase | 116 | 0 | 1 2 129 1 2 129 +@testcase | 116 | 0 | 1 2 typeid-1 1 2 typeid-1 @testcase | 117 | | (null) 1 3 -@testcase | 118 | 5 | 5 10 129 +@testcase | 118 | 5 | 5 10 typeid-1 @testcase | 118 | null | (null) (null) 0 -@testcase | 119 | | (null) (null) 1 2 129 100 +@testcase | 119 | | (null) (null) 1 2 typeid-1 100 @testcase | 120 | -1 | (null) (null) 0 -@testcase | 120 | 0 | 1 2 129 +@testcase | 120 | 0 | 1 2 typeid-1 @testcase | 121 | | [ 1 [ 3 4 ] ] @testcase | 122 | 0 | [ 1 [ 3 4 ] 4 (null) ] @testcase | 122 | -1 | [ 1 (null) 4 (null) ] -@testcase | 123 | | 1 3 4 132 -@testcase | 124 | 0 | 1 3 4 132 4 (null) (null) 0 +@testcase | 123 | | 1 3 4 typeid-4 +@testcase | 124 | 0 | 1 3 4 typeid-4 4 (null) (null) 0 @testcase | 124 | -1 | 1 (null) (null) 0 4 (null) (null) 0 @testcase | 125 | | 3 @testcase | 126 | | 1 (null) 2 @testcase | 127 | 1 | 1 (null) (null) 0 2 -@testcase | 127 | 2 | 1 2 3 129 4 +@testcase | 127 | 2 | 1 2 3 typeid-1 4 @testcase | 127 | 3 | 1 (null) (null) 0 5 -@testcase | 128 | 1 | 1 (null) (null) 0 2 139 +@testcase | 128 | 1 | 1 (null) (null) 0 2 typeid-11 @testcase | 128 | 2 | (null) (null) (null) (null) (null) 0 -@testcase | 128 | 3 | 1 2 3 129 4 139 +@testcase | 128 | 3 | 1 2 3 typeid-1 4 typeid-11 @testcase | 129 | 0 | 5 5 0 -1 1 2 0 -1 @testcase | 129 | -1 | 5 5 0 -1 (null) (null) 0 -1 -@testcase | 130 | 0 | 1 2 3 129 +@testcase | 130 | 0 | 1 2 3 typeid-1 @testcase | 130 | -1 | 1 (null) (null) 0 -@testcase | 131 | | 140 777 0 777 777 777 0 0 140 140 777 140 140 141 777 +@testcase | 131 | | typeid-12 777 0 777 777 777 0 0 typeid-12 typeid-12 777 typeid-12 typeid-12 typeid-13 777 @testcase | 132 | | -1 0 -1 0 777 (null) (null) -1 0 0 @testcase | 133 | | 60 -@testcase | 134 | | 11 21 129 -@testcase | 135 | | [ 10 ] [ (null) ] (null) 777 10 143 (null) 143 (null) 0 777 10 144 (null) 144 (null) 0 777 0 0 -1 0 0 -1 0 0 -1 777 0 -1 0 0 -1 0 +@testcase | 134 | | 11 21 typeid-1 +@testcase | 135 | | [ 10 ] [ (null) ] (null) 777 10 typeid-15 (null) typeid-15 (null) 0 777 10 typeid-16 (null) typeid-16 (null) 0 777 0 0 -1 0 0 -1 0 0 -1 777 0 -1 0 0 -1 0 @testcase | 136 | 9 | 9 0 @testcase | 136 | null | (null) -1 -@testcase | 140 | 8 9 | 8 9 145 (null) (null) 0 +@testcase | 140 | 8 9 | 8 9 typeid-17 (null) (null) 0 @testcase | 141 | | (null) 10 @testcase | 142 | | 3 3 1 2 @testcase | 143 | -1 0 | 1 2 1 3 777 (null) (null) (null) (null) 0 -@testcase | 143 | 0 -1 | 1 (null) 0 3 777 1 2 1 3 146 +@testcase | 143 | 0 -1 | 1 (null) 0 3 777 1 2 1 3 typeid-18 @fif_codegen """ diff --git a/tolk-tester/tests/pack-unpack-2.tolk b/tolk-tester/tests/pack-unpack-2.tolk index 75f20a5fd..2a17f7ca1 100644 --- a/tolk-tester/tests/pack-unpack-2.tolk +++ b/tolk-tester/tests/pack-unpack-2.tolk @@ -343,6 +343,11 @@ struct WriteWithBuilder { rest: builder; } +struct WriteWithSlice { + f1: int32; + rest: slice; +} + struct ReadWrittenWithBuilder { f1: int32; someInt: uint32; @@ -607,6 +612,8 @@ fun test_DifferentMix3() { fun test_WriteWithBuilderReadWithOther() { var b = beginCell().storeUint(55, 32).storeMaybeRef(null); var w: WriteWithBuilder = { f1: 10, rest: b }; + var w2: WriteWithSlice = { f1: 10, rest: stringHexToSlice("FFFF") }; + assert(w2.toCell().beginParse().skipBits(32).loadUint(16) == 0xFFFF, 100); return ReadWrittenWithBuilder.fromCell(w.toCell()); } @@ -664,18 +671,18 @@ fun main() { @testcase | 208 | | -1 @testcase | 209 | | -1 @testcase | 210 | | 123 555 46 (null) -@testcase | 211 | | (null) 132 123 -@testcase | 212 | | (null) 133 123 +@testcase | 211 | | (null) typeid-4 123 +@testcase | 212 | | (null) typeid-5 123 @testcase | 213 | | -1 @testcase | 214 | | -1 @testcase | 215 | | -1 @testcase | 216 | | -1 @testcase | 217 | | -1 @testcase | 218 | | -1 -@testcase | 219 | | 44 (null) (null) 46 143 -@testcase | 220 | | 0 0 (null) 99 136 137 99 1234 (null) 889129 14 +@testcase | 219 | | 44 (null) (null) 46 typeid-15 +@testcase | 220 | | 0 0 (null) 99 typeid-8 typeid-9 99 1234 (null) 889129 14 @testcase | 221 | | -1 -@testcase | 222 | | 510 567 9392843922 146 81923 81923 147 777 0 -1 (null) (null) 0 100000 100000 147 +@testcase | 222 | | 510 567 9392843922 typeid-18 81923 81923 typeid-19 777 0 -1 (null) (null) 0 100000 100000 typeid-19 @testcase | 223 | | 10 55 (null) @testcase | 224 | | 60 50000000 5 9 123 -1 @testcase | 225 | | 5 40 65535 8 50000000 diff --git a/tolk-tester/tests/pack-unpack-3.tolk b/tolk-tester/tests/pack-unpack-3.tolk index 74fa1b8e9..2e7d6ab2b 100644 --- a/tolk-tester/tests/pack-unpack-3.tolk +++ b/tolk-tester/tests/pack-unpack-3.tolk @@ -278,7 +278,7 @@ fun main() {} /** @testcase | 201 | | 80 800000000 @testcase | 202 | | 800000000 17 777 -1 0 -@testcase | 203 | | (null) 0 131 777 -1 0 +@testcase | 203 | | (null) 0 typeid-3 777 -1 0 @testcase | 204 | | -1 0 -1 @testcase | 205 | | 0 -1 0 -1 */ diff --git a/tolk-tester/tests/pack-unpack-6.tolk b/tolk-tester/tests/pack-unpack-6.tolk index 049109ad7..07ec41f85 100644 --- a/tolk-tester/tests/pack-unpack-6.tolk +++ b/tolk-tester/tests/pack-unpack-6.tolk @@ -147,14 +147,14 @@ RETALT """ IF:<{ DROP - 129 PUSHINT + 139 PUSHINT }>ELSE<{ x{03} SDBEGINSQ NIP IFNOTJMP:<{ 104 THROW }> - 130 PUSHINT + 140 PUSHINT }> """ @@ -199,24 +199,24 @@ IF:<{ @fif_codegen """ - test9 PROC:<{ // - x{010f} PUSHSLICE // s - x{01} SDBEGINSQ // s '6 - IF:<{ // s - DROP // - 129 PUSHINT // 'UTag=129 - }>ELSE<{ // s - x{03} SDBEGINSQ // s '6 - NIP // '6 - IFNOTJMP:<{ // + test9 PROC:<{ + x{010f} PUSHSLICE + x{01} SDBEGINSQ + IF:<{ + DROP + 139 PUSHINT + }>ELSE<{ + x{03} SDBEGINSQ + NIP + IFNOTJMP:<{ 16 PUSHPOW2DEC THROWANY }> - 130 PUSHINT // 'UTag=130 + 140 PUSHINT }> - 129 PUSHINT // 'UTag '17=129 - SWAP // '17=129 'UTag - EQUAL // '16 + 139 PUSHINT + SWAP + EQUAL }> """ diff --git a/tolk-tester/tests/parse-address.tolk b/tolk-tester/tests/parse-address.tolk index 3f1151f7a..0f8ad02d2 100644 --- a/tolk-tester/tests/parse-address.tolk +++ b/tolk-tester/tests/parse-address.tolk @@ -35,6 +35,91 @@ fun codegenAddrEq(a: address, b: address) { return 3; } +@inline_ref +fun buildAddrInShard_manual(a: address, options: AddressShardingOptions) { + var sb = options.closeTo as slice; + sb.skipBits(3); // addr_std$10 + anycast 0 + val wc_b = sb.loadInt(8); + val shardPrefix = sb.loadUint(options.fixedPrefixLength); + + var sa = a as slice; + sa.skipBits(3 + 8 + options.fixedPrefixLength); + + return beginCell() + .storeUint(0b100, 3) // addr_std$10 + anycast 0 + .storeInt(wc_b, 8) + .storeUint(shardPrefix, options.fixedPrefixLength) + .storeSlice(sa); +} + +@method_id(101) +fun test1() { + val a = address("0:00000000000000000000000000000000000000000000000000000000000000FF"); + val b = address("1:1100000000000000000000000000000000000000000000000000000000000000"); + val dd1 = buildAddrInShard_manual(a, {fixedPrefixLength: 8, closeTo: b}); + val dd2 = a.buildSameAddressInAnotherShard({fixedPrefixLength: 8, closeTo: b}); + assert(dd1.endCell().hash() == dd2.endCell().hash(), 400); + return dd1.endCell().beginParse() as address + == address("1:11000000000000000000000000000000000000000000000000000000000000FF"); +} + +@method_id(102) +fun test2() { + val a = address("0:1234aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + val b = address("0:FFFFbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); + val dd1 = buildAddrInShard_manual(a, {closeTo: b, fixedPrefixLength: 16}); + val dd2 = a.buildSameAddressInAnotherShard({fixedPrefixLength: 16, closeTo: b}); + assert((a as slice).remainingBitsCount() == 267, 267); + assert((b as slice).remainingBitsCount() == 267, 267); + assert(dd1.endCell().hash() == dd2.endCell().hash(), 400); + return address.fromValidBuilder(dd2) + == address("0:FFFFaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); +} + +@method_id(103) +fun test3() { + var t1_a = address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"); + var t1_b = address("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"); + var t2_a = address("EQCe4AYIBce1pAk2qJJPSs1OzyZRlKjkfq8zuC8D7erv6DUP"); + var t2_b = address("EQCtrtTXEAoSpoERmiqOnICe9LHxn2N89N4BH9qdHlrG-U0i"); + var t3_a = address("EQAUTbQiM522Y_XJ_T98QPhPhTmb4nV--VSPiha8kC6kRfPO"); + var t3_b = address("EQBlqsm144Dq6SjbPI4jjZvA1hqTIP3CvHovbIfW_t-SCALE"); + return ( + t1_a.buildSameAddressInAnotherShard({fixedPrefixLength: 8, closeTo: t1_b}).endCell().hash() & 0xFFFF, + t1_a.buildSameAddressInAnotherShard({fixedPrefixLength: 4, closeTo: t1_b}).endCell().hash() & 0xFFFF, + t2_a.buildSameAddressInAnotherShard({fixedPrefixLength: 2, closeTo: t2_b}).endCell().hash() & 0xFFFF, + t3_a.buildSameAddressInAnotherShard({fixedPrefixLength: 30, closeTo: t2_b}).endCell().hash() & 0xFFFF, + t2_a.buildSameAddressInAnotherShard({fixedPrefixLength: 1, closeTo: t3_b}).endCell().hash() & 0xFFFF, + t2_a.buildSameAddressInAnotherShard({fixedPrefixLength: 0, closeTo: t3_b}).endCell().hash() & 0xFFFF, + ) +} + +@method_id(104) +fun test4(shardDepth: int) { + var a = address("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"); + var b = address("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"); + return a.buildSameAddressInAnotherShard({fixedPrefixLength: shardDepth, closeTo: b}).endCell().hash() & 0xFFFF; +} + +@method_id(105) +fun test5() { + var b = beginCell() + .storeUint(0b100, 3) // std addr no anycast + .storeInt(MASTERCHAIN, 8) + .storeUint(0xFFFF, 256); + val a = address("-1:000000000000000000000000000000000000000000000000000000000000FFFF"); + return address.fromValidBuilder(b) == a; +} + +@method_id(106) +fun test6() { + val a = address("0:00000000000000000000000000000000000000000000000000000000000000FF"); + val b = address("1:1100000000000000000000000000000000000000000000000000000000000000"); + return address.fromValidBuilder(a.buildSameAddressInAnotherShard({fixedPrefixLength: 8, closeTo: b})) == + address("1:11000000000000000000000000000000000000000000000000000000000000FF"); +} + + fun main() { __expect_type(cc1, "address"); @@ -144,7 +229,14 @@ fun main() { } /** -@testcase | 0 | | -1 0 -1 0 -1 +@testcase | 0 | | -1 0 -1 0 -1 +@testcase | 101 | | -1 +@testcase | 102 | | -1 +@testcase | 103 | | 39876 24338 15241 50719 11252 15241 +@testcase | 104 | 2 | 24338 +@testcase | 104 | 9 | 39876 +@testcase | 105 | | -1 +@testcase | 106 | | -1 @fif_codegen """ diff --git a/tolk-tester/tests/send-msg-1.tolk b/tolk-tester/tests/send-msg-1.tolk new file mode 100644 index 000000000..07f736e20 --- /dev/null +++ b/tolk-tester/tests/send-msg-1.tolk @@ -0,0 +1,733 @@ +import "@stdlib/tvm-dicts" +import "@stdlib/tvm-lowlevel" + +/* +int_msg_info$0 1 + ihr_disabled:Bool // always 0, not implemented 1 + bounce:Bool // parameter 1 + bounced:Bool // always 0 on send 1 + src:MsgAddress // always 00 on send 2 + dest:MsgAddressInt // parameter 267 + value:CurrencyCollection // parameter 124 + 1 + ihr_fee:Grams // always 0, not implemented 4 + fwd_fee:Grams // always 0 on send 4 + created_lt:uint64 // always 0 on send 64 + created_at:uint32 // always 0 on send 32 + = CommonMsgInfoRelaxed; +_ split_depth:(Maybe (## 5)) 1 + 5 + special:(Maybe TickTock) 1 + 2 + code:(Maybe ^Cell) 1 + data:(Maybe ^Cell) 1 + library:(Maybe ^Cell) 1 + = StateInit; +message$_ {X:Type} + info:CommonMsgInfoRelaxed 502 + init:(Maybe (Either StateInit ^StateInit)) 12 + body:(Either X ^X) // body is either embedded or stored as ref + = MessageRelaxed X; + */ + +fun getMyAddressDev(): address + asm "x{80194DC6438F99D3D9DBE151944925D90B2492954BF6B9C070FBFF2DDED5F30547D_} PUSHSLICE"; + +@inline +fun calculateNftItemStateInitData(itemIndex: int): cell { + return beginCell() + .storeUint(itemIndex, 64) + .storeAddress(getMyAddressDev()) + .endCell(); +} + +@inline +fun calculateStateInitCell(code: cell, data: cell): cell { + return beginCell() + .storeUint(0, 2) // 0 split_depth, 0 special + .storeDict(code) + .storeDict(data) + .storeUint(0, 1) // 0 library + .endCell(); +} + +fun calculateNftItemAddress(workchain: int, stateInitCell: cell): address { + return beginCell() + .storeUint(0b100, 3) // addr_std$10 + 0 split_depth + .storeInt(workchain, 8) + .storeUint(stateInitCell.hash(), 256) + .endCell() + .beginParse() as address; +} + +struct(0x12345678) MyBody { + queryId: uint64; +} + +fun test1_manual() { + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA")) + .storeCoins(123) // value.grams + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + // MyBody: op + queryId + .storeUint(0x12345678, 32) + .storeUint(800, 64) + .endCell(); +} + +@method_id(101) +fun test1() { + val body: MyBody = { queryId: 800 }; + var b = createMessage({ + body: body, + bounce: true, + dest: address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), + value: 123, + }); + assert(b.hash() == test1_manual().hash(), 101); + return b.hash(); +} + +fun test2_manual() { + return beginCell() + .storeUint(0x10, 6) // no bounce + .storeAddress(address("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8")) + .storeCoins(90) // value.grams + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + // body: op + queryId + .storeUint(0x12345678, 32) + .storeUint(127493264572, 64) + .endCell(); +} + +@method_id(102) +fun test2() { + val body: MyBody = { queryId: 127493264572 }; + var b = createMessage({ + body, + bounce: false, + dest: address("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), + value: 90, + }); + assert(b.hash() == test2_manual().hash(), 102); + return b.hash(); +} + +fun test3_manual() { + val body_ref = beginCell() + .storeUint(0x12345678, 32) + .storeUint(127493264572, 64) + .endCell(); + return beginCell() + .storeUint(0x10, 6) // no bounce + .storeAddress(address("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8")) + .storeCoins(90) // value.grams + .storeUint(1, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 1 body (ref) + .storeRef(body_ref) + .endCell(); +} + +@method_id(103) +fun test3() { + var b = createMessage({ + body: MyBody{ queryId: 127493264572 }.toCell(), + bounce: false, + dest: address("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"), + value: 90, + }); + assert(b.hash() == test3_manual().hash(), 103); + return b.hash(); +} + +fun test4_manual(bodyCell: cell, dest: address, value: coins) { + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(dest) + .storeCoins(value) + .storeUint(1, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 1 body (ref) + .storeRef(bodyCell) + .endCell(); +} + +@method_id(104) +fun test4(value: coins) { + val bodyCell = beginCell() + .storeUint(0x03738FA9, 32) + .storeBool(true).storeBool(false).storeCoins(123) + .endCell(); + val dest = address("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff"); + var b = createMessage({ + body: bodyCell, + value, + bounce: true, + dest: dest, + }); + assert(b.hash() == test4_manual(bodyCell, dest, value).hash(), 104); + return b.hash(); +} + +fun test5_manual() { + var ec_dict = createEmptyDict(); + ec_dict.iDictSet(32, 1, "ec1"); + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA")) + .storeCoins(123) // value.grams + .storeMaybeRef(ec_dict) // value.extra + .storeUint(0, 0 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + // body: op + queryId + .storeUint(0x12345678, 32) + .storeUint(800, 64) + .endCell(); +} + +@method_id(105) +fun test5() { + var ec_dict = createEmptyDict(); + ec_dict.iDictSet(32, 1, "ec1"); + val body: MyBody = { queryId: 800 }; + var b = createMessage({ + body, + bounce: true, + dest: address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), + value: (123, ec_dict), + }); + assert(b.hash() == test5_manual().hash(), 105); + return b.hash(); +} + +fun test6_manual(value: coins, ec_dict: dict) { + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA")) + .storeCoins(value) // value.grams + .storeMaybeRef(ec_dict) // value.extra + .storeUint(0, 0 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + // body: op + queryId + .storeUint(0x12345678, 32) + .storeUint(800, 64) + .endCell(); +} + +@method_id(106) +fun test6(value: coins, dictKey: int?) { + var ec_dict = createEmptyDict(); + if (dictKey != null) { + ec_dict.iDictSet(32, dictKey, "ec1"); + } + val body: MyBody = { queryId: 800 }; + var b = createMessage({ + bounce: true, + dest: address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), + value: (value, ec_dict), + body, + }); + assert(b.hash() == test6_manual(value, ec_dict).hash(), 106); + return b.hash(); +} + +struct MyNftBody { + nftContent: cell; +} + +fun test7_manual(nftItemCode: cell, amount: coins) { + var (itemIndex: int, nftContent: cell) = (10, beginCell().endCell()); + val nftItemData = calculateNftItemStateInitData(itemIndex); + val nftAddress = calculateNftItemAddress(BASECHAIN, calculateStateInitCell(nftItemCode, nftItemData)); + + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(nftAddress) + .storeCoins(amount) + // 1 state init exists, 0 either left (state init embedded), 00110 (code and data), 0 either left (body inline) + .storeUint(0b10001100, 1 + 4 + 4 + 64 + 32 + (1 + 1 + 5 + 1)) + .storeRef(nftItemCode) + .storeRef(nftItemData) + .storeRef(nftContent) + .endCell(); +} + +@method_id(107) +fun test7(amount: coins) { + val nftItemCode: cell = beginCell().storeInt(0x273849723892, 94).endCell(); + var (itemIndex: int, nftContent: cell) = (10, beginCell().endCell()); + val stateInitData = calculateNftItemStateInitData(itemIndex); + + val body: MyNftBody = { nftContent }; + var b = createMessage({ + bounce: true, + body, + dest: { workchain: BASECHAIN, stateInit: { code: nftItemCode, data: stateInitData } }, + value: amount, + }); + assert(b.hash() == test7_manual(nftItemCode, amount).hash(), 107); + return b.hash(); +} + +struct(0x706c7567) RequestPaymentMessage { + queryId: uint64; + amount: coins; + someDict: dict; +} + +fun test8_manual(destAddr: address, requestedAmount: coins) { + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(destAddr) + .storeCoins(123) // value.grams + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + .storeUint(0x706c7567, 32) + .storeUint(0, 64) + .storeCoins(requestedAmount) + .storeDict(null) + .endCell(); +} + +@method_id(108) +fun test8(requestedAmount: coins) { + val destAddr = address("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf"); + var b = createMessage({ + bounce: true, + dest: destAddr, + value: 123, + body: RequestPaymentMessage { + queryId: 0, + amount: requestedAmount, + someDict: null, + } + }); + assert(b.hash() == test8_manual(destAddr, requestedAmount).hash(), 108); + return b.hash(); +} + +fun test9_manual(bounceable: bool) { + return beginCell() + .storeUint(0b01, 2).storeBool(bounceable).storeUint(0b000, 3) + .storeAddress(address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA")) + .storeCoins(0) // value.grams + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + .storeUint(0x12345678, 32) + .storeUint(800, 64) + .storeInt(-777, 13) + .endCell(); +} + +@method_id(109) +fun test9(bounceable: bool) { + var b = createMessage({ + bounce: bounceable, + dest: address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), + value: 0, + body: (0x12345678 as uint32, 800 as uint64, -777 as int13), + }); + assert(b.hash() == test9_manual(bounceable).hash(), 109); + return b.hash(); +} + +type Body500Bits = (uint250, uint250); // this body guaranteely fits into cell + +fun test10_manual(bd: Body500Bits) { + return beginCell() + .storeUint(0x10, 6) + .storeAddress(address("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6")) + .storeCoins(1 << 118) + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + .storeUint(bd.0, 250) + .storeUint(bd.1, 250) + .endCell(); +} + +@method_id(110) +fun test10(bd: Body500Bits) { + var b = createMessage({ + bounce: false, + body: bd, + dest: address("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6"), + value: 1 << 118, + }); + assert(b.hash() == test10_manual(bd).hash(), 110); + return b.hash(); +} + +type Body750Bits = (uint250, uint250, uint250); // this body is auto-ref + +fun test11_manual(bd: Body750Bits) { + val bodyRef = beginCell().storeUint(bd.0,250).storeUint(bd.1,250).storeUint(bd.2,250).endCell(); + return beginCell() + .storeUint(0x10, 6) + .storeAddress(address("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6")) + .storeCoins(1 << 110) + .storeUint(1, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 1 body (ref) + .storeRef(bodyRef) + .endCell(); +} + +@method_id(111) +fun test11(bd: Body750Bits) { + var b = createMessage({ + bounce: false, + body: bd, + dest: address("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6"), + value: 1 << 110, + }); + assert(b.hash() == test11_manual(bd).hash(), 111); + return b.hash(); +} + +fun test12_manual(data32: uint32) { + var init = ContractState { + data: beginCell().storeUint(data32,32).endCell(), + code: beginCell().endCell(), + }; + + return beginCell() + .storeUint(0x18, 6) // bounce + .storeUint(0b100, 3).storeInt(MASTERCHAIN, 8).storeUint(StateInit.calcHashCodeData(init.code, init.data), 256) + .storeCoins(ton("0.05")) + // 1 state init exists, 0 either left (state init embedded), 00110 (code and data), 0 either left (body inline) + .storeUint(0b10001100, 1 + 4 + 4 + 64 + 32 + (1 + 1 + 5 + 1)) + .storeRef(init.code) + .storeRef(init.data) + .storeUint(data32, 32) + .endCell(); +} + +@method_id(112) +fun test12(data32: uint32) { + var init = ContractState { + data: beginCell().storeUint(data32,32).endCell(), + code: beginCell().endCell(), + }; + + var b = createMessage({ + bounce: true, + body: data32, + dest: { workchain: MASTERCHAIN, stateInit: init }, + value: ton("0.05"), + }); + assert(b.hash() == test12_manual(data32).hash(), 112); + return b.hash(); +} + +fun test13_manual() { + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(address("EQDE0HBgfkOiqHezLtExBGTvOs8eitthHQosBjW3BmDy1y2K")) + .storeCoins(52 << 78) // value.grams + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + .endCell(); // no body at all +} + +@method_id(113) +fun test13() { + var b = createMessage({ + bounce: true, + dest: address("EQDE0HBgfkOiqHezLtExBGTvOs8eitthHQosBjW3BmDy1y2K"), + value: 52 << 78, + }); + assert(b.hash() == test13_manual().hash(), 113); + return b.hash(); +} + +fun test14_manual(stateInitCell: cell, amount: coins) { + val nftAddress = calculateNftItemAddress(BASECHAIN, stateInitCell); + + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(nftAddress) + .storeCoins(amount) + // 1 state init exists, 1 either right (state init ref), 0 either left (body inline) + .storeUint(0b110, 1 + 4 + 4 + 64 + 32 + (1 + 1 + 1)) + .storeRef(stateInitCell) + .endCell(); // no body +} + +@method_id(114) +fun test14(amount: coins) { + val stateInitCode = beginCell().storeInt(0x273849723892, 94).endCell(); + val stateInitData = calculateNftItemStateInitData(10); + val stateInitCell = calculateStateInitCell(stateInitCode, stateInitData); + + var b = createMessage({ + bounce: true, + dest: { stateInit: stateInitCell }, + value: amount, + }); + assert(b.hash() == test14_manual(stateInitCell, amount).hash(), 114); + return b.hash(); +} + +struct(0x1234) Body15 { + tens: Body750Bits; + more: int32; +} + +fun test15_manual(stateInitCell: cell, bd: Body15) { + val bodyRef = beginCell().storeUint(0x1234,16).storeUint(bd.tens.0,250).storeUint(bd.tens.1,250).storeUint(bd.tens.2,250).storeInt(bd.more,32).endCell(); + val nftAddress = calculateNftItemAddress(MASTERCHAIN, stateInitCell); + + return beginCell() + .storeUint(0x10, 6) // no bounce + .storeAddress(nftAddress) + .storeCoins(ton("100.0004")) + // 1 state init exists, 1 either right (state init ref), 1 either right (body ref) + .storeUint(0b111, 1 + 4 + 4 + 64 + 32 + (1 + 1 + 1)) + .storeRef(stateInitCell) + .storeRef(bodyRef) + .endCell(); // no body +} + +@method_id(115) +fun test15(tens0: int, tens1: int) { + val stateInitCode = beginCell().storeInt(0x273849723892, 94).endCell(); + val stateInitData = calculateNftItemStateInitData(10); + val stateInitCell = beginCell() + .storeUint(1, 1) // has split_depth + .storeUint(3, 5) // split_depth + .storeUint(0, 1) // 0 special + .storeDict(stateInitCode) + .storeDict(stateInitData) + .storeUint(0, 1) // 0 library + .endCell(); + var bd: Body15 = { + tens: (1 << 88, tens0, tens1), + more: max(tens0, tens1), + }; + + var b = createMessage({ + bounce: false, + dest: { workchain: MASTERCHAIN, stateInit: stateInitCell }, + body: bd, + value: ton("100.0004"), + }); + assert(b.hash() == test15_manual(stateInitCell, bd).hash(), 115); + return b.hash(); +} + +fun test18_manual(dest: builder, queryId: uint64) { + return beginCell() + .storeUint(0x18, 6) // bounce + .storeBuilder(dest) + .storeCoins(123) // value.grams + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + // MyBody: op + queryId + .storeUint(0x12345678, 32) + .storeUint(queryId, 64) + .endCell(); +} + +@method_id(118) +fun test18(queryId: uint64) { + val body: MyBody = { queryId }; + var b = createMessage({ + body: body, + bounce: true, + dest: beginCell().storeUint(0, 2), + value: 123, + }); + assert(b.hash() == test18_manual(beginCell().storeUint(0, 2), queryId).hash(), 118); + return b.hash(); +} + +fun test19_manual(bd: uint64) { + var destB = beginCell().storeUint(0, 2); + return beginCell() + .storeUint(0x10, 6) // no bounce + .storeBuilder(destB) + .storeCoins(ton("0.059")) + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + .storeUint(bd, 64) + .endCell(); +} + +@method_id(119) +fun test19(bd: uint64) { + var destB = beginCell().storeUint(0, 2); + var b = createMessage({ + bounce: 10 < 3, + body: bd, + dest: destB, + value: ton("0.059"), + }); + assert(b.hash() == test19_manual(bd).hash(), 119); + return b.hash(); +} + +fun test20_manual() { + var bodyB = beginCell().storeUint(0x12345678, 32).storeUint(800, 64); + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA")) + .storeCoins(123) // value.grams + .storeUint(1, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 1 body (ref) + .storeRef(bodyB.endCell()) + .endCell(); +} + +@method_id(120) +fun test20() { + var bodyB = beginCell().storeUint(0x12345678, 32).storeUint(800, 64); + var b = createMessage({ + body: bodyB, + bounce: true, + dest: address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"), + value: 123, + }); + assert(b.hash() == test20_manual().hash(), 120); + return b.hash(); +} + +struct(0x12345678) Body21 { + queryId: uint64; + payload: RemainingBitsAndRefs; +} + +fun test21_manual(bd: Body21) { + return beginCell() + .storeUint(0x10, 6) // no bounce + .storeAddress(address("EQAUzE-Nef80O9dLZy91HfPiOb6EEQ8YqyWKyIU-KeaYLNUi")) + .storeCoins(ton("0.1")) // value.grams + .storeDict(createEmptyDict()) + .storeUint(1, 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 1 body (ref) + .storeRef(bd.toCell()) // stored as ref due to remainder of unpredictable size + .endCell(); +} + +@method_id(121) +fun test21(queryId: uint64) { + var payloadSlice = beginCell().storeUint(0x12345678, 32).storeUint(800, 64).storeRef(createEmptyCell()).endCell().beginParse(); + var body: Body21 = { queryId, payload: payloadSlice }; + var b = createMessage({ + bounce: false, + dest: address("EQAUzE-Nef80O9dLZy91HfPiOb6EEQ8YqyWKyIU-KeaYLNUi"), + value: (ton("0.1"), createEmptyDict()), + body, + }); + assert(b.hash() == test21_manual(body).hash(), 121); + return b.hash(); +} + +fun test22_manual(unsafeB: builder) { + return beginCell() + .storeUint(0x18, 6) // bounce + .storeAddress(address("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff")) + .storeCoins(ton("0.08")) + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (no ref) + .storeBuilder(unsafeB) + .endCell(); +} + +@method_id(122) +fun test22(op: uint32, queryId: uint64) { + var bodyB = beginCell().storeUint(op, 32).storeUint(queryId, 64); + var b = createMessage({ + bounce: true, + dest: address("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff"), + value: ton("0.08"), + body: UnsafeBodyNoRef { + bodyForceNoRef: bodyB + }, + }); + assert(b.hash() == test22_manual(bodyB).hash(), 122); + return b.hash(); +} + +struct Body23Unlimited { + any1: (address, builder), + any2: RemainingBitsAndRefs; +} + +fun test23_manual(state32: uint32, body: Body23Unlimited) { + var init = ContractState { + data: beginCell().storeUint(state32,32).endCell(), + code: beginCell().endCell(), + }; + + return beginCell() + .storeUint(0x18, 6) // bounce + .storeUint(0b100, 3).storeInt(9, 8).storeUint(StateInit.calcHashCodeData(init.code, init.data), 256) + .storeCoins(ton("0.10009")) + // 1 state init exists, 0 either left (state init embedded), 00110 (code and data), 0 either left (body inline) + .storeUint(0b10001100, 1 + 4 + 4 + 64 + 32 + (1 + 1 + 5 + 1)) + .storeRef(init.code) + .storeRef(init.data) + .storeAddress(body.any1.0) + .storeBuilder(body.any1.1) + .storeSlice(body.any2) + .endCell(); +} + +@method_id(123) +fun test23(state32: uint32) { + var init = ContractState { + data: beginCell().storeUint(state32,32).endCell(), + code: beginCell().endCell(), + }; + var bd: Body23Unlimited = { + any1: (createAddressNone(), beginCell().storeRef(init.code)), + any2: beginCell().storeUint(80, 80).endCell().beginParse(), + }; + + var b = createMessage({ + bounce: true, + body: UnsafeBodyNoRef { + bodyForceNoRef: bd, + }, + dest: { workchain: 9, stateInit: init }, + value: ton("0.10009"), + }); + assert(b.hash() == test23_manual(state32, bd).hash(), 123); + return b.hash(); +} + +fun test24_manual(addrHash: uint256) { + return beginCell() + .storeUint(0x18, 6) // bounce + .storeUint(0b100, 3).storeInt(MASTERCHAIN, 8).storeUint(addrHash, 256) + .storeCoins(ton("0.6")) + .storeUint(0, 1 + 4 + 4 + 64 + 32 + 1 + 1) // ... + 0 init + 0 body (not ref) + .storeUint(0x12345678, 32) + .storeUint(800, 64) + .endCell(); +} + +@method_id(124) +fun test24() { + val addrHash = address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA").getWorkchainAndHash().1; + var b = createMessage({ + body: MyBody { queryId: 800 }, + bounce: true, + dest: (MASTERCHAIN, addrHash), + value: ton("0.6"), + }); + assert(b.hash() == test24_manual(addrHash).hash(), 124); + return b.hash(); +} + +fun main() { +} + +/** +@testcase | 101 | | 104134380371273907780196393444164225714229635235007677971195651971972203592811 +@testcase | 102 | | 20192939504955637835708665496574868659039935190188156593169026529135727309085 +@testcase | 103 | | 97654287980681401436727082042373998183264988661375688119718809500301413968039 +@testcase | 104 | 600000 | 755636693689039391990782995030008885663781699175339033402019272057515711062 +@testcase | 105 | | 24816341499673835567887890014278904436471582322802948121781418622643959482495 +@testcase | 106 | 123 null | 104134380371273907780196393444164225714229635235007677971195651971972203592811 +@testcase | 106 | 456 8888 | 39337586036945718311402746340438400160817844833530971545330721291986281100430 +@testcase | 107 | 1000 | 55093441331748625324828489600632232039914212774002148634088483962817636598198 +@testcase | 108 | 50000000 | 95023796475113775225029817428715936488418545169963429399979521091689824066088 +@testcase | 109 | 0 | 55999621586681214992294941423256376619779969729861696464321825639854258502733 +@testcase | 109 | -1 | 84087871798432599249441213206223143701565541307347047545146076475041341315422 +@testcase | 110 | 250 250 | 97468400996544929599099493087921300963923138788231489050737873840992619823773 +@testcase | 111 | 1 2 3 | 35151166799433266221446406287469019610025742512503320058804207122452431754243 +@testcase | 112 | 32 | 40256061350602595831367445407067573081836468277788226383346273736379122699330 +@testcase | 113 | | 8212062468046185391185852622213582155366804215840270337189205672457136520017 +@testcase | 114 | 100500 | 69207815800109735757177433421533576767812185821226447066044060358661780329301 +@testcase | 115 | 66 77 | 77952119695754644819736002369288963466111166174255607433142955344110346202253 +@testcase | 118 | 888 | 75462675913935779917745192355822465171309245151518256862408373999119088535160 +@testcase | 119 | 999 | 20451206881650273118327988889219529836875916996856547550069532964562229905067 +@testcase | 120 | | 86341416901030824925289599533989709413619468614523233983159539599569269903295 +@testcase | 121 | 2983742 | 29682465216061902145511914581895871811826194319877849112468723187093436476183 +@testcase | 122 | 88 12892 | 104095783372529353117379287284649790470404208891518326338200063605256169961669 +@testcase | 123 | 999 | 93691386126134034953606952841435771108648354412078750046821446829279390964584 +@testcase | 124 | | 21886688052816798288463190773103865772534937765373272039597499398551023701577 + */ diff --git a/tolk-tester/tests/send-msg-2.tolk b/tolk-tester/tests/send-msg-2.tolk new file mode 100644 index 000000000..b1e3af99c --- /dev/null +++ b/tolk-tester/tests/send-msg-2.tolk @@ -0,0 +1,337 @@ +/* +int_msg_info$0 1 + ihr_disabled:Bool // always 0, not implemented 1 + bounce:Bool // parameter 1 + bounced:Bool // always 0 on send 1 + src:MsgAddress // always 00 on send 2 + dest:MsgAddressInt // parameter 267 + value:CurrencyCollection // parameter 124 + 1 + ihr_fee:Grams // always 0, not implemented 4 + fwd_fee:Grams // always 0 on send 4 + created_lt:uint64 // always 0 on send 64 + created_at:uint32 // always 0 on send 32 + = CommonMsgInfoRelaxed; +_ split_depth:(Maybe (## 5)) 1 + 5 + special:(Maybe TickTock) 1 + 2 + code:(Maybe ^Cell) 1 + data:(Maybe ^Cell) 1 + library:(Maybe ^Cell) 1 + = StateInit; +message$_ {X:Type} + info:CommonMsgInfoRelaxed 502 + init:(Maybe (Either StateInit ^StateInit)) 12 + body:(Either X ^X) // body is either embedded or stored as ref + = MessageRelaxed X; + */ + +const SHARD_DEPTH = 8; + +fun getMyAddressDev(): address + asm "x{80194DC6438F99D3D9DBE151944925D90B2492954BF6B9C070FBFF2DDED5F30547D_} PUSHSLICE"; + +@inline +fun getAddressShard(address: address, shardLen: int): int { + // Skip workchain, load shard prefix + (address as slice).skipBits(3 + 8); + return (address as slice).loadUint(shardLen); +} + +@inline +fun packJettonWalletData(status: int, balance: int, ownerAddress: address, jettonMasterAddress: address): cell { + return beginCell() + .storeUint(status, 8) + .storeCoins(balance) + .storeAddress(ownerAddress) + .storeAddress(jettonMasterAddress) + .endCell(); +} + +@inline +fun calculateJettonWalletStateInitWithShard(ownerAddress: address, jettonMasterAddress: address, jettonWalletCode: cell): cell { + /* + https://github.com/ton-blockchain/ton/blob/8a9ff339927b22b72819c5125428b70c406da631/crypto/block/block.tlb#L144 + _ split_depth:(Maybe (## 5)) special:(Maybe TickTock) + code:(Maybe ^Cell) data:(Maybe ^Cell) + library:(Maybe ^Cell) = StateInit; + */ + return beginCell() + .storeUint(1, 1) + .storeUint(SHARD_DEPTH, 5) + .storeUint(0, 1) + .storeMaybeRef(jettonWalletCode) + .storeMaybeRef( + packJettonWalletData( + 0, // status + 0, // balance + ownerAddress, + jettonMasterAddress) + ) + .storeUint(0, 1) // Empty libraries + .endCell(); +} + +@inline +fun calculateJettonWalletAddress(shardPrefix: int, stateInitCell: cell): address { + var mask = (1 << (256 - SHARD_DEPTH)) - 1; + var prefixLess = stateInitCell.hash() & mask; + return beginCell() + .storeUint(4, 3) // addr_std$10 + anycast 0 + .storeInt(BASECHAIN, 8) + .storeUint(shardPrefix, SHARD_DEPTH) + .storeUint(prefixLess, 256 - SHARD_DEPTH) + .endCell() + .beginParse() as address; +} + +@inline +fun calculateAddressInAnotherShard(pivotAddress: address, shardPrefixLen: uint5, code: cell, data: cell): builder { + val stateInitCell = beginCell() + .storeUint(1, 1) + .storeUint(shardPrefixLen, 5) // shard depth + .storeUint(0, 1) + .storeMaybeRef(code) + .storeMaybeRef(data) + .storeUint(0, 1) // Empty libraries + .endCell(); + + val shardPrefix = getAddressShard(pivotAddress, shardPrefixLen); + var mask = (1 << (256 - shardPrefixLen)) - 1; + var prefixLess = stateInitCell.hash() & mask; + return beginCell() + .storeUint(4, 3) // addr_std$10 + anycast 0 + .storeInt(BASECHAIN, 8) + .storeUint(shardPrefix, shardPrefixLen) + .storeUint(prefixLess, 256 - shardPrefixLen); +} + +fun test1_manual(toAddress: address, masterMsg: cell, jettonWalletCode: cell) { + var shardPrefix = getAddressShard(toAddress, SHARD_DEPTH); + var stateInitCell = calculateJettonWalletStateInitWithShard(toAddress, getMyAddressDev(), jettonWalletCode); + var toWalletAddress = calculateJettonWalletAddress(shardPrefix, stateInitCell); + var jettonWalletData = packJettonWalletData(0, 0, toAddress, getMyAddressDev()); + + return beginCell() + .storeUint(0x18, 6) // bounceable + .storeAddress(toWalletAddress) // dest + .storeCoins(ton("0.05")) + // 1 state init exists, 0 either left (state init embedded), 1 either left (fixed_prefix_length exists) + .storeUint(0b101, 1 + 4 + 4 + 64 + 32 + 1 + 1 + 1) + .storeUint(SHARD_DEPTH, 5) + .storeUint(0b01101, 4 + 1) // 0 ticktock + 1 code + 1 data + 0 library + 1 body ref + .storeRef(jettonWalletCode) + .storeRef(jettonWalletData) + .storeRef(masterMsg) + .endCell(); +} + +@method_id(101) +fun test1() { + val toAddress = address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"); + val masterMsg = beginCell().storeUint(1<<14, 32).endCell(); + val jettonWalletCode = beginCell().storeInt(0x273849723892, 94).endCell(); + + var b = createMessage({ + bounce: true, + dest: { + stateInit: { + code: jettonWalletCode, + data: packJettonWalletData(0, 0, toAddress, getMyAddressDev()), + }, + toShard: { + fixedPrefixLength: SHARD_DEPTH, + closeTo: toAddress, + } + }, + body: masterMsg, + value: ton("0.05"), + }); + assert(b.hash() == test1_manual(toAddress, masterMsg, jettonWalletCode).hash(), 101); + return b.hash() +} + +fun test2_manual(toAddress: address, masterMsg: cell, jettonWalletCode: cell) { + var shardPrefix = getAddressShard(toAddress, SHARD_DEPTH); + var stateInit = calculateJettonWalletStateInitWithShard(toAddress, getMyAddressDev(), jettonWalletCode); + var toWalletAddress = calculateJettonWalletAddress(shardPrefix, stateInit); + + return beginCell() + .storeUint(0x10, 6) // not bounceable + .storeAddress(toWalletAddress) // dest + .storeCoins(ton("0.05")) + .storeUint(0b111, 1 + 4 + 4 + 64 + 32 + 1 + 1 + 1) // 1 state init exists + 1 state init ref + 1 body ref + .storeRef(stateInit) + .storeRef(masterMsg) + .endCell(); +} + +@method_id(102) +fun test2() { + val toAddress = address("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"); + val masterMsg = beginCell().endCell(); + val jettonWalletCode = beginCell().storeInt(0x273849723892, 94).endCell(); + var stateInitCell = calculateJettonWalletStateInitWithShard(toAddress, getMyAddressDev(), jettonWalletCode); + + var b = createMessage({ + bounce: false, + dest: { + workchain: BASECHAIN, + stateInit: stateInitCell, + toShard: { + fixedPrefixLength: SHARD_DEPTH, + closeTo: toAddress, + } + }, + body: masterMsg, + value: ton("0.05"), + }); + assert(b.hash() == test2_manual(toAddress, masterMsg, jettonWalletCode).hash(), 102); + return b.hash() +} + +struct Test3Body { + bigData: bits800; +} + +fun test3_manual(myCode: cell, myData: cell, msgBody: Test3Body) { + var addrInShard = calculateAddressInAnotherShard(getMyAddressDev(), 20, myCode, myData); + + return beginCell() + .storeUint(0x18, 6) // bounceable + .storeBuilder(addrInShard) // dest + .storeCoins(ton("0.001")) + .storeUint(0b10, 1 + 4 + 4 + 64 + 32 + 1 + 1) // 1 state init exists + 1 state init inline + .storeUint(1, 1) + .storeUint(20, 5) // shard depth + .storeUint(0b0110, 4) // code + data exist + .storeUint(1, 1) // body ref + .storeRef(myCode) + .storeRef(myData) + .storeRef(msgBody.toCell()) + .endCell(); +} + +@method_id(103) +fun test3() { + val myCode = beginCell().storeInt(0x273849723892, 94).endCell(); + val myData = beginCell().storeInt(0x273849723892, 200).storeMaybeRef(myCode).endCell(); + val builder800 = beginCell().storeInt(1, 250).storeInt(1, 250).storeInt(1, 250).storeInt(1, 50); + val body: Test3Body = { bigData: builder800.endCell().beginParse() as bits800 }; + + var b = createMessage({ + bounce: true, + dest: { + stateInit: { + code: myCode, + data: myData, + }, + toShard: { + fixedPrefixLength: 20, + closeTo: getMyAddressDev(), + } + }, + body, + value: ton("0.001"), + }); + assert(b.hash() == test3_manual(myCode, myData, body).hash(), 103); + return b.hash() +} + +struct Test4Body { + f1: int64; + f2: int64; +} + +@method_id(104) +fun test4() { + val myCode = beginCell().storeInt(0x273849723892, 94).endCell(); + val myData = beginCell().storeInt(0x273849723892, 200).endCell(); + val body: Test4Body = { f1: 123, f2: 456 }; + + var b = createMessage({ + bounce: true, + dest: { + stateInit: { + code: myCode, + data: myData, + }, + toShard: { + fixedPrefixLength: 30, + closeTo: getMyAddressDev(), + } + }, + body, + value: ton("0.1"), + }); + return b.hash() +} + +@overflow1023_policy("suppress") +struct Test5Body { + c1: coins; c2: coins; c3: coins; + c4: coins; c5: coins; c6: coins; + c7: coins; c8: coins; c9: coins; +} + +@method_id(105) +fun test5() { + val pivot = address("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"); + val myCode = beginCell().storeInt(0x273849723892, 94).endCell(); + val myData = beginCell().storeInt(0x273849723892, 200).endCell(); + val body: Test5Body = { c1: 1, c2: 12, c3: 13, c4: 41, c5: 15, c6: 66, c7: 777, c8: 8888, c9: 0x9999 }; + + var b = createMessage({ + bounce: true, + dest: { + stateInit: { + code: myCode, + data: myData, + }, + toShard: { + fixedPrefixLength: 2, + closeTo: pivot, + } + }, + body: UnsafeBodyNoRef { bodyForceNoRef: body }, + value: ton("0.1"), + }); + return b.hash() +} + +@method_id(106) +fun test6(shardLen: int) { + val pivot = address("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"); + val myCode = beginCell().storeInt(0x273849723892, 94).endCell(); + val myData = beginCell().storeInt(0x273849723892, 200).endCell(); + val body: Test5Body = { c1: 1, c2: 12, c3: 13, c4: 41, c5: 15, c6: 66, c7: 777, c8: 8888, c9: 0x9999 }; + + var b = createMessage({ + bounce: true, + dest: { + workchain: BASECHAIN, + stateInit: { + code: myCode, + data: myData, + }, + toShard: { + fixedPrefixLength: shardLen, + closeTo: pivot, + } + }, + body: body, + value: ton("0.1"), + }); + return b.hash() +} + +fun main() {} + + +/** +@testcase | 101 | | 114692697425779687591180352274328586758984823673144676907237399062035787598876 +@testcase | 102 | | 19141316219057792078125032524380927286847294424548966584483011017258286475706 +@testcase | 103 | | 33184074158306258754479533493449661031201063538320283251640171344508567641003 +@testcase | 104 | | 52060388210984960140400107426752119783720473036191582802152450095687953630545 +@testcase | 105 | | 18248582963085351579261363895401647616194364201057112283341027605548475235397 +@testcase | 106 | 2 | 73367500160949078368695742399322272058735198813159066885436981754975296689213 +@testcase | 106 | 9 | 26260034723231226694451786564335585031223880314377072827334911071856417531179 + */ diff --git a/tolk-tester/tests/send-msg-3.tolk b/tolk-tester/tests/send-msg-3.tolk new file mode 100644 index 000000000..69afed4cd --- /dev/null +++ b/tolk-tester/tests/send-msg-3.tolk @@ -0,0 +1,233 @@ +/* +ext_out_msg_info$11 + src:MsgAddress // 00 on send + dest:MsgAddressExt // parameter + created_lt:uint64 // 0 on send + created_at:uint32 // 0 on send + = CommonMsgInfoRelaxed; +message$_ {X:Type} + info:CommonMsgInfoRelaxed + init:(Maybe (Either StateInit ^StateInit)) // 0 on send + body:(Either X ^X) // body is either embedded or stored as ref + = MessageRelaxed X; +*/ + +struct(0x12345678) MyBody { + queryId: uint64; +} + +fun test1_manual(topic: int) { + return beginCell() + .storeUint(0b1100, 4) // ext_out_msg_info$11 src:MsgAddressInt () + .storeUint(1, 2) // addr_extern$01 + .storeUint(256, 9) // len:(## 9) + .storeUint(topic, 256) // external_address:(bits len) (assume it fits 248 bits) + .storeUint(0, 64 + 32 + 2) // created_lt, created_at, init:Maybe (0), body:Either left (inline) + // MyBody: op + queryId + .storeUint(0x12345678, 32) + .storeUint(800, 64) + .endCell(); +} + +@method_id(101) +fun test1(topic: int) { + val body: MyBody = { queryId: 800 }; + var b = createExternalLogMessage({ + body: body, + dest: ExtOutLogBucket { topic }, + }); + assert(b.hash() == test1_manual(topic).hash(), 101); + return b.hash(); +} + +type Body750Bits = (uint250, uint250, uint250); // this body is auto-ref + +fun test2_manual(bd: Body750Bits) { + val bodyRef = beginCell().storeUint(bd.0,250).storeUint(bd.1,250).storeUint(bd.2,250).endCell(); + return beginCell() + .storeUint(0b1100, 4) // ext_out_msg_info$11 src:MsgAddressInt () + .storeUint(1, 2) // addr_extern$01 + .storeUint(256, 9) // len:(## 9) + .storeUint(1 << 102, 256) // external_address:(bits len) + .storeUint(1, 64 + 32 + 2) // created_lt, created_at, init:Maybe (0), body:Either right (ref) + .storeRef(bodyRef) + .endCell(); +} + +@method_id(102) +fun test2(bd: Body750Bits) { + var b = createExternalLogMessage({ + body: bd, + dest: { topic: 1 << 102 }, + }); + assert(b.hash() == test2_manual(bd).hash(), 102); + return b.hash(); +} + +struct Body3 { + bodyAddress: address; + amount: coins; + someRef: cell; +} + +@method_id(103) +fun test3(eventId: int) { + val bodyAddress = address("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf"); + val someRef = MyBody { queryId: 800 }.toCell(); + var b = createExternalLogMessage({ + body: Body3 { bodyAddress, someRef, amount: ton("0.05") }, + dest: { topic: eventId }, + }); + return b.hash(); +} + +fun test4_manual(dest: address, body: cell) { + return beginCell() + .storeUint(0b1100, 4) // ext_out_msg_info$11 src:MsgAddressInt () + .storeAddress(dest) // dest (manually created external address) + .storeUint(1, 64 + 32 + 2) // created_lt, created_at, init:Maybe (0), body:Either right (ref) + .storeRef(body) + .endCell(); +} + +@method_id(104) +fun test4() { + val body: cell = Body3 { bodyAddress: createAddressNone(), amount: 0, someRef: createEmptyCell() }.toCell(); + val dest = beginCell().storeUint(0b01, 2).storeUint(12, 9).storeUint(88, 12).endCell() + .beginParse() as address; + var b = createExternalLogMessage({ + body, + dest, + }); + assert(b.hash() == test4_manual(dest, body).hash(), 104); + return b.hash(); +} + +@method_id(105) +fun test5(topic: int) { + var b = createExternalLogMessage({ + dest: { topic }, + }); + return b.hash(); +} + +@method_id(106) +fun test6() { + val dest = beginCell().storeUint(0b01, 2).storeUint(12, 9).storeUint(88, 12).endCell() + .beginParse() as address; + var b = createExternalLogMessage({ + dest, + }); + return b.hash(); +} + +fun test7_manual(dest: builder, queryId: uint64) { + return beginCell() + .storeUint(0b1100, 4) // ext_out_msg_info$11 src:MsgAddressInt () + .storeBuilder(dest) // manually created bits representing external address + .storeUint(0, 64 + 32 + 2) // created_lt, created_at, init:Maybe (0), body:Either left (inline) + // MyBody: op + queryId + .storeUint(0x12345678, 32) + .storeUint(queryId, 64) + .endCell(); +} + +@method_id(107) +fun test7(queryId: uint64) { + val body: MyBody = { queryId }; + val dest = beginCell().storeUint(0b01, 2).storeUint(12, 9).storeUint(88, 12); + var b = createExternalLogMessage({ + body, + dest, + }); + assert(b.hash() == test7_manual(dest, queryId).hash(), 107); + return b.hash(); +} + +@method_id(108) +fun test8(topic: int) { + var bodyB = beginCell().storeUint(0x12345678, 32).storeUint(800, 64); + var b = createExternalLogMessage({ + body: bodyB, + dest: { topic }, + }); + return b.hash(); +} + +struct Body9Unlimited { + any1: cell?, + any2: builder, +} + +@method_id(109) +fun test9(topic: int) { + var bd: Body9Unlimited = { + any1: null, + any2: beginCell().storeUint(80, 80).storeMaybeRef(null).storeRef(createEmptyCell()), + }; + var b = createExternalLogMessage({ + body: UnsafeBodyNoRef { + bodyForceNoRef: bd, + }, + dest: ExtOutLogBucket { topic: topic }, + }); + return b.hash(); +} + +fun test10_manual(topic: slice, queryIdRef: uint64) { + return beginCell() + .storeUint(0b1100, 4) // ext_out_msg_info$11 src:MsgAddressInt () + .storeUint(1, 2) // addr_extern$01 + .storeUint(256, 9) // len:(## 9) + .storeUint(0x00, 8) // prefix of ExtOutLogBucket + .storeSlice(topic) // assume this slice is bits248 + .storeUint(0, 64 + 32 + 2) // created_lt, created_at, init:Maybe (0), body:Either left (inline) + .storeRef(beginCell().storeUint(queryIdRef, 64).endCell()) + .endCell(); +} + +struct Body10 { + queryIdRef: Cell; +} + +@method_id(110) +fun test10() { + val queryIdRef = 123; + val topic = stringHexToSlice("012345678901234567890123456789012345678901234567890123456789FF"); + var b = createExternalLogMessage({ + body: Body10 { queryIdRef: (queryIdRef as uint64).toCell() }, + dest: ExtOutLogBucket { topic: topic as bits248 }, + }); + assert(b.hash() == test10_manual(topic, queryIdRef).hash(), 110); + return b.hash(); +} + +@method_id(111) +fun test11() { + try { + createExternalLogMessage({ + dest: ExtOutLogBucket { topic: "asdf" as bits248 } // wrong bits count + }); + return 0; + } catch (excno) { + return excno; + } +} + +fun main() { + +} + +/** +@testcase | 101 | 100500 | 8399954633429803564068357060778961153860921657197909539205508789211284886940 +@testcase | 102 | 1 2 3 | 64371641017422166665232911624005896033238822098768040861805756890339950064655 +@testcase | 103 | 829999 | 33039908261421267188072304726227928987040300641614103065684678216731595546507 +@testcase | 104 | | 94681566274361007141065727833953679134419126606639224444485202401996571399154 +@testcase | 105 | 329889 | 24202801754361933063453004029355483858376498707977868951519201997233775688005 +@testcase | 106 | | 17838299862930869444234264325786950440464929864630237676886309244018701054708 +@testcase | 107 | 777 | 77493748753370371314291958955179535305777482888906702570602071501642726442389 +@testcase | 108 | 0 | 69287572703823759594480986943855745035158339431740068559880301501812484359455 +@testcase | 109 | 12 | 72694183679327534355706107347199645597776533707531061585024123910830113048519 +@testcase | 110 | | 33186381366677843958628736306577552541950719844529663083708985644217867888324 +@testcase | 111 | | 9 + */ diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index 106d93840..779095302 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -733,16 +733,16 @@ fun main(x: int?): int { @testcase | 139 | | 16 @testcase | 140 | 5 | 25 @testcase | 141 | | 1 2 -@testcase | 142 | | 5 3 (null) (null) 0 133 3 (null) (null) 0 +@testcase | 142 | | 5 3 (null) (null) 0 typeid-5 3 (null) (null) 0 @testcase | 143 | | 10 11 (null) 10 11 (null) (null) 0 @testcase | 144 | | 10 11 (null) 10 11 (null) (null) 0 -@testcase | 145 | | 5 3 (null) (null) 0 133 3 (null) (null) (null) (null) 0 3 (null) (null) 0 -@testcase | 146 | | 3 4 5 3 4 5 132 +@testcase | 145 | | 5 3 (null) (null) 0 typeid-5 3 (null) (null) (null) (null) 0 3 (null) (null) 0 +@testcase | 146 | | 3 4 5 3 4 5 typeid-4 @testcase | 147 | | (null) (null) 100 (null) 100 (null) (null) 0 @testcase | 158 | | 123 10 123 5 @testcase | 160 | | 101 109 @testcase | 161 | 9 9 | (null) (null) 0 (null) (null) -@testcase | 161 | 19 0 | 19 0 129 19 0 +@testcase | 161 | 19 0 | 19 0 typeid-1 19 0 @stderr warning: expression of type `int` can never be `null`, this condition is always true @stderr warning: unreachable code diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index 46da0207c..c7f81e4c3 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -581,37 +581,37 @@ type PointAlias = Point; @testcase | 105 | | 35 30 45 @testcase | 106 | | 10 20 @testcase | 107 | | 5 5 10 20 15 -@testcase | 108 | | 777 131 777 0 777 777 777 131 777 132 777 777 +@testcase | 108 | | 777 typeid-3 777 0 777 777 777 typeid-3 777 typeid-4 777 777 @testcase | 109 | | 70 30 20 20 -80 @testcase | 110 | 0 | (null) (null) 0 -@testcase | 110 | -1 | 10 20 133 +@testcase | 110 | -1 | 10 20 typeid-5 @testcase | 111 | 0 | 0 2 10 20 30 @testcase | 111 | -1 | 0 0 3 4 7 @testcase | 112 | 0 | (null) (null) 0 -@testcase | 112 | -1 | 1 2 133 +@testcase | 112 | -1 | 1 2 typeid-5 @testcase | 113 | | (null) (null) 0 @testcase | 114 | | 1 2 @testcase | 115 | | (null) (null) (null) (null) 0 -@testcase | 116 | | -1 -4 [ 8 4 ] 7 (null) 0 [ 10 11 ] 135 +@testcase | 116 | | -1 -4 [ 8 4 ] 7 (null) 0 [ 10 11 ] typeid-7 @testcase | 117 | 5 | 5 5 5 5 5 @testcase | 117 | null | (null) (null) (null) -1 (null) @testcase | 118 | | 17 -@testcase | 119 | | 10 20 9 10 133 9 -@testcase | 120 | 0 | 80 9 8 80 133 +@testcase | 119 | | 10 20 9 10 typeid-5 9 +@testcase | 120 | 0 | 80 9 8 80 typeid-5 @testcase | 120 | -1 | 8 14 (null) (null) 0 @testcase | 121 | 0 | 17 17 8 136 @testcase | 121 | -1 | 8 (null) 8 8 @testcase | 122 | 100 | 101 50 106 212 100 101 101 @testcase | 124 | 66 | 66 -@testcase | 125 | | (null) (null) 40 60 133 +@testcase | 125 | | (null) (null) 40 60 typeid-5 @testcase | 126 | | 118 -@testcase | 127 | | 0 0 131 131 777 -1 -1 0 0 777 0 0 138 777 0 0 777 (null) (null) 0 -@testcase | 128 | -1 | (null) (null) 131 -1 0 0 777 (null) (null) 131 -1 0 0 -@testcase | 128 | 0 | 1 2 130 0 -1 0 777 0 131 138 0 -1 0 +@testcase | 127 | | 0 0 typeid-3 typeid-3 777 -1 -1 0 0 777 0 0 typeid-10 777 0 0 777 (null) (null) 0 +@testcase | 128 | -1 | (null) (null) typeid-3 -1 0 0 777 (null) (null) typeid-3 -1 0 0 +@testcase | 128 | 0 | 1 2 typeid-2 0 -1 0 777 0 typeid-3 typeid-10 0 -1 0 @testcase | 129 | | (null) -@testcase | 130 | -1 | 140 777 (null) 140 777 0 777 0 0 -@testcase | 130 | 0 | 139 777 4 1 777 139 777 0 0 -@testcase | 131 | | 5 141 (null) 141 (null) 0 777 0 0 -1 777 0 -1 +@testcase | 130 | -1 | typeid-12 777 (null) typeid-12 777 0 777 0 0 +@testcase | 130 | 0 | typeid-11 777 4 1 777 typeid-11 777 0 0 +@testcase | 131 | | 5 typeid-13 (null) typeid-13 (null) 0 777 0 0 -1 777 0 -1 @testcase | 132 | | 10 20 10 0 0 20 0 0 0 0 @testcase | 133 | | -1 0 5 (null) (null) (null) 0 0 46 @testcase | 134 | | 10 20 diff --git a/tolk-tester/tests/union-types-tests.tolk b/tolk-tester/tests/union-types-tests.tolk index 692c06e1c..422b08bdb 100644 --- a/tolk-tester/tests/union-types-tests.tolk +++ b/tolk-tester/tests/union-types-tests.tolk @@ -791,11 +791,11 @@ fun main() { @testcase | 105 | | 5 5 1 5 1 5 @testcase | 106 | 1 | (null) 0 @testcase | 106 | 2 | 2 1 -@testcase | 107 | 1 | (null) 2 3 129 (null) 2 3 129 -@testcase | 107 | 3 | 6 7 8 130 6 7 8 130 -@testcase | 108 | | (null) 2 3 129 (null) (null) (null) 0 (null) 2 3 129 (null) 2 3 129 (null) (null) (null) 0 -@testcase | 109 | | 6 7 8 130 0 (null) -1 (null) (null) (null) 0 -1 (null) (null) (null) 0 -@testcase | 110 | | 1 (null) (null) 0 (null) (null) 0 135 +@testcase | 107 | 1 | (null) 2 3 typeid-1 (null) 2 3 typeid-1 +@testcase | 107 | 3 | 6 7 8 typeid-2 6 7 8 typeid-2 +@testcase | 108 | | (null) 2 3 typeid-1 (null) (null) (null) 0 (null) 2 3 typeid-1 (null) 2 3 typeid-1 (null) (null) (null) 0 +@testcase | 109 | | 6 7 8 typeid-2 0 (null) -1 (null) (null) (null) 0 -1 (null) (null) (null) 0 +@testcase | 110 | | 1 (null) (null) 0 (null) (null) 0 typeid-7 @testcase | 120 | | 5 @testcase | 121 | | -1 0 0 -1 -1 0 @testcase | 122 | 0 0 1 | 36 @@ -807,21 +807,21 @@ fun main() { @testcase | 125 | 1 1 | 42 @testcase | 126 | | 42 @testcase | 127 | | 1 2 5 1 -@testcase | 128 | | (null) (null) 0 777 (null) 2 1 777 3 4 129 +@testcase | 128 | | (null) (null) 0 777 (null) 2 1 777 3 4 typeid-1 @testcase | 129 | | (null) (null) 0 (null) 5 1 -@testcase | 130 | 0 | (null) 5 1 777 (null) (null) 0 777 1 2 129 777 (null) [ 6 ] 136 777 (null) (null) 0 -@testcase | 130 | -1 | (null) (null) 0 777 (null) (null) 0 777 1 2 129 777 (null) [ 6 ] 136 777 (null) (null) 0 -@testcase | 131 | | (null) 777 5 777 1 2 777 (null) 777 5 777 (null) (null) 0 777 1 2 129 -@testcase | 132 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 129 777 (null) (null) (null) 0 -@testcase | 133 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 129 777 (null) (null) (null) 0 -@testcase | 134 | | (null) 5 1 777 (null) 5 1 777 1 2 129 777 1 2 129 777 (null) 5 42 777 5 42 777 5 42 -@testcase | 135 | | (null) 5 1 777 (null) (null) 1 2 129 777 (null) (null) 1 2 129 777 (null) (null) 0 -@testcase | 136 | | 1 2 129 777 1 2 140 777 1 2 141 777 (null) 1 2 142 +@testcase | 130 | 0 | (null) 5 1 777 (null) (null) 0 777 1 2 typeid-1 777 (null) [ 6 ] typeid-8 777 (null) (null) 0 +@testcase | 130 | -1 | (null) (null) 0 777 (null) (null) 0 777 1 2 typeid-1 777 (null) [ 6 ] typeid-8 777 (null) (null) 0 +@testcase | 131 | | (null) 777 5 777 1 2 777 (null) 777 5 777 (null) (null) 0 777 1 2 typeid-1 +@testcase | 132 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 typeid-1 777 (null) (null) (null) 0 +@testcase | 133 | | (null) (null) 5 1 777 (null) (null) 5 42 777 (null) 4 5 typeid-1 777 (null) (null) (null) 0 +@testcase | 134 | | (null) 5 1 777 (null) 5 1 777 1 2 typeid-1 777 1 2 typeid-1 777 (null) 5 42 777 5 42 777 5 42 +@testcase | 135 | | (null) 5 1 777 (null) (null) 1 2 typeid-1 777 (null) (null) 1 2 typeid-1 777 (null) (null) 0 +@testcase | 136 | | 1 2 typeid-1 777 1 2 typeid-12 777 1 2 typeid-13 777 (null) 1 2 typeid-14 @testcase | 137 | 1 2 | 1 2 777 1 2 777 1 2 777 [ 1 2 ] -@testcase | 138 | | 1 (null) (null) (null) (null) 0 2 146 777 1 (null) 0 2 147 777 1 5 1 2 147 777 1 (null) 5 1 2 148 +@testcase | 138 | | 1 (null) (null) (null) (null) 0 2 typeid-18 777 1 (null) 0 2 typeid-19 777 1 5 1 2 typeid-19 777 1 (null) 5 1 2 typeid-20 @testcase | 139 | | 1 (null) (null) (null) (null) 0 2 777 1 (null) 0 2 777 1 5 1 2 777 1 (null) 5 1 2 @testcase | 140 | 5 | 5 42 -@testcase | 140 | 15 | 15 131 +@testcase | 140 | 15 | 15 typeid-3 @testcase | 140 | 90 | 90 49 @testcase | 141 | 7 8 | 7 7 1 8 8 1 @testcase | 141 | null null | (null) (null) 0 (null) (null) 0 @@ -842,9 +842,9 @@ fun main() { @testcase | 154 | | 100 1 @testcase | 155 | | 5 1 5 1 @testcase | 156 | 1 2 -1 | 2 44 2 2 44 2 2 44 2 44 2 2 2 -@testcase | 157 | 1 | (null) 0 (null) 149 777 1 2 129 777 149 149 777 0 0 -1 0 -1 0 0 777 -1 0 -@testcase | 157 | 0 | (null) 0 (null) 149 777 (null) (null) 149 777 0 0 777 0 0 -1 0 0 -1 0 777 0 -1 -@testcase | 158 | | (null) (null) 149 (null) (null) 149 (null) 0 (null) +@testcase | 157 | 1 | (null) 0 (null) typeid-21 777 1 2 typeid-1 777 typeid-21 typeid-21 777 0 0 -1 0 -1 0 0 777 -1 0 +@testcase | 157 | 0 | (null) 0 (null) typeid-21 777 (null) (null) typeid-21 777 0 0 777 0 0 -1 0 0 -1 0 777 0 -1 +@testcase | 158 | | (null) (null) typeid-21 (null) (null) typeid-21 (null) 0 (null) @testcase | 159 | 0 4 | 456 @testcase | 159 | null 0 | 123 @testcase | 160 | | 10 0 diff --git a/tolk-tester/tests/var-apply-tests.tolk b/tolk-tester/tests/var-apply-tests.tolk index 825b48b96..d1361bd0c 100644 --- a/tolk-tester/tests/var-apply-tests.tolk +++ b/tolk-tester/tests/var-apply-tests.tolk @@ -338,7 +338,7 @@ fun main() {} @testcase | 107 | | 65537 @testcase | 108 | | 4 @testcase | 109 | | 0 3 0 7 -@testcase | 110 | 5 | 5 10 100 100 100 134 +@testcase | 110 | 5 | 5 10 100 100 100 typeid-6 @testcase | 110 | 0 | (null) (null) (null) (null) (null) 0 @testcase | 111 | 2 3 9 | -1 @testcase | 111 | 11 22 44 | -1 @@ -348,7 +348,7 @@ fun main() {} @testcase | 112 | -1 -10 -20 | -1 @testcase | 113 | 0 | 12 12 @testcase | 113 | -1 | 12 -1 -@testcase | 114 | -1 | 2 3 131 +@testcase | 114 | -1 | 2 3 typeid-3 @testcase | 114 | 0 | (null) 12 1 @testcase | 115 | | 7 6 7 @testcase | 116 | | 7 11 80 diff --git a/tolk-tester/tolk-tester.js b/tolk-tester/tolk-tester.js index 8b7000724..dcd667e6e 100644 --- a/tolk-tester/tolk-tester.js +++ b/tolk-tester/tolk-tester.js @@ -130,9 +130,12 @@ class TolkTestCaseInputOutput { this.expected_output = output_str } - check(/**string[]*/ stdout_lines, /**number*/ line_idx) { - if (stdout_lines[line_idx] !== this.expected_output) - throw new CompareOutputError(`error on case #${line_idx + 1} (${this.method_id} | ${this.input}):\n expect: ${this.expected_output}\n actual: ${stdout_lines[line_idx]}`, stdout_lines.join("\n")) + check(/**string[]*/ stdout_lines, /**number*/ line_idx, /**number*/ pivot_typeid) { + let expected_str = this.expected_output + if (expected_str.includes("typeid")) + expected_str = expected_str.replace(/typeid-(\d+)/g, (match, p1) => pivot_typeid + (+p1)) + if (stdout_lines[line_idx] !== expected_str) + throw new CompareOutputError(`error on case #${line_idx + 1} (${this.method_id} | ${this.input}):\n expect: ${expected_str}\n actual: ${stdout_lines[line_idx]}`, stdout_lines.join("\n")) } } @@ -275,6 +278,8 @@ class TolkTestFile { this.experimental_options = null /** @type {boolean} */ this.enable_tolk_lines_comments = false + /** @type {number} */ + this.pivot_typeid = 138 // may be changed when stdlib introduces new union types } parse_input_from_tolk_file() { @@ -394,7 +399,7 @@ class TolkTestFile { throw new CompareOutputError(`unexpected number of fift output: ${stdout_lines.length} lines, but ${this.input_output.length} testcases`, stdout) for (let i = 0; i < stdout_lines.length; ++i) - this.input_output[i].check(stdout_lines, i) + this.input_output[i].check(stdout_lines, i, this.pivot_typeid) if (this.fif_codegen.length) { const fif_output = fs.readFileSync(this.get_compiled_fif_filename(), 'utf-8').split(/\r?\n/) diff --git a/tolk-tester/tolk-tester.py b/tolk-tester/tolk-tester.py index 126b07930..f39d2d3ab 100644 --- a/tolk-tester/tolk-tester.py +++ b/tolk-tester/tolk-tester.py @@ -127,9 +127,12 @@ def __init__(self, method_id_str: str, input_str: str, output_str: str): self.input = " ".join(processed_inputs) self.expected_output = output_str - def check(self, stdout_lines: List[str], line_idx: int): - if stdout_lines[line_idx] != self.expected_output: - raise CompareOutputError("error on case #%d (%d | %s):\n expect: %s\n actual: %s" % (line_idx + 1, self.method_id, self.input, self.expected_output, stdout_lines[line_idx]), "\n".join(stdout_lines)) + def check(self, stdout_lines: List[str], line_idx: int, pivot_typeid: int): + expected_str = self.expected_output + if expected_str.find("typeid") != -1: + expected_str = re.sub(r'typeid-(\d+)', lambda m: str(pivot_typeid + int(m.group(1))), expected_str) + if stdout_lines[line_idx] != expected_str: + raise CompareOutputError("error on case #%d (%d | %s):\n expect: %s\n actual: %s" % (line_idx + 1, self.method_id, self.input, expected_str, stdout_lines[line_idx]), "\n".join(stdout_lines)) class TolkTestCaseStderr: @@ -261,6 +264,7 @@ def __init__(self, tolk_filename: str, artifacts_folder: str): self.expected_hash: TolkTestCaseExpectedHash | None = None self.experimental_options: str | None = None self.enable_tolk_lines_comments = False + self.pivot_typeid = 138 # may be changed when stdlib introduces new union types def parse_input_from_tolk_file(self): with open(self.tolk_filename, "r") as fd: @@ -374,7 +378,7 @@ def run_and_check(self): raise CompareOutputError("unexpected number of fift output: %d lines, but %d testcases" % (len(stdout_lines), len(self.input_output)), stdout) for i in range(len(stdout_lines)): - self.input_output[i].check(stdout_lines, i) + self.input_output[i].check(stdout_lines, i, self.pivot_typeid) if len(self.fif_codegen): with open(self.get_compiled_fif_filename()) as fd: diff --git a/tolk/CMakeLists.txt b/tolk/CMakeLists.txt index 7aca83688..2e0bf0cad 100644 --- a/tolk/CMakeLists.txt +++ b/tolk/CMakeLists.txt @@ -8,6 +8,7 @@ set(TOLK_SOURCE constant-evaluator.cpp pack-unpack-api.cpp pack-unpack-serializers.cpp + send-message-api.cpp pipe-discover-parse-sources.cpp pipe-register-symbols.cpp pipe-resolve-identifiers.cpp diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp index 6a82b432f..3696d550e 100644 --- a/tolk/builtins.cpp +++ b/tolk/builtins.cpp @@ -1228,6 +1228,7 @@ void define_builtins() { TypePtr Bool = TypeDataBool::create(); TypePtr Slice = TypeDataSlice::create(); TypePtr Builder = TypeDataBuilder::create(); + TypePtr Address = TypeDataAddress::create(); TypePtr Tuple = TypeDataTuple::create(); TypePtr Never = TypeDataNever::create(); @@ -1246,6 +1247,11 @@ void define_builtins() { TypePtr CellT = TypeDataUnknown::create(); TypePtr PackOptions = TypeDataUnknown::create(); TypePtr UnpackOptions = TypeDataUnknown::create(); + TypePtr CreateMessageOptions = TypeDataUnknown::create(); + TypePtr createExternalLogMessageOptions = TypeDataUnknown::create(); + TypePtr OutMessage = TypeDataUnknown::create(); + TypePtr AddressShardingOptions = TypeDataUnknown::create(); + const GenericsDeclaration* declTBody = new GenericsDeclaration(std::vector{{"TBody", nullptr}}, 0); // builtin operators // they are internally stored as functions, because at IR level, there is no difference @@ -1449,6 +1455,9 @@ void define_builtins() { define_builtin_method("tuple.set", Tuple, {Tuple, typeT, Int}, Unit, declGenericT, compile_tuple_set_at, FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf); + define_builtin_method("address.buildSameAddressInAnotherShard", Address, {Address, AddressShardingOptions}, Builder, nullptr, + compile_time_only_function, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf | FunctionData::flagCompileTimeGen); define_builtin_method("debug.print", debug, {typeT}, Unit, declGenericT, compile_debug_print_to_string, FunctionData::flagAllowAnyWidthT); @@ -1495,6 +1504,13 @@ void define_builtins() { compile_time_only_function, FunctionData::flagMarkedAsPure | FunctionData::flagCompileTimeGen | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf | FunctionData::flagHasMutateParams | FunctionData::flagAllowAnyWidthT); + define_builtin_func("createMessage", {CreateMessageOptions}, OutMessage, declTBody, + compile_time_only_function, + FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); + define_builtin_func("createExternalLogMessage", {createExternalLogMessageOptions}, OutMessage, declTBody, + compile_time_only_function, + FunctionData::flagCompileTimeGen | FunctionData::flagAllowAnyWidthT); + // functions not presented in stdlib at all // used in tolk-tester to check/expose internal compiler state // each of them is handled in a special way, search by its name @@ -1519,6 +1535,11 @@ void patch_builtins_after_stdlib_loaded() { lookup_function("debug.printString")->mutate()->receiver_type = debug; lookup_function("debug.dumpStack")->mutate()->receiver_type = debug; + StructPtr struct_ref_AddressShardingOptions = lookup_global_symbol("AddressShardingOptions")->try_as(); + TypePtr AddressShardingOptions = TypeDataStruct::create(struct_ref_AddressShardingOptions); + + lookup_function("address.buildSameAddressInAnotherShard")->mutate()->parameters[1].declared_type = AddressShardingOptions; + StructPtr struct_ref_CellT = lookup_global_symbol("Cell")->try_as(); StructPtr struct_ref_PackOptions = lookup_global_symbol("PackOptions")->try_as(); StructPtr struct_ref_UnpackOptions = lookup_global_symbol("UnpackOptions")->try_as(); @@ -1552,6 +1573,18 @@ void patch_builtins_after_stdlib_loaded() { lookup_function("slice.skipAny")->mutate()->parameters[1].default_value = v_empty_UnpackOptions; lookup_function("builder.storeAny")->mutate()->parameters[2].declared_type = PackOptions; lookup_function("builder.storeAny")->mutate()->parameters[2].default_value = v_empty_PackOptions; + + StructPtr struct_ref_CreateMessageOptions = lookup_global_symbol("CreateMessageOptions")->try_as(); + StructPtr struct_ref_createExternalLogMessageOptions = lookup_global_symbol("createExternalLogMessageOptions")->try_as(); + StructPtr struct_ref_OutMessage = lookup_global_symbol("OutMessage")->try_as(); + TypePtr CreateMessageOptions = TypeDataGenericTypeWithTs::create(struct_ref_CreateMessageOptions, nullptr, {TypeDataGenericT::create("TBody")}); + TypePtr createExternalLogMessageOptions = TypeDataGenericTypeWithTs::create(struct_ref_createExternalLogMessageOptions, nullptr, {TypeDataGenericT::create("TBody")}); + TypePtr OutMessage = TypeDataStruct::create(struct_ref_OutMessage); + + lookup_function("createMessage")->mutate()->parameters[0].declared_type = CreateMessageOptions; + lookup_function("createMessage")->mutate()->declared_return_type = OutMessage; + lookup_function("createExternalLogMessage")->mutate()->parameters[0].declared_type = createExternalLogMessageOptions; + lookup_function("createExternalLogMessage")->mutate()->declared_return_type = OutMessage; } } // namespace tolk diff --git a/tolk/pack-unpack-api.cpp b/tolk/pack-unpack-api.cpp index 6af6650f4..227a07066 100644 --- a/tolk/pack-unpack-api.cpp +++ b/tolk/pack-unpack-api.cpp @@ -135,13 +135,19 @@ class PackUnpackAvailabilityChecker { return {}; } - // `builder` can be used for writing, but not for reading + // `builder` and `slice` can be used for writing, but not for reading if (any_type == TypeDataBuilder::create()) { if (is_pack) { return {}; } return CantSerializeBecause("because type `builder` can not be used for reading, only for writing\nhint: use `bitsN` or `RemainingBitsAndRefs` for reading\nhint: using generics, you can substitute `builder` for writing and something other for reading"); } + if (any_type == TypeDataSlice::create()) { + if (is_pack) { + return {}; + } + return CantSerializeBecause("because type `slice` can not be used for reading, it doesn't define binary width\nhint: replace `slice` with `address` if it's an address, actually\nhint: replace `slice` with `bits128` and similar if it represents fixed-width data without refs"); + } // serialization not available // for common types, make a detailed explanation with a hint how to fix @@ -149,9 +155,6 @@ class PackUnpackAvailabilityChecker { if (any_type == TypeDataInt::create()) { return CantSerializeBecause("because type `int` is not serializable, it doesn't define binary width\nhint: replace `int` with `int32` / `uint64` / `coins` / etc."); } - if (any_type == TypeDataSlice::create()) { - return CantSerializeBecause("because type `slice` is not serializable, it doesn't define binary width\nhint: replace `slice` with `address` if it's an address, actually\nhint: replace `slice` with `bits128` and similar if it represents fixed-width data without refs"); - } if (any_type == TypeDataNullLiteral::create()) { return CantSerializeBecause("because type `null` is not serializable\nhint: `int32?` and other nullable types will work"); } diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index c0490c5c7..0ffce5fa2 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -77,6 +77,11 @@ void PackContext::storeUint(var_idx_t ir_idx, int len) const { code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), f_storeUint); } +void PackContext::storeUint_var(var_idx_t ir_idx, var_idx_t ir_len) const { + std::vector args = { ir_builder0, ir_idx, ir_len }; + code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), f_storeUint); +} + void PackContext::storeBool(var_idx_t ir_idx) const { std::vector args = { ir_builder0, ir_idx }; code.emplace_back(loc, Op::_Call, ir_builder, std::move(args), lookup_function("builder.storeBool")); @@ -454,6 +459,25 @@ struct S_Builder final : ISerializer { } }; +struct S_Slice final : ISerializer { + void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { + tolk_assert(rvect.size() == 1); + ctx->storeSlice(rvect[0]); + } + + std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + tolk_assert(false); // `slice` can only be used for writing, checked earlier + } + + void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { + tolk_assert(false); // `slice` can only be used for writing, checked earlier + } + + PackSize estimate(const EstimateContext* ctx) override { + return PackSize::unpredictable_infinity(); + } +}; + struct S_Null final : ISerializer { void pack(const PackContext* ctx, CodeBlob& code, SrcLocation loc, std::vector&& rvect) override { // while `null` itself is not serializable, it may be contained inside a union: @@ -941,6 +965,9 @@ static std::unique_ptr get_serializer_for_type(TypePtr any_type) { if (any_type == TypeDataBuilder::create()) { return std::make_unique(); } + if (any_type == TypeDataSlice::create()) { + return std::make_unique(); + } if (any_type == TypeDataNullLiteral::create()) { return std::make_unique(); } diff --git a/tolk/pack-unpack-serializers.h b/tolk/pack-unpack-serializers.h index c2c99ea2a..06163153b 100644 --- a/tolk/pack-unpack-serializers.h +++ b/tolk/pack-unpack-serializers.h @@ -72,6 +72,7 @@ class PackContext { void storeInt(var_idx_t ir_idx, int len) const; void storeUint(var_idx_t ir_idx, int len) const; + void storeUint_var(var_idx_t ir_idx, var_idx_t ir_len) const; void storeBool(var_idx_t ir_idx) const; void storeCoins(var_idx_t ir_idx) const; void storeRef(var_idx_t ir_idx) const; diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp index 1f5025918..8a6c2911b 100644 --- a/tolk/pipe-ast-to-legacy.cpp +++ b/tolk/pipe-ast-to-legacy.cpp @@ -24,7 +24,7 @@ #include "smart-casts-cfg.h" #include "pack-unpack-api.h" #include "generics-helpers.h" -#include +#include "send-message-api.h" /* * This pipe is the last one operating AST: it transforms AST to IR. @@ -589,6 +589,21 @@ static std::vector gen_compile_time_code_instead_of_fun_call(CodeBlob if (f_name == "T.estimatePackSize") { return generate_estimate_size_call(code, loc, typeT); } + + if (f_name == "createMessage") { + std::vector ir_msg_params = vars_per_arg[0]; + return generate_createMessage(code, loc, typeT->unwrap_alias(), std::move(ir_msg_params)); + } + if (f_name == "createExternalLogMessage") { + std::vector ir_msg_params = vars_per_arg[0]; + return generate_createExternalLogMessage(code, loc, typeT->unwrap_alias(), std::move(ir_msg_params)); + } + } + + if (called_f->name == "address.buildSameAddressInAnotherShard") { + std::vector ir_self_address = vars_per_arg[0]; + std::vector ir_shard_options = vars_per_arg[1]; + return generate_address_buildInAnotherShard(code, loc, std::move(ir_self_address), std::move(ir_shard_options)); } tolk_assert(false); diff --git a/tolk/pipe-check-serialized-fields.cpp b/tolk/pipe-check-serialized-fields.cpp index 3ed8fc33a..8f436c957 100644 --- a/tolk/pipe-check-serialized-fields.cpp +++ b/tolk/pipe-check-serialized-fields.cpp @@ -58,6 +58,11 @@ class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody static void check_struct_fits_cell_or_has_policy(const TypeDataStruct* t_struct) { StructPtr struct_ref = t_struct->struct_ref; + bool avoid_check = struct_ref->is_instantiation_of_generic_struct() && struct_ref->base_struct_ref->name == "UnsafeBodyNoRef"; + if (avoid_check) { + return; + } + PackSize size = estimate_serialization_size(t_struct); if (size.max_bits > 1023 && !size.is_unpredictable_infinity()) { if (struct_ref->overflow1023_policy == StructData::Overflow1023Policy::not_specified) { @@ -82,16 +87,18 @@ class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody TypePtr serialized_type = nullptr; bool is_pack = false; if (f_name == "Cell.load" || f_name == "T.fromSlice" || f_name == "T.fromCell" || f_name == "T.toCell" || - f_name == "T.loadAny" || f_name == "slice.skipAny" || f_name == "slice.storeAny" || f_name == "T.estimatePackSize") { + f_name == "T.loadAny" || f_name == "slice.skipAny" || f_name == "slice.storeAny" || f_name == "T.estimatePackSize" || + f_name == "createMessage" || f_name == "createExternalLogMessage") { serialized_type = fun_ref->substitutedTs->typeT_at(0); - is_pack = f_name == "T.toCell" || f_name == "slice.storeAny" || f_name == "T.estimatePackSize"; + is_pack = f_name == "T.toCell" || f_name == "slice.storeAny" || f_name == "T.estimatePackSize" || f_name == "createMessage" || f_name == "createExternalLogMessage"; } else { return; // not a serialization function } std::string because_msg; if (!check_struct_can_be_packed_or_unpacked(serialized_type, is_pack, because_msg)) { - fire(cur_f, v->loc, "auto-serialization via " + fun_ref->method_name + "() is not available for type `" + serialized_type->as_human_readable() + "`\n" + because_msg); + std::string via_name = fun_ref->is_method() ? fun_ref->method_name : fun_ref->base_fun_ref->name; + fire(cur_f, v->loc, "auto-serialization via " + via_name + "() is not available for type `" + serialized_type->as_human_readable() + "`\n" + because_msg); } check_type_fits_cell_or_has_policy(serialized_type); diff --git a/tolk/send-message-api.cpp b/tolk/send-message-api.cpp new file mode 100644 index 000000000..4fd8ab846 --- /dev/null +++ b/tolk/send-message-api.cpp @@ -0,0 +1,496 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "send-message-api.h" +#include "pack-unpack-serializers.h" +#include "type-system.h" + +namespace tolk { + +std::vector pre_compile_is_type(CodeBlob& code, TypePtr expr_type, TypePtr cmp_type, const std::vector& expr_ir_idx, SrcLocation loc, const char* debug_desc); +std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc); + +static bool is_type_UnsafeBodyNoRef_T(TypePtr bodyT) { + if (const TypeDataStruct* t_struct = bodyT->unwrap_alias()->try_as()) { + if (t_struct->struct_ref->is_instantiation_of_generic_struct() && t_struct->struct_ref->base_struct_ref->name == "UnsafeBodyNoRef") { + return true; + } + } + return false; +} + +// currently, there is no way to pass custom pack options to createMessage, using hardcoded ones +static std::vector create_default_PackOptions(CodeBlob& code, SrcLocation loc) { + StructPtr s_PackOptions = lookup_global_symbol("PackOptions")->try_as(); + std::vector ir_options = code.create_tmp_var(TypeDataStruct::create(s_PackOptions), loc, "(pack-options)"); + tolk_assert(ir_options.size() == 1); + + var_idx_t ir_zero = code.create_int(loc, 0, "(zero)"); + code.emplace_back(loc, Op::_Let, std::vector{ir_options[0]}, std::vector{ir_zero}); // skipBitsNFieldsValidation + return ir_options; +} + +std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, TypePtr bodyT, std::vector&& rvect) { + StructPtr s_Options = lookup_global_symbol("CreateMessageOptions")->try_as(); + StructPtr s_AutoDeployAddress = lookup_global_symbol("AutoDeployAddress")->try_as(); + + const TypeDataBool* t_bounce = s_Options->find_field("bounce")->declared_type->try_as(); + const TypeDataUnion* t_dest = s_Options->find_field("dest")->declared_type->try_as(); + const TypeDataUnion* t_value = s_Options->find_field("value")->declared_type->try_as(); + const TypeDataUnion* t_stateInit = s_AutoDeployAddress->find_field("stateInit")->declared_type->try_as(); + const TypeDataUnion* t_toShard = s_AutoDeployAddress->find_field("toShard")->declared_type->try_as(); + tolk_assert(t_bounce); + tolk_assert(t_dest && t_dest->get_width_on_stack() == (1+3+3+1) && t_dest->size() == 4); + tolk_assert(t_value && t_value->get_width_on_stack() == (2+1) && t_value->size() == 2); + tolk_assert(t_stateInit && t_stateInit->get_width_on_stack() == (2+1) && t_stateInit->size() == 2); + tolk_assert(t_toShard && t_toShard->get_width_on_stack() == (2+1) && t_toShard->or_null); + + int offset = 0; + auto next_slice = [&rvect, &offset](int width) -> std::vector { + int start = offset; + offset += width; + return std::vector(rvect.begin() + start, rvect.begin() + start + width); + }; + + std::vector ir_bounce = next_slice(t_bounce->get_width_on_stack()); + std::vector ir_value = next_slice(t_value->get_width_on_stack()); + std::vector ir_dest = next_slice(t_dest->get_width_on_stack()); + std::vector ir_body = next_slice(bodyT->get_width_on_stack()); + tolk_assert(offset == static_cast(rvect.size())); + + // field `dest` is `dest: address | AutoDeployAddress | (int8, uint256) | builder`; + // struct AutoDeployAddress { workchain: int8; stateInit: ContractState | cell; toShard: AddressShardingOptions?; } + // struct ContractState { code: cell; data: cell; } + // struct AddressShardingOptions { fixedPrefixLength: uint5; closeTo: address; } + std::vector ir_dest_is_address = pre_compile_is_type(code, t_dest, TypeDataAddress::create(), ir_dest, loc, "(is-address)"); + std::vector ir_dest_is_AutoDeploy = pre_compile_is_type(code, t_dest, TypeDataStruct::create(s_AutoDeployAddress), ir_dest, loc, "(is-address)"); + std::vector ir_dest_is_builder = pre_compile_is_type(code, t_dest, TypeDataBuilder::create(), ir_dest, loc, "(is-builder)"); + std::vector ir_dest_AutoDeployAddress = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataStruct::create(s_AutoDeployAddress), loc); + // dest.workchain + std::vector ir_dest_workchain(ir_dest_AutoDeployAddress.begin(), ir_dest_AutoDeployAddress.begin() + 1); + // dest.stateInit (code+data or cell) + std::vector ir_dest_stateInitUnion(ir_dest_AutoDeployAddress.begin() + 1, ir_dest_AutoDeployAddress.begin() + 1 + 3); + std::vector ir_is_ContractState = pre_compile_is_type(code, t_stateInit, t_stateInit->variants[0], ir_dest_stateInitUnion, loc, "(is-ContractState)"); + std::vector ir_ContractState = transition_to_target_type(std::vector(ir_dest_stateInitUnion), code, t_stateInit, t_stateInit->variants[0], loc); + std::vector ir_StateInitCell = transition_to_target_type(std::vector(ir_dest_stateInitUnion), code, t_stateInit, t_stateInit->variants[1], loc); + // dest.toShard (shardPrefix+closeTo or null) + std::vector ir_dest_toShardOrNull(ir_dest_AutoDeployAddress.begin() + 1 + 3, ir_dest_AutoDeployAddress.begin() + 1 + 3 + 3); + std::vector ir_is_AddressSharding = pre_compile_is_type(code, t_toShard, t_toShard->or_null, ir_dest_toShardOrNull, loc, "(is-AddressSharding)"); + std::vector ir_AddressSharding = transition_to_target_type(std::vector(ir_dest_toShardOrNull), code, t_toShard, t_toShard->or_null, loc); + + // currently, there is no way to pass PackOptions, defaults are used + std::vector ir_options = create_default_PackOptions(code, loc); + + FunctionPtr f_beginCell = lookup_function("beginCell"); + FunctionPtr f_endCell = lookup_function("builder.endCell"); + + // detect whether to store `body: (Either X ^X)` inline or as ref + // if it's small (guaranteed to fit), store it inside the same builder, without creating a cell + PackSize body_size = EstimateContext().estimate_any(bodyT); + // if `body` is already `cell` / `Cell` + bool body_already_ref = bodyT == TypeDataCell::create() || is_type_cellT(bodyT); + // if `body` is `UnsafeBodyNoRef` + bool body_force_no_ref = is_type_UnsafeBodyNoRef_T(bodyT); + // max size of all fields before body = 514 (502 CommonMsgInfoRelaxed + 12 StateInit), so 500 bits will fit + bool body_100p_fits_no_ref = body_size.max_bits <= 500 && body_size.max_refs < 2; + // final decision: 1 (^X) or 0 (X) + bool body_store_as_ref = body_already_ref || (!body_100p_fits_no_ref && !body_force_no_ref); + + // if we need to store body ref, convert it to a cell here, before creating a builder for the message; + // it's more optimal, since the `body` field is the topmost at the stack + if (body_store_as_ref && !body_already_ref) { + std::vector ir_ref_builder = code.create_var(TypeDataBuilder::create(), loc, "refb"); + code.emplace_back(loc, Op::_Call, ir_ref_builder, std::vector{}, f_beginCell); + PackContext ref_ctx(code, loc, ir_ref_builder, ir_options); + ref_ctx.generate_pack_any(bodyT, std::move(ir_body)); + std::vector ir_ref_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(ref-cell)"); + code.emplace_back(loc, Op::_Call, ir_ref_cell, std::move(ir_ref_builder), f_endCell); + ir_body = std::move(ir_ref_cell); + } + + std::vector ir_builder = code.create_var(TypeDataSlice::create(), loc, "b"); + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, f_beginCell); + PackContext ctx(code, loc, ir_builder, ir_options); + var_idx_t ir_zero = code.create_int(loc, 0, "(zero)"); + var_idx_t ir_one = code.create_int(loc, 1, "(one)"); + + // '0' prefix int_msg_info + ctx.storeUint(ir_zero, 1); + // fill `ihr_disabled:Bool` always 1 + ctx.storeUint(ir_one, 1); + // fill `bounce:Bool` from p.bounce (if it's constant (most likely), it will be concatenated with prev and next) + ctx.storeBool(ir_bounce[0]); + // fill `bounced:Bool` + `src:MsgAddress` 00 + ctx.storeUint(ir_zero, 1 + 2); + + // fill `dest:MsgAddressInt` from p.dest (complex union) + Op& if_address = code.emplace_back(loc, Op::_If, ir_dest_is_address); + { + // input is `dest: someAddress` + code.push_set_cur(if_address.block0); + std::vector ir_dest_address = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataAddress::create(), loc); + ctx.storeAddress(ir_dest_address[0]); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_address.block1); + Op& if_AutoDeploy = code.emplace_back(loc, Op::_If, ir_dest_is_AutoDeploy); + { + // input is `dest: { workchain, stateInit, [toShard] }`; + // then calculate hash equal to StateInit cell would be and fill "addr_std$10 + 0 anycast + workchain + hash"; + // and, if toShard, take first D bits from dest.toShard.closeTo and mix with 256-D bits of hash + code.push_set_cur(if_AutoDeploy.block0); + ctx.storeUint(code.create_int(loc, 0b100, "(addr-prefix)"), 3); // addr_std$10 + 0 anycast + ctx.storeInt(ir_dest_workchain[0], 8); + std::vector ir_hash = code.create_tmp_var(TypeDataInt::create(), loc, "(addr-hash)"); + Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_is_ContractState); + { + // input is `dest: { ... stateInit: { code, data } }` + code.push_set_cur(if_ContractState.block0); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_is_AddressSharding); + { + // input is `dest: { ... stateInit: { code, data }, toShard: { fixedPrefixLength, closeTo } }; + // then stateInitHash = (hash of StateInit = 0b1(depth)0110 (prefix + code + data)) + code.push_set_cur(if_sharded.block0); + std::vector ir_args_depth_code_data = { ir_AddressSharding[0], ir_ContractState[0], ir_ContractState[1] }; + code.emplace_back(loc, Op::_Call, ir_hash, std::move(ir_args_depth_code_data), lookup_function("StateInit.calcHashPrefixCodeData")); + code.close_pop_cur(loc); + } + { + // input is: `dest: { ... stateInit: { code, data } }` (toShard is null); + // then hash = (hash of StateInit = 0b00110 (only code + data)) + code.push_set_cur(if_sharded.block1); + code.emplace_back(loc, Op::_Call, ir_hash, ir_ContractState, lookup_function("StateInit.calcHashCodeData")); + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + { + // input is `dest: { ... stateInit: cell }` + code.push_set_cur(if_ContractState.block1); + code.emplace_back(loc, Op::_Call, ir_hash, ir_StateInitCell, lookup_function("cell.hash")); + code.close_pop_cur(loc); + } + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_is_AddressSharding); + { + // input is `dest: { ... toShard: { fixedPrefixLength, closeTo } }` + // we already calculated stateInitHash (ir_hash): either cell.hash() or based on prefix+code+data; + // now, we need: hash = (first D bits from dest.toShard.closeTo) + (last 256-D bits from stateInitHash); + // example for fixedPrefixLength (shard depth) = 8: + // | closeTo | 01010101...xxx | given as input, by user (it's address, internally slice) + // | shardPrefix | 01010101 | first 8 bits of closeTo + // | stateInitHash | yyyyyyyy...yyy | mask = (1 << (256-D)) - 1 = 00000000111...111 (8 zeroes) + // | hash (result) | 01010101...yyy | + // remember, that closeTo is addr_std$10 + 0 + workchain + xxx...xxx, so skip 11 bits and read 8 + code.push_set_cur(if_sharded.block0); + std::vector ir_shardPrefix = code.create_tmp_var(TypeDataSlice::create(), loc, "(shardPrefix)"); + std::vector args_subslice = { ir_AddressSharding[1], code.create_int(loc, 3+8, ""), ir_AddressSharding[0] }; + code.emplace_back(loc, Op::_Call, ir_shardPrefix, std::move(args_subslice), lookup_function("slice.getMiddleBits")); + ctx.storeSlice(ir_shardPrefix[0]); // first 8 bits of closeTo hash + std::vector ir_mask = code.create_tmp_var(TypeDataInt::create(), loc, "(mask)"); + code.emplace_back(loc, Op::_Call, ir_mask, std::vector{code.create_int(loc, 256, ""), ir_AddressSharding[0]}, lookup_function("_-_")); + code.emplace_back(loc, Op::_Call, ir_mask, std::vector{ir_one, ir_mask[0]}, lookup_function("_<<_")); + code.emplace_back(loc, Op::_Call, ir_mask, std::vector{ir_mask[0], ir_one}, lookup_function("_-_")); + code.emplace_back(loc, Op::_Call, ir_hash, std::vector{ir_hash[0], ir_mask[0]}, lookup_function("_&_")); + code.emplace_back(loc, Op::_Call, ir_mask, std::vector{code.create_int(loc, 256, ""), ir_AddressSharding[0]}, lookup_function("_-_")); + ctx.storeUint_var(ir_hash[0], ir_mask[0]); // 248 STU (stateInitHash & mask) + code.close_pop_cur(loc); + } + { + // input is `dest: { workchain, stateInit }` (toShard is null); + // we already calculated stateInitHash: either cell.hash() or based on code+data + code.push_set_cur(if_sharded.block1); + ctx.storeUint(ir_hash[0], 256); + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_AutoDeploy.block1); + Op& if_builder = code.emplace_back(loc, Op::_If, ir_dest_is_builder); + { + // input is `dest: someBuilder` + code.push_set_cur(if_builder.block0); + std::vector ir_dest_builder = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataBuilder::create(), loc); + ctx.storeBuilder(ir_dest_builder[0]); + code.close_pop_cur(loc); + } + { + // input is `dest: (workchain, hash)` + code.push_set_cur(if_builder.block1); + std::vector ir_dest_wc_hash = transition_to_target_type(std::vector(ir_dest), code, t_dest, t_dest->variants[2], loc); + ctx.storeUint(code.create_int(loc, 0b100, "(addr-prefix)"), 3); + ctx.storeInt(ir_dest_wc_hash[0], 8); // most likely, it's 0 (basechain), will be merged with above + ctx.storeUint(ir_dest_wc_hash[1], 256); + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + + // fill `value:CurrencyCollection` from p.value `coins | (coins, dict)` + std::vector ir_is_coins = pre_compile_is_type(code, t_value, TypeDataCoins::create(), ir_value, loc, "(is-coins)"); + Op& if_coins = code.emplace_back(loc, Op::_If, ir_is_coins); + { + code.push_set_cur(if_coins.block0); + std::vector ir_coins = transition_to_target_type(std::vector(ir_value), code, t_value, TypeDataCoins::create(), loc); + ctx.storeCoins(ir_coins[0]); + ctx.storeUint(ir_zero, 1); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_coins.block1); + std::vector ir_coins_dict = transition_to_target_type(std::move(ir_value), code, t_value, t_value->variants[1], loc); + ctx.storeCoins(ir_coins_dict[0]); + ctx.storeMaybeRef(ir_coins_dict[1]); + code.close_pop_cur(loc); + } + + // tail of CommonMsgInfoRelaxed: 4*0 ihr_fee + 4*0 fwd_fee + 64*0 created_lt + 32*0 created_at + ctx.storeUint(ir_zero, 4 + 4 + 64 + 32); + + // fill `init: (Maybe (Either StateInit ^StateInit))` + // it's present only if p.dest contains StateInit + // also fill the either bit of `body: (Either X ^X)` + Op& if_no_init = code.emplace_back(loc, Op::_If, ir_dest_is_AutoDeploy); + { + // when it's known at compile-time (always in practice), this `if` is simplified, and bits join with above + code.push_set_cur(if_no_init.block1); + ctx.storeUint(body_store_as_ref ? ir_one : ir_zero, 1 + 1); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_no_init.block0); + Op& if_ContractState = code.emplace_back(loc, Op::_If, ir_is_ContractState); + { + // input is `dest: { ... stateInit: { code, data } }` and need to compose TL/B StateInit; + // it's either just code+data OR (if `toShard: { ... }` is set) fixedPrefixLength+code+data + code.push_set_cur(if_ContractState.block0); + Op& if_sharded = code.emplace_back(loc, Op::_If, ir_is_AddressSharding); + { + // 1 (maybe true) + 0 (either left) + 1 (maybe true of StateInit) + fixedPrefixLength + 0110 + body ref or not + code.push_set_cur(if_sharded.block0); + ctx.storeUint(code.create_int(loc, 0b101, ""), 1 + 1 + 1); + ctx.storeUint(ir_AddressSharding[0], 5); // fixedPrefixLength (shard depth) + ctx.storeUint(code.create_int(loc, 0b01100 + body_store_as_ref, ""), 4 + 1); + code.close_pop_cur(loc); + // also, we used dest.toShard to fill CommonMsgInfoRelaxed.dest.address (with a mask for stateInitHash, see above) + } + { + // 1 (maybe true) + 0 (either left) + 00110 (only code and data from StateInit) + body ref or not + code.push_set_cur(if_sharded.block1); + var_idx_t ir_rest_bits = code.create_int(loc, 0b10001100 + body_store_as_ref, "(rest-bits)"); + ctx.storeUint(ir_rest_bits, 1 + 1 + 5 + 1); + code.close_pop_cur(loc); + } + ctx.storeRef(ir_ContractState[0]); // dest.stateInit.code + ctx.storeRef(ir_ContractState[1]); // dest.stateInit.data + code.close_pop_cur(loc); + } + { + // so, we have `dest: { stateInit: someCell }`, store it as ref + // 1 (maybe true) + 1 (either right) + body ref or not + code.push_set_cur(if_ContractState.block1); + var_idx_t ir_rest_bits = code.create_int(loc, 0b110 + body_store_as_ref, "(rest-bits)"); + ctx.storeUint(ir_rest_bits, 1 + 1 + 1); + ctx.storeRef(ir_StateInitCell[0]); + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + + // store body; previously, we've calculated whether to store is as a ref or not + if (body_size.max_bits == 0 && body_size.max_refs == 0) { + tolk_assert(ir_body.empty()); + } else if (body_store_as_ref) { + tolk_assert(ir_body.size() == 1); // it was either an input cell or a automatically created one + ctx.storeRef(ir_body[0]); + } else { + ctx.generate_pack_any(bodyT, std::move(ir_body)); + } + + std::vector ir_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(msg-cell)"); + code.emplace_back(loc, Op::_Call, ir_cell, std::move(ir_builder), f_endCell); + return ir_cell; +} + +std::vector generate_createExternalLogMessage(CodeBlob& code, SrcLocation loc, TypePtr bodyT, std::vector&& rvect) { + StructPtr s_Options = lookup_global_symbol("createExternalLogMessageOptions")->try_as(); + StructPtr s_ExtOutLogBucket = lookup_global_symbol("ExtOutLogBucket")->try_as(); + + const TypeDataUnion* t_dest = s_Options->find_field("dest")->declared_type->try_as(); + const TypeDataUnion* t_topic = s_ExtOutLogBucket->find_field("topic")->declared_type->try_as(); + tolk_assert(t_dest && t_dest->get_width_on_stack() == (2+1) && t_dest->size() == 3); + tolk_assert(t_topic && t_topic->get_width_on_stack() == (1+1) && t_topic->size() == 2); + + int offset = 0; + auto next_slice = [&rvect, &offset](int width) -> std::vector { + int start = offset; + offset += width; + return std::vector(rvect.begin() + start, rvect.begin() + start + width); + }; + + std::vector ir_dest = next_slice(t_dest->get_width_on_stack()); + std::vector ir_body = next_slice(bodyT->get_width_on_stack()); + tolk_assert(offset == static_cast(rvect.size())); + + // field `dest` is `dest: address | builder | ExtOutLogBucket`; + // struct ExtOutLogBucket { topic: uint248 | bits248; } + std::vector ir_dest_is_address = pre_compile_is_type(code, t_dest, TypeDataAddress::create(), ir_dest, loc, "(is-address)"); + std::vector ir_dest_is_builder = pre_compile_is_type(code, t_dest, TypeDataBuilder::create(), ir_dest, loc, "(is-builder)"); + std::vector ir_dest_ExtOutLogBucket = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataStruct::create(s_ExtOutLogBucket), loc); + // dest.topic (it's the only field in a struct) + std::vector ir_dest_topic = std::move(ir_dest_ExtOutLogBucket); + + std::vector ir_options = create_default_PackOptions(code, loc); + + FunctionPtr f_beginCell = lookup_function("beginCell"); + FunctionPtr f_endCell = lookup_function("builder.endCell"); + + // detect whether to store `body: (Either X ^X)` inline or as ref + // if it's small (guaranteed to fit), store it inside the same builder, without creating a cell + PackSize body_size = EstimateContext().estimate_any(bodyT); + // if `body` is already `cell` / `Cell` + bool body_already_ref = bodyT == TypeDataCell::create() || is_type_cellT(bodyT); + // if `body` is `UnsafeBodyNoRef` + bool body_force_no_ref = is_type_UnsafeBodyNoRef_T(bodyT); + // max size of all fields before body = 622 (621 CommonMsgInfoRelaxed + 1 StateInit), so 400 bits will fit + bool body_100p_fits_no_ref = body_size.max_bits < 400; + // final decision: 1 (^X) or 0 (X) + bool body_store_as_ref = body_already_ref || (!body_100p_fits_no_ref && !body_force_no_ref); + + // same as for createMessage: `body` field is the topmost at the stack, convert it to a cell before creating a builder + if (body_store_as_ref && !body_already_ref) { + std::vector ir_ref_builder = code.create_var(TypeDataBuilder::create(), loc, "refb"); + code.emplace_back(loc, Op::_Call, ir_ref_builder, std::vector{}, f_beginCell); + PackContext ref_ctx(code, loc, ir_ref_builder, ir_options); + ref_ctx.generate_pack_any(bodyT, std::move(ir_body)); + std::vector ir_ref_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(ref-cell)"); + code.emplace_back(loc, Op::_Call, ir_ref_cell, std::move(ir_ref_builder), f_endCell); + ir_body = std::move(ir_ref_cell); + } + + std::vector ir_builder = code.create_var(TypeDataBuilder::create(), loc, "b"); + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, f_beginCell); + PackContext ctx(code, loc, ir_builder, ir_options); + var_idx_t ir_zero = code.create_int(loc, 0, "(zero)"); + var_idx_t ir_one = code.create_int(loc, 1, "(one)"); + + // '11' prefix ext_out_msg_info + '00' src + ctx.storeUint(code.create_int(loc, 0b1100, "(out-prefix)"), 4); + + // fill `dest:MsgAddressExt` from p.dest (complex union) + Op& if_address = code.emplace_back(loc, Op::_If, ir_dest_is_address); + { + // input is `dest: someAddress` + code.push_set_cur(if_address.block0); + std::vector ir_dest_address = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataAddress::create(), loc); + ctx.storeAddress(ir_dest_address[0]); + code.close_pop_cur(loc); + } + { + code.push_set_cur(if_address.block1); + Op& if_builder = code.emplace_back(loc, Op::_If, ir_dest_is_builder); + { + // input is `dest: someBuilder` + code.push_set_cur(if_builder.block0); + std::vector ir_dest_builder = transition_to_target_type(std::vector(ir_dest), code, t_dest, TypeDataBuilder::create(), loc); + ctx.storeBuilder(ir_dest_builder[0]); + code.close_pop_cur(loc); + } + { + // input is `dest: ExtOutLogBucket`; + // fill addr_extern$01 + 256 (len 9 bit) + 0x00 (prefix) + 248 bits + code.push_set_cur(if_builder.block1); + ctx.storeUint(ir_one, 2); // addr_extern$01 + ctx.storeUint(code.create_int(loc, 256, "(addr-len)"), 9); // len:(## 9) = 256 + ctx.storeOpcode(s_ExtOutLogBucket->opcode); + std::vector ir_if_topic_uint = pre_compile_is_type(code, t_topic, t_topic->variants[0], ir_dest_topic, loc, "(topic-is-uint)"); + Op& if_topic_uint = code.emplace_back(loc, Op::_If, ir_if_topic_uint); + { + // input is `dest: ExtOutLogBucket { topic: uint248 }` + code.push_set_cur(if_topic_uint.block0); + ctx.storeUint(ir_dest_topic[0], 248); + code.close_pop_cur(loc); + } + { + // input is `dest: ExtOutLogBucket { topic: bits248 }` + // for this field, generate runtime check to ensure its length + code.push_set_cur(if_topic_uint.block1); + ctx.generate_pack_any(t_topic->variants[1], std::vector{ir_dest_topic[0]}); + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + code.close_pop_cur(loc); + } + + // tail of CommonMsgInfoRelaxed: 64*0 created_lt + 32*0 created_at + // plus, StateInit is empty (0 maybe bit) for external messages + ctx.storeUint(ir_zero, 64 + 32 + 1); + + // fill bit `body: (Either X ^X)` and store body + if (body_size.max_bits == 0 && body_size.max_refs == 0) { + // missing body of type `never` + tolk_assert(ir_body.empty()); + ctx.storeUint(ir_zero, 1); + } else if (body_store_as_ref) { + tolk_assert(ir_body.size() == 1); + ctx.storeUint(ir_one, 1); + ctx.storeRef(ir_body[0]); + } else { + ctx.storeUint(ir_zero, 1); + ctx.generate_pack_any(bodyT, std::move(ir_body)); + } + + std::vector ir_cell = code.create_tmp_var(TypeDataCell::create(), loc, "(msg-cell)"); + code.emplace_back(loc, Op::_Call, ir_cell, std::move(ir_builder), f_endCell); + return ir_cell; +} + +std::vector generate_address_buildInAnotherShard(CodeBlob& code, SrcLocation loc, std::vector&& ir_self_address, std::vector&& ir_shard_options) { + tolk_assert(ir_shard_options.size() == 2); + + // example for fixedPrefixLength (shard depth) = 8: + // | self (A) | aaaaaaaaaaa...aaa | + // | closeTo (B) | 01010101bbb...bbb | shardPrefix = 01010101 (depth 8) + // | result | 01010101aaa...aaa | address of A in same shard as B + + // the most effective way is not to calculate shardPrefix, but to: + // - take first 3+8+D bits of B: we'll have '100' (std addr no anycast) + workchainB + shardPrefix + // - take last 256-D bits of A: we'll have "aa...a" + // - concatenate: we'll result in '100' + workchainB + "bbaa...a" + + std::vector ir_offsetB = {code.create_int(loc, 3 + 8, "(offset-addrB)")}; + code.emplace_back(loc, Op::_Call, ir_offsetB, std::vector{ir_offsetB[0], ir_shard_options[0]}, lookup_function("_+_")); + std::vector ir_headB = code.create_tmp_var(TypeDataSlice::create(), loc, "(headB)"); + code.emplace_back(loc, Op::_Call, ir_headB, std::vector{ir_shard_options[1], ir_offsetB[0]}, lookup_function("slice.getFirstBits")); + + std::vector ir_builder = code.create_var(TypeDataBuilder::create(), loc, "b"); + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{}, lookup_function("beginCell")); + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{ir_builder[0], ir_headB[0]}, lookup_function("builder.storeSlice")); + + std::vector ir_restLenA = {code.create_int(loc, 256, "(last-addrA)")}; + code.emplace_back(loc, Op::_Call, ir_restLenA, std::vector{ir_restLenA[0], ir_shard_options[0]}, lookup_function("_-_")); + std::vector ir_tailA = code.create_tmp_var(TypeDataSlice::create(), loc, "(tailA)"); + code.emplace_back(loc, Op::_Call, ir_tailA, std::vector{ir_self_address[0], ir_restLenA[0]}, lookup_function("slice.getLastBits")); + code.emplace_back(loc, Op::_Call, ir_builder, std::vector{ir_builder[0], ir_tailA[0]}, lookup_function("builder.storeSlice")); + + return ir_builder; +} + +} // namespace tolk diff --git a/tolk/send-message-api.h b/tolk/send-message-api.h new file mode 100644 index 000000000..1af7389d4 --- /dev/null +++ b/tolk/send-message-api.h @@ -0,0 +1,28 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "tolk.h" + +namespace tolk { + +std::vector generate_createMessage(CodeBlob& code, SrcLocation loc, TypePtr bodyT, std::vector&& rvect); +std::vector generate_createExternalLogMessage(CodeBlob& code, SrcLocation loc, TypePtr bodyT, std::vector&& rvect); + +std::vector generate_address_buildInAnotherShard(CodeBlob& code, SrcLocation loc, std::vector&& ir_self_address, std::vector&& ir_shard_options); + +} // namespace tolk From c8a5b1ba8d72943c4ace58910ce7660135caeba8 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Tue, 10 Jun 2025 18:24:07 +0700 Subject: [PATCH 287/388] [Tolk] Fire an error on using `++/--` operators Before, `++i` was valid (and did nothing, --1 is 1), probably unexpected --- .../tests/invalid-syntax/err-3498.tolk | 12 ++++++++++ .../tests/invalid-syntax/err-3810.tolk | 14 +++++++++++ tolk-tester/tests/no-spaces.tolk | 14 +++++------ tolk-tester/tests/numbers-tests.tolk | 2 +- tolk/ast-from-tokens.cpp | 24 ++++++++++++++++--- tolk/lexer.cpp | 2 ++ tolk/lexer.h | 2 ++ 7 files changed, 59 insertions(+), 11 deletions(-) create mode 100644 tolk-tester/tests/invalid-syntax/err-3498.tolk create mode 100644 tolk-tester/tests/invalid-syntax/err-3810.tolk diff --git a/tolk-tester/tests/invalid-syntax/err-3498.tolk b/tolk-tester/tests/invalid-syntax/err-3498.tolk new file mode 100644 index 000000000..204a750cc --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3498.tolk @@ -0,0 +1,12 @@ + +fun main() { + var z = 5; + ++z; + return z; +} + +/** +@compilation_should_fail +@stderr Tolk has no increment operator +@stderr hint: use `i += 1`, not `i++` + */ diff --git a/tolk-tester/tests/invalid-syntax/err-3810.tolk b/tolk-tester/tests/invalid-syntax/err-3810.tolk new file mode 100644 index 000000000..a70ae569b --- /dev/null +++ b/tolk-tester/tests/invalid-syntax/err-3810.tolk @@ -0,0 +1,14 @@ + +struct A { + f: int; +} + +fun main(a: A) { + a.f--; +} + +/** +@compilation_should_fail +@stderr Tolk has no decrement operator +@stderr hint: use `i -= 1`, not `i--` + */ diff --git a/tolk-tester/tests/no-spaces.tolk b/tolk-tester/tests/no-spaces.tolk index 50b2761d7..857b71e3b 100644 --- a/tolk-tester/tests/no-spaces.tolk +++ b/tolk-tester/tests/no-spaces.tolk @@ -4,27 +4,27 @@ fun just10(): int { return int10; } fun eq(v: int): int { return`v` } @method_id(101,) fun `get_-1` (): int {return-1} -@method_id(102,) fun `get_--1` (): int {return--1} -@method_id(103,) fun `get_---1`(): int {return---1} -@method_id(104,) fun `get_+++1`(): int {return+++1} +@method_id(102,) fun `get_--1` (): int {return- -1} +@method_id(103,) fun `get_---1`(): int {return-(-(-1))} +@method_id(104,) fun `get_+++1`(): int {return+(+(+1))} @method_id(105,) fun `get_+-+1`(): int {return+-+1} global `some()var`:int; @method_id(110) fun `some_math`(): int { - `some()var`=--6; - return 1*-2*-3*-4*just10()*-5+-`some()var`+--`some()var`---`some()var`; + `some()var`=- -6; + return 1*-2*-3*-4*just10()*-5+-`some()var`+-(-`some()var`)-(-(-`some()var`)); } @method_id(111) fun `negative_nums`(a:int):int { var m$0:int=1; var m1:int=-(+0x1)*m$0; - return `a`*-1*-(1)*---(1)*+just10()+-`just10`()*m1*-m1+-eq(m1)----0x1; + return `a`*-1*-(1)*-(1)*+just10()+-`just10`()*m1*-m1+-eq(m1)-(-0x1); } @method_id(112) fun `bitwise~ops`(flags:int):[bool,bool] { return[ - (just10()-3==just10()-(4)--1)|((2==2)&(eq(eq(10)) -3==just10()--13)), + (just10()-3==just10()-(4)- -1)|((2==2)&(eq(eq(10)) -3==just10()-(-13))), ((flags&0xFF)!=0) ] } diff --git a/tolk-tester/tests/numbers-tests.tolk b/tolk-tester/tests/numbers-tests.tolk index 2dfdee8fd..7a08740ba 100644 --- a/tolk-tester/tests/numbers-tests.tolk +++ b/tolk-tester/tests/numbers-tests.tolk @@ -26,7 +26,7 @@ fun main() { 0b0 == 0, -0b1010100001010101010-0b1+0b1 == -344746, 0b00001 == 1, - --0b00101111101100 == 3052 + -(-0b00101111101100) == 3052 ); } diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp index 0ca8b3aeb..9a3a4e6fb 100644 --- a/tolk/ast-from-tokens.cpp +++ b/tolk/ast-from-tokens.cpp @@ -69,6 +69,16 @@ static void fire_error_mix_and_or_no_parenthesis(SrcLocation loc, std::string_vi "Use parenthesis to emphasize operator precedence."); } +// fire an error "Tolk does not have ++i operator" +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_no_increment_operator(SrcLocation loc, bool is_increment) { + std::string op_name = is_increment ? "increment" : "decrement"; + std::string op_wrong = is_increment ? "++" : "--"; + std::string op_right = is_increment ? "+=" : "-="; + throw ParseError(loc, std::string("Tolk has no ") + op_name + " operator\n" + + "hint: use `i " + op_right + " 1`, not `i" + op_wrong + "`"); +} + // diagnose when bitwise operators are used in a probably wrong way due to tricky precedence // example: "flags & 0xFF != 0" is equivalent to "flags & 1", most likely it's unexpected // the only way to suppress this error for the programmer is to use parenthesis @@ -847,7 +857,7 @@ static AnyExprV parse_expr100(Lexer& lex) { } } -// parse E(...) and E! having parsed E already (left-to-right) +// parse E(...) / E! / E++ / E-- having parsed E already (left-to-right) static AnyExprV parse_fun_call_postfix(Lexer& lex, AnyExprV lhs) { while (true) { if (lex.tok() == tok_oppar) { @@ -855,6 +865,8 @@ static AnyExprV parse_fun_call_postfix(Lexer& lex, AnyExprV lhs) { } else if (lex.tok() == tok_logical_not) { lex.next(); lhs = createV(lhs->loc, lhs); + } else if (lex.tok() == tok_double_plus || lex.tok() == tok_double_minus) { + fire_error_no_increment_operator(lex.cur_location(), lex.tok() == tok_double_plus); } else { break; } @@ -865,7 +877,7 @@ static AnyExprV parse_fun_call_postfix(Lexer& lex, AnyExprV lhs) { // parse E(...) and E! (left-to-right) static AnyExprV parse_expr90(Lexer& lex) { AnyExprV res = parse_expr100(lex); - if (lex.tok() == tok_oppar || lex.tok() == tok_logical_not) { + if (lex.tok() == tok_oppar || lex.tok() == tok_logical_not || lex.tok() == tok_double_plus || lex.tok() == tok_double_minus) { res = parse_fun_call_postfix(lex, res); } return res; @@ -892,7 +904,7 @@ static AnyExprV parse_expr80(Lexer& lex) { lex.unexpected("method name"); } lhs = createV(loc, lhs, v_ident, v_instantiationTs); - if (lex.tok() == tok_oppar || lex.tok() == tok_logical_not) { + if (lex.tok() == tok_oppar || lex.tok() == tok_logical_not || lex.tok() == tok_double_plus || lex.tok() == tok_double_minus) { lhs = parse_fun_call_postfix(lex, lhs); } } @@ -909,6 +921,12 @@ static AnyExprV parse_expr75(Lexer& lex) { AnyExprV rhs = parse_expr75(lex); return createV(loc, operator_name, t, rhs); } + if (t == tok_double_minus || t == tok_double_plus) { + SrcLocation loc = lex.cur_location(); + lex.next(); + parse_expr75(lex); + fire_error_no_increment_operator(loc, t == tok_double_plus); + } return parse_expr80(lex); } diff --git a/tolk/lexer.cpp b/tolk/lexer.cpp index c31f48762..94620dc25 100644 --- a/tolk/lexer.cpp +++ b/tolk/lexer.cpp @@ -523,6 +523,8 @@ struct TolkLanguageGrammar { register_token("^=", 2, tok_set_bitwise_xor); register_token("->", 2, tok_arrow); register_token("=>", 2, tok_double_arrow); + register_token("++", 2, tok_double_plus); + register_token("--", 2, tok_double_minus); register_token("<=>", 3, tok_spaceship); register_token("~>>", 3, tok_rshiftR); register_token("^>>", 3, tok_rshiftC); diff --git a/tolk/lexer.h b/tolk/lexer.h index 19ccb5c10..dbe0f34b7 100644 --- a/tolk/lexer.h +++ b/tolk/lexer.h @@ -64,6 +64,8 @@ enum TokenType { tok_set_div, tok_mod, tok_set_mod, + tok_double_plus, + tok_double_minus, tok_lshift, tok_set_lshift, tok_rshift, From 79bf8ebcda05ac888e7f0886dfd266e226aac577 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Wed, 11 Jun 2025 18:18:20 +0700 Subject: [PATCH 288/388] [Tolk] Fix unpacking RemainingBitsAndRefs when it's not the last --- tolk-tester/tests/pack-unpack-2.tolk | 22 ++++++++++++++++++++++ tolk/pack-unpack-serializers.cpp | 5 ++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/tolk-tester/tests/pack-unpack-2.tolk b/tolk-tester/tests/pack-unpack-2.tolk index 2a17f7ca1..789ed664f 100644 --- a/tolk-tester/tests/pack-unpack-2.tolk +++ b/tolk-tester/tests/pack-unpack-2.tolk @@ -375,6 +375,12 @@ struct ReadWriteMid { f3: coins; } +struct WithTwoRestFields { + i32: int32; + rest1: RemainingBitsAndRefs; + rest2: RemainingBitsAndRefs; +} + // --------------------------------------------- @@ -649,6 +655,20 @@ fun test_MidIsBuilderOrBitsN() { return (r.f1, mid.remainingBitsCount(), mid.loadAny(), mid.remainingBitsCount(), r.f3); } +@method_id(226) +fun test_MultipleRemainers() { + val m = WithTwoRestFields.fromSlice(stringHexToSlice("00000001FFFF")); + return (m.i32, m.rest1.remainingBitsCount(), m.rest2.remainingBitsCount()); +} + +@method_id(227) +fun test_mutatingRemainder() { + var cs = stringHexToSlice("00000001FFFF"); + val obj = cs.loadAny(); + cs.assertEnd(); + return obj.rest.remainingBitsCount(); +} + fun main() { var t: JustInt32 = { value: 10 }; @@ -686,4 +706,6 @@ fun main() { @testcase | 223 | | 10 55 (null) @testcase | 224 | | 60 50000000 5 9 123 -1 @testcase | 225 | | 5 40 65535 8 50000000 +@testcase | 226 | | 1 16 0 +@testcase | 227 | | 16 */ diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index 0ffce5fa2..61962615a 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -419,7 +419,10 @@ struct S_RemainingBitsAndRefs final : ISerializer { } std::vector unpack(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { - return ctx->ir_slice; + std::vector ir_rem_slice = code.create_tmp_var(TypeDataSlice::create(), loc, "(remainder)"); + code.emplace_back(loc, Op::_Let, ir_rem_slice, ctx->ir_slice); + code.emplace_back(loc, Op::_Call, ctx->ir_slice, std::vector{}, lookup_function("createEmptySlice")); + return ir_rem_slice; } void skip(const UnpackContext* ctx, CodeBlob& code, SrcLocation loc) override { From 973be6b5f6f5228d40a0778ebc06f19225271de0 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Wed, 11 Jun 2025 18:19:13 +0700 Subject: [PATCH 289/388] [Tolk] Bump version to v0.99 A pre-release one (although many optimizations are yet to come) --- crypto/smartcont/tolk-stdlib/common.tolk | 2 +- crypto/smartcont/tolk-stdlib/gas-payments.tolk | 2 +- crypto/smartcont/tolk-stdlib/lisp-lists.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-dicts.tolk | 2 +- crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk | 2 +- tolk/tolk-version.h | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk index e522bf2c0..30aabed3a 100644 --- a/crypto/smartcont/tolk-stdlib/common.tolk +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -1,7 +1,7 @@ // Standard library for Tolk (LGPL licence). // It contains common functions that are available out of the box, the user doesn't have to import anything. // More specific functions are required to be imported explicitly, like "@stdlib/tvm-dicts". -tolk 0.13 +tolk 0.99 /// In Tolk v1.x there would be a type `map`. /// Currently, working with dictionaries is still low-level, with raw cells. diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk index 8fde5f3c3..21de803ca 100644 --- a/crypto/smartcont/tolk-stdlib/gas-payments.tolk +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.13 +tolk 0.99 /** Gas and payment related primitives. diff --git a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk index 14d8f835b..13625c5ca 100644 --- a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk +++ b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.13 +tolk 0.99 /** Lisp-style lists are nested 2-elements tuples: `[1, [2, [3, null]]]` represents list `[1, 2, 3]`. diff --git a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk index 9d678e83d..dc2d3fd6b 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.13 +tolk 0.99 /** Dictionaries are represented as `cell` data type (cells can store anything, dicts in particular). diff --git a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk index 7281038d6..724464b70 100644 --- a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk +++ b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk @@ -1,5 +1,5 @@ // A part of standard library for Tolk -tolk 0.13 +tolk 0.99 /// Usually `c3` has a continuation initialized by the whole code of the contract. It is used for function calls. /// The primitive returns the current value of `c3`. diff --git a/tolk/tolk-version.h b/tolk/tolk-version.h index 5da1582d2..aaf1255d2 100644 --- a/tolk/tolk-version.h +++ b/tolk/tolk-version.h @@ -18,6 +18,6 @@ namespace tolk { -constexpr const char* TOLK_VERSION = "0.13.0"; +constexpr const char* TOLK_VERSION = "0.99.0"; } // namespace tolk From d6eb770f6269ac8292adb1b367071a05d64c08d2 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 13 Jun 2025 11:18:41 +0300 Subject: [PATCH 290/388] Fix initializing neighbor stats in collator --- validator/impl/collator.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index c165b7191..1c7ad1f27 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -1245,6 +1245,7 @@ bool Collator::add_trivial_neighbor_after_merge() { nb.set_queue_root(out_msg_queue_->get_root_cell()); nb.processed_upto = processed_upto_; nb.blk_.id.shard = get_shard(); + stats_.neighbors[i].shard = nb.blk_.shard_full(); LOG(DEBUG) << "adjusted neighbor #" << i << " : " << nb.blk_.to_str() << " with shard expansion (immediate after-merge adjustment)"; } else { @@ -1320,10 +1321,13 @@ bool Collator::add_trivial_neighbor() { CHECK(sibling_out_msg_queue_); CHECK(sibling_processed_upto_); neighbors_.emplace_back(*descr_ref); + stats_.neighbors.push_back(CollationStats::NeighborStats{ + .shard = descr_ref->shard(), .is_trivial = true, .is_local = true, .msg_limit = -1}); auto& nb2 = neighbors_.at(i); nb2.set_queue_root(sibling_out_msg_queue_->get_root_cell()); nb2.processed_upto = sibling_processed_upto_; nb2.blk_.id.shard = ton::shard_sibling(get_shard()); + stats_.neighbors[i].shard = nb2.blk_.shard_full(); LOG(DEBUG) << "adjusted neighbor #" << i << " : " << nb2.blk_.to_str() << " with shard shrinking to our sibling (immediate after-split adjustment)"; auto& nb1 = neighbors_.at(n); @@ -1343,6 +1347,8 @@ bool Collator::add_trivial_neighbor() { CHECK(!sibling_out_msg_queue_); CHECK(!sibling_processed_upto_); neighbors_.emplace_back(*descr_ref); + stats_.neighbors.push_back(CollationStats::NeighborStats{ + .shard = descr_ref->shard(), .is_trivial = true, .is_local = true, .msg_limit = -1}); auto& nb2 = neighbors_.at(i); auto sib_shard = ton::shard_sibling(shard_); // compute the part of virtual sibling's OutMsgQueue with destinations in our shard @@ -1362,6 +1368,7 @@ bool Collator::add_trivial_neighbor() { return fatal_error("error splitting ProcessedUpto for our virtual sibling"); } nb2.blk_.id.shard = ton::shard_sibling(get_shard()); + stats_.neighbors[i].shard = nb2.blk_.shard_full(); LOG(DEBUG) << "adjusted neighbor #" << i << " : " << nb2.blk_.to_str() << " with shard shrinking to our sibling (continued after-split adjustment)"; auto& nb1 = neighbors_.at(n); @@ -6410,6 +6417,14 @@ void Collator::finalize_stats() { }); } stats_.new_out_msg_queue_size = out_msg_queue_size_; + + auto neighbors_stats = std::move(stats_.neighbors); + stats_.neighbors.clear(); + for (size_t i = 0; i < neighbors_stats.size(); ++i) { + if (!neighbors_.at(i).is_disabled()) { + stats_.neighbors.push_back(std::move(neighbors_stats[i])); + } + } } /** From 6a7b2102cca1ca707675541dd3da77c8c3b615af Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Fri, 30 May 2025 16:38:16 -0400 Subject: [PATCH 291/388] Delete unused validator/db/archive-mover.{hpp,cpp} --- validator/db/archive-mover.cpp | 493 --------------------------------- validator/db/archive-mover.hpp | 166 ----------- 2 files changed, 659 deletions(-) delete mode 100644 validator/db/archive-mover.cpp delete mode 100644 validator/db/archive-mover.hpp diff --git a/validator/db/archive-mover.cpp b/validator/db/archive-mover.cpp deleted file mode 100644 index aa89570d9..000000000 --- a/validator/db/archive-mover.cpp +++ /dev/null @@ -1,493 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "archive-mover.hpp" -#include "td/actor/MultiPromise.h" -#include "validator/fabric.h" - -namespace ton { - -namespace validator { - -void ArchiveFileMover::start_up() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_handle1, std::move(R)); - }); - td::actor::send_closure(archive_manager_, &ArchiveManager::get_handle, block_id_, std::move(P)); -} - -void ArchiveFileMover::got_block_handle0(td::Result R) { - if (R.is_ok()) { - handle_ = R.move_as_ok(); - CHECK(handle_->moved_to_archive()); - CHECK(handle_->handle_moved_to_archive()); - finish_query(); - return; - } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_handle1, std::move(R)); - }); - td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read_handle, block_id_, std::move(P)); -} - -void ArchiveFileMover::got_block_handle1(td::Result R) { - if (R.is_ok()) { - handle_ = R.move_as_ok(); - got_block_handle(); - return; - } - - if (R.error().code() != ErrorCode::notready) { - abort_query(R.move_as_error()); - return; - } - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_handle1, std::move(R)); - }); - td::actor::send_closure(block_db_, &BlockDb::get_block_handle, std::move(P)); -} - -void ArchiveFileMover::got_block_handle2(td::Result R) { - if (R.is_ok()) { - handle_ = R.move_as_ok(); - got_block_handle(); - return; - } - - if (R.error().code() != ErrorCode::notready) { - abort_query(R.move_as_error()); - return; - } - - finish_query(); -} - -void ArchiveFileMover::got_block_handle() { - if (!handle_->is_applied()) { - finish_query(); - return; - } - if (handle_->id().seqno() == 0) { - processed_all_children(); - return; - } - - CHECK(handle_->inited_prev()); - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &ArchiveFileMover::abort_query, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &ArchiveFileMover::processed_child); - } - }); - - td::actor::create_actor("mover", handle_->one_prev(left_), mode_, block_db_, file_db_, - old_archive_db_, old_archive_manager_, archive_manager_, std::move(P)) - .release(); -} - -void ArchiveFileMover::processed_child() { - if (!left_ || !handle_->merge_before()) { - processed_all_children(); - return; - } - left_ = false; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &ArchiveFileMover::abort_query, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &ArchiveFileMover::processed_child); - } - }); - - td::actor::create_actor("mover", handle_->one_prev(left_), mode_, block_db_, file_db_, - old_archive_db_, old_archive_manager_, archive_manager_, std::move(P)) - .release(); -} - -void ArchiveFileMover::processed_all_children() { - if (!handle_->received()) { - got_block_data(td::BufferSlice{}); - } else { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_data, std::move(R)); - }); - - if (handle_->moved_to_archive()) { - CHECK(handle_->inited_unix_time()); - td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, handle_->unix_time(), - handle_->is_key_block(), FileDb::RefId{fileref::Block{handle_->id()}}, std::move(P)); - } else { - td::actor::send_closure(handle_->moved_to_storage() ? old_archive_db_ : file_db_, &FileDb::load_file, - FileDb::RefId{fileref::Block{handle_->id()}}, std::move(P)); - } - } -} - -void ArchiveFileMover::got_block_data(td::Result R) { - if (R.is_error()) { - if (R.error().code() != ErrorCode::notready) { - abort_query(R.move_as_error()); - return; - } - } else { - data_ = R.move_as_ok(); - } - if (!handle_->inited_proof()) { - got_block_proof(td::BufferSlice{}); - } else { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_proof, std::move(R)); - }); - - if (handle_->moved_to_archive()) { - CHECK(handle_->inited_unix_time()); - td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, handle_->unix_time(), - handle_->is_key_block(), FileDb::RefId{fileref::Proof{handle_->id()}}, std::move(P)); - } else { - td::actor::send_closure(handle_->moved_to_storage() ? old_archive_db_ : file_db_, &FileDb::load_file, - FileDb::RefId{fileref::Proof{handle_->id()}}, std::move(P)); - } - } -} - -void ArchiveFileMover::got_block_proof(td::Result R) { - if (R.is_error()) { - if (R.error().code() != ErrorCode::notready) { - abort_query(R.move_as_error()); - return; - } - } else { - proof_ = R.move_as_ok(); - } - if (!handle_->inited_proof_link()) { - got_block_proof_link(td::BufferSlice{}); - } else { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &ArchiveFileMover::got_block_proof_link, std::move(R)); - }); - - if (handle_->moved_to_archive()) { - CHECK(handle_->inited_unix_time()); - td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, handle_->unix_time(), - handle_->is_key_block(), FileDb::RefId{fileref::ProofLink{handle_->id()}}, std::move(P)); - } else { - td::actor::send_closure(handle_->moved_to_storage() ? old_archive_db_ : file_db_, &FileDb::load_file, - FileDb::RefId{fileref::ProofLink{handle_->id()}}, std::move(P)); - } - } -} - -void ArchiveFileMover::got_block_proof_link(td::Result R) { - if (R.is_error()) { - if (R.error().code() != ErrorCode::notready) { - abort_query(R.move_as_error()); - return; - } - } else { - proof_link_ = R.move_as_ok(); - } - - td::MultiPromise mp; - auto ig = mp.init_guard(); - ig.add_promise([SelfId = actor_id(this)](td::Result R) { - R.ensure(); - td::actor::send_closure(SelfId, &ArchiveFileMover::written_data); - }); - - if (data_.size() > 0) { - td::actor::send_closure(archive_manager_, &ArchiveManager::add_file, handle_, fileref::Block{block_id_}, - std::move(data_), ig.get_promise()); - } - if (proof_.size() > 0) { - td::actor::send_closure(archive_manager_, &ArchiveManager::add_file, handle_, fileref::Proof{block_id_}, - std::move(proof_), ig.get_promise()); - } - if (proof_link_.size() > 0) { - td::actor::send_closure(archive_manager_, &ArchiveManager::add_file, handle_, fileref::ProofLink{block_id_}, - std::move(proof_link_), ig.get_promise()); - } -} - -void ArchiveFileMover::written_data() { - handle_->set_moved_to_archive(); - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - R.ensure(); - td::actor::send_closure(SelfId, &ArchiveFileMover::written_handle); - }); - td::actor::send_closure(archive_manager_, &ArchiveManager::add_handle, handle_, std::move(P)); -} - -void ArchiveFileMover::written_handle() { - CHECK(handle_->handle_moved_to_archive()); - finish_query(); -} - -void ArchiveFileMover::abort_query(td::Status error) { - if (promise_) { - promise_.set_error(std::move(error)); - } - stop(); -} - -void ArchiveFileMover::finish_query() { - if (promise_) { - promise_.set_value(td::Unit()); - } - stop(); -} - -void ArchiveKeyBlockMover::start_up() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_ok()) { - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::skip_block_proof, R.move_as_ok()); - } else { - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::failed_to_get_proof0); - } - }); - if (proof_link_) { - td::actor::send_closure(archive_manager_, &ArchiveManager::get_file_short, fileref::ProofLink{block_id_}, - std::move(P)); - } else { - td::actor::send_closure(archive_manager_, &ArchiveManager::get_file_short, fileref::Proof{block_id_}, std::move(P)); - } -} - -void ArchiveKeyBlockMover::failed_to_get_proof0() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_ok()) { - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::got_block_proof, R.move_as_ok()); - } else { - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::failed_to_get_proof1); - } - }); - if (proof_link_) { - td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, fileref::ProofLink{block_id_}, - std::move(P)); - } else { - td::actor::send_closure(old_archive_manager_, &OldArchiveManager::read, fileref::Proof{block_id_}, std::move(P)); - } -} - -void ArchiveKeyBlockMover::failed_to_get_proof1() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_ok()) { - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::got_block_proof, R.move_as_ok()); - } else { - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::failed_to_get_proof2); - } - }); - if (proof_link_) { - td::actor::send_closure(old_archive_db_, &FileDb::load_file, fileref::ProofLink{block_id_}, std::move(P)); - } else { - td::actor::send_closure(old_archive_db_, &FileDb::load_file, fileref::Proof{block_id_}, std::move(P)); - } -} - -void ArchiveKeyBlockMover::failed_to_get_proof2() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_ok()) { - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::got_block_proof, R.move_as_ok()); - } else { - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::failed_to_get_proof3); - } - }); - if (proof_link_) { - td::actor::send_closure(file_db_, &FileDb::load_file, fileref::ProofLink{block_id_}, std::move(P)); - } else { - td::actor::send_closure(file_db_, &FileDb::load_file, fileref::Proof{block_id_}, std::move(P)); - } -} - -void ArchiveKeyBlockMover::failed_to_get_proof3() { - if (proof_link_) { - written_data(); - } else { - proof_link_ = true; - start_up(); - } -} - -void ArchiveKeyBlockMover::got_block_proof(td::BufferSlice data) { - data_ = std::move(data); - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - R.ensure(); - td::actor::send_closure(SelfId, &ArchiveKeyBlockMover::written_data); - }); - - if (proof_link_) { - auto p = create_proof_link(block_id_, data_.clone()).move_as_ok(); - auto h = p->get_basic_header_info().move_as_ok(); - td::actor::send_closure(archive_manager_, &ArchiveManager::add_key_block_proof, h.utime, - fileref::ProofLink{block_id_}, std::move(data_), std::move(P)); - } else { - auto p = create_proof(block_id_, data_.clone()).move_as_ok(); - auto h = p->get_basic_header_info().move_as_ok(); - td::actor::send_closure(archive_manager_, &ArchiveManager::add_key_block_proof, h.utime, fileref::Proof{block_id_}, - std::move(data_), std::move(P)); - } -} - -void ArchiveKeyBlockMover::skip_block_proof(td::BufferSlice data) { - data_ = std::move(data); - written_data(); -} - -void ArchiveKeyBlockMover::written_data() { - td::Ref proof_link; - if (proof_link_) { - auto p = create_proof_link(block_id_, data_.clone()).move_as_ok(); - proof_link = std::move(p); - } else { - auto p = create_proof(block_id_, data_.clone()).move_as_ok(); - proof_link = std::move(p); - } - auto ts = proof_link->get_basic_header_info().move_as_ok().utime; - auto te = ValidatorManager::persistent_state_ttl(ts); - if (te < td::Clocks::system()) { - finish_query(); - return; - } -} - -void ArchiveKeyBlockMover::abort_query(td::Status error) { - if (promise_) { - promise_.set_error(std::move(error)); - } - stop(); -} - -void ArchiveKeyBlockMover::finish_query() { - if (promise_) { - promise_.set_value(td::Unit()); - } - stop(); -} - -void ArchiveMover::start_up() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &ArchiveMover::abort_query, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &ArchiveMover::moved_blocks); - } - }); - td::actor::create_actor("fmover", masterchain_block_id_, block_db_.get(), file_db_.get(), - old_archive_db_.get(), old_archive_manager_.get(), archive_manager_.get(), - std::move(P)) - .release(); -} - -void ArchiveMover::moved_blocks() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - R.ensure(); - td::actor::send_closure(SelfId, &ArchiveMover::got_handle, R.move_as_ok()); - }); - td::actor::send_closure(archive_manager_, &ArchiveManager::get_handle, masterchain_block_id_, std::move(P)); -} - -void ArchiveMover::got_handle(BlockHandle handle) { - handle_ = std::move(handle); - CHECK(handle_->is_applied()); - CHECK(handle_->inited_state_boc()); - CHECK(!handle_->deleted_state_boc()); - auto P = td::PromiseCreator::lambda( - [handle = handle_, SelfId = actor_id(this)](td::Result> R) mutable { - R.ensure(); - auto S = create_shard_state(handle->id(), R.move_as_ok()); - S.ensure(); - td::actor::send_closure(SelfId, &ArchiveMover::got_state, td::Ref{S.move_as_ok()}); - }); - td::actor::send_closure(cell_db_, &CellDb::load_cell, handle_->state(), std::move(P)); -} - -void ArchiveMover::got_state(td::Ref state) { - state_ = std::move(state); - - td::MultiPromise mp; - auto ig = mp.init_guard(); - ig.add_promise([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &ArchiveMover::abort_query, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &ArchiveMover::moved_key_blocks); - } - }); - - auto k = state_->prev_key_block_id(std::numeric_limits::max()); - while (k.is_valid() && k.seqno() > 0) { - td::actor::create_actor("keymover", k, block_db_.get(), file_db_.get(), old_archive_db_.get(), - old_archive_manager_.get(), archive_manager_.get(), ig.get_promise()) - .release(); - k = state_->prev_key_block_id(k.seqno()); - } -} - -void ArchiveMover::moved_key_blocks() { - td::MultiPromise mp; - auto ig = mp.init_guard(); - ig.add_promise([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &ArchiveMover::abort_query, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &ArchiveMover::moved_key_blocks); - } - }); - - auto k = state_->prev_key_block_id(std::numeric_limits::max()); - while (k.is_valid() && k.seqno() > 0) { - td::actor::create_actor("keymover", k, block_db_.get(), file_db_.get(), old_archive_db_.get(), - old_archive_manager_.get(), archive_manager_.get(), ig.get_promise()) - .release(); - k = state_->prev_key_block_id(k.seqno()); - } -} - -void ArchiveMover::run() { - if (to_move_.empty() && to_check_.empty()) { - completed(); - return; - } - if (!to_check_.empty()) { - auto B = to_check_.back(); - CHECK(to_check_set_.count(B) == 1); - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - td::actor::send_closure(SelfId, &ArchiveMover::got_to_check_handle, std::move(R)); - }); - td::actor::send_closure(block_db_, &BlockDb::get_block_handle, B, std::move(P)); - return; - } - CHECK(!to_move_.empty()); -} - -void ArchiveMover::got_to_check_handle(td::Result R) { - if (R.is_error()) { - CHECK(R.error().code() == ErrorCode::notready); - run(); - return; - } - auto handle = R.move_as_ok(); -} - -} // namespace validator - -} // namespace ton diff --git a/validator/db/archive-mover.hpp b/validator/db/archive-mover.hpp deleted file mode 100644 index be0d8dc75..000000000 --- a/validator/db/archive-mover.hpp +++ /dev/null @@ -1,166 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once - -#include "td/actor/actor.h" -#include "filedb.hpp" -#include "blockdb.hpp" -#include "statedb.hpp" -#include "celldb.hpp" -#include "archive-db.hpp" -#include "archive-manager.hpp" - -#include -#include - -namespace ton { - -namespace validator { - -class ArchiveFileMover : public td::actor::Actor { - public: - ArchiveFileMover(BlockIdExt block_id, td::actor::ActorId block_db, td::actor::ActorId file_db, - td::actor::ActorId old_archive_db, td::actor::ActorId old_archive_manager, - td::actor::ActorId archive_manager, td::Promise promise) - : block_id_(block_id) - , block_db_(block_db) - , file_db_(file_db) - , old_archive_db_(old_archive_db) - , old_archive_manager_(old_archive_manager) - , archive_manager_(archive_manager) - , promise_(std::move(promise)) { - } - void start_up() override; - void got_block_handle0(td::Result R); - void got_block_handle1(td::Result R); - void got_block_handle2(td::Result R); - void got_block_handle(); - - void processed_child(); - void processed_all_children(); - - void got_block_data(td::Result R); - void got_block_proof(td::Result R); - void got_block_proof_link(td::Result R); - - void written_data(); - void written_handle(); - - void abort_query(td::Status error); - void finish_query(); - - private: - BlockIdExt block_id_; - BlockHandle handle_; - td::BufferSlice data_; - td::BufferSlice proof_; - td::BufferSlice proof_link_; - bool left_ = true; - - td::actor::ActorId block_db_; - td::actor::ActorId file_db_; - td::actor::ActorId old_archive_db_; - td::actor::ActorId old_archive_manager_; - td::actor::ActorId archive_manager_; - - td::Promise promise_; -}; - -class ArchiveKeyBlockMover : public td::actor::Actor { - public: - ArchiveKeyBlockMover(BlockIdExt block_id, td::actor::ActorId block_db, td::actor::ActorId file_db, - td::actor::ActorId old_archive_db, - td::actor::ActorId old_archive_manager, - td::actor::ActorId archive_manager, td::Promise promise) - : block_id_(block_id) - , block_db_(block_db) - , file_db_(file_db) - , old_archive_db_(old_archive_db) - , old_archive_manager_(old_archive_manager) - , archive_manager_(archive_manager) - , promise_(std::move(promise)) { - } - - void start_up() override; - void failed_to_get_proof0(); - void failed_to_get_proof1(); - void failed_to_get_proof2(); - void failed_to_get_proof3(); - void got_block_proof(td::BufferSlice data); - void skip_block_proof(td::BufferSlice data); - - void written_data(); - - void abort_query(td::Status error); - void finish_query(); - - private: - BlockIdExt block_id_; - td::BufferSlice data_; - bool proof_link_ = false; - - td::actor::ActorId block_db_; - td::actor::ActorId file_db_; - td::actor::ActorId old_archive_db_; - td::actor::ActorId old_archive_manager_; - td::actor::ActorId archive_manager_; - - td::Promise promise_; -}; - -class ArchiveMover : public td::actor::Actor { - public: - ArchiveMover(std::string db_root, BlockIdExt masterchain_block_id, BlockIdExt shard_block_id, - BlockIdExt key_block_id); - - void start_up() override; - void moved_blocks(); - void got_handle(BlockHandle handle); - void got_state(td::Ref state); - void moved_key_blocks(); - void run(); - void completed(); - void add_to_move(BlockIdExt block_id); - void add_to_check(BlockIdExt block_id); - - void got_to_check_handle(td::Result R); - - void abort_query(td::Status error); - void finish_query(); - - private: - std::string db_root_; - BlockHandle handle_; - td::Ref state_; - - td::actor::ActorOwn block_db_; - td::actor::ActorOwn file_db_; - td::actor::ActorOwn old_archive_db_; - td::actor::ActorOwn old_archive_manager_; - td::actor::ActorOwn archive_manager_; - td::actor::ActorOwn cell_db_; - - BlockIdExt masterchain_block_id_; - BlockIdExt shard_block_id_; - BlockIdExt key_block_id_; -}; - -} // namespace validator - -} // namespace ton From af42e201786d5f385501a96eeef94fc721056562 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Tue, 3 Jun 2025 22:24:19 -0400 Subject: [PATCH 292/388] Remove unused FileReference constructor --- validator/db/fileref.cpp | 31 ------------------------------- validator/db/fileref.hpp | 1 - 2 files changed, 32 deletions(-) diff --git a/validator/db/fileref.cpp b/validator/db/fileref.cpp index feefd1f9c..72aa32cfb 100644 --- a/validator/db/fileref.cpp +++ b/validator/db/fileref.cpp @@ -258,37 +258,6 @@ std::string BlockInfoShort::filename_short() const { } // namespace fileref -FileReference::FileReference(tl_object_ptr key) { - ton_api::downcast_call( - *key.get(), - td::overloaded( - [&](const ton_api::db_filedb_key_empty& key) { ref_ = fileref::Empty{}; }, - [&](const ton_api::db_filedb_key_blockFile& key) { ref_ = fileref::Block{create_block_id(key.block_id_)}; }, - [&](const ton_api::db_filedb_key_zeroStateFile& key) { - ref_ = fileref::ZeroState{create_block_id(key.block_id_)}; - }, - [&](const ton_api::db_filedb_key_persistentStateFile& key) { - ref_ = fileref::PersistentState{create_block_id(key.block_id_), create_block_id(key.masterchain_block_id_)}; - }, - [&](const ton_api::db_filedb_key_proof& key) { ref_ = fileref::Proof{create_block_id(key.block_id_)}; }, - [&](const ton_api::db_filedb_key_proofLink& key) { - ref_ = fileref::ProofLink{create_block_id(key.block_id_)}; - }, - [&](const ton_api::db_filedb_key_signatures& key) { - ref_ = fileref::Signatures{create_block_id(key.block_id_)}; - }, - [&](const ton_api::db_filedb_key_candidate& key) { - ref_ = fileref::Candidate{PublicKey{key.id_->source_}, create_block_id(key.id_->id_), - key.id_->collated_data_file_hash_}; - }, - [&](const ton_api::db_filedb_key_candidateRef& key) { - ref_ = fileref::CandidateRef{create_block_id(key.id_)}; - }, - [&](const ton_api::db_filedb_key_blockInfo& key) { - ref_ = fileref::BlockInfo{create_block_id(key.block_id_)}; - })); -} - FileReferenceShort FileReference::shortref() const { FileReferenceShort h; ref_.visit([&](const auto& obj) { h = obj.shortref(); }); diff --git a/validator/db/fileref.hpp b/validator/db/fileref.hpp index 6710cc063..14060fb45 100644 --- a/validator/db/fileref.hpp +++ b/validator/db/fileref.hpp @@ -382,7 +382,6 @@ class FileReference { } FileReference() : ref_(fileref::Empty{}) { } - FileReference(tl_object_ptr key); static td::Result create(std::string filename); From 774371bdc9f6107fd05106c1fd559e8903e0513d Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Thu, 29 May 2025 19:29:43 -0400 Subject: [PATCH 293/388] Store `archive_split_depth` in WorkchainDescr in config The parameter will be used in later commits to denote number of parts we split archived shard states into. --- crypto/block/block.tlb | 1 + crypto/block/mc-config.cpp | 1 + crypto/block/mc-config.h | 2 ++ validator/impl/shard.cpp | 8 ++++++++ validator/impl/shard.hpp | 1 + validator/interfaces/shard.h | 1 + 6 files changed, 14 insertions(+) diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 99dfdb522..93b1f3ad7 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -670,6 +670,7 @@ workchain_v2#a7 enabled_since:uint32 monitor_min_split:(## 8) zerostate_root_hash:bits256 zerostate_file_hash:bits256 version:uint32 format:(WorkchainFormat basic) split_merge_timings:WcSplitMergeTimings + persistent_state_split_depth:(## 8) { persistent_state_split_depth <= 63 } = WorkchainDescr; _ workchains:(HashmapE 32 WorkchainDescr) = ConfigParam 12; diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 78e2294f8..35c3df005 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2199,6 +2199,7 @@ bool WorkchainInfo::unpack(ton::WorkchainId wc, vm::CellSlice& cs) { split_merge_interval = rec.split_merge_interval; min_split_merge_interval = rec.min_split_merge_interval; max_split_merge_delay = rec.max_split_merge_delay; + persistent_state_split_depth = info.persistent_state_split_depth; return true; }; block::gen::WorkchainDescr::Record_workchain info_v1; diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 95e107352..34fab4df4 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -438,6 +438,8 @@ struct WorkchainInfo : public td::CntObject { unsigned min_split_merge_interval = 30; // split/merge interval must be at least 30 seconds unsigned max_split_merge_delay = 1000; // end of split/merge interval must be at most 1000 seconds in the future + td::uint32 persistent_state_split_depth = 0; + bool is_valid() const { return workchain != ton::workchainInvalid; } diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index d2d402dd6..d9a78b2de 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -524,6 +524,14 @@ bool MasterchainStateQ::check_old_mc_block_id(const ton::BlockIdExt& blkid, bool return config_ && config_->check_old_mc_block_id(blkid, strict); } +td::uint32 MasterchainStateQ::persistent_state_split_depth(WorkchainId workchain_id) const { + if (!config_) { + return 0; + } + auto wc_info = config_->get_workchain_info(workchain_id); + return wc_info.not_null() ? wc_info->persistent_state_split_depth : 0; +} + td::uint32 MasterchainStateQ::monitor_min_split_depth(WorkchainId workchain_id) const { if (!config_) { return 0; diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index fa36e1e66..a896f3eb0 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -124,6 +124,7 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { bool has_workchain(WorkchainId workchain) const { return config_ && config_->has_workchain(workchain); } + td::uint32 persistent_state_split_depth(WorkchainId workchain_id) const override; td::uint32 monitor_min_split_depth(WorkchainId workchain_id) const override; td::uint32 min_split_depth(WorkchainId workchain_id) const override; BlockSeqno min_ref_masterchain_seqno() const override; diff --git a/validator/interfaces/shard.h b/validator/interfaces/shard.h index 64aea9b62..96767cbf9 100644 --- a/validator/interfaces/shard.h +++ b/validator/interfaces/shard.h @@ -71,6 +71,7 @@ class MasterchainState : virtual public ShardState { virtual std::vector> get_shards() const = 0; virtual td::Ref get_shard_from_config(ShardIdFull shard) const = 0; virtual bool workchain_is_active(WorkchainId workchain_id) const = 0; + virtual td::uint32 persistent_state_split_depth(WorkchainId workchain_id) const = 0; virtual td::uint32 monitor_min_split_depth(WorkchainId workchain_id) const = 0; virtual td::uint32 min_split_depth(WorkchainId workchain_id) const = 0; virtual BlockSeqno min_ref_masterchain_seqno() const = 0; From 7fefaa9a627da246cc2684b113a438c15b60991e Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Tue, 3 Jun 2025 22:30:13 -0400 Subject: [PATCH 294/388] Add initial support for split persistent states in ArchiveManager For now, we only make implementation ready for them without changing public API in any way. In particular, it is not possible to create a split state or request it. --- tl/generate/scheme/ton_api.tl | 4 +- tl/generate/scheme/ton_api.tlo | Bin 103024 -> 103388 bytes validator/db/archive-manager.cpp | 63 ++++++++++++++----- validator/db/archive-manager.hpp | 3 +- validator/db/fileref.cpp | 57 +++++++++++++++++ validator/db/fileref.hpp | 78 +++++++++++++++++++++++- validator/interfaces/persistent-state.h | 38 ++++++++++++ 7 files changed, 223 insertions(+), 20 deletions(-) create mode 100644 validator/interfaces/persistent-state.h diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 580d74861..06f202a7f 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -542,7 +542,9 @@ db.candidate.id source:PublicKey id:tonNode.blockIdExt collated_data_file_hash:i db.filedb.key.empty = db.filedb.Key; db.filedb.key.blockFile block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.zeroStateFile block_id:tonNode.blockIdExt = db.filedb.Key; -db.filedb.key.persistentStateFile block_id:tonNode.blockIdExt masterchain_block_id:tonNode.blockIdExt = db.filedb.Key; +db.filedb.key.persistentStateFile block_id:tonNode.blockIdExt masterchain_block_id:tonNode.blockIdExt = db.filedb.Key; +db.filedb.key.splitAccountStateFile block_id:tonNode.blockIdExt masterchain_block_id:tonNode.blockIdExt effective_shard:long = db.filedb.Key; +db.filedb.key.splitPersistentStateFile block_id:tonNode.blockIdExt masterchain_block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.proof block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.proofLink block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.signatures block_id:tonNode.blockIdExt = db.filedb.Key; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index ae1785ebb566de1406a4792679287455daaccfc3..f30418f7bd78ff6a6f1b8519fe1c506e3f5999ad 100644 GIT binary patch delta 419 zcmeycgze69whac{q6_wz)bXYy>7`}nqyj1L)Jg^ho{5g)n`5|-=reBHtZ3bDF9A|? z^O%qi0}vEBv*~cywjBhCO+Hbj!2)I+s4;;sbgFgGM5a_jrOs5RID!;|OaXzba$GTK zOLLAXBdpI(t<)`*4-fNp%e@rK!ISd0%yEh^3|E=fhQfdT9fpc^61H>puU_eTx7Uw_n~ u`!%5!s>YyPV)Bw&5r`3cYN6^3%ApLII&`xd>TJM@n-A1&KTyZGK^OoTf~5ig delta 254 zcmcb!obAIBwhac{qBFH#DD$Qy>7`}nqyj1L)Jg^hu8EG~n`5|-=reBFtZ3bDF9A|? z^O%qi0}vEBv*~cywjBhCO+Hbj!2)K)RGUB;I@LO8B2!>eXR1>mQVVKTP}NMTLpLU+ u4&9hDFf|6{5|ah$MIa_<)I-%7ltURE_2_23skZ?uZq{kouG7G{K^Oqe_g{Yi diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index 9e6ea797e..926e7c3b5 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -311,9 +311,6 @@ void ArchiveManager::get_file(ConstBlockHandle handle, FileReference ref_id, td: } void ArchiveManager::register_perm_state(FileReferenceShort id) { - BlockSeqno masterchain_seqno = 0; - id.ref().visit(td::overloaded( - [&](const fileref::PersistentStateShort &x) { masterchain_seqno = x.masterchain_seqno; }, [&](const auto &) {})); td::uint64 size; auto r_stat = td::stat(db_root_ + "/archive/states/" + id.filename_short()); if (r_stat.is_error()) { @@ -322,7 +319,7 @@ void ArchiveManager::register_perm_state(FileReferenceShort id) { } else { size = r_stat.ok().size_; } - perm_states_[{masterchain_seqno, id.hash()}] = {.id = id, .size = size}; + perm_states_[{id.seqno_of_persistent_state(), id.hash()}] = {.id = id, .size = size}; } void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise promise) { @@ -347,6 +344,23 @@ void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, t .release(); } +namespace { + +FileReferenceShort create_persistent_state_id(BlockIdExt block_id, BlockIdExt mc_block_id, PersistentStateType type) { + FileReferenceShort result; + type.visit(td::overloaded( + [&](UnsplitStateType const &) { result = fileref::PersistentStateShort::create(block_id, mc_block_id); }, + [&](SplitAccountStateType const &account_state) { + result = fileref::SplitAccountState::create(block_id, mc_block_id, account_state.effective_shard_id); + }, + [&](SplitPersistentStateType const &persistent_state) { + result = fileref::SplitPersistentState::create(block_id, mc_block_id); + })); + return result; +} + +} // namespace + void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data, td::Promise promise) { auto create_writer = [&](std::string path, td::Promise P) { @@ -354,7 +368,9 @@ void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt master std::move(P)) .release(); }; - add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer)); + // TODO: Allow specifying state type. + add_persistent_state_impl(create_persistent_state_id(block_id, masterchain_block_id, UnsplitStateType{}), + std::move(promise), std::move(create_writer)); } void ArchiveManager::add_persistent_state_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, @@ -365,23 +381,22 @@ void ArchiveManager::add_persistent_state_gen(BlockIdExt block_id, BlockIdExt ma std::move(write_state), std::move(P)) .release(); }; - add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer)); + // TODO: Allow specifying state type. + add_persistent_state_impl(create_persistent_state_id(block_id, masterchain_block_id, UnsplitStateType{}), + std::move(promise), std::move(create_writer)); } void ArchiveManager::add_persistent_state_impl( - BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise, + FileReferenceShort const &id, td::Promise promise, std::function)> create_writer) { - auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; - BlockSeqno masterchain_seqno = masterchain_block_id.seqno(); - auto hash = id.hash(); - if (perm_states_.find({masterchain_seqno, hash}) != perm_states_.end()) { + if (perm_states_.find({id.seqno_of_persistent_state(), id.hash()}) != perm_states_.end()) { promise.set_value(td::Unit()); return; } auto path = db_root_ + "/archive/states/" + id.filename_short(); auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), id = id.shortref(), promise = std::move(promise)](td::Result R) mutable { + [SelfId = actor_id(this), id = id, promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); } else { @@ -436,7 +451,8 @@ void ArchiveManager::get_previous_persistent_state_files( void ArchiveManager::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) { - auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; + // TODO: Allow specifying state type. + auto id = create_persistent_state_id(block_id, masterchain_block_id, UnsplitStateType{}); auto hash = id.hash(); if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) { promise.set_error(td::Status::Error(ErrorCode::notready, "state file not in db")); @@ -449,7 +465,8 @@ void ArchiveManager::get_persistent_state(BlockIdExt block_id, BlockIdExt master void ArchiveManager::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, td::int64 max_size, td::Promise promise) { - auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; + // TODO: Allow specifying state type. + auto id = create_persistent_state_id(block_id, masterchain_block_id, UnsplitStateType{}); auto hash = id.hash(); if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) { promise.set_error(td::Status::Error(ErrorCode::notready, "state file not in db")); @@ -462,7 +479,8 @@ void ArchiveManager::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt void ArchiveManager::get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) { - auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; + // TODO: Allow specifying state type. + auto id = create_persistent_state_id(block_id, masterchain_block_id, UnsplitStateType{}); auto hash = id.hash(); auto it = perm_states_.find({masterchain_block_id.seqno(), hash}); if (it == perm_states_.end()) { @@ -918,6 +936,11 @@ void ArchiveManager::start_up() { R = FileReferenceShort::create(newfname); R.ensure(); } + if (!R.ok().is_state_like()) { + LOG(ERROR) << "deleting file that is not state-like '" << fname << "'"; + td::unlink(db_root_ + "/archive/states/" + fname.str()).ignore(); + return; + } register_perm_state(R.move_as_ok()); } }).ensure(); @@ -1037,6 +1060,14 @@ void ArchiveManager::persistent_state_gc(std::pair last) { res = 0; seqno = x.masterchain_seqno; }, + [&](const fileref::SplitAccountState &x) { + res = 0; + seqno = x.masterchain_seqno; + }, + [&](const fileref::SplitPersistentState &x) { + res = 0; + seqno = x.masterchain_seqno; + }, [&](const auto &obj) { res = -1; })); if (res == -1) { @@ -1315,6 +1346,8 @@ void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle han it->second.id.ref().visit(td::overloaded( [&](const fileref::ZeroStateShort &x) { res = -1; }, [&](const fileref::PersistentStateShort &x) { res = x.masterchain_seqno <= masterchain_seqno ? -1 : 1; }, + [&](const fileref::SplitPersistentState &x) { res = x.masterchain_seqno <= masterchain_seqno ? -1 : 1; }, + [&](const fileref::SplitAccountState &x) { res = x.masterchain_seqno <= masterchain_seqno ? -1 : 1; }, [&](const auto &obj) { res = 1; })); if (res <= 0) { it++; diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index 33dd1f3d6..361eaf516 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -19,6 +19,7 @@ #pragma once #include "archive-slice.hpp" +#include "interfaces/persistent-state.h" namespace ton { @@ -219,7 +220,7 @@ class ArchiveManager : public td::actor::Actor { PackageId get_max_temp_file_desc_idx(); PackageId get_prev_temp_file_desc_idx(PackageId id); - void add_persistent_state_impl(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise, + void add_persistent_state_impl(FileReferenceShort const &id, td::Promise promise, std::function)> create_writer); void register_perm_state(FileReferenceShort id); diff --git a/validator/db/fileref.cpp b/validator/db/fileref.cpp index 72aa32cfb..4e87327dc 100644 --- a/validator/db/fileref.cpp +++ b/validator/db/fileref.cpp @@ -106,6 +106,16 @@ std::string ZeroStateShort::filename_short() const { return PSTRING() << "zerostate_" << workchain << "_" << hash().to_hex(); } +std::string SplitAccountState::filename_short() const { + return PSTRING() << "stateaccount_" << masterchain_seqno << "_" << shard_id.workchain << "_" + << shard_to_str(shard_id.shard) << "_" << shard_to_str(effective_shard) << "_" << hash().to_hex(); +} + +std::string SplitPersistentState::filename_short() const { + return PSTRING() << "statesplit_" << masterchain_seqno << "_" << shard_id.workchain << "_" + << shard_to_str(shard_id.shard) << "_" << hash().to_hex(); +} + PersistentStateShort PersistentState::shortref() const { return PersistentStateShort{block_id.shard_full(), masterchain_block_id.seqno(), hash()}; } @@ -288,6 +298,24 @@ ShardIdFull FileReferenceShort::shard() const { return h; } +BlockSeqno FileReferenceShort::seqno_of_persistent_state() const { + BlockSeqno result; + ref_.visit(td::overloaded([&](const fileref::PersistentStateShort& x) { result = x.masterchain_seqno; }, + [&](const fileref::SplitAccountState& x) { result = x.masterchain_seqno; }, + [&](const fileref::SplitPersistentState& x) { result = x.masterchain_seqno; }, + [&](const fileref::ZeroStateShort) { result = 0; }, [&](const auto&) { CHECK(false); })); + return result; +} + +bool FileReferenceShort::is_state_like() const { + bool result = false; + ref_.visit(td::overloaded([&](const fileref::PersistentStateShort& x) { result = true; }, + [&](const fileref::SplitAccountState& x) { result = true; }, + [&](const fileref::SplitPersistentState& x) { result = true; }, + [&](const fileref::ZeroStateShort) { result = true; }, [&](const auto&) {})); + return result; +} + std::string FileReference::filename() const { std::string h; ref_.visit([&](const auto& obj) { h = obj.filename(); }); @@ -420,6 +448,35 @@ td::Result FileReferenceShort::create(std::string filename) } else { return td::Status::Error(ErrorCode::protoviolation, "too big file name"); } + } else if (token == "stateaccount") { + std::getline(ss, token, '_'); + TRY_RESULT(masterchain_seqno, td::to_integer_safe(token)); + std::getline(ss, token, '_'); + TRY_RESULT(workchain, td::to_integer_safe(token)); + std::getline(ss, token, '_'); + TRY_RESULT(shard, td::hex_to_integer_safe(token)); + std::getline(ss, token, '_'); + TRY_RESULT(effective_shard, td::hex_to_integer_safe(token)); + TRY_RESULT(vhash, get_token_hash(ss)); + if (!ss.eof()) { + return td::Status::Error(ErrorCode::protoviolation, "too big file name"); + } + if (!shard_is_proper_ancestor(shard, effective_shard)) { + return td::Status::Error(ErrorCode::protoviolation, "shard is not a proper ancestor of effective shard"); + } + return fileref::SplitAccountState{ShardIdFull{workchain, shard}, effective_shard, masterchain_seqno, vhash}; + } else if (token == "statesplit") { + std::getline(ss, token, '_'); + TRY_RESULT(masterchain_seqno, td::to_integer_safe(token)); + std::getline(ss, token, '_'); + TRY_RESULT(workchain, td::to_integer_safe(token)); + std::getline(ss, token, '_'); + TRY_RESULT(shard, td::hex_to_integer_safe(token)); + TRY_RESULT(vhash, get_token_hash(ss)); + if (!ss.eof()) { + return td::Status::Error(ErrorCode::protoviolation, "too big file name"); + } + return fileref::SplitPersistentState{ShardIdFull{workchain, shard}, masterchain_seqno, vhash}; } else if (token == "state") { std::getline(ss, token, '_'); TRY_RESULT(masterchain_seqno, td::to_integer_safe(token)); diff --git a/validator/db/fileref.hpp b/validator/db/fileref.hpp index 14060fb45..df368f759 100644 --- a/validator/db/fileref.hpp +++ b/validator/db/fileref.hpp @@ -110,8 +110,77 @@ class ZeroState { BlockIdExt block_id; }; +class SplitAccountState { + public: + static SplitAccountState create(BlockIdExt block_id, BlockIdExt masterchain_block_id, ShardId effective_shard) { + auto hash = create_hash_tl_object( + create_tl_block_id(block_id), create_tl_block_id(masterchain_block_id), effective_shard); + + return { + .shard_id = block_id.shard_full(), + .effective_shard = effective_shard, + .masterchain_seqno = masterchain_block_id.seqno(), + .hashv = hash, + }; + } + + FileHash hash() const { + return hashv; + } + + ShardIdFull shard() const { + return {shard_id.workchain, effective_shard}; + } + + std::string filename_short() const; + + ShardIdFull shard_id; + ShardId effective_shard; + BlockSeqno masterchain_seqno; + FileHash hashv; +}; + +class SplitPersistentState { + public: + static SplitPersistentState create(BlockIdExt block_id, BlockIdExt masterchain_block_id) { + auto hash = create_hash_tl_object( + create_tl_block_id(block_id), create_tl_block_id(masterchain_block_id)); + + return { + .shard_id = block_id.shard_full(), + .masterchain_seqno = masterchain_block_id.seqno(), + .hashv = hash, + }; + } + + FileHash hash() const { + return hashv; + } + + ShardIdFull shard() const { + return shard_id; + } + + std::string filename_short() const; + + ShardIdFull shard_id; + BlockSeqno masterchain_seqno; + FileHash hashv; +}; + class PersistentStateShort { public: + static PersistentStateShort create(BlockIdExt block_id, BlockIdExt masterchain_block_id) { + auto hash = create_hash_tl_object( + create_tl_block_id(block_id), create_tl_block_id(masterchain_block_id)); + + return { + .shard_id = block_id.shard_full(), + .masterchain_seqno = masterchain_block_id.seqno(), + .hashv = hash, + }; + } + FileHash hash() const { return hashv; } @@ -346,9 +415,10 @@ class BlockInfo { class FileReferenceShort { private: - td::Variant + td::Variant ref_; public: @@ -366,6 +436,8 @@ class FileReferenceShort { td::Bits256 hash() const; ShardIdFull shard() const; + BlockSeqno seqno_of_persistent_state() const; + bool is_state_like() const; std::string filename_short() const; }; diff --git a/validator/interfaces/persistent-state.h b/validator/interfaces/persistent-state.h new file mode 100644 index 000000000..4a03a8f2b --- /dev/null +++ b/validator/interfaces/persistent-state.h @@ -0,0 +1,38 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "td/utils/Variant.h" +#include "ton/ton-types.h" + +namespace ton { + +namespace validator { + +struct UnsplitStateType {}; + +struct SplitAccountStateType { + ShardId effective_shard_id; +}; + +struct SplitPersistentStateType {}; + +using PersistentStateType = td::Variant; + +} // namespace validator + +} // namespace ton From f3c43560de8e06b473a2ad36aba75e69dff66877 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Wed, 4 Jun 2025 09:52:36 -0400 Subject: [PATCH 295/388] Expose PersistentStateType and pass it through a million actors What are we, an enterprise? More seriously, the commit makes it a responsibility of (indirect) users of ArchiveManager to pass PersistentStateType. For now, we still hardcode UnsplitStateType everywhere. --- validator/db/archive-manager.cpp | 32 +++++++++++------------- validator/db/archive-manager.hpp | 18 ++++++------- validator/db/rootdb.cpp | 24 ++++++++++-------- validator/db/rootdb.hpp | 15 +++++------ validator/downloaders/download-state.cpp | 2 +- validator/full-node-master.cpp | 8 +++--- validator/full-node-shard.cpp | 8 +++--- validator/interfaces/db.h | 14 +++++++---- validator/interfaces/validator-manager.h | 4 ++- validator/manager-disk.cpp | 19 ++++++++------ validator/manager-disk.hpp | 17 +++++++------ validator/manager-hardfork.hpp | 17 +++++++------ validator/manager.cpp | 25 ++++++++++-------- validator/manager.hpp | 17 +++++++------ validator/net/download-state.cpp | 2 +- validator/state-serializer.cpp | 4 +-- validator/validator.h | 10 +++++--- 17 files changed, 127 insertions(+), 109 deletions(-) diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index 926e7c3b5..f3d8b9425 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -361,19 +361,20 @@ FileReferenceShort create_persistent_state_id(BlockIdExt block_id, BlockIdExt mc } // namespace -void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data, +void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::BufferSlice data, td::Promise promise) { auto create_writer = [&](std::string path, td::Promise P) { td::actor::create_actor("writefile", db_root_ + "/archive/tmp/", std::move(path), std::move(data), std::move(P)) .release(); }; - // TODO: Allow specifying state type. - add_persistent_state_impl(create_persistent_state_id(block_id, masterchain_block_id, UnsplitStateType{}), - std::move(promise), std::move(create_writer)); + add_persistent_state_impl(create_persistent_state_id(block_id, masterchain_block_id, type), std::move(promise), + std::move(create_writer)); } void ArchiveManager::add_persistent_state_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, std::function write_state, td::Promise promise) { auto create_writer = [&](std::string path, td::Promise P) { @@ -381,9 +382,8 @@ void ArchiveManager::add_persistent_state_gen(BlockIdExt block_id, BlockIdExt ma std::move(write_state), std::move(P)) .release(); }; - // TODO: Allow specifying state type. - add_persistent_state_impl(create_persistent_state_id(block_id, masterchain_block_id, UnsplitStateType{}), - std::move(promise), std::move(create_writer)); + add_persistent_state_impl(create_persistent_state_id(block_id, masterchain_block_id, type), std::move(promise), + std::move(create_writer)); } void ArchiveManager::add_persistent_state_impl( @@ -450,9 +450,8 @@ void ArchiveManager::get_previous_persistent_state_files( } void ArchiveManager::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { - // TODO: Allow specifying state type. - auto id = create_persistent_state_id(block_id, masterchain_block_id, UnsplitStateType{}); + PersistentStateType type, td::Promise promise) { + auto id = create_persistent_state_id(block_id, masterchain_block_id, type); auto hash = id.hash(); if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) { promise.set_error(td::Status::Error(ErrorCode::notready, "state file not in db")); @@ -463,10 +462,10 @@ void ArchiveManager::get_persistent_state(BlockIdExt block_id, BlockIdExt master td::actor::create_actor("readfile", path, 0, -1, 0, std::move(promise)).release(); } -void ArchiveManager::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_size, td::Promise promise) { - // TODO: Allow specifying state type. - auto id = create_persistent_state_id(block_id, masterchain_block_id, UnsplitStateType{}); +void ArchiveManager::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::int64 offset, td::int64 max_size, + td::Promise promise) { + auto id = create_persistent_state_id(block_id, masterchain_block_id, type); auto hash = id.hash(); if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) { promise.set_error(td::Status::Error(ErrorCode::notready, "state file not in db")); @@ -478,9 +477,8 @@ void ArchiveManager::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt } void ArchiveManager::get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { - // TODO: Allow specifying state type. - auto id = create_persistent_state_id(block_id, masterchain_block_id, UnsplitStateType{}); + PersistentStateType type, td::Promise promise) { + auto id = create_persistent_state_id(block_id, masterchain_block_id, type); auto hash = id.hash(); auto it = perm_states_.find({masterchain_block_id.seqno(), hash}); if (it == perm_states_.end()) { diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index 361eaf516..0d9a59df3 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -44,16 +44,16 @@ class ArchiveManager : public td::actor::Actor { void get_file(ConstBlockHandle handle, FileReference ref_id, td::Promise promise); void add_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise promise); - void add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data, - td::Promise promise); - void add_persistent_state_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_state, - td::Promise promise); + void add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::BufferSlice data, td::Promise promise); + void add_persistent_state_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + std::function write_state, td::Promise promise); void get_zero_state(BlockIdExt block_id, td::Promise promise); - void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise); - void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_size, td::Promise promise); - void get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::Promise promise); + void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::int64 offset, td::int64 max_size, td::Promise promise); + void get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise); void check_zero_state(BlockIdExt block_id, td::Promise promise); void get_previous_persistent_state_files(BlockSeqno cur_mc_seqno, diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 90546fbef..d041228c0 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -285,35 +285,37 @@ void RootDb::get_cell_db_reader(td::Promise> p td::actor::send_closure(cell_db_, &CellDb::get_cell_db_reader, std::move(promise)); } -void RootDb::store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, - td::Promise promise) { - td::actor::send_closure(archive_db_, &ArchiveManager::add_persistent_state, block_id, masterchain_block_id, +void RootDb::store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::BufferSlice state, td::Promise promise) { + td::actor::send_closure(archive_db_, &ArchiveManager::add_persistent_state, block_id, masterchain_block_id, type, std::move(state), std::move(promise)); } void RootDb::store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, std::function write_data, td::Promise promise) { - td::actor::send_closure(archive_db_, &ArchiveManager::add_persistent_state_gen, block_id, masterchain_block_id, + td::actor::send_closure(archive_db_, &ArchiveManager::add_persistent_state_gen, block_id, masterchain_block_id, type, std::move(write_data), std::move(promise)); } -void RootDb::get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, +void RootDb::get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) { - td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state, block_id, masterchain_block_id, + td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state, block_id, masterchain_block_id, type, std::move(promise)); } -void RootDb::get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_size, td::Promise promise) { +void RootDb::get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::int64 offset, td::int64 max_size, + td::Promise promise) { td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state_slice, block_id, masterchain_block_id, - offset, max_size, std::move(promise)); + type, offset, max_size, std::move(promise)); } void RootDb::get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { + PersistentStateType type, td::Promise promise) { td::actor::send_closure(archive_db_, &ArchiveManager::get_persistent_state_file_size, block_id, masterchain_block_id, - std::move(promise)); + type, std::move(promise)); } void RootDb::store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) { diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 2ccd7f942..57e560e1c 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -71,16 +71,17 @@ class RootDb : public Db { td::actor::send_closure(validator_manager_, &ValidatorManager::get_block_handle, id, force, std::move(promise)); } - void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, - td::Promise promise) override; - void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::BufferSlice state, td::Promise promise) override; + void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, std::function write_data, td::Promise promise) override; - void get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, + void get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) override; - void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_length, td::Promise promise) override; - void get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::int64 offset, td::int64 max_length, + td::Promise promise) override; + void get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) override; void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; void get_zero_state_file(BlockIdExt block_id, td::Promise promise) override; diff --git a/validator/downloaders/download-state.cpp b/validator/downloaders/download-state.cpp index 8473cb228..4335a2956 100644 --- a/validator/downloaders/download-state.cpp +++ b/validator/downloaders/download-state.cpp @@ -189,7 +189,7 @@ void DownloadShardState::checked_shard_state() { std::move(P)); } else { td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file, block_id_, masterchain_block_id_, - std::move(data_), std::move(P)); + UnsplitStateType{}, std::move(data_), std::move(P)); } } diff --git a/validator/full-node-master.cpp b/validator/full-node-master.cpp index 8dc2a6815..996ce2219 100644 --- a/validator/full-node-master.cpp +++ b/validator/full-node-master.cpp @@ -288,7 +288,7 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo auto block_id = create_block_id(query.block_); auto masterchain_block_id = create_block_id(query.masterchain_block_); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, - masterchain_block_id, std::move(P)); + masterchain_block_id, UnsplitStateType{}, std::move(P)); } void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, @@ -301,7 +301,7 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo auto block_id = create_block_id(query.block_); auto masterchain_block_id = create_block_id(query.masterchain_block_); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, - masterchain_block_id, std::move(P)); + masterchain_block_id, UnsplitStateType{}, std::move(P)); } void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, @@ -361,7 +361,7 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo auto block_id = create_block_id(query.block_); auto masterchain_block_id = create_block_id(query.masterchain_block_); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state, block_id, - masterchain_block_id, std::move(P)); + masterchain_block_id, UnsplitStateType{}, std::move(P)); } void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, @@ -378,7 +378,7 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo auto block_id = create_block_id(query.block_); auto masterchain_block_id = create_block_id(query.masterchain_block_); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_slice, block_id, - masterchain_block_id, query.offset_, query.max_size_, std::move(P)); + masterchain_block_id, UnsplitStateType{}, query.offset_, query.max_size_, std::move(P)); } void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 2543431a8..db629c898 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -523,7 +523,7 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod VLOG(FULL_NODE_DEBUG) << "Got query preparePersistentState " << block_id.to_str() << " " << masterchain_block_id.to_str() << " from " << src; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, - masterchain_block_id, std::move(P)); + masterchain_block_id, UnsplitStateType{}, std::move(P)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, @@ -538,7 +538,7 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod VLOG(FULL_NODE_DEBUG) << "Got query getPersistentStateSize " << block_id.to_str() << " " << masterchain_block_id.to_str() << " from " << src; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, - masterchain_block_id, std::move(P)); + masterchain_block_id, UnsplitStateType{}, std::move(P)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, @@ -607,7 +607,7 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod VLOG(FULL_NODE_DEBUG) << "Got query downloadPersistentState " << block_id.to_str() << " " << masterchain_block_id.to_str() << " from " << src; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_slice, block_id, - masterchain_block_id, 0, max_size + 1, std::move(P)); + masterchain_block_id, UnsplitStateType{}, 0, max_size + 1, std::move(P)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, @@ -631,7 +631,7 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod promise.set_value(R.move_as_ok()); }); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_slice, block_id, - masterchain_block_id, query.offset_, query.max_size_, std::move(P)); + masterchain_block_id, UnsplitStateType{}, query.offset_, query.max_size_, std::move(P)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index f21c6f902..afd087549 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -20,6 +20,7 @@ #include "ton/ton-types.h" #include "validator/interfaces/block-handle.h" +#include "validator/interfaces/persistent-state.h" #include "validator/interfaces/validator-manager.h" namespace ton { @@ -53,17 +54,20 @@ class Db : public td::actor::Actor { virtual void get_block_state(ConstBlockHandle handle, td::Promise> promise) = 0; virtual void get_cell_db_reader(td::Promise> promise) = 0; - virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, + virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::BufferSlice state, td::Promise promise) = 0; virtual void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, std::function write_data, td::Promise promise) = 0; - virtual void get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, + virtual void get_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) = 0; - virtual void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_length, td::Promise promise) = 0; + virtual void get_persistent_state_file_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::int64 offset, td::int64 max_length, + td::Promise promise) = 0; virtual void get_persistent_state_file_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) = 0; + PersistentStateType type, td::Promise promise) = 0; virtual void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) = 0; virtual void get_zero_state_file(BlockIdExt block_id, td::Promise promise) = 0; virtual void check_zero_state_file_exists(BlockIdExt block_id, td::Promise promise) = 0; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 077ce7849..2aebaca74 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -71,9 +71,11 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) = 0; virtual void get_cell_db_reader(td::Promise> promise) = 0; - virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, + virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::BufferSlice state, td::Promise promise) = 0; virtual void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, std::function write_data, td::Promise promise) = 0; virtual void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 82081cd11..c48528b48 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -201,13 +201,14 @@ void ValidatorManagerImpl::get_zero_state(BlockIdExt block_id, td::Promise promise) { - td::actor::send_closure(db_, &Db::get_persistent_state_file_size, block_id, masterchain_block_id, + PersistentStateType type, td::Promise promise) { + td::actor::send_closure(db_, &Db::get_persistent_state_file_size, block_id, masterchain_block_id, type, std::move(promise)); } void ValidatorManagerImpl::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { - td::actor::send_closure(db_, &Db::get_persistent_state_file, block_id, masterchain_block_id, std::move(promise)); + PersistentStateType type, td::Promise promise) { + td::actor::send_closure(db_, &Db::get_persistent_state_file, block_id, masterchain_block_id, type, + std::move(promise)); } void ValidatorManagerImpl::get_block_proof(BlockHandle handle, td::Promise promise) { @@ -692,15 +693,17 @@ void ValidatorManagerImpl::get_cell_db_reader(td::Promise promise) { - td::actor::send_closure(db_, &Db::store_persistent_state_file, block_id, masterchain_block_id, std::move(state), + PersistentStateType type, td::BufferSlice state, + td::Promise promise) { + td::actor::send_closure(db_, &Db::store_persistent_state_file, block_id, masterchain_block_id, type, std::move(state), std::move(promise)); } void ValidatorManagerImpl::store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_data, + PersistentStateType type, + std::function write_data, td::Promise promise) { - td::actor::send_closure(db_, &Db::store_persistent_state_file_gen, block_id, masterchain_block_id, + td::actor::send_closure(db_, &Db::store_persistent_state_file_gen, block_id, masterchain_block_id, type, std::move(write_data), std::move(promise)); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index cdf6c1cae..ee83fc850 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -108,12 +108,13 @@ class ValidatorManagerImpl : public ValidatorManager { void get_block_data(BlockHandle handle, td::Promise promise) override; void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) override; void get_zero_state(BlockIdExt block_id, td::Promise promise) override; - void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) override; - void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, + void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) override; - void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_length, td::Promise promise) override { + void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::int64 offset, td::int64 max_length, + td::Promise promise) override { UNREACHABLE(); } void get_previous_persistent_state_files( @@ -150,10 +151,10 @@ class ValidatorManagerImpl : public ValidatorManager { void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; void get_cell_db_reader(td::Promise> promise) override; - void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, - td::Promise promise) override; - void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_data, + void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::BufferSlice state, td::Promise promise) override; + void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + std::function write_data, td::Promise promise) override; void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index fbb40901f..ecfc8de7e 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -128,16 +128,17 @@ class ValidatorManagerImpl : public ValidatorManager { void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) override { UNREACHABLE(); } - void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) override { UNREACHABLE(); } - void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, + void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) override { UNREACHABLE(); } - void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_length, td::Promise promise) override { + void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::int64 offset, td::int64 max_length, + td::Promise promise) override { UNREACHABLE(); } void get_previous_persistent_state_files( @@ -176,12 +177,12 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } void get_cell_db_reader(td::Promise> promise) override; - void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, - td::Promise promise) override { + void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::BufferSlice state, td::Promise promise) override { UNREACHABLE(); } - void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_data, + void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + std::function write_data, td::Promise promise) override { UNREACHABLE(); } diff --git a/validator/manager.cpp b/validator/manager.cpp index a94f9c07c..86d794bc8 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -305,20 +305,21 @@ void ValidatorManagerImpl::get_zero_state(BlockIdExt block_id, td::Promise promise) { - td::actor::send_closure(db_, &Db::get_persistent_state_file_size, block_id, masterchain_block_id, + PersistentStateType type, td::Promise promise) { + td::actor::send_closure(db_, &Db::get_persistent_state_file_size, block_id, masterchain_block_id, type, std::move(promise)); } void ValidatorManagerImpl::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise) { - td::actor::send_closure(db_, &Db::get_persistent_state_file, block_id, masterchain_block_id, std::move(promise)); + PersistentStateType type, td::Promise promise) { + td::actor::send_closure(db_, &Db::get_persistent_state_file, block_id, masterchain_block_id, type, + std::move(promise)); } void ValidatorManagerImpl::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::int64 offset, td::int64 max_length, + PersistentStateType type, td::int64 offset, td::int64 max_length, td::Promise promise) { - td::actor::send_closure(db_, &Db::get_persistent_state_file_slice, block_id, masterchain_block_id, offset, max_length, - std::move(promise)); + td::actor::send_closure(db_, &Db::get_persistent_state_file_slice, block_id, masterchain_block_id, type, offset, + max_length, std::move(promise)); } void ValidatorManagerImpl::get_previous_persistent_state_files( @@ -1237,15 +1238,17 @@ void ValidatorManagerImpl::get_cell_db_reader(td::Promise promise) { - td::actor::send_closure(db_, &Db::store_persistent_state_file, block_id, masterchain_block_id, std::move(state), + PersistentStateType type, td::BufferSlice state, + td::Promise promise) { + td::actor::send_closure(db_, &Db::store_persistent_state_file, block_id, masterchain_block_id, type, std::move(state), std::move(promise)); } void ValidatorManagerImpl::store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_data, + PersistentStateType type, + std::function write_data, td::Promise promise) { - td::actor::send_closure(db_, &Db::store_persistent_state_file_gen, block_id, masterchain_block_id, + td::actor::send_closure(db_, &Db::store_persistent_state_file_gen, block_id, masterchain_block_id, type, std::move(write_data), std::move(promise)); } diff --git a/validator/manager.hpp b/validator/manager.hpp index a321684b7..ab86d2d37 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -378,12 +378,13 @@ class ValidatorManagerImpl : public ValidatorManager { void get_block_data(BlockHandle handle, td::Promise promise) override; void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) override; void get_zero_state(BlockIdExt block_id, td::Promise promise) override; - void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) override; - void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, + void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) override; - void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_length, td::Promise promise) override; + void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::int64 offset, td::int64 max_length, + td::Promise promise) override; void get_previous_persistent_state_files( BlockSeqno cur_mc_seqno, td::Promise>> promise) override; void get_block_proof(BlockHandle handle, td::Promise promise) override; @@ -410,10 +411,10 @@ class ValidatorManagerImpl : public ValidatorManager { void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; void get_cell_db_reader(td::Promise> promise) override; - void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, - td::Promise promise) override; - void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_data, + void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::BufferSlice state, td::Promise promise) override; + void store_persistent_state_file_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + std::function write_data, td::Promise promise) override; void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, diff --git a/validator/net/download-state.cpp b/validator/net/download-state.cpp index c481adbbf..3824fa01c 100644 --- a/validator/net/download-state.cpp +++ b/validator/net/download-state.cpp @@ -74,7 +74,7 @@ void DownloadState::start_up() { alarm_timestamp() = timeout_; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state, block_id_, - masterchain_block_id_, + masterchain_block_id_, UnsplitStateType{}, [SelfId = actor_id(this), block_id = block_id_](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &DownloadState::get_block_handle); diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index b8fd84d17..73ce440d3 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -418,7 +418,7 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state }); td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, masterchain_handle_->id(), - masterchain_handle_->id(), write_data, std::move(P)); + masterchain_handle_->id(), UnsplitStateType{}, write_data, std::move(P)); current_status_ = PSTRING() << "serializing masterchain state " << state->get_block_id().id.to_str(); current_status_ts_ = td::Timestamp::now(); @@ -488,7 +488,7 @@ void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Refid(), - masterchain_handle_->id(), write_data, std::move(P)); + masterchain_handle_->id(), UnsplitStateType{}, write_data, std::move(P)); current_status_ = PSTRING() << "serializing shard state " << next_idx_ << "/" << shards_.size() << " " << state->get_block_id().id.to_str(); current_status_ts_ = td::Timestamp::now(); diff --git a/validator/validator.h b/validator/validator.h index dc0f5761a..d988261a9 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -22,6 +22,7 @@ #include #include +#include "interfaces/persistent-state.h" #include "td/actor/actor.h" #include "ton/ton-types.h" @@ -231,12 +232,13 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_block_data(BlockHandle handle, td::Promise promise) = 0; virtual void check_zero_state_exists(BlockIdExt block_id, td::Promise promise) = 0; virtual void get_zero_state(BlockIdExt block_id, td::Promise promise) = 0; - virtual void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, + virtual void get_persistent_state_size(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) = 0; - virtual void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, + virtual void get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::Promise promise) = 0; - virtual void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, - td::int64 max_length, td::Promise promise) = 0; + virtual void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::int64 offset, td::int64 max_length, + td::Promise promise) = 0; virtual void get_previous_persistent_state_files( BlockSeqno cur_mc_seqno, td::Promise>> promise) = 0; virtual void get_block_proof(BlockHandle handle, td::Promise promise) = 0; From 0ed9c4a9c250affff99eb9389eabdbd11f6f8036 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Wed, 4 Jun 2025 15:46:48 -0400 Subject: [PATCH 296/388] Move around request handlers in validator/full-node-{master,shard}.cpp No functional change. This will make next commit easier to review. --- validator/full-node-master.cpp | 60 +++++++++++++------------- validator/full-node-shard.cpp | 78 +++++++++++++++++----------------- 2 files changed, 69 insertions(+), 69 deletions(-) diff --git a/validator/full-node-master.cpp b/validator/full-node-master.cpp index 996ce2219..8ca0aab09 100644 --- a/validator/full-node-master.cpp +++ b/validator/full-node-master.cpp @@ -291,19 +291,6 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo masterchain_block_id, UnsplitStateType{}, std::move(P)); } -void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, - td::Promise promise) { - auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - TRY_RESULT_PROMISE(promise, size, std::move(R)); - promise.set_value(create_serialize_tl_object(size)); - }); - auto block_id = create_block_id(query.block_); - auto masterchain_block_id = create_block_id(query.masterchain_block_); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, - masterchain_block_id, UnsplitStateType{}, std::move(P)); -} - void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, td::Promise promise) { auto cnt = static_cast(query.max_size_); @@ -364,23 +351,6 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo masterchain_block_id, UnsplitStateType{}, std::move(P)); } -void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, - td::Promise promise) { - auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error_prefix("failed to get state from db: ")); - return; - } - - promise.set_value(R.move_as_ok()); - }); - auto block_id = create_block_id(query.block_); - auto masterchain_block_id = create_block_id(query.masterchain_block_); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_slice, block_id, - masterchain_block_id, UnsplitStateType{}, query.offset_, query.max_size_, std::move(P)); -} - void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, td::Promise promise) { promise.set_value( @@ -432,6 +402,36 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo promise.set_value(create_serialize_tl_object()); } +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix("failed to get state from db: ")); + return; + } + + promise.set_value(R.move_as_ok()); + }); + auto block_id = create_block_id(query.block_); + auto masterchain_block_id = create_block_id(query.masterchain_block_); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_slice, block_id, + masterchain_block_id, UnsplitStateType{}, query.offset_, query.max_size_, std::move(P)); +} + +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, size, std::move(R)); + promise.set_value(create_serialize_tl_object(size)); + }); + auto block_id = create_block_id(query.block_); + auto masterchain_block_id = create_block_id(query.masterchain_block_); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, + masterchain_block_id, UnsplitStateType{}, std::move(P)); +} + void FullNodeMasterImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise) { auto BX = fetch_tl_prefix(query, true); diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index db629c898..baf55f052 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -526,21 +526,6 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod masterchain_block_id, UnsplitStateType{}, std::move(P)); } -void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, - td::Promise promise) { - auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - TRY_RESULT_PROMISE(promise, size, std::move(R)); - promise.set_value(create_serialize_tl_object(size)); - }); - auto block_id = create_block_id(query.block_); - auto masterchain_block_id = create_block_id(query.masterchain_block_); - VLOG(FULL_NODE_DEBUG) << "Got query getPersistentStateSize " << block_id.to_str() << " " - << masterchain_block_id.to_str() << " from " << src; - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, - masterchain_block_id, UnsplitStateType{}, std::move(P)); -} - void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, td::Promise promise) { auto cnt = static_cast(query.max_size_); @@ -610,30 +595,6 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod masterchain_block_id, UnsplitStateType{}, 0, max_size + 1, std::move(P)); } -void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, - td::Promise promise) { - auto block_id = create_block_id(query.block_); - auto masterchain_block_id = create_block_id(query.masterchain_block_); - VLOG(FULL_NODE_DEBUG) << "Got query downloadPersistentStateSlice " << block_id.to_str() << " " - << masterchain_block_id.to_str() << " " << query.offset_ << " " << query.max_size_ << " from " - << src; - if (query.max_size_ < 0 || query.max_size_ > (1 << 24)) { - promise.set_error(td::Status::Error(ErrorCode::protoviolation, "invalid max_size")); - return; - } - auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error_prefix("failed to get state from db: ")); - return; - } - - promise.set_value(R.move_as_ok()); - }); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_slice, block_id, - masterchain_block_id, UnsplitStateType{}, query.offset_, query.max_size_, std::move(P)); -} - void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, td::Promise promise) { VLOG(FULL_NODE_DEBUG) << "Got query getCapabilities from " << src; @@ -741,6 +702,45 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod }); } +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, + td::Promise promise) { + auto block_id = create_block_id(query.block_); + auto masterchain_block_id = create_block_id(query.masterchain_block_); + VLOG(FULL_NODE_DEBUG) << "Got query downloadPersistentStateSlice " << block_id.to_str() << " " + << masterchain_block_id.to_str() << " " << query.offset_ << " " << query.max_size_ << " from " + << src; + if (query.max_size_ < 0 || query.max_size_ > (1 << 24)) { + promise.set_error(td::Status::Error(ErrorCode::protoviolation, "invalid max_size")); + return; + } + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix("failed to get state from db: ")); + return; + } + + promise.set_value(R.move_as_ok()); + }); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_slice, block_id, + masterchain_block_id, UnsplitStateType{}, query.offset_, query.max_size_, std::move(P)); +} + +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, size, std::move(R)); + promise.set_value(create_serialize_tl_object(size)); + }); + auto block_id = create_block_id(query.block_); + auto masterchain_block_id = create_block_id(query.masterchain_block_); + VLOG(FULL_NODE_DEBUG) << "Got query getPersistentStateSize " << block_id.to_str() << " " + << masterchain_block_id.to_str() << " from " << src; + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, + masterchain_block_id, UnsplitStateType{}, std::move(P)); +} + void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise) { if (!active_) { From 27a4b367c98fcc47016d7b214cc48c5a3120fab9 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Sat, 14 Jun 2025 14:03:01 -0400 Subject: [PATCH 297/388] Report error if state is not found in getPersistentStateSize query This will allow us to omit preparePersistentState query in the future. --- tl/generate/scheme/ton_api.tl | 1 + tl/generate/scheme/ton_api.tlo | Bin 103388 -> 103464 bytes validator/full-node-master.cpp | 7 +++++-- validator/full-node-shard.cpp | 7 +++++-- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 06f202a7f..dacf442d8 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -410,6 +410,7 @@ tonNode.preparedProofLink = tonNode.PreparedProof; tonNode.preparedState = tonNode.PreparedState; tonNode.notFoundState = tonNode.PreparedState; tonNode.persistentStateSize size:long = tonNode.PersistentStateSize; +tonNode.persistentStateSizeNotFound = tonNode.PersistentStateSize; tonNode.prepared = tonNode.Prepared; tonNode.notFound = tonNode.Prepared; tonNode.data data:bytes = tonNode.Data; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index f30418f7bd78ff6a6f1b8519fe1c506e3f5999ad..96aee7ac06af55f9aa09e7eb9f4e00f65c4234a5 100644 GIT binary patch delta 129 zcmcb!oNdJlwhdQ|S#}B+aZJ8pEHlwTVl$742s`8U&5G9BA40?vb{mL-6fi)+RXMJh z>;qxHl&9~NVU$+z%P(=uFU?BQ promise) { auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - TRY_RESULT_PROMISE(promise, size, std::move(R)); - promise.set_value(create_serialize_tl_object(size)); + if (R.is_error()) { + promise.set_value(create_serialize_tl_object()); + } else { + promise.set_value(create_serialize_tl_object(R.move_as_ok())); + } }); auto block_id = create_block_id(query.block_); auto masterchain_block_id = create_block_id(query.masterchain_block_); diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index baf55f052..480c24b64 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -730,8 +730,11 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod td::Promise promise) { auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { - TRY_RESULT_PROMISE(promise, size, std::move(R)); - promise.set_value(create_serialize_tl_object(size)); + if (R.is_error()) { + promise.set_value(create_serialize_tl_object()); + } else { + promise.set_value(create_serialize_tl_object(R.move_as_ok())); + } }); auto block_id = create_block_id(query.block_); auto masterchain_block_id = create_block_id(query.masterchain_block_); From 8d2551ae289a3caaa0a9a0efa8d7ef8e7516889d Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Wed, 4 Jun 2025 15:40:48 -0400 Subject: [PATCH 298/388] Teach full node to response with split persistent states We introduce two additional ADNL requests that can be made between nodes: downloadPersistentStateSliceV2 and getPersistentStateSizeV2. They differ from "V1" versions by inclusion of "effective shard" parameter. The parameter denotes a split account state if block.shard is a proper ancestor of effective_shard. If block.shard == effective_shard, we treat this as a request for split state header, and if it is 0 (or invalid), we serve an unsplit state. Old requests are wired to call V2 with effective shard set to 0. --- tl/generate/scheme/ton_api.tl | 13 ++++--- tl/generate/scheme/ton_api.tlo | Bin 103464 -> 103992 bytes validator/full-node-master.cpp | 28 ++++++++++----- validator/full-node-master.hpp | 6 ++++ validator/full-node-shard.cpp | 38 +++++++++++++------- validator/full-node-shard.hpp | 4 +++ validator/interfaces/persistent-state.h | 44 ++++++++++++++++++++++++ 7 files changed, 108 insertions(+), 25 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index dacf442d8..9b535ba0e 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -409,6 +409,7 @@ tonNode.preparedProof = tonNode.PreparedProof; tonNode.preparedProofLink = tonNode.PreparedProof; tonNode.preparedState = tonNode.PreparedState; tonNode.notFoundState = tonNode.PreparedState; +tonNode.persistentStateIdV2 block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt effective_shard:long = tonNode.PersistentStateIdV2; tonNode.persistentStateSize size:long = tonNode.PersistentStateSize; tonNode.persistentStateSizeNotFound = tonNode.PersistentStateSize; tonNode.prepared = tonNode.Prepared; @@ -476,15 +477,11 @@ tonNode.prepareBlockProofs blocks:(vector tonNode.blockIdExt) allow_partial:Bool tonNode.prepareKeyBlockProofs blocks:(vector tonNode.blockIdExt) allow_partial:Bool = tonNode.PreparedProof; tonNode.prepareBlock block:tonNode.blockIdExt = tonNode.Prepared; tonNode.prepareBlocks blocks:(vector tonNode.blockIdExt) = tonNode.Prepared; -tonNode.preparePersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PreparedState; -tonNode.getPersistentStateSize block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PersistentStateSize; tonNode.prepareZeroState block:tonNode.blockIdExt = tonNode.PreparedState; tonNode.getNextKeyBlockIds block:tonNode.blockIdExt max_size:int = tonNode.KeyBlocks; tonNode.downloadNextBlockFull prev_block:tonNode.blockIdExt = tonNode.DataFull; tonNode.downloadBlockFull block:tonNode.blockIdExt = tonNode.DataFull; tonNode.downloadBlock block:tonNode.blockIdExt = tonNode.Data; -tonNode.downloadPersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.Data; -tonNode.downloadPersistentStateSlice block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt offset:long max_size:long = tonNode.Data; tonNode.downloadZeroState block:tonNode.blockIdExt = tonNode.Data; tonNode.downloadBlockProof block:tonNode.blockIdExt = tonNode.Data; tonNode.downloadKeyBlockProof block:tonNode.blockIdExt = tonNode.Data; @@ -496,6 +493,14 @@ tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data; tonNode.getOutMsgQueueProof dst_shard:tonNode.shardId blocks:(vector tonNode.blockIdExt) limits:tonNode.importedMsgQueueLimits = tonNode.OutMsgQueueProof; +tonNode.downloadPersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.Data; +tonNode.downloadPersistentStateSlice block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt offset:long max_size:long = tonNode.Data; +tonNode.getPersistentStateSize block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PersistentStateSize; +tonNode.preparePersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.PreparedState; + +tonNode.downloadPersistentStateSliceV2 state:tonNode.persistentStateIdV2 offset:long max_size:long = tonNode.Data; +tonNode.getPersistentStateSizeV2 state:tonNode.persistentStateIdV2 = tonNode.PersistentStateSize; + tonNode.getCapabilities = tonNode.Capabilities; tonNode.slave.sendExtMessage message:tonNode.externalMessage = tonNode.Success; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 96aee7ac06af55f9aa09e7eb9f4e00f65c4234a5..27bc598a75eb3a54807bba7b2ef4f942a1e368ad 100644 GIT binary patch delta 227 zcmZ3nf^EkdHr_|G^{p77KyxGSabtz#fH$1dCHZ-N`6;P-0jWjBnZ+fkc_qOmi6yC? zDPcwvH%M&0V(i1txMQ=T_4J3VVD-}#Wf{eJLDC@Z&0|77%+o)%GX_sSQ=`KI6575- zhEYJDv3L7kZ^rvJtO6}PZEDjGwlk{mgc&iksBndW^-L~&Ee=*beM2Xs!(@jqJd+P3 zaZGOzXB5~zr-QMFg;iqtB5#H1GkX~2_<$N27 promise) { + auto query_v2 = create_tl_object(persistent_state_id_from_v1_query(query)); + return process_query(src, *query_v2, std::move(promise)); +} + void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, td::Promise promise) { auto cnt = static_cast(query.max_size_); @@ -351,6 +357,13 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo masterchain_block_id, UnsplitStateType{}, std::move(P)); } +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, + td::Promise promise) { + auto query_v2 = create_tl_object( + persistent_state_id_from_v1_query(query), query.offset_, query.max_size_); + return process_query(src, *query_v2, std::move(promise)); +} + void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, td::Promise promise) { promise.set_value( @@ -402,7 +415,8 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo promise.set_value(create_serialize_tl_object()); } -void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, + ton_api::tonNode_downloadPersistentStateSliceV2 &query, td::Promise promise) { auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { @@ -413,13 +427,12 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo promise.set_value(R.move_as_ok()); }); - auto block_id = create_block_id(query.block_); - auto masterchain_block_id = create_block_id(query.masterchain_block_); + auto [block_id, mc_block_id, state_type] = persistent_state_from_v2_query(query); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_slice, block_id, - masterchain_block_id, UnsplitStateType{}, query.offset_, query.max_size_, std::move(P)); + mc_block_id, state_type, query.offset_, query.max_size_, std::move(P)); } -void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, +void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSizeV2 &query, td::Promise promise) { auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { @@ -429,10 +442,9 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo promise.set_value(create_serialize_tl_object(R.move_as_ok())); } }); - auto block_id = create_block_id(query.block_); - auto masterchain_block_id = create_block_id(query.masterchain_block_); + auto [block_id, mc_block_id, state_type] = persistent_state_from_v2_query(query); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, - masterchain_block_id, UnsplitStateType{}, std::move(P)); + mc_block_id, state_type, std::move(P)); } void FullNodeMasterImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, diff --git a/validator/full-node-master.hpp b/validator/full-node-master.hpp index d6160a94e..82815b48f 100644 --- a/validator/full-node-master.hpp +++ b/validator/full-node-master.hpp @@ -86,6 +86,12 @@ class FullNodeMasterImpl : public FullNodeMaster { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, td::Promise promise); + + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSliceV2 &query, + td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSizeV2 &query, + td::Promise promise); + // void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareNextKeyBlockProof &query, // td::Promise promise); void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise); diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 480c24b64..64d0d8121 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -526,6 +526,12 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod masterchain_block_id, UnsplitStateType{}, std::move(P)); } +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, + td::Promise promise) { + auto query_v2 = create_tl_object(persistent_state_id_from_v1_query(query)); + return process_query(src, *query_v2, std::move(promise)); +} + void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getNextKeyBlockIds &query, td::Promise promise) { auto cnt = static_cast(query.max_size_); @@ -595,6 +601,13 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod masterchain_block_id, UnsplitStateType{}, 0, max_size + 1, std::move(P)); } +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, + td::Promise promise) { + auto query_v2 = create_tl_object( + persistent_state_id_from_v1_query(query), query.offset_, query.max_size_); + return process_query(src, *query_v2, std::move(promise)); +} + void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, td::Promise promise) { VLOG(FULL_NODE_DEBUG) << "Got query getCapabilities from " << src; @@ -702,13 +715,12 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod }); } -void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSliceV2 &query, td::Promise promise) { - auto block_id = create_block_id(query.block_); - auto masterchain_block_id = create_block_id(query.masterchain_block_); - VLOG(FULL_NODE_DEBUG) << "Got query downloadPersistentStateSlice " << block_id.to_str() << " " - << masterchain_block_id.to_str() << " " << query.offset_ << " " << query.max_size_ << " from " - << src; + auto [block_id, mc_block_id, state_type] = persistent_state_from_v2_query(query); + VLOG(FULL_NODE_DEBUG) << "Got query downloadPersistentStateSlice " << block_id.to_str() << " " << mc_block_id.to_str() + << " (" << persistent_state_type_to_string(block_id.shard_full(), state_type) << ") " + << query.offset_ << " " << query.max_size_ << " from " << src; if (query.max_size_ < 0 || query.max_size_ > (1 << 24)) { promise.set_error(td::Status::Error(ErrorCode::protoviolation, "invalid max_size")); return; @@ -723,10 +735,10 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod promise.set_value(R.move_as_ok()); }); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_slice, block_id, - masterchain_block_id, UnsplitStateType{}, query.offset_, query.max_size_, std::move(P)); + mc_block_id, state_type, query.offset_, query.max_size_, std::move(P)); } -void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSize &query, +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSizeV2 &query, td::Promise promise) { auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { @@ -736,12 +748,12 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod promise.set_value(create_serialize_tl_object(R.move_as_ok())); } }); - auto block_id = create_block_id(query.block_); - auto masterchain_block_id = create_block_id(query.masterchain_block_); - VLOG(FULL_NODE_DEBUG) << "Got query getPersistentStateSize " << block_id.to_str() << " " - << masterchain_block_id.to_str() << " from " << src; + auto [block_id, mc_block_id, state_type] = persistent_state_from_v2_query(query); + VLOG(FULL_NODE_DEBUG) << "Got query getPersistentStateSize " << block_id.to_str() << " " << mc_block_id.to_str() + << " (" << persistent_state_type_to_string(block_id.shard_full(), state_type) << ") from " + << src; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_size, block_id, - masterchain_block_id, UnsplitStateType{}, std::move(P)); + mc_block_id, state_type, std::move(P)); } void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 8cbd614aa..5be6b6a5b 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -143,6 +143,10 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getOutMsgQueueProof &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSliceV2 &query, + td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getPersistentStateSizeV2 &query, + td::Promise promise); void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise); void receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data); diff --git a/validator/interfaces/persistent-state.h b/validator/interfaces/persistent-state.h index 4a03a8f2b..8e501398a 100644 --- a/validator/interfaces/persistent-state.h +++ b/validator/interfaces/persistent-state.h @@ -16,8 +16,11 @@ */ #pragma once +#include "auto/tl/ton_api.h" #include "td/utils/Variant.h" +#include "td/utils/overloaded.h" #include "ton/ton-types.h" +#include "ton/ton-shard.h" namespace ton { @@ -33,6 +36,47 @@ struct SplitPersistentStateType {}; using PersistentStateType = td::Variant; +auto persistent_state_id_from_v1_query(auto const &query) { + auto block = create_tl_block_id(create_block_id(query.block_)); + auto mc_block = create_tl_block_id(create_block_id(query.masterchain_block_)); + return create_tl_object(std::move(block), std::move(mc_block), 0); +} + +auto persistent_state_from_v2_query(auto const &query) { + auto block = create_block_id(query.state_->block_); + auto mc_block = create_block_id(query.state_->masterchain_block_); + ShardId effective_shard = static_cast(query.state_->effective_shard_); + + if (effective_shard == 0 || !shard_is_ancestor(block.shard_full().shard, effective_shard)) { + // The second condition is technically a "protocol" violation but since we don't really validate + // stuff here regardless, let's just map it to an unsplit state. + return std::tuple{block, mc_block, PersistentStateType{UnsplitStateType{}}}; + } + + if (effective_shard == block.shard_full().shard) { + return std::tuple{block, mc_block, PersistentStateType{SplitPersistentStateType{}}}; + } + + CHECK(shard_is_proper_ancestor(block.shard_full().shard, effective_shard)); + return std::tuple{block, mc_block, PersistentStateType{SplitAccountStateType{effective_shard}}}; +} + +inline std::string persistent_state_type_to_string(ShardIdFull const &shard, PersistentStateType const &state) { + std::string result; + state.visit(td::overloaded([&](UnsplitStateType) { result = "unsplit"; }, + [&](SplitAccountStateType type) { + int real_pfx_len = shard_prefix_length(shard.shard); + int effective_pfx_len = shard_prefix_length(type.effective_shard_id); + td::uint64 parts_count = 1 << (effective_pfx_len - real_pfx_len); + td::uint64 part_idx = + type.effective_shard_id >> (64 - effective_pfx_len) & (parts_count - 1); + result = + "part " + std::to_string(part_idx + 1) + " out of " + std::to_string(parts_count); + }, + [&](SplitPersistentStateType) { result = "split header"; })); + return result; +} + } // namespace validator } // namespace ton From d54065abe58c8ab9bbcf9cffa81f70da4dd16773 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Thu, 19 Jun 2025 10:20:54 +0300 Subject: [PATCH 299/388] Update Windows github action (#1714) * not always gh mac runners have nproc utility * deprecate Ubuntu 20.04 * fix linking issues of portable tonlibjson.dylib * Upgrade gh action win-2019->win-2022 * adjust gh path to MSVC --- .github/workflows/ton-x86-64-windows.yml | 10 +- assembly/native/build-windows-2022.bat | 207 ++++++++++++++++++ assembly/native/build-windows-github-2022.bat | 2 + 3 files changed, 214 insertions(+), 5 deletions(-) create mode 100644 assembly/native/build-windows-2022.bat create mode 100644 assembly/native/build-windows-github-2022.bat diff --git a/.github/workflows/ton-x86-64-windows.yml b/.github/workflows/ton-x86-64-windows.yml index b930d25a9..8de9e11be 100644 --- a/.github/workflows/ton-x86-64-windows.yml +++ b/.github/workflows/ton-x86-64-windows.yml @@ -9,7 +9,7 @@ defaults: jobs: build: - runs-on: windows-2019 + runs-on: windows-2022 steps: - name: Get Current OS version @@ -30,15 +30,15 @@ jobs: uses: actions/cache@v4 with: path: third_libs - key: ${{ runner.os }}-${{ runner.arch }}-windows-2019-3pp-${{ hashFiles('**/assembly/native/build-windows-2019.bat') }} + key: ${{ runner.os }}-${{ runner.arch }}-windows-2022-3pp-${{ hashFiles('**/assembly/native/build-windows-2022.bat') }} - name: Build TON run: | git submodule sync --recursive git submodule update - copy assembly\native\build-windows-github-2019.bat . - copy assembly\native\build-windows-2019.bat . - build-windows-github-2019.bat Enterprise + copy assembly\native\build-windows-github-2022.bat . + copy assembly\native\build-windows-2022.bat . + build-windows-github-2022.bat Enterprise ccache -sp # - name: Run Tests diff --git a/assembly/native/build-windows-2022.bat b/assembly/native/build-windows-2022.bat new file mode 100644 index 000000000..6ff7166af --- /dev/null +++ b/assembly/native/build-windows-2022.bat @@ -0,0 +1,207 @@ +REM execute this script inside elevated (Run as Administrator) console "x64 Native Tools Command Prompt for VS 2022" + +echo off + +echo Installing chocolatey windows package manager... +@"%SystemRoot%\System32\WindowsPowerShell\v1.0\powershell.exe" -NoProfile -InputFormat None -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" && SET "PATH=%PATH%;%ALLUSERSPROFILE%\chocolatey\bin" +choco -? +IF %errorlevel% NEQ 0 ( + echo Can't install chocolatey + exit /b %errorlevel% +) + +choco feature enable -n allowEmptyChecksums + +echo Installing pkgconfiglite... +choco install -y pkgconfiglite +IF %errorlevel% NEQ 0 ( + echo Can't install pkgconfiglite + exit /b %errorlevel% +) + +echo Installing ninja... +choco install -y ninja +IF %errorlevel% NEQ 0 ( + echo Can't install ninja + exit /b %errorlevel% +) + +echo Installing ccache... +choco install -y ccache +IF %errorlevel% NEQ 0 ( + echo Can't install ccache + exit /b %errorlevel% +) + +echo Installing nasm... +choco install -y nasm +where nasm +SET PATH=%PATH%;C:\Program Files\NASM +IF %errorlevel% NEQ 0 ( + echo Can't install nasm + exit /b %errorlevel% +) + +if not exist "third_libs" ( + mkdir "third_libs" +) +cd third_libs + +set third_libs=%cd% +echo %third_libs% + +if not exist "zlib" ( + git clone https://github.com/madler/zlib.git + cd zlib + git checkout v1.3.1 + cd contrib\vstudio\vc14 + msbuild zlibstat.vcxproj /p:Configuration=ReleaseWithoutAsm /p:platform=x64 -p:PlatformToolset=v143 + cd ..\..\..\.. +) else ( + echo Using zlib... +) + +if not exist "lz4" ( + git clone https://github.com/lz4/lz4.git + cd lz4 + git checkout v1.9.4 + cd build\VS2022\liblz4 + msbuild liblz4.vcxproj /p:Configuration=Release /p:platform=x64 -p:PlatformToolset=v143 + cd ..\..\..\.. +) else ( + echo Using lz4... +) + +if not exist "libsodium" ( + git clone https://github.com/jedisct1/libsodium + cd libsodium + git checkout 1.0.18-RELEASE + msbuild libsodium.vcxproj /p:Configuration=Release /p:platform=x64 -p:PlatformToolset=v143 + cd .. +) else ( + echo Using libsodium... +) + +if not exist "openssl" ( + git clone https://github.com/openssl/openssl.git + cd openssl + git checkout openssl-3.1.4 + where perl + perl Configure VC-WIN64A + IF %errorlevel% NEQ 0 ( + echo Can't configure openssl + exit /b %errorlevel% + ) + nmake + cd .. +) else ( + echo Using openssl... +) + +if not exist "libmicrohttpd" ( + git clone https://github.com/Karlson2k/libmicrohttpd.git + cd libmicrohttpd + git checkout v1.0.1 + cd w32\VS2022 + msbuild libmicrohttpd.vcxproj /p:Configuration=Release-static /p:platform=x64 -p:PlatformToolset=v143 + IF %errorlevel% NEQ 0 ( + echo Can't compile libmicrohttpd + exit /b %errorlevel% + ) + cd ../../.. +) else ( + echo Using libmicrohttpd... +) + +cd .. +echo Current dir %cd% + +mkdir build +cd build +cmake -GNinja -DCMAKE_BUILD_TYPE=Release ^ +-DPORTABLE=1 ^ +-DSODIUM_USE_STATIC_LIBS=1 ^ +-DSODIUM_LIBRARY_RELEASE=%third_libs%\libsodium\Build\Release\x64\libsodium.lib ^ +-DSODIUM_LIBRARY_DEBUG=%third_libs%\libsodium\Build\Release\x64\libsodium.lib ^ +-DSODIUM_INCLUDE_DIR=%third_libs%\libsodium\src\libsodium\include ^ +-DLZ4_FOUND=1 ^ +-DLZ4_INCLUDE_DIRS=%third_libs%\lz4\lib ^ +-DLZ4_LIBRARIES=%third_libs%\lz4\build\VS2022\liblz4\bin\x64_Release\liblz4_static.lib ^ +-DMHD_FOUND=1 ^ +-DMHD_LIBRARY=%third_libs%\libmicrohttpd\w32\VS2022\Output\x64\libmicrohttpd.lib ^ +-DMHD_INCLUDE_DIR=%third_libs%\libmicrohttpd\src\include ^ +-DZLIB_FOUND=1 ^ +-DZLIB_INCLUDE_DIR=%third_libs%\zlib ^ +-DZLIB_LIBRARIES=%third_libs%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib ^ +-DOPENSSL_FOUND=1 ^ +-DOPENSSL_INCLUDE_DIR=%third_libs%\openssl\include ^ +-DOPENSSL_CRYPTO_LIBRARY=%third_libs%\openssl\libcrypto_static.lib ^ +-DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj" .. + +IF %errorlevel% NEQ 0 ( + echo Can't configure TON + exit /b %errorlevel% +) + +IF "%1"=="-t" ( +ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ +tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ +json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ +test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net ^ +test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ +test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver +IF %errorlevel% NEQ 0 ( + echo Can't compile TON + exit /b %errorlevel% +) +) else ( +ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ +tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ +json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator proxy-liteserver +IF %errorlevel% NEQ 0 ( + echo Can't compile TON + exit /b %errorlevel% +) +) + +copy validator-engine\validator-engine.exe test +IF %errorlevel% NEQ 0 ( + echo validator-engine.exe does not exist + exit /b %errorlevel% +) + +echo Strip and copy artifacts +cd .. +echo where strip +where strip +mkdir artifacts +mkdir artifacts\smartcont +mkdir artifacts\lib + +for %%I in (build\storage\storage-daemon\storage-daemon.exe ^ + build\storage\storage-daemon\storage-daemon-cli.exe ^ + build\blockchain-explorer\blockchain-explorer.exe ^ + build\crypto\fift.exe ^ + build\crypto\tlbc.exe ^ + build\crypto\func.exe ^ + build\tolk\tolk.exe ^ + build\crypto\create-state.exe ^ + build\validator-engine-console\validator-engine-console.exe ^ + build\tonlib\tonlib-cli.exe ^ + build\tonlib\tonlibjson.dll ^ + build\http\http-proxy.exe ^ + build\rldp-http-proxy\rldp-http-proxy.exe ^ + build\dht-server\dht-server.exe ^ + build\lite-client\lite-client.exe ^ + build\validator-engine\validator-engine.exe ^ + build\utils\generate-random-id.exe ^ + build\utils\json2tlo.exe ^ + build\utils\proxy-liteserver.exe ^ + build\adnl\adnl-proxy.exe ^ + build\emulator\emulator.dll) do ( + echo strip -s %%I & copy %%I artifacts\ + strip -s %%I & copy %%I artifacts\ +) + +xcopy /e /k /h /i crypto\smartcont artifacts\smartcont +xcopy /e /k /h /i crypto\fift\lib artifacts\lib diff --git a/assembly/native/build-windows-github-2022.bat b/assembly/native/build-windows-github-2022.bat new file mode 100644 index 000000000..f1e4b884a --- /dev/null +++ b/assembly/native/build-windows-github-2022.bat @@ -0,0 +1,2 @@ +call "C:\Program Files\Microsoft Visual Studio\2022\%1\VC\Auxiliary\Build\vcvars64.bat" +call build-windows-2022.bat -t From 5a1d271b9b0dd2bbffea555561cdc385ed2672a6 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Sat, 7 Jun 2025 13:13:14 -0400 Subject: [PATCH 300/388] Add support for split persistent states in StateSerializer --- crypto/vm/dict.cpp | 6 ++ crypto/vm/dict.h | 1 + validator/state-serializer.cpp | 168 ++++++++++++++++++++++++++++----- validator/state-serializer.hpp | 16 +++- 4 files changed, 168 insertions(+), 23 deletions(-) diff --git a/crypto/vm/dict.cpp b/crypto/vm/dict.cpp index e1e683fe3..0450932a2 100644 --- a/crypto/vm/dict.cpp +++ b/crypto/vm/dict.cpp @@ -2790,6 +2790,12 @@ Ref AugmentedDictionary::get_root() const { return root; } +Ref AugmentedDictionary::get_wrapped_dict_root() const { + vm::CellBuilder cb; + cb.append_cellslice(get_root()); + return cb.finalize(); +} + Ref AugmentedDictionary::extract_root() && { if (!(flags & f_root_cached) && !compute_root()) { return {}; diff --git a/crypto/vm/dict.h b/crypto/vm/dict.h index e22b90946..4da56acd9 100644 --- a/crypto/vm/dict.h +++ b/crypto/vm/dict.h @@ -572,6 +572,7 @@ class AugmentedDictionary final : public DictionaryFixed { AugmentedDictionary(DictNonEmpty, Ref _root, int _n, const AugmentationData& _aug, bool validate = true); Ref get_empty_dictionary() const; Ref get_root() const; + Ref get_wrapped_dict_root() const; Ref extract_root() &&; bool append_dict_to_bool(CellBuilder& cb) &&; bool append_dict_to_bool(CellBuilder& cb) const &; diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index 73ce440d3..890d7e23f 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -17,11 +17,15 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "state-serializer.hpp" +#include "crypto/block/block-auto.h" +#include "crypto/block/block-parse.h" #include "td/utils/Random.h" +#include "td/utils/overloaded.h" #include "ton/ton-io.hpp" #include "common/delay.h" #include "td/utils/filesystem.h" #include "td/utils/HashSet.h" +#include "vm/cells/MerkleProof.h" namespace ton { @@ -199,7 +203,7 @@ void AsyncStateSerializer::next_iteration() { } if (next_idx_ < shards_.size()) { running_ = true; - request_shard_state(shards_[next_idx_]); + request_shard_state(shards_[next_idx_].block_id); return; } LOG(ERROR) << "finished serializing persistent state for " << masterchain_handle_->id().id.to_str(); @@ -324,7 +328,16 @@ class CachedCellDbReader : public vm::CellDbReader { td::uint64 bulk_reqs_ = 0; }; -void AsyncStateSerializer::PreviousStateCache::prepare_cache(ShardIdFull shard) { +void AsyncStateSerializer::PreviousStateCache::prepare_cache(ShardIdFull shard, PersistentStateType type) { + if (type.get_offset() == type.offset()) { + // Header of a split state is small, so not caching it is fine. + return; + } + + if (type.get_offset() == type.offset()) { + shard = {shard.workchain, type.get().effective_shard_id}; + } + std::vector prev_shards; for (const auto& [_, prev_shard] : state_files) { if (shard_intersects(shard, prev_shard)) { @@ -335,7 +348,6 @@ void AsyncStateSerializer::PreviousStateCache::prepare_cache(ShardIdFull shard) return; } cur_shards = std::move(prev_shards); - cache = {}; if (cur_shards.empty()) { return; } @@ -375,6 +387,25 @@ void AsyncStateSerializer::PreviousStateCache::prepare_cache(ShardIdFull shard) cache = std::make_shared(std::move(cells)); } +void AsyncStateSerializer::PreviousStateCache::add_new_cells(vm::CellDbReader& reader, Ref const& cell) { + if (!cell->is_loaded()) { + return; + } + if (reader.load_cell(cell->get_hash().as_slice()).is_ok()) { + return; + } + + auto [_, inserted] = cache->insert(cell); + if (!inserted) { + return; + } + + vm::CellSlice cs{vm::NoVm{}, cell}; + for (unsigned i = 0; i < cs.size_refs(); ++i) { + add_new_cells(reader, cs.prefetch_ref(i)); + } +} + void AsyncStateSerializer::got_masterchain_state(td::Ref state, std::shared_ptr cell_db_reader) { if (!opts_->get_state_serializer_enabled() || auto_disabled_) { @@ -389,7 +420,10 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state auto vec = state->get_shards(); for (auto &v : vec) { if (opts_->need_monitor(v->shard(), state)) { - shards_.push_back(v->top_block_id()); + shards_.push_back({ + .block_id = v->top_block_id(), + .split_depth = state->persistent_state_split_depth(v->shard().workchain), + }); } } @@ -401,7 +435,7 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state return vm::std_boc_serialize_to_file(root, fd, 31, std::move(cancellation_token)); } if (fast_serializer_enabled) { - previous_state_cache->prepare_cache(shard); + previous_state_cache->prepare_cache(shard, UnsplitStateType{}); } auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache->cache); auto res = vm::boc_serialize_to_file_large(new_cell_db_reader, root->get_hash(), fd, 31, std::move(cancellation_token)); @@ -455,43 +489,135 @@ void AsyncStateSerializer::got_shard_handle(BlockHandle handle) { td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle, std::move(P)); } +namespace { + +// Expects `ShardStateUnsplit` as `shard_state_cell`. +std::vector split_shard_state(ShardId shard_id, td::Ref shard_state_cell, int split_depth) { + CHECK(split_depth <= 63); + int shard_prefix_length = shard_pfx_len(shard_id); + if (shard_prefix_length >= static_cast(split_depth)) { + return {{UnsplitStateType{}, std::move(shard_state_cell)}}; + } + + block::gen::ShardStateUnsplit::Record unsplit_shard_state; + bool rc = tlb::unpack_cell(shard_state_cell, unsplit_shard_state); + CHECK(rc); + + std::vector result; + + auto unwrapped_accounts_root = unsplit_shard_state.accounts; + auto accounts_cut = std::make_shared(); + auto accounts_root = vm::UsageCell::create(unwrapped_accounts_root, accounts_cut->root_ptr()); + + // NOTE: Ref constructor expects caller to unwrap HashMapAugE. + vm::AugmentedDictionary accounts{ + vm::load_cell_slice_ref(accounts_root), + 256, + block::tlb::aug_ShardAccounts, + false, + }; + + // Build account dict parts + ShardId effective_shard = shard_id ^ (1ULL << (63 - shard_prefix_length)) ^ (1ULL << (63 - split_depth)); + ShardId increment = 1ULL << (64 - split_depth); + + for (int i = 0; i < (1 << (split_depth - shard_prefix_length)); ++i, effective_shard += increment) { + td::BitArray<64> prefix; + prefix.store_ulong(effective_shard); + auto account_dict_part = accounts; + account_dict_part.cut_prefix_subdict(prefix.bits(), split_depth); + + if (!account_dict_part.is_empty()) { + result.push_back({SplitAccountStateType{effective_shard}, account_dict_part.get_wrapped_dict_root()}); + } + } + + auto accounts_proof = vm::MerkleProof::generate_raw(unwrapped_accounts_root, accounts_cut.get()); + + // Build header + unsplit_shard_state.accounts = accounts_proof; + vm::CellBuilder unsplit_shard_state_cb; + rc = tlb::pack(unsplit_shard_state_cb, unsplit_shard_state); + CHECK(rc); + + auto header = unsplit_shard_state_cb.finalize(); + CHECK(header->get_level() <= 1 && header->get_hash(0) == shard_state_cell->get_hash()); + + result.push_back({SplitPersistentStateType{}, vm::CellBuilder::create_merkle_proof(header)}); + + return result; +} + +} // namespace + void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref state, std::shared_ptr cell_db_reader) { + int archive_split_depth = shards_[next_idx_].split_depth; next_idx_++; + if (!opts_->get_state_serializer_enabled() || auto_disabled_) { success_handler(); return; } LOG(ERROR) << "serializing shard state " << handle->id().id.to_str(); - auto write_data = [shard = state->get_shard(), root = state->root_cell(), cell_db_reader, - previous_state_cache = previous_state_cache_, - fast_serializer_enabled = opts_->get_fast_state_serializer_enabled(), + + auto parts = split_shard_state(state->get_shard().shard, state->root_cell(), archive_split_depth); + CHECK(!parts.empty()); + + write_shard_state(handle, state->get_shard(), cell_db_reader, + std::make_shared>(std::move(parts)), 0); + + current_status_ = PSTRING() << "serializing shard state " << next_idx_ << "/" << shards_.size() << " " + << state->get_block_id().id.to_str(); + current_status_ts_ = td::Timestamp::now(); +} + +void AsyncStateSerializer::write_shard_state(BlockHandle handle, ShardIdFull shard, + std::shared_ptr cell_db_reader, + std::shared_ptr> parts, size_t idx) { + auto [type, cell] = parts->at(idx); + + auto write_data = [=, this, cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable { + CHECK(running_); + + LOG(ERROR) << "serializing shard state " << handle->id().id.to_str() << " (" + << persistent_state_type_to_string(shard, type) << ")"; if (!cell_db_reader) { - return vm::std_boc_serialize_to_file(root, fd, 31, std::move(cancellation_token)); + return vm::std_boc_serialize_to_file(cell, fd, 31, std::move(cancellation_token)); } - if (fast_serializer_enabled) { - previous_state_cache->prepare_cache(shard); + if (opts_->get_fast_state_serializer_enabled()) { + previous_state_cache_->prepare_cache(shard, type); } - auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache->cache); - auto res = vm::boc_serialize_to_file_large(new_cell_db_reader, root->get_hash(), fd, 31, std::move(cancellation_token)); + if (!previous_state_cache_->cache) { + previous_state_cache_->cache = std::make_shared(); + } + previous_state_cache_->add_new_cells(*cell_db_reader, cell); + auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache_->cache); + auto res = + vm::boc_serialize_to_file_large(new_cell_db_reader, cell->get_hash(), fd, 31, std::move(cancellation_token)); new_cell_db_reader->print_stats(); return res; }; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result R) { + auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this)](td::Result R) { if (R.is_error() && R.error().code() == cancelled) { LOG(ERROR) << "Persistent state serialization cancelled"; + td::actor::send_closure(SelfId, &AsyncStateSerializer::success_handler); + return; + } + + R.ensure(); + LOG(ERROR) << "finished serializing shard state " << handle->id().id.to_str() << " (" + << persistent_state_type_to_string(shard, type) << ")"; + if (idx + 1 == parts->size()) { + td::actor::send_closure(SelfId, &AsyncStateSerializer::success_handler); } else { - R.ensure(); - LOG(ERROR) << "finished serializing shard state " << handle->id().id.to_str(); + td::actor::send_closure(SelfId, &AsyncStateSerializer::write_shard_state, handle, shard, cell_db_reader, parts, + idx + 1); } - td::actor::send_closure(SelfId, &AsyncStateSerializer::success_handler); }); td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, handle->id(), - masterchain_handle_->id(), UnsplitStateType{}, write_data, std::move(P)); - current_status_ = PSTRING() << "serializing shard state " << next_idx_ << "/" << shards_.size() << " " - << state->get_block_id().id.to_str(); - current_status_ts_ = td::Timestamp::now(); + masterchain_handle_->id(), type, write_data, std::move(P)); } void AsyncStateSerializer::fail_handler(td::Status reason) { diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index 406ac350a..baa9268db 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -27,6 +27,11 @@ namespace ton { namespace validator { +struct SerializablePart { + PersistentStateType type; + td::Ref cell; +}; + class AsyncStateSerializer : public td::actor::Actor { private: td::uint32 attempt_ = 0; @@ -52,13 +57,18 @@ class AsyncStateSerializer : public td::actor::Actor { bool stored_persistent_state_description_ = false; bool have_masterchain_state_ = false; - std::vector shards_; + struct ShardSerializationConfig { + BlockIdExt block_id; + td::uint32 split_depth; + }; + std::vector shards_; struct PreviousStateCache { std::vector> state_files; std::shared_ptr cache; std::vector cur_shards; - void prepare_cache(ShardIdFull shard); + void prepare_cache(ShardIdFull shard, PersistentStateType type); + void add_new_cells(vm::CellDbReader& reader, Ref const& cell); }; std::shared_ptr previous_state_cache_; @@ -93,6 +103,8 @@ class AsyncStateSerializer : public td::actor::Actor { void stored_masterchain_state(); void got_shard_handle(BlockHandle handle); void got_shard_state(BlockHandle handle, td::Ref state, std::shared_ptr cell_db_reader); + void write_shard_state(BlockHandle handle, ShardIdFull shard, std::shared_ptr cell_db_reader, + std::shared_ptr> parts, size_t idx); void get_masterchain_seqno(td::Promise promise) { promise.set_result(last_block_id_.id.seqno); From 3a93b225e2887d83f167f5886343990edef418c2 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Wed, 11 Jun 2025 19:22:00 -0400 Subject: [PATCH 301/388] Teach DownloadState how to download split states This way, we propagate hardcoding UnsplitStateType one layer up. --- validator/full-node-shard.cpp | 4 +- validator/interfaces/persistent-state.h | 8 +++ validator/net/download-state.cpp | 94 +++++++++++++++++++------ validator/net/download-state.hpp | 10 ++- 4 files changed, 91 insertions(+), 25 deletions(-) diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 64d0d8121..a6dd2346c 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -990,8 +990,8 @@ void FullNodeShardImpl::download_persistent_state(BlockIdExt id, BlockIdExt mast td::Timestamp timeout, td::Promise promise) { auto &b = choose_neighbour(); td::actor::create_actor(PSTRING() << "downloadstatereq" << id.id.to_str(), id, masterchain_block_id, - adnl_id_, overlay_id_, b.adnl_id, priority, timeout, validator_manager_, - rldp2_, overlays_, adnl_, client_, std::move(promise)) + UnsplitStateType{}, adnl_id_, overlay_id_, b.adnl_id, priority, timeout, + validator_manager_, rldp2_, overlays_, adnl_, client_, std::move(promise)) .release(); } diff --git a/validator/interfaces/persistent-state.h b/validator/interfaces/persistent-state.h index 8e501398a..36d9ef0bd 100644 --- a/validator/interfaces/persistent-state.h +++ b/validator/interfaces/persistent-state.h @@ -61,6 +61,14 @@ auto persistent_state_from_v2_query(auto const &query) { return std::tuple{block, mc_block, PersistentStateType{SplitAccountStateType{effective_shard}}}; } +inline ShardId persistent_state_to_effective_shard(ShardIdFull const &shard, PersistentStateType const &type) { + ShardId result = 0; + type.visit(td::overloaded([](UnsplitStateType) {}, + [&](SplitAccountStateType type) { result = type.effective_shard_id; }, + [&](SplitPersistentStateType) { result = shard.shard; })); + return result; +} + inline std::string persistent_state_type_to_string(ShardIdFull const &shard, PersistentStateType const &state) { std::string result; state.visit(td::overloaded([&](UnsplitStateType) { result = "unsplit"; }, diff --git a/validator/net/download-state.cpp b/validator/net/download-state.cpp index 3824fa01c..16d619b8e 100644 --- a/validator/net/download-state.cpp +++ b/validator/net/download-state.cpp @@ -28,15 +28,17 @@ namespace validator { namespace fullnode { -DownloadState::DownloadState(BlockIdExt block_id, BlockIdExt masterchain_block_id, adnl::AdnlNodeIdShort local_id, - overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, - td::uint32 priority, td::Timestamp timeout, +DownloadState::DownloadState(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + adnl::AdnlNodeIdShort local_id, overlay::OverlayIdShort overlay_id, + adnl::AdnlNodeIdShort download_from, td::uint32 priority, td::Timestamp timeout, td::actor::ActorId validator_manager, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId adnl, td::actor::ActorId client, td::Promise promise) : block_id_(block_id) , masterchain_block_id_(masterchain_block_id) + , type_(type) + , effective_shard_(persistent_state_to_effective_shard(block_id_.shard_full(), type)) , local_id_(local_id) , overlay_id_(overlay_id) , download_from_(download_from) @@ -48,6 +50,7 @@ DownloadState::DownloadState(BlockIdExt block_id, BlockIdExt masterchain_block_i , adnl_(adnl) , client_(client) , promise_(std::move(promise)) { + CHECK(masterchain_block_id_.is_valid() || effective_shard_ == 0); } void DownloadState::abort_query(td::Status reason) { @@ -74,7 +77,7 @@ void DownloadState::start_up() { alarm_timestamp() = timeout_; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state, block_id_, - masterchain_block_id_, UnsplitStateType{}, + masterchain_block_id_, type_, [SelfId = actor_id(this), block_id = block_id_](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &DownloadState::get_block_handle); @@ -124,22 +127,39 @@ void DownloadState::got_block_handle(BlockHandle handle) { void DownloadState::got_node_to_download(adnl::AdnlNodeIdShort node) { download_from_ = node; - LOG(WARNING) << "downloading state " << block_id_.to_str() << " from " << download_from_; + LOG(WARNING) << "downloading state " << block_id_.to_str() << " (" + << persistent_state_type_to_string(block_id_.shard_full(), type_) << ") from " << download_from_; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) mutable { - if (R.is_error()) { - td::actor::send_closure(SelfId, &DownloadState::abort_query, R.move_as_error()); + td::Promise P; + td::BufferSlice query; + + if (effective_shard_ == 0) { + P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) mutable { + if (R.is_error()) { + td::actor::send_closure(SelfId, &DownloadState::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &DownloadState::got_block_state_description, R.move_as_ok()); + } + }); + + if (masterchain_block_id_.is_valid()) { + query = create_serialize_tl_object( + create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_)); } else { - td::actor::send_closure(SelfId, &DownloadState::got_block_state_description, R.move_as_ok()); + query = create_serialize_tl_object(create_tl_block_id(block_id_)); } - }); - - td::BufferSlice query; - if (masterchain_block_id_.is_valid()) { - query = create_serialize_tl_object( - create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_)); } else { - query = create_serialize_tl_object(create_tl_block_id(block_id_)); + P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) mutable { + if (R.is_error()) { + td::actor::send_closure(SelfId, &DownloadState::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &DownloadState::got_state_size, R.move_as_ok()); + } + }); + + query = create_serialize_tl_object( + create_tl_object( + create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_), effective_shard_)); } if (client_.empty()) { @@ -195,6 +215,25 @@ void DownloadState::got_block_state_description(td::BufferSlice data) { })); } +void DownloadState::got_state_size(td::BufferSlice size_or_not_found) { + auto F = fetch_tl_object(std::move(size_or_not_found), true); + if (F.is_error()) { + abort_query(F.move_as_error()); + return; + } + prev_logged_timer_ = td::Timer(); + + ton_api::downcast_call(*F.move_as_ok().get(), + td::overloaded( + [&](ton_api::tonNode_persistentStateSizeNotFound &f) { + abort_query(td::Status::Error(ErrorCode::notready, "state not found")); + }, + [&](ton_api::tonNode_persistentStateSize &f) { + total_size_ = f.size_; + got_block_state_part(td::BufferSlice{}, 0); + })); +} + void DownloadState::request_total_size() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { @@ -207,8 +246,15 @@ void DownloadState::request_total_size() { td::actor::send_closure(SelfId, &DownloadState::got_total_size, res.ok()->size_); }); - td::BufferSlice query = create_serialize_tl_object( - create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_)); + td::BufferSlice query; + if (effective_shard_ == 0) { + query = create_serialize_tl_object( + create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_)); + } else { + query = create_serialize_tl_object( + create_tl_object( + create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_), effective_shard_)); + } if (client_.empty()) { td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, download_from_, local_id_, overlay_id_, "get size", std::move(P), td::Timestamp::in(3.0), std::move(query), @@ -275,8 +321,16 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques } }); - td::BufferSlice query = create_serialize_tl_object( - create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_), sum_, part_size); + td::BufferSlice query; + if (effective_shard_ == 0) { + query = create_serialize_tl_object( + create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_), sum_, part_size); + } else { + query = create_serialize_tl_object( + create_tl_object( + create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_), effective_shard_), + sum_, part_size); + } if (client_.empty()) { td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, download_from_, local_id_, overlay_id_, "download state", std::move(P), td::Timestamp::in(20.0), std::move(query), diff --git a/validator/net/download-state.hpp b/validator/net/download-state.hpp index 29854ce27..ebf119bd3 100644 --- a/validator/net/download-state.hpp +++ b/validator/net/download-state.hpp @@ -33,9 +33,10 @@ namespace fullnode { class DownloadState : public td::actor::Actor { public: - DownloadState(BlockIdExt block_id, BlockIdExt masterchain_block_id, adnl::AdnlNodeIdShort local_id, - overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, td::uint32 priority, - td::Timestamp timeout, td::actor::ActorId validator_manager, + DownloadState(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, + adnl::AdnlNodeIdShort local_id, overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, + td::uint32 priority, td::Timestamp timeout, + td::actor::ActorId validator_manager, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId adnl, td::actor::ActorId client, td::Promise promise); @@ -49,6 +50,7 @@ class DownloadState : public td::actor::Actor { void got_block_handle(BlockHandle handle); void got_node_to_download(adnl::AdnlNodeIdShort node); void got_block_state_description(td::BufferSlice data_description); + void got_state_size(td::BufferSlice size_or_not_found); void request_total_size(); void got_total_size(td::uint64 size); void got_block_state_part(td::BufferSlice data, td::uint32 requested_size); @@ -57,6 +59,8 @@ class DownloadState : public td::actor::Actor { private: BlockIdExt block_id_; BlockIdExt masterchain_block_id_; + PersistentStateType type_; + ShardId effective_shard_; adnl::AdnlNodeIdShort local_id_; overlay::OverlayIdShort overlay_id_; From ca80ed7ea2bc61286e27997a006bfbcfbf72b9fc Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Thu, 12 Jun 2025 09:28:54 -0400 Subject: [PATCH 302/388] Expose PersistentStateType to DownloadShardState This is the actor which will merge separate parts of a persistent state into a single one in the future commits. --- create-hardfork/create-hardfork.cpp | 4 ++-- test/test-ton-collator.cpp | 4 ++-- validator/downloaders/download-state.cpp | 2 +- validator/full-node-shard.cpp | 22 ++++++++++++---------- validator/full-node-shard.h | 5 +++-- validator/full-node-shard.hpp | 5 +++-- validator/full-node.cpp | 16 +++++++++------- validator/full-node.hpp | 4 ++-- validator/interfaces/validator-manager.h | 3 ++- validator/manager-disk.cpp | 2 +- validator/manager-disk.hpp | 4 ++-- validator/manager-hardfork.hpp | 4 ++-- validator/manager.cpp | 4 ++-- validator/manager.hpp | 4 ++-- validator/validator.h | 5 +++-- 15 files changed, 48 insertions(+), 40 deletions(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index 20e58f006..e2f77dd86 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -258,8 +258,8 @@ class HardforkCreator : public td::actor::Actor { td::Promise promise) override { } void download_persistent_state(ton::BlockIdExt block_id, ton::BlockIdExt masterchain_block_id, - td::uint32 priority, td::Timestamp timeout, - td::Promise promise) override { + ton::validator::PersistentStateType type, td::uint32 priority, + td::Timestamp timeout, td::Promise promise) override { } void download_block_proof(ton::BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) override { diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index a062ddbe5..89be586e6 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -359,8 +359,8 @@ class TestNode : public td::actor::Actor { td::Promise promise) override { } void download_persistent_state(ton::BlockIdExt block_id, ton::BlockIdExt masterchain_block_id, - td::uint32 priority, td::Timestamp timeout, - td::Promise promise) override { + ton::validator::PersistentStateType type, td::uint32 priority, + td::Timestamp timeout, td::Promise promise) override { } void download_block_proof(ton::BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) override { diff --git a/validator/downloaders/download-state.cpp b/validator/downloaders/download-state.cpp index 4335a2956..cc942d1ec 100644 --- a/validator/downloaders/download-state.cpp +++ b/validator/downloaders/download-state.cpp @@ -123,7 +123,7 @@ void DownloadShardState::checked_proof_link() { CHECK(masterchain_block_id_.is_valid()); CHECK(masterchain_block_id_.is_masterchain()); td::actor::send_closure(manager_, &ValidatorManager::send_get_persistent_state_request, block_id_, - masterchain_block_id_, priority_, std::move(P)); + masterchain_block_id_, UnsplitStateType{}, priority_, std::move(P)); } status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading state"); } diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index a6dd2346c..608a5ec8b 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -980,18 +980,20 @@ void FullNodeShardImpl::download_block(BlockIdExt id, td::uint32 priority, td::T void FullNodeShardImpl::download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) { - td::actor::create_actor(PSTRING() << "downloadstatereq" << id.id.to_str(), id, BlockIdExt{}, adnl_id_, - overlay_id_, adnl::AdnlNodeIdShort::zero(), priority, timeout, - validator_manager_, rldp_, overlays_, adnl_, client_, std::move(promise)) + td::actor::create_actor(PSTRING() << "downloadstatereq" << id.id.to_str(), id, BlockIdExt{}, + UnsplitStateType{}, adnl_id_, overlay_id_, adnl::AdnlNodeIdShort::zero(), + priority, timeout, validator_manager_, rldp_, overlays_, adnl_, client_, + std::move(promise)) .release(); } -void FullNodeShardImpl::download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Timestamp timeout, td::Promise promise) { +void FullNodeShardImpl::download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::uint32 priority, td::Timestamp timeout, + td::Promise promise) { auto &b = choose_neighbour(); td::actor::create_actor(PSTRING() << "downloadstatereq" << id.id.to_str(), id, masterchain_block_id, - UnsplitStateType{}, adnl_id_, overlay_id_, b.adnl_id, priority, timeout, - validator_manager_, rldp2_, overlays_, adnl_, client_, std::move(promise)) + type, adnl_id_, overlay_id_, b.adnl_id, priority, timeout, validator_manager_, + rldp2_, overlays_, adnl_, client_, std::move(promise)) .release(); } @@ -1007,9 +1009,9 @@ void FullNodeShardImpl::download_block_proof(BlockIdExt block_id, td::uint32 pri void FullNodeShardImpl::download_block_proof_link(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) { auto &b = choose_neighbour(); - td::actor::create_actor("downloadproofreq", block_id, true, false, adnl_id_, overlay_id_, - b.adnl_id, priority, timeout, validator_manager_, rldp_, - overlays_, adnl_, client_, create_neighbour_promise(b, std::move(promise))) + td::actor::create_actor("downloadproofreq", block_id, true, false, adnl_id_, overlay_id_, b.adnl_id, + priority, timeout, validator_manager_, rldp_, overlays_, adnl_, client_, + create_neighbour_promise(b, std::move(promise))) .release(); } diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 5898db80d..0c6aa441b 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -55,8 +55,9 @@ class FullNodeShard : public td::actor::Actor { td::Promise promise) = 0; virtual void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; - virtual void download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Timestamp timeout, td::Promise promise) = 0; + virtual void download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::uint32 priority, td::Timestamp timeout, + td::Promise promise) = 0; virtual void download_block_proof(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 5be6b6a5b..891d8edb5 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -178,8 +178,9 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise) override; void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) override; - void download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Timestamp timeout, td::Promise promise) override; + void download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::uint32 priority, td::Timestamp timeout, + td::Promise promise) override; void download_block_proof(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) override; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 953e0c599..d871c024b 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -374,16 +374,17 @@ void FullNodeImpl::download_zero_state(BlockIdExt id, td::uint32 priority, td::T td::actor::send_closure(shard, &FullNodeShard::download_zero_state, id, priority, timeout, std::move(promise)); } -void FullNodeImpl::download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Timestamp timeout, td::Promise promise) { +void FullNodeImpl::download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::uint32 priority, td::Timestamp timeout, + td::Promise promise) { auto shard = get_shard(id.shard_full()); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping download state diff query to unknown shard"; promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); return; } - td::actor::send_closure(shard, &FullNodeShard::download_persistent_state, id, masterchain_block_id, priority, timeout, - std::move(promise)); + td::actor::send_closure(shard, &FullNodeShard::download_persistent_state, id, masterchain_block_id, type, priority, + timeout, std::move(promise)); } void FullNodeImpl::download_block_proof(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, @@ -647,9 +648,10 @@ void FullNodeImpl::start_up() { td::Promise promise) override { td::actor::send_closure(id_, &FullNodeImpl::download_zero_state, id, priority, timeout, std::move(promise)); } - void download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Timestamp timeout, td::Promise promise) override { - td::actor::send_closure(id_, &FullNodeImpl::download_persistent_state, id, masterchain_block_id, priority, + void download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::uint32 priority, td::Timestamp timeout, + td::Promise promise) override { + td::actor::send_closure(id_, &FullNodeImpl::download_persistent_state, id, masterchain_block_id, type, priority, timeout, std::move(promise)); } void download_block_proof(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, diff --git a/validator/full-node.hpp b/validator/full-node.hpp index e2d5c72b0..e501e1e99 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -71,8 +71,8 @@ class FullNodeImpl : public FullNode { void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); - void download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Timestamp timeout, td::Promise promise); + void download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::uint32 priority, td::Timestamp timeout, td::Promise promise); void download_block_proof(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); void download_block_proof_link(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 2aebaca74..e18a7f18f 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -135,7 +135,8 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void send_get_block_request(BlockIdExt id, td::uint32 priority, td::Promise promise) = 0; virtual void send_get_zero_state_request(BlockIdExt id, td::uint32 priority, td::Promise promise) = 0; - virtual void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, + virtual void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::uint32 priority, td::Promise promise) = 0; virtual void send_get_block_proof_request(BlockIdExt block_id, td::uint32 priority, td::Promise promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index c48528b48..ed6f2ddae 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -911,7 +911,7 @@ void ValidatorManagerImpl::send_get_zero_state_request(BlockIdExt id, td::uint32 } void ValidatorManagerImpl::send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, - td::uint32 priority, + PersistentStateType type, td::uint32 priority, td::Promise promise) { UNREACHABLE(); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index ee83fc850..3729f909a 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -242,8 +242,8 @@ class ValidatorManagerImpl : public ValidatorManager { void send_get_block_request(BlockIdExt id, td::uint32 priority, td::Promise promise) override; void send_get_zero_state_request(BlockIdExt id, td::uint32 priority, td::Promise promise) override; - void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Promise promise) override; + void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::uint32 priority, td::Promise promise) override; void send_get_block_proof_request(BlockIdExt block_id, td::uint32 priority, td::Promise promise) override { UNREACHABLE(); diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index ecfc8de7e..b2c8de617 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -308,8 +308,8 @@ class ValidatorManagerImpl : public ValidatorManager { void send_get_zero_state_request(BlockIdExt id, td::uint32 priority, td::Promise promise) override { UNREACHABLE(); } - void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Promise promise) override { + void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::uint32 priority, td::Promise promise) override { UNREACHABLE(); } void send_get_block_proof_request(BlockIdExt block_id, td::uint32 priority, diff --git a/validator/manager.cpp b/validator/manager.cpp index 86d794bc8..5ca1581fc 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1609,9 +1609,9 @@ void ValidatorManagerImpl::send_get_zero_state_request(BlockIdExt id, td::uint32 } void ValidatorManagerImpl::send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, - td::uint32 priority, + PersistentStateType type, td::uint32 priority, td::Promise promise) { - callback_->download_persistent_state(id, masterchain_block_id, priority, td::Timestamp::in(3600 * 3), + callback_->download_persistent_state(id, masterchain_block_id, type, priority, td::Timestamp::in(3600 * 3), std::move(promise)); } diff --git a/validator/manager.hpp b/validator/manager.hpp index ab86d2d37..bee59dff7 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -502,8 +502,8 @@ class ValidatorManagerImpl : public ValidatorManager { void send_get_block_request(BlockIdExt id, td::uint32 priority, td::Promise promise) override; void send_get_zero_state_request(BlockIdExt id, td::uint32 priority, td::Promise promise) override; - void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Promise promise) override; + void send_get_persistent_state_request(BlockIdExt id, BlockIdExt masterchain_block_id, PersistentStateType type, + td::uint32 priority, td::Promise promise) override; void send_get_block_proof_request(BlockIdExt block_id, td::uint32 priority, td::Promise promise) override; void send_get_block_proof_link_request(BlockIdExt block_id, td::uint32 priority, diff --git a/validator/validator.h b/validator/validator.h index d988261a9..943c6e9c5 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -185,8 +185,9 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Promise promise) = 0; virtual void download_zero_state(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; - virtual void download_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::Timestamp timeout, td::Promise promise) = 0; + virtual void download_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, + PersistentStateType type, td::uint32 priority, td::Timestamp timeout, + td::Promise promise) = 0; virtual void download_block_proof(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; virtual void download_block_proof_link(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, From 716475342f243237f91c14583d59e53a8861425b Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Thu, 12 Jun 2025 19:49:28 -0400 Subject: [PATCH 303/388] Store state split depth in persistent state descriptions --- tl/generate/scheme/ton_api.tl | 2 ++ tl/generate/scheme/ton_api.tlo | Bin 103992 -> 104360 bytes ton/ton-types.h | 7 +++- validator/db/statedb.cpp | 57 ++++++++++++++++++++++++++------- validator/manager.cpp | 4 +-- validator/state-serializer.cpp | 5 ++- 6 files changed, 60 insertions(+), 15 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 9b535ba0e..42281547f 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -568,6 +568,8 @@ db.state.asyncSerializer block:tonNode.blockIdExt last:tonNode.blockIdExt last_t db.state.hardforks blocks:(vector tonNode.blockIdExt) = db.state.Hardforks; db.state.dbVersion version:int = db.state.DbVersion; db.state.persistentStateDescriptionShards shard_blocks:(vector tonNode.blockIdExt) = db.state.PersistentStateDescriptionShards; +db.state.persistentStateDescriptionShard block:tonNode.blockIdExt split_depth:int = db.state.PersistentStateDescriptionShard; +db.state.persistentStateDescriptionShardsV2 shard_blocks:(vector db.state.PersistentStateDescriptionShard) = db.state.PersistentStateDescriptionShards; db.state.persistentStateDescriptionHeader masterchain_id:tonNode.blockIdExt start_time:int end_time:int = db.state.PersistentStateDescriptionHeader; db.state.persistentStateDescriptionsList list:(vector db.state.persistentStateDescriptionHeader) = db.state.PersistentStateDescriptionsList; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 27bc598a75eb3a54807bba7b2ef4f942a1e368ad..5c47336ef5bd9b69ea119d49cddd58431303ec1f 100644 GIT binary patch delta 199 zcmdn7hHb@qHr_|G^{p77Kx-p!G(W3n*sn<%lXLksCQsy%W|3#&u$r91uQSm>V)F+6 z721ruHb+{wheA~;PEejafuEHfXaGptbcYT`9UkuDf}G5f_>|Owl8ng*>}6O$Qj@Pt zuz?sRGSNpAWGc+ot8!d1{?jb(XitWi#>N(A1h!RVqJ;!VD@FZQDVRWOK^I L?I{x(udo0BR`EIo delta 77 zcmZ3no^8h(Hr_|G^{p77KyxE+G(T&=6oY4)lXLksCT|p%-u#1qg*M}k%?qvDLnj}Y ZXutxJo&05@517$>WzzO5lNhhC008_L9pwN3 diff --git a/ton/ton-types.h b/ton/ton-types.h index c7aff6448..bbc4b2a24 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -515,8 +515,13 @@ struct ValidatorSessionConfig { }; struct PersistentStateDescription : public td::CntObject { + struct ShardBlock { + BlockIdExt block; + td::uint32 split_depth; + }; + BlockIdExt masterchain_id; - std::vector shard_blocks; + std::vector shard_blocks; UnixTime start_time, end_time; virtual CntObject* make_copy() const { diff --git a/validator/db/statedb.cpp b/validator/db/statedb.cpp index fe3b9d735..753ed7e64 100644 --- a/validator/db/statedb.cpp +++ b/validator/db/statedb.cpp @@ -277,16 +277,36 @@ void StateDb::add_persistent_state_description(td::Reflist_.resize(new_size); - std::vector> shard_blocks; - for (const BlockIdExt& block_id : desc->shard_blocks) { - shard_blocks.push_back(create_tl_block_id(block_id)); + bool can_be_stored_as_v1 = true; + for (auto const& [block_id, split_depth] : desc->shard_blocks) { + if (split_depth != 0) { + can_be_stored_as_v1 = false; + break; + } } + + td::BufferSlice serialized_shards; + + if (can_be_stored_as_v1) { + std::vector> shard_blocks; + for (auto const& [block_id, _] : desc->shard_blocks) { + shard_blocks.push_back(create_tl_block_id(block_id)); + } + serialized_shards = + create_serialize_tl_object(std::move(shard_blocks)); + } else { + std::vector> shard_blocks; + for (auto const& [block_id, split_depth] : desc->shard_blocks) { + shard_blocks.push_back(create_tl_object( + create_tl_block_id(block_id), static_cast(split_depth))); + } + serialized_shards = + create_serialize_tl_object(std::move(shard_blocks)); + } + auto key = create_hash_tl_object(desc->masterchain_id.seqno()); - kv_->set(key.as_slice(), - create_serialize_tl_object(std::move(shard_blocks)) - .as_slice()) - .ensure(); + kv_->set(key.as_slice(), serialized_shards.as_slice()).ensure(); list->list_.push_back(create_tl_object( create_tl_block_id(desc->masterchain_id), desc->start_time, desc->end_time)); @@ -325,11 +345,26 @@ void StateDb::get_persistent_state_descriptions(td::Promise(value, true); + auto F2 = fetch_tl_object(value, true); F2.ensure(); - for (const auto& block_id : F2.ok()->shard_blocks_) { - desc.shard_blocks.push_back(create_block_id(block_id)); - } + ton_api::downcast_call(*F2.ok().get(), + td::overloaded( + [&](ton_api::db_state_persistentStateDescriptionShards const& shards) { + for (auto const& block : shards.shard_blocks_) { + desc.shard_blocks.push_back({ + .block = create_block_id(block), + .split_depth = 0, + }); + } + }, + [&](ton_api::db_state_persistentStateDescriptionShardsV2 const& shards) { + for (auto const& shard_description : shards.shard_blocks_) { + desc.shard_blocks.push_back({ + .block = create_block_id(shard_description->block_), + .split_depth = static_cast(shard_description->split_depth_), + }); + } + })); result.push_back(td::Ref(true, std::move(desc))); } promise.set_result(std::move(result)); diff --git a/validator/manager.cpp b/validator/manager.cpp index 5ca1581fc..1ff686dbd 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -3254,7 +3254,7 @@ void ValidatorManagerImpl::add_persistent_state_description(td::Refsecond; if (prev_desc->end_time <= now) { - for (const BlockIdExt &block_id : prev_desc->shard_blocks) { + for (auto const &[block_id, _] : prev_desc->shard_blocks) { persistent_state_blocks_.erase(block_id); } it = persistent_state_descriptions_.erase(it); @@ -3271,7 +3271,7 @@ void ValidatorManagerImpl::add_persistent_state_description_impl(td::Refmasterchain_id.to_str() << " start_time=" << desc->start_time << " end_time=" << desc->end_time; - for (const BlockIdExt &block_id : desc->shard_blocks) { + for (auto const &[block_id, _] : desc->shard_blocks) { persistent_state_blocks_[block_id] = desc; LOG(DEBUG) << "Persistent state description: shard block " << block_id.to_str(); } diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index 890d7e23f..e65bad4c0 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -251,7 +251,10 @@ void AsyncStateSerializer::store_persistent_state_description(td::Refget_unix_time(); desc.end_time = ValidatorManager::persistent_state_ttl(desc.start_time); for (const auto &v : state->get_shards()) { - desc.shard_blocks.push_back(v->top_block_id()); + desc.shard_blocks.push_back({ + .block = v->top_block_id(), + .split_depth = state->persistent_state_split_depth(v->shard().workchain), + }); } td::actor::send_closure(manager_, &ValidatorManager::add_persistent_state_description, td::Ref(true, std::move(desc))); From 09dbf7250fbf6557460af9fbd73e9c61918840b1 Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Thu, 12 Jun 2025 20:01:15 -0400 Subject: [PATCH 304/388] Pass correct state split depth to DownloadShardState ... and let DownloadShardState do nothing with it. With this commit, we are expecting DownloadShardState to pass correct PersistentStateType from one side and are passing split depth from the other side. So, only state merging in that single actor remains to be done. --- validator/downloaders/download-state.cpp | 13 ++++++++++--- validator/downloaders/download-state.hpp | 3 ++- validator/downloaders/wait-block-state.cpp | 22 +++++++++++++++++++--- validator/import-db-slice.cpp | 6 ++++-- validator/manager-init.cpp | 6 +++--- validator/shard-client.cpp | 17 +++++++++++------ validator/shard-client.hpp | 7 ++++++- 7 files changed, 55 insertions(+), 19 deletions(-) diff --git a/validator/downloaders/download-state.cpp b/validator/downloaders/download-state.cpp index cc942d1ec..bebbe9e51 100644 --- a/validator/downloaders/download-state.cpp +++ b/validator/downloaders/download-state.cpp @@ -26,15 +26,22 @@ namespace ton { namespace validator { -DownloadShardState::DownloadShardState(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::uint32 priority, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise> promise) +DownloadShardState::DownloadShardState(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::uint32 split_depth, + td::uint32 priority, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise> promise) : block_id_(block_id) , masterchain_block_id_(masterchain_block_id) + , split_depth_(split_depth) , priority_(priority) , manager_(manager) , timeout_(timeout) , promise_(std::move(promise)) { + CHECK(masterchain_block_id_.is_valid() || split_depth_ == 0); + + int shard_prefix_length = shard_pfx_len(block_id_.shard_full().shard); + if (shard_prefix_length >= static_cast(split_depth_)) { + split_depth_ = 0; + } } void DownloadShardState::start_up() { diff --git a/validator/downloaders/download-state.hpp b/validator/downloaders/download-state.hpp index bde80aae1..9d3a495cb 100644 --- a/validator/downloaders/download-state.hpp +++ b/validator/downloaders/download-state.hpp @@ -27,7 +27,7 @@ namespace validator { class DownloadShardState : public td::actor::Actor { public: - DownloadShardState(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::uint32 priority, + DownloadShardState(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::uint32 split_depth, td::uint32 priority, td::actor::ActorId manager, td::Timestamp timeout, td::Promise> promise); @@ -58,6 +58,7 @@ class DownloadShardState : public td::actor::Actor { private: BlockIdExt block_id_; BlockIdExt masterchain_block_id_; + td::uint32 split_depth_; BlockHandle handle_; td::uint32 priority_; diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index c80e7d896..00085e960 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -117,10 +117,26 @@ void WaitBlockState::start() { td::actor::send_closure(SelfId, &WaitBlockState::written_state, R.move_as_ok()); } }); + BlockIdExt masterchain_id = persistent_state_desc_->masterchain_id; - td::actor::create_actor("downloadstate", handle_->id(), masterchain_id, priority_, manager_, - timeout_, std::move(P)) - .release(); + td::uint32 split_depth = 0; + bool block_found = false; + for (auto const& [block, block_split_depth] : persistent_state_desc_->shard_blocks) { + if (block == handle_->id()) { + split_depth = block_split_depth; + block_found = true; + break; + } + } + if (!block_found) { + LOG(ERROR) << "invalid persistent state description passed to WaitBlockState for block " + << handle_->id().to_str(); + P.set_error(td::Status::Error("invalid persistent state description")); + } else { + td::actor::create_actor("downloadstate", handle_->id(), masterchain_id, split_depth, + priority_, manager_, timeout_, std::move(P)) + .release(); + } } else if (!handle_->inited_prev() || (!handle_->inited_proof() && !handle_->inited_proof_link())) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result R) { if (R.is_error()) { diff --git a/validator/import-db-slice.cpp b/validator/import-db-slice.cpp index c0d502cb2..80fe141bb 100644 --- a/validator/import-db-slice.cpp +++ b/validator/import-db-slice.cpp @@ -418,8 +418,10 @@ void ArchiveImporter::apply_shard_block_cont1(BlockHandle handle, BlockIdExt mas if (handle->id().seqno() == 0) { auto P = td::PromiseCreator::lambda( [promise = std::move(promise)](td::Result>) mutable { promise.set_value(td::Unit()); }); - td::actor::create_actor("downloadstate", handle->id(), masterchain_block_id, 2, manager_, - td::Timestamp::in(3600), std::move(P)) + td::actor::create_actor( + "downloadstate", handle->id(), masterchain_block_id, + start_state_->persistent_state_split_depth(handle->id().shard_full().workchain), 2, manager_, + td::Timestamp::in(3600), std::move(P)) .release(); return; } diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index 6f304680b..004e5b120 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -65,7 +65,7 @@ void ValidatorManagerMasterchainReiniter::got_masterchain_handle(BlockHandle han R.ensure(); td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_masterchain_state); }); - td::actor::create_actor("downloadstate", handle_->id(), BlockIdExt{}, 2, manager_, + td::actor::create_actor("downloadstate", handle_->id(), BlockIdExt{}, 0, 2, manager_, td::Timestamp::in(3600), std::move(P)) .release(); return; @@ -80,7 +80,7 @@ void ValidatorManagerMasterchainReiniter::download_proof_link() { R.ensure(); td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::downloaded_zero_state); }); - td::actor::create_actor("downloadstate", handle_->id(), BlockIdExt{}, 2, manager_, + td::actor::create_actor("downloadstate", handle_->id(), BlockIdExt{}, 0, 2, manager_, td::Timestamp::in(3600), std::move(P)) .release(); } else { @@ -263,7 +263,7 @@ void ValidatorManagerMasterchainReiniter::download_masterchain_state() { R.move_as_ok()); } }); - td::actor::create_actor("downloadstate", block_id_, block_id_, 2, manager_, + td::actor::create_actor("downloadstate", block_id_, block_id_, 0, 2, manager_, td::Timestamp::in(3600 * 3), std::move(P)) .release(); } diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index ac86cf37d..e5fd0a2ac 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -74,24 +74,29 @@ void ShardClient::got_init_state_from_db(td::Ref state) { } void ShardClient::start_up_init_mode() { - std::vector shards; - for (const auto& s : masterchain_state_->get_shards()) { + std::vector shards; + for (const auto &s : masterchain_state_->get_shards()) { if (opts_->need_monitor(s->shard(), masterchain_state_)) { - shards.push_back(s->top_block_id()); + auto shard = s->top_block_id(); + shards.push_back({ + .shard = shard, + .split_depth = masterchain_state_->persistent_state_split_depth(shard.shard_full().workchain), + }); } } download_shard_states(masterchain_block_handle_->id(), std::move(shards), 0); } -void ShardClient::download_shard_states(BlockIdExt masterchain_block_id, std::vector shards, size_t idx) { +void ShardClient::download_shard_states(BlockIdExt masterchain_block_id, std::vector shards, + size_t idx) { if (idx >= shards.size()) { LOG(WARNING) << "downloaded all shard states"; applied_all_shards(); return; } - BlockIdExt block_id = shards[idx]; + auto [block_id, split_depth] = shards[idx]; td::actor::create_actor( - "downloadstate", block_id, masterchain_block_handle_->id(), 2, manager_, td::Timestamp::in(3600 * 5), + "downloadstate", block_id, masterchain_block_handle_->id(), split_depth, 2, manager_, td::Timestamp::in(3600 * 5), [=, SelfId = actor_id(this), shards = std::move(shards)](td::Result> R) { R.ensure(); td::actor::send_closure(SelfId, &ShardClient::download_shard_states, masterchain_block_id, std::move(shards), diff --git a/validator/shard-client.hpp b/validator/shard-client.hpp index 7c2c978c2..c13b625aa 100644 --- a/validator/shard-client.hpp +++ b/validator/shard-client.hpp @@ -27,6 +27,11 @@ namespace validator { class ShardClient : public td::actor::Actor { private: + struct DownloadableShard { + BlockIdExt shard; + td::uint32 split_depth; + }; + td::Ref opts_; BlockHandle masterchain_block_handle_; @@ -64,7 +69,7 @@ class ShardClient : public td::actor::Actor { void start_up() override; void start_up_init_mode(); - void download_shard_states(BlockIdExt masterchain_block_id, std::vector shards, size_t idx); + void download_shard_states(BlockIdExt masterchain_block_id, std::vector shards, size_t idx); void start(); void got_state_from_db(BlockIdExt masterchain_block_id); void got_init_handle_from_db(BlockHandle handle); From d3cec64d803ee29f48ae2ad90d4f365456aca69f Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Sun, 15 Jun 2025 10:13:37 -0400 Subject: [PATCH 305/388] Actually deserialize split states We don't yet store state parts in DB as this requires yet another instance of "change every single actor in the observable universe". --- crypto/vm/cells/MerkleProof.cpp | 6 + crypto/vm/cells/MerkleProof.h | 1 + validator/downloaders/download-state.cpp | 241 ++++++++++++++++++++++- validator/downloaders/download-state.hpp | 18 +- validator/fabric.h | 2 +- validator/impl/fabric.cpp | 2 +- 6 files changed, 257 insertions(+), 13 deletions(-) diff --git a/crypto/vm/cells/MerkleProof.cpp b/crypto/vm/cells/MerkleProof.cpp index 26dff7879..4c1fcbdb7 100644 --- a/crypto/vm/cells/MerkleProof.cpp +++ b/crypto/vm/cells/MerkleProof.cpp @@ -17,6 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "vm/cells/MerkleProof.h" +#include "td/utils/Status.h" #include "vm/cells/CellBuilder.h" #include "vm/cells/CellSlice.h" #include "vm/boc.h" @@ -152,6 +153,11 @@ Ref MerkleProof::virtualize(Ref cell, int virtualization) { return virtualize_raw(r_raw.move_as_ok(), {0 /*level*/, static_cast(virtualization)}); } +td::Result> MerkleProof::try_virtualize(Ref cell, int virtualization) { + TRY_RESULT(unpacked_cell, unpack_proof(std::move(cell))); + return unpacked_cell->virtualize({0 /*level*/, static_cast(virtualization)}); +} + class MerkleProofCombineFast { public: MerkleProofCombineFast(Ref a, Ref b) : a_(std::move(a)), b_(std::move(b)) { diff --git a/crypto/vm/cells/MerkleProof.h b/crypto/vm/cells/MerkleProof.h index fc2cb6ebd..0f0eaef09 100644 --- a/crypto/vm/cells/MerkleProof.h +++ b/crypto/vm/cells/MerkleProof.h @@ -36,6 +36,7 @@ class MerkleProof { // cell must have zero level and must be a MerkleProof static Ref virtualize(Ref cell, int virtualization); + static td::Result> try_virtualize(Ref cell, int virtualization = 1); static Ref combine(Ref a, Ref b); static td::Result> combine_status(Ref a, Ref b); diff --git a/validator/downloaders/download-state.cpp b/validator/downloaders/download-state.cpp index bebbe9e51..42f2c5c40 100644 --- a/validator/downloaders/download-state.cpp +++ b/validator/downloaders/download-state.cpp @@ -21,11 +21,103 @@ #include "common/checksum.h" #include "common/delay.h" #include "ton/ton-io.hpp" +#include "vm/cells/MerkleProof.h" +#include "crypto/block/block-auto.h" +#include "crypto/block/block-parse.h" namespace ton { namespace validator { +class SplitStateDeserializer { + public: + td::Result> get_effective_shards_from_header(ShardId shard_id, RootHash root_hash, + td::Ref wrapped_header, + td::uint32 split_depth) { + int shard_prefix_length = shard_pfx_len(shard_id); + CHECK(split_depth <= 63 && shard_prefix_length < static_cast(split_depth)); + + try { + TRY_RESULT(header, vm::MerkleProof::try_virtualize(wrapped_header)); + + if (RootHash{header->get_hash().bits()} != root_hash) { + return td::Status::Error("Hash mismatch in split state header"); + } + + auto shard_state_cs = vm::load_cell_slice(header); + bool rc = block::gen::t_ShardStateUnsplit.unpack(shard_state_cs, shard_state_); + if (!rc) { + return td::Status::Error("Cannot deserialize ShardStateUnsplit"); + } + + vm::AugmentedDictionary accounts{ + vm::load_cell_slice_ref(shard_state_.accounts), + 256, + block::tlb::aug_ShardAccounts, + false, + }; + + std::vector parts; + + // The following loop is the same as in state-serializer.cpp. + ShardId effective_shard = shard_id ^ (1ULL << (63 - shard_prefix_length)) ^ (1ULL << (63 - split_depth)); + ShardId increment = 1ULL << (64 - split_depth); + + for (int i = 0; i < (1 << (split_depth - shard_prefix_length)); ++i, effective_shard += increment) { + td::BitArray<64> prefix; + prefix.store_ulong(effective_shard); + auto account_dict_part = accounts; + account_dict_part.cut_prefix_subdict(prefix.bits(), split_depth); + + if (!account_dict_part.is_empty()) { + parts.push_back({effective_shard, account_dict_part.get_wrapped_dict_root()->get_hash()}); + } + } + + // Now check that header does not contain pruned cells outside of accounts dict. For that, we + // just replace account dict with an empty cell and see if header remains virtualized or not. + shard_state_.accounts = vm::DataCell::create("", 0, {}, false).move_as_ok(); + + vm::CellBuilder cb; + block::gen::t_ShardStateUnsplit.pack(cb, shard_state_); + if (cb.finalize()->get_virtualization() > 0) { + return td::Status::Error("State headers is pruned outside of account dict"); + } + + return parts; + } catch (vm::VmVirtError const&) { + return td::Status::Error("Insufficient number of cells in split state header"); + } + } + + td::Ref merge(std::vector> const& parts) { + vm::AugmentedDictionary accounts{256, block::tlb::aug_ShardAccounts}; + for (auto const& part_root : parts) { + vm::AugmentedDictionary part{ + vm::load_cell_slice_ref(part_root), + 256, + block::tlb::aug_ShardAccounts, + false, + }; + bool rc = accounts.combine_with(part); + LOG_CHECK(rc) << "Split state parts have been validated but merging them still resulted in a conflict"; + } + + CHECK(accounts.is_valid()); + + shard_state_.accounts = accounts.get_wrapped_dict_root(); + + vm::CellBuilder cb; + block::gen::t_ShardStateUnsplit.pack(cb, shard_state_); + auto state_root = cb.finalize(); + CHECK(state_root->get_virtualization() == 0); + return state_root; + } + + private: + block::gen::ShardStateUnsplit::Record shard_state_; +}; + DownloadShardState::DownloadShardState(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::uint32 split_depth, td::uint32 priority, td::actor::ActorId manager, td::Timestamp timeout, td::Promise> promise) @@ -42,8 +134,13 @@ DownloadShardState::DownloadShardState(BlockIdExt block_id, BlockIdExt mastercha if (shard_prefix_length >= static_cast(split_depth_)) { split_depth_ = 0; } + + LOG(INFO) << "requested to download state of " << block_id.to_str() << " referenced by " + << masterchain_block_id.to_str() << " with split depth " << split_depth; } +DownloadShardState::~DownloadShardState() = default; + void DownloadShardState::start_up() { status_ = ProcessStatus(manager_, "process.download_state"); alarm_timestamp() = timeout_; @@ -71,6 +168,8 @@ void DownloadShardState::got_block_handle(BlockHandle handle) { } void DownloadShardState::retry() { + deserializer_ = {}; + parts_.clear(); download_state(); } @@ -119,20 +218,35 @@ void DownloadShardState::checked_proof_link() { } }); td::actor::send_closure(manager_, &ValidatorManager::try_get_static_file, block_id_.file_hash, std::move(P)); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading zero state"); } else { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - fail_handler(SelfId, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &DownloadShardState::downloaded_shard_state, R.move_as_ok()); - } - }); CHECK(masterchain_block_id_.is_valid()); CHECK(masterchain_block_id_.is_masterchain()); - td::actor::send_closure(manager_, &ValidatorManager::send_get_persistent_state_request, block_id_, - masterchain_block_id_, UnsplitStateType{}, priority_, std::move(P)); + + if (split_depth_ == 0) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + fail_handler(SelfId, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &DownloadShardState::downloaded_shard_state, R.move_as_ok()); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::send_get_persistent_state_request, block_id_, + masterchain_block_id_, UnsplitStateType{}, priority_, std::move(P)); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading state"); + } else { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + fail_handler(SelfId, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &DownloadShardState::downloaded_split_state_header, R.move_as_ok()); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::send_get_persistent_state_request, block_id_, + masterchain_block_id_, SplitPersistentStateType{}, priority_, std::move(P)); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading state header"); + } } - status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading state"); } void DownloadShardState::download_zero_state() { @@ -200,6 +314,113 @@ void DownloadShardState::checked_shard_state() { } } +void DownloadShardState::downloaded_split_state_header(td::BufferSlice data) { + LOG(INFO) << "processing state header"; + status_.set_status(PSTRING() << block_id_.id.to_str() << " : processing state header"); + + deserializer_ = std::make_unique(); + + auto maybe_header = vm::std_boc_deserialize(data); + if (maybe_header.is_error()) { + fail_handler(actor_id(this), maybe_header.move_as_error()); + return; + } + + auto maybe_parts = deserializer_->get_effective_shards_from_header(block_id_.shard_full().shard, handle_->state(), + maybe_header.move_as_ok(), split_depth_); + if (maybe_parts.is_error()) { + fail_handler(actor_id(this), maybe_parts.move_as_error()); + return; + } + + parts_ = maybe_parts.move_as_ok(); + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &DownloadShardState::written_split_state_file); + }); + td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file, block_id_, masterchain_block_id_, + SplitPersistentStateType{}, std::move(data), std::move(P)); +} + +namespace { + +void retry_part_download(td::actor::ActorId SelfId, td::Status error) { + LOG(WARNING) << "failed to download state part : " << error; + delay_action([=]() { td::actor::send_closure(SelfId, &DownloadShardState::written_split_state_file); }, + td::Timestamp::in(1.0)); +} + +} // namespace + +void DownloadShardState::written_split_state_file() { + if (stored_parts_.size() == parts_.size()) { + auto state_root = deserializer_->merge(stored_parts_); + auto maybe_state = create_shard_state(block_id_, state_root); + + // We cannot rollback database changes here without significant elbow grease. + maybe_state.ensure(); + state_ = maybe_state.move_as_ok(); + CHECK(state_->root_hash() == handle_->state()); + + written_shard_state_file(); + return; + } + + size_t idx = stored_parts_.size(); + + LOG(INFO) << "downloading state part " << idx + 1 << " out of " << parts_.size(); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading state part (part " << idx + 1 << " out of " + << parts_.size() << ")"); + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + retry_part_download(SelfId, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &DownloadShardState::downloaded_state_part, R.move_as_ok()); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::send_get_persistent_state_request, block_id_, + masterchain_block_id_, SplitAccountStateType{parts_[idx].effective_shard}, priority_, + std::move(P)); +} + +void DownloadShardState::downloaded_state_part(td::BufferSlice data) { + size_t idx = stored_parts_.size(); + + LOG(INFO) << "processing state part " << idx + 1 << " out of " << parts_.size(); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : processing state part (part " << idx + 1 << " out of " + << parts_.size() << ")"); + + auto maybe_part = vm::std_boc_deserialize(data); + if (maybe_part.is_error()) { + retry_part_download(actor_id(this), maybe_part.move_as_error()); + return; + } + + auto root = maybe_part.move_as_ok(); + if (root->get_hash() != parts_[idx].root_hash) { + auto error_message = + "Hash mismatch for part " + + persistent_state_type_to_string(block_id_.shard_full(), SplitAccountStateType{parts_[idx].effective_shard}); + retry_part_download(actor_id(this), td::Status::Error(error_message)); + return; + } + + stored_parts_.push_back(root); + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &DownloadShardState::written_split_state_file); + }); + td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file, block_id_, masterchain_block_id_, + SplitAccountStateType{parts_[idx].effective_shard}, std::move(data), std::move(P)); + + LOG(INFO) << "storing state part to file " << idx + 1 << " out of " << parts_.size(); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : storing state part to file (part " << idx + 1 + << " out of " << parts_.size() << ")"); +} + void DownloadShardState::written_shard_state_file() { status_.set_status(PSTRING() << block_id_.id.to_str() << " : storing state to celldb"); LOG(WARNING) << "written shard state file " << block_id_.to_str(); diff --git a/validator/downloaders/download-state.hpp b/validator/downloaders/download-state.hpp index 9d3a495cb..0f66bad4b 100644 --- a/validator/downloaders/download-state.hpp +++ b/validator/downloaders/download-state.hpp @@ -25,11 +25,19 @@ namespace ton { namespace validator { +class SplitStateDeserializer; + +struct SplitStatePart { + ShardId effective_shard; + vm::CellHash root_hash; +}; + class DownloadShardState : public td::actor::Actor { public: DownloadShardState(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::uint32 split_depth, td::uint32 priority, td::actor::ActorId manager, td::Timestamp timeout, td::Promise> promise); + ~DownloadShardState(); void start_up() override; void got_block_handle(BlockHandle handle); @@ -43,8 +51,12 @@ class DownloadShardState : public td::actor::Actor { void downloaded_zero_state(td::BufferSlice data); void downloaded_shard_state(td::BufferSlice data); - void checked_shard_state(); + + void downloaded_split_state_header(td::BufferSlice data); + void written_split_state_file(); + void downloaded_state_part(td::BufferSlice data); + void written_shard_state_file(); void written_shard_state(td::Ref state); void written_block_handle(); @@ -67,6 +79,10 @@ class DownloadShardState : public td::actor::Actor { td::Timestamp timeout_; td::Promise> promise_; + std::unique_ptr deserializer_; + std::vector parts_; + std::vector> stored_parts_; + td::BufferSlice data_; td::Ref state_; diff --git a/validator/fabric.h b/validator/fabric.h index 2c39acebe..7776c8e89 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -39,7 +39,7 @@ td::Result> create_proof(BlockIdExt masterchain_block_id, td::Buf td::Result> create_proof_link(BlockIdExt block_id, td::BufferSlice proof); td::Result> create_signature_set(td::BufferSlice sig_set); td::Result> create_shard_state(BlockIdExt block_id, td::BufferSlice data); -td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell); +td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell); td::Result create_block_handle(td::BufferSlice data); td::Result create_block_handle(td::Slice data); td::Result create_temp_block_handle(td::BufferSlice data); diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index 65b922624..55b416859 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -84,7 +84,7 @@ td::Result> create_shard_state(BlockIdExt block_id, td::Buff } } -td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell) { +td::Result> create_shard_state(BlockIdExt block_id, td::Ref root_cell) { auto res = ShardStateQ::fetch(block_id, {}, std::move(root_cell)); if (res.is_error()) { return res.move_as_error(); From 39d5519c97bd6ac58741f6ac09bd6a7d5de0ae3a Mon Sep 17 00:00:00 2001 From: Dan Klishch Date: Sun, 15 Jun 2025 14:55:54 -0400 Subject: [PATCH 306/388] Save intermediate split state parts in DB This way, we don't need to keep full shard state at any point during deserialization. --- validator/db/rootdb.cpp | 5 +++++ validator/db/rootdb.hpp | 2 ++ validator/downloaders/download-state.cpp | 28 ++++++++++++++++++++---- validator/downloaders/download-state.hpp | 4 +++- validator/interfaces/db.h | 2 ++ validator/interfaces/validator-manager.h | 2 ++ validator/manager-disk.cpp | 5 +++++ validator/manager-disk.hpp | 2 ++ validator/manager-hardfork.hpp | 5 +++++ validator/manager.cpp | 5 +++++ validator/manager.hpp | 2 ++ 11 files changed, 57 insertions(+), 5 deletions(-) diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index d041228c0..8eb2b76a9 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -281,6 +281,11 @@ void RootDb::get_block_state(ConstBlockHandle handle, td::Promise cell, + td::Promise> promise) { + td::actor::send_closure(cell_db_, &CellDb::store_cell, BlockIdExt{effective_block}, cell, std::move(promise)); +} + void RootDb::get_cell_db_reader(td::Promise> promise) { td::actor::send_closure(cell_db_, &CellDb::get_cell_db_reader, std::move(promise)); } diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 57e560e1c..782e48532 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -63,6 +63,8 @@ class RootDb : public Db { void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; void get_block_state(ConstBlockHandle handle, td::Promise> promise) override; + void store_block_state_part(BlockId effective_block, td::Ref cell, + td::Promise> promise) override; void get_cell_db_reader(td::Promise> promise) override; void store_block_handle(BlockHandle handle, td::Promise promise) override; diff --git a/validator/downloaders/download-state.cpp b/validator/downloaders/download-state.cpp index 42f2c5c40..8e4cded06 100644 --- a/validator/downloaders/download-state.cpp +++ b/validator/downloaders/download-state.cpp @@ -337,7 +337,7 @@ void DownloadShardState::downloaded_split_state_header(td::BufferSlice data) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); - td::actor::send_closure(SelfId, &DownloadShardState::written_split_state_file); + td::actor::send_closure(SelfId, &DownloadShardState::download_next_part_or_finish); }); td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file, block_id_, masterchain_block_id_, SplitPersistentStateType{}, std::move(data), std::move(P)); @@ -347,13 +347,13 @@ namespace { void retry_part_download(td::actor::ActorId SelfId, td::Status error) { LOG(WARNING) << "failed to download state part : " << error; - delay_action([=]() { td::actor::send_closure(SelfId, &DownloadShardState::written_split_state_file); }, + delay_action([=]() { td::actor::send_closure(SelfId, &DownloadShardState::download_next_part_or_finish); }, td::Timestamp::in(1.0)); } } // namespace -void DownloadShardState::written_split_state_file() { +void DownloadShardState::download_next_part_or_finish() { if (stored_parts_.size() == parts_.size()) { auto state_root = deserializer_->merge(stored_parts_); auto maybe_state = create_shard_state(block_id_, state_root); @@ -411,7 +411,7 @@ void DownloadShardState::downloaded_state_part(td::BufferSlice data) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); - td::actor::send_closure(SelfId, &DownloadShardState::written_split_state_file); + td::actor::send_closure(SelfId, &DownloadShardState::written_state_part_file); }); td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file, block_id_, masterchain_block_id_, SplitAccountStateType{parts_[idx].effective_shard}, std::move(data), std::move(P)); @@ -421,6 +421,26 @@ void DownloadShardState::downloaded_state_part(td::BufferSlice data) { << " out of " << parts_.size() << ")"); } +void DownloadShardState::written_state_part_file() { + size_t idx = stored_parts_.size() - 1; + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + R.ensure(); + td::actor::send_closure(SelfId, &DownloadShardState::saved_state_part_into_celldb, R.move_as_ok()); + }); + td::actor::send_closure(manager_, &ValidatorManager::store_block_state_part, + BlockId{block_id_.shard_full().workchain, parts_[idx].effective_shard, block_id_.seqno()}, + stored_parts_.back(), std::move(P)); + LOG(INFO) << "saving to celldb state part " << idx + 1 << " out of " << parts_.size(); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : saving state part to celldb (part " << idx + 1 + << " out of " << parts_.size() << ")"); +} + +void DownloadShardState::saved_state_part_into_celldb(td::Ref cell) { + stored_parts_.back() = cell; + download_next_part_or_finish(); +} + void DownloadShardState::written_shard_state_file() { status_.set_status(PSTRING() << block_id_.id.to_str() << " : storing state to celldb"); LOG(WARNING) << "written shard state file " << block_id_.to_str(); diff --git a/validator/downloaders/download-state.hpp b/validator/downloaders/download-state.hpp index 0f66bad4b..2ba30e922 100644 --- a/validator/downloaders/download-state.hpp +++ b/validator/downloaders/download-state.hpp @@ -54,8 +54,10 @@ class DownloadShardState : public td::actor::Actor { void checked_shard_state(); void downloaded_split_state_header(td::BufferSlice data); - void written_split_state_file(); + void download_next_part_or_finish(); void downloaded_state_part(td::BufferSlice data); + void written_state_part_file(); + void saved_state_part_into_celldb(td::Ref cell); void written_shard_state_file(); void written_shard_state(td::Ref state); diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index afd087549..f8fce7ba9 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -52,6 +52,8 @@ class Db : public td::actor::Actor { virtual void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) = 0; virtual void get_block_state(ConstBlockHandle handle, td::Promise> promise) = 0; + virtual void store_block_state_part(BlockId effective_block, td::Ref cell, + td::Promise> promise) = 0; virtual void get_cell_db_reader(td::Promise> promise) = 0; virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index e18a7f18f..34d7d57c7 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -70,6 +70,8 @@ class ValidatorManager : public ValidatorManagerInterface { } virtual void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) = 0; + virtual void store_block_state_part(BlockId effective_block, td::Ref cell, + td::Promise> promise) = 0; virtual void get_cell_db_reader(td::Promise> promise) = 0; virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::BufferSlice state, diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index ed6f2ddae..3cc463a65 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -688,6 +688,11 @@ void ValidatorManagerImpl::set_block_state(BlockHandle handle, td::Ref cell, + td::Promise> promise) { + td::actor::send_closure(db_, &Db::store_block_state_part, effective_block, cell, std::move(promise)); +} + void ValidatorManagerImpl::get_cell_db_reader(td::Promise> promise) { td::actor::send_closure(db_, &Db::get_cell_db_reader, std::move(promise)); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 3729f909a..ebbdd8887 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -150,6 +150,8 @@ class ValidatorManagerImpl : public ValidatorManager { void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; + void store_block_state_part(BlockId effective_block, td::Ref cell, + td::Promise> promise) override; void get_cell_db_reader(td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::BufferSlice state, td::Promise promise) override; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index b2c8de617..20430d804 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -176,6 +176,11 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { UNREACHABLE(); } + void store_block_state_part(BlockId effective_block, td::Ref cell, + td::Promise> promise) override { + UNREACHABLE(); + } + void get_cell_db_reader(td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::BufferSlice state, td::Promise promise) override { diff --git a/validator/manager.cpp b/validator/manager.cpp index 1ff686dbd..31e757362 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1233,6 +1233,11 @@ void ValidatorManagerImpl::set_block_state(BlockHandle handle, td::Ref cell, + td::Promise> promise) { + td::actor::send_closure(db_, &Db::store_block_state_part, effective_block, cell, std::move(promise)); +} + void ValidatorManagerImpl::get_cell_db_reader(td::Promise> promise) { td::actor::send_closure(db_, &Db::get_cell_db_reader, std::move(promise)); } diff --git a/validator/manager.hpp b/validator/manager.hpp index bee59dff7..e90d5b347 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -410,6 +410,8 @@ class ValidatorManagerImpl : public ValidatorManager { void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; + void store_block_state_part(BlockId effective_block, td::Ref cell, + td::Promise> promise) override; void get_cell_db_reader(td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::BufferSlice state, td::Promise promise) override; From f42d13e4402b80fdf19c770779490785a0b36a05 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 20 Jun 2025 17:50:29 +0300 Subject: [PATCH 307/388] Various fixes in node, add tests for p256 signature (#1716) --- crypto/Ed25519.cpp | 2 +- crypto/ellcurve/p256.cpp | 5 ++++- crypto/test/fift.cpp | 4 ++++ crypto/test/fift/p256.fif | 25 +++++++++++++++++++++++++ lite-client/lite-client.cpp | 7 ++++++- storage/MicrochunkTree.cpp | 3 ++- tdutils/td/utils/BigNum.cpp | 7 +++++-- tdutils/td/utils/BigNum.h | 2 +- tdutils/td/utils/format.h | 5 +++-- test/regression-tests.ans | 1 + validator/impl/external-message.cpp | 1 + validator/manager.cpp | 2 -- 12 files changed, 53 insertions(+), 11 deletions(-) create mode 100644 crypto/test/fift/p256.fif diff --git a/crypto/Ed25519.cpp b/crypto/Ed25519.cpp index fa5b4f2f6..20fb8a3ca 100644 --- a/crypto/Ed25519.cpp +++ b/crypto/Ed25519.cpp @@ -237,7 +237,7 @@ Result Ed25519::compute_shared_secret(const PublicKey &public_key, BigNum::mod_sub(y2, p, y2, p, context); BigNum inverse_y_plus_1; - BigNum::mod_inverse(inverse_y_plus_1, y2, p, context); + TRY_STATUS(BigNum::mod_inverse(inverse_y_plus_1, y2, p, context)); BigNum u; BigNum::mod_mul(u, y, inverse_y_plus_1, p, context); diff --git a/crypto/ellcurve/p256.cpp b/crypto/ellcurve/p256.cpp index de5393723..684ac8ca0 100644 --- a/crypto/ellcurve/p256.cpp +++ b/crypto/ellcurve/p256.cpp @@ -58,7 +58,7 @@ td::Status p256_check_signature(td::Slice data, td::Slice public_key, td::Slice SCOPE_EXIT { EVP_MD_CTX_free(md_ctx); }; - if (EVP_DigestVerifyInit(md_ctx, nullptr, nullptr, nullptr, pkey) <= 0) { + if (EVP_DigestVerifyInit(md_ctx, nullptr, EVP_sha256(), nullptr, pkey) <= 0) { return td::Status::Error("Can't init DigestVerify"); } ECDSA_SIG* sig = ECDSA_SIG_new(); @@ -71,7 +71,10 @@ td::Status p256_check_signature(td::Slice data, td::Slice public_key, td::Slice BIGNUM* r = BN_bin2bn(buf, 33, nullptr); std::copy(signature.ubegin() + 32, signature.ubegin() + 64, buf + 1); BIGNUM* s = BN_bin2bn(buf, 33, nullptr); + CHECK(r != nullptr && s != nullptr); if (ECDSA_SIG_set0(sig, r, s) != 1) { + BN_free(r); + BN_free(s); return td::Status::Error("Invalid signature"); } unsigned char* signature_encoded = nullptr; diff --git a/crypto/test/fift.cpp b/crypto/test/fift.cpp index 38a9ddd92..55a6ded75 100644 --- a/crypto/test/fift.cpp +++ b/crypto/test/fift.cpp @@ -175,3 +175,7 @@ TEST(Fift, test_secp256k1) { TEST(Fift, test_get_extra_balance) { run_fift("get_extra_balance.fif"); } + +TEST(Fift, test_p256) { + run_fift("p256.fif"); +} diff --git a/crypto/test/fift/p256.fif b/crypto/test/fift/p256.fif new file mode 100644 index 000000000..ecd43d19b --- /dev/null +++ b/crypto/test/fift/p256.fif @@ -0,0 +1,25 @@ +"Asm.fif" include +"FiftExt.fif" include + +// Test vectors from https://datatracker.ietf.org/doc/html/rfc6979#appendix-A.2.5 + +x{73616d706c65} // "sample" +x{EFD48B2AACB6A8FD1140DD9CD45E81D69D2C877B56AAF991C34D0EA84EAF3716F7CB1C942D657C41D436C7A1B6E29F65F3E900DBB9AFF4064DC4AB2F843ACDA8} +x{0360FED4BA255A9D31C961EB74C6356D68C049B8923B61FA6CE669622E60F29FB6} + +<{ P256_CHKSIGNS }>s 64 runvmx .s // -1 0 +drop drop + +x{74657374} // "test" +x{F1ABB023518351CD71D881567B1EA663ED3EFCF6C5132B354F28D3B0B7D38367019F4113742A2B14BD25926B49C649155F267E60D3814B4C0CC84250E46F0083} +x{0360FED4BA255A9D31C961EB74C6356D68C049B8923B61FA6CE669622E60F29FB6} + +<{ P256_CHKSIGNS }>s 64 runvmx .s // -1 0 +drop drop + +x{74657374} // "test" +x{83910E8B48BB0C74244EBDF7F07A1C5413D61472BD941EF3920E623FBCCEBEB68DDBEC54CF8CD5874883841D712142A56A8D0F218F5003CB0296B6B509619F2C} +x{0360FED4BA255A9D31C961EB74C6356D68C049B8923B61FA6CE669622E60F29FB6} + +<{ P256_CHKSIGNS }>s 64 runvmx .s // 0 0 +drop drop diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index 1050e6d27..e97b42c80 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -422,7 +422,12 @@ void TestNode::got_server_mc_block_id(ton::BlockIdExt blkid, ton::ZeroStateIdExt } td::TerminalIO::out() << "latest masterchain block known to server is " << blkid.to_str(); if (created > 0) { - td::TerminalIO::out() << " created at " << created << " (" << now() - created << " seconds ago)\n"; + auto time = now(); + if (time >= created) { + td::TerminalIO::out() << " created at " << created << " (" << time - created << " seconds ago)\n"; + } else { + td::TerminalIO::out() << " created at " << created << " (" << created - time << " seconds in the future)\n"; + } } else { td::TerminalIO::out() << "\n"; } diff --git a/storage/MicrochunkTree.cpp b/storage/MicrochunkTree.cpp index 3249d25b4..1c7359336 100644 --- a/storage/MicrochunkTree.cpp +++ b/storage/MicrochunkTree.cpp @@ -188,7 +188,8 @@ td::Result> MicrochunkTree::get_proof(td::uint64 l, td::uint64 if (!torrent.inited_info()) { return td::Status::Error("Torrent info is not ready"); } - if (!torrent.get_info().piece_size % MICROCHUNK_SIZE != 0) { + // piece_size must be an exact multiple of MICROCHUNK_SIZE + if ((torrent.get_info().piece_size % MICROCHUNK_SIZE) != 0) { return td::Status::Error("Invalid piece size in torrent"); } td::Ref root_raw = vm::CellSlice(vm::NoVm(), root_proof_).prefetch_ref(); diff --git a/tdutils/td/utils/BigNum.cpp b/tdutils/td/utils/BigNum.cpp index 9de11fcae..22f524465 100644 --- a/tdutils/td/utils/BigNum.cpp +++ b/tdutils/td/utils/BigNum.cpp @@ -291,9 +291,12 @@ void BigNum::mod_mul(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumCon LOG_IF(FATAL, result != 1); } -void BigNum::mod_inverse(BigNum &r, BigNum &a, const BigNum &m, BigNumContext &context) { +td::Status BigNum::mod_inverse(BigNum &r, BigNum &a, const BigNum &m, BigNumContext &context) { auto result = BN_mod_inverse(r.impl_->big_num, a.impl_->big_num, m.impl_->big_num, context.impl_->big_num_context); - LOG_IF(FATAL, result != r.impl_->big_num); + if (result != r.impl_->big_num) { + return td::Status::Error("Failed to compute modulo inverse"); + } + return td::Status::OK(); } void BigNum::div(BigNum *quotient, BigNum *remainder, const BigNum ÷nd, const BigNum &divisor, diff --git a/tdutils/td/utils/BigNum.h b/tdutils/td/utils/BigNum.h index b0094659c..682b441ab 100644 --- a/tdutils/td/utils/BigNum.h +++ b/tdutils/td/utils/BigNum.h @@ -109,7 +109,7 @@ class BigNum { static void mod_mul(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumContext &context); - static void mod_inverse(BigNum &r, BigNum &a, const BigNum &m, BigNumContext &context); + static td::Status mod_inverse(BigNum &r, BigNum &a, const BigNum &m, BigNumContext &context); static void div(BigNum *quotient, BigNum *remainder, const BigNum ÷nd, const BigNum &divisor, BigNumContext &context); diff --git a/tdutils/td/utils/format.h b/tdutils/td/utils/format.h index 30d183abe..cf4447186 100644 --- a/tdutils/td/utils/format.h +++ b/tdutils/td/utils/format.h @@ -179,11 +179,12 @@ inline StringBuilder &operator<<(StringBuilder &logger, Time t) { double value; }; - static constexpr NamedValue durations[] = {{"ns", 1e-9}, {"us", 1e-6}, {"ms", 1e-3}, {"s", 1}}; + static constexpr NamedValue durations[] = {{"ns", 1e-9}, {"us", 1e-6}, {"ms", 1e-3}, + {"s", 1}, {"h", 3600}, {"d", 86400}}; static constexpr size_t durations_n = sizeof(durations) / sizeof(NamedValue); size_t i = 0; - while (i + 1 < durations_n && t.seconds_ > 10 * durations[i + 1].value) { + while (i + 1 < durations_n && std::abs(t.seconds_) > 10 * durations[i + 1].value) { i++; } logger << StringBuilder::FixedDouble(t.seconds_ / durations[i].value, 1) << durations[i].name; diff --git a/test/regression-tests.ans b/test/regression-tests.ans index 5831ebe74..16db1c9f1 100644 --- a/test/regression-tests.ans +++ b/test/regression-tests.ans @@ -21,6 +21,7 @@ Test_Fift_test_hash_ext_default 686fc5680feca5b3bb207768215b27f6872a95128762dee0 Test_Fift_test_hmap_default c269246882039824bb5822e896c3e6e82ef8e1251b6b251f5af8ea9fb8d05067 Test_Fift_test_levels_default 9fba4a7c98aec9000f42846d6e5fd820343ba61d68f9139dd16c88ccda757cf3 Test_Fift_test_namespaces_default e6419619c51332fb5e8bf22043ef415db686c47fe24f03061e5ad831014e7c6c +Test_Fift_test_p256_default e1948ddd3d2686baa9f70fdf376ffcebbc2ec5f20eeb366cd856254e61fbfa31 Test_Fift_test_rist255_default f4d7558f200a656934f986145c19b1dedbe2ad029292a5a975576d6891e25fc4 Test_Fift_test_secp256k1_default 3118450dace6af05fcdbd54a87d9446162ce11ac6ef6dfc57998cf113587d602 Test_Fift_test_sort2_default 9b57d47e6a10e7d1bbb565db35400debf2f963031f434742a702ec76555a5d3a diff --git a/validator/impl/external-message.cpp b/validator/impl/external-message.cpp index 8b1f5eb7f..54acb105e 100644 --- a/validator/impl/external-message.cpp +++ b/validator/impl/external-message.cpp @@ -112,6 +112,7 @@ void ExtMessageQ::run_message(td::Ref message, td::actor::ActorIdclone()}); } - LOG(ERROR) << "XXXX download " << id.id.to_str(); callback_->download_block(id, priority, td::Timestamp::in(10.0), std::move(promise)); } @@ -1632,7 +1631,6 @@ void ValidatorManagerImpl::send_get_block_proof_link_request(BlockIdExt block_id return; } } - LOG(ERROR) << "XXXX download link " << block_id.id.to_str(); callback_->download_block_proof_link(block_id, priority, td::Timestamp::in(10.0), std::move(promise)); } From 2e0378df313931c73c45011de197fa5938a7ada3 Mon Sep 17 00:00:00 2001 From: Dan Klishch <30951924+DanShaders@users.noreply.github.com> Date: Fri, 20 Jun 2025 10:52:54 -0400 Subject: [PATCH 308/388] Add more CMake options (#1715) * Enable colored diagnostics A well-known limitation of using ninja is that it requires the user to pass -fcolor-diagnostics=always flag to the compiler to get colored diagnostics. Starting from version 3.24, CMake can do this for us if we set CMAKE_COLOR_DIAGNOSTICS flag to a truthful value. * Add CMake option to disable GDB backtrace on fatal signal Backtraces we produce are very verbose and somewhat useless. They also are sometimes so long that they fill all of terminal scrollback and hide the actual error. * Add an off-by-default CMake option for using LLD as a linker Using LLD gives a 3-fold link-time improvement on my machine. --- CMakeLists.txt | 12 ++++++++++++ tdutils/CMakeLists.txt | 4 ++++ tdutils/td/utils/port/signals.cpp | 4 +++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5cc0a32da..c37f351d8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,6 +33,8 @@ set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED TRUE) set(CMAKE_CXX_EXTENSIONS FALSE) +set(CMAKE_COLOR_DIAGNOSTICS TRUE) + #BEGIN internal option(BUILD_SHARED_LIBS "Use \"ON\" to build shared libraries instead of static where it's not specified (not recommended)" OFF) option(USE_EMSCRIPTEN "Use \"ON\" for config building wasm." OFF) @@ -56,6 +58,16 @@ option(TON_USE_TSAN "Use \"ON\" to enable ThreadSanitizer." OFF) option(TON_USE_UBSAN "Use \"ON\" to enable UndefinedBehaviorSanitizer." OFF) set(TON_ARCH "native" CACHE STRING "Architecture, will be passed to -march=") +option(TON_PRINT_BACKTRACE_ON_CRASH "Attempt to print a backtrace when a fatal signal is caught" ON) + +option(TON_USE_LLD "Use LLD for linking" OFF) + +if (TON_USE_LLD) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=lld") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=lld") + set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=lld") +endif() + #BEGIN M1 support EXECUTE_PROCESS( COMMAND uname -m COMMAND tr -d '\n' OUTPUT_VARIABLE ARCHITECTURE ) diff --git a/tdutils/CMakeLists.txt b/tdutils/CMakeLists.txt index b4a7fec21..e1d0c87b8 100644 --- a/tdutils/CMakeLists.txt +++ b/tdutils/CMakeLists.txt @@ -319,6 +319,10 @@ set(TDUTILS_TEST_SOURCE PARENT_SCOPE ) +if (NOT TON_PRINT_BACKTRACE_ON_CRASH) + set_source_files_properties(td/utils/port/signals.cpp PROPERTIES COMPILE_DEFINITIONS TON_DISABLE_BACKTRACE) +endif() + #RULES #LIBRARIES add_library(tdutils STATIC ${TDUTILS_SOURCE}) diff --git a/tdutils/td/utils/port/signals.cpp b/tdutils/td/utils/port/signals.cpp index 1d60b2fa6..12962f29e 100644 --- a/tdutils/td/utils/port/signals.cpp +++ b/tdutils/td/utils/port/signals.cpp @@ -317,7 +317,7 @@ static void block_stdin() { #endif } -static void default_failure_signal_handler(int sig) { +[[maybe_unused]] static void default_failure_signal_handler(int sig) { Stacktrace::init(); signal_safe_write_signal_number(sig); @@ -334,9 +334,11 @@ Status set_default_failure_signal_handler() { Stdin(); // init static variables before atexit #endif std::atexit(block_stdin); +#ifndef TON_DISABLE_BACKTRACE TRY_STATUS(setup_signals_alt_stack()); TRY_STATUS(set_signal_handler(SignalType::Abort, default_failure_signal_handler)); TRY_STATUS(set_signal_handler(SignalType::Error, default_failure_signal_handler)); +#endif return Status::OK(); } From eb45c4849f4d1584b1070155e56d8bdaf01e7e46 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Fri, 20 Jun 2025 17:55:23 +0300 Subject: [PATCH 309/388] Smart cont try fetch config (#1717) * SmartContract: try fetch config from c7 if it's not set * feat: Added tvm_emulator_emulate_run_method clone with logs available * free * fix cast warning * bf --------- Co-authored-by: Marat S Co-authored-by: Yma Het --- crypto/smc-envelope/SmartContract.cpp | 42 ++++++++++++++++++++++++++ emulator/emulator-extern.cpp | 43 ++++++++++++++++++++++----- emulator/emulator-extern.h | 30 +++++++++++++++++++ emulator/emulator_export_list | 3 ++ 4 files changed, 111 insertions(+), 7 deletions(-) diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp index db2ab74ab..265c602d7 100644 --- a/crypto/smc-envelope/SmartContract.cpp +++ b/crypto/smc-envelope/SmartContract.cpp @@ -191,6 +191,42 @@ td::Ref prepare_vm_c7(SmartContract::Args args, td::Ref cod return vm::make_tuple_ref(std::move(tuple_ref)); } +std::shared_ptr try_fetch_config_from_c7(td::Ref c7) { + if (c7.is_null() || c7->size() < 1) { + return nullptr; + } + auto c7_tuple = c7->at(0).as_tuple(); + if (c7_tuple.is_null() || c7_tuple->size() < 10) { + return nullptr; + } + auto config_cell = c7_tuple->at(9).as_cell(); + if (config_cell.is_null()) { + return nullptr; + } + auto config_dict = std::make_unique(config_cell, 32); + auto config_addr_cell = config_dict->lookup_ref(td::BitArray<32>::zero()); + ton::StdSmcAddress config_addr; + if (config_addr_cell.is_null()) { + config_addr = ton::StdSmcAddress::zero(); + } else { + auto config_addr_cs = vm::load_cell_slice(std::move(config_addr_cell)); + if (config_addr_cs.size() != 0x100) { + LOG(WARNING) << "Config parameter 0 with config address has wrong size"; + config_addr = ton::StdSmcAddress::zero(); + } else { + config_addr_cs.fetch_bits_to(config_addr); + } + } + auto global_config = block::Config(config_cell, std::move(config_addr), + block::Config::needWorkchainInfo | block::Config::needSpecialSmc | block::Config::needCapabilities); + auto unpack_res = global_config.unpack(); + if (unpack_res.is_error()) { + LOG(ERROR) << "Failed to unpack config: " << unpack_res.error(); + return nullptr; + } + return std::make_shared(std::move(global_config)); +} + SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref stack, td::Ref c7, vm::GasLimits gas, bool ignore_chksig, td::Ref libraries, int vm_log_verbosity, bool debug_enabled, @@ -314,6 +350,9 @@ td::Ref SmartContract::get_init_state() const { } SmartContract::Answer SmartContract::run_method(Args args) { + if (args.c7 && !args.config) { + args.config = try_fetch_config_from_c7(args.c7.value()); + } if (!args.c7) { args.c7 = prepare_vm_c7(args, state_.code); } @@ -335,6 +374,9 @@ SmartContract::Answer SmartContract::run_method(Args args) { } SmartContract::Answer SmartContract::run_get_method(Args args) const { + if (args.c7 && !args.config) { + args.config = try_fetch_config_from_c7(args.c7.value()); + } if (!args.c7) { args.c7 = prepare_vm_c7(args, state_.code); } diff --git a/emulator/emulator-extern.cpp b/emulator/emulator-extern.cpp index c5f3c04e0..2addd353c 100644 --- a/emulator/emulator-extern.cpp +++ b/emulator/emulator-extern.cpp @@ -639,10 +639,16 @@ const char *tvm_emulator_run_get_method(void *tvm_emulator, int method_id, const return strdup(jb.string_builder().as_cslice().c_str()); } -const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc, int64_t gas_limit) { +struct TvmEulatorEmulateRunMethodResponse +{ + const char *response; + const char *log; +}; + +TvmEulatorEmulateRunMethodResponse emulate_run_method(uint32_t len, const char *params_boc, int64_t gas_limit) { auto params_cell = vm::std_boc_deserialize(td::Slice(params_boc, len)); if (params_cell.is_error()) { - return nullptr; + return { nullptr, nullptr }; } auto params_cs = vm::load_cell_slice(params_cell.move_as_ok()); auto code = params_cs.fetch_ref(); @@ -657,12 +663,12 @@ const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc td::Ref stack; if (!vm::Stack::deserialize_to(stack_cs, stack)) { - return nullptr; + return { nullptr, nullptr }; } td::Ref c7; if (!vm::Stack::deserialize_to(c7_cs, c7)) { - return nullptr; + return { nullptr, nullptr }; } auto emulator = new emulator::TvmEmulator(code, data); @@ -677,7 +683,7 @@ const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc vm::CellBuilder stack_cb; if (!result.stack->serialize(stack_cb)) { - return nullptr; + return { nullptr, nullptr }; } vm::CellBuilder cb; @@ -687,7 +693,7 @@ const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc auto ser = vm::std_boc_serialize(cb.finalize()); if (!ser.is_ok()) { - return nullptr; + return { nullptr, nullptr }; } auto sok = ser.move_as_ok(); @@ -696,7 +702,24 @@ const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc memcpy(rn, &sz, 4); memcpy(rn+4, sok.data(), sz); - return rn; + return { rn, strdup(result.vm_log.data()) }; +} + +const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc, int64_t gas_limit) { + auto result = emulate_run_method(len, params_boc, gas_limit); + return result.response; +} + +void *tvm_emulator_emulate_run_method_detailed(uint32_t len, const char *params_boc, int64_t gas_limit) { + auto result = emulate_run_method(len, params_boc, gas_limit); + return new TvmEulatorEmulateRunMethodResponse(result); +} + +void run_method_detailed_result_destroy(void *detailed_result) { + auto result = static_cast(detailed_result); + free(const_cast(result->response)); + free(const_cast(result->log)); + delete result; } const char *tvm_emulator_send_external_message(void *tvm_emulator, const char *message_body_boc) { @@ -773,6 +796,12 @@ void emulator_config_destroy(void *config) { delete static_cast(config); } +void string_destroy(const char *str) { + if (str != nullptr) { + free(const_cast(str)); + } +} + const char* emulator_version() { auto version_json = td::JsonBuilder(); auto obj = version_json.enter_object(); diff --git a/emulator/emulator-extern.h b/emulator/emulator-extern.h index 14879e1ec..2da767456 100644 --- a/emulator/emulator-extern.h +++ b/emulator/emulator-extern.h @@ -254,6 +254,27 @@ EMULATOR_EXPORT const char *tvm_emulator_run_get_method(void *tvm_emulator, int */ EMULATOR_EXPORT const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc, int64_t gas_limit); +/** + * @brief Optimized version of "run get method" with all passed parameters in a single call. Also returns log. + * @param len Length of params_boc buffer + * @param params_boc BoC serialized parameters, scheme: request$_ code:^Cell data:^Cell stack:^VmStack params:^[c7:^VmStack libs:^Cell] method_id:(## 32) + * @param gas_limit Gas limit + * @return Pointer to struct with two fields: + * - response: Char* with first 4 bytes defining length, and the rest BoC serialized result + * Scheme: result$_ exit_code:(## 32) gas_used:(## 32) stack:^VmStack + * - log: Char* with VM log string + */ +EMULATOR_EXPORT void *tvm_emulator_emulate_run_method_detailed(uint32_t len, const char *params_boc, int64_t gas_limit); + +/** + * @brief Destroy detailed result of "tvm_emulator_emulate_run_method_detailed" + * @param detailed_result Pointer to detailed result struct returned by "tvm_emulator_emulate_run_method_detailed" + * + * Caller should not use string_destroy() for fields of this struct, + * as they are already freed in this function. + */ +EMULATOR_EXPORT void run_method_detailed_result_destroy(void *detailed_result); + /** * @brief Send external message * @param tvm_emulator Pointer to TVM emulator @@ -315,6 +336,15 @@ EMULATOR_EXPORT void tvm_emulator_destroy(void *tvm_emulator); */ EMULATOR_EXPORT void emulator_config_destroy(void *config); +/** + * @brief Destroy string created by emulator library + * @param string Pointer to string to destroy + * + * This function should be used to free strings returned by emulator library functions. + * It is not safe to use caller's free() on them, as they may have been allocated using a different allocator. + */ +EMULATOR_EXPORT void string_destroy(const char *string); + /** * @brief Get git commit hash and date of the library */ diff --git a/emulator/emulator_export_list b/emulator/emulator_export_list index bd991cd73..2cec6fed9 100644 --- a/emulator/emulator_export_list +++ b/emulator/emulator_export_list @@ -27,4 +27,7 @@ _tvm_emulator_send_external_message _tvm_emulator_send_internal_message _tvm_emulator_destroy _tvm_emulator_emulate_run_method +_tvm_emulator_emulate_run_method_detailed +_run_method_detailed_result_destroy +_string_destroy _emulator_version From a174fb1044355f31ea0f3c5dcc3ae416cb97cf31 Mon Sep 17 00:00:00 2001 From: Kirill <43584399+Sumrachek@users.noreply.github.com> Date: Fri, 20 Jun 2025 21:42:45 +0400 Subject: [PATCH 310/388] Replaced hashmap by b-tree map in large boc serializer (#1686) --- crypto/test/test-db.cpp | 53 ++++++++++++++++++++++-------- crypto/vm/large-boc-serializer.cpp | 4 +-- tdutils/td/utils/HashMap.h | 6 ++++ 3 files changed, 48 insertions(+), 15 deletions(-) diff --git a/crypto/test/test-db.cpp b/crypto/test/test-db.cpp index f4dde8df7..c70738c7b 100644 --- a/crypto/test/test-db.cpp +++ b/crypto/test/test-db.cpp @@ -1691,6 +1691,31 @@ TEST(TonDb, BocDeserializerSimpleThreads) { test_boc_deserializer_threads(); } +class RandomTree { + public: + RandomTree(size_t size, td::Random::Xorshift128plus rnd) : rnd_(rnd) { + root_ = create(size); + } + Ref root() const { + return root_; + } + private: + Ref root_; + td::Random::Xorshift128plus rnd_; + Ref create(size_t size) { + CellBuilder cb; + cb.store_long(rnd_(), rnd_() % 63 + 1); + if (size > 0) { + td::uint64 rc = (rnd_() % 4) + 1; + for (td::uint64 i = 0; i < rc; i++) { + auto ref = create(size / rc); + cb.store_ref(std::move(ref)); + } + } + return cb.finalize(); + } +}; + class CompactArray { public: CompactArray(size_t size) { @@ -2987,11 +3012,8 @@ TEST(TonDb, DynamicBocRespectsUsageCell) { TEST(TonDb, LargeBocSerializer) { td::Random::Xorshift128plus rnd{123}; - size_t n = 1000000; - std::vector data(n); - std::iota(data.begin(), data.end(), 0); - vm::CompactArray arr(data); - auto root = arr.root(); + vm::RandomTree tree(100000, rnd); + auto root = tree.root(); std::string path = "serialization"; td::unlink(path).ignore(); auto fd = td::FileFd::open(path, td::FileFd::Flags::Create | td::FileFd::Flags::Truncate | td::FileFd::Flags::Write) @@ -3009,15 +3031,20 @@ TEST(TonDb, LargeBocSerializer) { dboc->commit(cell_storer); dboc->set_loader(std::make_unique(kv)); td::unlink(path).ignore(); - fd = td::FileFd::open(path, td::FileFd::Flags::Create | td::FileFd::Flags::Truncate | td::FileFd::Flags::Write) - .move_as_ok(); - boc_serialize_to_file_large(dboc->get_cell_db_reader(), root->get_hash(), fd, 31); - fd.close(); - auto b = td::read_file_str(path).move_as_ok(); - + td::string prev_b(""); auto a_cell = vm::deserialize_boc(td::BufferSlice(a)); - auto b_cell = vm::deserialize_boc(td::BufferSlice(b)); - ASSERT_EQ(a_cell->get_hash(), b_cell->get_hash()); + for (int i = 0; i < 4; i++) { + fd = td::FileFd::open(path, td::FileFd::Flags::Create | td::FileFd::Flags::Truncate | td::FileFd::Flags::Write) + .move_as_ok(); + boc_serialize_to_file_large(dboc->get_cell_db_reader(), root->get_hash(), fd, 31); + fd.close(); + auto b = td::read_file_str(path).move_as_ok(); + + auto b_cell = vm::deserialize_boc(td::BufferSlice(b)); + ASSERT_EQ(a_cell->get_hash(), b_cell->get_hash()); + if (i > 0) ASSERT_EQ(prev_b, b); + prev_b = b; + } } TEST(TonDb, DoNotMakeListsPrunned) { diff --git a/crypto/vm/large-boc-serializer.cpp b/crypto/vm/large-boc-serializer.cpp index 839e6235a..f6e35c6fe 100644 --- a/crypto/vm/large-boc-serializer.cpp +++ b/crypto/vm/large-boc-serializer.cpp @@ -115,7 +115,7 @@ td::Status LargeBocSerializer::import_cells() { td::Result LargeBocSerializer::import_cell(Hash root_hash, int root_depth) { const int start_ind = cell_count; - td::HashMap> current_depth_hashes; + td::BTreeMap> current_depth_hashes; auto existing_it = cells.find(root_hash); if (existing_it != cells.end()) { @@ -131,7 +131,7 @@ td::Result LargeBocSerializer::import_cell(Hash root_hash, int root_depth) } cell_list.resize(cell_list.size() + current_depth_hashes.size()); - td::HashMap> next_depth_hashes; + td::BTreeMap> next_depth_hashes; auto batch_start = current_depth_hashes.begin(); while (batch_start != current_depth_hashes.end()) { std::vector batch_hashes; diff --git a/tdutils/td/utils/HashMap.h b/tdutils/td/utils/HashMap.h index 1380f0291..d40006785 100644 --- a/tdutils/td/utils/HashMap.h +++ b/tdutils/td/utils/HashMap.h @@ -23,8 +23,10 @@ #if TD_HAVE_ABSL #include #include +#include #else #include +#include #endif namespace td { @@ -34,11 +36,15 @@ template > using HashMap = absl::flat_hash_map; template , class E = std::equal_to<>> using NodeHashMap = absl::node_hash_map; +template +using BTreeMap = absl::btree_map; #else template > using HashMap = std::unordered_map; template > using NodeHashMap = std::unordered_map; +template +using BTreeMap = std::map; #endif } // namespace td From 6c9de4626c4fdf73ff2d85e7ccd60c2c059e0a8a Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 20 Jun 2025 20:44:29 +0300 Subject: [PATCH 311/388] Optimized celldb and package import for archive nodes (#1653) * celldb: version 2 - thread safe cache - parallel commit - multiple optimizations - support of key-value merge operations - improved tests and benchmarks - in-memory version won't read from key value after start - uses vector in-memory table now - use rocksdb::WALRecoveryMode::kTolerateCorruptedTailRecords - do not silently ignore errors during recovery * celldb: add test for load nonexisting cell, test thread safeness of CellUsageTree, fixes * Cache validator sets for create_shard_state * Permanent celldb, improve importing archive slices, new flags for validator-engine * Fix persistent state lookup * Tool for generating liteserver desc for global config * Import proof for initial block * Perf timer for archive import * Do not recompute validator set in MasterchainStateQ::mc_reinit This does not improve performance (as unpack_validator_set result is cached regardless) but makes the code clearer. * Remove debug logs * Fix downloading initial proof * Fix error processing in CellDbIn::store_block_state_permanent_bulk * Improve some logs --------- Co-authored-by: birydrad <> Co-authored-by: Dan Klishch --- crypto/block/mc-config.h | 4 +- crypto/vm/db/CellStorage.cpp | 33 +- crypto/vm/db/CellStorage.h | 3 +- utils/CMakeLists.txt | 3 + utils/prepare-ls-slice-config.cpp | 266 ++++++++ validator-engine/validator-engine.cpp | 30 +- validator-engine/validator-engine.hpp | 12 + validator/CMakeLists.txt | 4 + validator/db/celldb.cpp | 209 +++++- validator/db/celldb.hpp | 8 + .../permanent-celldb-utils.cpp | 83 +++ .../permanent-celldb/permanent-celldb-utils.h | 37 + validator/db/rootdb.cpp | 35 + validator/db/rootdb.hpp | 4 + validator/downloaders/download-state.cpp | 16 +- validator/downloaders/download-state.hpp | 1 + validator/downloaders/wait-block-state.cpp | 28 +- validator/downloaders/wait-block-state.hpp | 11 +- validator/full-node-shard.cpp | 6 +- validator/impl/shard.cpp | 7 +- validator/import-db-slice-local.cpp | 640 ++++++++++++++++++ validator/import-db-slice-local.hpp | 103 +++ validator/import-db-slice.cpp | 8 +- validator/import-db-slice.hpp | 2 + validator/interfaces/db.h | 4 + validator/interfaces/validator-manager.h | 9 + validator/manager-disk.cpp | 14 +- validator/manager-disk.hpp | 4 + validator/manager-hardfork.cpp | 4 +- validator/manager-hardfork.hpp | 7 + validator/manager-init.cpp | 57 +- validator/manager-init.hpp | 2 +- validator/manager.cpp | 124 +++- validator/manager.hpp | 7 + validator/net/download-state.cpp | 24 +- validator/validator-options.cpp | 15 +- validator/validator-options.hpp | 18 +- validator/validator.h | 6 +- 38 files changed, 1739 insertions(+), 109 deletions(-) create mode 100644 utils/prepare-ls-slice-config.cpp create mode 100644 validator/db/permanent-celldb/permanent-celldb-utils.cpp create mode 100644 validator/db/permanent-celldb/permanent-celldb-utils.h create mode 100644 validator/import-db-slice-local.cpp create mode 100644 validator/import-db-slice-local.hpp diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 34fab4df4..d32a9945d 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -650,8 +650,8 @@ class Config { const WorkchainSet& get_workchain_list() const { return workchains_; } - const ValidatorSet* get_cur_validator_set() const { - return cur_validators_.get(); + std::shared_ptr const& get_cur_validator_set() const& { + return cur_validators_; } std::pair get_validator_set_start_stop(int next = 0) const; ton::ValidatorSessionConfig get_consensus_config() const; diff --git a/crypto/vm/db/CellStorage.cpp b/crypto/vm/db/CellStorage.cpp index f93e3fa59..33c18ab3f 100644 --- a/crypto/vm/db/CellStorage.cpp +++ b/crypto/vm/db/CellStorage.cpp @@ -32,8 +32,9 @@ namespace { class RefcntCellStorer { public: - RefcntCellStorer(td::int32 refcnt, const td::Ref &cell, bool as_boc) - : refcnt_(refcnt), cell_(cell), as_boc_(as_boc) { + RefcntCellStorer(td::int32 refcnt, const td::Ref &cell, bool as_boc, int max_level = vm::Cell::max_level) + : refcnt_(refcnt), cell_(cell), as_boc_(as_boc), max_level_(max_level) { + CHECK(!as_boc_ || max_level_ == vm::Cell::max_level); } template @@ -51,10 +52,25 @@ class RefcntCellStorer { CHECK(refcnt_ > 0); store(refcnt_, storer); CHECK(cell_.not_null()) - store(*cell_, storer); + if (max_level_ == vm::Cell::max_level) { + store(*cell_, storer); + } else { + auto level_mask = cell_->get_level_mask().apply(max_level_); + storer.template store_binary( + static_cast(cell_->get_refs_cnt() + 8 * cell_->is_special() + 32 * level_mask.get_mask())); + auto d2 = static_cast((cell_->get_bits() / 8) * 2); + if ((cell_->get_bits() & 7) != 0) { + d2 = static_cast(d2 + 1); + } + storer.template store_binary(d2); + storer.store_slice(td::Slice(cell_->get_data(), (cell_->get_bits() + 7) / 8)); + } + for (unsigned i = 0; i < cell_->size_refs(); i++) { auto cell = cell_->get_ref(i); - auto level_mask = cell->get_level_mask(); + auto level_mask = + cell->get_level_mask().apply(max_level_ + (cell_->special_type() == CellTraits::SpecialType::MerkleProof || + cell_->special_type() == CellTraits::SpecialType::MerkleUpdate)); auto level = level_mask.get_level(); td::uint8 x = static_cast(level_mask.get_mask()); storer.store_slice(td::Slice(&x, 1)); @@ -79,6 +95,7 @@ class RefcntCellStorer { td::int32 refcnt_; td::Ref cell_; bool as_boc_; + int max_level_; }; class RefcntCellParser { @@ -120,13 +137,13 @@ class RefcntCellParser { Ref refs[Cell::max_refs]; for (int i = 0; i < info.refs_cnt; i++) { if (data.size() < 1) { - return td::Status::Error("Not enought data"); + return td::Status::Error("Not enough data"); } Cell::LevelMask level_mask(data[0]); auto n = level_mask.get_hashes_count(); auto end_offset = 1 + n * (Cell::hash_bytes + Cell::depth_bytes); if (data.size() < end_offset) { - return td::Status::Error("Not enought data"); + return td::Status::Error("Not enough data"); } TRY_RESULT(ext_cell, ext_cell_creator.ext_cell(level_mask, data.substr(1, n * Cell::hash_bytes), @@ -243,8 +260,8 @@ td::Status CellStorer::erase(td::Slice hash) { return kv_.erase(hash); } -std::string CellStorer::serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc) { - return td::serialize(RefcntCellStorer(refcnt, cell, as_boc)); +std::string CellStorer::serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc, int max_level) { + return td::serialize(RefcntCellStorer(refcnt, cell, as_boc, max_level)); } td::Status CellStorer::set(td::int32 refcnt, const td::Ref &cell, bool as_boc) { diff --git a/crypto/vm/db/CellStorage.h b/crypto/vm/db/CellStorage.h index 7ae586793..e17fab5f2 100644 --- a/crypto/vm/db/CellStorage.h +++ b/crypto/vm/db/CellStorage.h @@ -72,7 +72,8 @@ class CellStorer { static void merge_refcnt_diffs(std::string &left, td::Slice right); static std::string serialize_refcnt_diffs(td::int32 refcnt_diff); - static std::string serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc); + static std::string serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc, + int max_level = vm::Cell::max_level); struct Diff { enum Type { Set, Erase, Merge } type{Set}; diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 99a4e92df..0afdf7389 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -22,4 +22,7 @@ target_include_directories(pack-viewer PUBLIC $. +*/ +#include "td/utils/filesystem.h" +#include "td/actor/actor.h" +#include "td/actor/MultiPromise.h" +#include "td/utils/OptionParser.h" +#include "td/utils/port/path.h" +#include "td/utils/port/signals.h" +#include "td/utils/port/IPAddress.h" +#include "td/utils/Random.h" +#include "td/utils/FileLog.h" +#include "git.h" +#include "auto/tl/ton_api.h" +#include "auto/tl/lite_api.h" +#include "tl-utils/lite-utils.hpp" +#include "auto/tl/ton_api_json.h" +#include "adnl/adnl.h" +#include "lite-client/ext-client.h" +#include "ton/lite-tl.hpp" + +#include "td/utils/overloaded.h" + +#include +#include "td/utils/tl_storers.h" +#include "vm/boc.h" +#include "vm/cells/MerkleProof.h" + +#include +#include "block/block-auto.h" +#include "block/mc-config.h" + +using namespace ton; + +std::string global_config_file; +td::optional start_mc_seqno, end_mc_seqno; +std::vector shards; + +class PrepareLsSliceConfig : public td::actor::Actor { + public: + void start_up() override { + if (start_mc_seqno && end_mc_seqno && start_mc_seqno.value() > end_mc_seqno.value()) { + LOG(FATAL) << "from-seqno is greater than to-seqno"; + } + + if (!start_mc_seqno && !end_mc_seqno) { + auto slice = create_tl_object(); + if (shards.empty()) { + slice->shards_.push_back(create_tl_shard_id(ShardIdFull{basechainId, shardIdAll})); + } else { + for (const ShardIdFull& shard : shards) { + if (!shard.is_masterchain()) { + slice->shards_.push_back(create_tl_shard_id(shard)); + } + } + } + print_result(*slice); + return; + } + + auto gc_s = td::read_file(global_config_file).move_as_ok(); + auto gc_j = td::json_decode(gc_s.as_slice()).move_as_ok(); + ton_api::liteclient_config_global gc; + ton_api::from_json(gc, gc_j.get_object()).ensure(); + auto r_servers = liteclient::LiteServerConfig::parse_global_config(gc); + r_servers.ensure(); + client_ = liteclient::ExtClient::create(r_servers.move_as_ok(), nullptr); + + slice_timed_ = create_tl_object(); + ++pending_; + request_shards_info(start_mc_seqno, true); + request_shards_info(end_mc_seqno, false); + dec_pending(); + } + + template + static td::BufferSlice create_query(Args&&... args) { + Type object(std::forward(args)...); + return create_serialize_tl_object(serialize_tl_object(&object, true)); + } + + template + static tl_object_ptr parse_response(const td::Result& R) { + R.ensure(); + auto err = fetch_tl_object(R.ok(), true); + if (err.is_ok()) { + LOG(FATAL) << "liteserver error: " << err.ok()->message_; + } + auto res = fetch_tl_object(R.ok(), true); + res.ensure(); + return res.move_as_ok(); + } + + void request_shards_info(td::optional seqno, bool is_start) { + if (!seqno) { + return; + } + ++pending_; + td::actor::send_closure( + client_, &liteclient::ExtClient::send_query, "q", + create_query( + 1, create_tl_object(masterchainId, shardIdAll, seqno.value()), 0, 0), + td::Timestamp::in(5.0), [=, client = client_.get(), SelfId = actor_id(this)](td::Result R) { + auto mc_header = parse_response(std::move(R)); + auto block_id = create_block_id(mc_header->id_); + td::actor::send_closure( + client, &liteclient::ExtClient::send_query, "q", + create_query(create_tl_lite_block_id(block_id)), + td::Timestamp::in(5.0), [=, mc_header = std::move(mc_header)](td::Result R) mutable { + auto shards_info = parse_response(std::move(R)); + td::actor::send_closure(SelfId, &PrepareLsSliceConfig::got_shards_info, std::move(mc_header), + std::move(shards_info), is_start); + }); + }); + } + + static tl_object_ptr parse_header(const lite_api::liteServer_blockHeader& obj, + bool is_start) { + auto res = create_tl_object(); + + BlockIdExt block_id = create_block_id(obj.id_); + res->shard_id_ = create_tl_shard_id(block_id.shard_full()); + res->seqno_ = block_id.seqno(); + + auto root = vm::std_boc_deserialize(obj.header_proof_).move_as_ok(); + root = vm::MerkleProof::virtualize(root, 1); + block::gen::Block::Record blk; + block::gen::BlockInfo::Record info; + CHECK(tlb::unpack_cell(root, blk) && tlb::unpack_cell(blk.info, info)); + res->utime_ = info.gen_utime; + res->lt_ = (is_start ? info.start_lt : info.end_lt); + + return res; + } + + void got_shards_info(tl_object_ptr mc_header, + tl_object_ptr shards_info, bool is_start) { + (is_start ? slice_timed_->shards_from_ : slice_timed_->shards_to_).push_back(parse_header(*mc_header, is_start)); + + auto root = vm::std_boc_deserialize(shards_info->data_).move_as_ok(); + block::ShardConfig sh_conf; + CHECK(sh_conf.unpack(vm::load_cell_slice_ref(root))); + auto ids = sh_conf.get_shard_hash_ids(true); + for (auto id : ids) { + BlockIdExt block_id = sh_conf.get_shard_hash(ton::ShardIdFull(id))->top_block_id(); + bool ok = shards.empty(); + for (const auto& our_shard : shards) { + if (shard_intersects(our_shard, block_id.shard_full())) { + ok = true; + break; + } + } + if (ok) { + ++pending_; + td::actor::send_closure( + client_, &liteclient::ExtClient::send_query, "q", + create_query(create_tl_lite_block_id(block_id), 0xffff), + td::Timestamp::in(5.0), [=, SelfId = actor_id(this)](td::Result R) mutable { + auto header = parse_response(std::move(R)); + td::actor::send_closure(SelfId, &PrepareLsSliceConfig::got_block_header, std::move(header), is_start); + }); + } + } + + dec_pending(); + } + + void got_block_header(tl_object_ptr header, bool is_start) { + (is_start ? slice_timed_->shards_from_ : slice_timed_->shards_to_).push_back(parse_header(*header, is_start)); + dec_pending(); + } + + void print_result(const ton_api::liteserver_descV2_Slice& result) { + auto s = td::json_encode(td::ToJson(result), true); + std::cout << s << "\n"; + std::cout.flush(); + exit(0); + } + + private: + td::actor::ActorOwn client_; + tl_object_ptr slice_timed_; + size_t pending_ = 0; + + void dec_pending() { + --pending_; + if (pending_ == 0) { + auto cmp = [](const tl_object_ptr& a, + const tl_object_ptr& b) { + return create_shard_id(a->shard_id_) < create_shard_id(b->shard_id_); + }; + std::sort(slice_timed_->shards_from_.begin(), slice_timed_->shards_from_.end(), cmp); + std::sort(slice_timed_->shards_to_.begin(), slice_timed_->shards_to_.end(), cmp); + print_result(*slice_timed_); + } + } +}; + +int main(int argc, char* argv[]) { + SET_VERBOSITY_LEVEL(verbosity_INFO); + + td::unique_ptr logger_; + SCOPE_EXIT { + td::log_interface = td::default_log_interface; + }; + + td::OptionParser p; + p.set_description( + "Generate liteserver.descV2.Slice for global-config.json from given shards and masterchain seqnos\n"); + p.add_option('v', "verbosity", "set verbosity level", [&](td::Slice arg) { + int v = VERBOSITY_NAME(FATAL) + (td::to_integer(arg)); + SET_VERBOSITY_LEVEL(v); + }); + p.add_option('V', "version", "show build information", [&]() { + std::cout << "prepare-ls-slice-config build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::exit(0); + }); + p.add_option('h', "help", "print help", [&]() { + char b[10240]; + td::StringBuilder sb(td::MutableSlice{b, 10000}); + sb << p; + std::cout << sb.as_cslice().c_str(); + std::exit(2); + }); + p.add_option('C', "global-config", "global TON configuration file (used to fetch shard configuration)", + [&](td::Slice arg) { global_config_file = arg.str(); }); + p.add_checked_option('f', "from-seqno", "starting masterchain seqno (default: none)", + [&](td::Slice arg) -> td::Status { + TRY_RESULT_ASSIGN(start_mc_seqno, td::to_integer_safe(arg)) + return td::Status::OK(); + }); + p.add_checked_option('t', "to-seqno", "ending masterchain seqno (default: none)", [&](td::Slice arg) -> td::Status { + TRY_RESULT_ASSIGN(end_mc_seqno, td::to_integer_safe(arg)) + return td::Status::OK(); + }); + p.add_checked_option('s', "shard", "shard in format 0:8000000000000000 (default: all shards)", + [&](td::Slice arg) -> td::Status { + TRY_RESULT(shard, ShardIdFull::parse(arg)); + if (!shard.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard " << arg); + } + shards.push_back(shard); + return td::Status::OK(); + }); + + p.run(argc, argv).ensure(); + td::actor::Scheduler scheduler({3}); + + scheduler.run_in_context([&] { td::actor::create_actor("main").release(); }); + while (scheduler.run(1)) { + } +} diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 582902782..b8a4f715a 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1489,6 +1489,8 @@ td::Status ValidatorEngine::load_global_config() { if (catchain_max_block_delay_slow_) { validator_options_.write().set_catchain_max_block_delay_slow(catchain_max_block_delay_slow_.value()); } + validator_options_.write().set_permanent_celldb(permanent_celldb_); + validator_options_.write().set_initial_sync_disabled(skip_key_sync_); std::vector h; for (auto &x : conf.validator_->hardforks_) { @@ -1516,16 +1518,23 @@ td::Status ValidatorEngine::load_global_config() { void ValidatorEngine::set_shard_check_function() { if (!not_all_shards_) { - validator_options_.write().set_shard_check_function([](ton::ShardIdFull shard) -> bool { return true; }); + validator_options_.write().set_shard_check_function( + [sync_shards_upto = sync_shards_upto_](ton::ShardIdFull shard, ton::BlockSeqno mc_seqno) -> bool { + return shard.is_masterchain() || !sync_shards_upto || mc_seqno <= sync_shards_upto.value(); + }); } else { std::vector shards = {ton::ShardIdFull(ton::masterchainId)}; - for (const auto& s : config_.shards_to_monitor) { + for (const auto &s : config_.shards_to_monitor) { shards.push_back(s); } std::sort(shards.begin(), shards.end()); shards.erase(std::unique(shards.begin(), shards.end()), shards.end()); validator_options_.write().set_shard_check_function( - [shards = std::move(shards)](ton::ShardIdFull shard) -> bool { + [shards = std::move(shards), sync_shards_upto = sync_shards_upto_](ton::ShardIdFull shard, + ton::BlockSeqno mc_seqno) -> bool { + if (!shard.is_masterchain() && sync_shards_upto && mc_seqno > sync_shards_upto.value()) { + return false; + } for (auto s : shards) { if (shard_intersects(shard, s)) { return true; @@ -4622,6 +4631,21 @@ int main(int argc, char *argv[]) { [&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_broadcast_speed_multiplier_private, v); }); return td::Status::OK(); }); + p.add_option( + '\0', "permanent-celldb", + "disable garbage collection in CellDb. This improves performance on archival nodes (once enabled, this option " + "cannot be disabled)", + [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_permanent_celldb, true); }); }); + p.add_option('\0', "skip-key-sync", + "don't select the best persistent state on initial sync, start on init_block from global config", [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_skip_key_sync, true); }); + }); + p.add_checked_option( + '\0', "sync-shards-upto", "stop syncing shards on this masterchain seqno", [&](td::Slice s) -> td::Status { + TRY_RESULT(v, td::to_integer_safe(s)); + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_sync_shards_upto, v); }); + return td::Status::OK(); + }); auto S = p.run(argc, argv); if (S.is_error()) { LOG(ERROR) << "failed to parse options: " << S.move_as_error(); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index de6c3ce19..61bbb72ff 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -234,6 +234,9 @@ class ValidatorEngine : public td::actor::Actor { double broadcast_speed_multiplier_catchain_ = 3.33; double broadcast_speed_multiplier_public_ = 3.33; double broadcast_speed_multiplier_private_ = 3.33; + bool permanent_celldb_ = false; + bool skip_key_sync_ = false; + td::optional sync_shards_upto_; std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; @@ -349,6 +352,15 @@ class ValidatorEngine : public td::actor::Actor { void set_broadcast_speed_multiplier_private(double value) { broadcast_speed_multiplier_private_ = value; } + void set_permanent_celldb(bool value) { + permanent_celldb_ = value; + } + void set_skip_key_sync(bool value) { + skip_key_sync_ = value; + } + void set_sync_shards_upto(ton::BlockSeqno seqno) { + sync_shards_upto_ = seqno; + } void start_up() override; ValidatorEngine() { diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index f7a4ce1d1..9fb0d190f 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -14,6 +14,8 @@ set(VALIDATOR_DB_SOURCE db/archive-slice.hpp db/celldb.cpp db/celldb.hpp + db/permanent-celldb/permanent-celldb-utils.cpp + db/permanent-celldb/permanent-celldb-utils.h db/files-async.hpp db/fileref.hpp db/fileref.cpp @@ -54,6 +56,7 @@ set(VALIDATOR_HEADERS invariants.hpp import-db-slice.hpp + import-db-slice-local.hpp queue-size-counter.hpp validator-telemetry.hpp @@ -74,6 +77,7 @@ set(VALIDATOR_SOURCE block-handle.cpp get-next-key-blocks.cpp import-db-slice.cpp + import-db-slice-local.cpp shard-client.cpp state-serializer.cpp token-manager.cpp diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index fc8735d31..f4a3a084b 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -27,6 +27,12 @@ #include "ton/ton-tl.hpp" #include "ton/ton-io.hpp" #include "common/delay.h" +#include "block/block-auto.h" +#include "permanent-celldb/permanent-celldb-utils.h" +#include "td/actor/MultiPromise.h" + +#include +#include #include #include @@ -331,6 +337,30 @@ void CellDbIn::start_up() { last_deleted_mc_state_ = r_value.move_as_ok(); } } + { + std::string key = "opts.permanent_mode", value; + auto R = boc_->meta_get(td::as_slice(key), value); + R.ensure(); + bool stored_permanent_mode = R.ok() == td::KeyValue::GetStatus::Ok; + permanent_mode_ = stored_permanent_mode || opts_->get_permanent_celldb(); + if (permanent_mode_) { + LOG(WARNING) << "Celldb is in permanent mode"; + if (!stored_permanent_mode) { + cell_db_->begin_write_batch().ensure(); + value = "1"; + vm::CellStorer stor{*cell_db_}; + boc_->meta_set(td::as_slice(key), td::as_slice(value)); + boc_->commit(stor).ensure(); + cell_db_->commit_write_batch().ensure(); + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + } + } + } + cell_db_statistics_.permanent_mode_ = permanent_mode_; + LOG_IF(FATAL, permanent_mode_ && opts_->get_celldb_in_memory()) + << "celldb permanent_mode and in_memory_mode are not compatible"; + } } void CellDbIn::load_cell(RootHash hash, td::Promise> promise) { @@ -366,7 +396,9 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi auto R = get_block(key_hash); // duplicate if (R.is_ok()) { - promise.set_result(boc_->load_cell(cell->get_hash().as_slice())); + delay_action([cell = boc_->load_cell(cell->get_hash().as_slice()), + promise = std::move(promise)]() mutable { promise.set_result(std::move(cell)); }, + td::Timestamp::now()); return; } @@ -377,7 +409,7 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi cell = std::move(cell)](td::Result Res) mutable { Res.ensure(); timer_prepare.pause(); - td::actor::send_lambda( + td::actor::send_lambda_later( SelfId, [=, this, timer = std::move(timer), promise = std::move(promise), cell = std::move(cell)]() mutable { TD_PERF_COUNTER(celldb_store_cell); auto empty = get_empty_key_hash(); @@ -414,7 +446,9 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); } - promise.set_result(boc_->load_cell(cell->get_hash().as_slice())); + delay_action([cell = boc_->load_cell(cell->get_hash().as_slice()), + promise = std::move(promise)]() mutable { promise.set_result(std::move(cell)); }, + td::Timestamp::now()); if (!opts_->get_disable_rocksdb_stats()) { cell_db_statistics_.store_cell_time_.insert(timer.elapsed() * 1e6); cell_db_statistics_.store_cell_prepare_time_.insert(timer_prepare.elapsed() * 1e6); @@ -437,6 +471,151 @@ void CellDbIn::get_cell_db_reader(td::Promise> promise.set_result(boc_->get_cell_db_reader()); } +void CellDbIn::store_block_state_permanent(td::Ref block, td::Promise> promise) { + if (!permanent_mode_) { + promise.set_error(td::Status::Error("celldb is not in permanent mode")); + return; + } + if (db_busy_) { + action_queue_.push( + [self = this, block = std::move(block), promise = std::move(promise)](td::Result R) mutable { + R.ensure(); + self->store_block_state_permanent(std::move(block), std::move(promise)); + }); + return; + } + auto key_hash = get_key_hash(block->block_id()); + auto R = get_block(key_hash); + // duplicate + if (R.is_ok()) { + delay_action([cell = boc_->load_cell(R.ok().root_hash.as_slice()), + promise = std::move(promise)]() mutable { promise.set_result(std::move(cell)); }, + td::Timestamp::now()); + return; + } + store_block_state_permanent_bulk( + {block}, [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_STATUS_PROMISE(promise, R.move_as_status()); + block::gen::Block::Record rec; + if (!block::gen::unpack_cell(block->root_cell(), rec)) { + promise.set_error(td::Status::Error("cannot unpack Block record")); + return; + } + bool spec; + vm::CellSlice update_cs = vm::load_cell_slice_special(rec.state_update, spec); + if (update_cs.special_type() != vm::CellTraits::SpecialType::MerkleUpdate) { + promise.set_error(td::Status::Error("invalid Merkle update in block")); + return; + } + td::Ref new_state_root = update_cs.prefetch_ref(1); + RootHash state_root_hash = new_state_root->get_hash(0).bits(); + td::actor::send_closure(SelfId, &CellDbIn::load_cell, state_root_hash, std::move(promise)); + }); +} + +void CellDbIn::store_block_state_permanent_bulk(std::vector> blocks, td::Promise promise) { + if (!permanent_mode_) { + promise.set_error(td::Status::Error("celldb is not in permanent mode")); + return; + } + if (db_busy_) { + action_queue_.push( + [self = this, blocks = std::move(blocks), promise = std::move(promise)](td::Result R) mutable { + R.ensure(); + self->store_block_state_permanent_bulk(std::move(blocks), std::move(promise)); + }); + return; + } + td::PerfWarningTimer timer{"storecellbulk", 0.1}; + td::Timer timer_prepare; + std::map> new_blocks; + for (auto& block : blocks) { + BlockIdExt block_id = block->block_id(); + if (new_blocks.contains(block_id)) { + continue; + } + if (get_block(get_key_hash(block_id)).is_ok()) { + continue; + } + new_blocks[block_id] = std::move(block); + } + if (new_blocks.empty()) { + promise.set_value(td::Unit{}); + return; + } + for (auto& [block_id, block] : new_blocks) { + std::vector prev; + BlockIdExt mc_blkid; + bool after_split; + TRY_STATUS_PROMISE(promise, + block::unpack_block_prev_blk_try(block->root_cell(), block_id, prev, mc_blkid, after_split)); + for (const BlockIdExt& prev_id : prev) { + if (!new_blocks.contains(prev_id) && get_block(get_key_hash(prev_id)).is_error()) { + promise.set_error(td::Status::Error("cannot store block state: previous block is not in db")); + return; + } + } + } + db_busy_ = true; + calculate_permanent_celldb_update( + new_blocks, async_executor, + [this, SelfId = actor_id(this), timer = std::move(timer), timer_prepare = std::move(timer_prepare), + promise = std::move(promise)](td::Result> R) mutable { + td::actor::send_lambda_later( + SelfId, [=, this, timer = std::move(timer), timer_prepare = std::move(timer_prepare), R = std::move(R), + promise = std::move(promise)]() mutable { + SCOPE_EXIT { + release_db(); + }; + TRY_RESULT_PROMISE(promise, updates, std::move(R)); + TD_PERF_COUNTER(celldb_store_cell_multi); + timer_prepare.pause(); + td::Timer timer_write; + vm::CellStorer stor{*cell_db_}; + cell_db_->begin_write_batch().ensure(); + + for (auto& update : updates) { + for (auto& [k, v] : update.to_store) { + cell_db_->set(k.as_slice(), v).ensure(); + } + } + + CHECK(!updates.empty()); + auto empty = get_empty_key_hash(); + auto E = get_block(empty).move_as_ok(); + for (size_t i = 0; i < updates.size(); ++i) { + KeyHash prev = i == 0 ? empty : get_key_hash(updates[i - 1].block_id); + KeyHash next = i + 1 == updates.size() ? E.next : get_key_hash(updates[i + 1].block_id); + DbEntry entry{updates[i].block_id, prev, next, updates[i].state_root_hash}; + set_block(get_key_hash(updates[i].block_id), std::move(entry)); + } + E.next = get_key_hash(updates[0].block_id); + if (E.prev == empty) { + E.prev = get_key_hash(updates.back().block_id); + } + set_block(empty, std::move(E)); + + boc_->commit(stor).ensure(); // Save meta + cell_db_->commit_write_batch().ensure(); + timer_write.pause(); + + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); + } + + if (!opts_->get_disable_rocksdb_stats()) { + cell_db_statistics_.store_cell_time_.insert(timer.elapsed() * 1e6); + cell_db_statistics_.store_cell_prepare_time_.insert(timer_prepare.elapsed() * 1e6); + cell_db_statistics_.store_cell_write_time_.insert(timer_write.elapsed() * 1e6); + cell_db_statistics_.store_cell_bulk_queries_++; + cell_db_statistics_.store_cell_bulk_total_blocks_ += updates.size(); + } + promise.set_result(td::Unit()); + }); + }); +} + std::vector> CellDbIn::prepare_stats() { TD_PERF_COUNTER(celldb_prepare_stats); auto r_boc_stats = boc_->get_stats(); @@ -504,6 +683,7 @@ void CellDbIn::flush_db_stats() { } td::RocksDb::reset_statistics(statistics_); cell_db_statistics_.clear(); + cell_db_statistics_.permanent_mode_ = permanent_mode_; } void CellDbIn::alarm() { @@ -522,6 +702,10 @@ void CellDbIn::alarm() { << " queue_size=" << cells_to_migrate_.size(); migration_stats_ = {}; } + if (permanent_mode_) { + skip_gc(); + return; + } auto E = get_block(get_empty_key_hash()).move_as_ok(); auto N = get_block(E.next).move_as_ok(); if (N.is_empty()) { @@ -575,6 +759,7 @@ void CellDbIn::gc_cont2(BlockHandle handle) { }); return; } + CHECK(!permanent_mode_); td::PerfWarningTimer timer{"gccell", 0.1}; td::PerfWarningTimer timer_all{"gccell_all", 0.05}; @@ -614,7 +799,7 @@ void CellDbIn::gc_cont2(BlockHandle handle) { P = std::move(P), N = std::move(N), cell = std::move(cell), timer = std::move(timer), timer_all = std::move(timer_all), handle](td::Result R) mutable { R.ensure(); - td::actor::send_lambda(SelfId, [this, timer_boc = std::move(timer_boc), F = std::move(F), key_hash, + td::actor::send_lambda_later(SelfId, [this, timer_boc = std::move(timer_boc), F = std::move(F), key_hash, P = std::move(P), N = std::move(N), cell = std::move(cell), timer = std::move(timer), timer_all = std::move(timer_all), handle]() mutable { TD_PERF_COUNTER(celldb_gc_cell); @@ -714,6 +899,9 @@ void CellDbIn::set_block(KeyHash key_hash, DbEntry e) { } void CellDbIn::migrate_cell(td::Bits256 hash) { + if (permanent_mode_) { + return; + } cells_to_migrate_.insert(hash); if (!migration_active_) { migration_active_ = true; @@ -826,6 +1014,14 @@ void CellDb::store_cell(BlockIdExt block_id, td::Ref cell, td::Promise td::actor::send_closure(cell_db_, &CellDbIn::store_cell, block_id, std::move(cell), std::move(promise)); } +void CellDb::store_block_state_permanent(td::Ref block, td::Promise> promise) { + td::actor::send_closure(cell_db_, &CellDbIn::store_block_state_permanent, std::move(block), std::move(promise)); +} + +void CellDb::store_block_state_permanent_bulk(std::vector> blocks, td::Promise promise) { + td::actor::send_closure(cell_db_, &CellDbIn::store_block_state_permanent_bulk, std::move(blocks), std::move(promise)); +} + void CellDb::get_cell_db_reader(td::Promise> promise) { td::actor::send_closure(cell_db_, &CellDbIn::get_cell_db_reader, std::move(promise)); } @@ -862,9 +1058,14 @@ td::BufferSlice CellDbIn::DbEntry::release() { std::vector> CellDbIn::CellDbStatistics::prepare_stats() { std::vector> stats; + stats.emplace_back("permanent_mode", PSTRING() << permanent_mode_); stats.emplace_back("store_cell.micros", PSTRING() << store_cell_time_.to_string()); stats.emplace_back("store_cell.prepare.micros", PSTRING() << store_cell_prepare_time_.to_string()); stats.emplace_back("store_cell.write.micros", PSTRING() << store_cell_write_time_.to_string()); + if (permanent_mode_) { + stats.emplace_back("store_cell.bulk.queries", PSTRING() << store_cell_bulk_queries_); + stats.emplace_back("store_cell.bulk.total_blocks", PSTRING() << store_cell_bulk_total_blocks_); + } stats.emplace_back("gc_cell.micros", PSTRING() << gc_cell_time_.to_string()); stats.emplace_back("total_time.micros", PSTRING() << (td::Timestamp::now().at() - stats_start_time_.at()) * 1e6); stats.emplace_back("in_memory", PSTRING() << bool(in_memory_load_time_)); diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index 1e1ccddab..f70f6aef8 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -66,6 +66,8 @@ class CellDbIn : public CellDbBase { void load_cell(RootHash hash, td::Promise> promise); void store_cell(BlockIdExt block_id, td::Ref cell, td::Promise> promise); void get_cell_db_reader(td::Promise> promise); + void store_block_state_permanent(td::Ref block, td::Promise> promise); + void store_block_state_permanent_bulk(std::vector> blocks, td::Promise promise); void migrate_cell(td::Bits256 hash); @@ -137,9 +139,12 @@ class CellDbIn : public CellDbBase { std::unique_ptr migration_stats_; struct CellDbStatistics { + bool permanent_mode_; PercentileStats store_cell_time_; PercentileStats store_cell_prepare_time_; PercentileStats store_cell_write_time_; + size_t store_cell_bulk_queries_ = 0; + size_t store_cell_bulk_total_blocks_ = 0; PercentileStats gc_cell_time_; td::Timestamp stats_start_time_ = td::Timestamp::now(); std::optional in_memory_load_time_; @@ -156,6 +161,7 @@ class CellDbIn : public CellDbBase { CellDbStatistics cell_db_statistics_; td::Timestamp statistics_flush_at_ = td::Timestamp::never(); BlockSeqno last_deleted_mc_state_ = 0; + bool permanent_mode_ = false; bool db_busy_ = false; std::queue> action_queue_; @@ -188,6 +194,8 @@ class CellDb : public CellDbBase { void prepare_stats(td::Promise>> promise); void load_cell(RootHash hash, td::Promise> promise); void store_cell(BlockIdExt block_id, td::Ref cell, td::Promise> promise); + void store_block_state_permanent(td::Ref block, td::Promise> promise); + void store_block_state_permanent_bulk(std::vector> blocks, td::Promise promise); void update_snapshot(std::unique_ptr snapshot) { CHECK(!opts_->get_celldb_in_memory()); if (!started_) { diff --git a/validator/db/permanent-celldb/permanent-celldb-utils.cpp b/validator/db/permanent-celldb/permanent-celldb-utils.cpp new file mode 100644 index 000000000..b1d19ceb4 --- /dev/null +++ b/validator/db/permanent-celldb/permanent-celldb-utils.cpp @@ -0,0 +1,83 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "permanent-celldb-utils.h" +#include "block/block-auto.h" +#include "td/actor/MultiPromise.h" +#include "td/utils/HashMap.h" +#include "vm/db/CellStorage.h" + +namespace ton::validator { + +void calculate_permanent_celldb_update(const std::map>& blocks, + std::shared_ptr executor, + td::Promise> promise) { + td::MultiPromise mp; + auto ig = mp.init_guard(); + auto updates = std::make_shared>(); + updates->reserve(blocks.size()); + for (auto& [_, block] : blocks) { + executor->execute_async([block = block, updates, executor, + promise = std::make_shared>(ig.get_promise())]() mutable { + block::gen::Block::Record rec; + if (!block::gen::unpack_cell(block->root_cell(), rec)) { + promise->set_error(td::Status::Error("cannot unpack Block record")); + return; + } + bool spec; + vm::CellSlice update_cs = vm::load_cell_slice_special(rec.state_update, spec); + if (update_cs.special_type() != vm::CellTraits::SpecialType::MerkleUpdate) { + promise->set_error(td::Status::Error("invalid Merkle update in block")); + return; + } + td::Ref new_state_root = update_cs.prefetch_ref(1); + td::HashMap visited; + PermanentCellDbUpdate update{.block_id = block->block_id(), + .state_root_hash = new_state_root->get_hash(0).bits()}; + std::function&, int)> dfs = [&](const td::Ref& cell, int merkle_depth) { + int& vis = visited[cell->get_hash()]; + if (vis & (1 << merkle_depth)) { + return; + } + vis |= (1 << merkle_depth); + vm::CellSlice cs{vm::NoVm(), cell}; + if (cs.special_type() == vm::CellTraits::SpecialType::PrunnedBranch && cell->get_level() == merkle_depth + 1) { + return; + } + update.to_store.emplace_back( + cell->get_hash(merkle_depth), + vm::CellStorer::serialize_value(1 << 29, cell->load_cell().move_as_ok().data_cell, false, merkle_depth)); + merkle_depth = cs.child_merkle_depth(merkle_depth); + for (unsigned i = 0; i < cs.size_refs(); ++i) { + dfs(cs.prefetch_ref(i), merkle_depth); + } + }; + dfs(new_state_root, 0); + executor->execute_sync( + [update = std::move(update), updates = std::move(updates), promise = std::move(promise)]() mutable { + updates->push_back(std::move(update)); + promise->set_result(td::Unit()); + }); + }); + } + + ig.add_promise([updates, promise = std::move(promise)](td::Result R) mutable { + TRY_STATUS_PROMISE(promise, R.move_as_status()); + promise.set_value(std::move(*updates)); + }); +} + +} // namespace ton::validator diff --git a/validator/db/permanent-celldb/permanent-celldb-utils.h b/validator/db/permanent-celldb/permanent-celldb-utils.h new file mode 100644 index 000000000..fad2ddfcd --- /dev/null +++ b/validator/db/permanent-celldb/permanent-celldb-utils.h @@ -0,0 +1,37 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "vm/cells.h" +#include "ton/ton-types.h" +#include "interfaces/block.h" +#include "vm/db/DynamicBagOfCellsDb.h" + +#include +#include + +namespace ton::validator { + +struct PermanentCellDbUpdate { + BlockIdExt block_id; + RootHash state_root_hash; + std::vector> to_store; +}; +void calculate_permanent_celldb_update(const std::map>& blocks, + std::shared_ptr executor, + td::Promise> promise); + +} // namespace ton::validator diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 8eb2b76a9..dfdbc72d0 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -259,6 +259,41 @@ void RootDb::store_block_state(BlockHandle handle, td::Ref state, } } +void RootDb::store_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) { + if (handle->id() != block->block_id()) { + promise.set_error(td::Status::Error("block id mismatch")); + return; + } + if (handle->moved_to_archive() || handle->inited_state_boc()) { + get_block_state(handle, std::move(promise)); + return; + } + auto P = td::PromiseCreator::lambda( + [b = archive_db_.get(), handle, promise = std::move(promise)](td::Result> R) mutable { + TRY_RESULT_PROMISE(promise, root, std::move(R)); + handle->set_state_root_hash(root->get_hash().bits()); + handle->set_state_boc(); + + auto S = create_shard_state(handle->id(), std::move(root)); + S.ensure(); + + auto P = td::PromiseCreator::lambda( + [promise = std::move(promise), state = S.move_as_ok()](td::Result R) mutable { + R.ensure(); + promise.set_value(std::move(state)); + }); + + td::actor::send_closure(b, &ArchiveManager::update_handle, std::move(handle), std::move(P)); + }); + td::actor::send_closure(cell_db_, &CellDb::store_block_state_permanent, std::move(block), std::move(P)); +} + +void RootDb::store_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) { + td::actor::send_closure(cell_db_, &CellDb::store_block_state_permanent_bulk, std::move(blocks), std::move(promise)); +} + void RootDb::get_block_state(ConstBlockHandle handle, td::Promise> promise) { if (handle->inited_state_boc()) { if (handle->deleted_state_boc()) { diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 782e48532..05d5cc027 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -62,6 +62,10 @@ class RootDb : public Db { void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; + void store_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) override; + void store_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) override; void get_block_state(ConstBlockHandle handle, td::Promise> promise) override; void store_block_state_part(BlockId effective_block, td::Ref cell, td::Promise> promise) override; diff --git a/validator/downloaders/download-state.cpp b/validator/downloaders/download-state.cpp index 8e4cded06..d60811d59 100644 --- a/validator/downloaders/download-state.cpp +++ b/validator/downloaders/download-state.cpp @@ -178,7 +178,22 @@ void DownloadShardState::download_state() { checked_proof_link(); return; } + status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading proof"); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id = block_id_](td::Result R) { + if (R.is_error()) { + LOG(DEBUG) << "Cannot get proof link from import: " << R.move_as_error(); + td::actor::send_closure(SelfId, &DownloadShardState::download_proof_link); + } else { + LOG(INFO) << "Got proof link for " << block_id.to_str() << " from import"; + td::actor::send_closure(SelfId, &DownloadShardState::downloaded_proof_link, R.move_as_ok()); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::get_block_proof_link_from_import, block_id_, + masterchain_block_id_, std::move(P)); +} + +void DownloadShardState::download_proof_link() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { fail_handler(SelfId, R.move_as_error()); @@ -188,7 +203,6 @@ void DownloadShardState::download_state() { }); td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_link_request, block_id_, priority_, std::move(P)); - status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading proof"); } void DownloadShardState::downloaded_proof_link(td::BufferSlice data) { diff --git a/validator/downloaders/download-state.hpp b/validator/downloaders/download-state.hpp index 2ba30e922..1c1be04c4 100644 --- a/validator/downloaders/download-state.hpp +++ b/validator/downloaders/download-state.hpp @@ -46,6 +46,7 @@ class DownloadShardState : public td::actor::Actor { void checked_proof_link(); void download_state(); + void download_proof_link(); void download_zero_state(); void downloaded_zero_state(td::BufferSlice data); diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index 00085e960..92a0720bc 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -68,6 +68,8 @@ void WaitBlockState::start() { return; } bool inited_proof = handle_->id().is_masterchain() ? handle_->inited_proof() : handle_->inited_proof_link(); + bool allow_download = + last_masterchain_state_.is_null() || opts_->need_monitor(handle_->id().shard_full(), last_masterchain_state_); if (handle_->received_state() && inited_proof) { reading_from_db_ = true; @@ -81,8 +83,8 @@ void WaitBlockState::start() { td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, handle_, std::move(P)); } else if (handle_->id().id.seqno == 0 && next_static_file_attempt_.is_in_past()) { next_static_file_attempt_ = td::Timestamp::in(60.0); - // id.file_hash contrains correct file hash of zero state - // => if file with this sha256 is found it is garanteed to be correct + // id.file_hash contains correct file hash of zero state + // => if file with this sha256 is found it is guaranteed to be correct // => if it is not, this error is permanent auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), id = handle_->id()](td::Result R) { if (R.is_error()) { @@ -108,7 +110,7 @@ void WaitBlockState::start() { }); td::actor::send_closure(manager_, &ValidatorManager::send_get_zero_state_request, handle_->id(), priority_, std::move(P)); - } else if (check_persistent_state_desc() && !handle_->received_state()) { + } else if (check_persistent_state_desc() && !handle_->received_state() && allow_download) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { LOG(WARNING) << "failed to get persistent state: " << R.move_as_error(); @@ -138,7 +140,11 @@ void WaitBlockState::start() { .release(); } } else if (!handle_->inited_prev() || (!handle_->inited_proof() && !handle_->inited_proof_link())) { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result R) { + if (!allow_download) { + abort_query(td::Status::Error(PSTRING() << "not monitoring shard " << handle_->id().shard_full())); + return; + } + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link); }, td::Timestamp::in(0.1)); @@ -164,6 +170,10 @@ void WaitBlockState::start() { td::actor::send_closure(manager_, &ValidatorManager::wait_prev_block_state, handle_, priority_, timeout_, std::move(P)); } else if (handle_->id().is_masterchain() && !handle_->inited_proof()) { + if (!allow_download) { + abort_query(td::Status::Error(PSTRING() << "not monitoring shard " << handle_->id().shard_full())); + return; + } auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result R) { if (R.is_error()) { delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof); }, @@ -177,6 +187,10 @@ void WaitBlockState::start() { td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_request, handle_->id(), priority_, std::move(P)); } else if (block_.is_null()) { + if (!allow_download && !handle_->received()) { + abort_query(td::Status::Error(PSTRING() << "not monitoring shard " << handle_->id().shard_full())); + return; + } auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &WaitBlockState::failed_to_get_block_data, @@ -261,6 +275,12 @@ void WaitBlockState::got_block_data(td::Ref data) { } void WaitBlockState::apply() { + if (opts_->get_permanent_celldb()) { + td::actor::send_closure(manager_, &ValidatorManager::set_block_state_from_data, handle_, block_, + std::move(promise_)); + stop(); + return; + } TD_PERF_COUNTER(apply_block_to_state); td::PerfWarningTimer t{"applyblocktostate", 0.1}; auto S = prev_state_.write().apply_block(handle_->id(), block_); diff --git a/validator/downloaders/wait-block-state.hpp b/validator/downloaders/wait-block-state.hpp index 6a14d909f..a9317381c 100644 --- a/validator/downloaders/wait-block-state.hpp +++ b/validator/downloaders/wait-block-state.hpp @@ -26,18 +26,21 @@ namespace validator { class WaitBlockState : public td::actor::Actor { public: - WaitBlockState(BlockHandle handle, td::uint32 priority, td::actor::ActorId manager, + WaitBlockState(BlockHandle handle, td::uint32 priority, td::Ref opts, + td::Ref last_masterchain_state, td::actor::ActorId manager, td::Timestamp timeout, td::Promise> promise, td::Ref persistent_state_desc = {}) : handle_(std::move(handle)) , priority_(priority) + , opts_(opts) + , last_masterchain_state_(last_masterchain_state) , manager_(manager) , timeout_(timeout) , promise_(std::move(promise)) , persistent_state_desc_(std::move(persistent_state_desc)) , perf_timer_("waitstate", 1.0, [manager](double duration) { - send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitstate", duration); - }) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitstate", duration); + }) { } void abort_query(td::Status reason); @@ -89,6 +92,8 @@ class WaitBlockState : public td::actor::Actor { td::uint32 priority_; + td::Ref opts_; + td::Ref last_masterchain_state_; td::actor::ActorId manager_; td::Timestamp timeout_; td::Promise> promise_; diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 608a5ec8b..00f6d0f25 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -541,7 +541,11 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod auto P = td::PromiseCreator::lambda([promise = std::move(promise), cnt](td::Result> R) mutable { if (R.is_error()) { - LOG(WARNING) << "getnextkey: " << R.move_as_error(); + if (R.error().code() == ErrorCode::notready) { + LOG(DEBUG) << "getnextkey: " << R.move_as_error(); + } else { + LOG(WARNING) << "getnextkey: " << R.move_as_error(); + } auto x = create_serialize_tl_object( std::vector>{}, false, true); promise.set_value(std::move(x)); diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index d9a78b2de..dd038c608 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -388,11 +388,8 @@ td::Status MasterchainStateQ::mc_reinit() { CHECK(config_); CHECK(config_->set_block_id_ext(get_block_id())); - auto cv_root = config_->get_config_param(35, 34); - if (cv_root.not_null()) { - TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(cv_root), true)); - cur_validators_ = std::move(validators); - } + cur_validators_ = config_->get_cur_validator_set(); + auto nv_root = config_->get_config_param(37, 36); if (nv_root.not_null()) { TRY_RESULT(validators, block::Config::unpack_validator_set(std::move(nv_root), true)); diff --git a/validator/import-db-slice-local.cpp b/validator/import-db-slice-local.cpp new file mode 100644 index 000000000..8c8db4ffb --- /dev/null +++ b/validator/import-db-slice-local.cpp @@ -0,0 +1,640 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "import-db-slice-local.hpp" + +#include "validator/db/fileref.hpp" +#include "td/utils/overloaded.h" +#include "validator/fabric.h" +#include "td/actor/MultiPromise.h" +#include "common/checksum.h" +#include "td/utils/port/path.h" +#include "ton/ton-io.hpp" +#include "downloaders/download-state.hpp" +#include "block/block-auto.h" + +#include + +namespace ton { + +namespace validator { + +ArchiveImporterLocal::ArchiveImporterLocal(std::string db_root, td::Ref state, + BlockSeqno shard_client_seqno, td::Ref opts, + td::actor::ActorId manager, + std::vector to_import_files, + td::Promise> promise) + : db_root_(std::move(db_root)) + , last_masterchain_state_(std::move(state)) + , shard_client_seqno_(shard_client_seqno) + , opts_(std::move(opts)) + , manager_(manager) + , to_import_files_(std::move(to_import_files)) + , promise_(std::move(promise)) + , perf_timer_("import-slice-local", 10.0, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "import-slice-local", duration); + }) { +} + +void ArchiveImporterLocal::start_up() { + LOG(WARNING) << "Importing archive for masterchain seqno #" << shard_client_seqno_ + 1 << " from disk"; + for (const std::string &path : to_import_files_) { + LOG(INFO) << "Importing file from disk " << path; + td::Status S = process_package(path); + if (S.is_error()) { + LOG(WARNING) << "Error processing package " << path << ": " << S; + } + } + + process_masterchain_blocks(); +} + +td::Status ArchiveImporterLocal::process_package(std::string path) { + LOG(DEBUG) << "Processing package " << path; + TRY_RESULT(p, Package::open(path, false, false)); + auto package = std::make_shared(std::move(p)); + + td::Status S = td::Status::OK(); + package->iterate([&](std::string filename, td::BufferSlice data, td::uint64) -> bool { + auto F = FileReference::create(filename); + if (F.is_error()) { + S = F.move_as_error(); + return false; + } + auto f = F.move_as_ok(); + + BlockIdExt block_id; + bool is_proof = false; + bool ignore = true; + + f.ref().visit(td::overloaded( + [&](const fileref::Proof &p) { + block_id = p.block_id; + ignore = !block_id.is_masterchain(); + is_proof = true; + }, + [&](const fileref::ProofLink &p) { + block_id = p.block_id; + ignore = block_id.is_masterchain(); + is_proof = true; + }, + [&](const fileref::Block &p) { + block_id = p.block_id; + ignore = false; + is_proof = false; + }, + [&](const auto &) { ignore = true; })); + + if (ignore || block_id.is_masterchain() && block_id.seqno() <= last_masterchain_state_->get_seqno()) { + return true; + } + + if (is_proof) { + if (block_id.is_masterchain()) { + auto R = create_proof(block_id, std::move(data)); + if (R.is_error()) { + S = R.move_as_error(); + return false; + } + blocks_[block_id].proof = R.move_as_ok(); + } else { + auto R = create_proof_link(block_id, std::move(data)); + if (R.is_error()) { + S = R.move_as_error(); + return false; + } + blocks_[block_id].proof_link = R.move_as_ok(); + } + } else { + if (td::sha256_bits256(data) != block_id.file_hash) { + S = td::Status::Error(ErrorCode::protoviolation, "bad block file hash"); + return false; + } + auto R = create_block(block_id, std::move(data)); + if (R.is_error()) { + S = R.move_as_error(); + return false; + } + blocks_[block_id].block = R.move_as_ok(); + } + if (block_id.is_masterchain()) { + masterchain_blocks_[block_id.seqno()] = block_id; + } + return true; + }); + return S; +} + +void ArchiveImporterLocal::process_masterchain_blocks() { + if (masterchain_blocks_.empty()) { + LOG(INFO) << "No masterchain blocks in the archive"; + checked_masterchain_proofs(); + return; + } + + if (masterchain_blocks_.begin()->first != last_masterchain_state_->get_seqno() + 1) { + abort_query(td::Status::Error(ErrorCode::notready, PSTRING() << "expected masterchain seqno " + << last_masterchain_state_->get_seqno() + 1 + << ", found " << masterchain_blocks_.begin()->first)); + return; + } + { + BlockSeqno expected_seqno = last_masterchain_state_->get_seqno() + 1; + for (auto &[seqno, _] : masterchain_blocks_) { + if (seqno != expected_seqno) { + abort_query( + td::Status::Error(ErrorCode::protoviolation, "non-consecutive masterchain blocks in the archive")); + return; + } + ++expected_seqno; + } + } + BlockInfo &first_block = blocks_[masterchain_blocks_.begin()->second]; + if (first_block.proof.is_null()) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block proof")); + return; + } + if (first_block.block.is_null()) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block data")); + return; + } + block::gen::Block::Record rec; + block::gen::BlockInfo::Record info; + if (!(block::gen::unpack_cell(first_block.block->root_cell(), rec) && block::gen::unpack_cell(rec.info, info))) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "cannot unpack masterchain block info")); + return; + } + if (info.key_block) { + import_first_key_block(); + return; + } + + process_masterchain_blocks_cont(); +} + +void ArchiveImporterLocal::import_first_key_block() { + BlockIdExt block_id = masterchain_blocks_.begin()->second; + BlockInfo &first_block = blocks_[block_id]; + LOG(INFO) << "First block in archive is key block : " << block_id.id.to_str(); + + auto P = + td::PromiseCreator::lambda([SelfId = actor_id(this), prev_block_id = last_masterchain_state_->get_block_id()]( + td::Result R) mutable { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + return; + } + auto handle = R.move_as_ok(); + CHECK(!handle->merge_before()); + if (handle->one_prev(true) != prev_block_id) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, + td::Status::Error(ErrorCode::protoviolation, "prev block mismatch")); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::checked_key_block_proof, std::move(handle)); + }); + + run_check_proof_query(block_id, first_block.proof, manager_, td::Timestamp::in(10.0), std::move(P), + last_masterchain_state_, opts_->is_hardfork(block_id)); +} + +void ArchiveImporterLocal::checked_key_block_proof(BlockHandle handle) { + BlockIdExt block_id = masterchain_blocks_.begin()->second; + CHECK(block_id == handle->id()); + BlockInfo &first_block = blocks_[block_id]; + run_apply_block_query( + handle->id(), first_block.block, handle->id(), manager_, td::Timestamp::in(600.0), + [SelfId = actor_id(this), manager = manager_, handle](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure( + manager, &ValidatorManager::get_shard_state_from_db, handle, [=](td::Result> R2) { + if (R2.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R2.move_as_error()); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::applied_key_block, + td::Ref{R2.move_as_ok()}); + }); + }); +} + +void ArchiveImporterLocal::applied_key_block(td::Ref state) { + CHECK(state->get_block_id() == masterchain_blocks_.begin()->second); + last_masterchain_state_ = state; + imported_any_ = true; + masterchain_blocks_.erase(masterchain_blocks_.begin()); + blocks_.erase(state->get_block_id()); + LOG(INFO) << "Imported key block " << state->get_block_id().id.to_str(); + if (masterchain_blocks_.empty()) { + LOG(INFO) << "No more masterchain blocks in the archive"; + checked_masterchain_proofs(); + return; + } + process_masterchain_blocks_cont(); +} + +void ArchiveImporterLocal::process_masterchain_blocks_cont() { + LOG(INFO) << "Importing masterchain blocks from " << masterchain_blocks_.begin()->first << " to " + << masterchain_blocks_.rbegin()->first; + + td::MultiPromise mp; + auto ig = mp.init_guard(); + + BlockIdExt prev_block_id = last_masterchain_state_->get_block_id(); + for (auto &[_, block_id] : masterchain_blocks_) { + auto &info = blocks_[block_id]; + info.import = true; + if (info.proof.is_null()) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block proof")); + return; + } + if (info.block.is_null()) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block data")); + return; + } + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), prev_block_id, promise = ig.get_promise()](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + CHECK(!handle->merge_before()); + if (handle->one_prev(true) != prev_block_id) { + promise.set_error(td::Status::Error(ErrorCode::protoviolation, "prev block mismatch")); + return; + } + promise.set_result(td::Unit()); + }); + run_check_proof_query(block_id, info.proof, manager_, td::Timestamp::in(10.0), std::move(P), + last_masterchain_state_, opts_->is_hardfork(block_id)); + prev_block_id = block_id; + } + ig.add_promise([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + } else { + LOG(INFO) << "Checked proofs for masterchain blocks"; + td::actor::send_closure(SelfId, &ArchiveImporterLocal::checked_masterchain_proofs); + } + }); +} + +void ArchiveImporterLocal::checked_masterchain_proofs() { + if (shard_client_seqno_ == last_masterchain_state_->get_seqno()) { + got_shard_client_state(last_masterchain_state_); + } else { + CHECK(shard_client_seqno_ < last_masterchain_state_->get_seqno()); + BlockIdExt block_id; + if (!last_masterchain_state_->get_old_mc_block_id(shard_client_seqno_, block_id)) { + abort_query(td::Status::Error("failed to get shard client block id")); + return; + } + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db_short, block_id, + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, + R.move_as_error_prefix("failed to get shard client state: ")); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::got_shard_client_state, + td::Ref{R.move_as_ok()}); + }); + } +} + +void ArchiveImporterLocal::got_shard_client_state(td::Ref state) { + CHECK(state->get_seqno() == shard_client_seqno_); + LOG(DEBUG) << "got_shard_client_state " << shard_client_seqno_; + shard_client_state_ = state; + new_shard_client_seqno_ = shard_client_seqno_; + current_shard_client_seqno_ = shard_client_seqno_; + for (auto &shard : state->get_shards()) { + visited_shard_blocks_.insert(shard->top_block_id()); + } + try_advance_shard_client_seqno(); +} + +void ArchiveImporterLocal::try_advance_shard_client_seqno() { + BlockSeqno seqno = new_shard_client_seqno_ + 1; + auto it = masterchain_blocks_.find(seqno); + if (it != masterchain_blocks_.end()) { + try_advance_shard_client_seqno_cont(blocks_[it->second].block); + return; + } + if (seqno > last_masterchain_state_->get_seqno()) { + processed_shard_blocks(); + return; + } + BlockIdExt block_id; + if (!last_masterchain_state_->get_old_mc_block_id(seqno, block_id)) { + abort_query(td::Status::Error("failed to get old mc block id")); + return; + } + td::actor::send_closure(manager_, &ValidatorManager::get_block_data_from_db_short, block_id, + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, + R.move_as_error_prefix("failed to get block data: ")); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::try_advance_shard_client_seqno_cont, + R.move_as_ok()); + }); +} + +void ArchiveImporterLocal::try_advance_shard_client_seqno_cont(td::Ref mc_block) { + CHECK(mc_block.not_null()); + CHECK(mc_block->block_id().seqno() == new_shard_client_seqno_ + 1); + LOG(DEBUG) << "try_advance_shard_client_seqno " << new_shard_client_seqno_ + 1; + + block::gen::Block::Record rec; + block::gen::BlockExtra::Record extra; + block::gen::McBlockExtra::Record mc_extra; + CHECK(block::gen::unpack_cell(mc_block->root_cell(), rec) && block::gen::unpack_cell(rec.extra, extra) && + block::gen::unpack_cell(extra.custom->prefetch_ref(), mc_extra)); + auto shard_config = std::make_unique(mc_extra.shard_hashes->prefetch_ref()); + + std::vector blocks_to_import; + std::function dfs = [&](const BlockIdExt &block_id) -> td::Status { + if (visited_shard_blocks_.contains(block_id)) { + return td::Status::OK(); + } + if (block_id.seqno() == 0) { + new_zerostates_.insert(block_id); + return td::Status::OK(); + } + visited_shard_blocks_.insert(block_id); + auto &info = blocks_[block_id]; + if (info.block.is_null()) { + return td::Status::Error(PSTRING() << "no shard block " << block_id.to_str()); + } + blocks_to_import.push_back(block_id); + + std::vector prev; + BlockIdExt mc_blkid; + bool after_split; + TRY_STATUS(block::unpack_block_prev_blk_try(info.block->root_cell(), block_id, prev, mc_blkid, after_split)); + for (const BlockIdExt &prev_block_id : prev) { + TRY_STATUS(dfs(prev_block_id)); + } + + return td::Status::OK(); + }; + td::Status S = td::Status::OK(); + std::vector top_shard_blocks; + shard_config->process_shard_hashes([&](block::McShardHash &shard) { + if (!opts_->need_monitor(shard.shard(), shard_client_state_)) { + return 0; + } + S = dfs(shard.top_block_id()); + top_shard_blocks.push_back(shard.top_block_id()); + if (S.is_error()) { + return -1; + } + return 0; + }); + if (S.is_error()) { + LOG(DEBUG) << "Cannot advance shard client seqno to " << new_shard_client_seqno_ + 1 << " : " << S; + processed_shard_blocks(); + return; + } + shard_configs_[mc_block->block_id().seqno()] = {mc_block->block_id(), std::move(top_shard_blocks)}; + ++new_shard_client_seqno_; + LOG(DEBUG) << "Advancing shard client seqno to " << new_shard_client_seqno_; + for (const BlockIdExt &block_id : blocks_to_import) { + blocks_[block_id].import = true; + } + td::actor::send_closure(actor_id(this), &ArchiveImporterLocal::try_advance_shard_client_seqno); +} + +void ArchiveImporterLocal::processed_shard_blocks() { + if (new_shard_client_seqno_ == shard_client_seqno_) { + LOG(INFO) << "No new shard blocks"; + } else { + LOG(INFO) << "New shard client seqno = " << new_shard_client_seqno_; + } + + td::MultiPromise mp; + auto ig = mp.init_guard(); + for (const BlockIdExt &block_id : new_zerostates_) { + LOG(INFO) << "Downloading zerostate " << block_id.to_str(); + td::actor::create_actor( + "downloadstate", block_id, shard_client_state_->get_block_id(), + shard_client_state_->persistent_state_split_depth(block_id.id.workchain), 2, manager_, td::Timestamp::in(3600), + ig.get_promise().wrap([](td::Ref &&) { return td::Unit(); })) + .release(); + } + ig.add_promise([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::store_data); + } + }); +} + +void ArchiveImporterLocal::store_data() { + td::MultiPromise mp; + auto ig = mp.init_guard(); + + if (opts_->get_permanent_celldb()) { + std::vector> blocks; + for (auto &[_, info] : blocks_) { + if (info.import) { + blocks.push_back(info.block); + } + } + td::actor::send_closure(manager_, &ValidatorManager::set_block_state_from_data_preliminary, std::move(blocks), + ig.get_promise()); + } + for (auto &[block_id, info] : blocks_) { + if (info.import) { + td::actor::send_closure( + manager_, &ValidatorManager::get_block_handle, block_id, true, + [promise = ig.get_promise(), block = info.block, manager = manager_](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + td::actor::send_closure(manager, &ValidatorManager::set_block_data, handle, std::move(block), + std::move(promise)); + }); + if (info.proof_link.not_null()) { + run_check_proof_link_query(block_id, info.proof_link, manager_, td::Timestamp::in(60.0), + ig.get_promise().wrap([](BlockHandle &&) { return td::Unit(); })); + } + } + } + + ig.add_promise([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::apply_next_masterchain_block); + } + }); +} + +void ArchiveImporterLocal::apply_next_masterchain_block() { + auto it = masterchain_blocks_.find(last_masterchain_state_->get_seqno() + 1); + if (it == masterchain_blocks_.end()) { + LOG(INFO) << "Applied masterchain blocks, last seqno = " << last_masterchain_state_->get_seqno(); + apply_shard_blocks(); + return; + } + BlockIdExt block_id = it->second; + LOG(DEBUG) << "Applying masterchain block " << block_id.to_str(); + BlockInfo &info = blocks_[block_id]; + run_apply_block_query(block_id, info.block, block_id, manager_, td::Timestamp::in(600.0), + [=, SelfId = actor_id(this), manager = manager_](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure( + manager, &ValidatorManager::get_shard_state_from_db_short, block_id, + [=](td::Result> R2) { + if (R2.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, + R2.move_as_error()); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::applied_next_masterchain_block, + td::Ref{R2.move_as_ok()}); + }); + }); +} + +void ArchiveImporterLocal::applied_next_masterchain_block(td::Ref state) { + last_masterchain_state_ = state; + imported_any_ = true; + LOG(DEBUG) << "Applied masterchain block " << state->get_block_id().to_str(); + apply_next_masterchain_block(); +} + +void ArchiveImporterLocal::apply_shard_blocks() { + if (current_shard_client_seqno_ == new_shard_client_seqno_) { + finish_query(); + return; + } + auto it = shard_configs_.find(current_shard_client_seqno_ + 1); + if (it == shard_configs_.end()) { + abort_query(td::Status::Error("no shard config for the next shard client seqno")); + return; + } + + td::MultiPromise mp; + auto ig = mp.init_guard(); + BlockIdExt mc_block_id = it->second.first; + LOG(DEBUG) << "Applying top shard blocks from " << current_shard_client_seqno_ + 1; + for (const BlockIdExt &block_id : it->second.second) { + apply_shard_block(block_id, mc_block_id, ig.get_promise()); + } + + ig.add_promise([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporterLocal::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure(SelfId, &ArchiveImporterLocal::applied_shard_blocks); + }); +} + +void ArchiveImporterLocal::applied_shard_blocks() { + LOG(DEBUG) << "Applied top shard blocks from " << current_shard_client_seqno_ + 1; + ++current_shard_client_seqno_; + imported_any_ = true; + apply_shard_blocks(); +} + +void ArchiveImporterLocal::apply_shard_block(BlockIdExt block_id, BlockIdExt mc_block_id, + td::Promise promise) { + td::actor::send_closure( + manager_, &ValidatorManager::get_block_handle, block_id, true, + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + R.ensure(); + td::actor::send_closure(SelfId, &ArchiveImporterLocal::apply_shard_block_cont1, R.move_as_ok(), mc_block_id, + std::move(promise)); + }); +} + +void ArchiveImporterLocal::apply_shard_block_cont1(BlockHandle handle, BlockIdExt mc_block_id, + td::Promise promise) { + if (handle->is_applied()) { + promise.set_value(td::Unit()); + return; + } + + promise = [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_STATUS_PROMISE(promise, R.move_as_status()); + td::actor::send_closure(SelfId, &ArchiveImporterLocal::apply_shard_block_cont2, handle, mc_block_id, + std::move(promise)); + }; + + if (!handle->merge_before() && handle->one_prev(true).shard_full() == handle->id().shard_full()) { + apply_shard_block(handle->one_prev(true), mc_block_id, std::move(promise)); + } else { + td::MultiPromise mp; + auto ig = mp.init_guard(); + ig.add_promise(std::move(promise)); + check_shard_block_applied(handle->one_prev(true), ig.get_promise()); + if (handle->merge_before()) { + check_shard_block_applied(handle->one_prev(false), ig.get_promise()); + } + } +} + +void ArchiveImporterLocal::apply_shard_block_cont2(BlockHandle handle, BlockIdExt mc_block_id, + td::Promise promise) { + td::Ref block = blocks_[handle->id()].block; + CHECK(block.not_null()); + LOG(DEBUG) << "Applying shard block " << handle->id().to_str(); + run_apply_block_query(handle->id(), std::move(block), mc_block_id, manager_, td::Timestamp::in(600.0), + std::move(promise)); +} + +void ArchiveImporterLocal::check_shard_block_applied(BlockIdExt block_id, td::Promise promise) { + td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, block_id, false, + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + if (!handle->is_applied()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "not applied")); + } else { + promise.set_value(td::Unit()); + } + }); +} + +void ArchiveImporterLocal::abort_query(td::Status error) { + if (!imported_any_) { + LOG(ERROR) << "Archive import: " << error; + promise_.set_error(std::move(error)); + stop(); + } else { + LOG(WARNING) << "Archive import: " << error; + finish_query(); + } +} + +void ArchiveImporterLocal::finish_query() { + LOG(WARNING) << "Imported archive in " << perf_timer_.elapsed() + << "s : mc_seqno=" << last_masterchain_state_->get_seqno() + << " shard_seqno=" << current_shard_client_seqno_; + promise_.set_value({last_masterchain_state_->get_seqno(), + std::min(last_masterchain_state_->get_seqno(), current_shard_client_seqno_)}); + stop(); +} + +} // namespace validator + +} // namespace ton diff --git a/validator/import-db-slice-local.hpp b/validator/import-db-slice-local.hpp new file mode 100644 index 000000000..e9e6efdb1 --- /dev/null +++ b/validator/import-db-slice-local.hpp @@ -0,0 +1,103 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "td/actor/actor.h" +#include "td/utils/port/path.h" +#include "validator/interfaces/validator-manager.h" +#include "validator/db/package.hpp" + +namespace ton { + +namespace validator { + +class ArchiveImporterLocal : public td::actor::Actor { + public: + ArchiveImporterLocal(std::string db_root, td::Ref state, BlockSeqno shard_client_seqno, + td::Ref opts, td::actor::ActorId manager, + std::vector to_import_files, + td::Promise> promise); + void start_up() override; + + void abort_query(td::Status error); + void finish_query(); + + td::Status process_package(std::string path); + + void process_masterchain_blocks(); + void process_masterchain_blocks_cont(); + + void import_first_key_block(); + void checked_key_block_proof(BlockHandle handle); + void applied_key_block(td::Ref state); + + void checked_masterchain_proofs(); + void got_shard_client_state(td::Ref state); + + void try_advance_shard_client_seqno(); + void try_advance_shard_client_seqno_cont(td::Ref mc_block); + + void processed_shard_blocks(); + void store_data(); + void apply_next_masterchain_block(); + void applied_next_masterchain_block(td::Ref state); + + void apply_shard_blocks(); + void applied_shard_blocks(); + + void apply_shard_block(BlockIdExt block_id, BlockIdExt mc_block_id, td::Promise promise); + void apply_shard_block_cont1(BlockHandle handle, BlockIdExt mc_block_id, td::Promise promise); + void apply_shard_block_cont2(BlockHandle handle, BlockIdExt mc_block_id, td::Promise promise); + void check_shard_block_applied(BlockIdExt block_id, td::Promise promise); + + private: + std::string db_root_; + td::Ref last_masterchain_state_; + BlockSeqno shard_client_seqno_; + + td::Ref opts_; + + td::actor::ActorId manager_; + + std::vector to_import_files_; + td::Promise> promise_; + + struct BlockInfo { + td::Ref block; + td::Ref proof; + td::Ref proof_link; + bool import = false; + }; + std::map blocks_; + std::map masterchain_blocks_; + + td::Ref shard_client_state_; + BlockSeqno new_shard_client_seqno_; + BlockSeqno current_shard_client_seqno_; + std::set visited_shard_blocks_; + std::set new_zerostates_; + + std::map>> shard_configs_; + + bool imported_any_ = false; + + td::PerfWarningTimer perf_timer_; +}; + +} // namespace validator + +} // namespace ton diff --git a/validator/import-db-slice.cpp b/validator/import-db-slice.cpp index 80fe141bb..58f77afb6 100644 --- a/validator/import-db-slice.cpp +++ b/validator/import-db-slice.cpp @@ -45,7 +45,10 @@ ArchiveImporter::ArchiveImporter(std::string db_root, td::Ref , manager_(manager) , to_import_files_(std::move(to_import_files)) , use_imported_files_(!to_import_files_.empty()) - , promise_(std::move(promise)) { + , promise_(std::move(promise)) + , perf_timer_("import-slice", 10.0, [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "import-slice", duration); + }) { } void ArchiveImporter::start_up() { @@ -85,7 +88,7 @@ void ArchiveImporter::downloaded_mc_archive(std::string path) { void ArchiveImporter::processed_mc_archive() { if (masterchain_blocks_.empty()) { - LOG(DEBUG) << "No masterhchain blocks in archive"; + LOG(DEBUG) << "No masterchain blocks in archive"; last_masterchain_seqno_ = last_masterchain_state_->get_seqno(); checked_all_masterchain_blocks(); return; @@ -527,6 +530,7 @@ void ArchiveImporter::finish_query() { td::unlink(f).ignore(); } if (promise_) { + LOG(INFO) << "Imported archive in " << perf_timer_.elapsed(); promise_.set_value({last_masterchain_state_->get_seqno(), std::min(last_masterchain_state_->get_seqno(), shard_client_seqno_)}); } diff --git a/validator/import-db-slice.hpp b/validator/import-db-slice.hpp index 04f22642d..4cf6ce40a 100644 --- a/validator/import-db-slice.hpp +++ b/validator/import-db-slice.hpp @@ -91,6 +91,8 @@ class ArchiveImporter : public td::actor::Actor { bool imported_any_ = false; bool have_shard_blocks_ = false; std::vector files_to_cleanup_; + + td::PerfWarningTimer perf_timer_; }; } // namespace validator diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index f8fce7ba9..00e7f31f8 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -51,6 +51,10 @@ class Db : public td::actor::Actor { virtual void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) = 0; + virtual void store_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) = 0; + virtual void store_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) = 0; virtual void get_block_state(ConstBlockHandle handle, td::Promise> promise) = 0; virtual void store_block_state_part(BlockId effective_block, td::Ref cell, td::Promise> promise) = 0; diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 34d7d57c7..abab4e6c9 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -72,6 +72,10 @@ class ValidatorManager : public ValidatorManagerInterface { td::Promise> promise) = 0; virtual void store_block_state_part(BlockId effective_block, td::Ref cell, td::Promise> promise) = 0; + virtual void set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) = 0; + virtual void set_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) = 0; virtual void get_cell_db_reader(td::Promise> promise) = 0; virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::BufferSlice state, @@ -157,6 +161,11 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) = 0; + virtual void get_block_proof_link_from_import(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { + promise.set_error(td::Status::Error("not supported")); + } + virtual void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_shard_client_state(bool from_db, td::Promise promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 3cc463a65..f1be05ca3 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -323,8 +323,8 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle->id(), std::move(R)); }); - auto id = td::actor::create_actor("waitstate", handle, 0, actor_id(this), td::Timestamp::in(10.0), - std::move(P)) + auto id = td::actor::create_actor("waitstate", handle, 0, opts_, last_masterchain_state_, + actor_id(this), td::Timestamp::in(10.0), std::move(P)) .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); @@ -693,6 +693,16 @@ void ValidatorManagerImpl::store_block_state_part(BlockId effective_block, td::R td::actor::send_closure(db_, &Db::store_block_state_part, effective_block, cell, std::move(promise)); } +void ValidatorManagerImpl::set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) { + td::actor::send_closure(db_, &Db::store_block_state_from_data, handle, block, std::move(promise)); +} + +void ValidatorManagerImpl::set_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) { + td::actor::send_closure(db_, &Db::store_block_state_from_data_preliminary, std::move(blocks), std::move(promise)); +} + void ValidatorManagerImpl::get_cell_db_reader(td::Promise> promise) { td::actor::send_closure(db_, &Db::get_cell_db_reader, std::move(promise)); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index ebbdd8887..dc08bf28b 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -152,6 +152,10 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void store_block_state_part(BlockId effective_block, td::Ref cell, td::Promise> promise) override; + void set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) override; + void set_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) override; void get_cell_db_reader(td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::BufferSlice state, td::Promise promise) override; diff --git a/validator/manager-hardfork.cpp b/validator/manager-hardfork.cpp index 91c598aa2..c3d6267be 100644 --- a/validator/manager-hardfork.cpp +++ b/validator/manager-hardfork.cpp @@ -171,8 +171,8 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle->id(), std::move(R)); }); - auto id = td::actor::create_actor("waitstate", handle, 0, actor_id(this), td::Timestamp::in(10.0), - std::move(P)) + auto id = td::actor::create_actor("waitstate", handle, 0, opts_, last_masterchain_state_, + actor_id(this), td::Timestamp::in(10.0), std::move(P)) .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 20430d804..0f25a5cfa 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -181,6 +181,13 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } + void set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) override { + UNREACHABLE(); + } + void set_block_state_from_data_preliminary(std::vector> blocks, td::Promise promise) { + UNREACHABLE(); + } void get_cell_db_reader(td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::BufferSlice state, td::Promise promise) override { diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index 004e5b120..f9a55c6ae 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -59,7 +59,7 @@ void ValidatorManagerMasterchainReiniter::got_masterchain_handle(BlockHandle han handle_ = std::move(handle); key_blocks_.push_back(handle_); - if (opts_->initial_sync_disabled()) { + if (opts_->initial_sync_disabled() && handle_->id().seqno() == 0) { status_.set_status(PSTRING() << "downloading masterchain state " << handle_->id().seqno()); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); @@ -74,7 +74,7 @@ void ValidatorManagerMasterchainReiniter::got_masterchain_handle(BlockHandle han download_proof_link(); } -void ValidatorManagerMasterchainReiniter::download_proof_link() { +void ValidatorManagerMasterchainReiniter::download_proof_link(bool try_local) { if (handle_->id().id.seqno == 0) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); @@ -84,45 +84,55 @@ void ValidatorManagerMasterchainReiniter::download_proof_link() { td::Timestamp::in(3600), std::move(P)) .release(); } else { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { - LOG(WARNING) << "failed to download proof link: " << R.move_as_error(); + if (try_local) { + LOG(DEBUG) << "failed to get proof link from local import: " << R.move_as_error(); + } else { + LOG(WARNING) << "failed to download proof link: " << R.move_as_error(); + } delay_action( - [SelfId]() { td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_proof_link); }, + [SelfId]() { + td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_proof_link, false); + }, td::Timestamp::in(1.0)); } else { td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::downloaded_proof_link, R.move_as_ok()); } }); - td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_link_request, handle_->id(), 2, - std::move(P)); + if (try_local) { + td::actor::send_closure(manager_, &ValidatorManager::get_block_proof_link_from_import, handle_->id(), + handle_->id(), std::move(P)); + } else { + td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_link_request, handle_->id(), 2, + std::move(P)); + } } } -void ValidatorManagerMasterchainReiniter::downloaded_proof_link(td::BufferSlice proof) { - auto pp = create_proof_link(handle_->id(), std::move(proof)); - if (pp.is_error()) { - LOG(WARNING) << "bad proof link: " << pp.move_as_error(); +void ValidatorManagerMasterchainReiniter::downloaded_proof_link(td::BufferSlice data) { + auto r_proof = create_proof(handle_->id(), std::move(data)); + if (r_proof.is_error()) { + LOG(WARNING) << "bad proof link: " << r_proof.move_as_error(); download_proof_link(); return; } + auto proof = r_proof.move_as_ok(); - auto proof_link = pp.move_as_ok(); - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), db = db_, proof_link](td::Result R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), db = db_, proof](td::Result R) { if (R.is_error()) { LOG(WARNING) << "downloaded proof link failed: " << R.move_as_error(); - td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_proof_link); + td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_proof_link, false); } else { auto P = td::PromiseCreator::lambda([SelfId, handle = R.move_as_ok()](td::Result R) { R.ensure(); td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::try_download_key_blocks, false); }); - td::actor::send_closure(db, &Db::add_key_block_proof_link, proof_link, std::move(P)); + td::actor::send_closure(db, &Db::add_key_block_proof_link, proof, std::move(P)); } }); - - run_check_proof_link_query(handle_->id(), proof_link, manager_, td::Timestamp::in(60.0), std::move(P)); + run_check_proof_query(handle_->id(), proof, manager_, td::Timestamp::in(60.0), std::move(P), + /* skip_check_signatures = */ true); } void ValidatorManagerMasterchainReiniter::downloaded_zero_state() { @@ -130,6 +140,10 @@ void ValidatorManagerMasterchainReiniter::downloaded_zero_state() { } void ValidatorManagerMasterchainReiniter::try_download_key_blocks(bool try_start) { + if (opts_->initial_sync_disabled()) { + download_masterchain_state(); + return; + } if (!download_new_key_blocks_until_) { if (opts_->allow_blockchain_init()) { download_new_key_blocks_until_ = td::Timestamp::in(60.0); @@ -183,8 +197,6 @@ void ValidatorManagerMasterchainReiniter::got_next_key_blocks(std::vector(key_blocks_.size()); key_blocks_.resize(key_blocks_.size() + vec.size(), nullptr); @@ -204,6 +216,11 @@ void ValidatorManagerMasterchainReiniter::got_key_block_handle(td::uint32 idx, B CHECK(!key_blocks_[idx]); CHECK(handle->inited_proof()); CHECK(handle->is_key_block()); + if (idx + 1 == key_blocks_.size()) { + int ago = (int)td::Clocks::system() - (int)handle->unix_time(); + LOG(WARNING) << "last key block is " << handle->id().to_str() << ", " << ago << "s ago"; + status_.set_status(PSTRING() << "last key block is " << handle->id().seqno() << ", " << ago << " s ago"); + } key_blocks_[idx] = std::move(handle); CHECK(pending_ > 0); if (!--pending_) { diff --git a/validator/manager-init.hpp b/validator/manager-init.hpp index 901b826bd..90dc5ccc9 100644 --- a/validator/manager-init.hpp +++ b/validator/manager-init.hpp @@ -44,7 +44,7 @@ class ValidatorManagerMasterchainReiniter : public td::actor::Actor { void start_up() override; void written_hardforks(); void got_masterchain_handle(BlockHandle handle); - void download_proof_link(); + void download_proof_link(bool try_local = true); void downloaded_proof_link(td::BufferSlice data); void downloaded_zero_state(); diff --git a/validator/manager.cpp b/validator/manager.cpp index 578b91323..7a2d63b7d 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -31,6 +31,7 @@ #include "state-serializer.hpp" #include "get-next-key-blocks.h" #include "import-db-slice.hpp" +#include "import-db-slice-local.hpp" #include "auto/tl/lite_api.h" #include "tl-utils/lite-utils.hpp" @@ -42,6 +43,7 @@ #include "td/utils/JsonBuilder.h" #include "common/delay.h" +#include "db/fileref.hpp" #include "td/utils/filesystem.h" #include "validator/stats-merger.h" @@ -708,10 +710,6 @@ void ValidatorManagerImpl::run_ext_query(td::BufferSlice data, td::Promise> promise) { - if (last_masterchain_state_.not_null() && !opts_->need_monitor(handle->id().shard_full(), last_masterchain_state_)) { - return promise.set_error( - td::Status::Error(PSTRING() << "not monitoring shard " << handle->id().shard_full().to_str())); - } auto it0 = block_state_cache_.find(handle->id()); if (it0 != block_state_cache_.end()) { it0->second.ttl_ = td::Timestamp::in(30.0); @@ -723,10 +721,11 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); }); - auto id = td::actor::create_actor("waitstate", handle, priority, actor_id(this), - td::Timestamp::at(timeout.at() + 10.0), std::move(P), - get_block_persistent_state_to_download(handle->id())) - .release(); + auto id = + td::actor::create_actor("waitstate", handle, priority, opts_, last_masterchain_state_, + actor_id(this), td::Timestamp::at(timeout.at() + 10.0), std::move(P), + get_block_persistent_state_to_download(handle->id())) + .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); } @@ -785,10 +784,6 @@ void ValidatorManagerImpl::wait_block_data_short(BlockIdExt block_id, td::uint32 void ValidatorManagerImpl::wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) { - if (last_masterchain_state_.not_null() && !opts_->need_monitor(left_id.shard_full(), last_masterchain_state_)) { - return promise.set_error( - td::Status::Error(PSTRING() << "not monitoring shard " << left_id.shard_full().to_str())); - } td::actor::create_actor("merge", left_id, right_id, priority, actor_id(this), timeout, std::move(promise)) .release(); @@ -1171,10 +1166,10 @@ void ValidatorManagerImpl::finished_wait_state(BlockHandle handle, td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); }); - auto id = - td::actor::create_actor("waitstate", handle, X.second, actor_id(this), X.first, - std::move(P), get_block_persistent_state_to_download(handle->id())) - .release(); + auto id = td::actor::create_actor("waitstate", handle, X.second, opts_, last_masterchain_state_, + actor_id(this), X.first, std::move(P), + get_block_persistent_state_to_download(handle->id())) + .release(); it->second.actor_ = id; return; } @@ -1238,6 +1233,16 @@ void ValidatorManagerImpl::store_block_state_part(BlockId effective_block, td::R td::actor::send_closure(db_, &Db::store_block_state_part, effective_block, cell, std::move(promise)); } +void ValidatorManagerImpl::set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) { + td::actor::send_closure(db_, &Db::store_block_state_from_data, handle, block, std::move(promise)); +} + +void ValidatorManagerImpl::set_block_state_from_data_preliminary(std::vector> blocks, + td::Promise promise) { + td::actor::send_closure(db_, &Db::store_block_state_from_data_preliminary, std::move(blocks), std::move(promise)); +} + void ValidatorManagerImpl::get_cell_db_reader(td::Promise> promise) { td::actor::send_closure(db_, &Db::get_cell_db_reader, std::move(promise)); } @@ -1693,6 +1698,67 @@ void ValidatorManagerImpl::send_download_archive_request(BlockSeqno mc_seqno, Sh callback_->download_archive(mc_seqno, shard_prefix, std::move(tmp_dir), timeout, std::move(promise)); } +void ValidatorManagerImpl::get_block_proof_link_from_import(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) { + auto it = to_import_all_.upper_bound(masterchain_block_id.seqno() + 1); + while (true) { + if (it == to_import_all_.begin()) { + promise.set_error(td::Status::Error("proof not found")); + return; + } + --it; + bool stop = false; + for (const std::string &path : it->second) { + td::BufferSlice result; + auto r_package = Package::open(path, false, false); + if (r_package.is_error()) { + LOG(WARNING) << "Cannot open package " << path << " : " << r_package.move_as_error(); + continue; + } + auto package = r_package.move_as_ok(); + package.iterate([&](std::string filename, td::BufferSlice data, td::uint64) -> bool { + auto F = FileReference::create(filename); + if (F.is_error()) { + return true; + } + auto f = F.move_as_ok(); + BlockIdExt id; + bool is_proof = false; + f.ref().visit(td::overloaded( + [&](const fileref::Block &p) { + id = p.block_id; + is_proof = false; + }, + [&](const fileref::Proof &p) { + id = p.block_id; + is_proof = true; + }, + [&](const fileref::ProofLink &p) { + id = p.block_id; + is_proof = true; + }, + [&](const auto &) {})); + if (is_proof && id == block_id) { + result = std::move(data); + return false; + } + if (shard_intersects(id.shard_full(), block_id.shard_full()) && id.seqno() < block_id.seqno()) { + stop = true; + } + return true; + }); + if (!result.empty()) { + promise.set_result(std::move(result)); + return; + } + } + if (block_id.is_masterchain() || stop) { + promise.set_error(td::Status::Error("proof not found")); + return; + } + } +} + void ValidatorManagerImpl::start_up() { db_ = create_db_actor(actor_id(this), db_root_, opts_); actor_stats_ = td::actor::create_actor("actor_stats"); @@ -1714,6 +1780,7 @@ void ValidatorManagerImpl::start_up() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::started, R.move_as_ok()); }); + size_t to_import_files = 0; auto to_import_dir = db_root_ + "/import"; auto S = td::WalkPath::run(to_import_dir, [&](td::CSlice cfname, td::WalkPath::Type t) -> void { auto fname = td::Slice(cfname); @@ -1733,26 +1800,31 @@ void ValidatorManagerImpl::start_up() { } fname = fname.substr(8); - while (fname.size() > 1 && fname[0] == '0') { - fname.remove_prefix(1); - } auto i = fname.find('.'); if (i == td::Slice::npos) { return; } fname = fname.substr(0, i); + while (fname.size() > 1 && fname[0] == '0') { + fname.remove_prefix(1); + } auto v = td::to_integer_safe(fname); if (v.is_error()) { return; } auto seqno = v.move_as_ok(); - LOG(INFO) << "found archive slice '" << cfname << "' for seqno " << seqno; + LOG(DEBUG) << "found archive slice '" << cfname << "' for seqno " << seqno; to_import_[seqno].push_back(cfname.str()); + ++to_import_files; } }); + to_import_all_ = to_import_; if (S.is_error()) { LOG(INFO) << "failed to load blocks from import dir: " << S; } + if (to_import_files > 0) { + LOG(INFO) << "found " << to_import_files << " files to import"; + } validator_manager_init(opts_, actor_id(this), db_.get(), std::move(P)); @@ -1935,9 +2007,15 @@ void ValidatorManagerImpl::download_next_archive() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::checked_archive_slice, R.ok().first, R.ok().second); } }); - td::actor::create_actor("archiveimport", db_root_, last_masterchain_state_, seqno, opts_, - actor_id(this), std::move(to_import_files), std::move(P)) - .release(); + if (to_import_files.empty()) { + td::actor::create_actor("archiveimport", db_root_, last_masterchain_state_, seqno, opts_, + actor_id(this), std::move(to_import_files), std::move(P)) + .release(); + } else { + td::actor::create_actor("archiveimport", db_root_, last_masterchain_state_, seqno, opts_, + actor_id(this), std::move(to_import_files), std::move(P)) + .release(); + } } void ValidatorManagerImpl::checked_archive_slice(BlockSeqno new_last_mc_seqno, BlockSeqno new_shard_client_seqno) { diff --git a/validator/manager.hpp b/validator/manager.hpp index e90d5b347..59ee1ff63 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -412,6 +412,9 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void store_block_state_part(BlockId effective_block, td::Ref cell, td::Promise> promise) override; + void set_block_state_from_data(BlockHandle handle, td::Ref block, + td::Promise> promise) override; + void set_block_state_from_data_preliminary(std::vector> blocks, td::Promise promise) override; void get_cell_db_reader(td::Promise> promise) override; void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, PersistentStateType type, td::BufferSlice state, td::Promise promise) override; @@ -523,6 +526,9 @@ class ValidatorManagerImpl : public ValidatorManager { void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, td::Timestamp timeout, td::Promise promise) override; + void get_block_proof_link_from_import(BlockIdExt block_id, BlockIdExt masterchain_block_id, + td::Promise promise) override; + void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; @@ -685,6 +691,7 @@ class ValidatorManagerImpl : public ValidatorManager { td::actor::ActorOwn serializer_; std::map> to_import_; + std::map> to_import_all_; private: std::unique_ptr callback_; diff --git a/validator/net/download-state.cpp b/validator/net/download-state.cpp index 16d619b8e..da6363f0d 100644 --- a/validator/net/download-state.cpp +++ b/validator/net/download-state.cpp @@ -76,16 +76,20 @@ void DownloadState::start_up() { status_ = ProcessStatus(validator_manager_, "process.download_state_net"); alarm_timestamp() = timeout_; - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state, block_id_, - masterchain_block_id_, type_, - [SelfId = actor_id(this), block_id = block_id_](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &DownloadState::get_block_handle); - } else { - LOG(WARNING) << "got block state from disk: " << block_id.to_str(); - td::actor::send_closure(SelfId, &DownloadState::got_block_state, R.move_as_ok()); - } - }); + td::Promise P = [SelfId = actor_id(this), block_id = block_id_](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &DownloadState::get_block_handle); + } else { + LOG(WARNING) << "got block state from disk: " << block_id.to_str(); + td::actor::send_closure(SelfId, &DownloadState::got_block_state, R.move_as_ok()); + } + }; + if (block_id_.seqno() == 0) { + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_zero_state, block_id_, std::move(P)); + } else { + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state, block_id_, + masterchain_block_id_, type_, std::move(P)); + } } void DownloadState::get_block_handle() { diff --git a/validator/validator-options.cpp b/validator/validator-options.cpp index cb26fe44d..a87562377 100644 --- a/validator/validator-options.cpp +++ b/validator/validator-options.cpp @@ -24,14 +24,13 @@ namespace ton { namespace validator { -td::Ref ValidatorManagerOptions::create( - BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard, bool allow_blockchain_init, - double sync_blocks_before, double block_ttl, double state_ttl, double max_mempool_num, - double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) { - return td::make_ref(zero_block_id, init_block_id, std::move(check_shard), - allow_blockchain_init, sync_blocks_before, block_ttl, state_ttl, - max_mempool_num, +td::Ref ValidatorManagerOptions::create(BlockIdExt zero_block_id, BlockIdExt init_block_id, + bool allow_blockchain_init, double sync_blocks_before, + double block_ttl, double state_ttl, + double max_mempool_num, double archive_ttl, + double key_proof_ttl, bool initial_sync_disabled) { + return td::make_ref(zero_block_id, init_block_id, allow_blockchain_init, + sync_blocks_before, block_ttl, state_ttl, max_mempool_num, archive_ttl, key_proof_ttl, initial_sync_disabled); } diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 55552635c..1fba6091d 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -34,7 +34,8 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { } bool need_monitor(ShardIdFull shard, const td::Ref& state) const override { td::uint32 min_split = state->monitor_min_split_depth(shard.workchain); - return check_shard_((td::uint32)shard.pfx_len() <= min_split ? shard : shard_prefix(shard, min_split)); + return check_shard_((td::uint32)shard.pfx_len() <= min_split ? shard : shard_prefix(shard, min_split), + state->get_seqno()); } bool allow_blockchain_init() const override { return allow_blockchain_init_; @@ -163,6 +164,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { double get_catchain_broadcast_speed_multiplier() const override { return catchain_broadcast_speed_multipliers_; } + bool get_permanent_celldb() const override { + return permanent_celldb_; + } void set_zero_block_id(BlockIdExt block_id) override { zero_block_id_ = block_id; @@ -170,7 +174,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_init_block_id(BlockIdExt block_id) override { init_block_id_ = block_id; } - void set_shard_check_function(std::function check_shard) override { + void set_shard_check_function(std::function check_shard) override { check_shard_ = std::move(check_shard); } void set_allow_blockchain_init(bool value) override { @@ -267,18 +271,19 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_catchain_broadcast_speed_multiplier(double value) override { catchain_broadcast_speed_multipliers_ = value; } + void set_permanent_celldb(bool value) override { + permanent_celldb_ = value; + } ValidatorManagerOptionsImpl *make_copy() const override { return new ValidatorManagerOptionsImpl(*this); } - ValidatorManagerOptionsImpl(BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard, bool allow_blockchain_init, + ValidatorManagerOptionsImpl(BlockIdExt zero_block_id, BlockIdExt init_block_id, bool allow_blockchain_init, double sync_blocks_before, double block_ttl, double state_ttl, double max_mempool_num, double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) : zero_block_id_(zero_block_id) , init_block_id_(init_block_id) - , check_shard_(std::move(check_shard)) , allow_blockchain_init_(allow_blockchain_init) , sync_blocks_before_(sync_blocks_before) , block_ttl_(block_ttl) @@ -292,7 +297,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { private: BlockIdExt zero_block_id_; BlockIdExt init_block_id_; - std::function check_shard_; + std::function check_shard_ = [](ShardIdFull, BlockSeqno) { return true; }; bool allow_blockchain_init_; double sync_blocks_before_; double block_ttl_; @@ -323,6 +328,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { td::Ref collator_options_{true}; bool fast_state_serializer_enabled_ = false; double catchain_broadcast_speed_multipliers_; + bool permanent_celldb_ = false; }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 943c6e9c5..afeb86cef 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -120,10 +120,11 @@ struct ValidatorManagerOptions : public td::CntObject { virtual td::Ref get_collator_options() const = 0; virtual bool get_fast_state_serializer_enabled() const = 0; virtual double get_catchain_broadcast_speed_multiplier() const = 0; + virtual bool get_permanent_celldb() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; - virtual void set_shard_check_function(std::function check_shard) = 0; + virtual void set_shard_check_function(std::function check_shard) = 0; virtual void set_allow_blockchain_init(bool value) = 0; virtual void set_sync_blocks_before(double value) = 0; virtual void set_block_ttl(double value) = 0; @@ -155,11 +156,10 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_collator_options(td::Ref value) = 0; virtual void set_fast_state_serializer_enabled(bool value) = 0; virtual void set_catchain_broadcast_speed_multiplier(double value) = 0; + virtual void set_permanent_celldb(bool value) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, - - std::function check_shard = [](ShardIdFull) { return true; }, bool allow_blockchain_init = false, double sync_blocks_before = 3600, double block_ttl = 86400, double state_ttl = 86400, double archive_ttl = 86400 * 7, double key_proof_ttl = 86400 * 3650, double max_mempool_num = 999999, bool initial_sync_disabled = false); From 5a0f54df45f913557f03bc2bc52bad4667ac241a Mon Sep 17 00:00:00 2001 From: neodix42 Date: Fri, 20 Jun 2025 20:45:11 +0300 Subject: [PATCH 312/388] Improve linux portable binaries (#1693) * not always gh mac runners have nproc utility * deprecate Ubuntu 20.04 * fix linking issues of portable tonlibjson.dylib * bind libgsl and libblas into portable binaries --- .github/workflows/build-ton-linux-x86-64-appimage.yml | 3 +-- assembly/appimage/create-appimages.sh | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-ton-linux-x86-64-appimage.yml b/.github/workflows/build-ton-linux-x86-64-appimage.yml index 43d989243..7ad873d22 100644 --- a/.github/workflows/build-ton-linux-x86-64-appimage.yml +++ b/.github/workflows/build-ton-linux-x86-64-appimage.yml @@ -21,8 +21,7 @@ jobs: - name: Install system libraries run: | sudo apt update - sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache - sudo apt remove libgsl-dev + sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev ccache libgsl-dev libblas-dev mkdir ~/.ccache 3pp - name: Install clang-16 diff --git a/assembly/appimage/create-appimages.sh b/assembly/appimage/create-appimages.sh index 2a8cd0ec6..7bd68b6fb 100644 --- a/assembly/appimage/create-appimages.sh +++ b/assembly/appimage/create-appimages.sh @@ -35,6 +35,8 @@ for file in ../artifacts/*; do /lib/$ARCH-linux-gnu/libmicrohttpd.so.12 \ /lib/$ARCH-linux-gnu/libreadline.so.8 \ /lib/$ARCH-linux-gnu/libstdc++.so.6 \ + /lib/$ARCH-linux-gnu/libgsl.so.27 \ + /lib/$ARCH-linux-gnu/libblas.so.3 \ $appName.AppDir/usr/lib/ chmod +x ./$appName.AppDir/usr/bin/$appName From 34cbbbb90fb9fab4348f0f8aa13a7e8364ab5136 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Fri, 20 Jun 2025 20:45:40 +0300 Subject: [PATCH 313/388] Add dht utils for troubleshooting complex deployments (#1685) * not always gh mac runners have nproc utility * deprecate Ubuntu 20.04 * fix linking issues of portable tonlibjson.dylib * add dht-ping-servers and dht-resolve utilities to artifacts * test gh builds * minor win fix * minor win fix --- Dockerfile | 4 +++- assembly/native/build-macos-portable.sh | 6 ++++-- assembly/native/build-macos-shared.sh | 19 +++++++++++++++++-- assembly/native/build-ubuntu-appimages.sh | 5 +++-- assembly/native/build-ubuntu-portable.sh | 5 +++-- assembly/native/build-ubuntu-shared.sh | 5 +++-- assembly/native/build-windows-2019.bat | 6 ++++-- assembly/native/build-windows.bat | 6 ++++-- 8 files changed, 41 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index f1b836bf9..7a53500db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,7 @@ RUN mkdir build && \ cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DTON_USE_JEMALLOC=ON .. && \ ninja storage-daemon storage-daemon-cli tonlibjson fift func validator-engine validator-engine-console \ generate-random-id dht-server lite-client tolk rldp-http-proxy dht-server proxy-liteserver create-state \ - blockchain-explorer emulator tonlibjson http-proxy adnl-proxy + blockchain-explorer emulator tonlibjson http-proxy adnl-proxy dht-ping-servers dht-resolve FROM ubuntu:22.04 ARG DEBIAN_FRONTEND=noninteractive @@ -50,6 +50,8 @@ COPY --from=builder /ton/build/blockchain-explorer/blockchain-explorer /usr/loca COPY --from=builder /ton/build/crypto/create-state /usr/local/bin/ COPY --from=builder /ton/build/utils/proxy-liteserver /usr/local/bin/ COPY --from=builder /ton/build/dht-server/dht-server /usr/local/bin/ +COPY --from=builder /ton/build/dht/dht-ping-servers /usr/local/bin/ +COPY --from=builder /ton/build/dht/dht-resolve /usr/local/bin/ COPY --from=builder /ton/build/rldp-http-proxy/rldp-http-proxy /usr/local/bin/ COPY --from=builder /ton/build/http/http-proxy /usr/local/bin/ COPY --from=builder /ton/build/adnl/adnl-proxy /usr/local/bin/ diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index e112ab499..351c4e7fb 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -150,7 +150,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli blockchain-explorer \ tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ - lite-client validator-engine-console generate-random-id json2tlo dht-server \ + lite-client validator-engine-console generate-random-id json2tlo dht-server dht-ping-servers dht-resolve \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \ test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont \ test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ @@ -159,7 +159,7 @@ if [ "$with_tests" = true ]; then else ninja storage-daemon storage-daemon-cli blockchain-explorer \ tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ - lite-client validator-engine-console generate-random-id json2tlo dht-server \ + lite-client validator-engine-console generate-random-id json2tlo dht-server dht-ping-servers dht-resolve \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi @@ -184,6 +184,8 @@ if [ "$with_artifacts" = true ]; then cp build/http/http-proxy artifacts/ cp build/rldp-http-proxy/rldp-http-proxy artifacts/ cp build/dht-server/dht-server artifacts/ + cp build/dht/dht-ping-servers artifacts/ + cp build/dht/dht-resolve artifacts/ cp build/lite-client/lite-client artifacts/ cp build/validator-engine/validator-engine artifacts/ cp build/utils/generate-random-id artifacts/ diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh index b21f21f89..f5aa15b4a 100644 --- a/assembly/native/build-macos-shared.sh +++ b/assembly/native/build-macos-shared.sh @@ -61,6 +61,19 @@ else echo "Using compiled lz4" fi +if [ ! -d "zlib" ]; then + git clone https://github.com/madler/zlib.git + cd zlib + zlibPath=`pwd` + ./configure --static + make -j4 + test $? -eq 0 || { echo "Can't compile zlib"; exit 1; } + cd .. +else + zlibPath=$(pwd)/zlib + echo "Using compiled zlib" +fi + brew unlink openssl@1.1 brew install openssl@3 brew unlink openssl@3 && brew link --overwrite openssl@3 @@ -75,7 +88,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli blockchain-explorer \ tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ - lite-client validator-engine-console generate-random-id json2tlo dht-server \ + lite-client validator-engine-console generate-random-id json2tlo dht-server dht-ping-servers dht-resolve \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \ test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont \ test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ @@ -84,7 +97,7 @@ if [ "$with_tests" = true ]; then else ninja storage-daemon storage-daemon-cli blockchain-explorer \ tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ - lite-client validator-engine-console generate-random-id json2tlo dht-server \ + lite-client validator-engine-console generate-random-id json2tlo dht-server dht-ping-servers dht-resolve \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi @@ -109,6 +122,8 @@ if [ "$with_artifacts" = true ]; then cp build/http/http-proxy artifacts/ cp build/rldp-http-proxy/rldp-http-proxy artifacts/ cp build/dht-server/dht-server artifacts/ + cp build/dht/dht-ping-servers artifacts/ + cp build/dht/dht-resolve artifacts/ cp build/lite-client/lite-client artifacts/ cp build/validator-engine/validator-engine artifacts/ cp build/utils/generate-random-id artifacts/ diff --git a/assembly/native/build-ubuntu-appimages.sh b/assembly/native/build-ubuntu-appimages.sh index 015b33cb7..2cdc6c652 100644 --- a/assembly/native/build-ubuntu-appimages.sh +++ b/assembly/native/build-ubuntu-appimages.sh @@ -61,7 +61,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client validator-engine-console blockchain-explorer \ - generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ + generate-random-id json2tlo dht-server http-proxy rldp-http-proxy dht-ping-servers dht-resolve \ adnl-proxy create-state emulator test-ed25519 test-bigint \ test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ @@ -71,7 +71,7 @@ else ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ - adnl-proxy create-state emulator proxy-liteserver + adnl-proxy create-state emulator proxy-liteserver dht-ping-servers dht-resolve test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi @@ -104,6 +104,7 @@ if [ "$with_artifacts" = true ]; then build/tonlib/libtonlibjson.so build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy \ build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine \ build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.so \ + build/dht/dht-ping-servers build/dht/dht-resolve \ artifacts test $? -eq 0 || { echo "Can't copy final binaries"; exit 1; } cp -R crypto/smartcont artifacts diff --git a/assembly/native/build-ubuntu-portable.sh b/assembly/native/build-ubuntu-portable.sh index 3a65ddd2b..a4f0bb233 100644 --- a/assembly/native/build-ubuntu-portable.sh +++ b/assembly/native/build-ubuntu-portable.sh @@ -135,7 +135,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client validator-engine-console blockchain-explorer \ - generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ + generate-random-id json2tlo dht-server http-proxy rldp-http-proxy dht-ping-servers dht-resolve \ adnl-proxy create-state emulator test-ed25519 test-bigint \ test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ @@ -145,7 +145,7 @@ else ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ - adnl-proxy create-state emulator proxy-liteserver + adnl-proxy create-state emulator proxy-liteserver dht-ping-servers dht-resolve test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi @@ -167,6 +167,7 @@ if [ "$with_artifacts" = true ]; then build/tonlib/libtonlibjson.so build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy \ build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine \ build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.so \ + build/dht/dht-ping-servers build/dht/dht-resolve \ artifacts test $? -eq 0 || { echo "Can't copy final binaries"; exit 1; } cp -R crypto/smartcont artifacts diff --git a/assembly/native/build-ubuntu-shared.sh b/assembly/native/build-ubuntu-shared.sh index 0bdf4072b..6ddbaf885 100644 --- a/assembly/native/build-ubuntu-shared.sh +++ b/assembly/native/build-ubuntu-shared.sh @@ -64,7 +64,7 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client validator-engine-console blockchain-explorer \ - generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ + generate-random-id json2tlo dht-server http-proxy rldp-http-proxy dht-ping-servers dht-resolve \ adnl-proxy create-state emulator test-ed25519 test-bigint \ test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ @@ -74,7 +74,7 @@ else ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ - adnl-proxy create-state emulator proxy-liteserver + adnl-proxy create-state emulator proxy-liteserver dht-ping-servers dht-resolve test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi @@ -99,6 +99,7 @@ if [ "$with_artifacts" = true ]; then build/tonlib/libtonlibjson.so build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy \ build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine \ build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.so \ + build/dht/dht-ping-servers build/dht/dht-resolve \ artifacts test $? -eq 0 || { echo "Can't copy final binaries"; exit 1; } cp -R crypto/smartcont artifacts diff --git a/assembly/native/build-windows-2019.bat b/assembly/native/build-windows-2019.bat index 2c635d3a6..e671d277c 100644 --- a/assembly/native/build-windows-2019.bat +++ b/assembly/native/build-windows-2019.bat @@ -149,14 +149,14 @@ tonlib-cli validator-engine lite-client validator-engine-console generate-random json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net ^ test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ -test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver +test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver dht-ping-servers dht-resolve IF %errorlevel% NEQ 0 ( echo Can't compile TON exit /b %errorlevel% ) ) else ( ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ -tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ +tonlib-cli validator-engine lite-client validator-engine-console generate-random-id dht-ping-servers dht-resolve ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator proxy-liteserver IF %errorlevel% NEQ 0 ( echo Can't compile TON @@ -192,6 +192,8 @@ for %%I in (build\storage\storage-daemon\storage-daemon.exe ^ build\http\http-proxy.exe ^ build\rldp-http-proxy\rldp-http-proxy.exe ^ build\dht-server\dht-server.exe ^ + build\dht\dht-ping-servers.exe ^ + build\dht\dht-resolve.exe ^ build\lite-client\lite-client.exe ^ build\validator-engine\validator-engine.exe ^ build\utils\generate-random-id.exe ^ diff --git a/assembly/native/build-windows.bat b/assembly/native/build-windows.bat index 6ff7166af..f0a65670b 100644 --- a/assembly/native/build-windows.bat +++ b/assembly/native/build-windows.bat @@ -149,14 +149,14 @@ tonlib-cli validator-engine lite-client validator-engine-console generate-random json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ test-ed25519 test-bigint test-vm test-fift test-cells test-smartcont test-net ^ test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ -test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver +test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver dht-ping-servers dht-resolve IF %errorlevel% NEQ 0 ( echo Can't compile TON exit /b %errorlevel% ) ) else ( ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ -tonlib-cli validator-engine lite-client validator-engine-console generate-random-id ^ +tonlib-cli validator-engine lite-client validator-engine-console generate-random-id dht-ping-servers dht-resolve ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator proxy-liteserver IF %errorlevel% NEQ 0 ( echo Can't compile TON @@ -192,6 +192,8 @@ for %%I in (build\storage\storage-daemon\storage-daemon.exe ^ build\http\http-proxy.exe ^ build\rldp-http-proxy\rldp-http-proxy.exe ^ build\dht-server\dht-server.exe ^ + build\dht\dht-ping-servers.exe ^ + build\dht\dht-resolve.exe ^ build\lite-client\lite-client.exe ^ build\validator-engine\validator-engine.exe ^ build\utils\generate-random-id.exe ^ From 95acdb1a4a625a62d8d599fdd0388f49083e4724 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Fri, 20 Jun 2025 22:09:28 +0300 Subject: [PATCH 314/388] Fix structured binding in StateSerializer lambda --- validator/state-serializer.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index e65bad4c0..b024dc100 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -578,7 +578,9 @@ void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref cell_db_reader, std::shared_ptr> parts, size_t idx) { - auto [type, cell] = parts->at(idx); + auto part = parts->at(idx); + auto type = part.type; + auto cell = part.cell; auto write_data = [=, this, cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable { From dfe997ee3254782cc2f06a0d23f1fe3c03c18d98 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Wed, 25 Jun 2025 21:54:55 +0300 Subject: [PATCH 315/388] Storage stat cache (#1724) Co-authored-by: SpyCheese --- crypto/block/account-storage-stat.h | 4 ++ crypto/block/transaction.cpp | 38 ++++++++++++++ crypto/block/transaction.h | 1 + tdutils/td/utils/LRUCache.h | 24 +++++---- validator/CMakeLists.txt | 2 + validator/impl/collator-impl.h | 4 ++ validator/impl/collator.cpp | 46 ++++++++++++++++- validator/impl/validate-query.cpp | 49 ++++++++++++++++++ validator/impl/validate-query.hpp | 4 ++ validator/interfaces/validator-manager.h | 7 +++ validator/manager.cpp | 1 + validator/manager.hpp | 10 ++++ validator/storage-stat-cache.cpp | 40 +++++++++++++++ validator/storage-stat-cache.hpp | 63 ++++++++++++++++++++++++ 14 files changed, 282 insertions(+), 11 deletions(-) create mode 100644 validator/storage-stat-cache.cpp create mode 100644 validator/storage-stat-cache.hpp diff --git a/crypto/block/account-storage-stat.h b/crypto/block/account-storage-stat.h index e939a0ff5..4ada076b8 100644 --- a/crypto/block/account-storage-stat.h +++ b/crypto/block/account-storage-stat.h @@ -57,6 +57,10 @@ class AccountStorageStat { return root.is_null() ? td::Bits256::zero() : td::Bits256{root->get_hash().bits()}; } + bool is_dict_ready() const { + return dict_up_to_date_; + } + void apply_child_stat(AccountStorageStat &&child); static constexpr int errorcode_limits_exceeded = 999; diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 8a72f8e48..7cc0813cd 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -561,6 +561,44 @@ bool Account::init_new(ton::UnixTime now) { return true; } +/** + * Initializes account_storage_stat of the account using the existing dict_root. + * This is not strictly necessary, as the storage stat is recomputed in Transaction. + * However, it can be used to optimize cell usage. + * This requires storage_dict_hash to be set, as it guarantees that the stored storage_used was computed recently + * (in older versions it included extra currency balance, in newer versions it does not). + * + * @param dict_root Root of the storage dictionary. + * + * @returns Status of the operation. + */ +td::Status Account::init_account_storage_stat(Ref dict_root) { + if (storage.is_null()) { + if (dict_root.not_null()) { + return td::Status::Error("storage is null, but dict_root is not null"); + } + account_storage_stat = {}; + return td::Status::OK(); + } + if (!storage_dict_hash) { + return td::Status::Error("cannot init storage dict: storage_dict_hash is not set"); + } + // Root of AccountStorage is not counted in AccountStorageStat + if (storage_used.cells < 1 || storage_used.bits < storage->size()) { + return td::Status::Error(PSTRING() << "storage_used is too small: cells=" << storage_used.cells + << " bits=" << storage_used.bits << " storage_root_bits=" << storage->size()); + } + AccountStorageStat new_stat(std::move(dict_root), storage->prefetch_all_refs(), storage_used.cells - 1, + storage_used.bits - storage->size()); + TRY_RESULT(root_hash, new_stat.get_dict_hash()); + if (storage_dict_hash.value() != root_hash) { + return td::Status::Error(PSTRING() << "invalid storage dict hash: computed " << root_hash.to_hex() << ", found " + << storage_dict_hash.value().to_hex()); + } + account_storage_stat = std::move(new_stat); + return td::Status::OK(); +} + /** * Resets the fixed prefix length of the account. * diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index 0b3a4358d..2f26ea3fb 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -298,6 +298,7 @@ struct Account { bool set_address(ton::WorkchainId wc, td::ConstBitPtr new_addr); bool unpack(Ref account, ton::UnixTime now, bool special); bool init_new(ton::UnixTime now); + td::Status init_account_storage_stat(Ref dict_root); bool deactivate(); bool recompute_tmp_addr(Ref& tmp_addr, int fixed_prefix_length, td::ConstBitPtr orig_addr_rewrite) const; td::RefInt256 compute_storage_fees(ton::UnixTime now, const std::vector& pricing) const; diff --git a/tdutils/td/utils/LRUCache.h b/tdutils/td/utils/LRUCache.h index 5c30f6693..d8f51525b 100644 --- a/tdutils/td/utils/LRUCache.h +++ b/tdutils/td/utils/LRUCache.h @@ -26,8 +26,7 @@ namespace td { template class LRUCache { public: - explicit LRUCache(size_t max_size) : max_size_(max_size) { - CHECK(max_size_ > 0); + explicit LRUCache(uint64 max_size) : max_size_(max_size) { } LRUCache(const LRUCache&) = delete; LRUCache& operator=(const LRUCache&) = delete; @@ -49,13 +48,14 @@ class LRUCache { return cache_.contains(key); } - bool put(const K& key, V value, bool update = true) { + bool put(const K& key, V value, bool update = true, uint64 weight = 1) { bool added = false; auto it = cache_.find(key); if (it == cache_.end()) { update = true; - it = cache_.insert(std::make_unique(key, std::move(value))).first; + it = cache_.insert(std::make_unique(key, std::move(value), weight)).first; added = true; + total_weight_ += weight; } else { (*it)->value = std::move(value); if (update) { @@ -69,11 +69,12 @@ class LRUCache { return added; } - V& get(const K& key, bool update = true) { + V& get(const K& key, bool update = true, uint64 weight = 1) { auto it = cache_.find(key); if (it == cache_.end()) { update = true; - it = cache_.insert(std::make_unique(key)).first; + it = cache_.insert(std::make_unique(key, weight)).first; + total_weight_ += weight; } else if (update) { (*it)->remove(); } @@ -87,12 +88,13 @@ class LRUCache { private: struct Entry : ListNode { - explicit Entry(K key) : key(std::move(key)) { + Entry(K key, uint64 weight) : key(std::move(key)), weight(weight) { } - Entry(K key, V value) : key(std::move(key)), value(std::move(value)) { + Entry(K key, V value, uint64 weight) : key(std::move(key)), value(std::move(value)), weight(weight) { } K key; V value; + uint64 weight; }; struct Cmp { using is_transparent = void; @@ -108,13 +110,15 @@ class LRUCache { }; std::set, Cmp> cache_; ListNode lru_; - size_t max_size_; + uint64 max_size_; + uint64 total_weight_ = 0; void cleanup() { - while (cache_.size() > max_size_) { + while (total_weight_ > max_size_ && cache_.size() > 1) { auto to_remove = (Entry*)lru_.get(); CHECK(to_remove); to_remove->remove(); + total_weight_ -= to_remove->weight; cache_.erase(cache_.find(to_remove->key)); } } diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index 9fb0d190f..78909f34f 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -59,6 +59,7 @@ set(VALIDATOR_HEADERS import-db-slice-local.hpp queue-size-counter.hpp validator-telemetry.hpp + storage-stat-cache.hpp manager-disk.h manager-disk.hpp @@ -87,6 +88,7 @@ set(VALIDATOR_SOURCE validator-options.cpp queue-size-counter.cpp validator-telemetry.cpp + storage-stat-cache.cpp downloaders/wait-block-data.cpp downloaders/wait-block-state.cpp diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 340e3a401..8b113c74e 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -227,6 +227,9 @@ class Collator final : public td::actor::Actor { bool deferring_messages_enabled_ = false; bool store_out_msg_queue_size_ = false; + std::function(const td::Bits256&)> storage_stat_cache_; + std::vector, td::uint32>> storage_stat_cache_update_; + td::PerfWarningTimer perf_timer_; // block::Account* lookup_account(td::ConstBitPtr addr) const; @@ -247,6 +250,7 @@ class Collator final : public td::actor::Actor { void after_get_shard_state(int idx, td::Result> res); void after_get_block_data(int idx, td::Result> res); void after_get_shard_blocks(td::Result>> res); + void after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res); bool preprocess_prev_mc_state(); bool register_mc_state(Ref other_mc_state); bool request_aux_mc_state(BlockSeqno seqno, Ref& state); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index cc9ee709c..614e0ecdb 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -277,7 +277,16 @@ void Collator::start_up() { td::actor::send_closure_later(std::move(self), &Collator::after_get_shard_blocks, std::move(res)); }); } - // 6. set timeout + // 6. get storage stat cache + ++pending; + LOG(DEBUG) << "sending get_storage_stat_cache() query to Manager"; + td::actor::send_closure_later( + manager, &ValidatorManager::get_storage_stat_cache, + [self = get_self()](td::Result(const td::Bits256&)>> res) { + LOG(DEBUG) << "got answer to get_storage_stat_cache() query"; + td::actor::send_closure_later(std::move(self), &Collator::after_get_storage_stat_cache, std::move(res)); + }); + // 7. set timeout alarm_timestamp() = timeout; CHECK(pending); } @@ -686,6 +695,22 @@ void Collator::after_get_shard_blocks(td::Result(const td::Bits256&)>> res) { + --pending; + if (res.is_error()) { + LOG(INFO) << "after_get_storage_stat_cache : " << res.error(); + } else { + LOG(DEBUG) << "after_get_storage_stat_cache : OK"; + storage_stat_cache_ = res.move_as_ok(); + } + check_pending(); +} + /** * Unpacks the last masterchain state and initializes the Collator object with the extracted configuration. * @@ -2448,6 +2473,20 @@ std::unique_ptr Collator::make_account_from(td::ConstBitPtr addr return nullptr; } ptr->block_lt = start_lt; + if (storage_stat_cache_ && ptr->storage_dict_hash) { + auto dict_root = storage_stat_cache_(ptr->storage_dict_hash.value()); + if (dict_root.not_null()) { + auto S = ptr->init_account_storage_stat(dict_root); + if (S.is_error()) { + fatal_error(S.move_as_error_prefix(PSTRING() << "failed to init storage stat from cache for account " + << addr.to_hex(256) << ": ")); + return nullptr; + } + LOG(DEBUG) << "Inited storage stat from cache for account " << addr.to_hex(256) << " (" << ptr->storage_used.cells + << " cells)"; + storage_stat_cache_update_.emplace_back(dict_root, ptr->storage_used.cells); + } + } return ptr; } @@ -2589,6 +2628,10 @@ bool Collator::combine_account_transactions() { } } } + if (acc.storage_dict_hash && acc.account_storage_stat && acc.account_storage_stat.value().is_dict_ready()) { + storage_stat_cache_update_.emplace_back(acc.account_storage_stat.value().get_dict_root().move_as_ok(), + acc.storage_used.cells); + } } else { if (acc.total_state->get_hash() != acc.orig_total_state->get_hash()) { return fatal_error(std::string{"total state of account "} + z.first.to_hex() + @@ -5685,6 +5728,7 @@ bool Collator::create_block_candidate() { stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt); td::actor::send_closure(manager, &ValidatorManager::record_collate_query_stats, block_candidate->id, work_time, cpu_work_time, std::move(stats_)); + td::actor::send_closure(manager, &ValidatorManager::update_storage_stat_cache, std::move(storage_stat_cache_update_)); return true; } diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 8c974b431..a191b0200 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -381,6 +381,15 @@ void ValidateQuery::start_up() { return; } } + // 5. get storage stat cache + ++pending; + LOG(DEBUG) << "sending get_storage_stat_cache() query to Manager"; + td::actor::send_closure_later( + manager, &ValidatorManager::get_storage_stat_cache, + [self = get_self()](td::Result(const td::Bits256&)>> res) { + LOG(DEBUG) << "got answer to get_storage_stat_cache() query"; + td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_storage_stat_cache, std::move(res)); + }); // ... CHECK(pending); } @@ -749,6 +758,26 @@ void ValidateQuery::got_mc_handle(td::Result res) { }); } +/** + * Callback function called after retrieving storage stat cache. + * + * @param res The retrieved storage stat cache. + */ +void ValidateQuery::after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res) { + --pending; + if (res.is_error()) { + LOG(INFO) << "after_get_storage_stat_cache : " << res.error(); + } else { + LOG(DEBUG) << "after_get_storage_stat_cache : OK"; + storage_stat_cache_ = res.move_as_ok(); + } + if (!pending) { + if (!try_validate()) { + fatal_error("cannot validate new block"); + } + } +} + /** * Callback function called after retrieving the shard state for a previous block. * @@ -5133,6 +5162,20 @@ std::unique_ptr ValidateQuery::make_account_from(td::ConstBitPtr return nullptr; } ptr->block_lt = start_lt_; + if (storage_stat_cache_ && ptr->storage_dict_hash) { + auto dict_root = storage_stat_cache_(ptr->storage_dict_hash.value()); + if (dict_root.not_null()) { + auto S = ptr->init_account_storage_stat(dict_root); + if (S.is_error()) { + fatal_error(S.move_as_error_prefix(PSTRING() << "failed to init storage stat from cache for account " + << addr.to_hex(256) << ": ")); + return nullptr; + } + LOG(DEBUG) << "Inited storage stat from cache for account " << addr.to_hex(256) << " (" << ptr->storage_used.cells + << " cells)"; + storage_stat_cache_update_.emplace_back(dict_root, ptr->storage_used.cells); + } + } return ptr; } @@ -5746,6 +5789,11 @@ bool ValidateQuery::check_account_transactions(const StdSmcAddress& acc_addr, Re })) { return reject_query("at least one Transaction of account "s + acc_addr.to_hex() + " is invalid"); } + if (account.storage_dict_hash && account.account_storage_stat && + account.account_storage_stat.value().is_dict_ready()) { + storage_stat_cache_update_.emplace_back(account.account_storage_stat.value().get_dict_root().move_as_ok(), + account.storage_used.cells); + } if (is_masterchain() && account.libraries_changed()) { return scan_account_libraries(account.orig_library, account.library, acc_addr); } else { @@ -6923,6 +6971,7 @@ bool ValidateQuery::save_candidate() { td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, id_, block_candidate.clone(), validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), std::move(P)); + td::actor::send_closure(manager, &ValidatorManager::update_storage_stat_cache, std::move(storage_stat_cache_update_)); return true; } diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 60f0cc8a4..14aee1a8e 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -236,6 +236,9 @@ class ValidateQuery : public td::actor::Actor { std::set account_expected_defer_all_messages_; td::uint64 old_out_msg_queue_size_ = 0, new_out_msg_queue_size_ = 0; + std::function(const td::Bits256&)> storage_stat_cache_; + std::vector, td::uint32>> storage_stat_cache_update_; + bool msg_metadata_enabled_ = false; bool deferring_messages_enabled_ = false; bool store_out_msg_queue_size_ = false; @@ -289,6 +292,7 @@ class ValidateQuery : public td::actor::Actor { void after_get_latest_mc_state(td::Result, BlockIdExt>> res); void after_get_mc_state(td::Result> res); void got_mc_handle(td::Result res); + void after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res); void after_get_shard_state(int idx, td::Result> res); bool process_mc_state(Ref mc_state); bool try_unpack_mc_state(); diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index abab4e6c9..44c74f07a 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -221,6 +221,13 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void add_persistent_state_description(td::Ref desc) = 0; + virtual void get_storage_stat_cache(td::Promise(const td::Bits256&)>> promise) { + promise.set_error(td::Status::Error("not implemented")); + } + virtual void update_storage_stat_cache(std::vector, td::uint32>> data) { + // not implemented + } + static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) { return ts / (1 << 17) != prev_ts / (1 << 17); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 7a2d63b7d..5acb2928a 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1764,6 +1764,7 @@ void ValidatorManagerImpl::start_up() { actor_stats_ = td::actor::create_actor("actor_stats"); lite_server_cache_ = create_liteserver_cache_actor(actor_id(this), db_root_); token_manager_ = td::actor::create_actor("tokenmanager"); + storage_stat_cache_ = td::actor::create_actor("storagestatcache"); td::mkdir(db_root_ + "/tmp/").ensure(); td::mkdir(db_root_ + "/catchains/").ensure(); diff --git a/validator/manager.hpp b/validator/manager.hpp index 59ee1ff63..4ea0d24f4 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -34,6 +34,7 @@ #include "rldp/rldp.h" #include "token-manager.h" #include "queue-size-counter.hpp" +#include "storage-stat-cache.hpp" #include "validator-telemetry.hpp" #include "impl/candidates-buffer.hpp" #include "td/utils/LRUCache.h" @@ -648,6 +649,13 @@ class ValidatorManagerImpl : public ValidatorManager { ++(success ? total_ls_queries_ok_ : total_ls_queries_error_)[lite_query_id]; } + void get_storage_stat_cache(td::Promise(const td::Bits256&)>> promise) override { + td::actor::send_closure(storage_stat_cache_, &StorageStatCache::get_cache, std::move(promise)); + } + void update_storage_stat_cache(std::vector, td::uint32>> data) override { + td::actor::send_closure(storage_stat_cache_, &StorageStatCache::update, std::move(data)); + } + private: td::Timestamp resend_shard_blocks_at_; td::Timestamp check_waiters_at_; @@ -773,6 +781,8 @@ class ValidatorManagerImpl : public ValidatorManager { std::map>>)>>> stats_providers_; + + td::actor::ActorOwn storage_stat_cache_; }; } // namespace validator diff --git a/validator/storage-stat-cache.cpp b/validator/storage-stat-cache.cpp new file mode 100644 index 000000000..6a8049b0d --- /dev/null +++ b/validator/storage-stat-cache.cpp @@ -0,0 +1,40 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "storage-stat-cache.hpp" + +namespace ton::validator { + +void StorageStatCache::get_cache(td::Promise(const td::Bits256&)>> promise) { + LOG(DEBUG) << "StorageStatCache::get_cache"; + promise.set_value( + [cache = cache_](const td::Bits256& hash) mutable -> td::Ref { return cache.lookup_ref(hash); }); +} + +void StorageStatCache::update(std::vector, td::uint32>> data) { + for (auto &[cell, size] : data) { + if (size < 4000) { + continue; + } + td::Bits256 hash = cell->get_hash().bits(); + LOG(DEBUG) << "StorageStatCache::update " << hash.to_hex() << " " << size; + cache_.set_ref(hash, cell); + lru_.put(hash, Deleter{hash, &cache_}, true, size); + } +} + +} // namespace ton::validator diff --git a/validator/storage-stat-cache.hpp b/validator/storage-stat-cache.hpp new file mode 100644 index 000000000..d32611fcd --- /dev/null +++ b/validator/storage-stat-cache.hpp @@ -0,0 +1,63 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "interfaces/validator-manager.h" +#include "td/utils/ConcurrentHashTable.h" +#include "td/utils/LRUCache.h" + +#include + +namespace ton::validator { + +class StorageStatCache : public td::actor::Actor { + public: + void get_cache(td::Promise(const td::Bits256&)>> promise); + + // (storage dict root, account total cells) + void update(std::vector, td::uint32>> data); + + private: + vm::Dictionary cache_{256}; + + struct Deleter { + Deleter(const td::Bits256& hash, vm::Dictionary* cache) : hash(hash), cache(cache) { + } + Deleter(const Deleter&) = delete; + Deleter(Deleter&& other) noexcept : hash(other.hash), cache(other.cache) { + other.cache = nullptr; + } + Deleter& operator=(const Deleter&) = delete; + Deleter& operator=(Deleter&& other) noexcept { + hash = other.hash; + cache = other.cache; + other.cache = nullptr; + return *this; + } + ~Deleter() { + if (cache) { + CHECK(cache->lookup_delete_ref(hash).not_null()); + LOG(DEBUG) << "StorageStatCache remove " << hash.to_hex(); + } + } + + td::Bits256 hash = td::Bits256::zero(); + vm::Dictionary* cache; + }; + td::LRUCache lru_{1 << 24}; +}; + +} // namespace ton::validator From 283186bf1453e00f4b206b3916240bc5b6c1d35d Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Thu, 26 Jun 2025 11:58:53 +0300 Subject: [PATCH 316/388] Update changelog --- Changelog.md | 14 +++++++++++++- recent_changelog.md | 17 ++++++++--------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/Changelog.md b/Changelog.md index 344b26cd3..d2f940c00 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,15 @@ +## 2025.06 Update + +1. ADNL and candidate broadcast optimization +2. [TVM version v11](./doc/GlobalVersions.md): new opcodes, and `c7` entry to improve developer experience. It also activates storage stats and `ihr_fee` nullification. +3. Fixed `start_lt` of tick transactions [see details on 01.06.2025 incident](https://telegra.ph/Report-on-June-1-2025-Operation-Incident-06-02). +4. Introduction of persistent state sharding, as well as making serialization of large BOCs more deterministic +5. Emulator improvements: in get methods, set config from provided `c7`; allow retrieval of logs from emulator runs for get methods +6. Optimized package import for archive nodes + +Besides the work of the core team, this update is based on the efforts of the RSquad team (deterministic large BOC serialization); AArayz, wy666444, Robinlzw, Lucian-code233 from TonBit (early discovery of the TVM 11 bug); @Skydev0h (uninitialized `BLOCKLT` in get methods); and @yma-het from TONWhales (emulator improvements). + + ## 2025.04 Update 1. Introduced substantial improvements of CellDB performance: celldb-v2, bloom filters. @@ -8,7 +20,7 @@ 6. [Added normalized hash](https://github.com/ton-blockchain/TEPs/pull/467) 7. Fix SDBEGINS(Q) in Asm.fif -Besides the work of the core team, this update is based on the efforts of @Stanislav-Povolotsky (tonlib fixes), @ice-charon (tonlib fixes), RSquad team (due payments improvements in v10), Arayz @ TonBit (improvements in RUNVM), @Skydev0h and @pyAndr3w (Asm.fif). +Besides the work of the core team, this update is based on the efforts of @Stanislav-Povolotsky (tonlib fixes); @ice-charon (tonlib fixes); RSquad team (due payments improvements in v10); Arayz, Robinlzw, @wy666444 @Lucian-code233 from TonBit (improvements in RUNVM); @Skydev0h and @pyAndr3w (Asm.fif). ## 2025.03 Update 1. New extracurrency behavior introduced, check [GlobalVersions.md](./doc/GlobalVersions.md#version-10) diff --git a/recent_changelog.md b/recent_changelog.md index 558960318..a178f38fa 100644 --- a/recent_changelog.md +++ b/recent_changelog.md @@ -1,12 +1,11 @@ -## 2025.04 Update +## 2025.06 Update -1. Introduced substantial improvements of CellDB performance: celldb-v2, bloom filters. -2. Accelerated a number of intrinsic node operations: SHA256, cell operations, large boc serialization, validator set checks. -3. [TVM version v10](./doc/GlobalVersions.md) -4. Overlay broadcast speed up and improved network stats. -5. Fixed some issues in tonlib -6. [Added normalized hash](https://github.com/ton-blockchain/TEPs/pull/467) -7. Fix SDBEGINS(Q) in Asm.fif +1. ADNL and candidate broadcast optimization +2. [TVM version v11](./doc/GlobalVersions.md): new opcodes, and `c7` entry to improve developer experience. It also activates storage stats and `ihr_fee` nullification. +3. Fixed `start_lt` of tick transactions [see details on 01.06.2025 incident](https://telegra.ph/Report-on-June-1-2025-Operation-Incident-06-02). +4. Introduction of persistent state sharding, as well as making serialization of large BOCs more deterministic +5. Emulator improvements: in get methods, set config from provided `c7`; allow retrieval of logs from emulator runs for get methods +6. Optimized package import for archive nodes -Besides the work of the core team, this update is based on the efforts of @Stanislav-Povolotsky (tonlib fixes), @ice-charon (tonlib fixes), RSquad team (due payments improvements in v10), Arayz @ TonBit (improvements in RUNVM), @Skydev0h and @pyAndr3w (Asm.fif). +Besides the work of the core team, this update is based on the efforts of the RSquad team (deterministic large BOC serialization); AArayz, wy666444, Robinlzw, Lucian-code233 from TonBit (early discovery of the TVM 11 bug); @Skydev0h (uninitialized `BLOCKLT` in get methods); and @yma-het from TONWhales (emulator improvements). From 75a75eb1f9283a958f6760f8b717f2eb49774a88 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Thu, 26 Jun 2025 16:38:30 +0400 Subject: [PATCH 317/388] use zero-autotools approach when compiling libsodium on mac (#1725) * Update build-macos-portable.sh update macro names * Update build-macos-portable.sh Make config.sub executable * Update build-macos-portable.sh Let's use zero-autotools approach when compiling libsodium on mac * Update build-macos-portable.sh refactor libsodium directory --- assembly/native/build-macos-portable.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index 351c4e7fb..feb5bf72f 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -65,19 +65,19 @@ else echo "Using compiled lz4" fi -if [ ! -d "../3pp/libsodium" ]; then +if [ ! -d "../3pp/libsodium-1.0.18" ]; then export LIBSODIUM_FULL_BUILD=1 - git clone https://github.com/jedisct1/libsodium.git ../3pp/libsodium - cd ../3pp/libsodium + cd ../3pp + curl -LO https://download.libsodium.org/libsodium/releases/libsodium-1.0.18.tar.gz + tar -xzf libsodium-1.0.18.tar.gz + cd libsodium-1.0.18 sodiumPath=`pwd` - git checkout 1.0.18 - ./autogen.sh ./configure --with-pic --enable-static make -j4 test $? -eq 0 || { echo "Can't compile libsodium"; exit 1; } cd ../../build else - sodiumPath=$(pwd)/../3pp/libsodium + sodiumPath=$(pwd)/../3pp/libsodium-1.0.18 echo "Using compiled libsodium" fi From 2bfe25a9ba20f1bff5d61a0f98e6e2a594487635 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 27 Jun 2025 12:00:07 +0300 Subject: [PATCH 318/388] Increase timeouts in ArchiveImporterLocal --- validator/import-db-slice-local.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/validator/import-db-slice-local.cpp b/validator/import-db-slice-local.cpp index 8c8db4ffb..945f41a6b 100644 --- a/validator/import-db-slice-local.cpp +++ b/validator/import-db-slice-local.cpp @@ -207,7 +207,7 @@ void ArchiveImporterLocal::import_first_key_block() { td::actor::send_closure(SelfId, &ArchiveImporterLocal::checked_key_block_proof, std::move(handle)); }); - run_check_proof_query(block_id, first_block.proof, manager_, td::Timestamp::in(10.0), std::move(P), + run_check_proof_query(block_id, first_block.proof, manager_, td::Timestamp::in(600.0), std::move(P), last_masterchain_state_, opts_->is_hardfork(block_id)); } @@ -278,7 +278,7 @@ void ArchiveImporterLocal::process_masterchain_blocks_cont() { } promise.set_result(td::Unit()); }); - run_check_proof_query(block_id, info.proof, manager_, td::Timestamp::in(10.0), std::move(P), + run_check_proof_query(block_id, info.proof, manager_, td::Timestamp::in(600.0), std::move(P), last_masterchain_state_, opts_->is_hardfork(block_id)); prev_block_id = block_id; } @@ -470,7 +470,7 @@ void ArchiveImporterLocal::store_data() { std::move(promise)); }); if (info.proof_link.not_null()) { - run_check_proof_link_query(block_id, info.proof_link, manager_, td::Timestamp::in(60.0), + run_check_proof_link_query(block_id, info.proof_link, manager_, td::Timestamp::in(600.0), ig.get_promise().wrap([](BlockHandle &&) { return td::Unit(); })); } } From aff4956078a0342f400bd4ab4bbab4a432ab9c49 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 1 Jul 2025 17:15:43 +0300 Subject: [PATCH 319/388] Storage stat cache for accelerator --- crypto/block/account-storage-stat.cpp | 6 +- crypto/block/account-storage-stat.h | 3 +- crypto/block/transaction.cpp | 6 +- crypto/block/transaction.h | 2 +- crypto/vm/cells/CellUsageTree.cpp | 4 +- crypto/vm/cells/CellUsageTree.h | 4 +- crypto/vm/cells/MerkleProof.h | 3 + validator/impl/collator-impl.h | 6 +- validator/impl/collator.cpp | 198 +++++++++++++++++++------- validator/impl/validate-query.cpp | 59 ++++---- validator/storage-stat-cache.cpp | 3 +- validator/storage-stat-cache.hpp | 7 +- 12 files changed, 206 insertions(+), 95 deletions(-) diff --git a/crypto/block/account-storage-stat.cpp b/crypto/block/account-storage-stat.cpp index e03341d4e..a6159c224 100644 --- a/crypto/block/account-storage-stat.cpp +++ b/crypto/block/account-storage-stat.cpp @@ -36,8 +36,7 @@ AccountStorageStat::AccountStorageStat(const AccountStorageStat* parent) CHECK(parent_->parent_ == nullptr); } -td::Status AccountStorageStat::replace_roots(std::vector> new_roots, bool check_merkle_depth, - td::HashSet* store_added) { +td::Status AccountStorageStat::replace_roots(std::vector> new_roots, bool check_merkle_depth) { std::erase_if(new_roots, [](const Ref& c) { return c.is_null(); }); if (new_roots.empty()) { roots_.clear(); @@ -74,9 +73,6 @@ td::Status AccountStorageStat::replace_roots(std::vector> new_root roots_ = std::move(new_roots); dict_up_to_date_ = false; for (auto& [_, e] : cache_) { - if (store_added && !e.exists && e.refcnt_diff > 0) { - store_added->insert(e.hash); - } TRY_STATUS(finalize_entry(e)); } return td::Status::OK(); diff --git a/crypto/block/account-storage-stat.h b/crypto/block/account-storage-stat.h index 12e922af1..4ada076b8 100644 --- a/crypto/block/account-storage-stat.h +++ b/crypto/block/account-storage-stat.h @@ -39,8 +39,7 @@ class AccountStorageStat { AccountStorageStat &operator=(const AccountStorageStat &other) = delete; AccountStorageStat &operator=(AccountStorageStat &&other) = default; - td::Status replace_roots(std::vector> new_roots, bool check_merkle_depth = false, - td::HashSet *store_added = nullptr); + td::Status replace_roots(std::vector> new_roots, bool check_merkle_depth = false); void add_hint(const td::HashSet &visited); td::uint64 get_total_cells() const { diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index 8e98f052e..ca34e14af 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -602,11 +602,9 @@ static td::Ref storage_without_extra_currencies(td::Ref> Account::compute_account_storage_dict(td::HashSet* storage_cells) const { +td::Result> Account::compute_account_storage_dict() const { if (storage.is_null()) { return td::Status::Error("cannot compute storage dict: empty storage"); } @@ -618,7 +616,7 @@ td::Result> Account::compute_account_storage_dict(td::HashSetprefetch_all_refs(), false, storage_cells)); + TRY_STATUS(stat.replace_roots(storage_for_stat->prefetch_all_refs())); // Root of AccountStorage is not counted in AccountStorageStat td::uint64 expected_cells = stat.get_total_cells() + 1; td::uint64 expected_bits = stat.get_total_bits() + storage->size(); diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index 528df4a00..1a7af2d12 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -301,7 +301,7 @@ struct Account { bool set_address(ton::WorkchainId wc, td::ConstBitPtr new_addr); bool unpack(Ref account, ton::UnixTime now, bool special); bool init_new(ton::UnixTime now); - td::Result> compute_account_storage_dict(td::HashSet* storage_cells = nullptr) const; + td::Result> compute_account_storage_dict() const; td::Status init_account_storage_stat(Ref dict_root); bool deactivate(); bool recompute_tmp_addr(Ref& tmp_addr, int fixed_prefix_length, td::ConstBitPtr orig_addr_rewrite) const; diff --git a/crypto/vm/cells/CellUsageTree.cpp b/crypto/vm/cells/CellUsageTree.cpp index 301ee7da4..05a4f0c45 100644 --- a/crypto/vm/cells/CellUsageTree.cpp +++ b/crypto/vm/cells/CellUsageTree.cpp @@ -99,11 +99,11 @@ void CellUsageTree::mark_path(NodeId node_id) { } } -CellUsageTree::NodeId CellUsageTree::get_parent(NodeId node_id) { +CellUsageTree::NodeId CellUsageTree::get_parent(NodeId node_id) const { return nodes_[node_id].parent; } -CellUsageTree::NodeId CellUsageTree::get_child(NodeId node_id, unsigned ref_id) { +CellUsageTree::NodeId CellUsageTree::get_child(NodeId node_id, unsigned ref_id) const { DCHECK(ref_id < CellTraits::max_refs); return nodes_[node_id].children[ref_id]; } diff --git a/crypto/vm/cells/CellUsageTree.h b/crypto/vm/cells/CellUsageTree.h index 902b98d75..cd2470dda 100644 --- a/crypto/vm/cells/CellUsageTree.h +++ b/crypto/vm/cells/CellUsageTree.h @@ -59,8 +59,8 @@ class CellUsageTree : public std::enable_shared_from_this { bool has_mark(NodeId node_id) const; void set_mark(NodeId node_id, bool mark = true); void mark_path(NodeId node_id); - NodeId get_parent(NodeId node_id); - NodeId get_child(NodeId node_id, unsigned ref_id); + NodeId get_parent(NodeId node_id) const; + NodeId get_child(NodeId node_id, unsigned ref_id) const; void set_use_mark_for_is_loaded(bool use_mark = true); NodeId create_child(NodeId node_id, unsigned ref_id); diff --git a/crypto/vm/cells/MerkleProof.h b/crypto/vm/cells/MerkleProof.h index f63683a59..68f0be7b9 100644 --- a/crypto/vm/cells/MerkleProof.h +++ b/crypto/vm/cells/MerkleProof.h @@ -74,6 +74,9 @@ class MerkleProofBuilder { void set_cell_load_callback(std::function f) { usage_tree->set_cell_load_callback(std::move(f)); } + const CellUsageTree &get_usage_tree() const { + return *usage_tree; + } }; } // namespace vm diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 91f3cd81c..fcc95caad 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -227,7 +227,6 @@ class Collator final : public td::actor::Actor { vm::ProofStorageStat proof_stat; bool add_to_collated_data = false; std::vector> storage_stat_updates; - td::HashSet original_storage_cells; }; std::map account_storage_dicts_; @@ -275,7 +274,8 @@ class Collator final : public td::actor::Actor { void after_get_shard_state(int idx, td::Result> res, td::PerfLogAction token); void after_get_block_data(int idx, td::Result> res, td::PerfLogAction token); void after_get_shard_blocks(td::Result>> res, td::PerfLogAction token); - void after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res); + void after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res, + td::PerfLogAction token); bool preprocess_prev_mc_state(); bool register_mc_state(Ref other_mc_state); bool request_aux_mc_state(BlockSeqno seqno, Ref& state); @@ -360,7 +360,7 @@ class Collator final : public td::actor::Actor { bool update_account_dict_estimation(const block::transaction::Transaction& trans); void update_account_storage_dict_info(const block::transaction::Transaction& trans); bool update_min_mc_seqno(ton::BlockSeqno some_mc_seqno); - bool process_account_storage_dict(const block::Account& account); + bool process_account_storage_dict(block::Account& account); bool combine_account_transactions(); bool update_public_libraries(); bool update_account_public_libraries(Ref orig_libs, Ref final_libs, const td::Bits256& addr); diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index f693d9554..acf9075b4 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -32,6 +32,7 @@ #include #include #include "fabric.h" +#include "storage-stat-cache.hpp" #include "validator-set.hpp" #include "top-shard-descr.hpp" #include @@ -298,12 +299,14 @@ void Collator::start_up() { // 6. get storage stat cache ++pending; LOG(DEBUG) << "sending get_storage_stat_cache() query to Manager"; - td::actor::send_closure_later( - manager, &ValidatorManager::get_storage_stat_cache, - [self = get_self()](td::Result(const td::Bits256&)>> res) { - LOG(DEBUG) << "got answer to get_storage_stat_cache() query"; - td::actor::send_closure_later(std::move(self), &Collator::after_get_storage_stat_cache, std::move(res)); - }); + td::actor::send_closure_later(manager, &ValidatorManager::get_storage_stat_cache, + [self = get_self(), token = perf_log_.start_action("get_storage_stat_cache")]( + td::Result(const td::Bits256&)>> res) mutable { + LOG(DEBUG) << "got answer to get_storage_stat_cache() query"; + td::actor::send_closure_later(std::move(self), + &Collator::after_get_storage_stat_cache, std::move(res), + std::move(token)); + }); // 7. set timeout alarm_timestamp() = timeout; CHECK(pending); @@ -741,8 +744,10 @@ void Collator::after_get_shard_blocks(td::Result(const td::Bits256&)>> res) { +void Collator::after_get_storage_stat_cache(td::Result(const td::Bits256&)>> res, + td::PerfLogAction token) { --pending; + token.finish(res); if (res.is_error()) { LOG(INFO) << "after_get_storage_stat_cache : " << res.error(); } else { @@ -2655,20 +2660,6 @@ std::unique_ptr Collator::make_account_from(td::ConstBitPtr addr if (!init_account_storage_dict(*ptr)) { return nullptr; } - if (storage_stat_cache_ && ptr->storage_dict_hash) { - auto dict_root = storage_stat_cache_(ptr->storage_dict_hash.value()); - if (dict_root.not_null()) { - auto S = ptr->init_account_storage_stat(dict_root); - if (S.is_error()) { - fatal_error(S.move_as_error_prefix(PSTRING() << "failed to init storage stat from cache for account " - << addr.to_hex(256) << ": ")); - return nullptr; - } - LOG(DEBUG) << "Inited storage stat from cache for account " << addr.to_hex(256) << " (" << ptr->storage_used.cells - << " cells)"; - storage_stat_cache_update_.emplace_back(dict_root, ptr->storage_used.cells); - } - } return ptr; } @@ -2679,29 +2670,54 @@ std::unique_ptr Collator::make_account_from(td::ConstBitPtr addr * @return True on success, False on failure */ bool Collator::init_account_storage_dict(block::Account& account) { - if (!full_collated_data_ || is_masterchain() || !account.storage_dict_hash || account.storage.is_null()) { + if (!account.storage_dict_hash || account.storage.is_null()) { return true; } td::Bits256 storage_dict_hash = account.storage_dict_hash.value(); if (storage_dict_hash.is_zero()) { return true; } + td::Ref cached_dict_root = + storage_stat_cache_ ? storage_stat_cache_(storage_dict_hash) : td::Ref{}; + if (cached_dict_root.not_null()) { + CHECK(td::Bits256{cached_dict_root->get_hash().bits()} == storage_dict_hash); + LOG(DEBUG) << "Inited storage stat from cache for account " << account.addr.to_hex() << " (" + << account.storage_used.cells << " cells)"; + storage_stat_cache_update_.emplace_back(cached_dict_root, account.storage_used.cells); + } + if (!full_collated_data_ || is_masterchain()) { + if (cached_dict_root.not_null()) { + auto S = account.init_account_storage_stat(cached_dict_root); + if (S.is_error()) { + return fatal_error(S.move_as_error_prefix(PSTRING() << "failed to init storage stat from cache for account " + << account.addr.to_hex() << ": ")); + } + } + return true; + } + AccountStorageDict& dict = account_storage_dicts_[storage_dict_hash]; if (!dict.inited) { dict.inited = true; - // don't mark cells in account state as loaded during compute_account_storage_dict - state_usage_tree_->set_ignore_loads(true); - auto res = account.compute_account_storage_dict(&dict.original_storage_cells); - state_usage_tree_->set_ignore_loads(false); - if (res.is_error()) { - return fatal_error(res.move_as_error_prefix(PSTRING() << "Failed to init account storage dict for " - << account.addr.to_hex() << ": ")); - } - if (res.ok().is_null()) { // Impossible if storage_dict_hash is not zero - return fatal_error(PSTRING() << "Failed to init account storage dict for " << account.addr.to_hex() - << ": dict is empty"); + td::Ref dict_root; + if (cached_dict_root.not_null()) { + dict_root = cached_dict_root; + } else { + // don't mark cells in account state as loaded during compute_account_storage_dict + state_usage_tree_->set_ignore_loads(true); + auto res = account.compute_account_storage_dict(); + state_usage_tree_->set_ignore_loads(false); + if (res.is_error()) { + return fatal_error(res.move_as_error_prefix(PSTRING() << "Failed to init account storage dict for " + << account.addr.to_hex() << ": ")); + } + dict_root = res.move_as_ok(); + if (dict_root.is_null()) { // Impossible if storage_dict_hash is not zero + return fatal_error(PSTRING() << "Failed to init account storage dict for " << account.addr.to_hex() + << ": dict is empty"); + } } - dict.mpb = vm::MerkleProofBuilder(res.move_as_ok()); + dict.mpb = vm::MerkleProofBuilder(std::move(dict_root)); dict.mpb.set_cell_load_callback([&](const vm::LoadedCell& cell) { on_cell_loaded(cell); }); @@ -2764,6 +2780,65 @@ td::Result Collator::make_account(td::ConstBitPtr addr, bool fo return ins.first->second.get(); } +/** + * Removes UsageCell's from new_root's subtree, replacing them with regular DataCell's + * + * @param old_root Original root (not wrapped in UsageCell) + * @param new_root Cell to remove UsageCell's from + * @param usage_tree CellUsageTree for old_root + * + * @returns new_root without UsageCell's + */ +static td::Ref clean_usage_cells(td::Ref old_root, td::Ref new_root, + const vm::CellUsageTree& usage_tree) { + if (new_root.is_null()) { + return {}; + } + + td::HashMap> visited_cells; + std::function, vm::CellUsageTree::NodeId)> dfs_old = [&](td::Ref cell, + vm::CellUsageTree::NodeId node_id) { + visited_cells[cell->get_hash()] = cell; + if (!usage_tree.is_loaded(node_id)) { + return; + } + vm::CellSlice cs{vm::NoVm(), cell}; + for (unsigned i = 0; i < cs.size_refs(); i++) { + dfs_old(cs.prefetch_ref(i), usage_tree.get_child(node_id, i)); + } + }; + if (old_root.not_null()) { + dfs_old(old_root, usage_tree.root_id()); + } + + std::function(td::Ref)> dfs = [&](td::Ref cell) -> td::Ref { + auto it = visited_cells.find(cell->get_hash()); + if (it != visited_cells.end()) { + return it->second; + } + auto loaded_cell = cell->load_cell().move_as_ok(); + CHECK(loaded_cell.virt.get_virtualization() == 0); + td::Ref data_cell = std::move(loaded_cell.data_cell); + td::Ref children[vm::Cell::max_refs]; + bool changed = false; + for (unsigned i = 0; i < data_cell->size_refs(); ++i) { + td::Ref child = data_cell->get_ref(i); + children[i] = dfs(child); + if (children[i] != child) { + changed = true; + } + } + if (changed) { + data_cell = vm::DataCell::create(td::Slice{data_cell->get_data(), (data_cell->size() + 7) / 8}, data_cell->size(), + {children, data_cell->size_refs()}, data_cell->is_special()) + .move_as_ok(); + CHECK(data_cell->get_hash() == cell->get_hash()); + } + return visited_cells[cell->get_hash()] = data_cell; + }; + return dfs(new_root); +} + /** * Decides whether to include storage dict proof to collated data for this account or not. * @@ -2771,17 +2846,34 @@ td::Result Collator::make_account(td::ConstBitPtr addr, bool fo * * @returns True if the operation is successful, false otherwise. */ -bool Collator::process_account_storage_dict(const block::Account& account) { +bool Collator::process_account_storage_dict(block::Account& account) { + bool store_dict_to_cache = account.storage_dict_hash && account.account_storage_stat && + account.account_storage_stat.value().is_dict_ready() && + account.storage_used.cells >= StorageStatCache::MIN_ACCOUNT_CELLS; if (!account.orig_storage_dict_hash) { + if (store_dict_to_cache) { + td::Ref dict_root = account.account_storage_stat.value().get_dict_root().move_as_ok(); + storage_stat_cache_update_.emplace_back(dict_root, account.storage_used.cells); + } return true; } td::Bits256 storage_dict_hash = account.orig_storage_dict_hash.value(); auto it = account_storage_dicts_.find(storage_dict_hash); if (it == account_storage_dicts_.end()) { + if (store_dict_to_cache) { + td::Ref dict_root = account.account_storage_stat.value().get_dict_root().move_as_ok(); + storage_stat_cache_update_.emplace_back(dict_root, account.storage_used.cells); + } return true; } CHECK(full_collated_data_ && !is_masterchain()); AccountStorageDict& dict = it->second; + td::Ref original_dict_root = dict.mpb.original_root(); + if (store_dict_to_cache) { + td::Ref dict_root = account.account_storage_stat.value().get_dict_root().move_as_ok(); + dict_root = clean_usage_cells(original_dict_root, dict_root, dict.mpb.get_usage_tree()); + storage_stat_cache_update_.emplace_back(dict_root, account.storage_used.cells); + } if (dict.add_to_collated_data) { LOG(DEBUG) << "Storage dict proof of account " << account.addr.to_hex() << " : already included"; return true; @@ -2794,12 +2886,13 @@ bool Collator::process_account_storage_dict(const block::Account& account) { td::HashSet visited; bool calculate_proof_size_diff = true; td::int64 proof_size_diff = 0; - std::function&)> dfs = [&](const Ref& cell) { + vm::Dictionary original_dict{original_dict_root, 256}; + std::function&)> dfs = [&](const Ref& cell) -> bool { if (cell.is_null() || !visited.emplace(cell->get_hash()).second) { - return; + return true; } auto loaded_cell = cell->load_cell().move_as_ok(); - if (dict.original_storage_cells.contains(cell->get_hash())) { + if (original_dict.lookup(cell->get_hash().bits(), 256).not_null()) { if (calculate_proof_size_diff) { switch (collated_data_stat.get_cell_status(cell->get_hash())) { case vm::ProofStorageStat::c_none: @@ -2812,26 +2905,36 @@ bool Collator::process_account_storage_dict(const block::Account& account) { case vm::ProofStorageStat::c_loaded: break; } + if (proof_size_diff > (td::int64)dict.proof_stat.estimate_proof_size()) { + return false; + } } else { collated_data_stat.add_loaded_cell(loaded_cell.data_cell, loaded_cell.virt.get_level()); } } vm::CellSlice cs{std::move(loaded_cell.data_cell)}; for (unsigned i = 0; i < cs.size_refs(); ++i) { - dfs(cs.prefetch_ref(i)); + if (!dfs(cs.prefetch_ref(i))) { + return false; + } } + return true; }; // Visit cells that were used in storage stat computation to calculate collated data increase + bool account_proof_too_big = false; state_usage_tree_->set_ignore_loads(true); for (const auto& cell : dict.storage_stat_updates) { - dfs(cell); + if (!dfs(cell)) { + account_proof_too_big = true; + break; + } } state_usage_tree_->set_ignore_loads(false); - if (proof_size_diff > (td::int64)dict.proof_stat.estimate_proof_size()) { + if (account_proof_too_big) { LOG(DEBUG) << "Storage dict proof of account " << account.addr.to_hex() - << " : account_proof_size=" << proof_size_diff + << " : account_proof_size>=" << proof_size_diff << ", dict_proof_size=" << dict.proof_stat.estimate_proof_size() << ", include dict in collated data"; dict.add_to_collated_data = true; collated_data_stat.add_loaded_cells(dict.proof_stat); @@ -2844,7 +2947,7 @@ bool Collator::process_account_storage_dict(const block::Account& account) { calculate_proof_size_diff = false; visited.clear(); for (const auto& cell : dict.storage_stat_updates) { - dfs(cell); + CHECK(dfs(cell)); } } @@ -2943,10 +3046,6 @@ bool Collator::combine_account_transactions() { if (!process_account_storage_dict(acc)) { return false; } - if (acc.storage_dict_hash && acc.account_storage_stat && acc.account_storage_stat.value().is_dict_ready()) { - storage_stat_cache_update_.emplace_back(acc.account_storage_stat.value().get_dict_root().move_as_ok(), - acc.storage_used.cells); - } } else { if (acc.total_state->get_hash() != acc.orig_total_state->get_hash()) { return fatal_error(std::string{"total state of account "} + z.first.to_hex() + @@ -6270,7 +6369,10 @@ bool Collator::create_block_candidate() { td::actor::send_closure_later(manager, &ValidatorManager::complete_external_messages, std::move(delay_ext_msgs_), std::move(bad_ext_msgs_)); } - td::actor::send_closure(manager, &ValidatorManager::update_storage_stat_cache, std::move(storage_stat_cache_update_)); + if (!storage_stat_cache_update_.empty()) { + td::actor::send_closure(manager, &ValidatorManager::update_storage_stat_cache, + std::move(storage_stat_cache_update_)); + } return true; } diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 7570c7899..ac7f400f3 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -32,6 +32,8 @@ #include "vm/cells/MerkleUpdate.h" #include "common/errorlog.h" #include "fabric.h" +#include "storage-stat-cache.hpp" + #include namespace ton { @@ -657,6 +659,7 @@ bool ValidateQuery::extract_collated_data_from(Ref croot, int idx) { if (!virt_account_storage_dicts_.emplace(virt_root->get_hash().bits(), virt_root).second) { return reject_query("duplicate AccountStorageDictProof"); } + full_collated_data_ = true; return true; } LOG(WARNING) << "collated datum # " << idx << " has unknown type (magic " << cs.prefetch_ulong(32) << "), ignoring"; @@ -5316,20 +5319,6 @@ std::unique_ptr ValidateQuery::make_account_from(td::ConstBitPtr return nullptr; } ptr->block_lt = start_lt_; - if (storage_stat_cache_ && ptr->storage_dict_hash) { - auto dict_root = storage_stat_cache_(ptr->storage_dict_hash.value()); - if (dict_root.not_null()) { - auto S = ptr->init_account_storage_stat(dict_root); - if (S.is_error()) { - fatal_error(S.move_as_error_prefix(PSTRING() << "failed to init storage stat from cache for account " - << addr.to_hex(256) << ": ")); - return nullptr; - } - LOG(DEBUG) << "Inited storage stat from cache for account " << addr.to_hex(256) << " (" << ptr->storage_used.cells - << " cells)"; - storage_stat_cache_update_.emplace_back(dict_root, ptr->storage_used.cells); - } - } return ptr; } @@ -5356,14 +5345,30 @@ std::unique_ptr ValidateQuery::unpack_account(td::ConstBitPtr ad return {}; } if (new_acc->storage_dict_hash) { - auto it = virt_account_storage_dicts_.find(new_acc->storage_dict_hash.value()); - if (it != virt_account_storage_dicts_.end()) { - LOG(DEBUG) << "Using account storage dict proof for account " << addr.to_hex(256) - << ", hash=" << it->second->get_hash().to_hex(); - auto S = new_acc->init_account_storage_stat(it->second); - if (S.is_error()) { - reject_query(PSTRING() << "Failed to init account storage stat for account " << addr.to_hex(256), std::move(S)); - return {}; + if (full_collated_data_ && !is_masterchain()) { + auto it = virt_account_storage_dicts_.find(new_acc->storage_dict_hash.value()); + if (it != virt_account_storage_dicts_.end()) { + LOG(DEBUG) << "Using account storage dict proof for account " << addr.to_hex(256) + << ", hash=" << it->second->get_hash().to_hex(); + auto S = new_acc->init_account_storage_stat(it->second); + if (S.is_error()) { + reject_query(PSTRING() << "Failed to init account storage stat for account " << addr.to_hex(256), + std::move(S)); + return {}; + } + } + } else if (storage_stat_cache_ && new_acc->storage_dict_hash) { + auto dict_root = storage_stat_cache_(new_acc->storage_dict_hash.value()); + if (dict_root.not_null()) { + auto S = new_acc->init_account_storage_stat(dict_root); + if (S.is_error()) { + fatal_error(S.move_as_error_prefix(PSTRING() << "failed to init storage stat from cache for account " + << addr.to_hex(256) << ": ")); + return {}; + } + LOG(DEBUG) << "Inited storage stat from cache for account " << addr.to_hex(256) << " (" + << new_acc->storage_used.cells << " cells)"; + storage_stat_cache_update_.emplace_back(dict_root, new_acc->storage_used.cells); } } } @@ -5955,8 +5960,9 @@ bool ValidateQuery::check_account_transactions(const StdSmcAddress& acc_addr, Re })) { return reject_query("at least one Transaction of account "s + acc_addr.to_hex() + " is invalid"); } - if (account.storage_dict_hash && account.account_storage_stat && - account.account_storage_stat.value().is_dict_ready()) { + if ((!full_collated_data_ || is_masterchain()) && account.storage_dict_hash && account.account_storage_stat && + account.account_storage_stat.value().is_dict_ready() && + account.storage_used.cells >= StorageStatCache::MIN_ACCOUNT_CELLS) { storage_stat_cache_update_.emplace_back(account.account_storage_stat.value().get_dict_root().move_as_ok(), account.storage_used.cells); } @@ -7156,7 +7162,10 @@ bool ValidateQuery::save_candidate() { td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, id_, block_candidate.clone(), validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), std::move(P)); - td::actor::send_closure(manager, &ValidatorManager::update_storage_stat_cache, std::move(storage_stat_cache_update_)); + if (!storage_stat_cache_update_.empty()) { + td::actor::send_closure(manager, &ValidatorManager::update_storage_stat_cache, + std::move(storage_stat_cache_update_)); + } return true; } diff --git a/validator/storage-stat-cache.cpp b/validator/storage-stat-cache.cpp index 6a8049b0d..a81d958f5 100644 --- a/validator/storage-stat-cache.cpp +++ b/validator/storage-stat-cache.cpp @@ -14,7 +14,6 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#pragma once #include "storage-stat-cache.hpp" namespace ton::validator { @@ -27,7 +26,7 @@ void StorageStatCache::get_cache(td::Promise(con void StorageStatCache::update(std::vector, td::uint32>> data) { for (auto &[cell, size] : data) { - if (size < 4000) { + if (size < MIN_ACCOUNT_CELLS) { continue; } td::Bits256 hash = cell->get_hash().bits(); diff --git a/validator/storage-stat-cache.hpp b/validator/storage-stat-cache.hpp index d32611fcd..194a0b79a 100644 --- a/validator/storage-stat-cache.hpp +++ b/validator/storage-stat-cache.hpp @@ -57,7 +57,12 @@ class StorageStatCache : public td::actor::Actor { td::Bits256 hash = td::Bits256::zero(); vm::Dictionary* cache; }; - td::LRUCache lru_{1 << 24}; + td::LRUCache lru_{MAX_CACHE_TOTAL_CELLS}; + + static constexpr td::uint64 MAX_CACHE_TOTAL_CELLS = 1 << 24; + + public: + static constexpr td::uint64 MIN_ACCOUNT_CELLS = 4000; }; } // namespace ton::validator From 21c191d2648c7a6d216afb4263a14a890d538d6d Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 1 Jul 2025 17:26:00 +0300 Subject: [PATCH 320/388] Improve TL scheme for overlay nodes V2 --- tl/generate/scheme/ton_api.tl | 4 ++-- tl/generate/scheme/ton_api.tlo | Bin 116024 -> 116024 bytes 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 30e8262d4..4da1ee301 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -232,7 +232,7 @@ overlay.node.toSignEx id:adnl.id.short overlay:int256 flags:int version:int = ov overlay.node id:PublicKey overlay:int256 version:int signature:bytes = overlay.Node; overlay.nodeV2 id:PublicKey overlay:int256 flags:int version:int signature:bytes certificate:overlay.MemberCertificate = overlay.NodeV2; overlay.nodes nodes:(vector overlay.node) = overlay.Nodes; -overlay.nodesV2 nodes:(vector overlay.NodeV2) = overlay.NodesV2; +overlay.nodesV2 nodes:(vector overlay.nodeV2) = overlay.NodesV2; overlay.pong = overlay.Pong; @@ -271,7 +271,7 @@ overlay.broadcastNotFound = overlay.Broadcast; ---functions--- overlay.getRandomPeers peers:overlay.nodes = overlay.Nodes; -overlay.getRandomPeersV2 peers:overlay.NodesV2 = overlay.NodesV2; +overlay.getRandomPeersV2 peers:overlay.nodesV2 = overlay.NodesV2; overlay.ping = overlay.Pong; overlay.query overlay:int256 = True; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 3caa8b14674e96e891645db8579517e543cebd5d..9959470322712e8a29e37b912ad8211c0c60d9d7 100644 GIT binary patch delta 167 zcmdnd#lEA9eZvp6iCaV_^Qh~vI7zTS*=(a;aZ&=r;?FNjEy_u()XU3HNi7aD0*Os7 zm~SAleC_PLj0_M^HFt0%7SgfX5jXSd5jO#0DE6JaR2}S delta 169 zcmdnd#lEA9eZvp6$w!z(CiAH4uvk31D7D!}z2c+ Date: Wed, 2 Jul 2025 11:50:28 +0300 Subject: [PATCH 321/388] Remove duplicate validator-engine options --- validator-engine/validator-engine.cpp | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index b28f349f9..c78e0acc3 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -5578,21 +5578,6 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_sync_shards_upto, v); }); return td::Status::OK(); }); - p.add_option( - '\0', "permanent-celldb", - "disable garbage collection in CellDb. This improves performance on archival nodes (once enabled, this option " - "cannot be disabled)", - [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_permanent_celldb, true); }); }); - p.add_option('\0', "skip-key-sync", - "don't select the best persistent state on initial sync, start on init_block from global config", [&]() { - acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_skip_key_sync, true); }); - }); - p.add_checked_option( - '\0', "sync-shards-upto", "stop syncing shards on this masterchain seqno", [&](td::Slice s) -> td::Status { - TRY_RESULT(v, td::to_integer_safe(s)); - acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_sync_shards_upto, v); }); - return td::Status::OK(); - }); p.add_checked_option('\0', "shard-block-retainer", "adnl id for shard block retainer (hex), or \"fullnode\" for full node id", [&](td::Slice arg) -> td::Status { From ee58f04c015dc971020c8470649d317dab347be1 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 4 Jul 2025 13:36:17 +0300 Subject: [PATCH 322/388] Properly add non-light validators to fast-sync overlays --- tl/generate/scheme/ton_api.tl | 2 +- tl/generate/scheme/ton_api.tlo | Bin 116024 -> 116068 bytes validator/full-node-fast-sync-overlays.cpp | 3 ++- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 4da1ee301..7db6ac0e7 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -768,7 +768,7 @@ engine.validator.shardOverlayStats.neighbour id:string verison_major:int version engine.validator.shardOverlayStats shard:string active:Bool neighbours:(vector engine.validator.shardOverlayStats.neighbour) = engine.validator.ShardOverlayStats; engine.validator.fastSyncOverlayStats shard:string validators_adnl:(vector int256) root_public_keys:(vector int256) - member_certificate:overlay.MemberCertificate = engine.validator.FastSyncOverlayStats; + member_certificate:overlay.MemberCertificate receive_broadcasts:Bool = engine.validator.FastSyncOverlayStats; engine.validator.onePerfTimerStat time:int min:double avg:double max:double = engine.validator.OnePerfTimerStat; engine.validator.perfTimerStatsByName name:string stats:(vector engine.validator.OnePerfTimerStat) = engine.validator.PerfTimerStatsByName; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 9959470322712e8a29e37b912ad8211c0c60d9d7..b4d84dd96804cee1ac61c758da3f1a1307cfb32f 100644 GIT binary patch delta 99 zcmdnd#r~v=eZveX*6f#m4y#TUl#<-MO={l~C^vAQ promise) if (!member_certificate_.empty()) { res->member_certificate_ = member_certificate_.tl(); } + res->receive_broadcasts_ = receive_broadcasts_; promise.set_result(td::json_encode(td::ToJson(*res), true)); } @@ -525,7 +526,7 @@ void FullNodeFastSyncOverlays::update_overlays(td::Ref state, // Update shard overlays for (ShardIdFull shard : all_shards) { - bool receive_broadcasts = overlays_info.is_validator_ ? shard.is_masterchain() : monitoring_shards.count(shard); + bool receive_broadcasts = monitoring_shards.contains(shard); auto &overlay = overlays_info.overlays_[shard]; if (overlay.empty()) { overlay = td::actor::create_actor( From 75ff7e6288d678ef209317f7b497dd64b2343bd5 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 4 Jul 2025 16:06:56 +0300 Subject: [PATCH 323/388] Enable fast-sync overlays by proto_version 5 --- validator/full-node.cpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 27278c3ee..df9123761 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -262,7 +262,11 @@ void FullNodeImpl::on_new_masterchain_block(td::Ref state, std } } + int proto_version = state->get_consensus_config().proto_version; + use_old_private_overlays_ = (proto_version < 5); + if (!use_old_private_overlays_) { + private_block_overlays_.clear(); std::set my_adnl_ids; my_adnl_ids.insert(adnl_id_); for (const auto &[adnl_id, _] : local_collator_nodes_) { @@ -690,14 +694,6 @@ void FullNodeImpl::update_validator_telemetry_collector() { } void FullNodeImpl::start_up() { - // TODO: enable fast sync overlays by other means (e.g. some config param) - // TODO: in the future - remove the old private overlay entirely - // This env var is for testing - auto fast_sync_env = getenv("TON_FAST_SYNC_OVERLAYS"); - if (fast_sync_env && !strcmp(fast_sync_env, "1")) { - use_old_private_overlays_ = false; - } - update_shard_actor(ShardIdFull{masterchainId}, true); if (local_id_.is_zero()) { if (adnl_id_.is_zero()) { From 0f4849b6671050581c48d43e624f3ba3d4ff9a39 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 4 Jul 2025 16:40:52 +0300 Subject: [PATCH 324/388] Set max fast sync overlay clients to 5 --- validator/full-node.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/full-node.h b/validator/full-node.h index 2c51038eb..dc29f67ca 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -117,7 +117,7 @@ class FullNode : public td::actor::Actor { } enum { broadcast_mode_public = 1, broadcast_mode_private_block = 2, broadcast_mode_custom = 4 }; - static constexpr td::int32 MAX_FAST_SYNC_OVERLAY_CLIENTS = 5000; // TODO: set lower limit (high limit for testing) + static constexpr td::int32 MAX_FAST_SYNC_OVERLAY_CLIENTS = 5; static td::actor::ActorOwn create( ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeOptions opts, From cbe79eb848ab554b5eb96e70c08e777cfa3b2297 Mon Sep 17 00:00:00 2001 From: krigga <25533192+krigga@users.noreply.github.com> Date: Sun, 6 Jul 2025 12:18:22 +0300 Subject: [PATCH 325/388] feat: add prev blocks info param to emscripten tx emulator (#1726) --- emulator/emulator-emscripten.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/emulator/emulator-emscripten.cpp b/emulator/emulator-emscripten.cpp index efb14eff9..7b8e9dd1e 100644 --- a/emulator/emulator-emscripten.cpp +++ b/emulator/emulator-emscripten.cpp @@ -11,6 +11,7 @@ struct TransactionEmulationParams { uint32_t utime; uint64_t lt; td::optional rand_seed_hex; + td::optional prev_blocks_info; bool ignore_chksig; bool is_tick_tock; bool is_tock; @@ -49,6 +50,11 @@ td::Result decode_transaction_emulation_params(const TRY_RESULT(is_tock, td::get_json_object_bool_field(obj, "is_tock", true, false)); params.is_tock = is_tock; + TRY_RESULT(prev_blocks_info_str, td::get_json_object_string_field(obj, "prev_blocks_info", true)); + if (prev_blocks_info_str.size() > 0) { + params.prev_blocks_info = prev_blocks_info_str; + } + if (is_tock && !is_tick_tock) { return td::Status::Error("Inconsistent parameters is_tick_tock=false, is_tock=true"); } @@ -200,12 +206,18 @@ const char *emulate_with_emulator(void* em, const char* libs, const char* accoun rand_seed_set = transaction_emulator_set_rand_seed(em, decoded_params.rand_seed_hex.unwrap().c_str()); } + bool prev_blocks_set = true; + if (decoded_params.prev_blocks_info) { + prev_blocks_set = transaction_emulator_set_prev_blocks_info(em, decoded_params.prev_blocks_info.unwrap().c_str()); + } + if (!transaction_emulator_set_libs(em, libs) || !transaction_emulator_set_lt(em, decoded_params.lt) || !transaction_emulator_set_unixtime(em, decoded_params.utime) || !transaction_emulator_set_ignore_chksig(em, decoded_params.ignore_chksig) || !transaction_emulator_set_debug_enabled(em, decoded_params.debug_enabled) || - !rand_seed_set) { + !rand_seed_set || + !prev_blocks_set) { transaction_emulator_destroy(em); return strdup(R"({"fail":true,"message":"Can't set params"})"); } From bd89b252ca5bee5fa9dee2f53accc0e8221e0c2f Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Thu, 24 Apr 2025 18:30:43 +0400 Subject: [PATCH 326/388] [Tolk] Optimize try-catch codegen with SETCONTMANY instruction --- tolk-tester/tests/try-catch-tests.tolk | 14 +++++++++++++- tolk/codegen.cpp | 11 +---------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/tolk-tester/tests/try-catch-tests.tolk b/tolk-tester/tests/try-catch-tests.tolk index be0362e39..8acb6f2c7 100644 --- a/tolk-tester/tests/try-catch-tests.tolk +++ b/tolk-tester/tests/try-catch-tests.tolk @@ -236,6 +236,17 @@ fun testCodegen3(numberId: int, paramVal: cell) { paramVal.beginParse(); } +@method_id(112) +fun testCatchUsesVars(x1: int) { + var x2 = x1 * 2; + try { alwaysThrow123(); } + catch (excno) { + var result = x1 + x2 + excno; + return result; + } + return -1; +} + @method_id(113) fun testBigExcno() { try { @@ -274,9 +285,10 @@ fun main() { @testcase | 110 | 0 | 5 @testcase | 111 | -1 | 123 @testcase | 111 | 0 | 456 +@testcase | 112 | 5 | 138 @testcase | 113 | | 2048 -@code_hash 26411074751358281917876479601714698321674833208179417883888954015536273820035 +@code_hash 100812963733861852648745958727639570706776197131971804296829155862118445447691 @fif_codegen """ diff --git a/tolk/codegen.cpp b/tolk/codegen.cpp index a7c84d27d..c635c7483 100644 --- a/tolk/codegen.cpp +++ b/tolk/codegen.cpp @@ -843,11 +843,6 @@ bool Op::generate_code_step(Stack& stack) { catch_stack.push_new_var(left[1]); stack.rearrange_top(loc, catch_vars, catch_last); stack.opt_show(); - stack.o << AsmOp::Custom(loc, "c1 PUSH"); - stack.o << AsmOp::Custom(loc, "c3 PUSH"); - stack.o << AsmOp::Custom(loc, "c4 PUSH"); - stack.o << AsmOp::Custom(loc, "c5 PUSH"); - stack.o << AsmOp::Custom(loc, "c7 PUSH"); stack.o << AsmOp::Custom(loc, "<{"); stack.o.indent(); if (block1->noreturn()) { @@ -858,11 +853,7 @@ bool Op::generate_code_step(Stack& stack) { catch_stack.opt_show(); stack.o.undent(); stack.o << AsmOp::Custom({}, "}>CONT"); - stack.o << AsmOp::Custom(loc, "c7 SETCONT"); - stack.o << AsmOp::Custom(loc, "c5 SETCONT"); - stack.o << AsmOp::Custom(loc, "c4 SETCONT"); - stack.o << AsmOp::Custom(loc, "c3 SETCONT"); - stack.o << AsmOp::Custom(loc, "c1 SETCONT"); + stack.o << AsmOp::Custom(loc, "0b10111010 SETCONTMANY"); for (size_t begin = catch_vars.size(), end = begin; end > 0; end = begin) { begin = end >= block_size ? end - block_size : 0; stack.o << AsmOp::Custom(loc, std::to_string(end - begin) + " PUSHINT"); From 487034a5a2e59c1e4ef8fd4ab0f742296f271234 Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Sun, 15 Jun 2025 14:53:55 +0700 Subject: [PATCH 327/388] [Tolk] More accurate serialization estimation for snake structs This allows express snake nesting like struct A { next: Cell? } --- .../tests/invalid-serialization/err-7563.tolk | 2 +- tolk-tester/tests/pack-unpack-5.tolk | 16 ++++++++++++++++ tolk/pack-unpack-api.cpp | 12 ++++++++++-- tolk/pack-unpack-serializers.cpp | 2 +- tolk/pipe-check-serialized-fields.cpp | 7 +------ 5 files changed, 29 insertions(+), 10 deletions(-) diff --git a/tolk-tester/tests/invalid-serialization/err-7563.tolk b/tolk-tester/tests/invalid-serialization/err-7563.tolk index cc83a0cd5..9934ebb6e 100644 --- a/tolk-tester/tests/invalid-serialization/err-7563.tolk +++ b/tolk-tester/tests/invalid-serialization/err-7563.tolk @@ -11,7 +11,7 @@ struct B { } fun main(b: B) { - b.toCell(); + B.fromSlice("").a.load(); } /** diff --git a/tolk-tester/tests/pack-unpack-5.tolk b/tolk-tester/tests/pack-unpack-5.tolk index 9644d7331..46b2a77b5 100644 --- a/tolk-tester/tests/pack-unpack-5.tolk +++ b/tolk-tester/tests/pack-unpack-5.tolk @@ -178,6 +178,21 @@ fun test10() { return (Test10_1.estimatePackSize(), Test10_2.estimatePackSize()); } +struct Test11_1 { + data: bits1022; + next: Cell?; +} + +struct Test11_2 { + self1: Cell; + self2: Cell; +} + +@method_id(111) +fun test11() { + return (Test11_1.estimatePackSize(), Test11_2.estimatePackSize()); +} + @method_id(120) fun test20() { return (Test7_1 .getDeclaredPackPrefixLen(), Test7_1 .getDeclaredPackPrefix(), @@ -199,5 +214,6 @@ fun main() { @testcase | 108 | | [ 10 275 0 0 ] [ 34 158 0 0 ] [ 47 1143 0 1 ] @testcase | 109 | | [ 34 130 0 0 ] [ 33 159 0 0 ] [ 4 8 0 0 ] [ 3 7 0 0 ] @testcase | 110 | | [ 32 9999 0 4 ] [ 33 9999 0 8 ] +@testcase | 111 | | [ 1023 1023 0 1 ] [ 0 0 2 2 ] @testcase | 120 | | 16 4128 1 1 */ diff --git a/tolk/pack-unpack-api.cpp b/tolk/pack-unpack-api.cpp index 227a07066..107130d59 100644 --- a/tolk/pack-unpack-api.cpp +++ b/tolk/pack-unpack-api.cpp @@ -50,8 +50,10 @@ struct CantSerializeBecause { }; class PackUnpackAvailabilityChecker { + std::vector already_checked; + public: - static std::optional detect_why_cant_serialize(TypePtr any_type, bool is_pack) { + std::optional detect_why_cant_serialize(TypePtr any_type, bool is_pack) { if (any_type->try_as()) { return {}; } @@ -76,6 +78,11 @@ class PackUnpackAvailabilityChecker { if (const auto* t_struct = any_type->try_as()) { StructPtr struct_ref = t_struct->struct_ref; + if (std::find(already_checked.begin(), already_checked.end(), struct_ref) != already_checked.end()) { + return {}; + } + already_checked.push_back(struct_ref); // prevent recursion and visiting one struct multiple times + for (StructFieldPtr field_ref : struct_ref->fields) { if (auto why = detect_why_cant_serialize(field_ref->declared_type, is_pack)) { return CantSerializeBecause("because field `" + struct_ref->name + "." + field_ref->name + "` of type `" + field_ref->declared_type->as_human_readable() + "` can't be serialized", why.value()); @@ -167,7 +174,8 @@ class PackUnpackAvailabilityChecker { }; bool check_struct_can_be_packed_or_unpacked(TypePtr any_type, bool is_pack, std::string& because_msg) { - if (auto why = PackUnpackAvailabilityChecker::detect_why_cant_serialize(any_type, is_pack)) { + PackUnpackAvailabilityChecker checker; + if (auto why = checker.detect_why_cant_serialize(any_type, is_pack)) { because_msg = why.value().because_msg; return false; } diff --git a/tolk/pack-unpack-serializers.cpp b/tolk/pack-unpack-serializers.cpp index 61962615a..d139a4526 100644 --- a/tolk/pack-unpack-serializers.cpp +++ b/tolk/pack-unpack-serializers.cpp @@ -959,7 +959,7 @@ static std::unique_ptr get_serializer_for_type(TypePtr any_type) { if (any_type == TypeDataBool::create()) { return std::make_unique(); } - if (any_type == TypeDataCell::create()) { + if (any_type == TypeDataCell::create() || is_type_cellT(any_type)) { return std::make_unique(); } if (any_type == TypeDataAddress::create()) { diff --git a/tolk/pipe-check-serialized-fields.cpp b/tolk/pipe-check-serialized-fields.cpp index 8f436c957..a13032e37 100644 --- a/tolk/pipe-check-serialized-fields.cpp +++ b/tolk/pipe-check-serialized-fields.cpp @@ -69,12 +69,7 @@ class CheckSerializedFieldsAndTypesVisitor final : public ASTVisitorFunctionBody fire_error_theoretical_overflow_1023(struct_ref, size); } } - for (StructFieldPtr field_ref : struct_ref->fields) { - if (is_type_cellT(field_ref->declared_type)) { - const TypeDataStruct* f_struct = field_ref->declared_type->try_as(); - check_type_fits_cell_or_has_policy(f_struct->struct_ref->substitutedTs->typeT_at(0)); - } - } + // don't check Cell fields for overflow of T: it would be checked on load() or other interaction with T } void visit(V v) override { From f825c353e0093067a5b7857d5ba23a092c951bfa Mon Sep 17 00:00:00 2001 From: tolk-vm Date: Fri, 6 Jun 2025 16:07:18 +0700 Subject: [PATCH 328/388] [Tolk] Auto-detect and inline functions at AST level Instead of generating PROCINLINE to Fift, perform compile-time inlining when possible --- tolk-tester/tests/a-tests.tolk | 2 +- .../tests/allow-post-modification.tolk | 8 + tolk-tester/tests/asm-arg-order.tolk | 6 +- tolk-tester/tests/bit-operators.tolk | 1 + tolk-tester/tests/constants-tests.tolk | 2 +- tolk-tester/tests/generics-1.tolk | 6 +- tolk-tester/tests/generics-2.tolk | 2 + tolk-tester/tests/generics-3.tolk | 4 +- tolk-tester/tests/generics-4.tolk | 3 + tolk-tester/tests/if-else-tests.tolk | 41 ++ tolk-tester/tests/indexed-access.tolk | 1 + tolk-tester/tests/inline-tests.tolk | 612 +++++++++++++++--- tolk-tester/tests/intN-tests.tolk | 1 + .../tests/invalid-semantics/err-4188.tolk | 13 + tolk-tester/tests/match-by-expr-tests.tolk | 52 ++ tolk-tester/tests/methods-tests.tolk | 4 + tolk-tester/tests/mutate-methods.tolk | 5 + tolk-tester/tests/no-spaces.tolk | 1 + tolk-tester/tests/nullable-tensors.tolk | 5 + tolk-tester/tests/op-priority.tolk | 3 + tolk-tester/tests/pack-unpack-2.tolk | 3 +- tolk-tester/tests/pack-unpack-3.tolk | 3 +- tolk-tester/tests/pack-unpack-4.tolk | 1 + tolk-tester/tests/parse-address.tolk | 1 + .../tests/remove-unused-functions.tolk | 1 + tolk-tester/tests/self-keyword.tolk | 4 + tolk-tester/tests/send-msg-1.tolk | 24 +- tolk-tester/tests/send-msg-2.tolk | 3 + tolk-tester/tests/send-msg-3.tolk | 5 + tolk-tester/tests/smart-cast-tests.tolk | 1 + tolk-tester/tests/some-tests-2.tolk | 2 +- tolk-tester/tests/some-tests-3.tolk | 1 + tolk-tester/tests/strings-tests.tolk | 2 +- tolk-tester/tests/struct-tests.tolk | 9 +- tolk-tester/tests/try-catch-tests.tolk | 10 +- tolk-tester/tests/union-types-tests.tolk | 1 + tolk/CMakeLists.txt | 1 + tolk/ast-from-tokens.cpp | 11 +- tolk/ast-replicator.h | 3 +- tolk/ast.cpp | 3 + tolk/ast.h | 6 +- tolk/builtins.cpp | 7 +- tolk/codegen.cpp | 2 +- tolk/fwd-declarations.h | 8 + tolk/generics-helpers.cpp | 2 +- tolk/pipe-ast-to-legacy.cpp | 126 +++- tolk/pipe-check-rvalue-lvalue.cpp | 21 +- tolk/pipe-detect-inline-in-place.cpp | 299 +++++++++ tolk/pipe-find-unused-symbols.cpp | 2 +- tolk/pipe-generate-fif-output.cpp | 29 +- tolk/pipe-register-symbols.cpp | 2 +- tolk/pipeline.h | 1 + tolk/symtable.cpp | 12 + tolk/symtable.h | 17 +- tolk/tolk.cpp | 1 + tolk/tolk.h | 4 +- 56 files changed, 1236 insertions(+), 164 deletions(-) create mode 100644 tolk-tester/tests/invalid-semantics/err-4188.tolk create mode 100644 tolk/pipe-detect-inline-in-place.cpp diff --git a/tolk-tester/tests/a-tests.tolk b/tolk-tester/tests/a-tests.tolk index 077303d64..a122ca39d 100644 --- a/tolk-tester/tests/a-tests.tolk +++ b/tolk-tester/tests/a-tests.tolk @@ -79,5 +79,5 @@ fun main(): int { method_id | in | out @testcase | 0 | | 31415926535897932384626433832795028841971693993751058209749445923078164 -@code_hash 84337043972311674339187056298873613816389434478842780265748859098303774481976 +@code_hash 103119459254804391561326789266022754176632268076661379738543746273668564328534 */ diff --git a/tolk-tester/tests/allow-post-modification.tolk b/tolk-tester/tests/allow-post-modification.tolk index 6d3abf057..0f8603cc6 100644 --- a/tolk-tester/tests/allow-post-modification.tolk +++ b/tolk-tester/tests/allow-post-modification.tolk @@ -1,18 +1,24 @@ fun unsafe_tuple(x: X): tuple asm "NOP"; +@noinline fun inc(x: int, y: int): (int, int) { return (x + y, y * 10); } +@noinline fun int.`~inc`(mutate self, y: int): int { val (newX, newY) = inc(self, y); self = newX; return newY; } +@noinline fun eq(v: X): X { return v; } +@noinline fun eq2(v: (int, int)) { return v; } +@noinline fun mul2(mutate dest: int, v: int): int { dest = v*2; return dest; } +@noinline fun (int, int).multens(mutate self, v: (int, int)): (int, int) { var (f, s) = self; var (m1, m2) = v; self = (f*m1, s*m2); return self; } @method_id(11) @@ -38,6 +44,7 @@ fun test_tuple_assign(x: int): (int, int, int, int, int, int, int) { return (x1, x2, x3, x4, x5, x6, x7); } +@noinline fun foo1(x1: int, x2: int, x3: int, x4: int, x5: int, x6: int, x7: int): (int, int, int, int, int, int, int) { return (x1, x2, x3, x4, x5, x6, x7); } @@ -47,6 +54,7 @@ fun test_call_1(x: int): (int, int, int, int, int, int, int) { return foo1(x, x.`~inc`(x / 20), x, x = x * 2, x, x += 1, x); } +@noinline fun foo2(x1: int, x2: int, x3456: (int, int, int, int), x7: int, ): (int, int, int, int, int, int, int) { var (x3: int, x4: int, x5: int, x6: int) = x3456; return (x1, x2, x3, x4, x5, x6, x7); diff --git a/tolk-tester/tests/asm-arg-order.tolk b/tolk-tester/tests/asm-arg-order.tolk index cb7b6f1e3..b8622060a 100644 --- a/tolk-tester/tests/asm-arg-order.tolk +++ b/tolk-tester/tests/asm-arg-order.tolk @@ -129,11 +129,15 @@ fun int.plus1TimesB(self, b: int): int asm(b self) "1 ADDCONST MUL"; @pure +@noinline fun get2Pure() { return 2; } @pure +@noinline fun get10Pure() { return 10; } +@noinline fun get2Impure() { return 2; } +@noinline fun get10Impure() { return 10; } global g2: int; @@ -272,5 +276,5 @@ fun main() { }> """ -@code_hash 93297578247504549901423325446957614083213743835412767087757830691482809332788 +@code_hash 88343225778124778743440967192570181022747908733235053033508198051414837052492 */ diff --git a/tolk-tester/tests/bit-operators.tolk b/tolk-tester/tests/bit-operators.tolk index 9ae04090c..692275bf6 100644 --- a/tolk-tester/tests/bit-operators.tolk +++ b/tolk-tester/tests/bit-operators.tolk @@ -88,6 +88,7 @@ fun testBoolNegateOptimized(x: bool) { return (x, !x, !!x, !!!x, !!!!true); } +@noinline fun eqX(x: bool) { return x; } @method_id(19) diff --git a/tolk-tester/tests/constants-tests.tolk b/tolk-tester/tests/constants-tests.tolk index 838a7dae3..607cccf0a 100644 --- a/tolk-tester/tests/constants-tests.tolk +++ b/tolk-tester/tests/constants-tests.tolk @@ -127,5 +127,5 @@ fun main() { @testcase | 105 | | -1 0 7 48 @testcase | 106 | | -1 0 0 -1 -@code_hash 85012002134196298607946339234948178079079623823648671175958399073436861460061 +@code_hash 39394999917297360167950120811797040756088634052063904555277102239104815314650 */ diff --git a/tolk-tester/tests/generics-1.tolk b/tolk-tester/tests/generics-1.tolk index 0984dcd51..39ddb2cdf 100644 --- a/tolk-tester/tests/generics-1.tolk +++ b/tolk-tester/tests/generics-1.tolk @@ -2,6 +2,7 @@ type MInt = int; type Tensor2Int = (int, int); type Tensor2IntN = (int, int)?; +@noinline fun eq1(value: X): X { return value; } fun eq2(value: X) { return value; } fun eq3(value: X): X { var cp: [X] = [eq1(value)]; var ((([v: X]))) = cp; return v; } @@ -240,7 +241,10 @@ fun main(x: int): (int, [Tup2Int]) { @fif_codegen DECLPROC eq1 @fif_codegen DECLPROC eq1 @fif_codegen DECLPROC eq1>> -@fif_codegen DECLPROC getTwo + +// was inlined +@fif_codegen_avoid DECLPROC getTwo +@fif_codegen_avoid getTwo @fif_codegen_avoid DECLPROC eq1 @fif_codegen_avoid DECLPROC eq2 diff --git a/tolk-tester/tests/generics-2.tolk b/tolk-tester/tests/generics-2.tolk index 2ee2851ae..65d76cebf 100644 --- a/tolk-tester/tests/generics-2.tolk +++ b/tolk-tester/tests/generics-2.tolk @@ -61,6 +61,7 @@ fun test2(c: Wrapper) { } @pure +@noinline fun getWrappervalue1(c: Wrapper) { return c.value; } @@ -71,6 +72,7 @@ fun Wrapper.getWrappervalue1(self) { } @pure +@noinline fun getWrappervalue2(c: T) { return c.value; } diff --git a/tolk-tester/tests/generics-3.tolk b/tolk-tester/tests/generics-3.tolk index a6fc74885..691b87835 100644 --- a/tolk-tester/tests/generics-3.tolk +++ b/tolk-tester/tests/generics-3.tolk @@ -138,8 +138,8 @@ fun main() { """ test3 PROC:<{ 10 PUSHINT // r.USlot1=10 - -1 PUSHINT // r.USlot1=10 '11=-1 - FALSE // r.USlot1=10 '11=-1 '13 + -1 PUSHINT // r.USlot1=10 '22=-1 + FALSE // r.USlot1=10 '22=-1 '24 }> """ */ diff --git a/tolk-tester/tests/generics-4.tolk b/tolk-tester/tests/generics-4.tolk index 70c70818d..2869ad1f5 100644 --- a/tolk-tester/tests/generics-4.tolk +++ b/tolk-tester/tests/generics-4.tolk @@ -1,4 +1,6 @@ +@noinline fun eqUnusedT(v: int) { return v; } +@noinline fun eqUnusedU(v: T) { if (v is int123) { __expect_type(v as U, "never"); } return v; } fun dup(x: T1, y: T2): (T1, T2) { return (x, y) } @@ -45,6 +47,7 @@ fun createParameters(bounce: bool, body: TBody, data return { bounce, body, init: { value: ton("0"), data } }; } +@noinline fun mySend(p: Parameters): int { var total = 0; if (p.bounce) { diff --git a/tolk-tester/tests/if-else-tests.tolk b/tolk-tester/tests/if-else-tests.tolk index 1656519af..d34dc178b 100644 --- a/tolk-tester/tests/if-else-tests.tolk +++ b/tolk-tester/tests/if-else-tests.tolk @@ -102,20 +102,24 @@ fun test8(op: int) { return (withNoElse(op), withElse(op), withMatch(op)); } +@noinline fun demo9_1(x: int) { if (x > 0) { t.push(123); } else { t.push(456) } } +@noinline fun demo9_2(x: int) { if (x > 0) { t.push(123); } } +@noinline fun demo9_3(x: int) { if (x > 0) { t.push(123); } return; } +@noinline fun demo9_4(x: int) { if (x != -100) { if (x > 0) { t.push(123); } @@ -130,6 +134,7 @@ fun test9(x: int) { return t; } +@noinline fun demo10_1(x: int) { match(x) { 0 => t.push(123), @@ -137,6 +142,7 @@ fun demo10_1(x: int) { } } +@noinline fun demo10_2(x: int) { if (x != -100) { match (x) { @@ -155,6 +161,7 @@ fun test10(x: int) { return t; } +@noinline fun demo_neg_11_1(mutate x: int) { match (x) { -1 => { x = 0; } @@ -182,6 +189,38 @@ fun test11(x: int) { return (x, t); } +fun demo12_1(x: int) { + __expect_inline(true); + if (x > 0) { t.push(123); } + else { t.push(456) } +} + +fun demo12_2(x: int) { + __expect_inline(true); + if (x > 0) { t.push(123); } +} + +fun demo12_3(x: int) { + __expect_inline(true); + if (x > 0) { t.push(123); } + return; +} + +fun demo12_4(x: int) { + __expect_inline(false); + if (x != -100) { + if (x > 0) { t.push(123); } + return; + } +} + +@method_id(112) +fun test12(x: int) { + t = createEmptyTuple(); + demo12_1(x); demo12_2(x); demo12_3(x); demo12_4(x); + return t; +} + fun main() { } @@ -212,6 +251,8 @@ fun main() { @testcase | 110 | 0 | [ 123 123 ] @testcase | 110 | 5 | [ 456 ] @testcase | 111 | 5 | 1 [ 456 100 ] +@testcase | 112 | 10 | [ 123 123 123 123 ] +@testcase | 112 | -10 | [ 456 ] @fif_codegen """ diff --git a/tolk-tester/tests/indexed-access.tolk b/tolk-tester/tests/indexed-access.tolk index 113124dd5..8cd4a74d1 100644 --- a/tolk-tester/tests/indexed-access.tolk +++ b/tolk-tester/tests/indexed-access.tolk @@ -174,6 +174,7 @@ fun test112(f: int, s: int) { } @pure +@noinline fun getConstTuple(): Tup2Int { return [1,2]; } diff --git a/tolk-tester/tests/inline-tests.tolk b/tolk-tester/tests/inline-tests.tolk index 13add753e..0645546a6 100644 --- a/tolk-tester/tests/inline-tests.tolk +++ b/tolk-tester/tests/inline-tests.tolk @@ -1,140 +1,560 @@ fun foo1(x: int): int { - if (x == 1) { - return 1; - } - return 2; + __expect_inline(false); // returns in the middle, leave it for Fift + if (x == 1) { + return 1; + } + return 2; } @inline fun foo2(x: int): int { - if (x == 1) { - return 11; - } - return 22; + __expect_inline(false); // returns in the middle, leave it for Fift + if (x == 1) { + return 11; + } + return 22; } @inline_ref fun foo3(x: int): int { - if (x == 1) { - return 111; - } - return 222; + if (x == 1) { + return 111; + } + return 222; +} + +@inline_ref +fun foo_light_but_ref() { + __expect_inline(false); + return 0; +} + +@noinline +fun foo_light_but_no() { + __expect_inline(false); + return 0; } @method_id(101) fun test1(x: int): (int, int, int) { - return (foo1(x)+1, foo2(x)+1, foo3(x)+1); + __expect_inline(false); // @method_id + return (foo1(x) + 1, foo2(x) + 1, foo3(x) + 1 + foo_light_but_ref() + foo_light_but_no()); } global g: int; @inline fun foo_repeat() { - g = 1; - repeat(5) { - g *= 2; - } + __expect_inline(true); + g = 1; + repeat (5) { + g *= 2; + } } @inline fun foo_until(): int { - g = 1; - var i: int = 0; - do { - g *= 2; - i += 1; - } while (i < 8); - return i; + __expect_inline(true); + g = 1; + var i: int = 0; + do { + g *= 2; + i += 1; + } while (i < 8); + return i; } @inline fun foo_while(): int { - g = 1; - var i: int = 0; - while (i < 10) { - g *= 2; - i += 1; - } - return i; + __expect_inline(true); + g = 1; + var i: int = 0; + while (i < 10) { + g *= 2; + i += 1; + } + return i; } @method_id(102) fun test2() { - foo_repeat(); - var x: int = g; - foo_until(); - var y: int = g; - foo_while(); - var z: int = g; - return (x, y, z); + foo_repeat(); + var x: int = g; + foo_until(); + var y: int = g; + foo_while(); + var z: int = g; + return (x, y, z); } @inline fun foo_big(x: int): int { - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - x = x * 10 + 1; - return x; + __expect_inline(true); + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + return x; +} + +@noinline +fun foo_big_not_annotated_called_once(x: int): int { + __expect_inline(true); // called only once, inline + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + return x; +} + +fun foo_big_not_annotated_called_twice(x: int): int { + __expect_inline(false); // too big for in-place auto-inlining when called multiple times + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; x = x * 10 + 1; + return x; } @method_id(103) fun test3(x: int): int { - return foo_big(x) * 10 + 5; + return foo_big(x) * 10 + 5; +} + +@method_id(104) +fun test4(x: int) { + return (foo_big_not_annotated_called_once(x) * 10, foo_big_not_annotated_called_twice(x) * 10, foo_big_not_annotated_called_twice(x)); +} + +struct Storage { + owner: address; + lastCall: int64; + extra: Cell<(int8, uint256)>; +} + +fun Storage.load() { + __expect_inline(true); + return Storage.fromCell(contract.getData()); +} + +fun Storage.save(self) { + __expect_inline(true); + contract.setData(self.toCell()); +} + +@method_id(105) +fun test5() { + var st = Storage.load(); + st.lastCall = 0; + st.save(); +} + +@inline +fun factorial(x: int): int { + __expect_inline(false); // due to recursion + if (x <= 1) { + return 1; + } + return x * factorial(x - 1); +} + +fun recurse1(x: int): int { __expect_inline(false); return recurse2(x + 1); } +fun recurse2(x: int): int { __expect_inline(false); return recurse3(x + 1); } +fun recurse3(x: int): int { __expect_inline(false); return recurse1(x + 1); } + +global g10: int; + +fun startCellWith(x: int) { + __expect_inline(true); + return beginCell().storeUint(x, 32); +} + +fun startCellWithG10() { + __expect_inline(true); + return beginCell().storeUint(g10, 32); +} + +fun builder.myBitsCount(self): int { + __expect_inline(true); + return self.bitsCount(); +} + +fun builder.store32(mutate self, v: int32): self { + __expect_inline(true); + return self.storeUint(v, 32); +} + +fun sum(a: int, b: int) { + __expect_inline(true); + return a + b; +} + +fun eq(v: T): T { + __expect_inline(true); + return v; +} + +fun get20() { return 20; } + +struct Point { + x: int; + y: int; +} + +fun Point.getX(self) { + __expect_inline(true); + return self.x; +} + +fun Point.create(x: int, y: int): Point { + __expect_inline(true); + return {x, y} +} + +fun increment(mutate x: int) { + __expect_inline(true); + x += 1; +} + +@method_id(106) +fun test6() { + var p = Point.create(10, 20); + increment(mutate p.x); + p.y = p.getX(); + return p; +} + +@method_id(107) +fun test7() { + g10 = 10; + return startCellWithG10().store32(20).myBitsCount(); +} + +@method_id(108) +fun test8() { + var x = 0; + return (startCellWith(x = 5).store32(x += 20).store32(x), x); +} + +@method_id(109) +fun test9(x: int) { + var p1: Point = {x, y: 0}; + var p2: Point = {x: 0, y: p1.getX()}; + var t = createEmptyTuple(); + t.pushPoint(p1); + t.pushPoint(p2); + return (t.popPoint(), t.popPoint(), t); +} + +fun tuple.pushPoint(mutate self, p: Point) { + __expect_inline(true); + var x = p.x; + self.push(x); + self.push(p.y); +} + +fun tuple.popPoint(mutate self): Point { + __expect_inline(true); + var y = self.pop(); + return {x: self.pop(), y}; +} + +@method_id(110) +fun test10() { + var (a, b, c) = usedIn10ButDeclaredBelow(5); + return (a, b, c.bitsCount()); +} + +global g11_1: int; +global g11_2: int; + +fun mutateGlobals(mutate v: int, inc11_2: int) { + __expect_inline(true); + v += 1; + g11_2 += inc11_2; +} + +@method_id(111) +fun test11() { + g11_1 = 0; + g11_2 = 0; + mutateGlobals(mutate g11_1, 5); // 1 5 + mutateGlobals(mutate g11_1, g11_2); // 2 10 + mutateGlobals(mutate g11_2, g11_1); // 2 11 + return (g11_1, g11_2); +} + +fun nested1(x: int) { + var y = x + 2; + x += 7; + return y; +} + +fun nested2(x: int) { + var y = x + 2; + var z = nested1(y); + y += z; + return y + z; +} + +@method_id(112) +fun test12(x: int) { + var r1 = nested2(10); + var r2 = nested2(x); + return (r1, r2); +} + +fun anotherMath(arg: int) { + __expect_inline(true); + var cp_x = arg; + increment(mutate arg); + return cp_x + arg; +} + +@method_id(113) +fun test13(p: Point) { + anotherMath(p.x); + var r = anotherMath(p.y); + return (p, r); +} + +fun evalPoint(p: Point) { + __expect_inline(true); + increment(mutate p.x); + p.y *= 2; + return max(p.x, p.y); +} + +@method_id(114) +fun test14(p2: Point) { + var p1: Point = { x: 0, y: 0 }; + var r1 = evalPoint(p1); + return (r1, p1, evalPoint(p2), p2); +} + +global t: tuple; + +fun logT(a: int) { + __expect_inline(true); + t.push(a); +} + +fun demoInlined1(x: int) { + if (x == -1) { + logT(10); + } else if (x == 0) { + logT(20); + } +} + +fun demoInlined2(x: int) { + __expect_inline(true); + match (x) { + -1 => logT(10), + else => logT(20), + } +} + +fun wrapperNotInlined(x: int?) { + if (x != null) { + demoInlined1(x); + return; + } + logT(100); +} + +fun someUsages(x: int?) { + __expect_inline(true); + wrapperNotInlined(x); + demoInlined2(x!); + return; +} + +@method_id(115) +fun test15(x: int?) { + t = createEmptyTuple(); + someUsages(x); + return t; +} + +struct (0x01) A16 { a: int8; } +struct (0x02) B16 {} +struct (0x03) C16 {} +type U16 = A16 | B16 | C16; + +fun check16(msg: U16) { + match (msg) { + A16 => { assert(msg.a > 0, 100); } + B16 => {} + C16 => {} + } +} + +fun wrap16(msg: U16, check: bool) { + if (check) { + check16(msg); + return; + } + throw 123; +} + +@method_id(116) +fun test16() { + wrap16(U16.fromSlice(stringHexToSlice("0170")), true); + return -1; } -fun main() {} + +fun main() { + // regardless of number of calls, it will be inlined, since it's lightweight in AST terms + Storage.load(); Storage.load(); Storage.load(); Storage.load(); Storage.load(); + Storage.load(); Storage.load(); Storage.load(); Storage.load(); Storage.load(); + Storage.load(); Storage.load(); Storage.load(); Storage.load(); Storage.load(); + Storage.load(); Storage.load(); Storage.load(); Storage.load(); Storage.load(); + Storage.load(); Storage.load(); Storage.load(); Storage.load(); Storage.load(); + usedIn10ButDeclaredBelow(10); +} + +fun usedIn10ButDeclaredBelow(x: int) { + __expect_inline(true); + var y = eq(x); + return (x, y, eq(startCellWith(x))); +} + + /** - method_id | in | out -@testcase | 101 | 1 | 2 12 112 -@testcase | 101 | 2 | 3 23 223 -@testcase | 102 | | 32 256 1024 -@testcase | 103 | 9 | 9111111111111111111111111111111111111111111111111115 + method_id | in | out +@testcase | 101 | 1 | 2 12 112 +@testcase | 101 | 2 | 3 23 223 +@testcase | 102 | | 32 256 1024 +@testcase | 103 | 9 | 911111111111111111111111111111111111111111111111115 +@testcase | 104 | 9 | 911111111111111111111111111110 911111111111111111111111111110 91111111111111111111111111111 +@testcase | 106 | | 11 11 +@testcase | 107 | | 64 +@testcase | 108 | | BC{0018000000050000001900000019} 25 +@testcase | 109 | 10 | 0 10 10 0 [] +@testcase | 110 | | 5 5 32 +@testcase | 111 | | 2 11 +@testcase | 112 | 10 | 40 40 +@testcase | 113 | 10 20 | 10 20 41 +@testcase | 114 | 10 20 | 1 0 0 40 10 20 +@testcase | 115 | -1 | [ 10 10 ] +@testcase | 115 | 1 | [ 20 ] +@testcase | 116 | | -1 + +@fif_codegen_avoid Storage.load +@fif_codegen_avoid Storage.save +@fif_codegen +""" + test5 PROC:<{ // + c4 PUSH // '7 + CTOS // s + LDMSGADDR // '11 s +""" + +@fif_codegen +""" + test6 PROC:<{ + 11 PUSHINT + DUP + }> +""" + +@fif_codegen +""" + test8 PROC:<{ // + 5 PUSHINT + NEWC + 32 STU // '3 + 25 PUSHINT // '3 '9 + SWAP // x self + 107374182425 PUSHINT + 64 STUR // x '3 + SWAP // '3 x + }> +""" + +@fif_codegen +""" + test9 PROC:<{ // x + 0 PUSHINT // p1.x p1.y=0 + s0 s1 PUSH2 // p1.x p1.y=0 p2.x=0 p2.y + NIL // x p1.y=0 p2.x=0 p2.y self + s0 s4 XCHG2 // p2.y p1.y=0 p2.x=0 self x + TPUSH // p2.y p1.y=0 p2.x=0 self + ROT // p2.y p2.x=0 self p1.y=0 + TPUSH // p2.y x=0 self + SWAP // p2.y self x=0 + TPUSH // p2.y self + SWAP // self p2.y + TPUSH // self + TPOP // self y + SWAP // y self + TPOP // '36 self '35 + SWAP // '36 '35 self + TPOP // '36 '35 self y + SWAP // '36 '35 y self + TPOP // '36 '35 '50 t '49 + s3 s4 XCHG // '35 '36 '50 t '49 + -ROT // '35 '36 '49 '50 t + }> +""" + +@fif_codegen +""" + test10 PROC:<{ + 5 PUSHINT // '3=5 + DUP // '3=5 y=5 + 5 PUSHINT + NEWC + 32 STU // a=5 b=5 c + BBITS // a=5 b=5 '18 + }> +""" + +@fif_codegen +""" + test12 PROC:<{ // x + 40 PUSHINT // x r1 + SWAP // r1 x + 2 ADDCONST // r1 y + ... +""" + +@fif_codegen +""" + wrapperNotInlined PROC:<{ // x + DUP // x x + ISNULL // x '1 + IFNOTJMP:<{ // x + DUP // x x + -1 EQINT // x '3 + IFJMP:<{ // x +""" + +@fif_codegen +""" + wrap16 PROC:<{ + IFJMP:<{ + 139 PUSHINT + OVER + EQUAL + IFJMP:<{ + DROP + 0 GTINT + 100 THROWIFNOT + }> + 140 PUSHINT + s2 POP + EQUAL + IFJMP:<{ + }> + }> + 123 THROW + }> +""" */ diff --git a/tolk-tester/tests/intN-tests.tolk b/tolk-tester/tests/intN-tests.tolk index 5b5d8563f..bf3981112 100644 --- a/tolk-tester/tests/intN-tests.tolk +++ b/tolk-tester/tests/intN-tests.tolk @@ -215,6 +215,7 @@ fun test14(firstComponent: int8?): (int, int16)? { return firstComponent! < 10 ? null : (firstComponent! as int, 2 as int16); } +@noinline fun assign0(mutate v: T) { v = 0; } fun main() { diff --git a/tolk-tester/tests/invalid-semantics/err-4188.tolk b/tolk-tester/tests/invalid-semantics/err-4188.tolk new file mode 100644 index 000000000..c908ed107 --- /dev/null +++ b/tolk-tester/tests/invalid-semantics/err-4188.tolk @@ -0,0 +1,13 @@ +fun recursive1(): int { + return recursive2(); +} + +fun recursive2(): int { + __expect_inline(true); + return recursive1(); +} + +/** +@compilation_should_fail +@stderr __expect_inline failed + */ diff --git a/tolk-tester/tests/match-by-expr-tests.tolk b/tolk-tester/tests/match-by-expr-tests.tolk index 559919353..ca3e21d83 100644 --- a/tolk-tester/tests/match-by-expr-tests.tolk +++ b/tolk-tester/tests/match-by-expr-tests.tolk @@ -22,6 +22,7 @@ fun test2() { } } +@noinline fun isGt10(x: int) { return x > 10; } @method_id(103) @@ -116,6 +117,26 @@ fun test11(x: bool) { } } +global g12: int; + +@noinline +fun helper12(x: int) { + // via codegen, check that `return` is added implicitly to every case, and `IFJMP` is produced + match (x) { + 1 => g12 = 1, + 2 => g12 = 2, + 3 => g12 = 3, + else => g12 = x, + } +} + +@method_id(112) +fun test12(x: int) { + g12 = 0; + helper12(x); + return g12 +} + type asdf = int; @@ -131,6 +152,7 @@ fun main() { } /** +@testcase | 0 | | 2 @testcase | 101 | 1 | 100 @testcase | 101 | 2 | 200 @testcase | 101 | 3 | 300 @@ -154,6 +176,8 @@ fun main() { @testcase | 110 | 50 | 100 @testcase | 111 | 0 | -1 @testcase | 111 | -1 | 0 +@testcase | 112 | 2 | 2 +@testcase | 112 | 20 | 20 @fif_codegen """ @@ -209,4 +233,32 @@ fun main() { }> """ +@fif_codegen +""" + helper12 PROC:<{ + DUP + 1 EQINT + IFJMP:<{ + DROP + 1 PUSHINT + g12 SETGLOB + }> + DUP + 2 EQINT + IFJMP:<{ + DROP + 2 PUSHINT + g12 SETGLOB + }> + DUP + 3 EQINT + IFJMP:<{ + DROP + 3 PUSHINT + g12 SETGLOB + }> + g12 SETGLOB + }> +""" + */ diff --git a/tolk-tester/tests/methods-tests.tolk b/tolk-tester/tests/methods-tests.tolk index a54fe9415..83118ef36 100644 --- a/tolk-tester/tests/methods-tests.tolk +++ b/tolk-tester/tests/methods-tests.tolk @@ -33,12 +33,14 @@ fun Pair.create0(): Pair { fun Pair.create0(): Pair { return { first: 0, second: 0 }; } +@noinline fun Pair.createFrom(first: U, second: V): Pair { return { first: first as A, second: second as B }; } fun Pair.is0(self) { return self.first == 0 && self.second == 0; } +@noinline fun Pair.compareWith(self, f: U, s: V) { return self.first == f && self.second == s; } @@ -243,9 +245,11 @@ struct CounterIncrement { inc_by: int; } struct CounterReset { initial_value: int; } type CounterMsg = CounterIncrement | CounterReset; +@noinline fun CounterReset.onInternalMessage(self) { throw 123; } +@noinline fun CounterMsg.onInternalMessage(self) { return match (self) { CounterIncrement => self.inc_by, diff --git a/tolk-tester/tests/mutate-methods.tolk b/tolk-tester/tests/mutate-methods.tolk index 18cb64404..a53b2c15c 100644 --- a/tolk-tester/tests/mutate-methods.tolk +++ b/tolk-tester/tests/mutate-methods.tolk @@ -1,17 +1,21 @@ +@noinline fun incrementInPlace(mutate x: int, byValue: int): void { x = x + byValue; } +@noinline fun int.incrementInPlace(mutate self, byValue: int): void { self = self + byValue; } +@noinline fun incrementTwoInPlace(mutate x: int, mutate y: int, byValue: int): int { x.incrementInPlace(byValue); y += byValue; return x + y; } +@noinline fun int.incrementTwoInPlace(mutate self, mutate y: int, byValue: int): int { self.incrementInPlace(byValue); y += byValue; @@ -40,6 +44,7 @@ fun testIncrement2() { } +@noinline fun load_next(mutate cs: slice): int { return cs.loadInt(32); } diff --git a/tolk-tester/tests/no-spaces.tolk b/tolk-tester/tests/no-spaces.tolk index 857b71e3b..016cf478a 100644 --- a/tolk-tester/tests/no-spaces.tolk +++ b/tolk-tester/tests/no-spaces.tolk @@ -37,6 +37,7 @@ global `some()var`:int; return [~-~~+-c3, ~+c3-~`c9`, -(-~+-c20-~c10+c3+~c38&39)]; } +@noinline fun add3(a: int, b: int, c: int) { return a+b+c; } @method_id(115) fun unary_const_check(): [int,int] { diff --git a/tolk-tester/tests/nullable-tensors.tolk b/tolk-tester/tests/nullable-tensors.tolk index d6934324e..56e111b82 100644 --- a/tolk-tester/tests/nullable-tensors.tolk +++ b/tolk-tester/tests/nullable-tensors.tolk @@ -9,14 +9,17 @@ fun sumOfNullableTensorComponents(t: (int, int)?): int { return t!.0 + t!.1; } +@noinline fun isTensorNull(t: (int, int)?) { return t == null; } +@noinline fun (int, int)?.isTensorNull(self) { return self == null; } +@noinline fun incrementNullableTensorComponents(mutate t: (int, int)?) { if (t != null) { t!.0 += 1; @@ -37,12 +40,14 @@ fun incrementTensorComponents(mutate t: (int, int)) { t.1 += 1; } +@noinline fun (int, int).incrementTensorComponents(mutate self): self { self.0 += 1; self.1 += 1; return self; } +@noinline fun assignFirstComponent(mutate t: (int, int), first: int) { t!.0 = first; } diff --git a/tolk-tester/tests/op-priority.tolk b/tolk-tester/tests/op-priority.tolk index 3a1dee21e..27b30c988 100644 --- a/tolk-tester/tests/op-priority.tolk +++ b/tolk-tester/tests/op-priority.tolk @@ -1,7 +1,10 @@ fun justTrue(): bool { return true; } +@noinline fun unary_minus_1(a: int, b: int, c: int): int{return -(a+b) *c;} +@noinline fun unary_minus_2(a: int, b: int, c: int): int{return(-(a+b))*c;} +@noinline fun unary_minus_3(a: int, b: int, c: int): int{return-((a+b) *c);} diff --git a/tolk-tester/tests/pack-unpack-2.tolk b/tolk-tester/tests/pack-unpack-2.tolk index 789ed664f..51432d0b4 100644 --- a/tolk-tester/tests/pack-unpack-2.tolk +++ b/tolk-tester/tests/pack-unpack-2.tolk @@ -41,11 +41,12 @@ fun assert_slice_is_44_and_ref45(s: slice) { s.assertEnd(); } -@inline +@inline_ref fun slice.appendRef(self, refSlice: slice): slice { return beginCell().storeSlice(self).storeRef(beginCell().storeSlice(refSlice).endCell()).endCell().beginParse(); } +@noinline fun run(input: TInputStruct, ans: slice) { repeat (2) { var s = input.toCell().beginParse(); diff --git a/tolk-tester/tests/pack-unpack-3.tolk b/tolk-tester/tests/pack-unpack-3.tolk index 2e7d6ab2b..310a3ed6c 100644 --- a/tolk-tester/tests/pack-unpack-3.tolk +++ b/tolk-tester/tests/pack-unpack-3.tolk @@ -19,7 +19,7 @@ fun slice.assertEqDeeply(self, rhs: slice): slice { return self; } -@inline +@inline_ref fun slice.appendRef(self, refSlice: slice): slice { return beginCell().storeSlice(self).storeRef(beginCell().storeSlice(refSlice).endCell()).endCell().beginParse(); } @@ -35,6 +35,7 @@ fun generateCell_44_with_ref45(): cell { } +@noinline fun run(input: TInputStruct, ans: slice) { repeat (2) { var s = input.toCell().beginParse(); diff --git a/tolk-tester/tests/pack-unpack-4.tolk b/tolk-tester/tests/pack-unpack-4.tolk index 992a0664a..d2bcfb618 100644 --- a/tolk-tester/tests/pack-unpack-4.tolk +++ b/tolk-tester/tests/pack-unpack-4.tolk @@ -10,6 +10,7 @@ fun slice.assertEqDeeply(self, rhs: slice): slice { } +@noinline fun run(input: TInputStruct, ans: slice) { repeat (2) { var s = input.toCell().beginParse(); diff --git a/tolk-tester/tests/parse-address.tolk b/tolk-tester/tests/parse-address.tolk index 0f8ad02d2..36fb2ec2b 100644 --- a/tolk-tester/tests/parse-address.tolk +++ b/tolk-tester/tests/parse-address.tolk @@ -25,6 +25,7 @@ fun address.createNone() { return createAddressNone() } +@noinline fun check0(addr: address) { assert (addr.getWorkchain() == 0) throw 111; } diff --git a/tolk-tester/tests/remove-unused-functions.tolk b/tolk-tester/tests/remove-unused-functions.tolk index 8e748ecf1..cd72fd08c 100644 --- a/tolk-tester/tests/remove-unused-functions.tolk +++ b/tolk-tester/tests/remove-unused-functions.tolk @@ -15,6 +15,7 @@ global used_gv: int; fun receiveGetter(): () -> int { return used_as_noncall2; } @pure +@noinline fun usedButOptimizedOut(x: int): int { return x + 2; } fun main(): (int, int, int) { diff --git a/tolk-tester/tests/self-keyword.tolk b/tolk-tester/tests/self-keyword.tolk index e504b5c48..e5941dc81 100644 --- a/tolk-tester/tests/self-keyword.tolk +++ b/tolk-tester/tests/self-keyword.tolk @@ -1,17 +1,21 @@ +@noinline fun int.incChained(mutate self): self { self = self + 1; return self; } +@noinline fun int.incChained2(mutate self): self { return self.incChained(); } +@noinline fun int.incChained3(mutate self): self { self.incChained(); return self; } +@noinline fun int.incChained4(mutate self): self { self.incChained(); return self; diff --git a/tolk-tester/tests/send-msg-1.tolk b/tolk-tester/tests/send-msg-1.tolk index 07f736e20..a7be4ec05 100644 --- a/tolk-tester/tests/send-msg-1.tolk +++ b/tolk-tester/tests/send-msg-1.tolk @@ -30,7 +30,7 @@ message$_ {X:Type} fun getMyAddressDev(): address asm "x{80194DC6438F99D3D9DBE151944925D90B2492954BF6B9C070FBFF2DDED5F30547D_} PUSHSLICE"; -@inline +@inline_ref fun calculateNftItemStateInitData(itemIndex: int): cell { return beginCell() .storeUint(itemIndex, 64) @@ -61,6 +61,7 @@ struct(0x12345678) MyBody { queryId: uint64; } +@noinline fun test1_manual() { return beginCell() .storeUint(0x18, 6) // bounce @@ -86,6 +87,7 @@ fun test1() { return b.hash(); } +@noinline fun test2_manual() { return beginCell() .storeUint(0x10, 6) // no bounce @@ -111,6 +113,7 @@ fun test2() { return b.hash(); } +@noinline fun test3_manual() { val body_ref = beginCell() .storeUint(0x12345678, 32) @@ -137,6 +140,7 @@ fun test3() { return b.hash(); } +@noinline fun test4_manual(bodyCell: cell, dest: address, value: coins) { return beginCell() .storeUint(0x18, 6) // bounce @@ -164,6 +168,7 @@ fun test4(value: coins) { return b.hash(); } +@noinline fun test5_manual() { var ec_dict = createEmptyDict(); ec_dict.iDictSet(32, 1, "ec1"); @@ -194,6 +199,7 @@ fun test5() { return b.hash(); } +@noinline fun test6_manual(value: coins, ec_dict: dict) { return beginCell() .storeUint(0x18, 6) // bounce @@ -228,6 +234,7 @@ struct MyNftBody { nftContent: cell; } +@noinline fun test7_manual(nftItemCode: cell, amount: coins) { var (itemIndex: int, nftContent: cell) = (10, beginCell().endCell()); val nftItemData = calculateNftItemStateInitData(itemIndex); @@ -268,6 +275,7 @@ struct(0x706c7567) RequestPaymentMessage { someDict: dict; } +@noinline fun test8_manual(destAddr: address, requestedAmount: coins) { return beginCell() .storeUint(0x18, 6) // bounce @@ -298,6 +306,7 @@ fun test8(requestedAmount: coins) { return b.hash(); } +@noinline fun test9_manual(bounceable: bool) { return beginCell() .storeUint(0b01, 2).storeBool(bounceable).storeUint(0b000, 3) @@ -324,6 +333,7 @@ fun test9(bounceable: bool) { type Body500Bits = (uint250, uint250); // this body guaranteely fits into cell +@noinline fun test10_manual(bd: Body500Bits) { return beginCell() .storeUint(0x10, 6) @@ -349,6 +359,7 @@ fun test10(bd: Body500Bits) { type Body750Bits = (uint250, uint250, uint250); // this body is auto-ref +@noinline fun test11_manual(bd: Body750Bits) { val bodyRef = beginCell().storeUint(bd.0,250).storeUint(bd.1,250).storeUint(bd.2,250).endCell(); return beginCell() @@ -372,6 +383,7 @@ fun test11(bd: Body750Bits) { return b.hash(); } +@noinline fun test12_manual(data32: uint32) { var init = ContractState { data: beginCell().storeUint(data32,32).endCell(), @@ -407,6 +419,7 @@ fun test12(data32: uint32) { return b.hash(); } +@noinline fun test13_manual() { return beginCell() .storeUint(0x18, 6) // bounce @@ -427,6 +440,7 @@ fun test13() { return b.hash(); } +@noinline fun test14_manual(stateInitCell: cell, amount: coins) { val nftAddress = calculateNftItemAddress(BASECHAIN, stateInitCell); @@ -460,6 +474,7 @@ struct(0x1234) Body15 { more: int32; } +@noinline fun test15_manual(stateInitCell: cell, bd: Body15) { val bodyRef = beginCell().storeUint(0x1234,16).storeUint(bd.tens.0,250).storeUint(bd.tens.1,250).storeUint(bd.tens.2,250).storeInt(bd.more,32).endCell(); val nftAddress = calculateNftItemAddress(MASTERCHAIN, stateInitCell); @@ -502,6 +517,7 @@ fun test15(tens0: int, tens1: int) { return b.hash(); } +@noinline fun test18_manual(dest: builder, queryId: uint64) { return beginCell() .storeUint(0x18, 6) // bounce @@ -527,6 +543,7 @@ fun test18(queryId: uint64) { return b.hash(); } +@noinline fun test19_manual(bd: uint64) { var destB = beginCell().storeUint(0, 2); return beginCell() @@ -551,6 +568,7 @@ fun test19(bd: uint64) { return b.hash(); } +@noinline fun test20_manual() { var bodyB = beginCell().storeUint(0x12345678, 32).storeUint(800, 64); return beginCell() @@ -580,6 +598,7 @@ struct(0x12345678) Body21 { payload: RemainingBitsAndRefs; } +@noinline fun test21_manual(bd: Body21) { return beginCell() .storeUint(0x10, 6) // no bounce @@ -605,6 +624,7 @@ fun test21(queryId: uint64) { return b.hash(); } +@noinline fun test22_manual(unsafeB: builder) { return beginCell() .storeUint(0x18, 6) // bounce @@ -635,6 +655,7 @@ struct Body23Unlimited { any2: RemainingBitsAndRefs; } +@noinline fun test23_manual(state32: uint32, body: Body23Unlimited) { var init = ContractState { data: beginCell().storeUint(state32,32).endCell(), @@ -678,6 +699,7 @@ fun test23(state32: uint32) { return b.hash(); } +@noinline fun test24_manual(addrHash: uint256) { return beginCell() .storeUint(0x18, 6) // bounce diff --git a/tolk-tester/tests/send-msg-2.tolk b/tolk-tester/tests/send-msg-2.tolk index b1e3af99c..1d11752db 100644 --- a/tolk-tester/tests/send-msg-2.tolk +++ b/tolk-tester/tests/send-msg-2.tolk @@ -104,6 +104,7 @@ fun calculateAddressInAnotherShard(pivotAddress: address, shardPrefixLen: uint5, .storeUint(prefixLess, 256 - shardPrefixLen); } +@noinline fun test1_manual(toAddress: address, masterMsg: cell, jettonWalletCode: cell) { var shardPrefix = getAddressShard(toAddress, SHARD_DEPTH); var stateInitCell = calculateJettonWalletStateInitWithShard(toAddress, getMyAddressDev(), jettonWalletCode); @@ -149,6 +150,7 @@ fun test1() { return b.hash() } +@noinline fun test2_manual(toAddress: address, masterMsg: cell, jettonWalletCode: cell) { var shardPrefix = getAddressShard(toAddress, SHARD_DEPTH); var stateInit = calculateJettonWalletStateInitWithShard(toAddress, getMyAddressDev(), jettonWalletCode); @@ -192,6 +194,7 @@ struct Test3Body { bigData: bits800; } +@noinline fun test3_manual(myCode: cell, myData: cell, msgBody: Test3Body) { var addrInShard = calculateAddressInAnotherShard(getMyAddressDev(), 20, myCode, myData); diff --git a/tolk-tester/tests/send-msg-3.tolk b/tolk-tester/tests/send-msg-3.tolk index 69afed4cd..07265ec96 100644 --- a/tolk-tester/tests/send-msg-3.tolk +++ b/tolk-tester/tests/send-msg-3.tolk @@ -16,6 +16,7 @@ struct(0x12345678) MyBody { queryId: uint64; } +@noinline fun test1_manual(topic: int) { return beginCell() .storeUint(0b1100, 4) // ext_out_msg_info$11 src:MsgAddressInt () @@ -42,6 +43,7 @@ fun test1(topic: int) { type Body750Bits = (uint250, uint250, uint250); // this body is auto-ref +@noinline fun test2_manual(bd: Body750Bits) { val bodyRef = beginCell().storeUint(bd.0,250).storeUint(bd.1,250).storeUint(bd.2,250).endCell(); return beginCell() @@ -81,6 +83,7 @@ fun test3(eventId: int) { return b.hash(); } +@noinline fun test4_manual(dest: address, body: cell) { return beginCell() .storeUint(0b1100, 4) // ext_out_msg_info$11 src:MsgAddressInt () @@ -121,6 +124,7 @@ fun test6() { return b.hash(); } +@noinline fun test7_manual(dest: builder, queryId: uint64) { return beginCell() .storeUint(0b1100, 4) // ext_out_msg_info$11 src:MsgAddressInt () @@ -174,6 +178,7 @@ fun test9(topic: int) { return b.hash(); } +@noinline fun test10_manual(topic: slice, queryIdRef: uint64) { return beginCell() .storeUint(0b1100, 4) // ext_out_msg_info$11 src:MsgAddressInt () diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk index 779095302..247431eb0 100644 --- a/tolk-tester/tests/smart-cast-tests.tolk +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -600,6 +600,7 @@ fun test54() { __expect_type(x3, "int?"); } +@noinline fun eq55(v: T) { return v; } fun test55() { diff --git a/tolk-tester/tests/some-tests-2.tolk b/tolk-tester/tests/some-tests-2.tolk index 592cf96ef..3568da56c 100644 --- a/tolk-tester/tests/some-tests-2.tolk +++ b/tolk-tester/tests/some-tests-2.tolk @@ -44,7 +44,7 @@ fun test89(last: int): (int, int, int, int) { return (t.get(0), t.get(t.size() - 1), t.first(), t.last()); } -@pure fun get10() { return 10; } +@pure @noinline fun get10() { return 10; } @method_id(91) fun touchCodegen2() { diff --git a/tolk-tester/tests/some-tests-3.tolk b/tolk-tester/tests/some-tests-3.tolk index 728b18d3f..8c53e33bf 100644 --- a/tolk-tester/tests/some-tests-3.tolk +++ b/tolk-tester/tests/some-tests-3.tolk @@ -15,6 +15,7 @@ fun main(cs: slice) { return (cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8)); } +@noinline fun f(cs: slice) { return (cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), diff --git a/tolk-tester/tests/strings-tests.tolk b/tolk-tester/tests/strings-tests.tolk index bbe458326..ab2facf8a 100644 --- a/tolk-tester/tests/strings-tests.tolk +++ b/tolk-tester/tests/strings-tests.tolk @@ -78,5 +78,5 @@ fun test1() { @testcase | 0 | | 0 @testcase | 101 | | [ 65 66 67 68 ] -@code_hash 61963905046482786665931036224265588524206004864815957521622605110240907698574 +@code_hash 15952581052628192793433894459656689089299796554419234117286198843831925928625 */ diff --git a/tolk-tester/tests/struct-tests.tolk b/tolk-tester/tests/struct-tests.tolk index c7f81e4c3..4a9a431e7 100644 --- a/tolk-tester/tests/struct-tests.tolk +++ b/tolk-tester/tests/struct-tests.tolk @@ -264,6 +264,7 @@ fun test17(x: JustInt?) { return (x, x!, x!.value, w1, w2); } +@noinline fun sumXY