Merge pull request #1320 from AntelopeIO/bump_to_4_0_3

[4.0] Bump Leap version to 4.0.3

.github/actions/parallel-ctest-containers/main.mjs

@@ -11,8 +11,10 @@ const error_log_paths = JSON.parse(core.getInput('error-log-paths', {required: t
 const log_tarball_prefix = core.getInput('log-tarball-prefix', {required: true});
 const tests_label = core.getInput('tests-label', {required: true});
 
+const repo_name = process.env.GITHUB_REPOSITORY.split('/')[1];
+
 try {
-   if(child_process.spawnSync("docker", ["run", "--name", "base", "-v", `${process.cwd()}/build.tar.zst:/build.tar.zst`, "--workdir", "/__w/leap/leap", container, "sh", "-c", "zstdcat /build.tar.zst | tar x"], {stdio:"inherit"}).status)
+   if(child_process.spawnSync("docker", ["run", "--name", "base", "-v", `${process.cwd()}/build.tar.zst:/build.tar.zst`, "--workdir", `/__w/${repo_name}/${repo_name}`, container, "sh", "-c", "zstdcat /build.tar.zst | tar x"], {stdio:"inherit"}).status)
       throw new Error("Failed to create base container");
    if(child_process.spawnSync("docker", ["commit", "base", "baseimage"], {stdio:"inherit"}).status)
       throw new Error("Failed to create base image");
@@ -45,13 +47,13 @@ try {
       let packer = tar.pack();
 
       extractor.on('entry', (header, stream, next) => {
-         if(!header.name.startsWith(`__w/leap/leap/build`)) {
+         if(!header.name.startsWith(`__w/${repo_name}/${repo_name}/build`)) {
             stream.on('end', () => next());
             stream.resume();
             return;
          }
 
-         header.name = header.name.substring(`__w/leap/leap/`.length);
+         header.name = header.name.substring(`__w/${repo_name}/${repo_name}/`.length);
          if(header.name !== "build/" && error_log_paths.filter(p => header.name.startsWith(p)).length === 0) {
             stream.on('end', () => next());
             stream.resume();

CMakeLists.txt

@@ -14,9 +14,9 @@ set( CMAKE_CXX_EXTENSIONS ON )
 set( CXX_STANDARD_REQUIRED ON)
 
 set(VERSION_MAJOR 4)
-set(VERSION_MINOR 1)
-set(VERSION_PATCH 0)
-set(VERSION_SUFFIX dev)
+set(VERSION_MINOR 0)
+set(VERSION_PATCH 3)
+#set(VERSION_SUFFIX rc3)
 
 if(VERSION_SUFFIX)
     set(VERSION_FULL "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}-${VERSION_SUFFIX}")
@@ -268,18 +268,8 @@ if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.21)
    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/lib/python3/dist-packages/TestHarness DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages COMPONENT dev EXCLUDE_FROM_ALL)
    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/share/leap_testing/bin DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing COMPONENT dev EXCLUDE_FROM_ALL)
 else()
-   # The following install(CODE ...) steps are necessary for `make dev-install` to work on cmake versions < 3.21.
-   # However, it can flag as an error if using `make package` instead of `make dev-install` for installation.
-   # This is due to those directories and symbolic links being created in the postinit script during the package install instead.
-
-   # Note If/when doing `make package`, it is not a true error if you see:
-   # Error creating directory "/<leap_install_dir>/lib/python3/dist-packages".
-   # CMake Error: failed to create symbolic link '/<leap_install_dir>/lib/python3/dist-packages/TestHarness': no such file or directory
-   # CMake Error: failed to create symbolic link '/<leap_install_dir>/share/leap_testing/bin': no such file or directory
-
-   install(CODE "execute_process( COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages)" COMPONENT dev EXCLUDE_FROM_ALL)
-   install(CODE "execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink ../../../share/leap_testing/tests/TestHarness ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages/TestHarness)" COMPONENT dev EXCLUDE_FROM_ALL)
-   install(CODE "execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink ../../bin ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing/bin)" COMPONENT dev EXCLUDE_FROM_ALL)
+   # The following install(SCRIPT ...) steps are necessary for `make dev-install` to work on cmake versions < 3.21.
+   install(SCRIPT ${CMAKE_CURRENT_BINARY_DIR}/scripts/install_testharness_symlinks.cmake COMPONENT dev EXCLUDE_FROM_ALL)
 
    # The `make package` installation of symlinks happens via the `postinst` script installed in cmake.package via the line below
 endif()

docs/01_nodeos/03_plugins/chain_plugin/index.md

@@ -120,7 +120,7 @@ Config Options for eosio::chain_plugin:
                                         applied to them (may specify multiple
                                         times)
   --read-mode arg (=head)               Database read mode ("head",
-                                        "irreversible").
+                                        "irreversible", "speculative").
                                         In "head" mode: database contains state
                                         changes up to the head block;
                                         transactions received by the node are
@@ -131,7 +131,14 @@ Config Options for eosio::chain_plugin:
                                         received via the P2P network are not
                                         relayed and transactions cannot be
                                         pushed via the chain API.
-
+                                        In "speculative" mode: (DEPRECATED:
+                                        head mode recommended) database
+                                        contains state changes by transactions
+                                        in the blockchain up to the head block
+                                        as well as some transactions not yet
+                                        included in the blockchain;
+                                        transactions received by the node are
+                                        relayed if valid.                                        
   --api-accept-transactions arg (=1)    Allow API transactions to be evaluated
                                         and relayed if valid.
   --validation-mode arg (=full)         Chain validation mode ("full" or

docs/01_nodeos/07_concepts/05_storage-and-read-modes.md

@@ -29,6 +29,7 @@ The `nodeos` service can be run in different "read" modes. These modes control h
 
 - `head`: this only includes the side effects of confirmed transactions, this mode processes unconfirmed transactions but does not include them.
 - `irreversible`: this mode also includes confirmed transactions only up to those included in the last irreversible block.
+- `speculative`: this includes the side effects of confirmed and unconfirmed transactions.
 
 A transaction is considered confirmed when a `nodeos` instance has received, processed, and written it to a block on the blockchain, i.e. it is in the head block or an earlier block.
 
@@ -44,6 +45,16 @@ When `nodeos` is configured to be in irreversible read mode, it will still track
 
 Clients such as `cleos` and the RPC API will see database state as of the current head block of the chain. It **will not** include changes made by transactions known to this node but not included in the chain, such as unconfirmed transactions.
 
+### Speculative Mode ( Deprecated )
+
+Clients such as `cleos` and the RPC API, will see database state as of the current head block plus changes made by all transactions known to this node but potentially not included in the chain, unconfirmed transactions for example.
+
+Speculative mode is low latency but fragile, there is no guarantee that the transactions reflected in the state will be included in the chain OR that they will reflected in the same order the state implies.
+
+This mode features the lowest latency, but is the least consistent.
+
+In speculative mode `nodeos` is able to execute transactions which have TaPoS (Transaction as Proof of Stake) pointing to any valid block in a fork considered to be the best fork by this node.
+
 ## How To Specify the Read Mode
 
 The mode in which `nodeos` is run can be specified using the `--read-mode` option from the `eosio::chain_plugin`.

libraries/chain/authorization_manager.cpp

@@ -559,8 +559,8 @@ namespace eosio { namespace chain {
 
       }
 
-      if( !allow_unused_keys  || check_but_dont_fail) {
-         EOS_ASSERT( checker.all_keys_used(), tx_irrelevant_sig,
+      if( !allow_unused_keys ) {
+         EOS_ASSERT( checker.all_keys_used() || check_but_dont_fail, tx_irrelevant_sig,
                      "transaction bears irrelevant signatures from these keys: ${keys}",
                      ("keys", checker.unused_keys()) );
       }

libraries/chain/block_log.cpp

@@ -496,19 +496,21 @@ namespace eosio { namespace chain {
 
       /// Would remove pre-existing block log and index, never write blocks into disk.
       struct empty_block_log final : block_log_impl {
+         uint32_t first_block_number = std::numeric_limits<uint32_t>::max();
+
          explicit empty_block_log(const bfs::path& log_dir) {
             fc::remove(log_dir / "blocks.log");
             fc::remove(log_dir / "blocks.index");
          }
 
-         uint32_t first_block_num() final { return head ? head->block_num() : 1; }
+         uint32_t first_block_num() final { return head ? head->block_num() : first_block_number; }
          void append(const signed_block_ptr& b, const block_id_type& id, const std::vector<char>& packed_block) final {
             update_head(b, id);
          }
 
          uint64_t get_block_pos(uint32_t block_num) final { return block_log::npos; }
          void reset(const genesis_state& gs, const signed_block_ptr& first_block) final { update_head(first_block); }
-         void reset(const chain_id_type& chain_id, uint32_t first_block_num) final {}
+         void reset(const chain_id_type& chain_id, uint32_t first_block_num) final { first_block_number = first_block_num; }
          void flush() final {}
 
          signed_block_ptr read_block_by_num(uint32_t block_num) final { return {}; };
@@ -1370,16 +1372,57 @@ namespace eosio { namespace chain {
    }
 
    // static
-   std::optional<genesis_state> block_log::extract_genesis_state(const fc::path& block_dir) {
-      boost::filesystem::path p(block_dir / "blocks.log");
-      for_each_file_in_dir_matches(block_dir, R"(blocks-1-\d+\.log)",
-                                   [&p](boost::filesystem::path log_path) { p = std::move(log_path); });
-      return block_log_data(p).get_genesis_state();
+   std::optional<block_log::chain_context> block_log::extract_chain_context( const fc::path& block_dir, const fc::path& retained_dir) {
+      boost::filesystem::path first_block_file;
+      if (!retained_dir.empty() && fc::exists(retained_dir)) {
+         for_each_file_in_dir_matches(retained_dir, R"(blocks-1-\d+\.log)",
+                                      [&](boost::filesystem::path log_path) {
+                                          first_block_file = std::move(log_path);
+                                      });
+      }
+
+      if (first_block_file.empty() && fc::exists(block_dir / "blocks.log")) {
+         first_block_file = block_dir / "blocks.log";
+      }
+
+      if (!first_block_file.empty()) {
+         return block_log_data(first_block_file).get_preamble().chain_context;
+      }
+      
+      if (!retained_dir.empty() && fc::exists(retained_dir)) {
+         const std::regex        my_filter(R"(blocks-\d+-\d+\.log)");
+         std::smatch             what;
+         bfs::directory_iterator end_itr; // Default ctor yields past-the-end
+         for (bfs::directory_iterator p(retained_dir); p != end_itr; ++p) {
+            // Skip if not a file
+            if (!bfs::is_regular_file(p->status()))
+               continue;
+            // skip if it does not match the pattern
+            if (!std::regex_match(p->path().filename().string(), what, my_filter))
+               continue;
+            return block_log_data(p->path()).chain_id();
+         }
+      }
+      return {};
    }
 
    // static
-   chain_id_type block_log::extract_chain_id(const fc::path& data_dir) {
-      return block_log_data(data_dir / "blocks.log").chain_id();
+   std::optional<genesis_state> block_log::extract_genesis_state(const fc::path& block_dir, const fc::path& retained_dir) {
+      auto context = extract_chain_context(block_dir, retained_dir);
+      if (!context || std::holds_alternative<chain_id_type>(*context))
+         return {};
+      return std::get<genesis_state>(*context);
+   }
+
+   // static
+   std::optional<chain_id_type> block_log::extract_chain_id(const fc::path& block_dir, const fc::path& retained_dir) {
+      auto context = extract_chain_context(block_dir, retained_dir);
+      if (!context)
+         return {};
+      return std::visit(overloaded{
+         [](const chain_id_type& id){ return id; },
+         [](const genesis_state& gs){ return gs.compute_chain_id(); }
+          } , *context);
    }
 
    // static
@@ -1530,10 +1573,14 @@ namespace eosio { namespace chain {
 
       block_log_bundle log_bundle(block_dir);
 
+      ilog("block log version= ${version}",("version", log_bundle.log_data.version()));
+      ilog("first block= ${first}",("first", log_bundle.log_data.first_block_num()));
+      ilog("last block= ${last}",("last", log_bundle.log_data.last_block_num()));
+
       ilog("blocks.log and blocks.index agree on number of blocks");
 
       if (interval == 0) {
-         interval = std::max((log_bundle.log_index.num_blocks() + 7u) >> 3, 1u);
+         interval = std::max((log_bundle.log_index.num_blocks() + 7) >> 3, 1U);
       }
       uint32_t expected_block_num = log_bundle.log_data.first_block_num();
 

libraries/chain/controller.cpp

@@ -210,6 +210,12 @@ struct pending_state {
 };
 
 struct controller_impl {
+   enum class app_window_type {
+      write, // Only main thread is running; read-only threads are not running.
+             // All read-write and read-only tasks are sequentially executed.
+      read   // Main thread and read-only threads are running read-ony tasks in parallel.
+             // Read-write tasks are not being executed.
+   };
 
    // LLVM sets the new handler, we need to reset this to throw a bad_alloc exception so we can possibly exit cleanly
    // and not just abort.
@@ -246,11 +252,10 @@ struct controller_impl {
 #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED)
    thread_local static vm::wasm_allocator wasm_alloc; // a copy for main thread and each read-only thread
 #endif
-   // Ideally wasmif should be thread_local which must be a static.
-   // Unittests can create multiple controller objects (testers) at the same time,
-   // which overwrites the same static wasmif, is used for eosvmoc too.
    wasm_interface  wasmif;  // used by main thread and all threads for EOSVMOC
-   thread_local static std::unique_ptr<wasm_interface> wasmif_thread_local; // a copy for each read-only thread, used by eos-vm and eos-vm-jit
+   std::mutex threaded_wasmifs_mtx;
+   std::unordered_map<std::thread::id, std::unique_ptr<wasm_interface>> threaded_wasmifs; // one for each read-only thread, used by eos-vm and eos-vm-jit
+   app_window_type app_window = app_window_type::write;
 
    typedef pair<scope_name,action_name>                   handler_key;
    map< account_name, map<handler_key, apply_handler> >   apply_handlers;
@@ -265,9 +270,7 @@ struct controller_impl {
          prev = fork_db.root();
       }
 
-      if ( read_mode == db_read_mode::HEAD ) {
-         EOS_ASSERT( head->block, block_validate_exception, "attempting to pop a block that was sparsely loaded from a snapshot");
-      }
+      EOS_ASSERT( head->block, block_validate_exception, "attempting to pop a block that was sparsely loaded from a snapshot");
 
       head = prev;
 
@@ -337,7 +340,12 @@ struct controller_impl {
       set_activation_handler<builtin_protocol_feature_t::crypto_primitives>();
 
       self.irreversible_block.connect([this](const block_state_ptr& bsp) {
-         get_wasm_interface().current_lib(bsp->block_num);
+         // producer_plugin has already asserted irreversible_block signal is
+         // called in write window
+         wasmif.current_lib(bsp->block_num);
+         for (auto& w: threaded_wasmifs) {
+            w.second->current_lib(bsp->block_num);
+         }
       });
 
 
@@ -415,7 +423,8 @@ struct controller_impl {
          EOS_ASSERT( root_id == log_head_id, fork_database_exception, "fork database root does not match block log head" );
       } else {
          EOS_ASSERT( fork_db.root()->block_num == lib_num, fork_database_exception,
-                     "empty block log expects the first appended block to build off a block that is not the fork database root. root block number: ${block_num}, lib: ${lib_num}", ("block_num", fork_db.root()->block_num) ("lib_num", lib_num) );
+                     "The first block ${lib_num} when starting with an empty block log should be the block after fork database root ${bn}.",
+                     ("lib_num", lib_num)("bn", fork_db.root()->block_num) );
       }
 
       const auto fork_head = fork_db_head();
@@ -497,9 +506,10 @@ struct controller_impl {
 
    void replay(std::function<bool()> check_shutdown) {
       auto blog_head = blog.head();
-      if( !blog_head && !fork_db.root() ) {
+      if( !fork_db.root() ) {
          fork_db.reset( *head );
-         return;
+         if (!blog_head)
+            return;
       }
 
       replaying = true;
@@ -549,7 +559,10 @@ struct controller_impl {
          ilog( "no irreversible blocks need to be replayed" );
       }
 
-      if( !except_ptr && !check_shutdown() && fork_db.head() ) {
+      if (snapshot_head_block != 0 && !blog_head) {
+         // loading from snapshot without a block log so fork_db can't be considered valid
+         fork_db.reset( *head );
+      } else if( !except_ptr && !check_shutdown() && fork_db.head() ) {
          auto head_block_num = head->block_num;
          auto branch = fork_db.fetch_branch( fork_db.head()->id );
          int rev = 0;
@@ -580,19 +593,21 @@ struct controller_impl {
    void startup(std::function<void()> shutdown, std::function<bool()> check_shutdown, const snapshot_reader_ptr& snapshot) {
       EOS_ASSERT( snapshot, snapshot_exception, "No snapshot reader provided" );
       this->shutdown = shutdown;
-      ilog( "Starting initialization from snapshot, this may take a significant amount of time" );
       try {
          snapshot->validate();
          if( auto blog_head = blog.head() ) {
+            ilog( "Starting initialization from snapshot and block log ${b}-${e}, this may take a significant amount of time",
+                  ("b", blog.first_block_num())("e", blog_head->block_num()) );
             read_from_snapshot( snapshot, blog.first_block_num(), blog_head->block_num() );
          } else {
+            ilog( "Starting initialization from snapshot and no block log, this may take a significant amount of time" );
             read_from_snapshot( snapshot, 0, std::numeric_limits<uint32_t>::max() );
-            const uint32_t lib_num = head->block_num;
-            EOS_ASSERT( lib_num > 0, snapshot_exception,
+            EOS_ASSERT( head->block_num > 0, snapshot_exception,
                         "Snapshot indicates controller head at block number 0, but that is not allowed. "
                         "Snapshot is invalid." );
-            blog.reset( chain_id, lib_num + 1 );
+            blog.reset( chain_id, head->block_num + 1 );
          }
+         ilog( "Snapshot loaded, lib: ${lib}", ("lib", head->block_num) );
 
          init(check_shutdown);
          ilog( "Finished initialization from snapshot" );
@@ -1624,7 +1639,7 @@ struct controller_impl {
             if ( trx->is_transient() ) {
                // remove trx from pending block by not canceling 'restore'
                trx_context.undo(); // this will happen automatically in destructor, but make it more explicit
-            } else if ( pending->_block_status == controller::block_status::ephemeral ) {
+            } else if ( read_mode != db_read_mode::SPECULATIVE && pending->_block_status == controller::block_status::ephemeral ) {
                // An ephemeral block will never become a full block, but on a producer node the trxs should be saved
                // in the un-applied transaction queue for execution during block production. For a non-producer node
                // save the trxs in the un-applied transaction queue for use during block validation to skip signature
@@ -2680,28 +2695,51 @@ struct controller_impl {
    // only called from non-main threads (read-only trx execution threads)
    // when producer_plugin starts them
    void init_thread_local_data() {
-      EOS_ASSERT( !is_main_thread(), misc_exception, "init_thread_local_data called on the main thread");
+      EOS_ASSERT( !is_on_main_thread(), misc_exception, "init_thread_local_data called on the main thread");
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
       if ( is_eos_vm_oc_enabled() )
          // EOSVMOC needs further initialization of its thread local data
          wasmif.init_thread_local_data();
       else
 #endif
+      {
+         std::lock_guard g(threaded_wasmifs_mtx);
          // Non-EOSVMOC needs a wasmif per thread
-         wasmif_thread_local = std::make_unique<wasm_interface>( conf.wasm_runtime, conf.eosvmoc_tierup, db, conf.state_dir, conf.eosvmoc_config, !conf.profile_accounts.empty());
+         threaded_wasmifs[std::this_thread::get_id()]  = std::make_unique<wasm_interface>( conf.wasm_runtime, conf.eosvmoc_tierup, db, conf.state_dir, conf.eosvmoc_config, !conf.profile_accounts.empty());
+      }
    }
 
-   bool is_main_thread() { return main_thread_id == std::this_thread::get_id(); };
+   bool is_on_main_thread() { return main_thread_id == std::this_thread::get_id(); };
+
+   void set_to_write_window() {
+      app_window = app_window_type::write;
+   }
+   void set_to_read_window() {
+      app_window = app_window_type::read;
+   }
+   bool is_write_window() const {
+      return app_window == app_window_type::write;
+   }
 
    wasm_interface& get_wasm_interface() {
-      if ( is_main_thread()
+      if ( is_on_main_thread()
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
           || is_eos_vm_oc_enabled()
 #endif
          )
          return wasmif;
       else
-         return *wasmif_thread_local;
+         return *threaded_wasmifs[std::this_thread::get_id()];
+   }
+
+   void code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num) {
+      // The caller of this function apply_eosio_setcode has already asserted that
+      // the transaction is not a read-only trx, which implies we are
+      // in write window. Safe to call threaded_wasmifs's code_block_num_last_used
+      wasmif.code_block_num_last_used(code_hash, vm_type, vm_version, block_num);
+      for (auto& w: threaded_wasmifs) {
+         w.second->code_block_num_last_used(code_hash, vm_type, vm_version, block_num);
+      }
    }
 
    block_state_ptr fork_db_head() const;
@@ -2711,7 +2749,6 @@ thread_local platform_timer controller_impl::timer;
 #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED)
 thread_local eosio::vm::wasm_allocator controller_impl::wasm_alloc;
 #endif
-thread_local std::unique_ptr<wasm_interface> controller_impl::wasmif_thread_local;
 
 const resource_limits_manager&   controller::get_resource_limits_manager()const
 {
@@ -3686,6 +3723,20 @@ void controller::init_thread_local_data() {
    my->init_thread_local_data();
 }
 
+void controller::set_to_write_window() {
+   my->set_to_write_window();
+}
+void controller::set_to_read_window() {
+   my->set_to_read_window();
+}
+bool controller::is_write_window() const {
+   return my->is_write_window();
+}
+
+void controller::code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num) {
+   return my->code_block_num_last_used(code_hash, vm_type, vm_version, block_num);
+}
+
 /// Protocol feature activation handlers:
 
 template<>

libraries/chain/eosio_contract.cpp

@@ -160,7 +160,7 @@ void apply_eosio_setcode(apply_context& context) {
       old_size  = (int64_t)old_code_entry.code.size() * config::setcode_ram_bytes_multiplier;
       if( old_code_entry.code_ref_count == 1 ) {
          db.remove(old_code_entry);
-         context.control.get_wasm_interface().code_block_num_last_used(account.code_hash, account.vm_type, account.vm_version, context.control.head_block_num() + 1);
+         context.control.code_block_num_last_used(account.code_hash, account.vm_type, account.vm_version, context.control.head_block_num() + 1);
       } else {
          db.modify(old_code_entry, [](code_object& o) {
             --o.code_ref_count;

libraries/chain/include/eosio/chain/block_log.hpp

@@ -82,9 +82,12 @@ namespace eosio { namespace chain {
 
          static fc::path repair_log( const fc::path& data_dir, uint32_t truncate_at_block = 0, const char* reversible_block_dir_name="" );
 
-         static std::optional<genesis_state> extract_genesis_state( const fc::path& data_dir );
+         using chain_context = std::variant<genesis_state, chain_id_type>;
+         static std::optional<chain_context> extract_chain_context( const fc::path& data_dir, const fc::path& retained_dir);
 
-         static chain_id_type extract_chain_id( const fc::path& data_dir );
+         static std::optional<genesis_state> extract_genesis_state( const fc::path& data_dir, const fc::path& retained_dir = fc::path{});
+
+         static std::optional<chain_id_type> extract_chain_id( const fc::path& data_dir, const fc::path& retained_dir = fc::path{});
 
          static void construct_index(const fc::path& block_file_name, const fc::path& index_file_name);
 

libraries/chain/include/eosio/chain/controller.hpp

@@ -49,7 +49,8 @@ namespace eosio { namespace chain {
 
    enum class db_read_mode {
       HEAD,
-      IRREVERSIBLE
+      IRREVERSIBLE,
+      SPECULATIVE
    };
 
    enum class validation_mode {
@@ -373,6 +374,10 @@ namespace eosio { namespace chain {
       void set_db_read_only_mode();
       void unset_db_read_only_mode();
       void init_thread_local_data();
+      void set_to_write_window();
+      void set_to_read_window();
+      bool is_write_window() const;
+      void code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num);
 
       private:
          friend class apply_context;

libraries/chain/include/eosio/chain/exceptions.hpp

@@ -361,6 +361,10 @@ namespace eosio { namespace chain {
                                     3080007, "Transaction exceeded the current greylisted account network usage limit" )
       FC_DECLARE_DERIVED_EXCEPTION( greylist_cpu_usage_exceeded, resource_exhausted_exception,
                                     3080008, "Transaction exceeded the current greylisted account CPU usage limit" )
+      FC_DECLARE_DERIVED_EXCEPTION( ro_trx_vm_oc_compile_temporary_failure, resource_exhausted_exception,
+                                    3080009, "Read-only transaction eos-vm-oc compile temporary failure" )
+      FC_DECLARE_DERIVED_EXCEPTION( ro_trx_vm_oc_compile_permanent_failure, resource_exhausted_exception,
+                                    3080010, "Read-only transaction eos-vm-oc compile permanent failure" )
 
       FC_DECLARE_DERIVED_EXCEPTION( leeway_deadline_exception, deadline_exception,
                                     3081001, "Transaction reached the deadline set due to leeway on account CPU limits" )

libraries/chain/include/eosio/chain/log_index.hpp

@@ -32,7 +32,7 @@ class log_index {
    bool is_open() const { return file_.is_open(); }
 
    uint64_t back() { return nth_block_position(num_blocks()-1); }
-   unsigned num_blocks() const { return num_blocks_; }
+   uint32_t num_blocks() const { return num_blocks_; }
    uint64_t nth_block_position(uint32_t n) {
       file_.seek(n*sizeof(uint64_t));
       uint64_t r;

libraries/chain/include/eosio/chain/thread_utils.hpp

@@ -37,9 +37,13 @@ namespace eosio { namespace chain {
 
       /// Spawn threads, can be re-started after stop().
       /// Assumes start()/stop() called from the same thread or externally protected.
+      /// Blocks until all threads are created and completed their init function, or an exception is thrown
+      ///  during thread startup or an init function. Exceptions thrown during these stages are rethrown from start()
+      ///  but some threads might still have been started. Calling stop() after such a failure is safe.
       /// @param num_threads is number of threads spawned
       /// @param on_except is the function to call if io_context throws an exception, is called from thread pool thread.
-      ///                  if an empty function then logs and rethrows exception on thread which will terminate.
+      ///                  if an empty function then logs and rethrows exception on thread which will terminate. Not called
+      ///                  for exceptions during the init function (such exceptions are rethrown from start())
       /// @param init is an optional function to call at startup to initialize any data.
       /// @throw assert_exception if already started and not stopped.
       void start( size_t num_threads, on_except_t on_except, init_t init = {} ) {
@@ -47,9 +51,25 @@ namespace eosio { namespace chain {
          _ioc_work.emplace( boost::asio::make_work_guard( _ioc ) );
          _ioc.restart();
          _thread_pool.reserve( num_threads );
-         for( size_t i = 0; i < num_threads; ++i ) {
-            _thread_pool.emplace_back( std::thread( &named_thread_pool::run_thread, this, i, on_except, init )  );
+
+         std::promise<void> start_complete;
+         std::atomic<uint32_t> threads_remaining = num_threads;
+         std::exception_ptr pending_exception;
+         std::mutex pending_exception_mutex;
+
+         try {
+            for( size_t i = 0; i < num_threads; ++i ) {
+               _thread_pool.emplace_back( std::thread( &named_thread_pool::run_thread, this, i, on_except, init, std::ref(start_complete),
+                                                       std::ref(threads_remaining), std::ref(pending_exception), std::ref(pending_exception_mutex) ) );
+            }
          }
+         catch( ... ) {
+            /// only an exception from std::thread's ctor should end up here. shut down all threads to ensure no
+            ///  potential access to the promise, atomic, etc above performed after throwing out of start
+            stop();
+            throw;
+         }
+         start_complete.get_future().get();
       }
 
       /// destroy work guard, stop io_context, join thread_pool
@@ -63,16 +83,42 @@ namespace eosio { namespace chain {
       }
 
    private:
-      void run_thread( size_t i, const on_except_t& on_except, const init_t& init ) {
-         std::string tn = boost::core::demangle(typeid(this).name());
-         auto offset = tn.rfind("::");
-         if (offset != std::string::npos)
-            tn.erase(0, offset+2);
-         tn = tn.substr(0, tn.find('>')) + "-" + std::to_string( i );
+      void run_thread( size_t i, const on_except_t& on_except, const init_t& init, std::promise<void>& start_complete,
+                       std::atomic<uint32_t>& threads_remaining, std::exception_ptr& pending_exception, std::mutex& pending_exception_mutex ) {
+
+         std::string tn;
+
+         auto decrement_remaining = [&]() {
+            if( !--threads_remaining ) {
+               if( pending_exception )
+                  start_complete.set_exception( pending_exception );
+               else
+                  start_complete.set_value();
+            }
+         };
+
+         try {
+            try {
+               tn = boost::core::demangle(typeid(this).name());
+               auto offset = tn.rfind("::");
+               if (offset != std::string::npos)
+                  tn.erase(0, offset+2);
+               tn = tn.substr(0, tn.find('>')) + "-" + std::to_string( i );
+               fc::set_os_thread_name( tn );
+               if ( init )
+                  init();
+            } FC_LOG_AND_RETHROW()
+         }
+         catch( ... ) {
+            std::lock_guard<std::mutex> l( pending_exception_mutex );
+            pending_exception = std::current_exception();
+            decrement_remaining();
+            return;
+         }
+
+         decrement_remaining();
+
          try {
-            fc::set_os_thread_name( tn );
-            if ( init )
-               init();
             _ioc.run();
          } catch( const fc::exception& e ) {
             if( on_except ) {

libraries/chain/include/eosio/chain/wasm_interface.hpp

@@ -63,9 +63,6 @@ namespace eosio { namespace chain {
          //Calls apply or error on a given code
          void apply(const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version, apply_context& context);
 
-         //Immediately exits currently running wasm. UB is called when no wasm running
-         void exit();
-
          //Returns true if the code is cached
          bool is_code_cached(const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) const;
 

libraries/chain/include/eosio/chain/wasm_interface_private.hpp

@@ -37,7 +37,6 @@ namespace eosio { namespace chain {
    struct wasm_interface_impl {
       struct wasm_cache_entry {
          digest_type                                          code_hash;
-         uint32_t                                             first_block_num_used;
          uint32_t                                             last_block_num_used;
          std::unique_ptr<wasm_instantiated_module_interface>  module;
          uint8_t                                              vm_type = 0;
@@ -109,24 +108,6 @@ namespace eosio { namespace chain {
          return it != wasm_instantiation_cache.end();
       }
 
-      std::vector<uint8_t> parse_initial_memory(const Module& module) {
-         std::vector<uint8_t> mem_image;
-
-         for(const DataSegment& data_segment : module.dataSegments) {
-            EOS_ASSERT(data_segment.baseOffset.type == InitializerExpression::Type::i32_const, wasm_exception, "");
-            EOS_ASSERT(module.memories.defs.size(), wasm_exception, "");
-            const U32 base_offset = data_segment.baseOffset.i32;
-            const Uptr memory_size = (module.memories.defs[0].type.size.min << IR::numBytesPerPageLog2);
-            if(base_offset >= memory_size || base_offset + data_segment.data.size() > memory_size)
-               FC_THROW_EXCEPTION(wasm_execution_error, "WASM data segment outside of valid memory range");
-            if(base_offset + data_segment.data.size() > mem_image.size())
-               mem_image.resize(base_offset + data_segment.data.size(), 0x00);
-            memcpy(mem_image.data() + base_offset, data_segment.data.data(), data_segment.data.size());
-         }
-
-         return mem_image;
-      }
-
       void code_block_num_last_used(const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version, const uint32_t& block_num) {
          wasm_cache_index::iterator it = wasm_instantiation_cache.find(boost::make_tuple(code_hash, vm_type, vm_version));
          if(it != wasm_instantiation_cache.end())
@@ -157,7 +138,6 @@ namespace eosio { namespace chain {
 
             it = wasm_instantiation_cache.emplace( wasm_interface_impl::wasm_cache_entry{
                                                       .code_hash = code_hash,
-                                                      .first_block_num_used = codeobject->first_block_used,
                                                       .last_block_num_used = UINT32_MAX,
                                                       .module = nullptr,
                                                       .vm_type = vm_type,
@@ -173,24 +153,8 @@ namespace eosio { namespace chain {
                trx_context.resume_billing_timer();
             });
             trx_context.pause_billing_timer();
-            IR::Module module;
-            std::vector<U8> bytes = {
-                (const U8*)codeobject->code.data(),
-                (const U8*)codeobject->code.data() + codeobject->code.size()};
-            try {
-               Serialization::MemoryInputStream stream((const U8*)bytes.data(),
-                                                       bytes.size());
-               WASM::scoped_skip_checks no_check;
-               WASM::serialize(stream, module);
-               module.userSections.clear();
-            } catch (const Serialization::FatalSerializationException& e) {
-               EOS_ASSERT(false, wasm_serialization_error, e.message.c_str());
-            } catch (const IR::ValidationException& e) {
-               EOS_ASSERT(false, wasm_serialization_error, e.message.c_str());
-            }
-
             wasm_instantiation_cache.modify(it, [&](auto& c) {
-               c.module = runtime_interface->instantiate_module((const char*)bytes.data(), bytes.size(), parse_initial_memory(module), code_hash, vm_type, vm_version);
+               c.module = runtime_interface->instantiate_module(codeobject->code.data(), codeobject->code.size(), code_hash, vm_type, vm_version);
             });
          }
          return it->module;
@@ -209,7 +173,6 @@ namespace eosio { namespace chain {
                   member<wasm_cache_entry, uint8_t,     &wasm_cache_entry::vm_version>
                >
             >,
-            ordered_non_unique<tag<by_first_block_num>, member<wasm_cache_entry, uint32_t, &wasm_cache_entry::first_block_num_used>>,
             ordered_non_unique<tag<by_last_block_num>, member<wasm_cache_entry, uint32_t, &wasm_cache_entry::last_block_num_used>>
          >
       > wasm_cache_index;

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc.hpp

@@ -30,10 +30,9 @@ class eosvmoc_runtime : public eosio::chain::wasm_runtime_interface {
    public:
       eosvmoc_runtime(const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db);
       ~eosvmoc_runtime();
-      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t> initial_memory,
+      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
                                                                              const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override;
 
-      void immediately_exit_currently_running_module() override;
       void init_thread_local_data() override;
 
       friend eosvmoc_instantiated_module;

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/code_cache.hpp

@@ -37,6 +37,7 @@ using allocator_t = bip::rbtree_best_fit<bip::null_mutex_family, bip::offset_ptr
 
 struct config;
 
+
 class code_cache_base {
    public:
       code_cache_base(const bfs::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db);
@@ -46,6 +47,12 @@ class code_cache_base {
 
       void free_code(const digest_type& code_id, const uint8_t& vm_version);
 
+      // get_descriptor_for_code failure reasons
+      enum class get_cd_failure {
+         temporary, // oc compile not done yet, users like read-only trxs can retry
+         permanent  // oc will not start, users should not retry
+      };
+
    protected:
       struct by_hash;
 
@@ -85,9 +92,6 @@ class code_cache_base {
 
       template <typename T>
       void serialize_cache_index(fc::datastream<T>& ds);
-
-      std::thread::id _main_thread_id;
-      bool is_main_thread() const;
 };
 
 class code_cache_async : public code_cache_base {
@@ -98,7 +102,7 @@ class code_cache_async : public code_cache_base {
       //If code is in cache: returns pointer & bumps to front of MRU list
       //If code is not in cache, and not blacklisted, and not currently compiling: return nullptr and kick off compile
       //otherwise: return nullptr
-      const code_descriptor* const get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version);
+      const code_descriptor* const get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version, bool is_write_window, get_cd_failure& failure);
 
    private:
       std::thread _monitor_reply_thread;
@@ -115,7 +119,7 @@ class code_cache_sync : public code_cache_base {
       ~code_cache_sync();
 
       //Can still fail and return nullptr if, for example, there is an expected instantiation failure
-      const code_descriptor* const get_descriptor_for_code_sync(const digest_type& code_id, const uint8_t& vm_version);
+      const code_descriptor* const get_descriptor_for_code_sync(const digest_type& code_id, const uint8_t& vm_version, bool is_write_window);
 };
 
 }}}

libraries/chain/include/eosio/chain/webassembly/eos-vm.hpp

@@ -43,15 +43,11 @@ template<typename Backend>
 class eos_vm_runtime : public eosio::chain::wasm_runtime_interface {
    public:
       eos_vm_runtime();
-      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t>,
+      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
                                                                              const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override;
 
-      void immediately_exit_currently_running_module() override;
-
    private:
       // todo: managing this will get more complicated with sync calls;
-      //       immediately_exit_currently_running_module() should probably
-      //       move from wasm_runtime_interface to wasm_instantiated_module_interface.
       eos_vm_backend_t<Backend>* _bkend = nullptr;  // non owning pointer to allow for immediate exit
 
    template<typename Impl>
@@ -61,10 +57,8 @@ class eos_vm_runtime : public eosio::chain::wasm_runtime_interface {
 class eos_vm_profile_runtime : public eosio::chain::wasm_runtime_interface {
    public:
       eos_vm_profile_runtime();
-      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t>,
+      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
                                                                              const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override;
-
-      void immediately_exit_currently_running_module() override;
 };
 
 }}}}// eosio::chain::webassembly::eos_vm_runtime

libraries/chain/include/eosio/chain/webassembly/runtime_interface.hpp

@@ -20,12 +20,9 @@ class wasm_instantiated_module_interface {
 
 class wasm_runtime_interface {
    public:
-      virtual std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t> initial_memory,
+      virtual std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
                                                                                      const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) = 0;
 
-      //immediately exit the currently running wasm_instantiated_module_interface. Yep, this assumes only one can possibly run at a time.
-      virtual void immediately_exit_currently_running_module() = 0;
-
       virtual ~wasm_runtime_interface();
 
       // eosvmoc_runtime needs this

libraries/chain/wasm_interface.cpp

@@ -92,8 +92,9 @@ namespace eosio { namespace chain {
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
       if(my->eosvmoc) {
          const chain::eosvmoc::code_descriptor* cd = nullptr;
+         chain::eosvmoc::code_cache_base::get_cd_failure failure = chain::eosvmoc::code_cache_base::get_cd_failure::temporary;
          try {
-            cd = my->eosvmoc->cc.get_descriptor_for_code(code_hash, vm_version);
+            cd = my->eosvmoc->cc.get_descriptor_for_code(code_hash, vm_version, context.control.is_write_window(), failure);
          }
          catch(...) {
             //swallow errors here, if EOS VM OC has gone in to the weeds we shouldn't bail: continue to try and run baseline
@@ -107,15 +108,18 @@ namespace eosio { namespace chain {
             my->eosvmoc->exec->execute(*cd, my->eosvmoc->mem, context);
             return;
          }
+         else if (context.trx_context.is_read_only()) {
+            if (failure == chain::eosvmoc::code_cache_base::get_cd_failure::temporary) {
+               EOS_ASSERT(false, ro_trx_vm_oc_compile_temporary_failure, "get_descriptor_for_code failed with temporary failure");
+            } else {
+               EOS_ASSERT(false, ro_trx_vm_oc_compile_permanent_failure, "get_descriptor_for_code failed with permanent failure");
+            }
+         }
       }
 #endif
       my->get_instantiated_module(code_hash, vm_type, vm_version, context.trx_context)->apply(context);
    }
 
-   void wasm_interface::exit() {
-      my->runtime_interface->immediately_exit_currently_running_module();
-   }
-
    bool wasm_interface::is_code_cached(const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) const {
       return my->is_code_cached(code_hash, vm_type, vm_version);
    }

libraries/chain/webassembly/cf_system.cpp

@@ -45,7 +45,8 @@ namespace eosio { namespace chain { namespace webassembly {
       }
    }
 
+   //be aware that EOS VM OC handles eosio_exit internally and this function will not be called by OC
    void interface::eosio_exit( int32_t code ) const {
-      context.control.get_wasm_interface().exit();
+      throw wasm_exit{};
    }
 }}} // ns eosio::chain::webassembly

libraries/chain/webassembly/runtimes/eos-vm-oc.cpp

@@ -28,7 +28,7 @@ class eosvmoc_instantiated_module : public wasm_instantiated_module_interface {
       bool is_main_thread() { return _main_thread_id == std::this_thread::get_id(); };
 
       void apply(apply_context& context) override {
-         const code_descriptor* const cd = _eosvmoc_runtime.cc.get_descriptor_for_code_sync(_code_hash, _vm_version);
+         const code_descriptor* const cd = _eosvmoc_runtime.cc.get_descriptor_for_code_sync(_code_hash, _vm_version, context.control.is_write_window());
          EOS_ASSERT(cd, wasm_execution_error, "EOS VM OC instantiation failed");
 
          if ( is_main_thread() )
@@ -50,14 +50,11 @@ eosvmoc_runtime::eosvmoc_runtime(const boost::filesystem::path data_dir, const e
 eosvmoc_runtime::~eosvmoc_runtime() {
 }
 
-std::unique_ptr<wasm_instantiated_module_interface> eosvmoc_runtime::instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t> initial_memory,
+std::unique_ptr<wasm_instantiated_module_interface> eosvmoc_runtime::instantiate_module(const char* code_bytes, size_t code_size,
                                                                                         const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) {
    return std::make_unique<eosvmoc_instantiated_module>(code_hash, vm_type, *this);
 }
 
-//never called. EOS VM OC overrides eosio_exit to its own implementation
-void eosvmoc_runtime::immediately_exit_currently_running_module() {}
-
 void eosvmoc_runtime::init_thread_local_data() {
    exec_thread_local = std::make_unique<eosvmoc::executor>(cc);
 }

libraries/chain/webassembly/runtimes/eos-vm-oc/code_cache.cpp

@@ -107,10 +107,11 @@ std::tuple<size_t, size_t> code_cache_async::consume_compile_thread_queue() {
 }
 
 
-const code_descriptor* const code_cache_async::get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version) {
+const code_descriptor* const code_cache_async::get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version, bool is_write_window, get_cd_failure& failure) {
    //if there are any outstanding compiles, process the result queue now
-   //do this only on main thread (which is in single threaded write window)
-   if(is_main_thread() && _outstanding_compiles_and_poison.size()) {
+   //When app is in write window, all tasks are running sequentially and read-only threads
+   //are not running. Safe to update cache entries.
+   if(is_write_window && _outstanding_compiles_and_poison.size()) {
       auto [count_processed, bytes_remaining] = consume_compile_thread_queue();
 
       if(count_processed)
@@ -136,37 +137,48 @@ const code_descriptor* const code_cache_async::get_descriptor_for_code(const dig
    //check for entry in cache
    code_cache_index::index<by_hash>::type::iterator it = _cache_index.get<by_hash>().find(boost::make_tuple(code_id, vm_version));
    if(it != _cache_index.get<by_hash>().end()) {
-      if (is_main_thread())
+      if (is_write_window)
          _cache_index.relocate(_cache_index.begin(), _cache_index.project<0>(it));
       return &*it;
    }
-   if(!is_main_thread()) // on read-only thread
+   if(!is_write_window) {
+      failure = get_cd_failure::temporary; // Compile might not be done yet
       return nullptr;
+   }
 
    const code_tuple ct = code_tuple{code_id, vm_version};
 
-   if(_blacklist.find(ct) != _blacklist.end())
+   if(_blacklist.find(ct) != _blacklist.end()) {
+      failure = get_cd_failure::permanent; // Compile will not start
       return nullptr;
+   }
    if(auto it = _outstanding_compiles_and_poison.find(ct); it != _outstanding_compiles_and_poison.end()) {
+      failure = get_cd_failure::temporary; // Compile might not be done yet
       it->second = false;
       return nullptr;
    }
-   if(_queued_compiles.find(ct) != _queued_compiles.end())
+   if(_queued_compiles.find(ct) != _queued_compiles.end()) {
+      failure = get_cd_failure::temporary; // Compile might not be done yet
       return nullptr;
+   }
 
    if(_outstanding_compiles_and_poison.size() >= _threads) {
       _queued_compiles.emplace(ct);
+      failure = get_cd_failure::temporary; // Compile might not be done yet
       return nullptr;
    }
 
    const code_object* const codeobject = _db.find<code_object,by_code_hash>(boost::make_tuple(code_id, 0, vm_version));
-   if(!codeobject) //should be impossible right?
+   if(!codeobject) { //should be impossible right?
+      failure = get_cd_failure::permanent; // Compile will not start
       return nullptr;
+   }
 
    _outstanding_compiles_and_poison.emplace(ct, false);
    std::vector<wrapped_fd> fds_to_pass;
    fds_to_pass.emplace_back(memfd_for_bytearray(codeobject->code));
    write_message_with_fds(_compile_monitor_write_socket, compile_wasm_message{ ct }, fds_to_pass);
+   failure = get_cd_failure::temporary; // Compile might not be done yet
    return nullptr;
 }
 
@@ -179,15 +191,15 @@ code_cache_sync::~code_cache_sync() {
       elog("unexpected response from EOS VM OC compile monitor during shutdown");
 }
 
-const code_descriptor* const code_cache_sync::get_descriptor_for_code_sync(const digest_type& code_id, const uint8_t& vm_version) {
+const code_descriptor* const code_cache_sync::get_descriptor_for_code_sync(const digest_type& code_id, const uint8_t& vm_version, bool is_write_window) {
    //check for entry in cache
    code_cache_index::index<by_hash>::type::iterator it = _cache_index.get<by_hash>().find(boost::make_tuple(code_id, vm_version));
    if(it != _cache_index.get<by_hash>().end()) {
-      if (is_main_thread())
+      if (is_write_window)
          _cache_index.relocate(_cache_index.begin(), _cache_index.project<0>(it));
       return &*it;
    }
-   if(!is_main_thread())
+   if(!is_write_window)
       return nullptr;
 
    const code_object* const codeobject = _db.find<code_object,by_code_hash>(boost::make_tuple(code_id, 0, vm_version));
@@ -212,8 +224,7 @@ const code_descriptor* const code_cache_sync::get_descriptor_for_code_sync(const
 
 code_cache_base::code_cache_base(const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db) :
    _db(db),
-   _cache_file_path(data_dir/"code_cache.bin"),
-   _main_thread_id(std::this_thread::get_id())
+   _cache_file_path(data_dir/"code_cache.bin")
 {
    static_assert(sizeof(allocator_t) <= header_offset, "header offset intersects with allocator");
 
@@ -390,8 +401,4 @@ void code_cache_base::check_eviction_threshold(size_t free_bytes) {
    if(free_bytes < _free_bytes_eviction_threshold)
       run_eviction_round();
 }
-
-bool code_cache_base::is_main_thread() const {
-   return _main_thread_id == std::this_thread::get_id();
-}
 }}}

libraries/chain/webassembly/runtimes/eos-vm.cpp

@@ -234,12 +234,7 @@ template<typename Impl>
 eos_vm_runtime<Impl>::eos_vm_runtime() {}
 
 template<typename Impl>
-void eos_vm_runtime<Impl>::immediately_exit_currently_running_module() {
-   throw wasm_exit{};
-}
-
-template<typename Impl>
-std::unique_ptr<wasm_instantiated_module_interface> eos_vm_runtime<Impl>::instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t>,
+std::unique_ptr<wasm_instantiated_module_interface> eos_vm_runtime<Impl>::instantiate_module(const char* code_bytes, size_t code_size,
                                                                                              const digest_type&, const uint8_t&, const uint8_t&) {
 
    using backend_t = eos_vm_backend_t<Impl>;
@@ -261,11 +256,7 @@ template class eos_vm_runtime<eosio::vm::jit>;
 
 eos_vm_profile_runtime::eos_vm_profile_runtime() {}
 
-void eos_vm_profile_runtime::immediately_exit_currently_running_module() {
-   throw wasm_exit{};
-}
-
-std::unique_ptr<wasm_instantiated_module_interface> eos_vm_profile_runtime::instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t>,
+std::unique_ptr<wasm_instantiated_module_interface> eos_vm_profile_runtime::instantiate_module(const char* code_bytes, size_t code_size,
                                                                                                const digest_type&, const uint8_t&, const uint8_t&) {
 
    using backend_t = eosio::vm::backend<eos_vm_host_functions_t, eosio::vm::jit_profile, webassembly::eos_vm_runtime::apply_options, vm::profile_instr_map>;

libraries/custom_appbase/include/eosio/chain/application.hpp

@@ -1,30 +1,33 @@
 #pragma once
 
 #include <appbase/application_base.hpp>
-#include <appbase/execution_priority_queue.hpp>
+#include <eosio/chain/exec_pri_queue.hpp>
+#include <mutex>
 
 /*
- * Custmomize appbase to support two-queue exector.
+ * Customize appbase to support two-queue executor.
 */
 namespace appbase { 
 
 enum class exec_window {
-   read,  // the window during which operations from read_only_trx_safe queue
-          // can be executed in app thread in parallel with read-only transactions
-          // in read-only transaction excuting threads.
-   write, // the window during which operations from both general and
-          // read_only_trx_safe queues can be executed in app thread,
-          // while read-only transactions are not executed in read-only
-          // transaction excuting threads.
+   read,  // the window during which operations from read_only queue
+          // can be executed in parallel in the read-only thread pool
+          // as well as in the app thread.
+   write, // the window during which operations from both read_write and
+          // parallel queues can be executed in app thread,
+          // while read-only operations are not executed in read-only
+          // thread pool. The read-only thread pool is not active; only
+          // the main app thread is active.
 };
 
 enum class exec_queue {
-   read_only_trx_safe, // the queue storing operations which are safe to execute
-                       // on app thread in parallel with read-only transactions
-                       // in read-only transaction excuting threads.
-   general             // the queue storing operations which can be only executed
-                       // on the app thread while read-only transactions are
-                       // not being executed in read-only threads
+   read_only,          // the queue storing tasks which are safe to execute
+                       // in parallel with other read-only tasks in the read-only
+                       // thread pool as well as on the main app thread.
+                       // Multi-thread safe as long as nothing is executed from the read_write queue.
+   read_write          // the queue storing tasks which can be only executed
+                       // on the app thread while read-only tasks are
+                       // not being executed in read-only threads. Single threaded.
 };
 
 class two_queue_executor {
@@ -32,56 +35,64 @@ public:
 
    template <typename Func>
    auto post( int priority, exec_queue q, Func&& func ) {
-      if ( q == exec_queue::general )
-         return boost::asio::post(io_serv_, general_queue_.wrap(priority, --order_, std::forward<Func>(func)));
+      if ( q == exec_queue::read_write )
+         return boost::asio::post(io_serv_, read_write_queue_.wrap(priority, --order_, std::forward<Func>(func)));
       else
-         return boost::asio::post(io_serv_, read_only_trx_safe_queue_.wrap(priority, --order_, std::forward<Func>(func)));
+         return boost::asio::post( io_serv_, read_only_queue_.wrap( priority, --order_, std::forward<Func>( func)));
    }
 
    // Legacy and deprecated. To be removed after cleaning up its uses in base appbase
    template <typename Func>
    auto post( int priority, Func&& func ) {
-      // safer to use general queue for unknown type of operation since operations
-      // from general queue are not executed in parallel with read-only transactions
-      return boost::asio::post(io_serv_, general_queue_.wrap(priority, --order_, std::forward<Func>(func)));
+      // safer to use read_write queue for unknown type of operation since operations
+      // from read_write queue are not executed in parallel with read-only operations
+      return boost::asio::post(io_serv_, read_write_queue_.wrap(priority, --order_, std::forward<Func>(func)));
    }
 
    boost::asio::io_service& get_io_service() { return io_serv_; }
 
    bool execute_highest() {
       if ( exec_window_ == exec_window::write ) {
-         if( !general_queue_.empty() && ( read_only_trx_safe_queue_.empty() || *read_only_trx_safe_queue_.top() < *general_queue_.top()) )  {
-            // general_queue_'s top function's priority greater than read_only_trx_safe_queue_'s top function's, or general_queue_ empty
-            general_queue_.execute_highest();
-         } else if( !read_only_trx_safe_queue_.empty() ) {
-            read_only_trx_safe_queue_.execute_highest();
+         // During write window only main thread is accessing anything in two_queue_executor, no locking required
+         if( !read_write_queue_.empty() && (read_only_queue_.empty() || *read_only_queue_.top() < *read_write_queue_.top()) )  {
+            // read_write_queue_'s top function's priority greater than read_only_queue_'s top function's, or read_only_queue_ empty
+            read_write_queue_.execute_highest();
+         } else if( !read_only_queue_.empty() ) {
+            read_only_queue_.execute_highest();
          }
-         return !read_only_trx_safe_queue_.empty() || !general_queue_.empty();
+         return !read_only_queue_.empty() || !read_write_queue_.empty();
       } else {
-         return read_only_trx_safe_queue_.execute_highest();
+         // When in read window, multiple threads including main app thread are accessing two_queue_executor, locking required
+         return read_only_queue_.execute_highest_locked(false);
       }
    }
 
+   bool execute_highest_read_only() {
+      return read_only_queue_.execute_highest_locked(true);
+   }
+
    template <typename Function>
-   boost::asio::executor_binder<Function, appbase::execution_priority_queue::executor>
+   boost::asio::executor_binder<Function, appbase::exec_pri_queue::executor>
    wrap(int priority, exec_queue q, Function&& func ) {
-      if ( q == exec_queue::general )
-         return general_queue_.wrap(priority, --order_, std::forward<Function>(func));
+      if ( q == exec_queue::read_write )
+         return read_write_queue_.wrap(priority, --order_, std::forward<Function>(func));
       else
-         return read_only_trx_safe_queue_.wrap(priority, --order_, std::forward<Function>(func));
+         return read_only_queue_.wrap( priority, --order_, std::forward<Function>( func));
    }
      
    void clear() {
-      read_only_trx_safe_queue_.clear();
-      general_queue_.clear();
+      read_only_queue_.clear();
+      read_write_queue_.clear();
    }
 
-   void set_to_read_window() {
+   void set_to_read_window(uint32_t num_threads, std::function<bool()> should_exit) {
       exec_window_ = exec_window::read;
+      read_only_queue_.enable_locking(num_threads, std::move(should_exit));
    }
 
    void set_to_write_window() {
       exec_window_ = exec_window::write;
+      read_only_queue_.disable_locking();
    }
 
    bool is_read_window() const {
@@ -92,15 +103,15 @@ public:
       return exec_window_ == exec_window::write;
    }
 
-   auto& read_only_trx_safe_queue() { return read_only_trx_safe_queue_; }
+   auto& read_only_queue() { return read_only_queue_; }
 
-   auto& general_queue() { return general_queue_; }
+   auto& read_write_queue() { return read_write_queue_; }
 
    // members are ordered taking into account that the last one is destructed first
 private:
    boost::asio::io_service            io_serv_;
-   appbase::execution_priority_queue  read_only_trx_safe_queue_;
-   appbase::execution_priority_queue  general_queue_;
+   appbase::exec_pri_queue            read_only_queue_;
+   appbase::exec_pri_queue            read_write_queue_;
    std::atomic<std::size_t>           order_ { std::numeric_limits<size_t>::max() }; // to maintain FIFO ordering in both queues within priority
    exec_window                        exec_window_ { exec_window::write };
 };

libraries/custom_appbase/include/eosio/chain/exec_pri_queue.hpp

@@ -0,0 +1,233 @@
+#pragma once
+#include <boost/asio.hpp>
+
+#include <condition_variable>
+#include <mutex>
+#include <queue>
+
+namespace appbase {
+// adapted from: https://www.boost.org/doc/libs/1_69_0/doc/html/boost_asio/example/cpp11/invocation/prioritised_handlers.cpp
+
+// Locking has to be coordinated by caller, use with care.
+class exec_pri_queue : public boost::asio::execution_context
+{
+public:
+
+   void enable_locking(uint32_t num_threads, std::function<bool()> should_exit) {
+      assert(num_threads > 0 && num_waiting_ == 0);
+      lock_enabled_ = true;
+      max_waiting_ = num_threads;
+      should_exit_ = std::move(should_exit);
+      exiting_blocking_ = false;
+   }
+
+   void disable_locking() {
+      lock_enabled_ = false;
+      should_exit_ = [](){ assert(false); return true; }; // should not be called when locking is disabled
+   }
+
+   // called from appbase::application_base::exec poll_one() or run_one()
+   template <typename Function>
+   void add(int priority, size_t order, Function function)
+   {
+      std::unique_ptr<queued_handler_base> handler(new queued_handler<Function>(priority, order, std::move(function)));
+      if (lock_enabled_) {
+         std::lock_guard g( mtx_ );
+         handlers_.push( std::move( handler ) );
+         if (num_waiting_)
+            cond_.notify_one();
+      } else {
+         handlers_.push( std::move( handler ) );
+      }
+   }
+
+   // only call when no lock required
+   void clear()
+   {
+      handlers_ = prio_queue();
+   }
+
+   // only call when no lock required
+   bool execute_highest()
+   {
+      if( !handlers_.empty() ) {
+         handlers_.top()->execute();
+         handlers_.pop();
+      }
+
+      return !handlers_.empty();
+   }
+
+private:
+   // has to be defined before use, auto return type
+   auto pop() {
+      auto t = std::move(const_cast<std::unique_ptr<queued_handler_base>&>(handlers_.top()));
+      handlers_.pop();
+      return t;
+   }
+
+public:
+
+   bool execute_highest_locked(bool should_block) {
+      std::unique_lock g(mtx_);
+      if (should_block) {
+         ++num_waiting_;
+         cond_.wait(g, [this](){
+            bool exit = exiting_blocking_ || should_exit_();
+            bool empty = handlers_.empty();
+            if (empty || exit) {
+               if (((empty && num_waiting_ == max_waiting_) || exit) && !exiting_blocking_) {
+                  cond_.notify_all();
+                  exiting_blocking_ = true;
+               }
+               return exit || exiting_blocking_; // same as calling should_exit(), but faster
+            }
+            return true;
+         });
+         --num_waiting_;
+         if (exiting_blocking_ || should_exit_())
+            return false;
+      }
+      if( handlers_.empty() )
+         return false;
+      auto t = pop();
+      g.unlock();
+      t->execute();
+      return true;
+   }
+
+   // Only call when locking disabled
+   size_t size() const { return handlers_.size(); }
+
+   // Only call when locking disabled
+   bool empty() const { return handlers_.empty(); }
+
+   // Only call when locking disabled
+   const auto& top() const { return handlers_.top(); }
+
+   class executor
+   {
+   public:
+      executor(exec_pri_queue& q, int p, size_t o)
+            : context_(q), priority_(p), order_(o)
+      {
+      }
+
+      exec_pri_queue& context() const noexcept
+      {
+         return context_;
+      }
+
+      template <typename Function, typename Allocator>
+      void dispatch(Function f, const Allocator&) const
+      {
+         context_.add(priority_, order_, std::move(f));
+      }
+
+      template <typename Function, typename Allocator>
+      void post(Function f, const Allocator&) const
+      {
+         context_.add(priority_, order_, std::move(f));
+      }
+
+      template <typename Function, typename Allocator>
+      void defer(Function f, const Allocator&) const
+      {
+         context_.add(priority_, order_, std::move(f));
+      }
+
+      void on_work_started() const noexcept {}
+      void on_work_finished() const noexcept {}
+
+      bool operator==(const executor& other) const noexcept
+      {
+         return order_ == other.order_ && &context_ == &other.context_ && priority_ == other.priority_;
+      }
+
+      bool operator!=(const executor& other) const noexcept
+      {
+         return !operator==(other);
+      }
+
+   private:
+      exec_pri_queue& context_;
+      int priority_;
+      size_t order_;
+   };
+
+   template <typename Function>
+   boost::asio::executor_binder<Function, executor>
+   wrap(int priority, size_t order, Function&& func)
+   {
+      return boost::asio::bind_executor( executor(*this, priority, order), std::forward<Function>(func) );
+   }
+
+private:
+   class queued_handler_base
+   {
+   public:
+      queued_handler_base( int p, size_t order )
+            : priority_( p )
+            , order_( order )
+      {
+      }
+
+      virtual ~queued_handler_base() = default;
+
+      virtual void execute() = 0;
+
+      int priority() const { return priority_; }
+      // C++20
+      // friend std::weak_ordering operator<=>(const queued_handler_base&,
+      //                                       const queued_handler_base&) noexcept = default;
+      friend bool operator<(const queued_handler_base& a,
+                            const queued_handler_base& b) noexcept
+      {
+         return std::tie( a.priority_, a.order_ ) < std::tie( b.priority_, b.order_ );
+      }
+
+   private:
+      int priority_;
+      size_t order_;
+   };
+
+   template <typename Function>
+   class queued_handler : public queued_handler_base
+   {
+   public:
+      queued_handler(int p, size_t order, Function f)
+            : queued_handler_base( p, order )
+            , function_( std::move(f) )
+      {
+      }
+
+      void execute() override
+      {
+         function_();
+      }
+
+   private:
+      Function function_;
+   };
+
+   struct deref_less
+   {
+      template<typename Pointer>
+      bool operator()(const Pointer& a, const Pointer& b) noexcept(noexcept(*a < *b))
+      {
+         return *a < *b;
+      }
+   };
+
+   bool lock_enabled_ = false;
+   std::mutex mtx_;
+   std::condition_variable cond_;
+   uint32_t num_waiting_{0};
+   uint32_t max_waiting_{0};
+   bool exiting_blocking_{false};
+   std::function<bool()> should_exit_; // called holding mtx_
+   using prio_queue = std::priority_queue<std::unique_ptr<queued_handler_base>, std::deque<std::unique_ptr<queued_handler_base>>, deref_less>;
+   prio_queue handlers_;
+};
+
+} // appbase

libraries/custom_appbase/tests/custom_appbase_tests.cpp

@@ -27,28 +27,28 @@ BOOST_AUTO_TEST_CASE( default_exec_window ) {
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::read_only_trx_safe,  [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_only,  [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[7]=seq_num; ++seq_num; } );
 
    // Stop app. Use the lowest priority to make sure this function to execute the last
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
-      // read_only_trx_safe_queue should only contain the current lambda function,
-      // and general_queue should have execute all its functions
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
-      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 0 );
+   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
+      // read_only_queue should only contain the current lambda function,
+      // and read_write_queue should have executed all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 1); // pop()s after execute
+      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 0 );
       app->quit();
       } );
    app_thread.join();
 
    // both queues are cleared after execution 
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
 
    // exactly number of both queues' functions processed
    BOOST_REQUIRE_EQUAL( rslts.size(), 8 );
@@ -64,40 +64,40 @@ BOOST_AUTO_TEST_CASE( default_exec_window ) {
    BOOST_CHECK_LT( rslts[6], rslts[7] );
 }
 
-// verify functions only from read_only_trx_safe queue are processed during read window
+// verify functions only from read_only queue are processed during read window
 BOOST_AUTO_TEST_CASE( execute_from_read_queue ) {
    appbase::scoped_app app;
    auto app_thread = start_app_thread(app);
    
-   // set to run functions from read_only_trx_safe queue only
-   app->executor().set_to_read_window();
+   // set to run functions from read_only queue only
+   app->executor().set_to_read_window(1, [](){return false;});
 
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[7]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[8]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[9]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[9]=seq_num; ++seq_num; } );
 
    // stop application. Use lowest at the end to make sure this executes the last
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
-      // read_queue should have current function and write_queue should have all its functions
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
-      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 4 );
+   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
+      // read_queue should be empty (read window pops before execute) and write_queue should have all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 0); // pop()s before execute
+      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 4 );
       app->quit();
       } );
    app_thread.join();
 
    // both queues are cleared after execution 
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
 
    // exactly number of posts processed
    BOOST_REQUIRE_EQUAL( rslts.size(), 6 );
@@ -109,40 +109,40 @@ BOOST_AUTO_TEST_CASE( execute_from_read_queue ) {
    BOOST_CHECK_LT( rslts[3], rslts[4] );
 }
 
-// verify no functions are executed during  read window if read_only_trx_safe queue is empty
+// verify no functions are executed during  read window if read_only queue is empty
 BOOST_AUTO_TEST_CASE( execute_from_empty_read_queue ) {
    appbase::scoped_app app;
    auto app_thread = start_app_thread(app);
    
-   // set to run functions from read_only_trx_safe queue only
-   app->executor().set_to_read_window();
+   // set to run functions from read_only queue only
+   app->executor().set_to_read_window(1, [](){return false;});
 
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::general, [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[8]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[9]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_write, [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_write, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[9]=seq_num; ++seq_num; } );
 
    // Stop application. Use lowest at the end to make sure this executes the last
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
-      // read_queue should have current function and write_queue should have all its functions
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
-      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 10 );
+   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
+      // read_queue should be empty (read window pops before execute) and write_queue should have all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 0); // pop()s before execute
+      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 10 );
       app->quit();
       } );
    app_thread.join();
 
    // both queues are cleared after execution 
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
 
    // no results
    BOOST_REQUIRE_EQUAL( rslts.size(), 0 );
@@ -159,32 +159,32 @@ BOOST_AUTO_TEST_CASE( execute_from_both_queues ) {
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::read_only_trx_safe,  [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[8]=seq_num; ++seq_num; } );
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[9]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[10]=seq_num; ++seq_num; } );
-   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[11]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_only,  [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only,  [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only,  [&]() { rslts[9]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[10]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[11]=seq_num; ++seq_num; } );
 
    // stop application. Use lowest at the end to make sure this executes the last
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
+   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
       // read_queue should have current function and write_queue's functions are all executed 
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
-      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 0 );
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 1); // pop()s after execute
+      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 0 );
       app->quit();
       } );
 
    app_thread.join();
 
    // queues are emptied after quit
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
 
    // exactly number of posts processed
    BOOST_REQUIRE_EQUAL( rslts.size(), 12 );

libraries/libfc/CMakeLists.txt

@@ -57,8 +57,6 @@ set( fc_sources
      src/crypto/blake2.cpp
      src/crypto/k1_recover.cpp
      src/network/ip.cpp
-     src/network/resolve.cpp
-     src/network/udp_socket.cpp
      src/network/url.cpp
      src/network/http/http_client.cpp
      src/compress/zlib.cpp

libraries/libfc/include/fc/network/resolve.hpp

@@ -1,9 +0,0 @@
-#pragma once
-#include <fc/vector.hpp>
-#include <fc/network/ip.hpp>
-
-namespace fc
-{
-  std::vector<boost::asio::ip::udp::endpoint> resolve(boost::asio::io_service& io_service,
-                                                      const std::string& host, uint16_t port);
-}

libraries/libfc/include/fc/network/udp_socket.hpp

@@ -1,39 +0,0 @@
-#pragma once
-#include <fc/utility.hpp>
-#include <memory>
-
-#include <boost/asio.hpp>
-
-namespace fc {
-  namespace ip {
-    class endpoint;
-    class address;
-  }
-
-  /**
-   *  The udp_socket class has reference semantics, all copies will
-   *  refer to the same underlying socket.
-   */
-  class udp_socket {
-    public:
-      udp_socket();
-      udp_socket( const udp_socket& s );
-      ~udp_socket();
-
-      void initialize(boost::asio::io_service &);
-      void open();
-      void send_to(const char* b, size_t l, boost::asio::ip::udp::endpoint &to);
-      void send_to(const std::shared_ptr<const char>& b, size_t l, boost::asio::ip::udp::endpoint &to);
-      void close();
-
-      void set_reuse_address(bool);
-
-      void connect(const boost::asio::ip::udp::endpoint& e);
-      const boost::asio::ip::udp::endpoint local_endpoint() const;
-
-    private:
-      class                impl;
-      std::shared_ptr<impl> my;
-  };
-
-}

libraries/libfc/src/log/gelf_appender.cpp

@@ -1,6 +1,3 @@
-#include <fc/network/udp_socket.hpp>
-#include <fc/network/ip.hpp>
-#include <fc/network/resolve.hpp>
 #include <fc/exception/exception.hpp>
 #include <fc/log/gelf_appender.hpp>
 #include <fc/reflect/variant.hpp>
@@ -10,6 +7,7 @@
 #include <fc/compress/zlib.hpp>
 #include <fc/log/logger_config.hpp>
 
+#include <boost/asio.hpp>
 #include <boost/lexical_cast.hpp>
 #include <iomanip>
 #include <iostream>
@@ -19,14 +17,6 @@
 
 namespace fc
 {
-  namespace detail
-  {
-    boost::asio::ip::udp::endpoint to_asio_ep( const fc::ip::endpoint& e )
-    {
-      return boost::asio::ip::udp::endpoint(boost::asio::ip::address_v4(e.get_address()), e.port() );
-    }
-  }
-
   const std::vector<std::string> gelf_appender::config::reserved_field_names  = {
       "_id",            // per GELF specification
       "_timestamp_ns",  // Remaining names all populated by appender
@@ -36,23 +26,21 @@ namespace fc
       "_method_name",
       "_thread_name",
       "_task_name"
-   };
+  };
 
-   const std::regex gelf_appender::config::user_field_name_pattern{"^_[\\w\\.\\-]*$"}; // per GELF specification
+  const std::regex gelf_appender::config::user_field_name_pattern{"^_[\\w\\.\\-]*$"}; // per GELF specification
 
   class gelf_appender::impl
   {
   public:
     using work_guard_t = boost::asio::executor_work_guard<boost::asio::io_context::executor_type>;
     config                                         cfg;
-    std::optional<boost::asio::ip::udp::endpoint>  gelf_endpoint;
     std::thread                                    thread;
     boost::asio::io_context                        io_context;
     work_guard_t work_guard =                      boost::asio::make_work_guard(io_context);
-    udp_socket                                     gelf_socket;
+    boost::asio::ip::udp::socket                   gelf_socket;
 
-
-    impl(const variant& c)
+    impl(const variant& c) : gelf_socket(io_context)
     {
       mutable_variant_object mvo;
       from_variant(c, mvo);
@@ -84,6 +72,35 @@ namespace fc
           thread.join();
        }
     }
+
+    static std::shared_ptr<std::vector<char>> make_new_bufer(boost::asio::const_buffer buf) {
+      const char* p = static_cast<const char*>(buf.data());
+      return std::make_shared<std::vector<char>>(p, p+buf.size());
+    }
+
+    static std::shared_ptr<std::vector<char>> make_new_bufer(const std::array<boost::asio::const_buffer, 2>& bufs) {
+      auto new_buf = std::make_shared<std::vector<char>>();
+      new_buf->reserve(bufs[0].size() + bufs[1].size());
+      for (int i = 0; i < 2; ++i) {
+          const char* p = static_cast<const char*>(bufs[i].data());
+          new_buf->insert(new_buf->end(), p, p + bufs[i].size());
+      }
+      return new_buf;
+    }
+
+    template <typename Buffers>
+    void send(Buffers&& bufs) {
+      boost::system::error_code ec;
+      gelf_socket.send(std::forward<Buffers>(bufs), 0, ec);
+      if (ec == boost::asio::error::would_block) {
+          auto new_buf = make_new_bufer(std::forward<Buffers>(bufs));
+          gelf_socket.async_send(boost::asio::buffer(*new_buf),
+                                    [new_buf](const boost::system::error_code& /*ec*/, std::size_t /*bytes_transferred*/) {
+                                        // Swallow errors.  Currently only used for GELF logging, so depend on local
+                                        // log to catch anything that doesn't make it across the network.
+                                    });
+      }
+    }
   };
 
   gelf_appender::gelf_appender(const variant& args) :
@@ -95,58 +112,40 @@ namespace fc
   {
     try
     {
-      try
-      {
-        // if it's a numeric address:port, this will parse it
-        my->gelf_endpoint = detail::to_asio_ep(ip::endpoint::from_string(my->cfg.endpoint));
+      if (my->cfg.endpoint.empty()) {
+        fprintf(stderr, "The logging destination is not specified\n");
+        return;
       }
-      catch (...)
-      {
-      }
-      if (!my->gelf_endpoint)
-      {
-        // couldn't parse as a numeric ip address, try resolving as a DNS name.
-        // This can yield, so don't do it in the catch block above
-        string::size_type colon_pos = my->cfg.endpoint.find(':');
-        try
-        {
-          FC_ASSERT(colon_pos != std::string::npos, "The logging destination port is not specified");
-          string port = my->cfg.endpoint.substr(colon_pos + 1);
 
-          string hostname = my->cfg.endpoint.substr( 0, colon_pos );
+      std::string_view endpoint = my->cfg.endpoint;
+      string::size_type colon_pos = endpoint.rfind(':');
+      FC_ASSERT(colon_pos != std::string::npos, "The logging destination port is not specified");
+      auto port = endpoint.substr(colon_pos + 1);
 
-          boost::asio::ip::udp::resolver resolver{ my->io_context };
-          auto endpoints = resolver.resolve(hostname, port);
+      auto hostname = (endpoint[0] == '[' && colon_pos >= 2)  ? endpoint.substr( 1, colon_pos-2 ) : endpoint.substr( 0, colon_pos );
 
-          if (endpoints.empty())
-              FC_THROW_EXCEPTION(unknown_host_exception, "The logging destination host name can not be resolved: ${hostname}",
-                                 ("hostname", hostname));
+      boost::asio::ip::udp::resolver resolver{ my->io_context };
+      auto endpoints = resolver.resolve(hostname, port);
 
-          my->gelf_endpoint = *endpoints.begin();
+      if (endpoints.empty())
+          FC_THROW_EXCEPTION(unknown_host_exception, "The logging destination host name can not be resolved: ${hostname}",
+                              ("hostname", std::string(hostname)));
+
+      my->gelf_socket.connect(*endpoints.begin());
+      std::cerr << "opened GELF socket to endpoint " << my->cfg.endpoint << "\n";
+      my->gelf_socket.non_blocking(true);
+
+      my->thread = std::thread([this] {
+        try {
+          fc::set_os_thread_name("gelf");
+          my->io_context.run();
+        } catch (std::exception& ex) {
+          fprintf(stderr, "GELF logger caught exception at %s:%d : %s\n", __FILE__, __LINE__, ex.what());
+        } catch (...) {
+          fprintf(stderr, "GELF logger caught exception unknown exception %s:%d\n", __FILE__, __LINE__);
         }
-        catch (const boost::bad_lexical_cast&)
-        {
-          FC_THROW("Bad port: ${port}", ("port", my->cfg.endpoint.substr(colon_pos + 1, my->cfg.endpoint.size())));
-        }
-      }
+      });
 
-      if (my->gelf_endpoint)
-      {
-        my->gelf_socket.initialize(my->io_context);
-        my->gelf_socket.open();
-        std::cerr << "opened GELF socket to endpoint " << my->cfg.endpoint << "\n";
-
-        my->thread = std::thread([this] {
-          try {
-            fc::set_os_thread_name("gelf");
-            my->io_context.run();
-          } catch (std::exception& ex) {
-            fprintf(stderr, "GELF logger caught exception at %s:%d : %s\n", __FILE__, __LINE__, ex.what());
-          } catch (...) {
-            fprintf(stderr, "GELF logger caught exception unknown exception %s:%d\n", __FILE__, __LINE__);
-          }
-        });
-      }
     }
     catch (...)
     {
@@ -221,57 +220,44 @@ namespace fc
 
     if (gelf_message_as_string.size() <= max_payload_size)
     {
-      // no need to split
-      std::shared_ptr<char> send_buffer(new char[gelf_message_as_string.size()],
-                                        [](char* p){ delete[] p; });
-      memcpy(send_buffer.get(), gelf_message_as_string.c_str(),
-             gelf_message_as_string.size());
-
-      my->gelf_socket.send_to(send_buffer, gelf_message_as_string.size(),
-                              *my->gelf_endpoint);
+      my->send(boost::asio::buffer(gelf_message_as_string));
     }
     else
     {
       // split the message
+      struct gelf_header {
+        uint8_t magic[2] = { 0x1e, 0x0f};
+        uint64_t message_id;
+        uint8_t seq = 0;
+        uint8_t count = 0;
+      } header;
+
       // we need to generate an 8-byte ID for this message.
       // city hash should do
-      uint64_t message_id = city_hash64(gelf_message_as_string.c_str(), gelf_message_as_string.size());
-      const unsigned header_length = 2 /* magic */ + 8 /* msg id */ + 1 /* seq */ + 1 /* count */;
-      const unsigned body_length = max_payload_size - header_length;
-      unsigned total_number_of_packets = (gelf_message_as_string.size() + body_length - 1) / body_length;
+      header.message_id = city_hash64(gelf_message_as_string.c_str(), gelf_message_as_string.size());
+      const unsigned body_length = max_payload_size - sizeof(header);
+      header.count = (gelf_message_as_string.size() + body_length - 1) / body_length;
       unsigned bytes_sent = 0;
-      unsigned number_of_packets_sent = 0;
+
       while (bytes_sent < gelf_message_as_string.size())
       {
         unsigned bytes_to_send = std::min((unsigned)gelf_message_as_string.size() - bytes_sent,
                                           body_length);
 
-        std::shared_ptr<char> send_buffer(new char[max_payload_size],
-                                          [](char* p){ delete[] p; });
-        char* ptr = send_buffer.get();
-        // magic number for chunked message
-        *(unsigned char*)ptr++ = 0x1e;
-        *(unsigned char*)ptr++ = 0x0f;
-
-        // message id
-        memcpy(ptr, (char*)&message_id, sizeof(message_id));
-        ptr += sizeof(message_id);
-
-        *(unsigned char*)(ptr++) = number_of_packets_sent;
-        *(unsigned char*)(ptr++) = total_number_of_packets;
-        memcpy(ptr, gelf_message_as_string.c_str() + bytes_sent,
-               bytes_to_send);
-        my->gelf_socket.send_to(send_buffer, header_length + bytes_to_send,
-                                *my->gelf_endpoint);
-        ++number_of_packets_sent;
+        std::array<boost::asio::const_buffer,2> bufs = {
+          boost::asio::const_buffer(&header, sizeof(header)),
+          boost::asio::const_buffer(gelf_message_as_string.c_str() + bytes_sent, bytes_to_send)
+        };
+        my->send(bufs);
+        ++header.seq;
         bytes_sent += bytes_to_send;
       }
-      FC_ASSERT(number_of_packets_sent == total_number_of_packets);
+      FC_ASSERT(header.seq == header.count);
     }
   }
 
   void gelf_appender::log(const log_message& message) {
-    if (!my->gelf_endpoint)
+    if (!my->thread.joinable())
       return;
 
     // use now() instead of context.get_timestamp() because log_message construction can include user provided long running calls

libraries/libfc/src/network/resolve.cpp

@@ -1,33 +0,0 @@
-#include <boost/asio.hpp>
-#include <fc/exception/exception.hpp>
-
-namespace fc
-{
-  std::vector<boost::asio::ip::udp::endpoint> resolve(boost::asio::io_service& io_service,
-                                                     const std::string& host, uint16_t port)
-  {
-    using q = boost::asio::ip::udp::resolver::query;
-    using b = boost::asio::ip::resolver_query_base;
-    boost::asio::ip::udp::resolver res(io_service);
-    boost::system::error_code ec;
-    auto ep = res.resolve(q(host, std::to_string(uint64_t(port)),
-                            b::address_configured | b::numeric_service), ec);
-    if(!ec)
-    {
-      std::vector<boost::asio::ip::udp::endpoint> eps;
-      while(ep != boost::asio::ip::udp::resolver::iterator())
-      {
-        if(ep->endpoint().address().is_v4())
-        {
-         eps.push_back(*ep);
-        }
-        // TODO: add support for v6
-        ++ep;
-      }
-      return eps;
-    }
-    FC_THROW_EXCEPTION(unknown_host_exception,
-                       "name resolution failed: ${reason}",
-                       ("reason", ec.message()));
-  }
-}

libraries/libfc/src/network/udp_socket.cpp

@@ -1,111 +0,0 @@
-#include <fc/network/udp_socket.hpp>
-#include <fc/network/ip.hpp>
-
-
-namespace fc
-{
-  class udp_socket::impl
-  {
-    public:
-      impl(){}
-      ~impl(){}
-
-      std::shared_ptr<boost::asio::ip::udp::socket> _sock;
-  };
-
-  udp_socket::udp_socket()
-    : my(new impl())
-  {
-  }
-
-  void udp_socket::initialize(boost::asio::io_service& service)
-  {
-    my->_sock.reset(new boost::asio::ip::udp::socket(service));
-  }
-
-  udp_socket::~udp_socket() 
-  {
-    try 
-    {
-      if(my->_sock)
-        my->_sock->close(); //close boost socket to make any pending reads run their completion handler
-    }
-    catch (...) //avoid destructor throw and likely this is just happening because socket wasn't open.
-    {
-    }
-  }
-
-  void udp_socket::send_to(const char* buffer, size_t length, boost::asio::ip::udp::endpoint& to)
-  {
-    try 
-    {
-      my->_sock->send_to(boost::asio::buffer(buffer, length), to);
-      return;
-    } 
-    catch(const boost::system::system_error& e)
-    {
-      if(e.code() == boost::asio::error::would_block)
-      {
-          auto send_buffer_ptr = std::make_shared<std::vector<char>>(buffer, buffer+length);
-          my->_sock->async_send_to(boost::asio::buffer(send_buffer_ptr.get(), length), to,
-                                   [send_buffer_ptr](const boost::system::error_code& /*ec*/, std::size_t /*bytes_transferred*/)
-          {
-            // Swallow errors.  Currently only used for GELF logging, so depend on local
-            // log to catch anything that doesn't make it across the network.
-          });
-       }
-       // All other exceptions ignored.
-    }
-  }
-
-  void udp_socket::send_to(const std::shared_ptr<const char>& buffer, size_t length,
-                           boost::asio::ip::udp::endpoint& to)
-  {
-    try
-    {
-      my->_sock->send_to(boost::asio::buffer(buffer.get(), length), to);
-      return;
-    } 
-    catch(const boost::system::system_error& e)
-    {
-      if(e.code() == boost::asio::error::would_block)
-      {
-          auto preserved_buffer_ptr = buffer;
-          my->_sock->async_send_to(boost::asio::buffer(preserved_buffer_ptr.get(), length), to,
-                                   [preserved_buffer_ptr](const boost::system::error_code& /*ec*/, std::size_t /*bytes_transferred*/)
-          {
-            // Swallow errors.  Currently only used for GELF logging, so depend on local
-            // log to catch anything that doesn't make it across the network.
-          });
-       }
-       // All other exceptions ignored.
-    }
-  }
-
-  void udp_socket::open()
-  {
-    my->_sock->open(boost::asio::ip::udp::v4());
-    my->_sock->non_blocking(true);
-  }
-
-  void udp_socket::close()
-  {
-    my->_sock->close();
-  }
-
-  const boost::asio::ip::udp::endpoint udp_socket::local_endpoint() const
-  {
-    return my->_sock->local_endpoint();
-  }
-
-  void udp_socket::connect(const boost::asio::ip::udp::endpoint& e)
-  {
-    my->_sock->connect(e);
-  }
-
-  void udp_socket::set_reuse_address( bool s )
-  {
-    my->_sock->set_option( boost::asio::ip::udp::socket::reuse_address(s) );
-  }
-
-}

libraries/state_history/include/eosio/state_history/log.hpp

@@ -89,7 +89,7 @@ namespace state_history {
       fc::path retained_dir       = "retained";
       fc::path archive_dir        = "archive";
       uint32_t stride             = 1000000;
-      uint32_t max_retained_files = 10;
+      uint32_t max_retained_files = UINT32_MAX;
    };
 } // namespace state_history
 
@@ -129,7 +129,7 @@ struct locked_decompress_stream {
 
 namespace detail {
 
-std::vector<char> zlib_decompress(fc::cfile& file, uint64_t compressed_size) {
+inline std::vector<char> zlib_decompress(fc::cfile& file, uint64_t compressed_size) {
    if (compressed_size) {
       std::vector<char> compressed(compressed_size);
       file.read(compressed.data(), compressed_size);
@@ -138,7 +138,7 @@ std::vector<char> zlib_decompress(fc::cfile& file, uint64_t compressed_size) {
    return {};
 }
 
-std::vector<char> zlib_decompress(fc::datastream<const char*>& strm, uint64_t compressed_size) {
+inline std::vector<char> zlib_decompress(fc::datastream<const char*>& strm, uint64_t compressed_size) {
    if (compressed_size) {
       return state_history::zlib_decompress({strm.pos(), compressed_size});
    }
@@ -283,7 +283,7 @@ private:
 class state_history_log {
  private:
    const char* const       name = "";
-   state_history_log_config config;
+   state_history_log_config _config;
 
    // provide exclusive access to all data of this object since accessed from the main thread and the ship thread
    mutable std::mutex      _mx;
@@ -305,7 +305,7 @@ class state_history_log {
    state_history_log(const char* name, const fc::path& log_dir,
                      state_history_log_config conf = {})
        : name(name)
-       , config(std::move(conf)) {
+       , _config(std::move(conf)) {
 
       log.set_file_path(log_dir/(std::string(name) + ".log"));
       index.set_file_path(log_dir/(std::string(name) + ".index"));
@@ -327,7 +327,7 @@ class state_history_log {
                _begin_block = _end_block = catalog.last_block_num() +1;
             }
          }
-      }, config);
+      }, _config);
 
       //check for conversions to/from pruned log, as long as log contains something
       if(_begin_block != _end_block) {
@@ -335,7 +335,7 @@ class state_history_log {
          log.seek(0);
          read_header(first_header);
 
-         auto prune_config = std::get_if<state_history::prune_config>(&config);
+         auto prune_config = std::get_if<state_history::prune_config>(&_config);
 
          if((is_ship_log_pruned(first_header.magic) == false) && prune_config) {
             //need to convert non-pruned to pruned; first prune any ranges we can (might be none)
@@ -362,7 +362,7 @@ class state_history_log {
       if(_begin_block == _end_block)
          return;
 
-      auto prune_config = std::get_if<state_history::prune_config>(&config);
+      auto prune_config = std::get_if<state_history::prune_config>(&_config);
       if(!prune_config || !prune_config->vacuum_on_close)
          return;
 
@@ -372,6 +372,10 @@ class state_history_log {
          vacuum();
    }
 
+   const state_history_log_config& config() const {
+      return _config;
+   }
+
    //        begin     end
    std::pair<uint32_t, uint32_t> block_range() const {
       std::lock_guard g(_mx);
@@ -500,7 +504,7 @@ class state_history_log {
          }
       }
 
-      auto prune_config = std::get_if<state_history::prune_config>(&config);
+      auto prune_config = std::get_if<state_history::prune_config>(&_config);
       if (block_num < _end_block) {
          // This is typically because of a fork, and we need to truncate the log back to the beginning of the fork.
          static uint32_t start_block_num = block_num;
@@ -550,7 +554,10 @@ class state_history_log {
          fc::raw::pack(log, num_blocks_in_log);
       }
 
-      auto partition_config = std::get_if<state_history::partition_config>(&config);
+      log.flush();
+      index.flush();
+
+      auto partition_config = std::get_if<state_history::partition_config>(&_config);
       if (partition_config && block_num % partition_config->stride == 0) {
          split_log();
       }
@@ -606,7 +613,7 @@ class state_history_log {
    }
 
    void prune(const fc::log_level& loglevel) {
-      auto prune_config = std::get_if<state_history::prune_config>(&config);
+      auto prune_config = std::get_if<state_history::prune_config>(&_config);
 
       if(!prune_config)
          return;

libraries/testing/tester.cpp

@@ -272,11 +272,14 @@ namespace eosio { namespace testing {
       if( !expected_chain_id ) {
          expected_chain_id = controller::extract_chain_id_from_db( cfg.state_dir );
          if( !expected_chain_id ) {
-            if( fc::is_regular_file( cfg.blocks_dir / "blocks.log" ) ) {
-               expected_chain_id = block_log::extract_chain_id( cfg.blocks_dir );
-            } else {
-               expected_chain_id = genesis_state().compute_chain_id();
+            fc::path retained_dir;
+            auto partitioned_config = std::get_if<partitioned_blocklog_config>(&cfg.blog);
+            if (partitioned_config) {
+               retained_dir = partitioned_config->retained_dir;
+               if (retained_dir.is_relative())
+                  retained_dir = cfg.blocks_dir/retained_dir;
             }
+            expected_chain_id = block_log::extract_chain_id( cfg.blocks_dir, retained_dir );
          }
       }
 

libraries/wasm-jit/Include/IR/Operators.h

@@ -12,11 +12,10 @@ namespace IR
 	struct NoImm {};
 	struct MemoryImm {};
 
-	PACKED_STRUCT(
 	struct ControlStructureImm
 	{
 		ResultType resultType{};
-	});
+	};
 
 	struct BranchImm
 	{
@@ -676,19 +675,4 @@ namespace IR
 	};
 
 	IR_API const char* getOpcodeName(Opcode opcode);
-}
-
-//paranoia for future platforms
-static_assert(sizeof(IR::OpcodeAndImm<IR::NoImm>) == 2);
-static_assert(sizeof(IR::OpcodeAndImm<IR::MemoryImm>) == 2);
-static_assert(sizeof(IR::OpcodeAndImm<IR::ControlStructureImm>) == 3);
-static_assert(sizeof(IR::OpcodeAndImm<IR::BranchImm>) == 6);
-static_assert(sizeof(IR::OpcodeAndImm<IR::BranchTableImm>) == 18);
-static_assert(sizeof(IR::OpcodeAndImm<IR::LiteralImm<I32>>) == 6);
-static_assert(sizeof(IR::OpcodeAndImm<IR::LiteralImm<I64>>) == 10);
-static_assert(sizeof(IR::OpcodeAndImm<IR::LiteralImm<F32>>) == 6);
-static_assert(sizeof(IR::OpcodeAndImm<IR::LiteralImm<F64>>) == 10);
-static_assert(sizeof(IR::OpcodeAndImm<IR::GetOrSetVariableImm<false>>) == 6);
-static_assert(sizeof(IR::OpcodeAndImm<IR::CallImm>) == 6);
-static_assert(sizeof(IR::OpcodeAndImm<IR::CallIndirectImm>) == 6);
-static_assert(sizeof(IR::OpcodeAndImm<IR::LoadOrStoreImm<0>>) == 10);
+}

plugins/chain_api_plugin/chain.swagger.yaml

@@ -122,10 +122,10 @@ paths:
                   description: Compression used, usually false
                 packed_context_free_data:
                   type: string
-                  description: json to hex
+                  description: JSON to hex
                 packed_trx:
                   type: string
-                  description: Transaction object json to hex
+                  description: Transaction object JSON to hex
       responses:
         "200":
           description: OK
@@ -154,10 +154,10 @@ paths:
                   description: Compression used, usually false
                 packed_context_free_data:
                   type: string
-                  description: json to hex
+                  description: JSON to hex
                 packed_trx:
                   type: string
-                  description: Transaction object json to hex
+                  description: Transaction object JSON to hex
 
       responses:
         "200":
@@ -345,29 +345,18 @@ paths:
               schema:
                 title: "GetProducersResponse"
                 type: object
-                additionalProperties: false
-                minProperties:  3
-                required:
-                  - active
-                  - pending
-                  - proposed
                 properties:
-                  active:
+                  rows:
                     type: array
                     nullable: true
                     items:
-                      $ref: "https://docs.eosnetwork.com/openapi/v2.0/ProducerSchedule.yaml"
-                  pending:
-                    type: array
-                    nullable: true
-                    items:
-                      $ref: "https://docs.eosnetwork.com/openapi/v2.0/ProducerSchedule.yaml"
-                  proposed:
-                    type: array
-                    nullable: true
-                    items:
-                      $ref: "https://docs.eosnetwork.com/openapi/v2.0/ProducerSchedule.yaml"
-
+                      $ref: "https://docs.eosnetwork.com/openapi/v2.0/Producer.yaml"
+                  total_producer_vote_weight: 
+                    type: string
+                    description: The sum of all producer votes.
+                  more:
+                    type: string
+                    description: If not all producers were returned with the first request, more contains the lower bound to use for the next request.
 
   /get_raw_code_and_abi:
     post:
@@ -724,3 +713,218 @@ paths:
                         threshold:
                           type: "integer"
                           description: the sum of weights that must be met or exceeded to satisfy the permission
+  /get_transaction_status:
+    post:
+      description: Attempts to get current blockchain state and, if available, transaction information given the transaction id. For query to work, the transaction finality status feature must be enabled by configuring the chain plugin with the config option '--transaction-finality-status-max-storage-size-gb' in nodeos.
+      operationId: get_transaction_status
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              required:
+                - id
+              properties:
+                id:
+                  type: string
+                  description: The transaction ID of the transaction to retrieve the status for. 
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: "https://docs.eosnetwork.com/openapi/v2.0/TransactionStatus.yaml"
+
+  /send_transaction2:
+    post:
+      description: Attempts to apply a transaction to the blockchain specified in JSON format. It supports returning the full trace of a failed transaction and automatic nodeos-mediated retry if it is enabled on the node. When transaction retry is enabled on an API node, it will monitor incoming API transactions and ensure they are resubmitted additional times into the P2P network until they expire or are included in a block. Warning, full failure traces are now returned by default instead of exceptions. Be careful to not confuse a returned trace as an indication of speculative execution success. Verify 'receipt' and 'except' fields of the returned trace.
+      operationId: send_transaction2
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                return_failure_trace:
+                  type: boolean
+                  description: If true, then embed transaction exceptions into the returned transaction trace.
+                retry_trx:
+                  type: boolean
+                  description: If true, requests to retry transaction until gets in a block of given height, see retry_trx_num_blocks as well, or it is irreversible or expires.
+                retry_trx_num_blocks:
+                  type: integer
+                  description: If retry_trx is true, requests to retry transaction until in a block of given height, or lib if not specified.
+                transaction:
+                  type: object
+                  properties:
+                    signatures:
+                      type: array
+                      description: array of signatures required to authorize transaction.
+                      items:
+                        $ref: "https://docs.eosnetwork.com/openapi/v2.0/Signature.yaml"
+                    compression:
+                      type: boolean
+                      description: Compression used, usually false
+                    packed_context_free_data:
+                      type: string
+                      description: JSON to hex
+                    packed_trx:
+                      type: string
+                      description: Transaction object JSON to hex
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                description: Returns Nothing
+
+  /compute_transaction:
+    post:
+      description: Executes specified transaction and creates a transaction trace, including resource usage, and then reverts all state changes but not contribute to the subjective billing for the account. If the transaction has signatures, they are processed, but any failures are ignored. Transactions which fail always include the transaction failure trace. Warning, users with exposed nodes who have enabled the compute_transaction endpoint should implement some throttling to protect from Denial of Service attacks.
+      operationId: compute_transaction
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                signatures:
+                  type: array
+                  description: array of signatures required to authorize transaction
+                  items:
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Signature.yaml"
+                compression:
+                  type: boolean
+                  description: Compression used, usually false
+                packed_context_free_data:
+                  type: string
+                  description: JSON to hex
+                packed_trx:
+                  type: string
+                  description: Transaction object, JSON to hex
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                description: Returns Nothing
+
+  /get_code_hash:
+    post:
+      description: Retrieves the code hash for a smart contract deployed on the blockchain. Once you have the code hash of a contract, you can compare it with a known or expected value to ensure that the contract code has not been modified or tampered with.
+      operationId: get_code_hash
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                account_name:
+                  description: The name of the account for which you want to retrieve the code hash. It represents the account that owns the smart contract code.
+                  type: string
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  account_name:
+                    description: The name of the account where the smart contract was deployed.
+                    type: string
+                  code_hash:
+                    type: string
+                    description: A string that represents the hash value of the specified account's smart contract code.
+
+  /get_transaction_id:
+    post:
+      description: Retrieves the transaction ID (also known as the transaction hash) of a specified transaction on the blockchain.
+      operationId: get_transaction_id
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              description: The transaction in JSON format for which the ID should be retrieved.
+              $ref: "https://docs.eosnetwork.com/openapi/v2.0/Transaction.yaml"
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: string
+                description: The transaction ID.
+
+  /get_producer_schedule:
+    post:
+      description: Retrieves the current producer schedule from the blockchain, which includes the list of active producers and their respective rotation schedule.
+      operationId: get_producer_schedule
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  active:
+                    description: A JSON object that encapsulates the list of active producers schedule and its version.
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/ProducerSchedule.yaml"                              
+                  pending:
+                    description: A JSON object that encapsulates the list of pending producers schedule and its version.
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/ProducerSchedule.yaml"
+                  proposed:
+                    description: A JSON object that encapsulates the list of proposed producers schedule and its version.
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/ProducerSchedule.yaml"
+
+  /send_read_only_transaction:
+    post:
+      description: Sends a read-only transaction in JSON format to the blockchain. This transaction is not intended for inclusion in the blockchain. When a user sends a transaction, which modifies the blockchain state, the connected node will fail the transaction.
+      operationId: send_read_only_transaction
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                transaction:
+                  type: object
+                  properties:
+                    compression:
+                      type: boolean
+                      description: Compression used, usually false
+                    packed_context_free_data:
+                      type: string
+                      description: JSON to hex
+                    packed_trx:
+                      type: string
+                      description: Transaction object JSON to hex
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                description: Returns Nothing
+
+  /push_block:
+    post:
+      description: Sends a block to the blockchain.
+      operationId: push_block
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "https://docs.eosnetwork.com/openapi/v2.0/Block.yaml"
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                description: Returns Nothing

plugins/chain_api_plugin/chain_api_plugin.cpp

@@ -41,6 +41,55 @@ parse_params<chain_apis::read_only::get_transaction_status_params, http_params_t
    }
 }
 
+// if actions.data & actions.hex_data provided, use the hex_data since only currently support unexploded data
+template<>
+chain_apis::read_only::get_transaction_id_params
+parse_params<chain_apis::read_only::get_transaction_id_params, http_params_types::params_required>(const std::string& body) {
+   if (body.empty()) {
+      EOS_THROW(chain::invalid_http_request, "A Request body is required");
+   }
+
+   try {
+      fc::variant trx_var =  fc::json::from_string( body );
+      if( trx_var.is_object() ) {
+         fc::variant_object& vo = trx_var.get_object();
+         if( vo.contains("actions") && vo["actions"].is_array() ) {
+            fc::mutable_variant_object mvo = vo;
+            fc::variants& action_variants = mvo["actions"].get_array();
+            for( auto& action_v : action_variants ) {
+               if( action_v.is_object() ) {
+                 fc::variant_object& action_vo = action_v.get_object();
+                  if( action_vo.contains( "data" ) && action_vo.contains( "hex_data" ) ) {
+                     fc::mutable_variant_object maction_vo = action_vo;
+                     maction_vo["data"] = maction_vo["hex_data"];
+                     action_vo = maction_vo;
+                     vo = mvo;
+                  } else if( action_vo.contains( "data" ) ) {
+                     if( !action_vo["data"].is_string() ) {
+                        EOS_THROW(chain::invalid_http_request, "Request supports only un-exploded 'data' (hex form)");
+                     }
+                  }
+               }
+               else {
+                  EOS_THROW(chain::invalid_http_request, "Transaction contains invalid or empty action");
+               }
+            } 
+         }
+         else {
+            EOS_THROW(chain::invalid_http_request, "Transaction actions are missing or invalid");
+         }
+      }
+      else {
+         EOS_THROW(chain::invalid_http_request, "Transaction object is missing or invalid");
+      }
+      auto trx = trx_var.as<chain_apis::read_only::get_transaction_id_params>();
+      if( trx.id() == transaction().id() ) {
+         EOS_THROW(chain::invalid_http_request, "Invalid transaction object");
+      }
+      return trx;
+   } EOS_RETHROW_EXCEPTIONS(chain::invalid_http_request, "Invalid transaction");
+}
+
 #define CALL_WITH_400(api_name, api_handle, api_namespace, call_name, http_response_code, params_type) \
 {std::string("/v1/" #api_name "/" #call_name), \
    [api_handle](string&&, string&& body, url_response_callback&& cb) mutable { \
@@ -75,7 +124,7 @@ void chain_api_plugin::plugin_startup() {
    ro_api.set_shorten_abi_errors( !http_plugin::verbose_errors() );
 
    _http_plugin.add_api( {
-      CHAIN_RO_CALL(get_info, 200, http_params_types::no_params)}, appbase::exec_queue::read_only_trx_safe, appbase::priority::medium_high);
+      CHAIN_RO_CALL(get_info, 200, http_params_types::no_params)}, appbase::exec_queue::read_only, appbase::priority::medium_high);
    _http_plugin.add_api({
       CHAIN_RO_CALL(get_activated_protocol_features, 200, http_params_types::possible_no_params),
       CHAIN_RO_CALL(get_block_info, 200, http_params_types::params_required),
@@ -96,19 +145,19 @@ void chain_api_plugin::plugin_startup() {
       CHAIN_RO_CALL(get_scheduled_transactions, 200, http_params_types::params_required),
       CHAIN_RO_CALL(get_required_keys, 200, http_params_types::params_required),
       CHAIN_RO_CALL(get_transaction_id, 200, http_params_types::params_required),
+      // transaction related APIs will be posted to read_write queue after keys are recovered, they are safe to run in parallel until they post to the read_write queue
       CHAIN_RO_CALL_ASYNC(send_read_only_transaction, chain_apis::read_only::send_read_only_transaction_results, 200, http_params_types::params_required),
       CHAIN_RO_CALL_ASYNC(compute_transaction, chain_apis::read_only::compute_transaction_results, 200, http_params_types::params_required),
-      // transaction related APIs will be posted to general queue after keys are recovered
       CHAIN_RW_CALL_ASYNC(push_transaction, chain_apis::read_write::push_transaction_results, 202, http_params_types::params_required),
       CHAIN_RW_CALL_ASYNC(push_transactions, chain_apis::read_write::push_transactions_results, 202, http_params_types::params_required),
       CHAIN_RW_CALL_ASYNC(send_transaction, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required),
       CHAIN_RW_CALL_ASYNC(send_transaction2, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required)
-   }, appbase::exec_queue::read_only_trx_safe);
+   }, appbase::exec_queue::read_only);
 
    // Not safe to run in parallel with read-only transactions
    _http_plugin.add_api({
       CHAIN_RW_CALL_ASYNC(push_block, chain_apis::read_write::push_block_results, 202, http_params_types::params_required)
-   }, appbase::exec_queue::general, appbase::priority::medium_low );
+   }, appbase::exec_queue::read_write, appbase::priority::medium_low );
 
    if (chain.account_queries_enabled()) {
       _http_plugin.add_async_api({
@@ -124,7 +173,7 @@ void chain_api_plugin::plugin_startup() {
    if (chain.transaction_finality_status_enabled()) {
       _http_plugin.add_api({
          CHAIN_RO_CALL_WITH_400(get_transaction_status, 200, http_params_types::params_required),
-      }, appbase::exec_queue::read_only_trx_safe);
+      }, appbase::exec_queue::read_only);
    }
 
    _http_plugin.add_api({
@@ -160,7 +209,7 @@ void chain_api_plugin::plugin_startup() {
            }
         }
       }
-   }, appbase::exec_queue::read_only_trx_safe);
+   }, appbase::exec_queue::read_only);
 }
 
 void chain_api_plugin::plugin_shutdown() {}

plugins/chain_plugin/chain_plugin.cpp

@@ -51,6 +51,8 @@ std::ostream& operator<<(std::ostream& osm, eosio::chain::db_read_mode m) {
       osm << "head";
    } else if ( m == eosio::chain::db_read_mode::IRREVERSIBLE ) {
       osm << "irreversible";
+   } else if ( m == eosio::chain::db_read_mode::SPECULATIVE ) {
+      osm << "speculative";
    }
 
    return osm;
@@ -70,10 +72,12 @@ void validate(boost::any& v,
   // one string, it's an error, and exception will be thrown.
   std::string const& s = validators::get_single_string(values);
 
- if ( s == "head" ) {
+  if ( s == "head" ) {
      v = boost::any(eosio::chain::db_read_mode::HEAD);
   } else if ( s == "irreversible" ) {
      v = boost::any(eosio::chain::db_read_mode::IRREVERSIBLE);
+  } else if ( s == "speculative" ) {
+     v = boost::any(eosio::chain::db_read_mode::SPECULATIVE);
   } else {
      throw validation_error(validation_error::invalid_option_value);
   }
@@ -286,10 +290,12 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip
          ("sender-bypass-whiteblacklist", boost::program_options::value<vector<string>>()->composing()->multitoken(),
           "Deferred transactions sent by accounts in this list do not have any of the subjective whitelist/blacklist checks applied to them (may specify multiple times)")
          ("read-mode", boost::program_options::value<eosio::chain::db_read_mode>()->default_value(eosio::chain::db_read_mode::HEAD),
-          "Database read mode (\"head\", \"irreversible\").\n"
+          "Database read mode (\"head\", \"irreversible\", \"speculative\").\n"
           "In \"head\" mode: database contains state changes up to the head block; transactions received by the node are relayed if valid.\n"
           "In \"irreversible\" mode: database contains state changes up to the last irreversible block; "
           "transactions received via the P2P network are not relayed and transactions cannot be pushed via the chain API.\n"
+          "In \"speculative\" mode: (DEPRECATED: head mode recommended) database contains state changes by transactions in the blockchain "
+          "up to the head block as well as some transactions not yet included in the blockchain; transactions received by the node are relayed if valid.\n"
           )
          ( "api-accept-transactions", bpo::value<bool>()->default_value(true), "Allow API transactions to be evaluated and relayed if valid.")
          ("validation-mode", boost::program_options::value<eosio::chain::validation_mode>()->default_value(eosio::chain::validation_mode::FULL),
@@ -638,21 +644,56 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
       if( options.count( "terminate-at-block" ))
          my->chain_config->terminate_at_block = options.at( "terminate-at-block" ).as<uint32_t>();
 
+      // move fork_db to new location
+      upgrade_from_reversible_to_fork_db( my.get() );
+
+      bool has_partitioned_block_log_options = options.count("blocks-retained-dir") ||  options.count("blocks-archive-dir")
+         || options.count("blocks-log-stride") || options.count("max-retained-block-files");
+      bool has_retain_blocks_option = options.count("block-log-retain-blocks");
+
+      EOS_ASSERT(!has_partitioned_block_log_options || !has_retain_blocks_option, plugin_config_exception,
+         "block-log-retain-blocks cannot be specified together with blocks-retained-dir, blocks-archive-dir or blocks-log-stride or max-retained-block-files.");
+
+      fc::path retained_dir;
+      if (has_partitioned_block_log_options) {
+         retained_dir = options.count("blocks-retained-dir") ? options.at("blocks-retained-dir").as<bfs::path>()
+                                                                 : bfs::path("");
+         if (retained_dir.is_relative())
+            retained_dir = fc::path{my->blocks_dir}/retained_dir;
+         my->chain_config->blog = eosio::chain::partitioned_blocklog_config{
+            .retained_dir = retained_dir,
+            .archive_dir  = options.count("blocks-archive-dir") ? options.at("blocks-archive-dir").as<bfs::path>()
+                                                                : bfs::path("archive"),
+            .stride       = options.count("blocks-log-stride") ? options.at("blocks-log-stride").as<uint32_t>()
+                                                               : UINT32_MAX,
+            .max_retained_files = options.count("max-retained-block-files")
+                                        ? options.at("max-retained-block-files").as<uint32_t>()
+                                        : UINT32_MAX,
+         };
+      } else if(has_retain_blocks_option) {
+         uint32_t block_log_retain_blocks = options.at("block-log-retain-blocks").as<uint32_t>();
+         if (block_log_retain_blocks == 0)
+            my->chain_config->blog = eosio::chain::empty_blocklog_config{};
+         else {
+            EOS_ASSERT(cfile::supports_hole_punching(), plugin_config_exception,
+                       "block-log-retain-blocks cannot be greater than 0 because the file system does not support hole "
+                       "punching");
+            my->chain_config->blog = eosio::chain::prune_blocklog_config{ .prune_blocks = block_log_retain_blocks };
+         }
+      }
+
+      
+
       if( options.count( "extract-genesis-json" ) || options.at( "print-genesis-json" ).as<bool>()) {
          std::optional<genesis_state> gs;
-
-         if( fc::exists( my->blocks_dir / "blocks.log" )) {
-            gs = block_log::extract_genesis_state( my->blocks_dir );
-            EOS_ASSERT( gs,
-                        plugin_config_exception,
-                        "Block log at '${path}' does not contain a genesis state, it only has the chain-id.",
-                        ("path", (my->blocks_dir / "blocks.log").generic_string())
-            );
-         } else {
-            wlog( "No blocks.log found at '${p}'. Using default genesis state.",
-                  ("p", (my->blocks_dir / "blocks.log").generic_string()));
-            gs.emplace();
-         }
+         
+         gs = block_log::extract_genesis_state( my->blocks_dir, retained_dir );
+         EOS_ASSERT( gs,
+                     plugin_config_exception,
+                     "Block log at '${path}' does not contain a genesis state, it only has the chain-id.",
+                     ("path", (my->blocks_dir / "blocks.log").generic_string())
+         );
+         
 
          if( options.at( "print-genesis-json" ).as<bool>()) {
             ilog( "Genesis JSON:\n${genesis}", ("genesis", json::to_pretty_string( *gs )));
@@ -677,41 +718,6 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
          EOS_THROW( extract_genesis_state_exception, "extracted genesis state from blocks.log" );
       }
 
-      // move fork_db to new location
-      upgrade_from_reversible_to_fork_db( my.get() );
-
-      bool has_partitioned_block_log_options = options.count("blocks-retained-dir") ||  options.count("blocks-archive-dir")
-         || options.count("blocks-log-stride") || options.count("max-retained-block-files");
-      bool has_retain_blocks_option = options.count("block-log-retain-blocks");
-
-      EOS_ASSERT(!has_partitioned_block_log_options || !has_retain_blocks_option, plugin_config_exception,
-         "block-log-retain-blocks cannot be specified together with blocks-retained-dir, blocks-archive-dir or blocks-log-stride or max-retained-block-files.");
-
-
-      if (has_partitioned_block_log_options) {
-         my->chain_config->blog = eosio::chain::partitioned_blocklog_config{
-            .retained_dir = options.count("blocks-retained-dir") ? options.at("blocks-retained-dir").as<bfs::path>()
-                                                                 : bfs::path(""),
-            .archive_dir  = options.count("blocks-archive-dir") ? options.at("blocks-archive-dir").as<bfs::path>()
-                                                                : bfs::path("archive"),
-            .stride       = options.count("blocks-log-stride") ? options.at("blocks-log-stride").as<uint32_t>()
-                                                               : UINT32_MAX,
-            .max_retained_files = options.count("max-retained-block-files")
-                                        ? options.at("max-retained-block-files").as<uint32_t>()
-                                        : UINT32_MAX,
-         };
-      } else if(has_retain_blocks_option) {
-         uint32_t block_log_retain_blocks = options.at("block-log-retain-blocks").as<uint32_t>();
-         if (block_log_retain_blocks == 0)
-            my->chain_config->blog = eosio::chain::empty_blocklog_config{};
-         else {
-            EOS_ASSERT(cfile::supports_hole_punching(), plugin_config_exception,
-                       "block-log-retain-blocks cannot be greater than 0 because the file system does not support hole "
-                       "punching");
-            my->chain_config->blog = eosio::chain::prune_blocklog_config{ .prune_blocks = block_log_retain_blocks };
-         }
-      }
-
       if( options.at( "delete-all-blocks" ).as<bool>()) {
          ilog( "Deleting state database and blocks" );
          if( options.at( "truncate-at-block" ).as<uint32_t>() > 0 )
@@ -755,41 +761,35 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
                  plugin_config_exception,
                  "Snapshot can only be used to initialize an empty database." );
 
-         if( fc::is_regular_file( my->blocks_dir / "blocks.log" )) {
-            auto block_log_genesis = block_log::extract_genesis_state(my->blocks_dir);
-            if( block_log_genesis ) {
-               const auto& block_log_chain_id = block_log_genesis->compute_chain_id();
-               EOS_ASSERT( *chain_id == block_log_chain_id,
-                           plugin_config_exception,
-                           "snapshot chain ID (${snapshot_chain_id}) does not match the chain ID from the genesis state in the block log (${block_log_chain_id})",
-                           ("snapshot_chain_id",  *chain_id)
-                           ("block_log_chain_id", block_log_chain_id)
-               );
-            } else {
-               const auto& block_log_chain_id = block_log::extract_chain_id(my->blocks_dir);
-               EOS_ASSERT( *chain_id == block_log_chain_id,
+         auto block_log_chain_id = block_log::extract_chain_id(my->blocks_dir, retained_dir);
+
+         if (block_log_chain_id) {
+            EOS_ASSERT( *chain_id == *block_log_chain_id,
                            plugin_config_exception,
                            "snapshot chain ID (${snapshot_chain_id}) does not match the chain ID (${block_log_chain_id}) in the block log",
                            ("snapshot_chain_id",  *chain_id)
-                           ("block_log_chain_id", block_log_chain_id)
+                           ("block_log_chain_id", *block_log_chain_id)
                );
-            }
          }
 
       } else {
 
          chain_id = controller::extract_chain_id_from_db( my->chain_config->state_dir );
 
+         auto chain_context = block_log::extract_chain_context( my->blocks_dir, retained_dir );
          std::optional<genesis_state> block_log_genesis;
-         std::optional<chain_id_type> block_log_chain_id;
+         std::optional<chain_id_type> block_log_chain_id;  
 
-         if( fc::is_regular_file( my->blocks_dir / "blocks.log" ) ) {
-            block_log_genesis = block_log::extract_genesis_state( my->blocks_dir );
-            if( block_log_genesis ) {
-               block_log_chain_id = block_log_genesis->compute_chain_id();
-            } else {
-               block_log_chain_id = block_log::extract_chain_id( my->blocks_dir );
-            }
+         if (chain_context) {
+            std::visit(overloaded {
+               [&](const genesis_state& gs) {
+                  block_log_genesis = gs;
+                  block_log_chain_id = gs.compute_chain_id();
+               },
+               [&](const chain_id_type& id) {
+                  block_log_chain_id = id;
+               } 
+            }, *chain_context);
 
             if( chain_id ) {
                EOS_ASSERT( *block_log_chain_id == *chain_id, block_log_exception,
@@ -2686,6 +2686,11 @@ fc::variant chain_plugin::get_log_trx(const transaction& trx) const {
     }
     return pretty_output;
 }
+
+const controller::config& chain_plugin::chain_config() const {
+   EOS_ASSERT(my->chain_config.has_value(), plugin_exception, "chain_config not initialized");
+   return *my->chain_config;
+}
 } // namespace eosio
 
 FC_REFLECT( eosio::chain_apis::detail::ram_market_exchange_state_t, (ignore1)(ignore2)(ignore3)(core_symbol)(ignore4) )

plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp

@@ -829,7 +829,7 @@ public:
             // the following will convert the input to array of 2 uint128_t in little endian, i.e. 50f0fa8360ec998f4bb65b00c86282f5 fb54b91bfed2fe7fe39a92d999d002c5
             // which is the format used by secondary index
             chain::key256_t k;
-            uint8_t buffer[32];
+            uint8_t buffer[32] = {};
             boost::multiprecision::export_bits(v, buffer, 8, false);
             memcpy(&k[0], buffer + 16, 16);
             memcpy(&k[1], buffer, 16);
@@ -886,6 +886,8 @@ public:
    // return variant of trx for logging, trace is modified to minimize log output
    fc::variant get_log_trx(const transaction& trx) const;
 
+   const controller::config& chain_config() const;
+
 private:
    static void log_guard_exception(const chain::guard_exception& e);
 

plugins/chain_plugin/test/CMakeLists.txt

@@ -6,6 +6,6 @@ add_executable( test_trx_retry_db test_trx_retry_db.cpp )
 target_link_libraries( test_trx_retry_db chain_plugin eosio_testing)
 add_test(NAME test_trx_retry_db COMMAND plugins/chain_plugin/test/test_trx_retry_db WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 
-add_executable( test_trx_finality_status_processing test_trx_finality_status_processing.cpp )
+add_executable( test_trx_finality_status_processing test_trx_finality_status_processing.cpp plugin_config_test.cpp)
 target_link_libraries( test_trx_finality_status_processing chain_plugin eosio_testing)
 add_test(NAME test_trx_finality_status_processing COMMAND plugins/chain_plugin/test/test_trx_finality_status_processing WORKING_DIRECTORY ${CMAKE_BINARY_DIR})

plugins/chain_plugin/test/plugin_config_test.cpp

@@ -0,0 +1,22 @@
+#include <array>
+#include <boost/test/unit_test.hpp>
+#include <eosio/chain/application.hpp>
+#include <eosio/chain_plugin/chain_plugin.hpp>
+#include <stdint.h>
+
+BOOST_AUTO_TEST_CASE(chain_plugin_default_tests) {
+   appbase::scoped_app app;
+   fc::temp_directory  tmp;
+
+   auto tmp_path = tmp.path().string();
+   std::array          args = {
+       "test_chain_plugin", "--blocks-log-stride", "10", "--data-dir", tmp_path.c_str(),
+   };
+
+   BOOST_CHECK(app->initialize<eosio::chain_plugin>(args.size(), const_cast<char**>(args.data())));
+   auto& plugin = app->get_plugin<eosio::chain_plugin>();
+
+   auto* config = std::get_if<eosio::chain::partitioned_blocklog_config>(&plugin.chain_config().blog);
+   BOOST_REQUIRE(config);
+   BOOST_CHECK_EQUAL(config->max_retained_files, UINT32_MAX);
+}

plugins/db_size_api_plugin/db_size_api_plugin.cpp

@@ -28,7 +28,7 @@ using namespace eosio;
 void db_size_api_plugin::plugin_startup() {
    app().get_plugin<http_plugin>().add_api({
        CALL_WITH_400(db_size, this, get,  INVOKE_R_V(this, get), 200),
-   }, appbase::exec_queue::read_only_trx_safe);
+   }, appbase::exec_queue::read_only);
 }
 
 db_size_stats db_size_api_plugin::get() {

plugins/http_plugin/http_plugin.cpp

@@ -28,7 +28,7 @@ namespace eosio {
    using std::regex;
    using boost::asio::ip::tcp;
    using std::shared_ptr;
-   
+
    static http_plugin_defaults current_http_plugin_defaults;
    static bool verbose_http_errors = false;
 
@@ -36,6 +36,10 @@ namespace eosio {
       current_http_plugin_defaults = config;
    }
 
+   std::string http_plugin::get_server_header() {
+      return current_http_plugin_defaults.server_header;
+   }
+
    using http_plugin_impl_ptr = std::shared_ptr<class http_plugin_impl>;
 
    class http_plugin_impl : public std::enable_shared_from_this<http_plugin_impl> {
@@ -47,9 +51,9 @@ namespace eosio {
 
          http_plugin_impl& operator=(const http_plugin_impl&) = delete;
          http_plugin_impl& operator=(http_plugin_impl&&) = delete;
-         
+
          std::optional<tcp::endpoint>  listen_endpoint;
-         
+
          std::optional<asio::local::stream_protocol::endpoint> unix_endpoint;
 
          shared_ptr<beast_http_listener<plain_session, tcp, tcp_socket_t > >  beast_server;
@@ -272,7 +276,7 @@ namespace eosio {
 
          my->plugin_state->server_header = current_http_plugin_defaults.server_header;
 
-         
+
          //watch out for the returns above when adding new code here
       } FC_LOG_AND_RETHROW()
    }
@@ -309,7 +313,7 @@ namespace eosio {
             if(my->unix_endpoint) {
                try {
                   my->create_beast_server(true);
-                  
+
                   my->beast_unix_server->listen(*my->unix_endpoint);
                   my->beast_unix_server->start_accept();
                } catch ( const fc::exception& e ){
@@ -334,8 +338,8 @@ namespace eosio {
                      handle_exception("node", "get_supported_apis", body.empty() ? "{}" : body, cb);
                   }
                }
-            }}, appbase::exec_queue::read_only_trx_safe);
-            
+            }}, appbase::exec_queue::read_only);
+
          } catch (...) {
             fc_elog(logger(), "http_plugin startup fails, shutting down");
             app().quit();
@@ -381,22 +385,30 @@ namespace eosio {
          boost::asio::post( my->plugin_state->thread_pool.get_executor(), f );
    }
 
-   void http_plugin::handle_exception( const char *api_name, const char *call_name, const string& body, const url_response_callback& cb) {
+   void http_plugin::handle_exception( const char* api_name, const char* call_name, const string& body, const url_response_callback& cb) {
       try {
          try {
             throw;
          } catch (chain::unknown_block_exception& e) {
             error_results results{400, "Unknown Block", error_results::error_info(e, verbose_http_errors)};
             cb( 400, fc::time_point::maximum(), fc::variant( results ));
+            fc_dlog( logger(), "Unknown block while processing ${api}.${call}: ${e}",
+                     ("api", api_name)("call", call_name)("e", e.to_detail_string()) );
          } catch (chain::invalid_http_request& e) {
             error_results results{400, "Invalid Request", error_results::error_info(e, verbose_http_errors)};
             cb( 400, fc::time_point::maximum(), fc::variant( results ));
+            fc_dlog( logger(), "Invalid http request while processing ${api}.${call}: ${e}",
+                     ("api", api_name)("call", call_name)("e", e.to_detail_string()) );
          } catch (chain::unsatisfied_authorization& e) {
             error_results results{401, "UnAuthorized", error_results::error_info(e, verbose_http_errors)};
             cb( 401, fc::time_point::maximum(), fc::variant( results ));
+            fc_dlog( logger(), "Auth error while processing ${api}.${call}: ${e}",
+                     ("api", api_name)("call", call_name)("e", e.to_detail_string()) );
          } catch (chain::tx_duplicate& e) {
             error_results results{409, "Conflict", error_results::error_info(e, verbose_http_errors)};
             cb( 409, fc::time_point::maximum(), fc::variant( results ));
+            fc_dlog( logger(), "Duplicate trx while processing ${api}.${call}: ${e}",
+                     ("api", api_name)("call", call_name)("e", e.to_detail_string()) );
          } catch (fc::eof_exception& e) {
             error_results results{422, "Unprocessable Entity", error_results::error_info(e, verbose_http_errors)};
             cb( 422, fc::time_point::maximum(), fc::variant( results ));
@@ -454,9 +466,9 @@ namespace eosio {
    fc::microseconds http_plugin::get_max_response_time()const {
       return my->plugin_state->max_response_time;
    }
-   
+
    size_t http_plugin::get_max_body_size()const {
       return my->plugin_state->max_body_size;
    }
-   
+
 }

plugins/http_plugin/include/eosio/http_plugin/beast_http_listener.hpp

@@ -121,9 +121,13 @@ private:
                fail(ec, "accept", self->plugin_state_->logger, "closing connection");
             } else {
                // Create the session object and run it
+               boost::system::error_code re_ec;
+               auto re = self->socket_.remote_endpoint(re_ec);
+               std::string remote_endpoint = re_ec ? "unknown" : boost::lexical_cast<std::string>(re);
                std::make_shared<session_type>(
                   std::move(self->socket_),
-                  self->plugin_state_)
+                  self->plugin_state_,
+                  std::move(remote_endpoint))
                   ->run_session();
             }
             

plugins/http_plugin/include/eosio/http_plugin/beast_http_session.hpp

@@ -3,6 +3,9 @@
 #include <eosio/http_plugin/common.hpp>
 #include <fc/io/json.hpp>
 
+#include <boost/iostreams/device/array.hpp>
+#include <boost/iostreams/stream.hpp>
+
 #include <memory>
 #include <string>
 #include <charconv>
@@ -74,6 +77,26 @@ bool allow_host(const http::request<http::string_body>& req, T& session,
 
 // Handle HTTP connection using boost::beast for TCP communication
 // Subclasses of this class (plain_session, ssl_session (now removed), etc.)
+// T can be request or response or anything serializable to boost iostreams
+template<typename T>
+std::string to_log_string(const T& req, size_t max_size = 1024) {
+   assert( max_size > 4 );
+   std::string buffer( max_size, '\0' );
+   {
+      boost::iostreams::array_sink sink( buffer.data(), buffer.size() );
+      boost::iostreams::stream stream( sink );
+      stream << req;
+   }
+   buffer.resize( std::strlen( buffer.data() ) );
+   if( buffer.size() == max_size ) {
+      buffer[max_size - 3] = '.';
+      buffer[max_size - 2] = '.';
+      buffer[max_size - 1] = '.';
+   }
+   std::replace_if( buffer.begin(), buffer.end(), []( unsigned char c ) { return c == '\r' || c == '\n'; }, ' ' );
+   return buffer;
+}
+
 // use the Curiously Recurring Template Pattern so that
 // the same code works with both regular TCP sockets and UNIX sockets
 template<class Derived>
@@ -92,6 +115,7 @@ protected:
    std::optional<http::response<http::string_body>> res_;
 
    std::shared_ptr<http_plugin_state> plugin_state_;
+   std::string remote_endpoint_;
 
    // whether response should be sent back to client when an exception occurs
    bool is_send_exception_response_ = true;
@@ -130,8 +154,12 @@ protected:
       }
 
       try {
-         if(!derived().allow_host(req))
+         if(!derived().allow_host(req)) {
+            error_results results{static_cast<uint16_t>(http::status::bad_request), "Disallowed HTTP HOST header in the request"};
+            send_response( fc::json::to_string( results, fc::time_point::maximum() ),
+                        static_cast<unsigned int>(http::status::bad_request) );
             return;
+         }
 
          if(!plugin_state_->access_control_allow_origin.empty()) {
             res_->set("Access-Control-Allow-Origin", plugin_state_->access_control_allow_origin);
@@ -152,6 +180,9 @@ protected:
             return;
          }
 
+         fc_dlog( plugin_state_->logger, "Request:  ${ep} ${r}",
+                  ("ep", remote_endpoint_)("r", to_log_string(req)) );
+
          std::string resource = std::string(req.target());
          // look for the URL handler to handle this resource
          auto handler_itr = plugin_state_->url_handlers.find(resource);
@@ -245,9 +276,9 @@ public:
    // shared_from_this() requires default constructor
    beast_http_session() = default;
 
-   beast_http_session(
-         std::shared_ptr<http_plugin_state> plugin_state)
-       : plugin_state_(std::move(plugin_state)) {
+   beast_http_session(std::shared_ptr<http_plugin_state> plugin_state, std::string remote_endpoint)
+       : plugin_state_(std::move(plugin_state)),
+         remote_endpoint_(std::move(remote_endpoint)) {
       plugin_state_->requests_in_flight += 1;
       req_parser_.emplace();
       req_parser_->body_limit(plugin_state_->max_body_size);
@@ -468,6 +499,9 @@ public:
       // Determine if we should close the connection after
       bool close = !(plugin_state_->keep_alive) || res_->need_eof();
 
+      fc_dlog( plugin_state_->logger, "Response: ${ep} ${b}",
+               ("ep", remote_endpoint_)("b", to_log_string(*res_)) );
+
       // Write the response
       http::async_write(
          derived().stream(),
@@ -498,8 +532,9 @@ public:
    // Create the session
    plain_session(
          tcp_socket_t socket,
-         std::shared_ptr<http_plugin_state> plugin_state)
-       : beast_http_session<plain_session>(std::move(plugin_state)), socket_(std::move(socket)) {}
+         std::shared_ptr<http_plugin_state> plugin_state,
+         std::string remote_endpoint)
+       : beast_http_session<plain_session>(std::move(plugin_state), std::move(remote_endpoint)), socket_(std::move(socket)) {}
 
    tcp_socket_t& stream() { return socket_; }
    tcp_socket_t& socket() { return socket_; }
@@ -542,8 +577,9 @@ class unix_socket_session
 
 public:
    unix_socket_session(stream_protocol::socket sock,
-                       std::shared_ptr<http_plugin_state> plugin_state)
-       : beast_http_session(std::move(plugin_state)), socket_(std::move(sock)) {}
+                       std::shared_ptr<http_plugin_state> plugin_state,
+                       std::string remote_endpoint)
+       : beast_http_session(std::move(plugin_state), std::move(remote_endpoint)), socket_(std::move(sock)) {}
 
    virtual ~unix_socket_session() = default;
 

plugins/http_plugin/include/eosio/http_plugin/http_plugin.hpp

@@ -78,6 +78,7 @@ namespace eosio {
 
         //must be called before initialize
         static void set_defaults(const http_plugin_defaults& config);
+        static std::string get_server_header();
 
         APPBASE_PLUGIN_REQUIRES()
         void set_program_options(options_description&, options_description& cfg) override;
@@ -121,7 +122,7 @@ namespace eosio {
         void register_metrics_listener(chain::plugin_interface::metrics_listener listener);
 
         size_t get_max_body_size()const;
-      
+
    private:
         std::shared_ptr<class http_plugin_impl> my;
    };
@@ -175,7 +176,7 @@ namespace eosio {
    };
 
    /**
-    * @brief Used to trim whitespace from body. 
+    * @brief Used to trim whitespace from body.
     * Returned string_view valid only for lifetime of body
     */
    inline std::string_view make_trimmed_string_view(const std::string& body) {

plugins/http_plugin/tests/unit_tests.cpp

@@ -52,7 +52,7 @@ public:
                   cb(200, fc::time_point::maximum(), fc::variant(ok ? string("yes") : string("no")));
                }
             },
-         }, appbase::exec_queue::general);
+         }, appbase::exec_queue::read_write);
    }
 
 private:

plugins/net_api_plugin/net_api_plugin.cpp

@@ -53,11 +53,7 @@ void net_api_plugin::plugin_startup() {
    ilog("starting net_api_plugin");
    // lifetime of plugin is lifetime of application
    auto& net_mgr = app().get_plugin<net_plugin>();
-   app().get_plugin<http_plugin>().add_api({
-    //   CALL(net, net_mgr, set_timeout,
-    //        INVOKE_V_R(net_mgr, set_timeout, int64_t), 200),
-    //   CALL(net, net_mgr, sign_transaction,
-    //        INVOKE_R_R_R_R(net_mgr, sign_transaction, chain::signed_transaction, flat_set<public_key_type>, chain::chain_id_type), 201),
+   app().get_plugin<http_plugin>().add_async_api({
        CALL_WITH_400(net, net_mgr, connect,
             INVOKE_R_R(net_mgr, connect, std::string), 201),
        CALL_WITH_400(net, net_mgr, disconnect,
@@ -66,9 +62,7 @@ void net_api_plugin::plugin_startup() {
             INVOKE_R_R(net_mgr, status, std::string), 201),
        CALL_WITH_400(net, net_mgr, connections,
             INVOKE_R_V(net_mgr, connections), 201),
-    //   CALL(net, net_mgr, open,
-    //        INVOKE_V_R(net_mgr, open, std::string), 200),
-   }, appbase::exec_queue::read_only_trx_safe, appbase::priority::medium_high);
+   } );
 }
 
 void net_api_plugin::plugin_initialize(const variables_map& options) {

plugins/net_plugin/include/eosio/net_plugin/protocol.hpp

@@ -7,9 +7,6 @@ namespace eosio {
    using namespace chain;
    using namespace fc;
 
-   static_assert(sizeof(std::chrono::system_clock::duration::rep) >= 8, "system_clock is expected to be at least 64 bits");
-   typedef std::chrono::system_clock::duration::rep tstamp;
-
    struct chain_size_message {
       uint32_t                   last_irreversible_block_num = 0;
       block_id_type              last_irreversible_block_id;
@@ -83,10 +80,10 @@ namespace eosio {
   };
 
   struct time_message {
-            tstamp  org{0};       //!< origin timestamp
-            tstamp  rec{0};       //!< receive timestamp
-            tstamp  xmt{0};       //!< transmit timestamp
-    mutable tstamp  dst{0};       //!< destination timestamp
+            int64_t  org{0};       //!< origin timestamp, in nanoseconds
+            int64_t  rec{0};       //!< receive timestamp, in nanoseconds
+            int64_t  xmt{0};       //!< transmit timestamp, in nanoseconds
+    mutable int64_t  dst{0};       //!< destination timestamp, in nanoseconds
   };
 
   enum id_list_modes {

plugins/net_plugin/net_plugin.cpp

@@ -51,6 +51,9 @@ namespace eosio {
    using connection_ptr = std::shared_ptr<connection>;
    using connection_wptr = std::weak_ptr<connection>;
 
+   static constexpr int64_t block_interval_ns =
+       std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::milliseconds(config::block_interval_ms)).count();
+
    const fc::string logger_name("net_plugin_impl");
    fc::logger logger;
    std::string peer_log_format;
@@ -121,9 +124,6 @@ namespace eosio {
          in_sync
       };
 
-      static constexpr int64_t block_interval_ns =
-            std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::milliseconds(config::block_interval_ms)).count();
-
       mutable std::mutex sync_mtx;
       uint32_t       sync_known_lib_num{0};
       uint32_t       sync_last_requested_num{0};
@@ -607,8 +607,7 @@ namespace eosio {
       bool is_transactions_only_connection()const { return connection_type == transactions_only; }
       bool is_blocks_only_connection()const { return connection_type == blocks_only; }
       void set_heartbeat_timeout(std::chrono::milliseconds msec) {
-         std::chrono::system_clock::duration dur = msec;
-         hb_timeout = dur.count();
+         hb_timeout = msec;
       }
 
    private:
@@ -682,15 +681,15 @@ namespace eosio {
        *  @{
        */
       // Members set from network data
-      tstamp                         org{0};          //!< originate timestamp
-      tstamp                         rec{0};          //!< receive timestamp
-      tstamp                         dst{0};          //!< destination timestamp
-      tstamp                         xmt{0};          //!< transmit timestamp
+      std::chrono::nanoseconds               org{0}; //!< origin timestamp. Time at the client when the request departed for the server.
+      // std::chrono::nanoseconds (not used) rec{0}; //!< receive timestamp. Time at the server when the request arrived from the client.
+      std::chrono::nanoseconds               xmt{0}; //!< transmit timestamp, Time at the server when the response left for the client.
+      // std::chrono::nanoseconds (not used) dst{0}; //!< destination timestamp, Time at the client when the reply arrived from the server.
       /** @} */
       // timestamp for the lastest message
-      tstamp                         latest_msg_time{0};
-      tstamp                         hb_timeout{std::chrono::milliseconds{def_keepalive_interval}.count()};
-      tstamp                         latest_blk_time{0};
+      std::chrono::system_clock::time_point       latest_msg_time{std::chrono::system_clock::time_point::min()};
+      std::chrono::milliseconds                   hb_timeout{std::chrono::milliseconds{def_keepalive_interval}};
+      std::chrono::system_clock::time_point       latest_blk_time{std::chrono::system_clock::time_point::min()};
 
       bool connected();
       bool current();
@@ -728,7 +727,7 @@ namespace eosio {
        */
       /**  \brief Check heartbeat time and send Time_message
        */
-      void check_heartbeat( tstamp current_time );
+      void check_heartbeat( std::chrono::system_clock::time_point current_time );
       /**  \brief Populate and queue time_message
        */
       void send_time();
@@ -742,8 +741,8 @@ namespace eosio {
        * packet is placed on the send queue.  Calls the kernel time of
        * day routine and converts to a (at least) 64 bit integer.
        */
-      static tstamp get_time() {
-         return std::chrono::system_clock::now().time_since_epoch().count();
+      static std::chrono::nanoseconds get_time() {
+         return std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::system_clock::now().time_since_epoch());
       }
       /** @} */
 
@@ -1051,6 +1050,8 @@ namespace eosio {
       if( !shutdown) my_impl->sync_master->sync_reset_lib_num( self->shared_from_this(), true );
       peer_ilog( self, "closing" );
       self->cancel_wait();
+      self->latest_msg_time = std::chrono::system_clock::time_point::min();
+      self->latest_blk_time = std::chrono::system_clock::time_point::min();
 
       if( reconnect && !shutdown ) {
          my_impl->start_conn_timer( std::chrono::milliseconds( 100 ), connection_wptr() );
@@ -1165,8 +1166,8 @@ namespace eosio {
    }
 
    // called from connection strand
-   void connection::check_heartbeat( tstamp current_time ) {
-      if( latest_msg_time > 0 ) {
+   void connection::check_heartbeat( std::chrono::system_clock::time_point current_time ) {
+      if( latest_msg_time > std::chrono::system_clock::time_point::min() ) {
          if( current_time > latest_msg_time + hb_timeout ) {
             no_retry = benign_other;
             if( !peer_address().empty() ) {
@@ -1178,7 +1179,7 @@ namespace eosio {
             }
             return;
          } else {
-            const tstamp timeout = std::max(hb_timeout/2, 2*std::chrono::milliseconds(config::block_interval_ms).count());
+            const std::chrono::milliseconds timeout = std::max(hb_timeout/2, 2*std::chrono::milliseconds(config::block_interval_ms));
             if ( current_time > latest_blk_time + timeout ) {
                send_handshake();
                return;
@@ -1191,20 +1192,27 @@ namespace eosio {
 
    // called from connection strand
    void connection::send_time() {
-      time_message xpkt;
-      xpkt.org = rec;
-      xpkt.rec = dst;
-      xpkt.xmt = get_time();
-      org = xpkt.xmt;
-      enqueue(xpkt);
+      if (org == std::chrono::nanoseconds{0}) { // do not send if there is already a time loop in progress
+         org = get_time();
+         // xpkt.org == 0 means we are initiating a ping. Actual origin time is in xpkt.xmt.
+         time_message xpkt{
+             .org = 0,
+             .rec = 0,
+             .xmt = org.count(),
+             .dst = 0 };
+         peer_dlog(this, "send init time_message: ${t}", ("t", xpkt));
+         enqueue(xpkt);
+      }
    }
 
    // called from connection strand
    void connection::send_time(const time_message& msg) {
-      time_message xpkt;
-      xpkt.org = msg.xmt;
-      xpkt.rec = msg.dst;
-      xpkt.xmt = get_time();
+      time_message xpkt{
+          .org = msg.xmt,
+          .rec = msg.dst,
+          .xmt = get_time().count(),
+          .dst = 0 };
+      peer_dlog( this, "send time_message: ${t}, org: ${o}", ("t", xpkt)("o", org.count()) );
       enqueue(xpkt);
    }
 
@@ -1434,7 +1442,7 @@ namespace eosio {
 
       block_buffer_factory buff_factory;
       auto sb = buff_factory.get_send_buffer( b );
-      latest_blk_time = get_time();
+      latest_blk_time = std::chrono::system_clock::now();
       enqueue_buffer( sb, no_reason, to_sync_queue);
    }
 
@@ -1590,7 +1598,7 @@ namespace eosio {
          } );
          sync_known_lib_num = highest_lib_num;
 
-         // if closing the connection we are currently syncing from, then reset our last requested and next expected.
+         // if closing the connection we are currently syncing from then request from a diff peer
          if( c == sync_source ) {
             reset_last_requested_num(g);
             // if starting to sync need to always start from lib as we might be on our own fork
@@ -1777,7 +1785,9 @@ namespace eosio {
       auto current_time_ns = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
       int64_t network_latency_ns = current_time_ns - msg.time; // net latency in nanoseconds
       if( network_latency_ns < 0 ) {
-         peer_wlog(c, "Peer sent a handshake with a timestamp skewed by at least ${t}ms", ("t", network_latency_ns/1000000));
+         if (network_latency_ns < -(block_interval_ns/2)) {
+            peer_wlog(c, "Peer sent a handshake with a timestamp skewed by at least ${t}ms", ("t", network_latency_ns / 1000000));
+         }
          network_latency_ns = 0;
       }
       // number of blocks syncing node is behind from a peer node, round up
@@ -1935,7 +1945,12 @@ namespace eosio {
                verify_catchup( c, msg.known_blocks.pending, id );
             } else {
                // we already have the block, so update peer with our view of the world
-               c->send_handshake();
+               auto chain_info = my_impl->get_chain_info();
+               std::unique_lock g_conn( c->conn_mtx );
+               if (chain_info.head_id != c->last_handshake_sent.head_id) { // no need to send handshake if nothing new to report
+                  g_conn.unlock();
+                  c->send_handshake();
+               }
             }
          }
       } else if (msg.known_blocks.mode == last_irr_catch_up) {
@@ -1953,7 +1968,10 @@ namespace eosio {
    void sync_manager::rejected_block( const connection_ptr& c, uint32_t blk_num ) {
       c->block_status_monitor_.rejected();
       std::unique_lock<std::mutex> g( sync_mtx );
-      reset_last_requested_num(g);
+      sync_last_requested_num = 0;
+      if (blk_num < sync_next_expected_num) {
+         sync_next_expected_num = my_impl->get_chain_lib_num();
+      }
       if( c->block_status_monitor_.max_events_violated()) {
          peer_wlog( c, "block ${bn} not accepted, closing connection", ("bn", blk_num) );
          sync_source.reset();
@@ -1961,6 +1979,7 @@ namespace eosio {
          c->close();
       } else {
          g.unlock();
+         peer_dlog(c, "rejected block ${bn}, sending handshake", ("bn", blk_num));
          c->send_handshake();
       }
    }
@@ -2143,7 +2162,7 @@ namespace eosio {
          send_buffer_type sb = buff_factory.get_send_buffer( b );
 
          cp->strand.post( [cp, bnum, sb{std::move(sb)}]() {
-            cp->latest_blk_time = cp->get_time();
+            cp->latest_blk_time = std::chrono::system_clock::now();
             bool has_block = cp->peer_lib_num >= bnum;
             if( !has_block ) {
                peer_dlog( cp, "bcast block ${b}", ("b", bnum) );
@@ -2563,14 +2582,14 @@ namespace eosio {
    // called from connection strand
    bool connection::process_next_message( uint32_t message_length ) {
       try {
-         latest_msg_time = get_time();
+         latest_msg_time = std::chrono::system_clock::now();
 
          // if next message is a block we already have, exit early
          auto peek_ds = pending_message_buffer.create_peek_datastream();
          unsigned_int which{};
          fc::raw::unpack( peek_ds, which );
          if( which == signed_block_which ) {
-            latest_blk_time = get_time();
+            latest_blk_time = std::chrono::system_clock::now();
             return process_next_block_message( message_length );
 
          } else if( which == packed_transaction_which ) {
@@ -2993,38 +3012,53 @@ namespace eosio {
       close( retry ); // reconnect if wrong_version
    }
 
-   void connection::handle_message( const time_message& msg ) {
-      peer_ilog( this, "received time_message" );
+   // some clients before leap 5.0 provided microsecond epoch instead of nanosecond epoch
+   std::chrono::nanoseconds normalize_epoch_to_ns(int64_t x) {
+      //        1686211688888 milliseconds - 2023-06-08T08:08:08.888, 5yrs from EOS genesis 2018-06-08T08:08:08.888
+      //     1686211688888000 microseconds
+      //  1686211688888000000 nanoseconds
+      if (x >= 1686211688888000000) // nanoseconds
+         return std::chrono::nanoseconds{x};
+      if (x >= 1686211688888000) // microseconds
+         return std::chrono::nanoseconds{x*1000};
+      if (x >= 1686211688888) // milliseconds
+         return std::chrono::nanoseconds{x*1000*1000};
+      if (x >= 1686211688) // seconds
+         return std::chrono::nanoseconds{x*1000*1000*1000};
+      return std::chrono::nanoseconds{0}; // unknown or is zero
+   }
 
-      /* We've already lost however many microseconds it took to dispatch
-       * the message, but it can't be helped.
-       */
-      msg.dst = get_time();
+   void connection::handle_message( const time_message& msg ) {
+      peer_dlog( this, "received time_message: ${t}, org: ${o}", ("t", msg)("o", org.count()) );
+
+      // We've already lost however many microseconds it took to dispatch the message, but it can't be helped.
+      msg.dst = get_time().count();
 
       // If the transmit timestamp is zero, the peer is horribly broken.
       if(msg.xmt == 0)
          return;                 /* invalid timestamp */
 
-      if(msg.xmt == xmt)
+      auto msg_xmt = normalize_epoch_to_ns(msg.xmt);
+      if(msg_xmt == xmt)
          return;                 /* duplicate packet */
 
-      xmt = msg.xmt;
-      rec = msg.rec;
-      dst = msg.dst;
+      xmt = msg_xmt;
 
       if( msg.org == 0 ) {
          send_time( msg );
          return;  // We don't have enough data to perform the calculation yet.
       }
 
-      double offset = (double(rec - org) + double(msg.xmt - dst)) / 2;
-      double NsecPerUsec{1000};
+      if (org != std::chrono::nanoseconds{0}) {
+         auto rec = normalize_epoch_to_ns(msg.rec);
+         int64_t offset = (double((rec - org).count()) + double(msg_xmt.count() - msg.dst)) / 2.0;
 
-      if( logger.is_enabled( fc::log_level::all ) )
-         logger.log( FC_LOG_MESSAGE( all, "Clock offset is ${o}ns (${us}us)",
-                                     ("o", offset)( "us", offset / NsecPerUsec ) ) );
-      org = 0;
-      rec = 0;
+         if (std::abs(offset) > block_interval_ns) {
+            peer_wlog(this, "Clock offset is ${of}us, calculation: (rec ${r} - org ${o} + xmt ${x} - dst ${d})/2",
+                      ("of", offset / 1000)("r", rec.count())("o", org.count())("x", msg_xmt.count())("d", msg.dst));
+         }
+      }
+      org = std::chrono::nanoseconds{0};
 
       std::unique_lock<std::mutex> g_conn( conn_mtx );
       if( last_handshake_recv.generation == 0 ) {
@@ -3056,15 +3090,13 @@ namespace eosio {
       switch (msg.known_trx.mode) {
       case none:
          break;
-      case last_irr_catch_up: {
+      case last_irr_catch_up:
+      case catch_up : {
          std::unique_lock<std::mutex> g_conn( conn_mtx );
          last_handshake_recv.head_num = msg.known_blocks.pending;
          g_conn.unlock();
          break;
       }
-      case catch_up : {
-         break;
-      }
       case normal: {
          my_impl->dispatcher->recv_notice( shared_from_this(), msg, false );
       }
@@ -3225,22 +3257,23 @@ namespace eosio {
             return;
          }
 
-         bool valid_block_header = !!bsp;
 
-         if( valid_block_header ) {
+         uint32_t block_num = bsp ? bsp->block_num : 0;
+
+         if( block_num != 0 ) {
             fc_dlog( logger, "validated block header, broadcasting immediately, connection ${cid}, blk num = ${num}, id = ${id}",
-                     ("cid", cid)("num", bsp->block_num)("id", bsp->id) );
+                     ("cid", cid)("num", block_num)("id", bsp->id) );
             my_impl->dispatcher->add_peer_block( bsp->id, cid ); // no need to send back to sender
             my_impl->dispatcher->bcast_block( bsp->block, bsp->id );
          }
 
-         app().executor().post(priority::medium, exec_queue::general, [ptr{std::move(ptr)}, bsp{std::move(bsp)}, id, c{std::move(c)}]() mutable {
+         app().executor().post(priority::medium, exec_queue::read_write, [ptr{std::move(ptr)}, bsp{std::move(bsp)}, id, c{std::move(c)}]() mutable {
             c->process_signed_block( id, std::move(ptr), std::move(bsp) );
          });
 
-         if( valid_block_header ) {
+         if( block_num != 0 ) {
             // ready to process immediately, so signal producer to interrupt start_block
-            my_impl->producer_plug->received_block();
+            my_impl->producer_plug->received_block(block_num);
          }
       });
    }
@@ -3252,18 +3285,6 @@ namespace eosio {
       // use c in this method instead of this to highlight that all methods called on c-> must be thread safe
       connection_ptr c = shared_from_this();
 
-      // if we have closed connection then stop processing
-      if( !c->socket_is_open() ) {
-         if( bsp ) {
-            // valid bsp means add_peer_block already called, need to remove it since we are not going to process the block
-            // call on dispatch strand to serialize with the add_peer_block calls
-            my_impl->dispatcher->strand.post( [blk_id]() {
-               my_impl->dispatcher->rm_block( blk_id );
-            } );
-         }
-         return;
-      }
-
       try {
          if( cc.fetch_block_by_id(blk_id) ) {
             c->strand.post( [sync_master = my_impl->sync_master.get(),
@@ -3386,7 +3407,7 @@ namespace eosio {
                fc_wlog( logger, "Peer keepalive ticked sooner than expected: ${m}", ("m", ec.message()) );
             }
 
-            tstamp current_time = connection::get_time();
+            auto current_time = std::chrono::system_clock::now();
             my->for_each_connection( [current_time]( auto& c ) {
                if( c->socket_is_open() ) {
                   c->strand.post([c, current_time]() {

plugins/producer_api_plugin/producer.swagger.yaml

@@ -342,11 +342,11 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/component/schema/Error"
+                $ref: "#/components/schemas/Error"
   /producer/schedule_snapshot:
     post:
       summary: schedule_snapshot
-      description: Submits a request to generate a schedule for automated snapshot with given parameters. If request body is empty, triest to execute snapshot immediately. Returns error when unable to create snapshot.
+      description: Submits a request to automatically generate snapshots according to a schedule specified with given parameters. If request body is empty, schedules immediate snapshot generation. Returns error when unable to accept schedule.
       operationId: schedule_snapshot
       requestBody:
         content:
@@ -377,6 +377,12 @@ paths:
             application/json:
               schema:
                 type: object
+                required:
+                    - snapshot_request_id
+                    - block_spacing
+                    - start_block_num
+                    - end_block_num
+                    - snapshot_description
                 properties:
                   snapshot_request_id:
                     type: integer
@@ -401,12 +407,12 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/component/schema/Error"
+                $ref: "#/components/schemas/Error"
   /producer/get_snapshot_requests:
     post:
       summary: get_snapshot_requests
-      description: Returns a list of scheduled snapshots
-      operationId: get_snapshot_status
+      description: Returns a list of scheduled snapshots.
+      operationId: get_snapshot_requests
       responses:
         "201":
           description: OK
@@ -417,9 +423,16 @@ paths:
                 properties:
                   snapshot_requests:
                     type: array
-                    description: An array of scheduled snapshots
+                    description: An array of scheduled snapshot requests
                     items:
                       type: object
+                      required:
+                          - snapshot_request_id
+                          - block_spacing
+                          - start_block_num
+                          - end_block_num
+                          - snapshot_description
+                          - pending_snapshots
                       properties:
                         snapshot_request_id:
                           type: integer
@@ -444,6 +457,12 @@ paths:
                           description: List of pending snapshots
                           items:
                             type: object
+                            required:
+                                - head_block_id
+                                - head_block_num
+                                - head_block_time
+                                - version
+                                - snapshot_name
                             properties:
                               head_block_id:
                                 $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
@@ -468,11 +487,11 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/component/schema/Error"
+                $ref: "#/components/schemas/Error"
   /producer/unschedule_snapshot:
     post:
       summary: unschedule_snapshot
-      description: Submits a request to remove identified by id recurring snapshot from schedule. Returns error when unable to create unschedule.
+      description: Removes snapshot request identified by id. Returns error if referenced snapshot request does not exist.
       operationId: unschedule_snapshot
       requestBody:
         required: true
@@ -491,6 +510,12 @@ paths:
             application/json:
               schema:
                 type: object
+                required:
+                  - snapshot_request_id
+                  - block_spacing
+                  - start_block_num
+                  - end_block_num
+                  - snapshot_description
                 properties:
                   snapshot_request_id:
                     type: integer

plugins/producer_api_plugin/producer_api_plugin.cpp

@@ -118,9 +118,11 @@ void producer_api_plugin::plugin_startup() {
             INVOKE_R_R(producer, get_account_ram_corrections, producer_plugin::get_account_ram_corrections_params), 201),
        CALL_WITH_400(producer, producer, get_unapplied_transactions,
                      INVOKE_R_R_D(producer, get_unapplied_transactions, producer_plugin::get_unapplied_transactions_params), 200),
-   }, appbase::exec_queue::read_only_trx_safe, appbase::priority::medium_high);
+       CALL_WITH_400(producer, producer, get_snapshot_requests,
+                     INVOKE_R_V(producer, get_snapshot_requests), 201),
+   }, appbase::exec_queue::read_only, appbase::priority::medium_high);
 
-   // Not safe to run in parallel with read-only transactions
+   // Not safe to run in parallel
    app().get_plugin<http_plugin>().add_api({
        CALL_WITH_400(producer, producer, pause,
             INVOKE_V_V(producer, pause), 201),
@@ -137,16 +139,14 @@ void producer_api_plugin::plugin_startup() {
        CALL_ASYNC(producer, producer, create_snapshot, producer_plugin::snapshot_information,
             INVOKE_R_V_ASYNC(producer, create_snapshot), 201),
        CALL_WITH_400(producer, producer, schedule_snapshot,
-            INVOKE_V_R_II(producer, schedule_snapshot, producer_plugin::snapshot_request_information), 201),
-       CALL_WITH_400(producer, producer, get_snapshot_requests,
-            INVOKE_R_V(producer, get_snapshot_requests), 201),
+            INVOKE_R_R_II(producer, schedule_snapshot, producer_plugin::snapshot_request_information), 201),
        CALL_WITH_400(producer, producer, unschedule_snapshot,
-            INVOKE_V_R(producer, unschedule_snapshot, producer_plugin::snapshot_request_id_information), 201),
+            INVOKE_R_R(producer, unschedule_snapshot, producer_plugin::snapshot_request_id_information), 201),
        CALL_WITH_400(producer, producer, get_integrity_hash,
             INVOKE_R_V(producer, get_integrity_hash), 201),
        CALL_WITH_400(producer, producer, schedule_protocol_feature_activations,
             INVOKE_V_R(producer, schedule_protocol_feature_activations, producer_plugin::scheduled_protocol_feature_activations), 201),
-   }, appbase::exec_queue::general, appbase::priority::medium_high);
+   }, appbase::exec_queue::read_write, appbase::priority::medium_high);
 }
 
 void producer_api_plugin::plugin_initialize(const variables_map& options) {

plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp

@@ -94,8 +94,11 @@ public:
       uint32_t snapshot_request_id = 0;
    };
 
+   struct snapshot_schedule_result : public snapshot_request_id_information, public snapshot_request_information {
+   };
+
    struct snapshot_schedule_information : public snapshot_request_id_information, public snapshot_request_information {
-      std::optional<std::vector<snapshot_information>> pending_snapshots;
+      std::vector<snapshot_information> pending_snapshots;
    };
 
    struct get_snapshot_requests_result {
@@ -159,8 +162,8 @@ public:
    integrity_hash_information get_integrity_hash() const;
 
    void create_snapshot(next_function<snapshot_information> next);
-   void schedule_snapshot(const snapshot_request_information& schedule);
-   void unschedule_snapshot(const snapshot_request_id_information& schedule);
+   snapshot_schedule_result schedule_snapshot(const snapshot_request_information& schedule);
+   snapshot_schedule_result unschedule_snapshot(const snapshot_request_id_information& schedule);
    get_snapshot_requests_result get_snapshot_requests() const;
 
    scheduled_protocol_feature_activations get_scheduled_protocol_feature_activations() const;
@@ -202,11 +205,14 @@ public:
    void register_metrics_listener(metrics_listener listener);
 
    // thread-safe, called when a new block is received
-   void received_block();
+   void received_block(uint32_t block_num);
 
-const std::set<account_name>& producer_accounts() const;
+   const std::set<account_name>& producer_accounts() const;
 
+   static void set_test_mode(bool m) { test_mode_ = m; }
  private:
+   inline static bool test_mode_{false}; // to be moved into appbase (application_base)
+
    std::shared_ptr<class producer_plugin_impl> my;
 };
 
@@ -221,6 +227,7 @@ FC_REFLECT(eosio::producer_plugin::snapshot_request_information, (block_spacing)
 FC_REFLECT(eosio::producer_plugin::snapshot_request_id_information, (snapshot_request_id))
 FC_REFLECT(eosio::producer_plugin::get_snapshot_requests_result, (snapshot_requests))
 FC_REFLECT_DERIVED(eosio::producer_plugin::snapshot_schedule_information, (eosio::producer_plugin::snapshot_request_id_information)(eosio::producer_plugin::snapshot_request_information), (pending_snapshots))
+FC_REFLECT_DERIVED(eosio::producer_plugin::snapshot_schedule_result, (eosio::producer_plugin::snapshot_request_id_information)(eosio::producer_plugin::snapshot_request_information),)
 FC_REFLECT(eosio::producer_plugin::scheduled_protocol_feature_activations, (protocol_features_to_activate))
 FC_REFLECT(eosio::producer_plugin::get_supported_protocol_features_params, (exclude_disabled)(exclude_unactivatable))
 FC_REFLECT(eosio::producer_plugin::get_account_ram_corrections_params, (lower_bound)(upper_bound)(limit)(reverse))

plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp

@@ -45,7 +45,7 @@ private:
    void x_serialize() {
       auto& vec = _snapshot_requests.get<as_vector>();
       std::vector<producer_plugin::snapshot_schedule_information> sr(vec.begin(), vec.end());
-      _snapshot_db << sr;   
+      _snapshot_db << sr;
    }
 
 public:
@@ -54,7 +54,7 @@ public:
    // snapshot_scheduler_listener
    void on_start_block(uint32_t height) {
       bool serialize_needed  = false;
-      bool snapshot_executed = false; 
+      bool snapshot_executed = false;
 
       auto execute_snapshot_with_log = [this, height, &snapshot_executed](const auto & req) {
          // one snapshot per height
@@ -64,18 +64,18 @@ public:
                ("end_block_num",   req.end_block_num)
                ("block_spacing",   req.block_spacing)
                ("height",          height));
-               
+
             execute_snapshot(req.snapshot_request_id);
             snapshot_executed = true;
          }
-      };    
+      };
 
       std::vector<uint32_t> unschedule_snapshot_request_ids;
       for(const auto& req: _snapshot_requests.get<0>()) {
          // -1 since its called from start block
          bool recurring_snapshot =  req.block_spacing &&  (height >= req.start_block_num + 1) && (!((height - req.start_block_num - 1) % req.block_spacing));
          bool onetime_snapshot   = (!req.block_spacing) && (height == req.start_block_num + 1);
-        
+
          // assume "asap" for snapshot with missed/zero start, it can have spacing
          if(!req.start_block_num) {
             // update start_block_num with current height only if this is recurring
@@ -84,16 +84,16 @@ public:
                auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
                auto it = snapshot_by_id.find(req.snapshot_request_id);
                _snapshot_requests.modify(it, [&height](auto& p) { p.start_block_num = height - 1; });
-               serialize_needed = true;              
+               serialize_needed = true;
             }
             execute_snapshot_with_log(req);
          } else if(recurring_snapshot || onetime_snapshot) {
             execute_snapshot_with_log(req);
-         }       
+         }
 
          // cleanup - remove expired (or invalid) request
-         if((!req.start_block_num && !req.block_spacing) ||         
-            (!req.block_spacing && height >= (req.start_block_num + 1)) || 
+         if((!req.start_block_num && !req.block_spacing) ||
+            (!req.block_spacing && height >= (req.start_block_num + 1)) ||
             (req.end_block_num > 0 && height >= (req.end_block_num + 1))) {
             unschedule_snapshot_request_ids.push_back(req.snapshot_request_id);
          }
@@ -108,7 +108,7 @@ public:
    }
 
    // snapshot_scheduler_handler
-   void schedule_snapshot(const producer_plugin::snapshot_request_information& sri) {
+   producer_plugin::snapshot_schedule_result schedule_snapshot(const producer_plugin::snapshot_request_information& sri) {
       auto& snapshot_by_value = _snapshot_requests.get<by_snapshot_value>();
       auto existing = snapshot_by_value.find(std::make_tuple(sri.block_spacing, sri.start_block_num, sri.end_block_num));
       EOS_ASSERT(existing == snapshot_by_value.end(), chain::duplicate_snapshot_request, "Duplicate snapshot request");
@@ -122,19 +122,27 @@ public:
          }
       }
 
-      _snapshot_requests.emplace(producer_plugin::snapshot_schedule_information {{_snapshot_id++},{sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description},{}});
+      _snapshot_requests.emplace(producer_plugin::snapshot_schedule_information {{_snapshot_id++}, {sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description},{}});
       x_serialize();
+
+      // returning snapshot_schedule_result
+      return  producer_plugin::snapshot_schedule_result{{_snapshot_id - 1}, {sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description}};
    }
 
-   virtual void unschedule_snapshot(uint32_t sri) {
+   producer_plugin::snapshot_schedule_result unschedule_snapshot(uint32_t sri) {
       auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
       auto existing = snapshot_by_id.find(sri);
       EOS_ASSERT(existing != snapshot_by_id.end(), chain::snapshot_request_not_found, "Snapshot request not found");
+
+      producer_plugin::snapshot_schedule_result result{{existing->snapshot_request_id}, {existing->block_spacing, existing->start_block_num, existing->end_block_num, existing->snapshot_description}};
       _snapshot_requests.erase(existing);
       x_serialize();
+
+      // returning snapshot_schedule_result
+      return result;
    }
 
-   virtual producer_plugin::get_snapshot_requests_result get_snapshot_requests() {
+   producer_plugin::get_snapshot_requests_result get_snapshot_requests() {
       producer_plugin::get_snapshot_requests_result result;
       auto& asvector = _snapshot_requests.get<as_vector>();
       result.snapshot_requests.reserve(asvector.size());
@@ -160,11 +168,8 @@ public:
       auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
       auto  snapshot_req   = snapshot_by_id.find(_inflight_sid);
       if (snapshot_req != snapshot_by_id.end()) {
-         _snapshot_requests.modify(snapshot_req, [&si](auto& p) { 
-            if (!p.pending_snapshots) {
-               p.pending_snapshots = std::vector<producer_plugin::snapshot_information>();
-            }
-            p.pending_snapshots->emplace_back(si);       
+         _snapshot_requests.modify(snapshot_req, [&si](auto& p) {
+            p.pending_snapshots.emplace_back(si);
          });
       }
    }
@@ -193,14 +198,12 @@ public:
             auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
             auto  snapshot_req   = snapshot_by_id.find(srid);
 
-            if (snapshot_req != snapshot_by_id.end()) {               
-               if (auto pending = snapshot_req->pending_snapshots; pending) {                 
-                  auto it = std::remove_if(pending->begin(), pending->end(), [&snapshot_info](const producer_plugin::snapshot_information & s){ return s.head_block_num <=  snapshot_info.head_block_num; });
-                  pending->erase(it, pending->end());
-                  _snapshot_requests.modify(snapshot_req, [&pending](auto& p) { 
-                     p.pending_snapshots = std::move(pending);
-                  });
-               }
+            if (snapshot_req != snapshot_by_id.end()) {
+               _snapshot_requests.modify(snapshot_req, [&](auto& p) {
+                  auto & pending = p.pending_snapshots;
+                  auto it = std::remove_if(pending.begin(), pending.end(), [&snapshot_info](const producer_plugin::snapshot_information & s){ return s.head_block_num <=  snapshot_info.head_block_num; });
+                  pending.erase(it, pending.end());
+               });
             }
          }
       };

plugins/producer_plugin/include/eosio/producer_plugin/subjective_billing.hpp

@@ -157,7 +157,10 @@ public:
       const auto time_ordinal = time_ordinal_for(now);
       const auto orig_count = _account_subjective_bill_cache.size();
       remove_subjective_billing( bsp, time_ordinal );
-      fc_dlog( log, "Subjective billed accounts ${n} removed ${r}", ("n", orig_count)("r", orig_count - _account_subjective_bill_cache.size()) );
+      if (orig_count > 0) {
+         fc_dlog( log, "Subjective billed accounts ${n} removed ${r}",
+                  ("n", orig_count)("r", orig_count - _account_subjective_bill_cache.size()) );
+      }
    }
 
    template <typename Yield>

plugins/producer_plugin/producer_plugin.cpp

@@ -14,6 +14,7 @@
 #include <fc/io/json.hpp>
 #include <fc/log/logger_config.hpp>
 #include <fc/scoped_exit.hpp>
+#include <fc/time.hpp>
 
 #include <boost/asio.hpp>
 #include <boost/date_time/posix_time/posix_time.hpp>
@@ -27,7 +28,6 @@
 #include <boost/multi_index/member.hpp>
 #include <boost/multi_index/hashed_index.hpp>
 #include <boost/multi_index/ordered_index.hpp>
-#include <boost/multi_index/random_access_index.hpp>
 #include <boost/signals2/connection.hpp>
 
 namespace bmi = boost::multi_index;
@@ -103,7 +103,8 @@ namespace {
       auto code = e.code();
       return (code == block_cpu_usage_exceeded::code_value) ||
              (code == block_net_usage_exceeded::code_value) ||
-             (code == deadline_exception::code_value);
+             (code == deadline_exception::code_value) ||
+             (code == ro_trx_vm_oc_compile_temporary_failure::code_value);
    }
 }
 
@@ -256,39 +257,53 @@ struct block_time_tracker {
       block_idle_time += idle;
    }
 
-   void add_fail_time( const fc::microseconds& fail_time ) {
-      trx_fail_time += fail_time;
-      ++trx_fail_num;
+   void add_fail_time( const fc::microseconds& fail_time, bool is_transient ) {
+      if( is_transient ) {
+         // transient time includes both success and fail time
+         transient_trx_time += fail_time;
+         ++transient_trx_num;
+      } else {
+         trx_fail_time += fail_time;
+         ++trx_fail_num;
+      }
    }
 
-   void add_success_time( const fc::microseconds& time ) {
-      trx_success_time += time;
-      ++trx_success_num;
+   void add_success_time( const fc::microseconds& time, bool is_transient ) {
+      if( is_transient ) {
+         transient_trx_time += time;
+         ++transient_trx_num;
+      } else {
+         trx_success_time += time;
+         ++trx_success_num;
+      }
    }
 
    void report( const fc::time_point& idle_trx_time, uint32_t block_num ) {
       if( _log.is_enabled( fc::log_level::debug ) ) {
          auto now = fc::time_point::now();
          add_idle_time( now - idle_trx_time );
-         fc_dlog( _log, "Block #${n} trx idle: ${i}us out of ${t}us, success: ${sn}, ${s}us, fail: ${fn}, ${f}us, other: ${o}us",
+         fc_dlog( _log, "Block #${n} trx idle: ${i}us out of ${t}us, success: ${sn}, ${s}us, fail: ${fn}, ${f}us, transient: ${trans_trx_num}, ${trans_trx_time}us, other: ${o}us",
                   ("n", block_num)
                   ("i", block_idle_time)("t", now - clear_time)("sn", trx_success_num)("s", trx_success_time)
                   ("fn", trx_fail_num)("f", trx_fail_time)
-                  ("o", (now - clear_time) - block_idle_time - trx_success_time - trx_fail_time) );
+                  ("trans_trx_num", transient_trx_num)("trans_trx_time", transient_trx_time)
+                  ("o", (now - clear_time) - block_idle_time - trx_success_time - trx_fail_time - transient_trx_time) );
       }
    }
 
    void clear() {
-      block_idle_time = trx_fail_time = trx_success_time = fc::microseconds{};
-      trx_fail_num = trx_success_num = 0;
+      block_idle_time = trx_fail_time = trx_success_time = transient_trx_time = fc::microseconds{};
+      trx_fail_num = trx_success_num = transient_trx_num = 0;
       clear_time = fc::time_point::now();
    }
 
    fc::microseconds block_idle_time;
    uint32_t trx_success_num = 0;
    uint32_t trx_fail_num = 0;
+   uint32_t transient_trx_num = 0;
    fc::microseconds trx_success_time;
    fc::microseconds trx_fail_time;
+   fc::microseconds transient_trx_time;
    fc::time_point clear_time{fc::time_point::now()};
 };
 
@@ -299,8 +314,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       producer_plugin_impl(boost::asio::io_service& io)
       :_timer(io)
       ,_transaction_ack_channel(app().get_channel<compat::channels::transaction_ack>())
-      ,_ro_write_window_timer(io)
-      ,_ro_read_window_timer(io)
+      ,_ro_timer(io)
       {
       }
 
@@ -324,9 +338,9 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       push_result push_transaction( const fc::time_point& block_deadline,
                                     const transaction_metadata_ptr& trx,
                                     bool api_trx, bool return_failure_trace,
-                                    next_function<transaction_trace_ptr> next );
+                                    const next_function<transaction_trace_ptr>& next );
       push_result handle_push_result( const transaction_metadata_ptr& trx,
-                                      next_function<transaction_trace_ptr> next,
+                                      const next_function<transaction_trace_ptr>& next,
                                       const fc::time_point& start,
                                       const chain::controller& chain,
                                       const transaction_trace_ptr& trace,
@@ -356,7 +370,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       named_thread_pool<struct prod>                            _thread_pool;
 
       std::atomic<int32_t>                                      _max_transaction_time_ms; // modified by app thread, read by net_plugin thread pool
-      std::atomic<bool>                                         _received_block{false}; // modified by net_plugin thread pool and app thread
+      std::atomic<uint32_t>                                     _received_block{0}; // modified by net_plugin thread pool
       fc::microseconds                                          _max_irreversible_block_age_us;
       int32_t                                                   _produce_time_offset_us = 0;
       int32_t                                                   _last_block_time_offset_us = 0;
@@ -411,42 +425,65 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
 
       // async snapshot scheduler
       snapshot_scheduler _snapshot_scheduler;
-      
+
       // ro for read-only
       struct ro_trx_t {
-         packed_transaction_ptr  trx;
-         next_func_t             next;
+         transaction_metadata_ptr trx;
+         next_func_t              next;
       };
-      // The queue storing read-only transactions to be executed by read-only threads
-      struct ro_trx_queue_t {
-         std::mutex              mtx;
-         std::deque<ro_trx_t>    queue;
+      // The queue storing previously exhausted read-only transactions to be re-executed by read-only threads
+      // thread-safe
+      class ro_trx_queue_t {
+      public:
+         void push_front(ro_trx_t&& t) {
+            std::lock_guard g(mtx);
+            queue.push_front(std::move(t));
+         }
+
+         bool empty() const {
+            std::lock_guard g(mtx);
+            return queue.empty();
+         }
+
+         bool pop_front(ro_trx_t& t) {
+            std::unique_lock g(mtx);
+            if (queue.empty())
+               return false;
+            t = queue.front();
+            queue.pop_front();
+            return true;
+         }
+
+      private:
+         mutable std::mutex      mtx;
+         deque<ro_trx_t>         queue;  // boost deque which is faster than std::deque
       };
-      uint16_t                        _ro_thread_pool_size{ 0 };
-      static constexpr uint16_t       _ro_max_eos_vm_oc_threads_allowed{ 8 }; // Due to uncertainty to get total virtual memory size on a 5-level paging system, set a hard limit
+
+      uint32_t                        _ro_thread_pool_size{ 0 };
+       // Due to uncertainty to get total virtual memory size on a 5-level paging system for eos-vm-oc and
+       // possible memory exhuastion for large number of contract usage for non-eos-vm-oc, set a hard limit
+      static constexpr uint32_t       _ro_max_threads_allowed{ 8 };
       named_thread_pool<struct read>  _ro_thread_pool;
       fc::microseconds                _ro_write_window_time_us{ 200000 };
       fc::microseconds                _ro_read_window_time_us{ 60000 };
       static constexpr fc::microseconds _ro_read_window_minimum_time_us{ 10000 };
       fc::microseconds                _ro_read_window_effective_time_us{ 0 }; // calculated during option initialization
-      boost::asio::deadline_timer     _ro_write_window_timer;
-      boost::asio::deadline_timer     _ro_read_window_timer;
+      std::atomic<int64_t>            _ro_all_threads_exec_time_us; // total time spent by all threads executing transactions. use atomic for simplicity and performance
+      fc::time_point                  _ro_read_window_start_time;
+      fc::time_point                  _ro_window_deadline; // only modified on app thread, read-window deadline or write-window deadline
+      boost::asio::deadline_timer     _ro_timer; // only accessible from the main thread
       fc::microseconds                _ro_max_trx_time_us{ 0 }; // calculated during option initialization
-      ro_trx_queue_t                  _ro_trx_queue;
-      std::atomic<uint32_t>            _ro_num_active_trx_exec_tasks{ 0 };
-      std::vector<std::future<bool>>  _ro_trx_exec_tasks_fut;
+      ro_trx_queue_t                  _ro_exhausted_trx_queue;
+      std::atomic<uint32_t>           _ro_num_active_exec_tasks{ 0 };
+      std::vector<std::future<bool>>  _ro_exec_tasks_fut;
 
       void start_write_window();
       void switch_to_write_window();
       void switch_to_read_window();
-      bool read_only_trx_execution_task();
-      bool process_read_only_transaction(const packed_transaction_ptr& trx,
-                                         const next_function<transaction_trace_ptr>& next,
-                                         const fc::time_point& read_window_start_time);
-      bool push_read_only_transaction(const transaction_metadata_ptr& trx,
-                                      next_function<transaction_trace_ptr> next,
-                                      const fc::time_point& read_window_start_time);
-      
+      bool read_only_execution_task(uint32_t pending_block_num);
+      void repost_exhausted_transactions(const fc::time_point& deadline);
+      bool push_read_only_transaction(transaction_metadata_ptr trx, next_function<transaction_trace_ptr> next);
+
       void consider_new_watermark( account_name producer, uint32_t block_num, block_timestamp_type timestamp) {
          auto itr = _producer_watermarks.find( producer );
          if( itr != _producer_watermarks.end() ) {
@@ -480,8 +517,9 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       }
 
       void on_irreversible_block( const signed_block_ptr& lib ) {
-         _irreversible_block_time = lib->timestamp.to_time_point();
          const chain::controller& chain = chain_plug->chain();
+         EOS_ASSERT(chain.is_write_window(), producer_exception, "write window is expected for on_irreversible_block signal");
+         _irreversible_block_time = lib->timestamp.to_time_point();
 
          // promote any pending snapshots
          auto& snapshots_by_height = _pending_snapshot_index.get<by_height>();
@@ -567,7 +605,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
          // push the new block
          auto handle_error = [&](const auto& e)
          {
-            elog((e.to_detail_string()));
+            elog("Exception on block ${bn}: ${e}", ("bn", blk_num)("e", e.to_detail_string()));
             app().get_channel<channels::rejected_block>().publish( priority::medium, block );
             throw;
          };
@@ -639,12 +677,12 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
                                          transaction_metadata::trx_type trx_type,
                                          bool return_failure_traces,
                                          next_function<transaction_trace_ptr> next) {
-         if ( trx_type == transaction_metadata::trx_type::read_only && _ro_thread_pool_size > 0 ) {
-            // Parallel read-only trx execution enabled.
-            // Store the transaction in read-only-trx queue so that it is
-            // executed in read window
-            std::lock_guard<std::mutex> g( _ro_trx_queue.mtx );
-            _ro_trx_queue.queue.push_back({trx, std::move(next)});
+         if ( trx_type == transaction_metadata::trx_type::read_only ) {
+            // Post all read only trxs to read_only queue for execution.
+            auto trx_metadata = transaction_metadata::create_no_recover_keys( trx, transaction_metadata::trx_type::read_only );
+            app().executor().post(priority::low, exec_queue::read_only, [this, trx{std::move(trx_metadata)}, next{std::move(next)}]() mutable {
+               push_read_only_transaction( std::move(trx), std::move(next) );
+            } );
             return;
          }
 
@@ -677,7 +715,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
                                                           next{std::move(next)}, trx=trx]() mutable {
             if( future.valid() ) {
                future.wait();
-               app().executor().post( priority::low, exec_queue::general, [self, future{std::move(future)}, api_trx, is_transient, next{std::move( next )}, trx{std::move(trx)}, return_failure_traces]() mutable {
+               app().executor().post( priority::low, exec_queue::read_write, [self, future{std::move(future)}, api_trx, is_transient, next{std::move( next )}, trx{std::move(trx)}, return_failure_traces]() mutable {
                   auto start = fc::time_point::now();
                   auto idle_time = start - self->_idle_trx_time;
                   self->_time_tracker.add_idle_time( idle_time );
@@ -688,7 +726,8 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
                      self->log_trx_results( trx, nullptr, ex, 0, start, is_transient );
                      next( std::move(ex) );
                      self->_idle_trx_time = fc::time_point::now();
-                     self->_time_tracker.add_fail_time(self->_idle_trx_time - start);
+                     auto dur = self->_idle_trx_time - start;
+                     self->_time_tracker.add_fail_time(dur, is_transient);
                   };
                   try {
                      auto result = future.get();
@@ -709,7 +748,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       bool process_incoming_transaction_async(const transaction_metadata_ptr& trx,
                                               bool api_trx,
                                               bool return_failure_trace,
-                                              next_function<transaction_trace_ptr> next) {
+                                              const next_function<transaction_trace_ptr>& next) {
          bool exhausted = false;
          chain::controller& chain = chain_plug->chain();
          try {
@@ -789,7 +828,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
          exhausted
       };
 
-      inline bool should_interrupt_start_block( const fc::time_point& deadline ) const;
+      inline bool should_interrupt_start_block( const fc::time_point& deadline, uint32_t pending_block_num ) const;
       start_block_result start_block();
 
       fc::time_point calculate_pending_block_time() const;
@@ -892,12 +931,12 @@ void producer_plugin::set_program_options(
           "Number of worker threads in producer thread pool")
          ("snapshots-dir", bpo::value<bfs::path>()->default_value("snapshots"),
           "the location of the snapshots directory (absolute path or relative to application data dir)")
-         ("read-only-threads", bpo::value<uint16_t>()->default_value(my->_ro_thread_pool_size),
-          "Number of worker threads in read-only transaction execution thread pool")
+         ("read-only-threads", bpo::value<uint32_t>(),
+          "Number of worker threads in read-only execution thread pool. Max 8.")
          ("read-only-write-window-time-us", bpo::value<uint32_t>()->default_value(my->_ro_write_window_time_us.count()),
-          "time in microseconds the write window lasts")
+          "Time in microseconds the write window lasts.")
          ("read-only-read-window-time-us", bpo::value<uint32_t>()->default_value(my->_ro_read_window_time_us.count()),
-          "time in microseconds the read window lasts")
+          "Time in microseconds the read window lasts.")
          ;
    config_file_options.add(producer_options);
 }
@@ -1073,11 +1112,23 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
       }
    }
 
-   my->_ro_thread_pool_size = options.at( "read-only-threads" ).as<uint16_t>();
+   if ( options.count( "read-only-threads" ) ) {
+      my->_ro_thread_pool_size = options.at( "read-only-threads" ).as<uint32_t>();
+   } else if ( my->_producers.empty() ) {
+      if( options.count( "plugin" ) ) {
+         const auto& v = options.at( "plugin" ).as<std::vector<std::string>>();
+         auto i = std::find_if( v.cbegin(), v.cend(), []( const std::string& p ) { return p == "eosio::chain_api_plugin"; } );
+         if( i != v.cend() ) {
+            // default to 3 threads for non producer nodes running chain_api_plugin if not specified
+            my->_ro_thread_pool_size = 3;
+            ilog( "chain_api_plugin configured, defaulting read-only-threads to ${t}", ("t", my->_ro_thread_pool_size) );
+         }
+      }
+   }
+   EOS_ASSERT( test_mode_ || my->_ro_thread_pool_size == 0 || my->_producers.empty(), plugin_config_exception, "read-only-threads not allowed on producer node" );
+
    // only initialize other read-only options when read-only thread pool is enabled
    if ( my->_ro_thread_pool_size > 0 ) {
-      EOS_ASSERT( my->_producers.empty(), plugin_config_exception, "--read-only-threads not allowed on producer node" );
-
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
       if (chain.is_eos_vm_oc_enabled()) {
          // EOS VM OC requires 4.2TB Virtual for each executing thread. Make sure the memory
@@ -1098,37 +1149,42 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
          }
 
          EOS_ASSERT( vm_total_kb > 0, plugin_config_exception, "Unable to get system virtual memory size (not a Linux?), therefore cannot determine if the system has enough virtual memory for multi-threaded read-only transactions on EOS VM OC");
+         EOS_ASSERT( vm_total_kb > vm_used_kb, plugin_config_exception, "vm total (${t}) must be greater than vm used (${u})", ("t", vm_total_kb)("u", vm_used_kb));
+         uint32_t num_threads_supported = (vm_total_kb - vm_used_kb) / 4200000000;
          // reserve 1 for the app thread, 1 for anything else which might use VM
-         int num_threads_supported = (vm_total_kb - vm_used_kb) / 4200000000 - 2;
-         ilog("vm total in kb: ${total}, vm used in kb: ${used}, number of EOS VM OC threads supported ((vm total - vm used)/4.2 TB - 2): ${supp}", ("total", vm_total_kb) ("used", vm_used_kb) ("supp", num_threads_supported));
-         EOS_ASSERT( num_threads_supported >= my->_ro_thread_pool_size, plugin_config_exception, "--read-only-threads (${th}) greater than number of threads supported for EOS VM OC (${supp}) by the system virtual memory size", ("th", my->_ro_thread_pool_size) ("supp", num_threads_supported) );
-
-         if ( my->_ro_thread_pool_size > my->_ro_max_eos_vm_oc_threads_allowed ) {
-            wlog("--read-only-threads (${th}) greater than maximum number of threads allowed (${allowed}) for EOS Vm OC. Set it to ${allowed}", ("th", my->_ro_thread_pool_size) ("allowed", my->_ro_max_eos_vm_oc_threads_allowed));
-            my->_ro_thread_pool_size = my->_ro_max_eos_vm_oc_threads_allowed;
-         }
+         EOS_ASSERT( num_threads_supported > 2, plugin_config_exception, "With the EOS VM OC configured, there is not enough system virtual memory to support the required minimum of 3 threads (1 for main thread, 1 for read-only, and 1 for anything else), vm total: ${t}, vm used: ${u}", ("t", vm_total_kb)("u", vm_used_kb));
+         num_threads_supported -= 2;
+         auto actual_threads_allowed = std::min(my->_ro_max_threads_allowed, num_threads_supported);
+         ilog("vm total in kb: ${total}, vm used in kb: ${used}, number of EOS VM OC threads supported ((vm total - vm used)/4.2 TB - 2): ${supp}, max allowed: ${max}, actual allowed: ${actual}", ("total", vm_total_kb) ("used", vm_used_kb) ("supp", num_threads_supported) ("max", my->_ro_max_threads_allowed)("actual", actual_threads_allowed));
+         EOS_ASSERT( my->_ro_thread_pool_size <= actual_threads_allowed, plugin_config_exception, "read-only-threads (${th}) greater than number of threads allowed for EOS VM OC (${allowed})", ("th", my->_ro_thread_pool_size) ("allowed", actual_threads_allowed) );
       }
 #endif
+      EOS_ASSERT( my->_ro_thread_pool_size <= my->_ro_max_threads_allowed, plugin_config_exception, "read-only-threads (${th}) greater than the number of threads allowed (${allowed})", ("th", my->_ro_thread_pool_size) ("allowed", my->_ro_max_threads_allowed) );
 
       my->_ro_write_window_time_us = fc::microseconds( options.at( "read-only-write-window-time-us" ).as<uint32_t>() );
       my->_ro_read_window_time_us = fc::microseconds( options.at( "read-only-read-window-time-us" ).as<uint32_t>() );
-      EOS_ASSERT( my->_ro_read_window_time_us > my->_ro_read_window_minimum_time_us, plugin_config_exception, "minimum of --read-only-read-window-time-us (${read}) must be ${min} microseconds", ("read", my->_ro_read_window_time_us) ("min", my->_ro_read_window_minimum_time_us) );
+      EOS_ASSERT( my->_ro_read_window_time_us > my->_ro_read_window_minimum_time_us, plugin_config_exception, "read-only-read-window-time-us (${read}) must be at least greater than  ${min} us", ("read", my->_ro_read_window_time_us) ("min", my->_ro_read_window_minimum_time_us) );
       my->_ro_read_window_effective_time_us = my->_ro_read_window_time_us - my->_ro_read_window_minimum_time_us;
 
       // Make sure a read-only transaction can finish within the read
       // window if scheduled at the very beginning of the window.
-      // Use _ro_read_window_effective_time_us instead of _ro_read_window_time_us
-      // for safety marging
+      // Add _ro_read_window_minimum_time_us for safety margin.
       if ( my->_max_transaction_time_ms.load() > 0 ) {
-         EOS_ASSERT( my->_ro_read_window_effective_time_us > fc::milliseconds(my->_max_transaction_time_ms.load()), plugin_config_exception, "--read-only-read-window-time-us (${read}) must be greater than --max-transaction-time ${trx_time} ms plus a margin of ${min} us", ("read", my->_ro_read_window_time_us) ("trx_time", my->_max_transaction_time_ms.load()) ("min", my->_ro_read_window_minimum_time_us) );
-         my->_ro_max_trx_time_us = fc::milliseconds(my->_max_transaction_time_ms.load());
-      } else {
-         // _max_transaction_time_ms can be set to negative in testing (for unlimited)
-         my->_ro_max_trx_time_us = my->_ro_read_window_effective_time_us;
+         EOS_ASSERT( my->_ro_read_window_time_us > ( fc::milliseconds(my->_max_transaction_time_ms.load()) + my->_ro_read_window_minimum_time_us ), plugin_config_exception, "read-only-read-window-time-us (${read} us) must be greater than max-transaction-time (${trx_time} us) plus ${min} us, required: ${read} us > (${trx_time} us + ${min} us).", ("read", my->_ro_read_window_time_us) ("trx_time", my->_max_transaction_time_ms.load() * 1000) ("min", my->_ro_read_window_minimum_time_us) );
       }
-      ilog("_ro_thread_pool_size ${s},  _ro_write_window_time_us ${ww}, _ro_read_window_time_us ${rw}, _ro_max_trx_time_us ${t}, _ro_read_window_effective_time_us ${w}", ("s", my->_ro_thread_pool_size) ("ww", my->_ro_write_window_time_us) ("rw", my->_ro_read_window_time_us) ("t", my->_ro_max_trx_time_us) ("w", my->_ro_read_window_effective_time_us));
+      ilog("read-only-write-window-time-us: ${ww} us, read-only-read-window-time-us: ${rw} us, effective read window time to be used: ${w} us",
+           ("ww", my->_ro_write_window_time_us)("rw", my->_ro_read_window_time_us)("w", my->_ro_read_window_effective_time_us));
    }
 
+   // Make sure _ro_max_trx_time_us is alwasys set.
+   if ( my->_max_transaction_time_ms.load() > 0 ) {
+      my->_ro_max_trx_time_us = fc::milliseconds(my->_max_transaction_time_ms.load());
+   } else {
+      // max-transaction-time can be set to negative for unlimited time
+     my->_ro_max_trx_time_us = fc::microseconds::maximum();
+   }
+   ilog("read-only-threads ${s}, max read-only trx time to be enforced: ${t} us", ("s", my->_ro_thread_pool_size)("t", my->_ro_max_trx_time_us));
+
    my->_incoming_block_sync_provider = app().get_method<incoming::methods::block_sync>().register_provider(
          [this](const signed_block_ptr& block, const std::optional<block_id_type>& block_id, const block_state_ptr& bsp) {
       return my->on_incoming_block(block, block_id, bsp);
@@ -1164,6 +1220,7 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
    my->_snapshot_scheduler.set_create_snapshot_fn([this](producer_plugin::next_function<producer_plugin::snapshot_information> next){create_snapshot(next);});
 } FC_LOG_AND_RETHROW() }
 
+using namespace std::chrono_literals;
 void producer_plugin::plugin_startup()
 { try {
    try {
@@ -1211,7 +1268,7 @@ void producer_plugin::plugin_startup()
    if ( my->_ro_thread_pool_size > 0 ) {
       my->_ro_thread_pool.start( my->_ro_thread_pool_size,
          []( const fc::exception& e ) {
-            fc_elog( _log, "Exception in read-only transaction thread pool, exiting: ${e}", ("e", e.to_detail_string()) );
+            fc_elog( _log, "Exception in read-only thread pool, exiting: ${e}", ("e", e.to_detail_string()) );
             app().quit();
          },
          [&]() {
@@ -1506,14 +1563,14 @@ void producer_plugin::create_snapshot(producer_plugin::next_function<producer_pl
    }
 }
 
-void producer_plugin::schedule_snapshot(const snapshot_request_information& sri)
+producer_plugin::snapshot_schedule_result producer_plugin::schedule_snapshot(const snapshot_request_information& sri)
 {
-   my->_snapshot_scheduler.schedule_snapshot(sri);
+   return my->_snapshot_scheduler.schedule_snapshot(sri);
 }
 
-void producer_plugin::unschedule_snapshot(const snapshot_request_id_information& sri)
+producer_plugin::snapshot_schedule_result producer_plugin::unschedule_snapshot(const snapshot_request_id_information& sri)
 {
-   my->_snapshot_scheduler.unschedule_snapshot(sri.snapshot_request_id);
+   return my->_snapshot_scheduler.unschedule_snapshot(sri.snapshot_request_id);
 }
 
 producer_plugin::get_snapshot_requests_result producer_plugin::get_snapshot_requests() const
@@ -1777,12 +1834,12 @@ fc::time_point producer_plugin_impl::calculate_block_deadline( const fc::time_po
    }
 }
 
-bool producer_plugin_impl::should_interrupt_start_block( const fc::time_point& deadline ) const {
+bool producer_plugin_impl::should_interrupt_start_block( const fc::time_point& deadline, uint32_t pending_block_num ) const {
    if( _pending_block_mode == pending_block_mode::producing ) {
       return deadline <= fc::time_point::now();
    }
    // if we can produce then honor deadline so production starts on time
-   return (!_producers.empty() && deadline <= fc::time_point::now()) || _received_block;
+   return (!_producers.empty() && deadline <= fc::time_point::now()) || (_received_block >= pending_block_num);
 }
 
 producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
@@ -1802,6 +1859,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
 
    const fc::time_point now = fc::time_point::now();
    const fc::time_point block_time = calculate_pending_block_time();
+   const uint32_t pending_block_num = hbs->block_num + 1;
    const fc::time_point preprocess_deadline = calculate_block_deadline(block_time);
 
    const pending_block_mode previous_pending_mode = _pending_block_mode;
@@ -1867,7 +1925,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
    if (_pending_block_mode == pending_block_mode::producing) {
       const auto start_block_time = block_time - fc::microseconds( config::block_interval_us );
       if( now < start_block_time ) {
-         fc_dlog(_log, "Not producing block waiting for production window ${n} ${bt}", ("n", hbs->block_num + 1)("bt", block_time) );
+         fc_dlog(_log, "Not producing block waiting for production window ${n} ${bt}", ("n", pending_block_num)("bt", block_time) );
          // start_block_time instead of block_time because schedule_delayed_production_loop calculates next block time from given time
          schedule_delayed_production_loop(weak_from_this(), calculate_producer_wake_up_time(start_block_time));
          return start_block_result::waiting_for_production;
@@ -1881,7 +1939,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
    }
 
    fc_dlog(_log, "Starting block #${n} at ${time} producer ${p}",
-           ("n", hbs->block_num + 1)("time", now)("p", scheduled_producer.producer_name));
+           ("n", pending_block_num)("time", now)("p", scheduled_producer.producer_name));
 
    try {
       uint16_t blocks_to_confirm = 0;
@@ -1945,7 +2003,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
             std::swap( features_to_activate, protocol_features_to_activate );
             _protocol_features_signaled = true;
             ilog( "signaling activation of the following protocol features in block ${num}: ${features_to_activate}",
-                  ("num", hbs->block_num + 1)("features_to_activate", features_to_activate) );
+                  ("num", pending_block_num)("features_to_activate", features_to_activate) );
          }
       }
 
@@ -1971,7 +2029,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
          if( !remove_expired_blacklisted_trxs( preprocess_deadline ) )
             return start_block_result::exhausted;
          if( !_subjective_billing.remove_expired( _log, chain.pending_block_time(), fc::time_point::now(),
-                                                  [&](){ return should_interrupt_start_block( preprocess_deadline ); } ) ) {
+                                                  [&](){ return should_interrupt_start_block( preprocess_deadline, pending_block_num ); } ) ) {
             return start_block_result::exhausted;
          }
 
@@ -1994,9 +2052,11 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
             process_scheduled_and_incoming_trxs( scheduled_trx_deadline, incoming_itr );
          }
 
+         repost_exhausted_transactions( preprocess_deadline );
+
          if( app().is_quiting() ) // db guard exception above in LOG_AND_DROP could have called app().quit()
             return start_block_result::failed;
-         if ( should_interrupt_start_block( preprocess_deadline ) || block_is_exhausted() ) {
+         if ( should_interrupt_start_block( preprocess_deadline, pending_block_num ) || block_is_exhausted() ) {
             return start_block_result::exhausted;
          }
 
@@ -2023,11 +2083,12 @@ bool producer_plugin_impl::remove_expired_trxs( const fc::time_point& deadline )
 {
    chain::controller& chain = chain_plug->chain();
    auto pending_block_time = chain.pending_block_time();
+   auto pending_block_num = chain.pending_block_num();
 
    // remove all expired transactions
    size_t num_expired = 0;
    size_t orig_count = _unapplied_transactions.size();
-   bool exhausted = !_unapplied_transactions.clear_expired( pending_block_time, [&](){ return should_interrupt_start_block(deadline); },
+   bool exhausted = !_unapplied_transactions.clear_expired( pending_block_time, [&](){ return should_interrupt_start_block(deadline, pending_block_num); },
          [&num_expired]( const packed_transaction_ptr& packed_trx_ptr, trx_enum_type trx_type ) {
             // expired exception is logged as part of next() call
             ++num_expired;
@@ -2051,12 +2112,13 @@ bool producer_plugin_impl::remove_expired_blacklisted_trxs( const fc::time_point
    if(!blacklist_by_expiry.empty()) {
       const chain::controller& chain = chain_plug->chain();
       const auto lib_time = chain.last_irreversible_block_time();
+      const auto pending_block_num = chain.pending_block_num();
 
       int num_expired = 0;
       int orig_count = _blacklisted_transactions.size();
 
       while (!blacklist_by_expiry.empty() && blacklist_by_expiry.begin()->expiry <= lib_time) {
-         if ( should_interrupt_start_block( deadline ) ) {
+         if ( should_interrupt_start_block( deadline, pending_block_num ) ) {
             exhausted = true;
             break;
          }
@@ -2195,9 +2257,10 @@ producer_plugin_impl::push_transaction( const fc::time_point& block_deadline,
                                         const transaction_metadata_ptr& trx,
                                         bool api_trx,
                                         bool return_failure_trace,
-                                        next_function<transaction_trace_ptr> next )
+                                        const next_function<transaction_trace_ptr>& next )
 {
    auto start = fc::time_point::now();
+   EOS_ASSERT(!trx->is_read_only(), producer_exception, "Unexpected read-only trx");
 
    bool disable_subjective_enforcement = (api_trx && _disable_subjective_api_billing)
                                          || (!api_trx && _disable_subjective_p2p_billing)
@@ -2214,7 +2277,7 @@ producer_plugin_impl::push_transaction( const fc::time_point& block_deadline,
          log_trx_results( trx, except_ptr );
          next( except_ptr );
       }
-      _time_tracker.add_fail_time(fc::time_point::now() - start);
+      _time_tracker.add_fail_time(fc::time_point::now() - start, trx->is_transient());
       return push_result{.failed = true};
    }
 
@@ -2234,14 +2297,6 @@ producer_plugin_impl::push_transaction( const fc::time_point& block_deadline,
       }
    }
 
-   // Make sure db_read_only_mode always to be unset
-   auto db_read_only_mode_guard = fc::make_scoped_exit([trx, &chain]{
-      if( trx->is_read_only() )
-         chain.unset_db_read_only_mode();
-   });
-   if( trx->is_read_only() )
-      chain.set_db_read_only_mode();
-
    auto trace = chain.push_transaction( trx, block_deadline, max_trx_time, prev_billed_cpu_time_us, false, sub_bill );
 
    return handle_push_result(trx, next, start, chain, trace, return_failure_trace, disable_subjective_enforcement, first_auth, sub_bill, prev_billed_cpu_time_us);
@@ -2249,7 +2304,7 @@ producer_plugin_impl::push_transaction( const fc::time_point& block_deadline,
 
 producer_plugin_impl::push_result
 producer_plugin_impl::handle_push_result( const transaction_metadata_ptr& trx,
-                                          next_function<transaction_trace_ptr> next,
+                                          const next_function<transaction_trace_ptr>& next,
                                           const fc::time_point& start,
                                           const chain::controller& chain,
                                           const transaction_trace_ptr& trace,
@@ -2261,14 +2316,20 @@ producer_plugin_impl::handle_push_result( const transaction_metadata_ptr& trx,
    auto end = fc::time_point::now();
    push_result pr;
    if( trace->except ) {
-      if ( !trx->is_read_only() )
-         _time_tracker.add_fail_time(end - start);
+      // Transient trxs are dry-run or read-only.
+      // Dry-run trxs only run in write window. Read-only trxs can run in
+      // both write and read windows; time spent in read window is counted
+      // by read window summary.
+      if ( chain.is_write_window() ) {
+         auto dur = end - start;
+         _time_tracker.add_fail_time(dur, trx->is_transient());
+      }
       if( exception_is_exhausted( *trace->except ) ) {
          if( _pending_block_mode == pending_block_mode::producing ) {
-            fc_dlog(_trx_failed_trace_log, "[TRX_TRACE] Block ${block_num} for producer ${prod} COULD NOT FIT, tx: ${txid} RETRYING ",
+            fc_dlog(trx->is_transient() ? _transient_trx_failed_trace_log : _trx_failed_trace_log, "[TRX_TRACE] Block ${block_num} for producer ${prod} COULD NOT FIT, tx: ${txid} RETRYING ",
                     ("block_num", chain.head_block_num() + 1)("prod", get_pending_block_producer())("txid", trx->id()));
          } else {
-            fc_dlog(_trx_failed_trace_log, "[TRX_TRACE] Speculative execution COULD NOT FIT tx: ${txid} RETRYING", ("txid", trx->id()));
+            fc_dlog(trx->is_transient() ? _transient_trx_failed_trace_log : _trx_failed_trace_log, "[TRX_TRACE] Speculative execution COULD NOT FIT tx: ${txid} RETRYING", ("txid", trx->id()));
          }
          if ( !trx->is_read_only() )
             pr.block_exhausted = block_is_exhausted(); // smaller trx might fit
@@ -2302,8 +2363,14 @@ producer_plugin_impl::handle_push_result( const transaction_metadata_ptr& trx,
    } else {
       fc_tlog( _log, "Subjective bill for success ${a}: ${b} elapsed ${t}us, time ${r}us",
                ("a",first_auth)("b",sub_bill)("t",trace->elapsed)("r", end - start));
-      if ( !trx->is_read_only() )
-         _time_tracker.add_success_time(end - start);
+      // Transient trxs are dry-run or read-only.
+      // Dry-run trxs only run in write window. Read-only trxs can run in
+      // both write and read windows; time spent in read window is counted
+      // by read window summary.
+      if ( chain.is_write_window() ) {
+         auto dur = end - start;
+         _time_tracker.add_success_time(dur, trx->is_transient());
+      }
       log_trx_results( trx, trace, start );
       // if producing then trx is in objective cpu account billing
       if (!disable_subjective_enforcement && _pending_block_mode != pending_block_mode::producing) {
@@ -2319,12 +2386,14 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin
 {
    bool exhausted = false;
    if( !_unapplied_transactions.empty() ) {
+      const chain::controller& chain = chain_plug->chain();
+      const auto pending_block_num = chain.pending_block_num();
       int num_applied = 0, num_failed = 0, num_processed = 0;
       auto unapplied_trxs_size = _unapplied_transactions.size();
       auto itr     = _unapplied_transactions.unapplied_begin();
       auto end_itr = _unapplied_transactions.unapplied_end();
       while( itr != end_itr ) {
-         if( should_interrupt_start_block( deadline ) ) {
+         if( should_interrupt_start_block( deadline, pending_block_num ) ) {
             exhausted = true;
             break;
          }
@@ -2446,7 +2515,7 @@ void producer_plugin_impl::process_scheduled_and_incoming_trxs( const fc::time_p
          auto trace = chain.push_scheduled_transaction(trx_id, deadline, max_trx_time, 0, false);
          auto end = fc::time_point::now();
          if (trace->except) {
-            _time_tracker.add_fail_time(end - start);
+            _time_tracker.add_fail_time(end - start, false); // delayed transaction cannot be transient
             if (exception_is_exhausted(*trace->except)) {
                if( block_is_exhausted() ) {
                   exhausted = true;
@@ -2466,7 +2535,7 @@ void producer_plugin_impl::process_scheduled_and_incoming_trxs( const fc::time_p
                num_failed++;
             }
          } else {
-            _time_tracker.add_success_time(end - start);
+            _time_tracker.add_success_time(end - start, false); // delayed transaction cannot be transient
             fc_dlog(_trx_successful_trace_log,
                     "[TRX_TRACE] Block ${block_num} for producer ${prod} is ACCEPTING scheduled tx: ${txid}, time: ${r}, auth: ${a}, cpu: ${cpu}",
                     ("block_num", chain.head_block_num() + 1)("prod", get_pending_block_producer())
@@ -2499,8 +2568,10 @@ bool producer_plugin_impl::process_incoming_trxs( const fc::time_point& deadline
    if( itr != end ) {
       size_t processed = 0;
       fc_dlog( _log, "Processing ${n} pending transactions", ("n", _unapplied_transactions.incoming_size()) );
+      const chain::controller& chain = chain_plug->chain();
+      const auto pending_block_num = chain.pending_block_num();
       while( itr != end ) {
-         if ( should_interrupt_start_block( deadline ) ) {
+         if ( should_interrupt_start_block( deadline, pending_block_num ) ) {
             exhausted = true;
             break;
          }
@@ -2543,7 +2614,6 @@ bool producer_plugin_impl::block_is_exhausted() const {
 // -> Idle
 // --> Start block B (block time y.000) at time x.500
 void producer_plugin_impl::schedule_production_loop() {
-   _received_block = false;
    _timer.cancel();
 
    auto result = start_block();
@@ -2555,7 +2625,7 @@ void producer_plugin_impl::schedule_production_loop() {
       _timer.expires_from_now( boost::posix_time::microseconds( config::block_interval_us  / 10 ));
 
       // we failed to start a block, so try again later?
-      _timer.async_wait( app().executor().wrap( priority::high, exec_queue::general,
+      _timer.async_wait( app().executor().wrap( priority::high, exec_queue::read_write,
           [weak_this = weak_from_this(), cid = ++_timer_corelation_id]( const boost::system::error_code& ec ) {
              auto self = weak_this.lock();
              if( self && ec != boost::asio::error::operation_aborted && cid == self->_timer_corelation_id ) {
@@ -2608,7 +2678,7 @@ void producer_plugin_impl::schedule_maybe_produce_block( bool exhausted ) {
                ("num", chain.head_block_num() + 1)("desc", block_is_exhausted() ? "Exhausted" : "Deadline exceeded") );
    }
 
-   _timer.async_wait( app().executor().wrap( priority::high, exec_queue::general,
+   _timer.async_wait( app().executor().wrap( priority::high, exec_queue::read_write,
          [&chain, weak_this = weak_from_this(), cid=++_timer_corelation_id](const boost::system::error_code& ec) {
             auto self = weak_this.lock();
             if( self && ec != boost::asio::error::operation_aborted && cid == self->_timer_corelation_id ) {
@@ -2650,7 +2720,7 @@ void producer_plugin_impl::schedule_delayed_production_loop(const std::weak_ptr<
       fc_dlog(_log, "Scheduling Speculative/Production Change at ${time}", ("time", wake_up_time));
       static const boost::posix_time::ptime epoch(boost::gregorian::date(1970, 1, 1));
       _timer.expires_at(epoch + boost::posix_time::microseconds(wake_up_time->time_since_epoch().count()));
-      _timer.async_wait( app().executor().wrap( priority::high, exec_queue::general,
+      _timer.async_wait( app().executor().wrap( priority::high, exec_queue::read_write,
          [weak_this,cid=++_timer_corelation_id](const boost::system::error_code& ec) {
             auto self = weak_this.lock();
             if( self && ec != boost::asio::error::operation_aborted && cid == self->_timer_corelation_id ) {
@@ -2751,8 +2821,8 @@ void producer_plugin_impl::produce_block() {
         ("confs", new_bs->header.confirmed));
 }
 
-void producer_plugin::received_block() {
-   my->_received_block = true;
+void producer_plugin::received_block(uint32_t block_num) {
+   my->_received_block = block_num;
 }
 
 void producer_plugin::log_failed_transaction(const transaction_id_type& trx_id, const packed_transaction_ptr& packed_trx_ptr, const char* reason) const {
@@ -2764,34 +2834,45 @@ void producer_plugin::log_failed_transaction(const transaction_id_type& trx_id,
             ("entire_trx", packed_trx_ptr ? my->chain_plug->get_log_trx(packed_trx_ptr->get_transaction()) : fc::variant{trx_id}));
 }
 
-// Called from app thread
+// Called from only one read_only thread
 void producer_plugin_impl::switch_to_write_window() {
+   if ( _log.is_enabled( fc::log_level::debug ) ) {
+      auto now = fc::time_point::now();
+      fc_dlog( _log, "Read-only threads ${n}, read window ${r}us, total all threads ${t}us",
+               ("n", _ro_thread_pool_size)
+               ("r", now - _ro_read_window_start_time)
+               ("t", _ro_all_threads_exec_time_us.load()));
+   }
+
+   chain::controller& chain = chain_plug->chain();
+
    // this method can be called from multiple places. it is possible
    // we are already in write window.
-   if ( app().executor().is_write_window() ) {
+   if ( chain.is_write_window() ) {
       return;
    }
 
-   EOS_ASSERT(_ro_num_active_trx_exec_tasks.load() == 0 && _ro_trx_exec_tasks_fut.empty(), producer_exception, "no read-only tasks should be running before switching to write window");
-   _ro_read_window_timer.cancel();
-   _ro_write_window_timer.cancel();
+   EOS_ASSERT(_ro_num_active_exec_tasks.load() == 0 && _ro_exec_tasks_fut.empty(), producer_exception, "no read-only tasks should be running before switching to write window");
 
    start_write_window();
 }
 
-// Called from app thread
+// Called from app thread on plugin_startup
+// Called from only one read_only thread & called from app thread, but not concurrently
 void producer_plugin_impl::start_write_window() {
    chain::controller& chain = chain_plug->chain();
 
    app().executor().set_to_write_window();
+   chain.set_to_write_window();
    chain.unset_db_read_only_mode();
-   _idle_trx_time = fc::time_point::now();
+   _idle_trx_time = _ro_window_deadline = fc::time_point::now();
 
+   _ro_window_deadline += _ro_write_window_time_us; // not allowed on block producers, so no need to limit to block deadline
    auto expire_time = boost::posix_time::microseconds(_ro_write_window_time_us.count());
-   _ro_write_window_timer.expires_from_now( expire_time );
-   _ro_write_window_timer.async_wait( app().executor().wrap(  // stay on app thread
+   _ro_timer.expires_from_now( expire_time );
+   _ro_timer.async_wait( app().executor().wrap(  // stay on app thread
       priority::high,
-      exec_queue::read_only_trx_safe, // placed in read_only_trx_safe queue so it is ensured to be executed in either window
+      exec_queue::read_write, // placed in read_write so only called from main thread
       [weak_this = weak_from_this()]( const boost::system::error_code& ec ) {
          auto self = weak_this.lock();
          if( self && ec != boost::asio::error::operation_aborted ) {
@@ -2800,144 +2881,154 @@ void producer_plugin_impl::start_write_window() {
       }));
 }
 
-// Called from app thread
+// Called only from app thread
 void producer_plugin_impl::switch_to_read_window() {
-   EOS_ASSERT(app().executor().is_write_window(),  producer_exception, "expected to be in write window");
-   EOS_ASSERT(_ro_num_active_trx_exec_tasks.load() == 0 && _ro_trx_exec_tasks_fut.empty(),  producer_exception, "_ro_trx_exec_tasks_fut expected to be empty" );
+   chain::controller& chain = chain_plug->chain();
+   EOS_ASSERT(chain.is_write_window(),  producer_exception, "expected to be in write window");
+   EOS_ASSERT( _ro_num_active_exec_tasks.load() == 0 && _ro_exec_tasks_fut.empty(), producer_exception, "_ro_exec_tasks_fut expected to be empty" );
 
-   _ro_write_window_timer.cancel();
-   _ro_read_window_timer.cancel();
    _time_tracker.add_idle_time( fc::time_point::now() - _idle_trx_time );
 
    // we are in write window, so no read-only trx threads are processing transactions.
-   // _ro_trx_queue is not being accessed. No need to lock.
-   if ( _ro_trx_queue.queue.empty() ) { // no read-only trxs to process. stay in write window
+   if ( app().executor().read_only_queue().empty() ) { // no read-only tasks to process. stay in write window
       start_write_window(); // restart write window timer for next round
       return;
    }
 
-   app().executor().set_to_read_window();
-   chain_plug->chain().set_db_read_only_mode();
-   _received_block = false;
+   uint32_t pending_block_num = chain.head_block_num() + 1;
+   _ro_read_window_start_time = fc::time_point::now();
+   _ro_window_deadline = _ro_read_window_start_time + _ro_read_window_effective_time_us;
+   app().executor().set_to_read_window(_ro_thread_pool_size,
+      [received_block=&_received_block, pending_block_num, ro_window_deadline=_ro_window_deadline]() {
+         return fc::time_point::now() >= ro_window_deadline || (received_block->load() >= pending_block_num); // should_exit()
+      });
+   chain.set_to_read_window();
+   chain.set_db_read_only_mode();
+   _ro_all_threads_exec_time_us = 0;
 
-   // start a read-only transaction execution task in each thread in the thread pool
-   _ro_num_active_trx_exec_tasks = _ro_thread_pool_size;
-   for (auto i = 0; i < _ro_thread_pool_size; ++i ) {
-      _ro_trx_exec_tasks_fut.emplace_back( post_async_task( _ro_thread_pool.get_executor(), [self = this] () {
-         return self->read_only_trx_execution_task();
+   // start a read-only execution task in each thread in the thread pool
+   _ro_num_active_exec_tasks = _ro_thread_pool_size;
+   _ro_exec_tasks_fut.resize(0);
+   for (uint32_t i = 0; i < _ro_thread_pool_size; ++i ) {
+      _ro_exec_tasks_fut.emplace_back( post_async_task( _ro_thread_pool.get_executor(), [self = this, pending_block_num] () {
+         return self->read_only_execution_task(pending_block_num);
       }) );
    }
 
    auto expire_time = boost::posix_time::microseconds(_ro_read_window_time_us.count());
-   _ro_read_window_timer.expires_from_now( expire_time );
-   _ro_read_window_timer.async_wait( app().executor().wrap(  // stay on app thread
+   _ro_timer.expires_from_now( expire_time );
+   // Needs to be on read_only because that is what is being processed until switch_to_write_window().
+   _ro_timer.async_wait( app().executor().wrap(
       priority::high,
-      exec_queue::read_only_trx_safe,
+      exec_queue::read_only,
       [weak_this = weak_from_this()]( const boost::system::error_code& ec ) {
          auto self = weak_this.lock();
          if( self && ec != boost::asio::error::operation_aborted ) {
             // use future to make sure all read-only tasks finished before switching to write window
-            for ( auto& task: self->_ro_trx_exec_tasks_fut ) {
+            for ( auto& task: self->_ro_exec_tasks_fut ) {
                task.get();
             }
-            self->_ro_trx_exec_tasks_fut.clear();
+            self->_ro_exec_tasks_fut.clear();
+            // will be executed from the main app thread because all read-only threads are idle now
             self->switch_to_write_window();
           } else if ( self ) {
-             self->_ro_trx_exec_tasks_fut.clear();
+             self->_ro_exec_tasks_fut.clear();
           }
        }));
 }
 
-// Called from a read only trx thread. Run in parallel with app and other read only trx threads
-bool producer_plugin_impl::read_only_trx_execution_task() {
-   auto start = fc::time_point::now();
-   auto read_window_deadline = start + _ro_read_window_effective_time_us;
-   // We have 4 ways to break out the while loop:
+// Called from a read only thread. Run in parallel with app and other read only threads
+bool producer_plugin_impl::read_only_execution_task(uint32_t pending_block_num) {
+   // We have 3 ways to break out the while loop:
    // 1. pass read window deadline
-   // 2. Net_plugin receives a block
-   // 3. No more transactions in the read-only trx queue
-   // 4. A transaction execution is exhaused
-   while ( fc::time_point::now() < read_window_deadline && !_received_block ) {
-      std::unique_lock<std::mutex> lck( _ro_trx_queue.mtx );
-      if ( _ro_trx_queue.queue.empty() ) {
-         break;
-      }
-      auto trx = _ro_trx_queue.queue.front();
-      _ro_trx_queue.queue.pop_front();
-      lck.unlock();
-      
-      auto retry = process_read_only_transaction( trx.trx, trx.next, start );
-      if ( retry ) {
-         lck.lock();
-         _ro_trx_queue.queue.push_front(trx);
-         // Do not schedule new execution
+   // 2. net_plugin receives a block
+   // 3. no read-only tasks to execute
+   while ( fc::time_point::now() < _ro_window_deadline && _received_block < pending_block_num ) {
+      bool more = app().executor().execute_highest_read_only(); // blocks until all read only threads are idle
+      if ( !more ) {
          break;
       }
    }
 
    // If all tasks are finished, do not wait until end of read window; switch to write window now.
-   if ( --_ro_num_active_trx_exec_tasks == 0 ) {
-      // Do switching on app thread to serialize
-      app().executor().post( priority::high, exec_queue::read_only_trx_safe, [self=this]() {
-         self->_ro_trx_exec_tasks_fut.clear();
+   if ( --_ro_num_active_exec_tasks == 0 ) {
+      // Needs to be on read_only because that is what is being processed until switch_to_write_window().
+      app().executor().post( priority::high, exec_queue::read_only, [self=this]() {
+         self->_ro_exec_tasks_fut.clear();
+         // will be executed from the main app thread because all read-only threads are idle now
          self->switch_to_write_window();
       } );
+      // last thread post any exhausted back into read_only queue with slightly higher priority (low+1) so they are executed first
+      ro_trx_t t;
+      while( _ro_exhausted_trx_queue.pop_front(t) ) {
+         app().executor().post(priority::low+1, exec_queue::read_only, [this, trx{std::move(t.trx)}, next{std::move(t.next)}]() mutable {
+            push_read_only_transaction( std::move(trx), std::move(next) );
+         } );
+      }
    }
 
    return true;
 }
 
-// Called from a read only trx thread. Run in parallel with app and other read only trx threads
-// Return whether the trx needs to be retried in next read window
-bool producer_plugin_impl::process_read_only_transaction(const packed_transaction_ptr& trx, const next_function<transaction_trace_ptr>& next, const fc::time_point& read_window_start_time) {
-   chain::controller& chain = chain_plug->chain();
-   auto future = transaction_metadata::start_recover_keys( trx, _thread_pool.get_executor(),
-                                                           chain.get_chain_id(), fc::microseconds::maximum(),
-                                                           transaction_metadata::trx_type::read_only,
-                                                           chain.configured_subjective_signature_length_limit() );
-   auto exception_handler = [&next](fc::exception_ptr ex) {
-      next( std::move(ex) );
-      return false;
-   };
-   future.wait();
-   try {
-      auto trx_metadata = future.get();
-      return push_read_only_transaction( trx_metadata, next, read_window_start_time );
-   } CATCH_AND_CALL(exception_handler);
-   return false;
+// Called from app thread during start block.
+// Reschedule any exhausted read-only transactions from the last block
+void producer_plugin_impl::repost_exhausted_transactions(const fc::time_point& deadline) {
+   if ( !_ro_exhausted_trx_queue.empty() ) {
+      chain::controller& chain = chain_plug->chain();
+      uint32_t pending_block_num = chain.pending_block_num();
+      // post any exhausted back into read_only queue with slightly higher priority (low+1) so they are executed first
+      ro_trx_t t;
+      while( !should_interrupt_start_block( deadline, pending_block_num ) && _ro_exhausted_trx_queue.pop_front(t) ) {
+         app().executor().post(priority::low+1, exec_queue::read_only, [this, trx{std::move(t.trx)}, next{std::move(t.next)}]() mutable {
+            push_read_only_transaction( std::move(trx), std::move(next) );
+         } );
+      }
+   }
 }
 
-// Called from a read_only_trx execution thread
+// Called from a read_only_trx execution thread, or from app thread when executing exclusively
 // Return whether the trx needs to be retried in next read window
-bool producer_plugin_impl::push_read_only_transaction(
-            const transaction_metadata_ptr& trx,
-            next_function<transaction_trace_ptr> next,
-            const fc::time_point& read_window_start_time) {
+bool producer_plugin_impl::push_read_only_transaction(transaction_metadata_ptr trx, next_function<transaction_trace_ptr> next) {
    auto retry = false;
-   chain::controller& chain = chain_plug->chain();
-
-   if( !chain.is_building_block() ) {
-      // try next round
-      return true;
-   }
 
    try {
-      const auto block_deadline = calculate_block_deadline( chain.pending_block_time() );
       auto start = fc::time_point::now();
-      auto time_used_in_read_window_us = ( start - read_window_start_time );
-      if ( time_used_in_read_window_us >= _ro_read_window_time_us ) {
-         // already passed read-window deadline, try next round
+      chain::controller& chain = chain_plug->chain();
+      if ( !chain.is_building_block() ) {
+         _ro_exhausted_trx_queue.push_front( {std::move(trx), std::move(next)} );
          return true;
       }
 
-      auto remaining_time_in_read_window_us = _ro_read_window_time_us - time_used_in_read_window_us;
-      // Ensure the trx to finish by the end of read-window.
-      auto window_deadline = std::min( start + remaining_time_in_read_window_us, block_deadline );
+      // When executing a read-only trx on the main thread while in the write window,
+      // need to switch db mode to read only.
+      auto db_read_only_mode_guard = fc::make_scoped_exit([&]{
+         if( chain.is_write_window() )
+            chain.unset_db_read_only_mode();
+      });
+
+      if ( chain.is_write_window() ) {
+         chain.set_db_read_only_mode();
+         auto idle_time = fc::time_point::now() - _idle_trx_time;
+         _time_tracker.add_idle_time( idle_time );
+      }
+
+      // use read-window/write-window deadline if there are read/write windows, otherwise use block_deadline if only the app thead
+      auto window_deadline = (_ro_thread_pool_size != 0) ? _ro_window_deadline : calculate_block_deadline( chain.pending_block_time() );
+
+      // Ensure the trx to finish by the end of read-window or write-window or block_deadline depending on
       auto trace = chain.push_transaction( trx, window_deadline, _ro_max_trx_time_us, 0, false, 0 );
+      _ro_all_threads_exec_time_us += (fc::time_point::now() - start).count();
       auto pr = handle_push_result(trx, next, start, chain, trace, true /*return_failure_trace*/, true /*disable_subjective_enforcement*/, {} /*first_auth*/, 0 /*sub_bill*/, 0 /*prev_billed_cpu_time_us*/);
       // If a transaction was exhausted, that indicates we are close to
       // the end of read window. Retry in next round.
       retry = pr.trx_exhausted;
+      if( retry ) {
+         _ro_exhausted_trx_queue.push_front( {std::move(trx), std::move(next)} );
+      }
+
+      if ( chain.is_write_window() ) {
+         _idle_trx_time = fc::time_point::now();
+      }
    } catch ( const guard_exception& e ) {
       chain_plugin::handle_guard_exception(e);
    } catch ( boost::interprocess::bad_alloc& ) {

plugins/producer_plugin/test/test_read_only_trx.cpp

@@ -50,7 +50,9 @@ auto make_unique_trx( const chain_id_type& chain_id ) {
 
 BOOST_AUTO_TEST_SUITE(read_only_trxs)
 
-void error_handling_common(std::vector<const char*>& specific_args) {
+enum class app_init_status { failed, succeeded };
+
+void test_configs_common(std::vector<const char*>& specific_args, app_init_status expected_status) {
    appbase::scoped_app app;
    fc::temp_directory temp;
    auto temp_dir_str = temp.path().string();
@@ -59,19 +61,30 @@ void error_handling_common(std::vector<const char*>& specific_args) {
    std::vector<const char*> argv =
       {"test", "--data-dir", temp_dir_str.c_str(), "--config-dir", temp_dir_str.c_str()};
    argv.insert( argv.end(), specific_args.begin(), specific_args.end() );
-   BOOST_CHECK_EQUAL( app->initialize<producer_plugin>( argv.size(), (char**) &argv[0]), false );
+
+   // app->initialize() returns a boolean. BOOST_CHECK_EQUAL cannot compare
+   // a boolean with a app_init_status directly
+   bool rc = (expected_status == app_init_status::succeeded) ? true : false;
+   BOOST_CHECK_EQUAL( app->initialize<producer_plugin>( argv.size(), (char**) &argv[0]), rc );
 }
 
 // --read-only-thread not allowed on producer node
 BOOST_AUTO_TEST_CASE(read_only_on_producer) {
    std::vector<const char*> specific_args = {"-p", "eosio", "-e", "--read-only-threads", "2" };
-   error_handling_common(specific_args);
+   test_configs_common(specific_args, app_init_status::failed);
 }
 
 // read_window_time must be greater than max_transaction_time + 10ms
 BOOST_AUTO_TEST_CASE(invalid_read_window_time) {
    std::vector<const char*> specific_args = { "--read-only-threads", "2", "--max-transaction-time", "10", "--read-only-write-window-time-us", "50000", "--read-only-read-window-time-us", "20000" }; // 20000 not greater than --max-transaction-time (10ms) + 10000us (minimum margin)
-   error_handling_common(specific_args);
+   test_configs_common(specific_args, app_init_status::failed);
+}
+
+// if --read-only-threads is not configured, read-only trx related configs should
+// not be checked
+BOOST_AUTO_TEST_CASE(not_check_configs_if_no_read_only_threads) {
+   std::vector<const char*> specific_args = { "--max-transaction-time", "10", "--read-only-write-window-time-us", "50000", "--read-only-read-window-time-us", "20000" }; // 20000 not greater than --max-transaction-time (10ms) + 10000us (minimum margin)
+   test_configs_common(specific_args, app_init_status::succeeded);
 }
 
 void test_trxs_common(std::vector<const char*>& specific_args) {
@@ -79,6 +92,7 @@ void test_trxs_common(std::vector<const char*>& specific_args) {
    appbase::scoped_app app;
    fc::temp_directory temp;
    auto temp_dir_str = temp.path().string();
+   producer_plugin::set_test_mode(true);
    
    std::promise<std::tuple<producer_plugin*, chain_plugin*>> plugin_promise;
    std::future<std::tuple<producer_plugin*, chain_plugin*>> plugin_fut = plugin_promise.get_future();
@@ -96,14 +110,19 @@ void test_trxs_common(std::vector<const char*>& specific_args) {
    auto chain_id = chain_plug->get_chain_id();
 
    std::atomic<size_t> next_calls = 0;
+   std::atomic<size_t> num_get_account_calls = 0;
    std::atomic<size_t> num_posts = 0;
    std::atomic<size_t> trace_with_except = 0;
    std::atomic<bool> trx_match = true;
-   const size_t num_pushes = 2558;
+   const size_t num_pushes = 4242;
 
    for( size_t i = 1; i <= num_pushes; ++i ) {
       auto ptrx = make_unique_trx( chain_id );
-      app->executor().post( priority::low, exec_queue::general, [ptrx, &next_calls, &num_posts, &trace_with_except, &trx_match, &app]() {
+      app->executor().post( priority::low, exec_queue::read_only, [&chain_plug=chain_plug, &num_get_account_calls]() {
+         chain_plug->get_read_only_api(fc::seconds(90)).get_account(chain_apis::read_only::get_account_params{.account_name=config::system_account_name}, fc::time_point::now()+fc::seconds(90));
+         ++num_get_account_calls;
+      });
+      app->executor().post( priority::low, exec_queue::read_write, [ptrx, &next_calls, &num_posts, &trace_with_except, &trx_match, &app]() {
          ++num_posts;
          bool return_failure_traces = true;
          app->get_method<plugin_interface::incoming::methods::transaction_async>()(ptrx,
@@ -126,12 +145,15 @@ void test_trxs_common(std::vector<const char*>& specific_args) {
                ++next_calls;
         });
       });
+      app->executor().post( priority::low, exec_queue::read_only, [&chain_plug=chain_plug]() {
+         chain_plug->get_read_only_api(fc::seconds(90)).get_consensus_parameters(chain_apis::read_only::get_consensus_parameters_params{}, fc::time_point::now()+fc::seconds(90));
+      });
    }
 
    // Wait long enough such that all transactions are executed
    auto start = fc::time_point::now();
    auto hard_deadline = start + fc::seconds(10); // To protect against waiting forever
-   while ( next_calls < num_pushes && fc::time_point::now() < hard_deadline ){
+   while ( (next_calls < num_pushes || num_get_account_calls < num_pushes) && fc::time_point::now() < hard_deadline ){
       std::this_thread::sleep_for( 100ms );;
    }
 
@@ -141,13 +163,39 @@ void test_trxs_common(std::vector<const char*>& specific_args) {
    BOOST_CHECK_EQUAL( trace_with_except, 0 ); // should not have any traces with except in it
    BOOST_CHECK_EQUAL( num_pushes, num_posts );
    BOOST_CHECK_EQUAL( num_pushes, next_calls.load() );
-   BOOST_CHECK( trx_match.load() );  // trace should match the transaction  
+   BOOST_CHECK_EQUAL( num_pushes, num_get_account_calls.load() );
+   BOOST_CHECK( trx_match.load() );  // trace should match the transaction
 }
 
 // test read-only trxs on main thread (no --read-only-threads)
 BOOST_AUTO_TEST_CASE(no_read_only_threads) {
-   std::vector<const char*> specific_args = { "-p", "eosio", "-e" };
+   std::vector<const char*> specific_args = { "-p", "eosio", "-e", "--abi-serializer-max-time-ms=999" };
    test_trxs_common(specific_args);
 }
 
+// test read-only trxs on 1 threads (with --read-only-threads)
+BOOST_AUTO_TEST_CASE(with_1_read_only_threads) {
+   std::vector<const char*> specific_args = { "-p", "eosio", "-e",
+                                              "--read-only-threads=1",
+                                              "--max-transaction-time=10",
+                                              "--abi-serializer-max-time-ms=999",
+                                              "--read-only-write-window-time-us=100000",
+                                              "--read-only-read-window-time-us=40000",
+                                              "--disable-subjective-billing=true" };
+   test_trxs_common(specific_args);
+}
+
+// test read-only trxs on 8 separate threads (with --read-only-threads)
+BOOST_AUTO_TEST_CASE(with_8_read_only_threads) {
+   std::vector<const char*> specific_args = { "-p", "eosio", "-e",
+                                              "--read-only-threads=8",
+                                              "--max-transaction-time=10",
+                                              "--abi-serializer-max-time-ms=999",
+                                              "--read-only-write-window-time-us=100000",
+                                              "--read-only-read-window-time-us=40000",
+                                              "--disable-subjective-billing=true" };
+   test_trxs_common(specific_args);
+}
+
+
 BOOST_AUTO_TEST_SUITE_END()

plugins/producer_plugin/test/test_snapshot_scheduler.cpp

@@ -67,7 +67,7 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
             fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::debug);
             std::vector<const char*> argv =
                   {"test", "--data-dir", temp.c_str(), "--config-dir", temp.c_str(),
-                   "-p", "eosio", "-e", "--max-transaction-time", "475", "--disable-subjective-billing=true"};
+                   "-p", "eosio", "-e", "--disable-subjective-billing=true"};
             appbase::app().initialize<chain_plugin, producer_plugin>(argv.size(), (char**) &argv[0]);
             appbase::app().startup();
             plugin_promise.set_value(
@@ -93,18 +93,20 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
          auto bs = chain_plug->chain().block_start.connect([&pp](uint32_t bn) {
             // catching pending snapshot
             if (!pp->get_snapshot_requests().snapshot_requests.empty()) {
-               auto& pending = pp->get_snapshot_requests().snapshot_requests.begin()->pending_snapshots;
-               if (pending && pending->size()==1) {
-                  // lets check the head block num of it, it should be 8 + 1 = 9
-                  // this means we are getting a snapshot for correct block # as well
-                  BOOST_CHECK_EQUAL(9, pending->begin()->head_block_num);   
+               const auto& snapshot_requests = pp->get_snapshot_requests().snapshot_requests;
+               auto it = find_if(snapshot_requests.begin(), snapshot_requests.end(), [](const producer_plugin::snapshot_schedule_information& obj) {return obj.snapshot_request_id == 0;});
+               // we should have a pending snapshot for request id = 0
+               BOOST_REQUIRE(it != snapshot_requests.end());
+               auto& pending = it->pending_snapshots;
+               if (pending.size()==1) {
+                  BOOST_CHECK_EQUAL(9, pending.begin()->head_block_num);   
                }
             }
          });
        
-         producer_plugin::snapshot_request_information sri1 = {.block_spacing = 8, .start_block_num = 1, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 2"};
-         producer_plugin::snapshot_request_information sri2 = {.block_spacing = 5000, .start_block_num = 100000, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 2"};
-         producer_plugin::snapshot_request_information sri3 = {.block_spacing = 2, .start_block_num = 0, .end_block_num = 3, .snapshot_description = "Example of recurring snapshot 1"};
+         producer_plugin::snapshot_request_information sri1 = {.block_spacing = 8, .start_block_num = 1, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 1"};
+         producer_plugin::snapshot_request_information sri2 = {.block_spacing = 5000, .start_block_num = 100000, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 2 that will never happen"};
+         producer_plugin::snapshot_request_information sri3 = {.block_spacing = 2, .start_block_num = 0, .end_block_num = 3, .snapshot_description = "Example of recurring snapshot 3 that will expire"};
 
          pp->schedule_snapshot(sri1);
          pp->schedule_snapshot(sri2);
@@ -118,8 +120,13 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
          // one of the snapshots is done here and request, corresponding to it should be deleted
          BOOST_CHECK_EQUAL(2, pp->get_snapshot_requests().snapshot_requests.size());
 
-         // check whether no pending snapshots present
-         BOOST_CHECK_EQUAL(0, pp->get_snapshot_requests().snapshot_requests.begin()->pending_snapshots->size());
+         // check whether no pending snapshots present for a snapshot with id 0
+         const auto& snapshot_requests = pp->get_snapshot_requests().snapshot_requests;
+         auto it = find_if(snapshot_requests.begin(), snapshot_requests.end(),[](const producer_plugin::snapshot_schedule_information& obj) {return obj.snapshot_request_id == 0;});
+
+         // snapshot request with id = 0 should be found and should not have any pending snapshots
+         BOOST_REQUIRE(it != snapshot_requests.end());
+         BOOST_CHECK(!it->pending_snapshots.size());
 
          // quit app
          appbase::app().quit();

plugins/producer_plugin/test/test_trx_full.cpp

@@ -112,7 +112,7 @@ BOOST_AUTO_TEST_CASE(producer) {
          fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::debug);
          std::vector<const char*> argv =
                {"test", "--data-dir", temp_dir_str.c_str(), "--config-dir", temp_dir_str.c_str(),
-                "-p", "eosio", "-e", "--max-transaction-time", "475", "--disable-subjective-billing=true" };
+                "-p", "eosio", "-e", "--disable-subjective-billing=true" };
          app->initialize<chain_plugin, producer_plugin>( argv.size(), (char**) &argv[0] );
          app->startup();
          plugin_promise.set_value(

plugins/prometheus_plugin/include/eosio/prometheus_plugin/simple_rest_server.hpp

@@ -0,0 +1,234 @@
+#pragma once
+
+#include <boost/beast/core.hpp>
+#include <boost/beast/http.hpp>
+#include <boost/beast/version.hpp>
+
+namespace eosio { namespace rest {
+
+   // The majority of the code here are derived from boost source
+   // libs/beast/example/http/server/async/http_server_async.cpp
+   // with minimum modification and yet reusable.
+
+   namespace beast = boost::beast;         // from <boost/beast.hpp>
+   namespace http  = beast::http;          // from <boost/beast/http.hpp>
+   namespace net   = boost::asio;          // from <boost/asio.hpp>
+   using tcp       = boost::asio::ip::tcp; // from <boost/asio/ip/tcp.hpp>
+   template <typename T>
+   class simple_server {
+      T* self() { return static_cast<T*>(this); }
+
+      void fail(beast::error_code ec, char const* what) { self()->log_error(what, ec.message()); }
+      // Return a response for the given request.
+      http::response<http::string_body> handle_request(http::request<http::string_body>&& req) {
+         auto server_header = self()->server_header();
+         // Returns a bad request response
+         auto const bad_request = [&req, &server_header](std::string_view why) {
+            http::response<http::string_body> res{ http::status::bad_request, req.version() };
+            res.set(http::field::server, server_header);
+            res.set(http::field::content_type, "text/plain");
+            res.keep_alive(req.keep_alive());
+            res.body() = std::string(why);
+            res.prepare_payload();
+            return res;
+         };
+
+         // Returns a not found response
+         auto const not_found = [&req, &server_header](std::string_view target) {
+            http::response<http::string_body> res{ http::status::not_found, req.version() };
+            res.set(http::field::server, server_header);
+            res.set(http::field::content_type, "text/plain");
+            res.keep_alive(req.keep_alive());
+            res.body() = "The resource '" + std::string(target) + "' was not found.";
+            res.prepare_payload();
+            return res;
+         };
+
+         // Returns a server error response
+         auto const server_error = [&req, &server_header](std::string_view what) {
+            http::response<http::string_body> res{ http::status::internal_server_error, req.version() };
+            res.set(http::field::server, server_header);
+            res.set(http::field::content_type, "text/plain");
+            res.keep_alive(req.keep_alive());
+            res.body() = "An error occurred: '" + std::string(what) + "'";
+            res.prepare_payload();
+            return res;
+         };
+
+         // Make sure we can handle the method
+         if (!self()->allow_method(req.method()))
+            return bad_request("Unknown HTTP-method");
+
+         // Request path must be absolute and not contain "..".
+         std::string_view target{req.target().data(), req.target().size()};
+         if (target.empty() || target[0] != '/' || target.find("..") != std::string_view::npos)
+            return bad_request("Illegal request-target");
+
+         try {
+            auto res = self()->on_request(std::move(req));
+            if (!res)
+               return not_found(target);
+            return *res;
+         } catch (std::exception& ex) { return server_error(ex.what()); }
+      }
+
+      class session : public std::enable_shared_from_this<session> {
+         tcp::socket                                        socket_;
+         boost::asio::io_context::strand                    strand_;
+         beast::flat_buffer                                 buffer_;
+         http::request<http::string_body>                   req_;
+         simple_server*                                     server_;
+         std::shared_ptr<http::response<http::string_body>> res_;
+
+       public:
+         // Take ownership of the stream
+         session(net::io_context& ioc, tcp::socket&& socket, simple_server* server)
+             : socket_(std::move(socket)), strand_(ioc), server_(server) {}
+
+         // Start the asynchronous operation
+         void run() { do_read(); }
+
+         void do_read() {
+            // Make the request empty before reading,
+            // otherwise the operation behavior is undefined.
+            req_ = {};
+
+            // Read a request
+            http::async_read(
+                  socket_, buffer_, req_,
+                  boost::asio::bind_executor(strand_, [self = this->shared_from_this()](beast::error_code ec,
+                                                                                        std::size_t bytes_transferred) {
+                     self->on_read(ec, bytes_transferred);
+                  }));
+         }
+
+         void on_read(beast::error_code ec, std::size_t bytes_transferred) {
+            boost::ignore_unused(bytes_transferred);
+
+            // This means they closed the connection
+            if (ec == http::error::end_of_stream)
+               return do_close();
+
+            if (ec)
+               return server_->fail(ec, "read");
+
+            // Send the response
+            send_response(server_->handle_request(std::move(req_)));
+         }
+
+         void send_response(http::response<http::string_body>&& msg) {
+            // The lifetime of the message has to extend
+            // for the duration of the async operation so
+            // we use a shared_ptr to manage it.
+            res_ = std::make_shared<http::response<http::string_body>>(std::move(msg));
+
+            // Write the response
+            http::async_write(socket_, *res_,
+                              boost::asio::bind_executor(socket_.get_executor(),
+                                                         [self = this->shared_from_this(), close = res_->need_eof()](
+                                                               beast::error_code ec, std::size_t bytes_transferred) {
+                                                            self->on_write(ec, bytes_transferred, close);
+                                                         }));
+         }
+
+         void on_write(boost::system::error_code ec, std::size_t bytes_transferred, bool close) {
+            boost::ignore_unused(bytes_transferred);
+
+            if (ec)
+               return server_->fail(ec, "write");
+
+            if (close) {
+               // This means we should close the connection, usually because
+               // the response indicated the "Connection: close" semantic.
+               return do_close();
+            }
+
+            // We're done with the response so delete it
+            res_ = nullptr;
+
+            // Read another request
+            do_read();
+         }
+
+         void do_close() {
+            // Send a TCP shutdown
+            beast::error_code ec;
+            socket_.shutdown(tcp::socket::shutdown_send, ec);
+
+            // At this point the connection is closed gracefully
+         }
+      };
+
+      //------------------------------------------------------------------------------
+
+      // Accepts incoming connections and launches the sessions
+      class listener : public std::enable_shared_from_this<listener> {
+         net::io_context& ioc_;
+         tcp::acceptor    acceptor_;
+         tcp::socket      socket_;
+         simple_server*   server_;
+
+       public:
+         listener(net::io_context& ioc, tcp::endpoint endpoint, simple_server* server)
+             : ioc_(ioc), acceptor_(ioc), socket_(ioc), server_(server) {
+            boost::system::error_code ec;
+
+            // Open the acceptor
+            acceptor_.open(endpoint.protocol(), ec);
+            if (ec) {
+               server_->fail(ec, "open");
+               return;
+            }
+
+            // Allow address reuse
+            acceptor_.set_option(net::socket_base::reuse_address(true), ec);
+            if (ec) {
+               server_->fail(ec, "set_option");
+               return;
+            }
+
+            // Bind to the server address
+            acceptor_.bind(endpoint, ec);
+            if (ec) {
+               server_->fail(ec, "bind");
+               return;
+            }
+
+            // Start listening for connections
+            acceptor_.listen(net::socket_base::max_listen_connections, ec);
+            if (ec) {
+               server_->fail(ec, "listen");
+               return;
+            }
+         }
+
+         // Start accepting incoming connections
+         void run() {
+            if (!acceptor_.is_open())
+               return;
+            do_accept();
+         }
+
+       private:
+         void do_accept() {
+            acceptor_.async_accept(
+                  socket_, [self = this->shared_from_this()](boost::system::error_code ec) { self->on_accept(ec); });
+         }
+
+         void on_accept(boost::system::error_code ec) {
+            if (ec) {
+               server_->fail(ec, "accept");
+            } else {
+               // Create the session and run it
+               std::make_shared<session>(ioc_, std::move(socket_), server_)->run();
+            }
+
+            // Accept another connection
+            do_accept();
+         }
+      };
+
+    public:
+      void run(net::io_context& ioc, tcp::endpoint endpoint) { std::make_shared<listener>(ioc, endpoint, this)->run(); }
+   };
+}} // namespace eosio::rest

plugins/prometheus_plugin/prometheus_plugin.cpp

@@ -1,8 +1,11 @@
 #include <eosio/prometheus_plugin/prometheus_plugin.hpp>
+#include <eosio/prometheus_plugin/simple_rest_server.hpp>
+
 #include <eosio/chain/plugin_interface.hpp>
 #include <eosio/chain/thread_utils.hpp>
 #include <eosio/http_plugin/macros.hpp>
 #include <eosio/net_plugin/net_plugin.hpp>
+#include <eosio/http_plugin/http_plugin.hpp>
 #include <eosio/producer_plugin/producer_plugin.hpp>
 
 #include <fc/log/logger.hpp>
@@ -14,8 +17,8 @@
 #include <prometheus/text_serializer.h>
 #include <prometheus/registry.h>
 
-
 namespace eosio {
+   static const char* prometheus_api_name = "/v1/prometheus/metrics";
    using namespace prometheus;
    using namespace chain::plugin_interface;
 
@@ -110,12 +113,41 @@ namespace eosio {
       }
    };
 
-   struct prometheus_plugin_impl {
+   namespace http = boost::beast::http;
+   struct prometheus_plugin_impl : rest::simple_server<prometheus_plugin_impl> {
+
+      std::string server_header() const {
+         return http_plugin::get_server_header();
+      }
+
+      void log_error(char const* what, const std::string& message) {
+         elog("${what}: ${message}", ("what", what)("message", message));
+      }
+
+      bool allow_method(http::verb method) const {
+         return method == http::verb::get;
+      }
+
+      std::optional<http::response<http::string_body>>
+       on_request(http::request<http::string_body>&& req) {
+         if(req.target() != prometheus_api_name)
+            return {};
+         http::response<http::string_body> res{ http::status::ok, req.version() };
+         // Respond to GET request
+         res.set(http::field::server, server_header());
+         res.set(http::field::content_type, "text/plain");
+         res.keep_alive(req.keep_alive());
+         res.body() = metrics();
+         res.prepare_payload();
+         return res;
+      }
+
       eosio::chain::named_thread_pool<struct prom> _prometheus_thread_pool;
       boost::asio::io_context::strand _prometheus_strand;
       prometheus_plugin_metrics _metrics;
 
       map<std::string, vector<runtime_metric>> _plugin_metrics;
+      boost::asio::ip::tcp::endpoint           _endpoint;
 
       prometheus_plugin_impl(): _prometheus_strand(_prometheus_thread_pool.get_executor()){ }
 
@@ -179,32 +211,11 @@ namespace eosio {
          return body;
       }
 
-      void metrics_async(chain::plugin_interface::next_function<std::string> results) {
-         _prometheus_strand.post([self=this, results=std::move(results)]() {
-            results(self->metrics());
-         });
+      void start() {
+         run(_prometheus_thread_pool.get_executor(), _endpoint);
+         _prometheus_thread_pool.start(
+               1, [](const fc::exception& e) { elog("Prometheus exception ${e}", ("e", e)); });
       }
-
-   };
-
-   using metrics_params = fc::variant_object;
-
-   struct prometheus_api {
-      prometheus_plugin_impl& _pp;
-      fc::microseconds _max_response_time_us;
-
-      fc::time_point start() const {
-         return fc::time_point::now() + _max_response_time_us;
-      }
-
-      void metrics(const metrics_params&, chain::plugin_interface::next_function<std::string> results) {
-         _pp.metrics_async(std::move(results));
-      }
-
-      prometheus_api(prometheus_plugin_impl& plugin, const fc::microseconds& max_response_time)
-      : _pp(plugin)
-      , _max_response_time_us(max_response_time){}
-
    };
 
    prometheus_plugin::prometheus_plugin()
@@ -214,24 +225,35 @@ namespace eosio {
    prometheus_plugin::~prometheus_plugin() = default;
 
    void prometheus_plugin::set_program_options(options_description&, options_description& cfg) {
+      cfg.add_options()
+         ("prometheus-exporter-address", bpo::value<string>()->default_value("127.0.0.1:9101"),
+            "The local IP and port to listen for incoming prometheus metrics http request.");
    }
 
    void prometheus_plugin::plugin_initialize(const variables_map& options) {
       my->initialize_metrics();
 
-      auto& _http_plugin = app().get_plugin<http_plugin>();
-      fc::microseconds max_response_time = _http_plugin.get_max_response_time();
+      string lipstr = options.at("prometheus-exporter-address").as<string>();
+      EOS_ASSERT(lipstr.size() > 0, chain::plugin_config_exception, "prometheus-exporter-address must have a value");
 
-      prometheus_api handle(*my, max_response_time);
-      app().get_plugin<http_plugin>().add_async_api({
-        CALL_ASYNC_WITH_400(prometheus, handle, eosio, metrics, std::string, 200, http_params_types::no_params)}, http_content_type::plaintext);
+      string host = lipstr.substr(0, lipstr.find(':'));
+      string port = lipstr.substr(host.size() + 1, lipstr.size());
+
+      boost::system::error_code ec;
+      using tcp = boost::asio::ip::tcp;
+      tcp::resolver resolver(app().get_io_service());
+
+      my->_endpoint = *resolver.resolve(tcp::v4(), host, port, ec);
+      if (!ec) {
+         fc_ilog(logger(), "configured prometheus metrics exporter to listen on ${h}", ("h", lipstr));
+      } else {
+         fc_elog(logger(), "failed to configure prometheus metrics exporter to listen on ${h} (${m})",
+                 ("h", lipstr)("m", ec.message()));
+      }
    }
 
    void prometheus_plugin::plugin_startup() {
-      my->_prometheus_thread_pool.start(1, []( const fc::exception& e ) {
-         elog("Prometheus excpetion ${e}:${l}", ("e", e));
-      } );
-
+      my->start();
       ilog("Prometheus plugin started.");
    }
 

plugins/state_history_plugin/include/eosio/state_history_plugin/session.hpp

@@ -103,7 +103,7 @@ public:
    void pop_entry(bool call_send = true) {
       send_queue.erase(send_queue.begin());
       sending = false;
-      if (call_send)
+      if (call_send || !send_queue.empty())
          send();
    }
 
@@ -491,7 +491,10 @@ private:
          return;
       }
 
-      auto block_id = plugin->get_block_id(to_send_block_num);
+      // not just an optimization, on accepted_block signal may not be able to find block_num in forkdb as it has not been validated
+      // until after the accepted_block signal
+      std::optional<chain::block_id_type> block_id =
+          (block_state && block_state->block_num == to_send_block_num) ? block_state->id : plugin->get_block_id(to_send_block_num);
 
       if (block_id && position_it && (*position_it)->block_num == to_send_block_num) {
          // This branch happens when the head block of nodeos is behind the head block of connecting client.
@@ -550,6 +553,7 @@ private:
 
       state_history::get_blocks_result_v0 result;
       result.head = {block_state->block_num, block_state->id};
+      to_send_block_num = std::min(block_state->block_num, to_send_block_num);
       send_update(std::move(result), block_state);
    }
 

plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp

@@ -3,6 +3,7 @@
 
 #include <eosio/chain_plugin/chain_plugin.hpp>
 #include <eosio/state_history/types.hpp>
+#include <eosio/state_history/log.hpp>
 
 namespace fc {
 class variant;
@@ -11,7 +12,6 @@ class variant;
 namespace eosio {
 using chain::bytes;
 using std::shared_ptr;
-
 typedef shared_ptr<struct state_history_plugin_impl> state_history_ptr;
 
 class state_history_plugin : public plugin<state_history_plugin> {
@@ -29,6 +29,9 @@ class state_history_plugin : public plugin<state_history_plugin> {
 
    void handle_sighup() override;
 
+   const state_history_log* trace_log() const;
+   const state_history_log* chain_state_log() const;
+
  private:
    state_history_ptr my;
 };

plugins/state_history_plugin/state_history_plugin.cpp

@@ -100,6 +100,8 @@ struct state_history_plugin_impl : std::enable_shared_from_this<state_history_pl
 
    named_thread_pool<struct ship> thread_pool;
 
+   bool  plugin_started = false;
+
    static fc::logger& logger() { return _log; }
 
    std::optional<state_history_log>& get_trace_log() { return trace_log; }
@@ -292,9 +294,15 @@ struct state_history_plugin_impl : std::enable_shared_from_this<state_history_pl
              "the process");
       }
 
-      boost::asio::post(get_ship_executor(), [self = this->shared_from_this(), block_state]() {
-         self->session_mgr.send_update(block_state);
-      });
+      // avoid accumulating all these posts during replay before ship threads started
+      // that can lead to a large memory consumption and failures
+      // this is safe as there are no clients connected until after replay is complete
+      // this method is called from the main thread and "plugin_started" is set on the main thread as well when plugin is started 
+      if (plugin_started) {
+         boost::asio::post(get_ship_executor(), [self = this->shared_from_this(), block_state]() {
+            self->session_mgr.send_update(block_state);
+         });
+      }
 
    }
 
@@ -455,7 +463,7 @@ void state_history_plugin::plugin_initialize(const variables_map& options) {
 
       state_history_log_config ship_log_conf;
       if (options.count("state-history-log-retain-blocks")) {
-         auto ship_log_prune_conf = ship_log_conf.emplace<state_history::prune_config>();
+         auto& ship_log_prune_conf = ship_log_conf.emplace<state_history::prune_config>();
          ship_log_prune_conf.prune_blocks = options.at("state-history-log-retain-blocks").as<uint32_t>();
          //the arbitrary limit of 1000 here is mainly so that there is enough buffer for newly applied forks to be delivered to clients
          // before getting pruned out. ideally pruning would have been smart enough to know not to prune reversible blocks
@@ -495,7 +503,8 @@ void state_history_plugin::plugin_startup() {
       my->thread_pool.start( 1, [](const fc::exception& e) {
          fc_elog( _log, "Exception in SHiP thread pool, exiting: ${e}", ("e", e.to_detail_string()) );
          app().quit();
-      } );
+      });
+      my->plugin_started = true; 
    } catch (std::exception& ex) {
       appbase::app().quit();
    }
@@ -512,4 +521,12 @@ void state_history_plugin::handle_sighup() {
    fc::logger::update(logger_name, _log);
 }
 
+const state_history_log* state_history_plugin::trace_log() const {
+   return my->trace_log ? std::addressof(*my->trace_log) : nullptr;
+}
+
+const state_history_log* state_history_plugin::chain_state_log() const {
+   return my->chain_state_log ? std::addressof(*my->chain_state_log) : nullptr;
+}
+
 } // namespace eosio

plugins/state_history_plugin/tests/CMakeLists.txt

@@ -1,5 +1,5 @@
-add_executable( test_state_history_session session_test.cpp )
-target_link_libraries(test_state_history_session state_history Boost::unit_test_framework)
-target_include_directories( test_state_history_session PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../include" )
+add_executable( test_state_history session_test.cpp plugin_config_test.cpp)
+target_link_libraries(test_state_history state_history_plugin Boost::unit_test_framework)
+target_include_directories( test_state_history PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../include" )
 
-add_test(test_state_history_session test_state_history_session)
+add_test(test_state_history test_state_history)

plugins/state_history_plugin/tests/plugin_config_test.cpp

@@ -0,0 +1,40 @@
+#include <array>
+#include <boost/test/unit_test.hpp>
+#include <eosio/state_history/log.hpp>
+#include <eosio/state_history_plugin/state_history_plugin.hpp>
+#include <fc/filesystem.hpp>
+#include <stdint.h>
+
+BOOST_AUTO_TEST_CASE(state_history_plugin_default_tests) {
+   fc::temp_directory  tmp;
+   appbase::scoped_app app;
+
+   auto tmp_path = tmp.path().string();
+   std::array args = {"test_state_history",    "--trace-history", "--state-history-stride", "10",
+                      "--disable-replay-opts", "--data-dir",      tmp_path.c_str()};
+
+   BOOST_CHECK(app->initialize<eosio::state_history_plugin>(args.size(), const_cast<char**>(args.data())));
+   auto& plugin = app->get_plugin<eosio::state_history_plugin>();
+
+   BOOST_REQUIRE(plugin.trace_log());
+   auto* config = std::get_if<eosio::state_history::partition_config>(&plugin.trace_log()->config());
+   BOOST_REQUIRE(config);
+   BOOST_CHECK_EQUAL(config->max_retained_files, UINT32_MAX);
+}
+
+BOOST_AUTO_TEST_CASE(state_history_plugin_retain_blocks_tests) {
+   fc::temp_directory  tmp;
+   appbase::scoped_app app;
+
+   auto tmp_path = tmp.path().string();
+   std::array args = {"test_state_history",    "--trace-history", "--state-history-log-retain-blocks", "4242",
+                      "--disable-replay-opts", "--data-dir",      tmp_path.c_str()};
+
+   BOOST_CHECK(app->initialize<eosio::state_history_plugin>(args.size(), const_cast<char**>(args.data())));
+   auto& plugin = app->get_plugin<eosio::state_history_plugin>();
+
+   BOOST_REQUIRE(plugin.trace_log());
+   auto* config = std::get_if<eosio::state_history::prune_config>(&plugin.trace_log()->config());
+   BOOST_REQUIRE(config);
+   BOOST_CHECK_EQUAL(config->prune_blocks, 4242);
+}

plugins/test_control_api_plugin/test_control_api_plugin.cpp

@@ -51,7 +51,7 @@ void test_control_api_plugin::plugin_startup() {
 
    app().get_plugin<http_plugin>().add_api({
       TEST_CONTROL_RW_CALL(kill_node_on_producer, 202, http_params_types::params_required)
-   }, appbase::exec_queue::general);
+   }, appbase::exec_queue::read_write);
 }
 
 void test_control_api_plugin::plugin_shutdown() {}

plugins/wallet_api_plugin/wallet_api_plugin.cpp

@@ -115,7 +115,7 @@ void wallet_api_plugin::plugin_startup() {
             INVOKE_R_R_R(wallet_mgr, list_keys, std::string, std::string), 200),
        CALL_WITH_400(wallet, wallet_mgr, get_public_keys,
             INVOKE_R_V(wallet_mgr, get_public_keys), 200)
-   }, appbase::exec_queue::general);
+   }, appbase::exec_queue::read_write);
 }
 
 void wallet_api_plugin::plugin_initialize(const variables_map& options) {

programs/keosd/main.cpp

@@ -111,7 +111,7 @@ int main(int argc, char** argv)
                        [&a=app](string, string, url_response_callback cb) {
          cb(200, fc::time_point::maximum(), fc::variant(fc::variant_object()));
          a->quit();
-      }, appbase::exec_queue::general );
+      }, appbase::exec_queue::read_write );
       app->startup();
       app->exec();
    } catch (const fc::exception& e) {

programs/leap-util/actions/blocklog.cpp

@@ -88,19 +88,19 @@ void blocklog_actions::setup(CLI::App& app) {
    auto* extract_blocks = sub->add_subcommand("extract-blocks", "Extract range of blocks from blocks.log and write to output-dir.  Must give 'first' and/or 'last'.")->callback([err_guard]() { err_guard(&blocklog_actions::extract_blocks); });
    extract_blocks->add_option("--first,-f", opt->first_block, "The first block number to keep.")->required();
    extract_blocks->add_option("--last,-l", opt->last_block, "The last block number to keep.")->required();
-   extract_blocks->add_option("--output-dir", opt->output_dir, "The output directory for the block log extracted from blocks-dir.");
+   extract_blocks->add_option("--output-dir", opt->output_dir, "The output directory for the block log extracted from blocks-dir.")->required();
 
    // subcommand - split blocks
    auto* split_blocks = sub->add_subcommand("split-blocks", "Split the blocks.log based on the stride and store the result in the specified 'output-dir'.")->callback([err_guard]() { err_guard(&blocklog_actions::split_blocks); });
    split_blocks->add_option("--blocks-dir", opt->blocks_dir, "The location of the blocks directory (absolute path or relative to the current directory).");
-   split_blocks->add_option("--output-dir", opt->output_dir, "The output directory for the split block log.");
+   split_blocks->add_option("--output-dir", opt->output_dir, "The output directory for the split block log.")->required();
    split_blocks->add_option("--stride", opt->stride, "The number of blocks to split into each file.")->required();
 
    // subcommand - merge blocks
    auto* merge_blocks = sub->add_subcommand("merge-blocks", "Merge block log files in 'blocks-dir' with the file pattern 'blocks-\\d+-\\d+.[log,index]' to 'output-dir' whenever possible."
           "The files in 'blocks-dir' will be kept without change.")->callback([err_guard]() { err_guard(&blocklog_actions::merge_blocks); });
    merge_blocks->add_option("--blocks-dir", opt->blocks_dir, "The location of the blocks directory (absolute path or relative to the current directory).");
-   merge_blocks->add_option("--output-dir", opt->output_dir, "The output directory for the merged block log.");
+   merge_blocks->add_option("--output-dir", opt->output_dir, "The output directory for the merged block log.")->required();
 
    // subcommand - smoke test
    sub->add_subcommand("smoke-test", "Quick test that blocks.log and blocks.index are well formed and agree with each other.")->callback([err_guard]() { err_guard(&blocklog_actions::smoke_test); });
@@ -169,32 +169,33 @@ int blocklog_actions::extract_blocks() {
 }
 
 int blocklog_actions::do_genesis() {
-   std::optional<genesis_state> gs;
    bfs::path bld = opt->blocks_dir;
-   auto full_path = (bld / "blocks.log").generic_string();
 
-   if(fc::exists(bld / "blocks.log")) {
-      gs = block_log::extract_genesis_state(opt->blocks_dir);
-      if(!gs) {
-         std::cerr << "Block log at '" << full_path
-                   << "' does not contain a genesis state, it only has the chain-id." << std::endl;
-         return -1;
-      }
-   } else {
-      std::cerr << "No blocks.log found at '" << full_path << "'." << std::endl;
+   auto context = block_log::extract_chain_context(opt->blocks_dir,opt->blocks_dir);
+   
+   if (!context) {
+      std::cerr << "No blocks log found at '" << opt->blocks_dir.c_str() << "'." << std::endl;
       return -1;
    }
 
+   if(!std::holds_alternative<genesis_state>(*context)) {
+      std::cerr << "Block log at '" << opt->blocks_dir.c_str()
+                  << "' does not contain a genesis state, it only has the chain-id." << std::endl;
+      return -1;
+   } 
+
+   const genesis_state& gs = std::get<genesis_state>(*context);
+
    // just print if output not set
    if(opt->output_file.empty()) {
-      std::cout << json::to_pretty_string(*gs) << std::endl;
+      std::cout << json::to_pretty_string(gs) << std::endl;
    } else {
       bfs::path p = opt->output_file;
       if(p.is_relative()) {
          p = bfs::current_path() / p;
       }
 
-      if(!fc::json::save_to_file(*gs, p, true)) {
+      if(!fc::json::save_to_file(gs, p, true)) {
          std::cerr << "Error occurred while writing genesis JSON to '" << p.generic_string() << "'" << std::endl;
          return -1;
       }

scripts/CMakeLists.txt

@@ -6,3 +6,4 @@ configure_file(eosio-tn_up.sh eosio-tn_up.sh COPYONLY)
 configure_file(abi_is_json.py abi_is_json.py COPYONLY)
 configure_file(postinst .)
 configure_file(prerm .)
+configure_file(install_testharness_symlinks.cmake .)

scripts/install_testharness_symlinks.cmake

@@ -0,0 +1,5 @@
+execute_process( COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages  ERROR_QUIET RESULT_VARIABLE ret)
+if(ret EQUAL "0")
+    execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink ../../../share/leap_testing/tests/TestHarness ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages/TestHarness)
+    execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink ../../bin ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing/bin)
+endif()

tests/CMakeLists.txt

@@ -68,6 +68,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/large-lib-test.py ${CMAKE_CURRENT_BIN
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/launcher.py ${CMAKE_CURRENT_BINARY_DIR}/launcher.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/auto_bp_peering_test.py ${CMAKE_CURRENT_BINARY_DIR}/auto_bp_peering_test.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/auto_bp_peering_test_shape.json ${CMAKE_CURRENT_BINARY_DIR}/auto_bp_peering_test_shape.json COPYONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/split_blocklog_replay_test.py ${CMAKE_CURRENT_BINARY_DIR}/split_blocklog_replay_test.py COPYONLY)
 
 if(DEFINED ENV{GITHUB_ACTIONS})
   set(UNSHARE "--unshared")
@@ -113,8 +114,8 @@ set_property(TEST ship_test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME ship_test_unix COMMAND tests/ship_test.py -v --num-clients 10 --num-requests 5000 --clean-run ${UNSHARE} --unix-socket WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST ship_test_unix PROPERTY LABELS nonparallelizable_tests)
 
-add_test(NAME ship_streamer_test COMMAND tests/ship_streamer_test.py -v --num-clients 10 --num-blocks 50 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
-set_property(TEST ship_streamer_test PROPERTY LABELS nonparallelizable_tests)
+add_test(NAME ship_streamer_test COMMAND tests/ship_streamer_test.py -v --num-clients 10 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+set_property(TEST ship_streamer_test PROPERTY LABELS long_running_tests)
 
 add_test(NAME p2p_dawn515_test COMMAND tests/p2p_tests/dawn_515/test.sh WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST p2p_dawn515_test PROPERTY LABELS nonparallelizable_tests)
@@ -125,11 +126,11 @@ add_test(NAME nodeos_protocol_feature_test COMMAND tests/nodeos_protocol_feature
 set_property(TEST nodeos_protocol_feature_test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME compute_transaction_test COMMAND tests/compute_transaction_test.py -v -p 2 -n 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST compute_transaction_test PROPERTY LABELS nonparallelizable_tests)
-add_test(NAME read-only-trx-basic-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME read-only-trx-basic-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --read-only-threads 0 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST read-only-trx-basic-test PROPERTY LABELS nonparallelizable_tests)
-add_test(NAME read-only-trx-parallel-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --read-only-threads 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME read-only-trx-parallel-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --read-only-threads 3 --num-test-runs 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST read-only-trx-parallel-test PROPERTY LABELS nonparallelizable_tests)
-add_test(NAME read-only-trx-parallel-eos-vm-oc-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --eos-vm-oc-enable --read-only-threads 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME read-only-trx-parallel-eos-vm-oc-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --eos-vm-oc-enable --read-only-threads 3 --num-test-runs 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST read-only-trx-parallel-eos-vm-oc-test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME subjective_billing_test COMMAND tests/subjective_billing_test.py -v -p 2 -n 4 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST subjective_billing_test PROPERTY LABELS nonparallelizable_tests)
@@ -138,6 +139,8 @@ set_property(TEST get_account_test PROPERTY LABELS nonparallelizable_tests)
 
 add_test(NAME distributed-transactions-test COMMAND tests/distributed-transactions-test.py -d 2 -p 4 -n 6 -v --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST distributed-transactions-test PROPERTY LABELS nonparallelizable_tests)
+add_test(NAME distributed-transactions-speculative-test COMMAND tests/distributed-transactions-test.py -d 2 -p 4 -n 6 --speculative -v --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+set_property(TEST distributed-transactions-speculative-test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME restart-scenarios-test-resync COMMAND tests/restart-scenarios-test.py -c resync -p4 -v --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST restart-scenarios-test-resync PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME restart-scenarios-test-hard_replay COMMAND tests/restart-scenarios-test.py -c hardReplay -p4 -v --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
@@ -257,6 +260,9 @@ set_property(TEST light_validation_sync_test PROPERTY LABELS nonparallelizable_t
 add_test(NAME auto_bp_peering_test COMMAND tests/auto_bp_peering_test.py ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST auto_bp_peering_test PROPERTY LABELS long_running_tests)
 
+add_test(NAME split_blocklog_replay_test COMMAND tests/split_blocklog_replay_test.py ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+set_property(TEST split_blocklog_replay_test PROPERTY LABELS nonparallelizable_tests)
+
 if(ENABLE_COVERAGE_TESTING)
 
   set(Coverage_NAME ${PROJECT_NAME}_coverage)

tests/TestHarness/Cluster.py

@@ -22,8 +22,11 @@ from .Node import BlockType
 from .Node import Node
 from .WalletMgr import WalletMgr
 from .launch_transaction_generators import TransactionGeneratorsLauncher, TpsTrxGensConfig
-from .libc import unshare, CLONE_NEWNET
-from .interfaces import getInterfaceFlags, setInterfaceUp, IFF_LOOPBACK
+try:
+    from .libc import unshare, CLONE_NEWNET
+    from .interfaces import getInterfaceFlags, setInterfaceUp, IFF_LOOPBACK
+except:
+    pass
 
 # Protocol Feature Setup Policy
 class PFSetupPolicy:
@@ -1447,7 +1450,7 @@ class Cluster(object):
     def killall(self, kill=True, silent=True, allInstances=False):
         """Kill cluster nodeos instances. allInstances will kill all nodeos instances running on the system."""
         signalNum=9 if kill else 15
-        cmd="%s -k %d --nogen -p 1 -n 1 --nodeos-log-path %s" % (f"python3 {str(self.launcherPath)}", signalNum, self.nodeosLogPath)
+        cmd="%s -k %d --nogen -p 1 -n 1 --nodeos-log-path %s" % (f"{sys.executable} {str(self.launcherPath)}", signalNum, self.nodeosLogPath)
         if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
         if 0 != subprocess.call(cmd.split(), stdout=Utils.FNull):
             if not silent: Utils.Print("Launcher failed to shut down eos cluster.")

tests/TestHarness/Node.py

@@ -61,7 +61,7 @@ class Node(Transactions):
             self.fetchBlock = lambda blockNum: self.processUrllibRequest("trace_api", "get_block", {"block_num":blockNum}, silentErrors=False, exitOnError=True)
             self.fetchKeyCommand = lambda: "[transaction][transaction_header][ref_block_num]"
             self.fetchRefBlock = lambda trans: trans["transaction_header"]["ref_block_num"]
-            self.cleosLimit = "--time-limit 99999"
+            self.cleosLimit = "--time-limit 999"
             self.fetchHeadBlock = lambda node, headBlock: node.processUrllibRequest("chain", "get_block_info", {"block_num":headBlock}, silentErrors=False, exitOnError=True)
 
     def __str__(self):

tests/TestHarness/WalletMgr.py

@@ -12,9 +12,9 @@ from .testUtils import Utils
 Wallet=namedtuple("Wallet", "name password host port")
 # pylint: disable=too-many-instance-attributes
 class WalletMgr(object):
-    __walletLogOutFile=f"{Utils.TestLogRoot}/test_keosd_out.log"
-    __walletLogErrFile=f"{Utils.TestLogRoot}/test_keosd_err.log"
     __walletDataDir=f"{Utils.TestLogRoot}/test_wallet_0"
+    __walletLogOutFile=f"{__walletDataDir}/test_keosd_out.log"
+    __walletLogErrFile=f"{__walletDataDir}/test_keosd_err.log"
     __MaxPort=9999
 
     # pylint: disable=too-many-arguments
@@ -83,6 +83,9 @@ class WalletMgr(object):
         cmd="%s --data-dir %s --config-dir %s --unlock-timeout=999999 --http-server-address=%s:%d --http-max-response-time-ms 99999 --verbose-http-errors" % (
             Utils.EosWalletPath, WalletMgr.__walletDataDir, WalletMgr.__walletDataDir, self.host, self.port)
         if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
+        if not os.path.isdir(WalletMgr.__walletDataDir):
+            if Utils.Debug: Utils.Print(f"Creating dir {WalletMgr.__walletDataDir} in dir: {os.getcwd()}")
+            os.mkdir(WalletMgr.__walletDataDir)
         with open(WalletMgr.__walletLogOutFile, 'w') as sout, open(WalletMgr.__walletLogErrFile, 'w') as serr:
             popen=subprocess.Popen(cmd.split(), stdout=sout, stderr=serr)
             self.__walletPid=popen.pid
@@ -299,6 +302,5 @@ class WalletMgr(object):
 
     @staticmethod
     def cleanup():
-        dataDir=WalletMgr.__walletDataDir
-        if os.path.isdir(dataDir) and os.path.exists(dataDir):
+        if os.path.isdir(WalletMgr.__walletDataDir) and os.path.exists(WalletMgr.__walletDataDir):
             shutil.rmtree(WalletMgr.__walletDataDir)

tests/TestHarness/queries.py

@@ -596,15 +596,18 @@ class NodeosQueries:
 
         return trans
 
-    def processUrllibRequest(self, resource, command, payload={}, silentErrors=False, exitOnError=False, exitMsg=None, returnType=ReturnType.json, endpoint=None):
+    def processUrllibRequest(self, resource, command, payload={}, silentErrors=False, exitOnError=False, exitMsg=None, returnType=ReturnType.json, method="POST", endpoint=None):
         if not endpoint:
             endpoint = self.endpointHttp
         cmd = f"{endpoint}/v1/{resource}/{command}"
-        req = urllib.request.Request(cmd, method="POST")
-        req.add_header('Content-Type', 'application/json')
-        data = payload
-        data = json.dumps(data)
-        data = data.encode()
+        req = urllib.request.Request(cmd, method=method)
+        if len(payload):
+            req.add_header('Content-Type', 'application/json')
+            data = payload
+            data = json.dumps(data)
+            data = data.encode()
+        else:
+            data = None
         if Utils.Debug: Utils.Print("cmd: %s %s" % (cmd, payload))
         rtn=None
         start=time.perf_counter()
@@ -640,6 +643,8 @@ class NodeosQueries:
                         rtn = ex.read()
                     else:
                         unhandledEnumType(returnType)
+            elif returnType==ReturnType.raw:
+                return ex.code
             else:
                 return None
 
@@ -772,4 +777,3 @@ class NodeosQueries:
     def getActivatedProtocolFeatures(self):
         latestBlockHeaderState = self.getLatestBlockHeaderState()
         return latestBlockHeaderState["activated_protocol_features"]["protocol_features"]
-

tests/auto_bp_peering_test.py

@@ -35,8 +35,7 @@ args = TestHelper.parse_args({
 Utils.Debug = args.v
 killAll = args.clean_run
 dumpErrorDetails = args.dump_error_details
-# dontKill = args.leave_running
-dontKill = True
+dontKill = args.leave_running
 killEosInstances = not dontKill
 killWallet = not dontKill
 keepLogs = args.keep_logs

tests/cli_test.py

@@ -11,7 +11,7 @@ import time
 import shutil
 import signal
 
-from TestHarness import Account, Cluster, Node, ReturnType, Utils, WalletMgr
+from TestHarness import Account, Node, ReturnType, Utils, WalletMgr
 
 testSuccessful=False
 
@@ -352,7 +352,7 @@ def abi_file_with_nodeos_test():
         os.makedirs(data_dir, exist_ok=True)
         walletMgr = WalletMgr(True)
         walletMgr.launch()
-        node = Node('localhost', 8888, nodeId, cmd="./programs/nodeos/nodeos -e -p eosio --plugin eosio::trace_api_plugin --trace-no-abis --plugin eosio::producer_plugin --plugin eosio::producer_api_plugin --plugin eosio::chain_api_plugin --plugin eosio::chain_plugin --plugin eosio::http_plugin --access-control-allow-origin=* --http-validate-host=false --resource-monitor-not-shutdown-on-threshold-exceeded " + "--data-dir " + data_dir + " --config-dir " + data_dir, walletMgr=walletMgr)
+        node = Node('localhost', 8888, nodeId, cmd="./programs/nodeos/nodeos -e -p eosio --plugin eosio::trace_api_plugin --trace-no-abis --plugin eosio::producer_plugin --plugin eosio::producer_api_plugin --plugin eosio::chain_api_plugin --plugin eosio::chain_plugin --plugin eosio::http_plugin --access-control-allow-origin=* --http-validate-host=false --max-transaction-time=-1 --resource-monitor-not-shutdown-on-threshold-exceeded " + "--data-dir " + data_dir + " --config-dir " + data_dir, walletMgr=walletMgr)
         node.verifyAlive() # Setting node state to not alive
         node.relaunch(newChain=True, cachePopen=True)
         node.waitForBlock(1)
@@ -403,7 +403,7 @@ def abi_file_with_nodeos_test():
                     os.kill(node.pid, signal.SIGKILL)
         if testSuccessful:
             Utils.Print("Cleanup nodeos data.")
-            shutil.rmtree(data_dir)
+            shutil.rmtree(Utils.DataPath)
 
         if malicious_token_abi_path:
             if os.path.exists(malicious_token_abi_path):

tests/distributed-transactions-test.py

@@ -3,6 +3,7 @@
 import random
 
 from TestHarness import Cluster, TestHelper, Utils, WalletMgr
+from TestHarness.TestHelper import AppArgs
 
 ###############################################################
 # distributed-transactions-test
@@ -19,8 +20,10 @@ from TestHarness import Cluster, TestHelper, Utils, WalletMgr
 Print=Utils.Print
 errorExit=Utils.errorExit
 
-args=TestHelper.parse_args({"-p","-n","-d","-s","--nodes-file","--seed"
-                           ,"--dump-error-details","-v","--leave-running","--clean-run","--keep-logs","--unshared"})
+appArgs = AppArgs()
+extraArgs = appArgs.add_bool(flag="--speculative", help="Run nodes in read-mode=speculative")
+args=TestHelper.parse_args({"-p","-n","-d","-s","--nodes-file","--seed", "--speculative"
+                           ,"--dump-error-details","-v","--leave-running","--clean-run","--keep-logs","--unshared"}, applicationSpecificArgs=appArgs)
 
 pnodes=args.p
 topo=args.s
@@ -34,6 +37,7 @@ dontKill=args.leave_running
 dumpErrorDetails=args.dump_error_details
 killAll=args.clean_run
 keepLogs=args.keep_logs
+speculative=args.speculative
 
 killWallet=not dontKill
 killEosInstances=not dontKill
@@ -71,7 +75,11 @@ try:
                (pnodes, total_nodes-pnodes, topo, delay))
 
         Print("Stand up cluster")
-        if cluster.launch(pnodes=pnodes, totalNodes=total_nodes, topo=topo, delay=delay) is False:
+        extraNodeosArgs = ""
+        if speculative:
+           extraNodeosArgs = " --read-mode speculative "
+
+        if cluster.launch(pnodes=pnodes, totalNodes=total_nodes, topo=topo, delay=delay, extraNodeosArgs=extraNodeosArgs) is False:
             errorExit("Failed to stand up eos cluster.")
 
         Print ("Wait for Cluster stabilization")

tests/http_plugin_test.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 
 from TestHarness import Cluster, Node, ReturnType, TestHelper, Utils
+import urllib.request
+import sys
 
 ###############################################################
 # http_plugin_tests.py
@@ -22,7 +24,7 @@ dumpErrorDetails = dumpErrorDetails=args.dump_error_details
 
 
 Utils.Debug=debug
-cluster=Cluster(walletd=True,unshared=args.unshared)
+cluster=Cluster(host="127.0.0.1",walletd=True,unshared=args.unshared)
 
 testSuccessful=False
 
@@ -34,17 +36,34 @@ try:
     TestHelper.printSystemInfo("BEGIN")
 
     Print("Stand up cluster")
-
-    if cluster.launch(dontBootstrap=True, loadSystemContract=False) is False:
+    node0_extra_config = "--http-validate-host true --http-server-address 127.0.0.1:8888"
+    if cluster.launch(dontBootstrap=True, loadSystemContract=False, specificExtraNodeosArgs = {0: node0_extra_config}) is False:
         cmdError("launcher")
         errorExit("Failed to stand up eos cluster.")
-
     Print("Getting cluster info")
     cluster.getInfos()
+
+    node0 = cluster.nodes[0]
+    ## HTTP plugin listens to 127.0.0.1:8888 by default. With the --http-validate-host=true,
+    ## the HTTP request to "http://localhost:8888" should fail because the HOST header doesn't
+    ## match "127.0.0.1".
+
+    def get_info_status(url):
+        try:
+            req = urllib.request.Request( url, method = "GET")
+            return urllib.request.urlopen(req, data=None).code
+        except urllib.error.HTTPError as response:
+            return response.code
+        except:
+            e = sys.exc_info()[0]
+            return e
+    url = node0.endpointHttp.replace("127.0.0.1", "localhost") + "/v1/chain/get_info"
+    code = get_info_status(url)
+    assert code == 400, f"Expected HTTP returned code 400, got {code}"
     testSuccessful = True
+
 finally:
     TestHelper.shutdown(cluster, None, testSuccessful, killEosInstances, True, keepLogs, killAll, dumpErrorDetails)
 
 exitCode = 0 if testSuccessful else 1
 exit(exitCode)
-        

tests/launcher.py

@@ -174,7 +174,7 @@ class launcher(object):
         parser.add_argument('-i', '--timestamp', help='set the timestamp for the first block.  Use "now" to indicate the current time')
         parser.add_argument('-l', '--launch', choices=['all', 'none', 'local'], help='select a subset of nodes to launch.  If not set, the default is to launch all unless an output file is named, in which case none are started.', default='all')
         parser.add_argument('-o', '--output', help='save a copy of the generated topology in this file and exit without launching', dest='topology_filename')
-        parser.add_argument('-k', '--kill', help='retrieve the list of previously started process ids and issue a kill to each')
+        parser.add_argument('-k', '--kill', type=int, help='kill a network as specified in arguments with given signal')
         parser.add_argument('--down', type=comma_separated, help='comma-separated list of node numbers that will be shut down', default=[])
         parser.add_argument('--bounce', type=comma_separated, help='comma-separated list of node numbers that will be restarted', default=[])
         parser.add_argument('--roll', type=comma_separated, help='comma-separated list of host names where the nodes will be rolled to a new version')
@@ -613,6 +613,17 @@ plugin = eosio::chain_api_plugin
                     if relaunch:
                         self.launch(node)
 
+    def kill(self, signum):
+        errorCode = 0
+        for node in self.network.nodes.values():
+            try:
+                with open(node.data_dir_name / f'{Utils.EosServerName}.pid', 'r') as f:
+                    pid = int(f.readline())
+                    self.terminate_wait_pid(pid, signum, raise_if_missing=False)
+            except FileNotFoundError as err:
+                errorCode = 1
+        return errorCode
+
     def start_all(self):
         if self.args.launch.lower() != 'none':
             for instance in self.network.nodes.values():
@@ -628,8 +639,9 @@ plugin = eosio::chain_api_plugin
                     f.write(f'"{node.dot_label}"->"{pname}" [dir="forward"];\n')
             f.write('}')
 
-    def terminate_wait_pid(self, pid, raise_if_missing=True):
-        '''Terminate a non-child process with SIGTERM and wait for it to exit.'''
+    def terminate_wait_pid(self, pid, signum = signal.SIGTERM, raise_if_missing=True):
+        '''Terminate a non-child process with given signal number or with SIGTERM if not
+           provided and wait for it to exit.'''
         if sys.version_info >= (3, 9) and platform.system() == 'Linux': # on our supported platforms, Python 3.9 accompanies a kernel > 5.3
             try:
                 fd = os.pidfd_open(pid)
@@ -640,7 +652,7 @@ plugin = eosio::chain_api_plugin
                 po = select.poll()
                 po.register(fd, select.POLLIN)
                 try:
-                    os.kill(pid, signal.SIGTERM)
+                    os.kill(pid, signum)
                 except ProcessLookupError:
                     if raise_if_missing:
                         raise
@@ -663,7 +675,7 @@ plugin = eosio::chain_api_plugin
                     return min(delay * 2, 0.04)
                 delay = 0.0001
                 try:
-                    os.kill(pid, signal.SIGTERM)
+                    os.kill(pid, signum)
                 except ProcessLookupError:
                     if raise_if_missing:
                         raise
@@ -676,12 +688,16 @@ plugin = eosio::chain_api_plugin
                         return
 
 if __name__ == '__main__':
+    errorCode = 0
     l = launcher(sys.argv[1:])
     if len(l.args.down):
         l.down(l.args.down)
     elif len(l.args.bounce):
         l.bounce(l.args.bounce)
+    elif l.args.kill:
+        errorCode = l.kill(l.args.kill)
     elif l.args.launch == 'all' or l.args.launch == 'local':
         l.start_all()
     for f in glob.glob(Utils.DataPath):
         shutil.rmtree(f)
+    sys.exit(errorCode)

tests/nodeos_contrl_c_test.py

@@ -115,7 +115,7 @@ try:
         errorExit("Failed to kill the seed node")
 
 finally:
-    TestHelper.shutdown(cluster, walletMgr, testSuccessful=testSuccessful, killEosInstances=True, killWallet=True, keepLogs=True, cleanRun=True, dumpErrorDetails=True)
+    TestHelper.shutdown(cluster, walletMgr, testSuccessful=testSuccessful, killEosInstances=True, killWallet=True, keepLogs=False, cleanRun=True, dumpErrorDetails=True)
 
 errorCode = 0 if testSuccessful else 1
 exit(errorCode)

tests/nodeos_forked_chain_test.py

@@ -5,8 +5,10 @@ from datetime import timedelta
 import time
 import json
 import signal
+import os
 
 from TestHarness import Cluster, Node, TestHelper, Utils, WalletMgr, CORE_SYMBOL
+from TestHarness.TestHelper import AppArgs
 
 ###############################################################
 # nodeos_forked_chain_test
@@ -27,6 +29,8 @@ from TestHarness import Cluster, Node, TestHelper, Utils, WalletMgr, CORE_SYMBOL
 #   Time is allowed to progress so that the "bridge" node can catchup and both producer nodes to come to consensus
 #   The block log is then checked for both producer nodes to verify that the 10 producer fork is selected and that
 #       both nodes are in agreement on the block log.
+#   This test also runs a state_history_plugin (SHiP) on node 0 and uses ship_streamer to verify all blocks are received
+#   across the fork.
 #
 ###############################################################
 
@@ -121,9 +125,10 @@ def getMinHeadAndLib(prodNodes):
     return (headBlockNum, libNum)
 
 
-
+appArgs = AppArgs()
+extraArgs = appArgs.add(flag="--num-ship-clients", type=int, help="How many ship_streamers should be started", default=2)
 args = TestHelper.parse_args({"--prod-count","--dump-error-details","--keep-logs","-v","--leave-running","--clean-run",
-                              "--wallet-port","--unshared"})
+                              "--wallet-port","--unshared"}, applicationSpecificArgs=appArgs)
 Utils.Debug=args.v
 totalProducerNodes=2
 totalNonProducerNodes=1
@@ -137,6 +142,7 @@ dontKill=args.leave_running
 prodCount=args.prod_count
 killAll=args.clean_run
 walletPort=args.wallet_port
+num_clients=args.num_ship_clients
 
 walletMgr=WalletMgr(True, port=walletPort)
 testSuccessful=False
@@ -154,6 +160,9 @@ try:
     cluster.cleanup()
     Print("Stand up cluster")
     specificExtraNodeosArgs={}
+    shipNodeNum = 0
+    specificExtraNodeosArgs[shipNodeNum]="--plugin eosio::state_history_plugin --disable-replay-opts"
+
     # producer nodes will be mapped to 0 through totalProducerNodes-1, so the number totalProducerNodes will be the non-producing node
     specificExtraNodeosArgs[totalProducerNodes]="--plugin eosio::test_control_api_plugin"
 
@@ -286,6 +295,31 @@ try:
     timestampStr=Node.getBlockAttribute(block, "timestamp", blockNum)
     timestamp=datetime.strptime(timestampStr, Utils.TimeFmt)
 
+    shipNode = cluster.getNode(0)
+    block_range = 1000
+    start_block_num = blockNum
+    end_block_num = start_block_num + block_range
+
+    shipClient = "tests/ship_streamer"
+    cmd = f"{shipClient} --start-block-num {start_block_num} --end-block-num {end_block_num} --fetch-block --fetch-traces --fetch-deltas"
+    if Utils.Debug: Utils.Print(f"cmd: {cmd}")
+    clients = []
+    files = []
+    shipTempDir = os.path.join(Utils.DataDir, "ship")
+    os.makedirs(shipTempDir, exist_ok = True)
+    shipClientFilePrefix = os.path.join(shipTempDir, "client")
+
+    starts = []
+    for i in range(0, num_clients):
+        start = time.perf_counter()
+        outFile = open(f"{shipClientFilePrefix}{i}.out", "w")
+        errFile = open(f"{shipClientFilePrefix}{i}.err", "w")
+        Print(f"Start client {i}")
+        popen=Utils.delayedCheckOutput(cmd, stdout=outFile, stderr=errFile)
+        starts.append(time.perf_counter())
+        clients.append((popen, cmd))
+        files.append((outFile, errFile))
+        Utils.Print(f"Client {i} started, Ship node head is: {shipNode.getBlockNum()}")
 
     # ***   Identify what the production cycle is   ***
 
@@ -559,6 +593,25 @@ try:
         Utils.errorExit("Did not find find block %s (the original divergent block) in blockProducers0, test setup is wrong.  blockProducers0: %s" % (killBlockNum, ", ".join(blockProducers0)))
     Print("Fork resolved and determined producer %s for block %s" % (resolvedKillBlockProducer, killBlockNum))
 
+    Print(f"Stopping all {num_clients} clients")
+    for index, (popen, _), (out, err), start in zip(range(len(clients)), clients, files, starts):
+        popen.wait()
+        Print(f"Stopped client {index}.  Ran for {time.perf_counter() - start:.3f} seconds.")
+        out.close()
+        err.close()
+        outFile = open(f"{shipClientFilePrefix}{index}.out", "r")
+        data = json.load(outFile)
+        block_num = start_block_num
+        for i in data:
+            # fork can cause block numbers to be repeated
+            this_block_num = i['get_blocks_result_v0']['this_block']['block_num']
+            if this_block_num < block_num:
+                block_num = this_block_num
+            assert block_num == this_block_num, f"{block_num} != {this_block_num}"
+            assert isinstance(i['get_blocks_result_v0']['block'], str) # verify block in result
+            block_num += 1
+        assert block_num-1 == end_block_num, f"{block_num-1} != {end_block_num}"
+
     blockProducers0=[]
     blockProducers1=[]
 

tests/nodeos_irreversible_mode_test.py

@@ -21,7 +21,7 @@ errorExit = Utils.errorExit
 cmdError = Utils.cmdError
 relaunchTimeout = 30
 numOfProducers = 4
-totalNodes = 10
+totalNodes = 20
 
 # Parse command line arguments
 args = TestHelper.parse_args({"-v","--clean-run","--dump-error-details","--leave-running","--keep-logs","--unshared"})
@@ -32,6 +32,8 @@ dontKill=args.leave_running
 killEosInstances=not dontKill
 killWallet=not dontKill
 keepLogs=args.keep_logs
+speculativeReadMode="head"
+blockLogRetainBlocks="10000"
 
 # Setup cluster and it's wallet manager
 walletMgr=WalletMgr(True)
@@ -174,7 +176,18 @@ try:
          0:"--enable-stale-production",
          4:"--read-mode irreversible",
          6:"--read-mode irreversible",
-         9:"--plugin eosio::producer_api_plugin"})
+         9:"--plugin eosio::producer_api_plugin",
+        10:"--read-mode speculative",
+        11:"--read-mode irreversible",
+        12:"--read-mode speculative",
+        13:"--read-mode irreversible",
+        14:"--read-mode speculative --plugin eosio::producer_api_plugin",
+        15:"--read-mode speculative",
+        16:"--read-mode irreversible",
+        17:"--read-mode speculative",
+        18:"--read-mode irreversible",
+        19:"--read-mode speculative --plugin eosio::producer_api_plugin"
+        })
 
    producingNodeId = 0
    producingNode = cluster.getNode(producingNodeId)
@@ -254,7 +267,7 @@ try:
 
       # Kill and relaunch in irreversible mode
       nodeToTest.kill(signal.SIGTERM)
-      relaunchNode(nodeToTest, chainArg=" --read-mode irreversible")
+      relaunchNode(nodeToTest, addSwapFlags={"--read-mode": "irreversible", "--block-log-retain-blocks":blockLogRetainBlocks})
 
       # Ensure the node condition is as expected after relaunch
       confirmHeadLibAndForkDbHeadOfIrrMode(nodeToTest, headLibAndForkDbHeadBeforeSwitchMode)
@@ -267,7 +280,7 @@ try:
 
       # Kill and relaunch in speculative mode
       nodeToTest.kill(signal.SIGTERM)
-      relaunchNode(nodeToTest, addSwapFlags={"--read-mode": "head"})
+      relaunchNode(nodeToTest, addSwapFlags={"--read-mode": speculativeReadMode, "--block-log-retain-blocks":blockLogRetainBlocks})
 
       # Ensure the node condition is as expected after relaunch
       confirmHeadLibAndForkDbHeadOfSpecMode(nodeToTest, headLibAndForkDbHeadBeforeSwitchMode)
@@ -283,7 +296,7 @@ try:
          # Kill and relaunch in irreversible mode
          nodeToTest.kill(signal.SIGTERM)
          waitForBlksProducedAndLibAdvanced() # Wait for some blks to be produced and lib advance
-         relaunchNode(nodeToTest, chainArg=" --read-mode irreversible")
+         relaunchNode(nodeToTest, addSwapFlags={"--read-mode": "irreversible", "--block-log-retain-blocks":blockLogRetainBlocks})
 
          # Ensure the node condition is as expected after relaunch
          ensureHeadLibAndForkDbHeadIsAdvancing(nodeToTest)
@@ -302,7 +315,7 @@ try:
          # Kill and relaunch in irreversible mode
          nodeToTest.kill(signal.SIGTERM)
          waitForBlksProducedAndLibAdvanced() # Wait for some blks to be produced and lib advance)
-         relaunchNode(nodeToTest, addSwapFlags={"--read-mode": "head"})
+         relaunchNode(nodeToTest, addSwapFlags={"--read-mode": speculativeReadMode, "--block-log-retain-blocks":blockLogRetainBlocks})
 
          # Ensure the node condition is as expected after relaunch
          ensureHeadLibAndForkDbHeadIsAdvancing(nodeToTest)
@@ -360,7 +373,7 @@ try:
          backupBlksDir(nodeIdOfNodeToTest)
 
          # Relaunch in irreversible mode and create the snapshot
-         relaunchNode(nodeToTest, chainArg=" --read-mode irreversible")
+         relaunchNode(nodeToTest, addSwapFlags={"--read-mode": "irreversible"})
          confirmHeadLibAndForkDbHeadOfIrrMode(nodeToTest)
          nodeToTest.createSnapshot()
          nodeToTest.kill(signal.SIGTERM)
@@ -368,7 +381,7 @@ try:
          # Start from clean data dir, recover back up blocks, and then relaunch with irreversible snapshot
          removeState(nodeIdOfNodeToTest)
          recoverBackedupBlksDir(nodeIdOfNodeToTest) # this function will delete the existing blocks dir first
-         relaunchNode(nodeToTest, chainArg=" --snapshot {}".format(getLatestSnapshot(nodeIdOfNodeToTest)), addSwapFlags={"--read-mode": "head"})
+         relaunchNode(nodeToTest, chainArg=" --snapshot {}".format(getLatestSnapshot(nodeIdOfNodeToTest)), addSwapFlags={"--read-mode": speculativeReadMode})
          confirmHeadLibAndForkDbHeadOfSpecMode(nodeToTest)
          # Ensure it automatically replays "reversible blocks", i.e. head lib and fork db should be the same
          headLibAndForkDbHeadAfterRelaunch = getHeadLibAndForkDbHead(nodeToTest)
@@ -394,6 +407,50 @@ try:
       finally:
          stopProdNode()
 
+   # 10th test case: Load an irreversible snapshot into a node running without a block log
+   # Expectation: Node launches successfully
+   #              and the head and lib should be advancing after some blocks produced
+   def switchToNoBlockLogWithIrrModeSnapshot(nodeIdOfNodeToTest, nodeToTest):
+      try:
+         # Kill node and backup blocks directory of speculative mode
+         headLibAndForkDbHeadBeforeShutdown = getHeadLibAndForkDbHead(nodeToTest)
+         nodeToTest.kill(signal.SIGTERM)
+
+         # Relaunch in irreversible mode and create the snapshot
+         relaunchNode(nodeToTest, addSwapFlags={"--read-mode": "irreversible", "--block-log-retain-blocks":"0"})
+         confirmHeadLibAndForkDbHeadOfIrrMode(nodeToTest)
+         nodeToTest.createSnapshot()
+         nodeToTest.kill(signal.SIGTERM)
+
+         # Start from clean data dir and then relaunch with irreversible snapshot, no block log means that fork_db will be reset
+         removeState(nodeIdOfNodeToTest)
+         relaunchNode(nodeToTest, chainArg=" --snapshot {}".format(getLatestSnapshot(nodeIdOfNodeToTest)), addSwapFlags={"--read-mode": speculativeReadMode, "--block-log-retain-blocks":"0"})
+         confirmHeadLibAndForkDbHeadOfSpecMode(nodeToTest)
+         # Ensure it does not replay "reversible blocks", i.e. head and lib should be different
+         headLibAndForkDbHeadAfterRelaunch = getHeadLibAndForkDbHead(nodeToTest)
+         assert headLibAndForkDbHeadBeforeShutdown != headLibAndForkDbHeadAfterRelaunch, \
+            "1: Head, Lib, and Fork Db same after relaunch {} vs {}".format(headLibAndForkDbHeadBeforeShutdown, headLibAndForkDbHeadAfterRelaunch)
+
+         # Start production and wait until lib advance, ensure everything is alright
+         startProdNode()
+         ensureHeadLibAndForkDbHeadIsAdvancing(nodeToTest)
+
+         # Note the head, lib and fork db head
+         stopProdNode()
+         headLibAndForkDbHeadBeforeShutdown = getHeadLibAndForkDbHead(nodeToTest)
+         nodeToTest.kill(signal.SIGTERM)
+
+         # Relaunch the node again (using the same snapshot)
+         # The end result should be the same as before shutdown
+         removeState(nodeIdOfNodeToTest)
+         relaunchNode(nodeToTest)
+         headLibAndForkDbHeadAfterRelaunch2 = getHeadLibAndForkDbHead(nodeToTest)
+         assert headLibAndForkDbHeadAfterRelaunch == headLibAndForkDbHeadAfterRelaunch2, \
+            "2: Head, Lib, and Fork Db after relaunch is different {} vs {}".format(headLibAndForkDbHeadAfterRelaunch, headLibAndForkDbHeadAfterRelaunch2)
+      finally:
+         stopProdNode()
+
+
    # Start executing test cases here
    testSuccessful = executeTest(1, replayInIrrModeWithRevBlks)
    testSuccessful = testSuccessful and executeTest(2, replayInIrrModeWithoutRevBlks)
@@ -405,6 +462,23 @@ try:
    testSuccessful = testSuccessful and executeTest(8, replayInIrrModeWithoutRevBlksAndConnectedToProdNode)
    testSuccessful = testSuccessful and executeTest(9, switchToSpecModeWithIrrModeSnapshot)
 
+   # retest with read-mode speculative instead of head
+   speculativeReadMode="speculative"
+   testSuccessful = testSuccessful and executeTest(10, switchSpecToIrrMode)
+   testSuccessful = testSuccessful and executeTest(11, switchIrrToSpecMode)
+   testSuccessful = testSuccessful and executeTest(12, switchSpecToIrrModeWithConnectedToProdNode)
+   testSuccessful = testSuccessful and executeTest(13, switchIrrToSpecModeWithConnectedToProdNode)
+   testSuccessful = testSuccessful and executeTest(14, switchToSpecModeWithIrrModeSnapshot)
+
+   # retest with read-mode head and no block log
+   speculativeReadMode="head"
+   blockLogRetainBlocks="0"
+   testSuccessful = testSuccessful and executeTest(15, switchSpecToIrrMode)
+   testSuccessful = testSuccessful and executeTest(16, switchIrrToSpecMode)
+   testSuccessful = testSuccessful and executeTest(17, switchSpecToIrrModeWithConnectedToProdNode)
+   testSuccessful = testSuccessful and executeTest(18, switchIrrToSpecModeWithConnectedToProdNode)
+   testSuccessful = testSuccessful and executeTest(19, switchToNoBlockLogWithIrrModeSnapshot)
+
 finally:
    TestHelper.shutdown(cluster, walletMgr, testSuccessful, killEosInstances, killWallet, keepLogs, killAll, dumpErrorDetails)
    # Print test result

tests/nodeos_read_terminate_at_block_test.py

@@ -191,7 +191,7 @@ try:
         0 : "--enable-stale-production",
         1 : "--read-mode irreversible --terminate-at-block 75",
         2 : "--read-mode head --terminate-at-block 100",
-        3 : "--read-mode head --terminate-at-block 125"
+        3 : "--read-mode speculative --terminate-at-block 125"
     }
 
     # Kill any existing instances and launch cluster

tests/nodeos_snapshot_forked_test.py

@@ -171,7 +171,7 @@ try:
         errorExit(f"Failure - (non-production) node {nonProdNode.nodeNum} should have restarted")
 
     Print("Wait for LIB to move, which indicates prodC has forked out the branch")
-    assert prodC.waitForLibToAdvance(), \
+    assert prodC.waitForLibToAdvance(60), \
         "ERROR: Network did not reach consensus after bridge node was restarted."
  
     for prodNode in prodNodes:

tests/performance_tests/performance_test_basic.py

@@ -497,15 +497,17 @@ class PerformanceTestBasic:
             traceback.print_exc()
 
         finally:
+            # Despite keepLogs being hardcoded to False, logs will still appear on test failure in TestLogs
+            # due to testSuccessful being False
             TestHelper.shutdown(
-                self.cluster,
-                self.walletMgr,
-                testSuccessful,
-                self.testHelperConfig._killEosInstances,
-                self.testHelperConfig._killWallet,
-                self.testHelperConfig.keepLogs,
-                self.testHelperConfig.killAll,
-                self.testHelperConfig.dumpErrorDetails
+                cluster=self.cluster,
+                walletMgr=self.walletMgr,
+                testSuccessful=testSuccessful,
+                killEosInstances=self.testHelperConfig._killEosInstances,
+                killWallet=self.testHelperConfig._killWallet,
+                keepLogs=False,
+                cleanRun=self.testHelperConfig.killAll,
+                dumpErrorDetails=self.testHelperConfig.dumpErrorDetails
                 )
 
             if self.ptbConfig.delPerfLogs:

tests/plugin_http_api_test.py

@@ -40,8 +40,8 @@ class PluginHttpTest(unittest.TestCase):
         self.keosd.killall(True)
         WalletMgr.cleanup()
         Node.killAllNodeos()
-        if os.path.exists(self.data_dir):
-            shutil.rmtree(self.data_dir)
+        if os.path.exists(Utils.DataPath):
+            shutil.rmtree(Utils.DataPath)
         if os.path.exists(self.config_dir):
             shutil.rmtree(self.config_dir)
         time.sleep(self.sleep_s)
@@ -56,7 +56,7 @@ class PluginHttpTest(unittest.TestCase):
                         "http_plugin", "db_size_api_plugin", "prometheus_plugin"]
         nodeos_plugins = "--plugin eosio::" +  " --plugin eosio::".join(plugin_names)
         nodeos_flags = (" --data-dir=%s --config-dir=%s --trace-dir=%s --trace-no-abis --access-control-allow-origin=%s "
-                        "--contracts-console --http-validate-host=%s --verbose-http-errors --abi-serializer-max-time-ms 30000 --http-max-response-time-ms 30000 "
+                        "--contracts-console --http-validate-host=%s --verbose-http-errors --max-transaction-time -1 --abi-serializer-max-time-ms 30000 --http-max-response-time-ms 30000 "
                         "--p2p-peer-address localhost:9011 --resource-monitor-not-shutdown-on-threshold-exceeded ") % (self.data_dir, self.config_dir, self.data_dir, "\'*\'", "false")
         start_nodeos_cmd = ("%s -e -p eosio %s %s ") % (Utils.EosServerPath, nodeos_plugins, nodeos_flags)
         self.nodeos.launchCmd(start_nodeos_cmd, self.node_id)
@@ -600,6 +600,35 @@ class PluginHttpTest(unittest.TestCase):
         ret_json = self.nodeos.processUrllibRequest(resource, command, self.http_post_invalid_param)
         self.assertEqual(ret_json["code"], 400)
         self.assertEqual(ret_json["error"]["code"], 3200006)
+        # get_transaction_id with missing actions
+        payload_no_actions = {"expiration":"2020-08-01T07:15:49",
+                "ref_block_num": 34881,
+                "ref_block_prefix":2972818865,
+                "max_net_usage_words":0,
+                "max_cpu_usage_ms":0,
+                "delay_sec":0,
+                "context_free_actions":[],
+                "transaction_extensions": [],
+                "signatures": ["SIG_K1_KeqfqiZu1GwUxQb7jzK9Fdks6HFaVBQ9AJtCZZj56eG9qGgvVMVtx8EerBdnzrhFoX437sgwtojf2gfz6S516Ty7c22oEp"],
+                "context_free_data": []}
+        ret_json = self.nodeos.processUrllibRequest(resource, command, payload_no_actions)
+        self.assertEqual(ret_json["code"], 400)
+        self.assertEqual(ret_json["error"]["code"], 3200006)
+        # get_transaction_id with invalid actions
+        payload_invalid_actions = {"expiration":"2020-08-01T07:15:49",
+                "ref_block_num": 34881,
+                "ref_block_prefix":2972818865,
+                "max_net_usage_words":0,
+                "max_cpu_usage_ms":0,
+                "delay_sec":0,
+                "context_free_actions":[],
+                "actions":  "hello_world",
+                "transaction_extensions": [],
+                "signatures": ["SIG_K1_KeqfqiZu1GwUxQb7jzK9Fdks6HFaVBQ9AJtCZZj56eG9qGgvVMVtx8EerBdnzrhFoX437sgwtojf2gfz6S516Ty7c22oEp"],
+                "context_free_data": []}
+        ret_json = self.nodeos.processUrllibRequest(resource, command, payload_invalid_actions)
+        self.assertEqual(ret_json["code"], 400)
+        self.assertEqual(ret_json["error"]["code"], 3200006)
         # get_transaction_id with valid parameter
         payload = {"expiration":"2020-08-01T07:15:49",
                    "ref_block_num": 34881,
@@ -614,6 +643,23 @@ class PluginHttpTest(unittest.TestCase):
                    "context_free_data": []}
         ret_str = self.nodeos.processUrllibRequest(resource, command, payload, returnType=ReturnType.raw).decode('ascii')
         self.assertEqual(ret_str, "\"0be762a6406bab15530e87f21e02d1c58e77944ee55779a76f4112e3b65cac48\"")
+        # transaction that has hex_data
+        payload_hex = {"expiration":"2020-08-01T07:15:49",
+                "ref_block_num": 34881,
+                "ref_block_prefix":2972818865,
+                "max_net_usage_words":0,
+                "max_cpu_usage_ms":0,
+                "delay_sec":0,
+                "context_free_actions":[],
+                "actions":[{"account":"eosio.token","name": "transfer","authorization": [{"actor": "han","permission": "active"}],
+                "data": "{\"entry\":774831,\"miner\":\"eosminer1111\",\"nonce\":139429}\"}",
+                "hex_data": "000000000000a6690000000000ea305501000000000000000453595300000000016d"}],
+                "transaction_extensions": [],
+                "signatures": ["SIG_K1_KeqfqiZu1GwUxQb7jzK9Fdks6HFaVBQ9AJtCZZj56eG9qGgvVMVtx8EerBdnzrhFoX437sgwtojf2gfz6S516Ty7c22oEp"],
+                "context_free_data": []}
+        ret_str = self.nodeos.processUrllibRequest(resource, command, payload_hex, returnType=ReturnType.raw).decode('ascii')
+        self.assertEqual(ret_str, "\"0be762a6406bab15530e87f21e02d1c58e77944ee55779a76f4112e3b65cac48\"")
+
 
         # push_block with empty parameter
         command = "push_block"
@@ -1341,8 +1387,8 @@ class PluginHttpTest(unittest.TestCase):
     def test_prometheusApi(self) :
         resource = "prometheus"
         command = "metrics"
-
-        ret_text = self.nodeos.processUrllibRequest(resource, command, returnType = ReturnType.raw ).decode()
+        endpointPrometheus = f'http://{self.nodeos.host}:9101'
+        ret_text = self.nodeos.processUrllibRequest(resource, command, returnType = ReturnType.raw, method="GET", endpoint=endpointPrometheus).decode()
         # filter out all empty lines or lines starting with '#'
         data_lines = filter(lambda line: len(line) > 0 and line[0]!='#', ret_text.split('\n'))
         # converting each line into a key value pair and then construct a dictionay out of all the pairs
@@ -1352,6 +1398,8 @@ class PluginHttpTest(unittest.TestCase):
         self.assertTrue(int(metrics["blocks_produced"]) > 1)
         self.assertTrue(int(metrics["last_irreversible"]) > 1)
 
+        ret = self.nodeos.processUrllibRequest(resource, "m", returnType = ReturnType.raw, method="GET", silentErrors= True, endpoint=endpointPrometheus)
+        self.assertTrue(ret == 404)
 
     def test_multipleRequests(self):
         """Test keep-alive ability of HTTP plugin.  Handle multiple requests in a single session"""

tests/read_only_trx_test.py

@@ -21,6 +21,7 @@ errorExit=Utils.errorExit
 
 appArgs=AppArgs()
 appArgs.add(flag="--read-only-threads", type=int, help="number of read-only threads", default=0)
+appArgs.add(flag="--num-test-runs", type=int, help="number of times to run the tests", default=1)
 appArgs.add_bool(flag="--eos-vm-oc-enable", help="enable eos-vm-oc")
 appArgs.add(flag="--wasm-runtime", type=str, help="if set to eos-vm-oc, must compile with EOSIO_EOS_VM_OC_DEVELOPER", default="eos-vm-jit")
 
@@ -44,6 +45,7 @@ dontKill=args.leave_running
 dumpErrorDetails=args.dump_error_details
 killAll=args.clean_run
 keepLogs=args.keep_logs
+numTestRuns=args.num_test_runs
 
 killWallet=not dontKill
 killEosInstances=not dontKill
@@ -91,9 +93,8 @@ try:
     specificExtraNodeosArgs={}
     # producer nodes will be mapped to 0 through pnodes-1, so the number pnodes is the no-producing API node
     specificExtraNodeosArgs[pnodes]=" --plugin eosio::net_api_plugin"
-    if args.read_only_threads > 0:
-        specificExtraNodeosArgs[pnodes]+=" --read-only-threads "
-        specificExtraNodeosArgs[pnodes]+=str(args.read_only_threads)
+    specificExtraNodeosArgs[pnodes]+=" --read-only-threads "
+    specificExtraNodeosArgs[pnodes]+=str(args.read_only_threads)
     if args.eos_vm_oc_enable:
         specificExtraNodeosArgs[pnodes]+=" --eos-vm-oc-enable"
     if args.wasm_runtime:
@@ -259,12 +260,12 @@ try:
         runReadOnlyTrxAndRpcInParallel("chain", "get_raw_code_and_abi", "account_name", expectedValue=testAccountName, payload = {"account_name":testAccountName})
         runReadOnlyTrxAndRpcInParallel("chain", "get_raw_abi", "account_name", expectedValue=testAccountName, payload = {"account_name":testAccountName})
         runReadOnlyTrxAndRpcInParallel("chain", "get_producers", "rows", payload = {"json":"true","lower_bound":""})
-        runReadOnlyTrxAndRpcInParallel("chain", "get_table_rows", code=500, payload = {"json":"true","table":"noauth"})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_table_rows", "rows", payload = {"json":"true","code":"eosio","scope":"eosio","table":"global"})
         runReadOnlyTrxAndRpcInParallel("chain", "get_table_by_scope", fieldIn="rows", payload = {"json":"true","table":"noauth"})
         runReadOnlyTrxAndRpcInParallel("chain", "get_currency_balance", code=200, payload = {"code":"eosio.token", "account":testAccountName})
         runReadOnlyTrxAndRpcInParallel("chain", "get_currency_stats", fieldIn="SYS", payload = {"code":"eosio.token", "symbol":"SYS"})
         runReadOnlyTrxAndRpcInParallel("chain", "get_required_keys", code=400)
-        runReadOnlyTrxAndRpcInParallel("chain", "get_transaction_id", code=200, payload = {"ref_block_num":"1"})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_transaction_id", code=400, payload = {"ref_block_num":"1"})
         runReadOnlyTrxAndRpcInParallel("chain", "push_block", code=202, payload = {"block":"signed_block"})
         runReadOnlyTrxAndRpcInParallel("chain", "get_producer_schedule", "active")
         runReadOnlyTrxAndRpcInParallel("chain", "get_scheduled_transactions", "transactions", payload = {"json":"true","lower_bound":""})
@@ -276,13 +277,26 @@ try:
         runReadOnlyTrxAndRpcInParallel("net", "connect", code=201, payload = "localhost")
         runReadOnlyTrxAndRpcInParallel("net", "disconnect", code=201, payload = "localhost")
 
+    def runEverythingParallel():
+        threadList = []
+        threadList.append(threading.Thread(target = multiReadOnlyTests))
+        threadList.append(threading.Thread(target = chainApiTests))
+        threadList.append(threading.Thread(target = netApiTests))
+        threadList.append(threading.Thread(target = mixedOpsTest))
+        for thr in threadList:
+            thr.start()
+        for thr in threadList:
+            thr.join()
+
     basicTests()
 
     if args.read_only_threads > 0: # Save test time. No need to run other tests if multi-threaded is not enabled
-        multiReadOnlyTests()
-        chainApiTests()
-        netApiTests()
-        mixedOpsTest()
+        for i in range(numTestRuns):
+            multiReadOnlyTests()
+            chainApiTests()
+            netApiTests()
+            mixedOpsTest()
+            runEverythingParallel()
 
     testSuccessful = True
 finally:

tests/ship_client.cpp

@@ -125,6 +125,8 @@ int main(int argc, char* argv[]) {
             eosio::check(result_doucment[1]["head"].IsObject(),                                 "'head' is not an object");
             eosio::check(result_doucment[1]["head"].HasMember("block_num"),                     "'head' does not contain 'block_num'");
             eosio::check(result_doucment[1]["head"]["block_num"].IsUint(),                      "'head.block_num' isn't a number");
+            eosio::check(result_doucment[1]["head"].HasMember("block_id"),                      "'head' does not contain 'block_id'");
+            eosio::check(result_doucment[1]["head"]["block_id"].IsString(),                     "'head.block_id' isn't a string");
 
             uint32_t this_block_num = result_doucment[1]["head"]["block_num"].GetUint();
 

tests/ship_streamer.cpp

@@ -10,6 +10,8 @@
 #include <boost/beast.hpp>
 #include <boost/program_options.hpp>
 
+#include <map>
+#include <set>
 #include <iostream>
 #include <string>
 
@@ -111,45 +113,81 @@ int main(int argc, char* argv[]) {
       stream.write(boost::asio::buffer(request_type.json_to_bin(request_sb.GetString(), [](){})));
       stream.read_message_max(0);
 
+      // Each block_num can have multiple block_ids since forks are possible
+      //       block_num,         block_id
+      std::map<uint32_t, std::set<std::string>> block_ids;
       bool is_first = true;
       for(;;) {
          boost::beast::flat_buffer buffer;
          stream.read(buffer);
 
          eosio::input_stream is((const char*)buffer.data().data(), buffer.data().size());
-         rapidjson::Document result_doucment;
-         result_doucment.Parse(result_type.bin_to_json(is).c_str());
+         rapidjson::Document result_document;
+         result_document.Parse(result_type.bin_to_json(is).c_str());
 
-         eosio::check(!result_doucment.HasParseError(),                                      "Failed to parse result JSON from abieos");
-         eosio::check(result_doucment.IsArray(),                                             "result should have been an array (variant) but it's not");
-         eosio::check(result_doucment.Size() == 2,                                           "result was an array but did not contain 2 items like a variant should");
-         eosio::check(std::string(result_doucment[0].GetString()) == "get_blocks_result_v0", "result type doesn't look like get_blocks_result_v0");
-         eosio::check(result_doucment[1].IsObject(),                                         "second item in result array is not an object");
-         eosio::check(result_doucment[1].HasMember("head"),                                  "cannot find 'head' in result");
-         eosio::check(result_doucment[1]["head"].IsObject(),                                 "'head' is not an object");
-         eosio::check(result_doucment[1]["head"].HasMember("block_num"),                     "'head' does not contain 'block_num'");
-         eosio::check(result_doucment[1]["head"]["block_num"].IsUint(),                      "'head.block_num' isn't a number");
-
-         uint32_t this_block_num = 0;
-         if( result_doucment[1].HasMember("this_block") && result_doucment[1]["this_block"].IsObject() ) {
-            if( result_doucment[1]["this_block"].HasMember("block_num") && result_doucment[1]["this_block"]["block_num"].IsUint() ) {
-               this_block_num = result_doucment[1]["this_block"]["block_num"].GetUint();
-            }
-         }
+         eosio::check(!result_document.HasParseError(),                                      "Failed to parse result JSON from abieos");
+         eosio::check(result_document.IsArray(),                                             "result should have been an array (variant) but it's not");
+         eosio::check(result_document.Size() == 2,                                           "result was an array but did not contain 2 items like a variant should");
+         eosio::check(std::string(result_document[0].GetString()) == "get_blocks_result_v0", "result type doesn't look like get_blocks_result_v0");
+         eosio::check(result_document[1].IsObject(),                                         "second item in result array is not an object");
+         eosio::check(result_document[1].HasMember("head"),                                  "cannot find 'head' in result");
+         eosio::check(result_document[1]["head"].IsObject(),                                 "'head' is not an object");
+         eosio::check(result_document[1]["head"].HasMember("block_num"),                     "'head' does not contain 'block_num'");
+         eosio::check(result_document[1]["head"]["block_num"].IsUint(),                      "'head.block_num' isn't a number");
+         eosio::check(result_document[1]["head"].HasMember("block_id"),                      "'head' does not contain 'block_id'");
+         eosio::check(result_document[1]["head"]["block_id"].IsString(),                     "'head.block_id' isn't a string");
 
+         // stream what was received
          if(is_first) {
-            std::cout << "[" << std::endl;
-            is_first = false;
+           std::cout << "[" << std::endl;
+           is_first = false;
          } else {
-            std::cout << "," << std::endl;
+           std::cout << "," << std::endl;
          }
          std::cout << "{ \"get_blocks_result_v0\":" << std::endl;
 
          rapidjson::StringBuffer result_sb;
          rapidjson::PrettyWriter<rapidjson::StringBuffer> result_writer(result_sb);
-         result_doucment[1].Accept(result_writer);
+         result_document[1].Accept(result_writer);
          std::cout << result_sb.GetString() << std::endl << "}" << std::endl;
 
+         // validate after streaming, so that invalid entry is included in the output
+         uint32_t this_block_num = 0;
+         if( result_document[1].HasMember("this_block") && result_document[1]["this_block"].IsObject() ) {
+            if( result_document[1]["this_block"].HasMember("block_num") && result_document[1]["this_block"]["block_num"].IsUint() ) {
+               this_block_num = result_document[1]["this_block"]["block_num"].GetUint();
+            }
+            std::string this_block_id;
+            if( result_document[1]["this_block"].HasMember("block_id") && result_document[1]["this_block"]["block_id"].IsString() ) {
+               this_block_id = result_document[1]["this_block"]["block_id"].GetString();
+            }
+            std::string prev_block_id;
+            if( result_document[1]["prev_block"].HasMember("block_id") && result_document[1]["prev_block"]["block_id"].IsString() ) {
+               prev_block_id = result_document[1]["prev_block"]["block_id"].GetString();
+            }
+            if( !irreversible_only && !this_block_id.empty() && !prev_block_id.empty() ) {
+               // verify forks were sent
+               if (block_ids.count(this_block_num-1)) {
+                  if (block_ids[this_block_num-1].count(prev_block_id) == 0) {
+                     std::cerr << "Received block: << " << this_block_num << " that does not link to previous: ";
+                     std::copy(block_ids[this_block_num-1].begin(), block_ids[this_block_num-1].end(), std::ostream_iterator<std::string>(std::cerr, " "));
+                     std::cerr << std::endl;
+                     return 1;
+                  }
+               }
+               block_ids[this_block_num].insert(this_block_id);
+
+               if( result_document[1]["last_irreversible"].HasMember("block_num") && result_document[1]["last_irreversible"]["block_num"].IsUint() ) {
+                  uint32_t lib_num = result_document[1]["last_irreversible"]["block_num"].GetUint();
+                  auto i = block_ids.lower_bound(lib_num);
+                  if (i != block_ids.end()) {
+                     block_ids.erase(block_ids.begin(), i);
+                  }
+               }
+            }
+
+         }
+
          if( this_block_num == end_block_num ) break;
       }
 

tests/ship_streamer_test.py

@@ -7,34 +7,33 @@ import shutil
 import signal
 import sys
 
-from TestHarness import Cluster, TestHelper, Utils, WalletMgr
+from TestHarness import Cluster, TestHelper, Utils, WalletMgr, CORE_SYMBOL
 from TestHarness.TestHelper import AppArgs
 
 ###############################################################
 # ship_streamer_test
 # 
-# This test sets up <-p> producing node(s) and <-n - -p>
-#   non-producing node(s). One of the non-producing nodes
-#   is configured with the state_history_plugin.  An instance
-#   of node will be started with ship_streamer to exercise
-#   the SHiP API.
+# This test sets up 2 producing nodes and one "bridge" node using test_control_api_plugin.
+#   One producing node has 3 of the elected producers and the other has 1 of the elected producers.
+#   All the producers are named in alphabetical order, so that the 3 producers, in the one production node, are
+#       scheduled first, followed by the 1 producer in the other producer node. Each producing node is only connected
+#       to the other producing node via the "bridge" node.
+#   The bridge node has the test_control_api_plugin, that the test uses to kill
+#       the "bridge" node to generate a fork.
+#   ship_streamer is used to connect to the state_history_plugin and verify that blocks receive link to previous
+#   blocks. If the blocks do not link then ship_streamer will exit with an error causing this test to generate an
+#   error. The fork generated by nodeos should be sent to the ship_streamer so it is able to correctly observe the
+#   fork.
 #
 ###############################################################
 
 Print=Utils.Print
 
 appArgs = AppArgs()
-extraArgs = appArgs.add(flag="--num-blocks", type=int, help="How many blocsk to stream from ship_streamer", default=20)
 extraArgs = appArgs.add(flag="--num-clients", type=int, help="How many ship_streamers should be started", default=1)
-args = TestHelper.parse_args({"-p", "-n","--dump-error-details","--keep-logs","-v","--leave-running","--clean-run","--unshared"}, applicationSpecificArgs=appArgs)
+args = TestHelper.parse_args({"--dump-error-details","--keep-logs","-v","--leave-running","--clean-run","--unshared"}, applicationSpecificArgs=appArgs)
 
 Utils.Debug=args.v
-totalProducerNodes=args.p
-totalNodes=args.n
-if totalNodes<=totalProducerNodes:
-    totalNodes=totalProducerNodes+1
-totalNonProducerNodes=totalNodes-totalProducerNodes
-totalProducers=totalProducerNodes
 cluster=Cluster(walletd=True,unshared=args.unshared)
 dumpErrorDetails=args.dump_error_details
 keepLogs=args.keep_logs
@@ -42,6 +41,12 @@ dontKill=args.leave_running
 killAll=args.clean_run
 walletPort=TestHelper.DEFAULT_WALLET_PORT
 
+totalProducerNodes=2
+totalNonProducerNodes=1
+totalNodes=totalProducerNodes+totalNonProducerNodes
+maxActiveProducers=21
+totalProducers=maxActiveProducers
+
 walletMgr=WalletMgr(True, port=walletPort)
 testSuccessful=False
 killEosInstances=not dontKill
@@ -57,12 +62,20 @@ try:
     cluster.killall(allInstances=killAll)
     cluster.cleanup()
     Print("Stand up cluster")
-    specificExtraNodeosArgs={}
-    # non-producing nodes are at the end of the cluster's nodes, so reserving the last one for state_history_plugin
-    shipNodeNum = totalNodes - 1
-    specificExtraNodeosArgs[shipNodeNum]="--plugin eosio::state_history_plugin --disable-replay-opts --trace-history --sync-fetch-span 200 --plugin eosio::net_api_plugin "
 
-    if cluster.launch(pnodes=totalProducerNodes,
+
+    # ***   setup topogrophy   ***
+
+    # "bridge" shape connects defprocera through defproducerc (3 in node0) to each other and defproduceru (1 in node1)
+    # and the only connection between those 2 groups is through the bridge node
+
+    shipNodeNum = 1
+    specificExtraNodeosArgs={}
+    specificExtraNodeosArgs[shipNodeNum]="--plugin eosio::state_history_plugin --disable-replay-opts --trace-history --chain-state-history --plugin eosio::net_api_plugin "
+    # producer nodes will be mapped to 0 through totalProducerNodes-1, so the number totalProducerNodes will be the non-producing node
+    specificExtraNodeosArgs[totalProducerNodes]="--plugin eosio::test_control_api_plugin  "
+
+    if cluster.launch(topo="bridge", pnodes=totalProducerNodes,
                       totalNodes=totalNodes, totalProducers=totalProducers,
                       specificExtraNodeosArgs=specificExtraNodeosArgs) is False:
         Utils.cmdError("launcher")
@@ -70,19 +83,81 @@ try:
 
     # ***   identify each node (producers and non-producing node)   ***
 
-    shipNode = cluster.getNode(shipNodeNum)
-    prodNode = cluster.getNode(0)
-
     #verify nodes are in sync and advancing
     cluster.waitOnClusterSync(blockAdvancing=5)
     Print("Cluster in Sync")
 
+    prodNode = cluster.getNode(0)
+    prodNode0 = prodNode
+    prodNode1 = cluster.getNode(1)
+    nonProdNode = cluster.getNode(2)
+    shipNode = cluster.getNode(shipNodeNum)
+
+
+    accounts=cluster.createAccountKeys(6)
+    if accounts is None:
+        Utils.errorExit("FAILURE - create keys")
+
+    accounts[0].name="testeraaaaaa"
+    accounts[1].name="tester111111" # needed for voting
+    accounts[2].name="tester222222" # needed for voting
+    accounts[3].name="tester333333" # needed for voting
+    accounts[4].name="tester444444" # needed for voting
+    accounts[5].name="tester555555" # needed for voting
+
+    testWalletName="test"
+
+    Print(f"Creating wallet {testWalletName}.")
+    testWallet=walletMgr.create(testWalletName, [cluster.eosioAccount,accounts[0],accounts[1],accounts[2],accounts[3],accounts[4],accounts[5]])
+
+    for _, account in cluster.defProducerAccounts.items():
+        walletMgr.importKey(account, testWallet, ignoreDupKeyWarning=True)
+
+    for i in range(0, totalNodes):
+        node=cluster.getNode(i)
+        node.producers=Cluster.parseProducers(i)
+        for prod in node.producers:
+            prodName = cluster.defProducerAccounts[prod].name
+            if prodName == "defproducera" or prodName == "defproducerb" or prodName == "defproducerc" or prodName == "defproduceru":
+                Print(f"Register producer {prodName}")
+                trans=node.regproducer(cluster.defProducerAccounts[prod], "http://mysite.com", 0, waitForTransBlock=False, exitOnError=True)
+
+    # create accounts via eosio as otherwise a bid is needed
+    for account in accounts:
+        Print(f"Create new account {account.name} via {cluster.eosioAccount.name} with private key: {account.activePrivateKey}")
+        trans=nonProdNode.createInitializeAccount(account, cluster.eosioAccount, stakedDeposit=0, waitForTransBlock=True, stakeNet=10000, stakeCPU=10000, buyRAM=10000000, exitOnError=True)
+        transferAmount="100000000.0000 {0}".format(CORE_SYMBOL)
+        Print(f"Transfer funds {transferAmount} from account {cluster.eosioAccount.name} to {account.name}")
+        nonProdNode.transferFunds(cluster.eosioAccount, account, transferAmount, "test transfer", waitForTransBlock=True)
+        trans=nonProdNode.delegatebw(account, 20000000.0000, 20000000.0000, waitForTransBlock=False, exitOnError=True)
+
+    # ***   vote using accounts   ***
+
+    cluster.waitOnClusterSync(blockAdvancing=3)
     start_block_num = shipNode.getBlockNum()
-    end_block_num = start_block_num + args.num_blocks
+
+    # vote a,b,c (node0)  u (node1)
+    voteProducers=[]
+    voteProducers.append("defproducera")
+    voteProducers.append("defproducerb")
+    voteProducers.append("defproducerc")
+    voteProducers.append("defproduceru")
+    for account in accounts:
+        Print(f"Account {account.name} vote for producers={voteProducers}")
+        trans=prodNode.vote(account, voteProducers, exitOnError=True, waitForTransBlock=False)
+
+    #verify nodes are in sync and advancing
+    cluster.waitOnClusterSync(blockAdvancing=3)
+    Print("Shutdown unneeded bios node")
+    cluster.biosNode.kill(signal.SIGTERM)
+    prodNode0.waitForProducer("defproducerc")
+
+    block_range = 350
+    end_block_num = start_block_num + block_range
 
     shipClient = "tests/ship_streamer"
-    cmd = "%s --start-block-num %d --end-block-num %d --fetch-block --fetch-traces --fetch-deltas" % (shipClient, start_block_num, end_block_num)
-    if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
+    cmd = f"{shipClient} --start-block-num {start_block_num} --end-block-num {end_block_num} --fetch-block --fetch-traces --fetch-deltas"
+    if Utils.Debug: Utils.Print(f"cmd: {cmd}")
     clients = []
     files = []
     shipTempDir = os.path.join(Utils.DataDir, "ship")
@@ -92,28 +167,61 @@ try:
     starts = []
     for i in range(0, args.num_clients):
         start = time.perf_counter()
-        outFile = open("%s%d.out" % (shipClientFilePrefix, i), "w")
-        errFile = open("%s%d.err" % (shipClientFilePrefix, i), "w")
-        Print("Start client %d" % (i))
+        outFile = open(f"{shipClientFilePrefix}{i}.out", "w")
+        errFile = open(f"{shipClientFilePrefix}{i}.err", "w")
+        Print(f"Start client {i}")
         popen=Utils.delayedCheckOutput(cmd, stdout=outFile, stderr=errFile)
         starts.append(time.perf_counter())
         clients.append((popen, cmd))
         files.append((outFile, errFile))
-        Print("Client %d started, Ship node head is: %s" % (i, shipNode.getBlockNum()))
+        Print(f"Client {i} started, Ship node head is: {shipNode.getBlockNum()}")
 
-    Print("Stopping all %d clients" % (args.num_clients))
+    # Generate a fork
+    forkAtProducer="defproducera"
+    prodNode1Prod="defproduceru"
+    preKillBlockNum=nonProdNode.getBlockNum()
+    preKillBlockProducer=nonProdNode.getBlockProducerByNum(preKillBlockNum)
+    nonProdNode.killNodeOnProducer(producer=forkAtProducer, whereInSequence=1)
+    Print(f"Current block producer {preKillBlockProducer} fork will be at producer {forkAtProducer}")
+    prodNode0.waitForProducer(forkAtProducer)
+    prodNode1.waitForProducer(prodNode1Prod)
+    if nonProdNode.verifyAlive(): # if on defproducera, need to wait again
+        prodNode0.waitForProducer(forkAtProducer)
+        prodNode1.waitForProducer(prodNode1Prod)
+    if nonProdNode.verifyAlive():
+        Utils.errorExit("Bridge did not shutdown");
+    Print("Fork started")
 
+    prodNode0.waitForProducer("defproducerb") # wait for fork to progress a bit
+
+    Print("Restore fork")
+    Print("Relaunching the non-producing bridge node to connect the producing nodes again")
+    if nonProdNode.verifyAlive():
+        Utils.errorExit("Bridge is already running");
+    if not nonProdNode.relaunch():
+        Utils.errorExit(f"Failure - (non-production) node {nonProdNode.nodeNum} should have restarted")
+
+    nonProdNode.waitForProducer(forkAtProducer)
+    nonProdNode.waitForProducer(prodNode1Prod)
+    afterForkBlockNum = nonProdNode.getBlockNum()
+    if int(afterForkBlockNum) > int(end_block_num):
+        Utils.errorExit(f"Did not stream long enough {end_block_num} to cover the fork {afterForkBlockNum}, increase block_range {block_range}")
+
+    Print(f"Stopping all {args.num_clients} clients")
     for index, (popen, _), (out, err), start in zip(range(len(clients)), clients, files, starts):
         popen.wait()
-        Print("Stopped client %d.  Ran for %.3f seconds." % (index, time.perf_counter() - start))
+        Print(f"Stopped client {index}.  Ran for {time.perf_counter() - start:.3f} seconds.")
         out.close()
         err.close()
-        outFile = open("%s%d.out" % (shipClientFilePrefix, index), "r")
+        outFile = open(f"{shipClientFilePrefix}{index}.out", "r")
         data = json.load(outFile)
         block_num = start_block_num
         for i in data:
-            print(i)
-            assert block_num == i['get_blocks_result_v0']['this_block']['block_num'], f"{block_num} != {i['get_blocks_result_v0']['this_block']['block_num']}"
+            # fork can cause block numbers to be repeated
+            this_block_num = i['get_blocks_result_v0']['this_block']['block_num']
+            if this_block_num < block_num:
+                block_num = this_block_num
+            assert block_num == this_block_num, f"{block_num} != {this_block_num}"
             assert isinstance(i['get_blocks_result_v0']['block'], str) # verify block in result
             block_num += 1
         assert block_num-1 == end_block_num, f"{block_num-1} != {end_block_num}"

tests/ship_test.py

@@ -132,7 +132,7 @@ try:
             try:
                 statuses = json.loads(" ".join(lines))
             except json.decoder.JSONDecodeError as er:
-                Utils.errorExit("javascript client output was malformed in %s. Exception: %s" % (shipClientErrorFile, er))
+                Utils.errorExit("ship_client output was malformed in %s. Exception: %s" % (shipClientErrorFile, er))
 
             for status in statuses:
                 statusDesc = status["status"]
@@ -143,7 +143,7 @@ try:
                     maxFirstBN = max(maxFirstBN, firstBlockNum)
                     minLastBN = min(minLastBN, lastBlockNum)
                 if statusDesc == "error":
-                    Utils.errorExit("javascript client reporting error see: %s." % (shipClientErrorFile))
+                    Utils.errorExit("ship_client reporting error see: %s." % (shipClientErrorFile))
 
         assert done, Print("ERROR: Did not find a \"done\" status for client %d" % (i))
 

tests/split_blocklog_replay_test.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+import os
+import shutil
+import time
+import signal
+from TestHarness import Node, TestHelper, Utils
+
+node_id = 1
+nodeos = Node(TestHelper.LOCAL_HOST, TestHelper.DEFAULT_PORT, node_id)
+data_dir = Utils.getNodeDataDir(node_id)
+config_dir = Utils.getNodeConfigDir(node_id)
+if os.path.exists(data_dir):
+    shutil.rmtree(data_dir)
+os.makedirs(data_dir)
+if not os.path.exists(config_dir):
+    os.makedirs(config_dir)
+
+try:
+    start_nodeos_cmd = f"{Utils.EosServerPath} -e -p eosio --data-dir={data_dir} --config-dir={config_dir} --blocks-log-stride 10" \
+                        " --plugin=eosio::http_plugin --plugin=eosio::chain_api_plugin --http-server-address=localhost:8888"
+
+    nodeos.launchCmd(start_nodeos_cmd, node_id)
+    time.sleep(2)
+    nodeos.waitForBlock(30)
+    nodeos.kill(signal.SIGTERM)
+
+    nodeos.relaunch(chainArg="--replay-blockchain")
+
+    time.sleep(2)
+    assert nodeos.waitForBlock(31)
+finally:
+    # clean up
+    Node.killAllNodeos()

tests/trace_plugin_test.py

@@ -8,6 +8,8 @@ import os
 
 from TestHarness import Cluster, Node, TestHelper, Utils, WalletMgr, CORE_SYMBOL
 
+testSuccessful = True
+
 class TraceApiPluginTest(unittest.TestCase):
     sleep_s = 1
     cluster=Cluster(walletd=True, defproduceraPrvtKey=None)
@@ -101,6 +103,8 @@ class TraceApiPluginTest(unittest.TestCase):
                     self.assertIn('memo', prms)
                 break
         self.assertTrue(isTrxInBlockFromTraceApi)
+        global testSuccessful
+        testSuccessful = True
 
     @classmethod
     def setUpClass(self):
@@ -109,7 +113,7 @@ class TraceApiPluginTest(unittest.TestCase):
 
     @classmethod
     def tearDownClass(self):
-        self.cleanEnv(self, shouldCleanup=False)   # not cleanup to save log in case for further investigation
+        self.cleanEnv(self, shouldCleanup=testSuccessful)
 
 if __name__ == "__main__":
     unittest.main()

tests/trx_finality_status_forked_test.py

@@ -184,7 +184,7 @@ try:
         errorExit(f"Failure - (non-production) node {nonProdNode.nodeNum} should have restarted")
 
     Print("Wait for LIB to move, which indicates prodC has forked out the branch")
-    assert prodC.waitForLibToAdvance(), \
+    assert prodC.waitForLibToAdvance(60), \
         "ERROR: Network did not reach consensus after bridge node was restarted."
 
     retStatus = prodC.getTransactionStatus(transId)

unittests/block_log_extract.cpp

@@ -58,7 +58,9 @@ BOOST_FIXTURE_TEST_CASE(extract_from_middle, block_log_extract_fixture) try {
    block_log new_log(output_dir.path());
 
    auto id = gs.compute_chain_id();
-   BOOST_REQUIRE_EQUAL(new_log.extract_chain_id(output_dir.path()), id);
+   auto extracted_id = new_log.extract_chain_id(output_dir.path());
+   BOOST_REQUIRE(extracted_id.has_value());
+   BOOST_REQUIRE_EQUAL(*extracted_id, id);
    BOOST_REQUIRE_EQUAL(new_log.first_block_num(), 3);
    BOOST_REQUIRE_EQUAL(new_log.head()->block_num(), 7);
 
@@ -74,7 +76,9 @@ BOOST_FIXTURE_TEST_CASE(extract_from_start, block_log_extract_fixture) try {
    block_log new_log(output_dir.path());
 
    auto id = gs.compute_chain_id();
-   BOOST_REQUIRE_EQUAL(new_log.extract_chain_id(output_dir.path()), id);
+   auto extracted_id = new_log.extract_chain_id(output_dir.path());
+   BOOST_REQUIRE(extracted_id.has_value());
+   BOOST_REQUIRE_EQUAL(*extracted_id, id);
    BOOST_REQUIRE_EQUAL(new_log.first_block_num(), 1);
    BOOST_REQUIRE_EQUAL(new_log.head()->block_num(), 7);
 
@@ -93,7 +97,9 @@ BOOST_FIXTURE_TEST_CASE(reextract_from_start, block_log_extract_fixture) try {
    block_log new_log(output_dir2.path());
 
    auto id = gs.compute_chain_id();
-   BOOST_REQUIRE_EQUAL(new_log.extract_chain_id(output_dir2.path()), id);
+   auto extracted_id = new_log.extract_chain_id(output_dir2.path());
+   BOOST_REQUIRE(extracted_id.has_value());
+   BOOST_REQUIRE_EQUAL(*extracted_id, id);
    BOOST_REQUIRE_EQUAL(new_log.first_block_num(), 1);
    BOOST_REQUIRE_EQUAL(new_log.head()->block_num(), 6);
 
@@ -108,7 +114,9 @@ BOOST_FIXTURE_TEST_CASE(extract_to_end, block_log_extract_fixture) try {
    block_log new_log(output_dir.path());
 
    auto id = gs.compute_chain_id();
-   BOOST_REQUIRE_EQUAL(new_log.extract_chain_id(output_dir.path()), id);
+   auto extracted_id = new_log.extract_chain_id(output_dir.path());
+   BOOST_REQUIRE(extracted_id.has_value());
+   BOOST_REQUIRE_EQUAL(*extracted_id, id);
    BOOST_REQUIRE_EQUAL(new_log.first_block_num(), 5);
    BOOST_REQUIRE_EQUAL(new_log.head()->block_num(), 12);