replace thread_local platform_timer with local variable

[4.0] prometheus on different port

CMakeLists.txt

@@ -16,7 +16,7 @@ set( CXX_STANDARD_REQUIRED ON)
 set(VERSION_MAJOR 4)
 set(VERSION_MINOR 0)
 set(VERSION_PATCH 0)
-set(VERSION_SUFFIX rc1)
+set(VERSION_SUFFIX rc2)
 
 if(VERSION_SUFFIX)
     set(VERSION_FULL "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}-${VERSION_SUFFIX}")
@@ -268,18 +268,8 @@ if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.21)
    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/lib/python3/dist-packages/TestHarness DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages COMPONENT dev EXCLUDE_FROM_ALL)
    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/share/leap_testing/bin DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing COMPONENT dev EXCLUDE_FROM_ALL)
 else()
-   # The following install(CODE ...) steps are necessary for `make dev-install` to work on cmake versions < 3.21.
-   # However, it can flag as an error if using `make package` instead of `make dev-install` for installation.
-   # This is due to those directories and symbolic links being created in the postinit script during the package install instead.
-
-   # Note If/when doing `make package`, it is not a true error if you see:
-   # Error creating directory "/<leap_install_dir>/lib/python3/dist-packages".
-   # CMake Error: failed to create symbolic link '/<leap_install_dir>/lib/python3/dist-packages/TestHarness': no such file or directory
-   # CMake Error: failed to create symbolic link '/<leap_install_dir>/share/leap_testing/bin': no such file or directory
-
-   install(CODE "execute_process( COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages)" COMPONENT dev EXCLUDE_FROM_ALL)
-   install(CODE "execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink ../../../share/leap_testing/tests/TestHarness ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages/TestHarness)" COMPONENT dev EXCLUDE_FROM_ALL)
-   install(CODE "execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink ../../bin ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing/bin)" COMPONENT dev EXCLUDE_FROM_ALL)
+   # The following install(SCRIPT ...) steps are necessary for `make dev-install` to work on cmake versions < 3.21.
+   install(SCRIPT ${CMAKE_CURRENT_BINARY_DIR}/scripts/install_testharness_symlinks.cmake COMPONENT dev EXCLUDE_FROM_ALL)
 
    # The `make package` installation of symlinks happens via the `postinst` script installed in cmake.package via the line below
 endif()

README.md

@@ -72,7 +72,7 @@ git clone --recursive https://github.com/AntelopeIO/leap.git
 git clone --recursive git@github.com:AntelopeIO/leap.git
 ```
 
-> ℹ️ **HTTPS vs. SSH Clone** ℹ️
+> ℹ️ **HTTPS vs. SSH Clone** ℹ️  
 Both an HTTPS or SSH git clone will yield the same result - a folder named `leap` containing our source code. It doesn't matter which type you use.
 
 Navigate into that folder:
@@ -96,13 +96,13 @@ git submodule update --init --recursive
 ### Step 3 - Build
 Select build instructions below for a [pinned build](#pinned-build) (preferred) or an [unpinned build](#unpinned-build).
 
-> ℹ️ **Pinned vs. Unpinned Build** ℹ️
+> ℹ️ **Pinned vs. Unpinned Build** ℹ️  
 We have two types of builds for Leap: "pinned" and "unpinned." The only difference is that pinned builds use specific versions for some dependencies hand-picked by the Leap engineers - they are "pinned" to those versions. In contrast, unpinned builds use the default dependency versions available on the build system at the time. We recommend performing a "pinned" build to ensure the compiler and boost versions remain the same between builds of different Leap versions. Leap requires these versions to remain the same, otherwise its state might need to be recovered from a portable snapshot or the chain needs to be replayed.
 
-> ⚠️ **A Warning On Parallel Compilation Jobs (`-j` flag)** ⚠️
+> ⚠️ **A Warning On Parallel Compilation Jobs (`-j` flag)** ⚠️  
 When building C/C++ software, often the build is performed in parallel via a command such as `make -j "$(nproc)"` which uses all available CPU threads. However, be aware that some compilation units (`*.cpp` files) in Leap will consume nearly 4GB of memory. Failures due to memory exhaustion will typically, but not always, manifest as compiler crashes. Using all available CPU threads may also prevent you from doing other things on your computer during compilation. For these reasons, consider reducing this value.
 
-> 🐋 **Docker and `sudo`** 🐋
+> 🐋 **Docker and `sudo`** 🐋  
 If you are in an Ubuntu docker container, omit `sudo` from all commands because you run as `root` by default. Most other docker containers also exclude `sudo`, especially Debian-family containers. If your shell prompt is a hash tag (`#`), omit `sudo`.
 
 #### Pinned Build
@@ -111,12 +111,14 @@ Make sure you are in the root of the `leap` repo, then run the `install_depts.sh
 sudo scripts/install_deps.sh
 ```
 
-Next, run the pinned build script. You have to give it three arguments, in the following order:
-  - A temporary folder, for all dependencies that need to be built from source.
-  - A build folder, where the binaries you need to install will be built to.
-  - The number of jobs or CPU cores/threads to use (note the [jobs flag](#step-3---build) warning above).
+Next, run the pinned build script. You have to give it three arguments in the following order:
+1. A temporary folder, for all dependencies that need to be built from source.
+1. A build folder, where the binaries you need to install will be built to.
+1. The number of jobs or CPU cores/threads to use (note the [jobs flag](#step-3---build) warning above).
 
-The following command runs the `pinned_build.sh` script, specifies a `deps` and `build` folder in the root of the Leap repo for the first two arguments, then builds the packages using all of your computer's CPU threads (Note: you don't need `sudo` for this command):
+> 🔒 You do not need to run this script with `sudo` or as root.
+
+For example, the following command runs the `pinned_build.sh` script, specifies a `deps` and `build` folder in the root of the Leap repo for the first two arguments, then builds the packages using all of your computer's CPU threads:
 ```bash
 scripts/pinned_build.sh deps build "$(nproc)"
 ```

libraries/chain/controller.cpp

@@ -210,6 +210,12 @@ struct pending_state {
 };
 
 struct controller_impl {
+   enum class app_window_type {
+      write, // Only main thread is running; read-only threads are not running.
+             // All read-write and read-only tasks are sequentially executed.
+      read   // Main thread and read-only threads are running read-ony tasks in parallel.
+             // Read-write tasks are not being executed.
+   };
 
    // LLVM sets the new handler, we need to reset this to throw a bad_alloc exception so we can possibly exit cleanly
    // and not just abort.
@@ -242,15 +248,13 @@ struct controller_impl {
    bool                            okay_to_print_integrity_hash_on_stop = false;
 
    std::thread::id                 main_thread_id;
-   thread_local static platform_timer timer; // a copy for main thread and each read-only thread
 #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED)
    thread_local static vm::wasm_allocator wasm_alloc; // a copy for main thread and each read-only thread
 #endif
-   // Ideally wasmif should be thread_local which must be a static.
-   // Unittests can create multiple controller objects (testers) at the same time,
-   // which overwrites the same static wasmif, is used for eosvmoc too.
    wasm_interface  wasmif;  // used by main thread and all threads for EOSVMOC
-   thread_local static std::unique_ptr<wasm_interface> wasmif_thread_local; // a copy for each read-only thread, used by eos-vm and eos-vm-jit
+   std::mutex threaded_wasmifs_mtx;
+   std::unordered_map<std::thread::id, std::unique_ptr<wasm_interface>> threaded_wasmifs; // one for each read-only thread, used by eos-vm and eos-vm-jit
+   app_window_type app_window = app_window_type::write;
 
    typedef pair<scope_name,action_name>                   handler_key;
    map< account_name, map<handler_key, apply_handler> >   apply_handlers;
@@ -337,7 +341,12 @@ struct controller_impl {
       set_activation_handler<builtin_protocol_feature_t::crypto_primitives>();
 
       self.irreversible_block.connect([this](const block_state_ptr& bsp) {
-         get_wasm_interface().current_lib(bsp->block_num);
+         // producer_plugin has already asserted irreversible_block signal is
+         // called in write window
+         wasmif.current_lib(bsp->block_num);
+         for (auto& w: threaded_wasmifs) {
+            w.second->current_lib(bsp->block_num);
+         }
       });
 
 
@@ -497,9 +506,10 @@ struct controller_impl {
 
    void replay(std::function<bool()> check_shutdown) {
       auto blog_head = blog.head();
-      if( !blog_head && !fork_db.root() ) {
+      if( !fork_db.root() ) {
          fork_db.reset( *head );
-         return;
+         if (!blog_head)
+            return;
       }
 
       replaying = true;
@@ -1163,6 +1173,7 @@ struct controller_impl {
          etrx.set_reference_block( self.head_block_id() );
       }
 
+      platform_timer timer;
       transaction_checktime_timer trx_timer(timer);
       const packed_transaction trx( std::move( etrx ) );
       transaction_context trx_context( self, trx, std::move(trx_timer), start );
@@ -1330,6 +1341,7 @@ struct controller_impl {
 
       uint32_t cpu_time_to_bill_us = billed_cpu_time_us;
 
+      platform_timer timer;
       transaction_checktime_timer trx_timer( timer );
       transaction_context trx_context( self, *trx->packed_trx(), std::move(trx_timer) );
       trx_context.leeway =  fc::microseconds(0); // avoid stealing cpu resource
@@ -1544,6 +1556,7 @@ struct controller_impl {
          }
 
          const signed_transaction& trn = trx->packed_trx()->get_signed_transaction();
+         platform_timer timer;
          transaction_checktime_timer trx_timer(timer);
          transaction_context trx_context(self, *trx->packed_trx(), std::move(trx_timer), start, trx->get_trx_type());
          if ((bool)subjective_cpu_leeway && self.is_speculative_block()) {
@@ -2680,38 +2693,59 @@ struct controller_impl {
    // only called from non-main threads (read-only trx execution threads)
    // when producer_plugin starts them
    void init_thread_local_data() {
-      EOS_ASSERT( !is_main_thread(), misc_exception, "init_thread_local_data called on the main thread");
+      EOS_ASSERT( !is_on_main_thread(), misc_exception, "init_thread_local_data called on the main thread");
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
       if ( is_eos_vm_oc_enabled() )
          // EOSVMOC needs further initialization of its thread local data
          wasmif.init_thread_local_data();
       else
 #endif
+      {
+         std::lock_guard g(threaded_wasmifs_mtx);
          // Non-EOSVMOC needs a wasmif per thread
-         wasmif_thread_local = std::make_unique<wasm_interface>( conf.wasm_runtime, conf.eosvmoc_tierup, db, conf.state_dir, conf.eosvmoc_config, !conf.profile_accounts.empty());
+         threaded_wasmifs[std::this_thread::get_id()]  = std::make_unique<wasm_interface>( conf.wasm_runtime, conf.eosvmoc_tierup, db, conf.state_dir, conf.eosvmoc_config, !conf.profile_accounts.empty());
+      }
    }
 
-   bool is_main_thread() { return main_thread_id == std::this_thread::get_id(); };
+   bool is_on_main_thread() { return main_thread_id == std::this_thread::get_id(); };
+
+   void set_to_write_window() {
+      app_window = app_window_type::write;
+   }
+   void set_to_read_window() {
+      app_window = app_window_type::read;
+   }
+   bool is_write_window() const {
+      return app_window == app_window_type::write;
+   }
 
    wasm_interface& get_wasm_interface() {
-      if ( is_main_thread()
+      if ( is_on_main_thread()
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
           || is_eos_vm_oc_enabled()
 #endif
          )
          return wasmif;
       else
-         return *wasmif_thread_local;
+         return *threaded_wasmifs[std::this_thread::get_id()];
+   }
+
+   void code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num) {
+      // The caller of this function apply_eosio_setcode has already asserted that
+      // the transaction is not a read-only trx, which implies we are
+      // in write window. Safe to call threaded_wasmifs's code_block_num_last_used
+      wasmif.code_block_num_last_used(code_hash, vm_type, vm_version, block_num);
+      for (auto& w: threaded_wasmifs) {
+         w.second->code_block_num_last_used(code_hash, vm_type, vm_version, block_num);
+      }
    }
 
    block_state_ptr fork_db_head() const;
 }; /// controller_impl
 
-thread_local platform_timer controller_impl::timer;
 #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED)
 thread_local eosio::vm::wasm_allocator controller_impl::wasm_alloc;
 #endif
-thread_local std::unique_ptr<wasm_interface> controller_impl::wasmif_thread_local;
 
 const resource_limits_manager&   controller::get_resource_limits_manager()const
 {
@@ -3686,6 +3720,20 @@ void controller::init_thread_local_data() {
    my->init_thread_local_data();
 }
 
+void controller::set_to_write_window() {
+   my->set_to_write_window();
+}
+void controller::set_to_read_window() {
+   my->set_to_read_window();
+}
+bool controller::is_write_window() const {
+   return my->is_write_window();
+}
+
+void controller::code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num) {
+   return my->code_block_num_last_used(code_hash, vm_type, vm_version, block_num);
+}
+
 /// Protocol feature activation handlers:
 
 template<>

libraries/chain/eosio_contract.cpp

@@ -160,7 +160,7 @@ void apply_eosio_setcode(apply_context& context) {
       old_size  = (int64_t)old_code_entry.code.size() * config::setcode_ram_bytes_multiplier;
       if( old_code_entry.code_ref_count == 1 ) {
          db.remove(old_code_entry);
-         context.control.get_wasm_interface().code_block_num_last_used(account.code_hash, account.vm_type, account.vm_version, context.control.head_block_num() + 1);
+         context.control.code_block_num_last_used(account.code_hash, account.vm_type, account.vm_version, context.control.head_block_num() + 1);
       } else {
          db.modify(old_code_entry, [](code_object& o) {
             --o.code_ref_count;

libraries/chain/include/eosio/chain/controller.hpp

@@ -373,6 +373,10 @@ namespace eosio { namespace chain {
       void set_db_read_only_mode();
       void unset_db_read_only_mode();
       void init_thread_local_data();
+      void set_to_write_window();
+      void set_to_read_window();
+      bool is_write_window() const;
+      void code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num);
 
       private:
          friend class apply_context;

libraries/chain/include/eosio/chain/wasm_interface_private.hpp

@@ -37,7 +37,6 @@ namespace eosio { namespace chain {
    struct wasm_interface_impl {
       struct wasm_cache_entry {
          digest_type                                          code_hash;
-         uint32_t                                             first_block_num_used;
          uint32_t                                             last_block_num_used;
          std::unique_ptr<wasm_instantiated_module_interface>  module;
          uint8_t                                              vm_type = 0;
@@ -109,24 +108,6 @@ namespace eosio { namespace chain {
          return it != wasm_instantiation_cache.end();
       }
 
-      std::vector<uint8_t> parse_initial_memory(const Module& module) {
-         std::vector<uint8_t> mem_image;
-
-         for(const DataSegment& data_segment : module.dataSegments) {
-            EOS_ASSERT(data_segment.baseOffset.type == InitializerExpression::Type::i32_const, wasm_exception, "");
-            EOS_ASSERT(module.memories.defs.size(), wasm_exception, "");
-            const U32 base_offset = data_segment.baseOffset.i32;
-            const Uptr memory_size = (module.memories.defs[0].type.size.min << IR::numBytesPerPageLog2);
-            if(base_offset >= memory_size || base_offset + data_segment.data.size() > memory_size)
-               FC_THROW_EXCEPTION(wasm_execution_error, "WASM data segment outside of valid memory range");
-            if(base_offset + data_segment.data.size() > mem_image.size())
-               mem_image.resize(base_offset + data_segment.data.size(), 0x00);
-            memcpy(mem_image.data() + base_offset, data_segment.data.data(), data_segment.data.size());
-         }
-
-         return mem_image;
-      }
-
       void code_block_num_last_used(const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version, const uint32_t& block_num) {
          wasm_cache_index::iterator it = wasm_instantiation_cache.find(boost::make_tuple(code_hash, vm_type, vm_version));
          if(it != wasm_instantiation_cache.end())
@@ -157,7 +138,6 @@ namespace eosio { namespace chain {
 
             it = wasm_instantiation_cache.emplace( wasm_interface_impl::wasm_cache_entry{
                                                       .code_hash = code_hash,
-                                                      .first_block_num_used = codeobject->first_block_used,
                                                       .last_block_num_used = UINT32_MAX,
                                                       .module = nullptr,
                                                       .vm_type = vm_type,
@@ -173,24 +153,8 @@ namespace eosio { namespace chain {
                trx_context.resume_billing_timer();
             });
             trx_context.pause_billing_timer();
-            IR::Module module;
-            std::vector<U8> bytes = {
-                (const U8*)codeobject->code.data(),
-                (const U8*)codeobject->code.data() + codeobject->code.size()};
-            try {
-               Serialization::MemoryInputStream stream((const U8*)bytes.data(),
-                                                       bytes.size());
-               WASM::scoped_skip_checks no_check;
-               WASM::serialize(stream, module);
-               module.userSections.clear();
-            } catch (const Serialization::FatalSerializationException& e) {
-               EOS_ASSERT(false, wasm_serialization_error, e.message.c_str());
-            } catch (const IR::ValidationException& e) {
-               EOS_ASSERT(false, wasm_serialization_error, e.message.c_str());
-            }
-
             wasm_instantiation_cache.modify(it, [&](auto& c) {
-               c.module = runtime_interface->instantiate_module((const char*)bytes.data(), bytes.size(), parse_initial_memory(module), code_hash, vm_type, vm_version);
+               c.module = runtime_interface->instantiate_module(codeobject->code.data(), codeobject->code.size(), code_hash, vm_type, vm_version);
             });
          }
          return it->module;
@@ -209,7 +173,6 @@ namespace eosio { namespace chain {
                   member<wasm_cache_entry, uint8_t,     &wasm_cache_entry::vm_version>
                >
             >,
-            ordered_non_unique<tag<by_first_block_num>, member<wasm_cache_entry, uint32_t, &wasm_cache_entry::first_block_num_used>>,
             ordered_non_unique<tag<by_last_block_num>, member<wasm_cache_entry, uint32_t, &wasm_cache_entry::last_block_num_used>>
          >
       > wasm_cache_index;

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc.hpp

@@ -30,7 +30,7 @@ class eosvmoc_runtime : public eosio::chain::wasm_runtime_interface {
    public:
       eosvmoc_runtime(const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db);
       ~eosvmoc_runtime();
-      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t> initial_memory,
+      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
                                                                              const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override;
 
       void immediately_exit_currently_running_module() override;

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/code_cache.hpp

@@ -85,9 +85,6 @@ class code_cache_base {
 
       template <typename T>
       void serialize_cache_index(fc::datastream<T>& ds);
-
-      std::thread::id _main_thread_id;
-      bool is_main_thread() const;
 };
 
 class code_cache_async : public code_cache_base {
@@ -98,7 +95,7 @@ class code_cache_async : public code_cache_base {
       //If code is in cache: returns pointer & bumps to front of MRU list
       //If code is not in cache, and not blacklisted, and not currently compiling: return nullptr and kick off compile
       //otherwise: return nullptr
-      const code_descriptor* const get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version);
+      const code_descriptor* const get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version, bool is_write_window);
 
    private:
       std::thread _monitor_reply_thread;
@@ -115,7 +112,7 @@ class code_cache_sync : public code_cache_base {
       ~code_cache_sync();
 
       //Can still fail and return nullptr if, for example, there is an expected instantiation failure
-      const code_descriptor* const get_descriptor_for_code_sync(const digest_type& code_id, const uint8_t& vm_version);
+      const code_descriptor* const get_descriptor_for_code_sync(const digest_type& code_id, const uint8_t& vm_version, bool is_write_window);
 };
 
 }}}

libraries/chain/include/eosio/chain/webassembly/eos-vm.hpp

@@ -43,7 +43,7 @@ template<typename Backend>
 class eos_vm_runtime : public eosio::chain::wasm_runtime_interface {
    public:
       eos_vm_runtime();
-      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t>,
+      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
                                                                              const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override;
 
       void immediately_exit_currently_running_module() override;
@@ -61,7 +61,7 @@ class eos_vm_runtime : public eosio::chain::wasm_runtime_interface {
 class eos_vm_profile_runtime : public eosio::chain::wasm_runtime_interface {
    public:
       eos_vm_profile_runtime();
-      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t>,
+      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
                                                                              const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override;
 
       void immediately_exit_currently_running_module() override;

libraries/chain/include/eosio/chain/webassembly/runtime_interface.hpp

@@ -20,7 +20,7 @@ class wasm_instantiated_module_interface {
 
 class wasm_runtime_interface {
    public:
-      virtual std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t> initial_memory,
+      virtual std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
                                                                                      const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) = 0;
 
       //immediately exit the currently running wasm_instantiated_module_interface. Yep, this assumes only one can possibly run at a time.

libraries/chain/wasm_interface.cpp

@@ -93,7 +93,7 @@ namespace eosio { namespace chain {
       if(my->eosvmoc) {
          const chain::eosvmoc::code_descriptor* cd = nullptr;
          try {
-            cd = my->eosvmoc->cc.get_descriptor_for_code(code_hash, vm_version);
+            cd = my->eosvmoc->cc.get_descriptor_for_code(code_hash, vm_version, context.control.is_write_window());
          }
          catch(...) {
             //swallow errors here, if EOS VM OC has gone in to the weeds we shouldn't bail: continue to try and run baseline

libraries/chain/webassembly/runtimes/eos-vm-oc.cpp

@@ -28,7 +28,7 @@ class eosvmoc_instantiated_module : public wasm_instantiated_module_interface {
       bool is_main_thread() { return _main_thread_id == std::this_thread::get_id(); };
 
       void apply(apply_context& context) override {
-         const code_descriptor* const cd = _eosvmoc_runtime.cc.get_descriptor_for_code_sync(_code_hash, _vm_version);
+         const code_descriptor* const cd = _eosvmoc_runtime.cc.get_descriptor_for_code_sync(_code_hash, _vm_version, context.control.is_write_window());
          EOS_ASSERT(cd, wasm_execution_error, "EOS VM OC instantiation failed");
 
          if ( is_main_thread() )
@@ -50,7 +50,7 @@ eosvmoc_runtime::eosvmoc_runtime(const boost::filesystem::path data_dir, const e
 eosvmoc_runtime::~eosvmoc_runtime() {
 }
 
-std::unique_ptr<wasm_instantiated_module_interface> eosvmoc_runtime::instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t> initial_memory,
+std::unique_ptr<wasm_instantiated_module_interface> eosvmoc_runtime::instantiate_module(const char* code_bytes, size_t code_size,
                                                                                         const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) {
    return std::make_unique<eosvmoc_instantiated_module>(code_hash, vm_type, *this);
 }

libraries/chain/webassembly/runtimes/eos-vm-oc/code_cache.cpp

@@ -107,10 +107,11 @@ std::tuple<size_t, size_t> code_cache_async::consume_compile_thread_queue() {
 }
 
 
-const code_descriptor* const code_cache_async::get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version) {
+const code_descriptor* const code_cache_async::get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version, bool is_write_window) {
    //if there are any outstanding compiles, process the result queue now
-   //do this only on main thread (which is in single threaded write window)
-   if(is_main_thread() && _outstanding_compiles_and_poison.size()) {
+   //When app is in write window, all tasks are running sequentially and read-only threads
+   //are not running. Safe to update cache entries.
+   if(is_write_window && _outstanding_compiles_and_poison.size()) {
       auto [count_processed, bytes_remaining] = consume_compile_thread_queue();
 
       if(count_processed)
@@ -136,11 +137,11 @@ const code_descriptor* const code_cache_async::get_descriptor_for_code(const dig
    //check for entry in cache
    code_cache_index::index<by_hash>::type::iterator it = _cache_index.get<by_hash>().find(boost::make_tuple(code_id, vm_version));
    if(it != _cache_index.get<by_hash>().end()) {
-      if (is_main_thread())
+      if (is_write_window)
          _cache_index.relocate(_cache_index.begin(), _cache_index.project<0>(it));
       return &*it;
    }
-   if(!is_main_thread()) // on read-only thread
+   if(!is_write_window)
       return nullptr;
 
    const code_tuple ct = code_tuple{code_id, vm_version};
@@ -179,15 +180,15 @@ code_cache_sync::~code_cache_sync() {
       elog("unexpected response from EOS VM OC compile monitor during shutdown");
 }
 
-const code_descriptor* const code_cache_sync::get_descriptor_for_code_sync(const digest_type& code_id, const uint8_t& vm_version) {
+const code_descriptor* const code_cache_sync::get_descriptor_for_code_sync(const digest_type& code_id, const uint8_t& vm_version, bool is_write_window) {
    //check for entry in cache
    code_cache_index::index<by_hash>::type::iterator it = _cache_index.get<by_hash>().find(boost::make_tuple(code_id, vm_version));
    if(it != _cache_index.get<by_hash>().end()) {
-      if (is_main_thread())
+      if (is_write_window)
          _cache_index.relocate(_cache_index.begin(), _cache_index.project<0>(it));
       return &*it;
    }
-   if(!is_main_thread())
+   if(!is_write_window)
       return nullptr;
 
    const code_object* const codeobject = _db.find<code_object,by_code_hash>(boost::make_tuple(code_id, 0, vm_version));
@@ -212,8 +213,7 @@ const code_descriptor* const code_cache_sync::get_descriptor_for_code_sync(const
 
 code_cache_base::code_cache_base(const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db) :
    _db(db),
-   _cache_file_path(data_dir/"code_cache.bin"),
-   _main_thread_id(std::this_thread::get_id())
+   _cache_file_path(data_dir/"code_cache.bin")
 {
    static_assert(sizeof(allocator_t) <= header_offset, "header offset intersects with allocator");
 
@@ -390,8 +390,4 @@ void code_cache_base::check_eviction_threshold(size_t free_bytes) {
    if(free_bytes < _free_bytes_eviction_threshold)
       run_eviction_round();
 }
-
-bool code_cache_base::is_main_thread() const {
-   return _main_thread_id == std::this_thread::get_id();
-}
 }}}

libraries/chain/webassembly/runtimes/eos-vm.cpp

@@ -239,7 +239,7 @@ void eos_vm_runtime<Impl>::immediately_exit_currently_running_module() {
 }
 
 template<typename Impl>
-std::unique_ptr<wasm_instantiated_module_interface> eos_vm_runtime<Impl>::instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t>,
+std::unique_ptr<wasm_instantiated_module_interface> eos_vm_runtime<Impl>::instantiate_module(const char* code_bytes, size_t code_size,
                                                                                              const digest_type&, const uint8_t&, const uint8_t&) {
 
    using backend_t = eos_vm_backend_t<Impl>;
@@ -265,7 +265,7 @@ void eos_vm_profile_runtime::immediately_exit_currently_running_module() {
    throw wasm_exit{};
 }
 
-std::unique_ptr<wasm_instantiated_module_interface> eos_vm_profile_runtime::instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t>,
+std::unique_ptr<wasm_instantiated_module_interface> eos_vm_profile_runtime::instantiate_module(const char* code_bytes, size_t code_size,
                                                                                                const digest_type&, const uint8_t&, const uint8_t&) {
 
    using backend_t = eosio::vm::backend<eos_vm_host_functions_t, eosio::vm::jit_profile, webassembly::eos_vm_runtime::apply_options, vm::profile_instr_map>;

libraries/custom_appbase/include/eosio/chain/application.hpp

@@ -1,30 +1,33 @@
 #pragma once
 
 #include <appbase/application_base.hpp>
-#include <appbase/execution_priority_queue.hpp>
+#include <eosio/chain/exec_pri_queue.hpp>
+#include <mutex>
 
 /*
- * Custmomize appbase to support two-queue exector.
+ * Customize appbase to support two-queue executor.
 */
 namespace appbase { 
 
 enum class exec_window {
-   read,  // the window during which operations from read_only_trx_safe queue
-          // can be executed in app thread in parallel with read-only transactions
-          // in read-only transaction excuting threads.
-   write, // the window during which operations from both general and
-          // read_only_trx_safe queues can be executed in app thread,
-          // while read-only transactions are not executed in read-only
-          // transaction excuting threads.
+   read,  // the window during which operations from read_only queue
+          // can be executed in parallel in the read-only thread pool
+          // as well as in the app thread.
+   write, // the window during which operations from both read_write and
+          // parallel queues can be executed in app thread,
+          // while read-only operations are not executed in read-only
+          // thread pool. The read-only thread pool is not active; only
+          // the main app thread is active.
 };
 
 enum class exec_queue {
-   read_only_trx_safe, // the queue storing operations which are safe to execute
-                       // on app thread in parallel with read-only transactions
-                       // in read-only transaction excuting threads.
-   general             // the queue storing operations which can be only executed
-                       // on the app thread while read-only transactions are
-                       // not being executed in read-only threads
+   read_only,          // the queue storing tasks which are safe to execute
+                       // in parallel with other read-only tasks in the read-only
+                       // thread pool as well as on the main app thread.
+                       // Multi-thread safe as long as nothing is executed from the read_write queue.
+   read_write          // the queue storing tasks which can be only executed
+                       // on the app thread while read-only tasks are
+                       // not being executed in read-only threads. Single threaded.
 };
 
 class two_queue_executor {
@@ -32,56 +35,64 @@ public:
 
    template <typename Func>
    auto post( int priority, exec_queue q, Func&& func ) {
-      if ( q == exec_queue::general )
-         return boost::asio::post(io_serv_, general_queue_.wrap(priority, --order_, std::forward<Func>(func)));
+      if ( q == exec_queue::read_write )
+         return boost::asio::post(io_serv_, read_write_queue_.wrap(priority, --order_, std::forward<Func>(func)));
       else
-         return boost::asio::post(io_serv_, read_only_trx_safe_queue_.wrap(priority, --order_, std::forward<Func>(func)));
+         return boost::asio::post( io_serv_, read_only_queue_.wrap( priority, --order_, std::forward<Func>( func)));
    }
 
    // Legacy and deprecated. To be removed after cleaning up its uses in base appbase
    template <typename Func>
    auto post( int priority, Func&& func ) {
-      // safer to use general queue for unknown type of operation since operations
-      // from general queue are not executed in parallel with read-only transactions
-      return boost::asio::post(io_serv_, general_queue_.wrap(priority, --order_, std::forward<Func>(func)));
+      // safer to use read_write queue for unknown type of operation since operations
+      // from read_write queue are not executed in parallel with read-only operations
+      return boost::asio::post(io_serv_, read_write_queue_.wrap(priority, --order_, std::forward<Func>(func)));
    }
 
    boost::asio::io_service& get_io_service() { return io_serv_; }
 
    bool execute_highest() {
       if ( exec_window_ == exec_window::write ) {
-         if( !general_queue_.empty() && ( read_only_trx_safe_queue_.empty() || *read_only_trx_safe_queue_.top() < *general_queue_.top()) )  {
-            // general_queue_'s top function's priority greater than read_only_trx_safe_queue_'s top function's, or general_queue_ empty
-            general_queue_.execute_highest();
-         } else if( !read_only_trx_safe_queue_.empty() ) {
-            read_only_trx_safe_queue_.execute_highest();
+         // During write window only main thread is accessing anything in two_queue_executor, no locking required
+         if( !read_write_queue_.empty() && (read_only_queue_.empty() || *read_only_queue_.top() < *read_write_queue_.top()) )  {
+            // read_write_queue_'s top function's priority greater than read_only_queue_'s top function's, or read_only_queue_ empty
+            read_write_queue_.execute_highest();
+         } else if( !read_only_queue_.empty() ) {
+            read_only_queue_.execute_highest();
          }
-         return !read_only_trx_safe_queue_.empty() || !general_queue_.empty();
+         return !read_only_queue_.empty() || !read_write_queue_.empty();
       } else {
-         return read_only_trx_safe_queue_.execute_highest();
+         // When in read window, multiple threads including main app thread are accessing two_queue_executor, locking required
+         return read_only_queue_.execute_highest_locked(false);
       }
    }
 
+   bool execute_highest_read_only() {
+      return read_only_queue_.execute_highest_locked(true);
+   }
+
    template <typename Function>
-   boost::asio::executor_binder<Function, appbase::execution_priority_queue::executor>
+   boost::asio::executor_binder<Function, appbase::exec_pri_queue::executor>
    wrap(int priority, exec_queue q, Function&& func ) {
-      if ( q == exec_queue::general )
-         return general_queue_.wrap(priority, --order_, std::forward<Function>(func));
+      if ( q == exec_queue::read_write )
+         return read_write_queue_.wrap(priority, --order_, std::forward<Function>(func));
       else
-         return read_only_trx_safe_queue_.wrap(priority, --order_, std::forward<Function>(func));
+         return read_only_queue_.wrap( priority, --order_, std::forward<Function>( func));
    }
      
    void clear() {
-      read_only_trx_safe_queue_.clear();
-      general_queue_.clear();
+      read_only_queue_.clear();
+      read_write_queue_.clear();
    }
 
-   void set_to_read_window() {
+   void set_to_read_window(uint32_t num_threads, std::function<bool()> should_exit) {
       exec_window_ = exec_window::read;
+      read_only_queue_.enable_locking(num_threads, std::move(should_exit));
    }
 
    void set_to_write_window() {
       exec_window_ = exec_window::write;
+      read_only_queue_.disable_locking();
    }
 
    bool is_read_window() const {
@@ -92,15 +103,15 @@ public:
       return exec_window_ == exec_window::write;
    }
 
-   auto& read_only_trx_safe_queue() { return read_only_trx_safe_queue_; }
+   auto& read_only_queue() { return read_only_queue_; }
 
-   auto& general_queue() { return general_queue_; }
+   auto& read_write_queue() { return read_write_queue_; }
 
    // members are ordered taking into account that the last one is destructed first
 private:
    boost::asio::io_service            io_serv_;
-   appbase::execution_priority_queue  read_only_trx_safe_queue_;
-   appbase::execution_priority_queue  general_queue_;
+   appbase::exec_pri_queue            read_only_queue_;
+   appbase::exec_pri_queue            read_write_queue_;
    std::atomic<std::size_t>           order_ { std::numeric_limits<size_t>::max() }; // to maintain FIFO ordering in both queues within priority
    exec_window                        exec_window_ { exec_window::write };
 };

libraries/custom_appbase/include/eosio/chain/exec_pri_queue.hpp

@@ -0,0 +1,233 @@
+#pragma once
+#include <boost/asio.hpp>
+
+#include <condition_variable>
+#include <mutex>
+#include <queue>
+
+namespace appbase {
+// adapted from: https://www.boost.org/doc/libs/1_69_0/doc/html/boost_asio/example/cpp11/invocation/prioritised_handlers.cpp
+
+// Locking has to be coordinated by caller, use with care.
+class exec_pri_queue : public boost::asio::execution_context
+{
+public:
+
+   void enable_locking(uint32_t num_threads, std::function<bool()> should_exit) {
+      assert(num_threads > 0 && num_waiting_ == 0);
+      lock_enabled_ = true;
+      max_waiting_ = num_threads;
+      should_exit_ = std::move(should_exit);
+      exiting_blocking_ = false;
+   }
+
+   void disable_locking() {
+      lock_enabled_ = false;
+      should_exit_ = [](){ assert(false); return true; }; // should not be called when locking is disabled
+   }
+
+   // called from appbase::application_base::exec poll_one() or run_one()
+   template <typename Function>
+   void add(int priority, size_t order, Function function)
+   {
+      std::unique_ptr<queued_handler_base> handler(new queued_handler<Function>(priority, order, std::move(function)));
+      if (lock_enabled_) {
+         std::lock_guard g( mtx_ );
+         handlers_.push( std::move( handler ) );
+         if (num_waiting_)
+            cond_.notify_one();
+      } else {
+         handlers_.push( std::move( handler ) );
+      }
+   }
+
+   // only call when no lock required
+   void clear()
+   {
+      handlers_ = prio_queue();
+   }
+
+   // only call when no lock required
+   bool execute_highest()
+   {
+      if( !handlers_.empty() ) {
+         handlers_.top()->execute();
+         handlers_.pop();
+      }
+
+      return !handlers_.empty();
+   }
+
+private:
+   // has to be defined before use, auto return type
+   auto pop() {
+      auto t = std::move(const_cast<std::unique_ptr<queued_handler_base>&>(handlers_.top()));
+      handlers_.pop();
+      return t;
+   }
+
+public:
+
+   bool execute_highest_locked(bool should_block) {
+      std::unique_lock g(mtx_);
+      if (should_block) {
+         ++num_waiting_;
+         cond_.wait(g, [this](){
+            bool exit = exiting_blocking_ || should_exit_();
+            bool empty = handlers_.empty();
+            if (empty || exit) {
+               if (((empty && num_waiting_ == max_waiting_) || exit) && !exiting_blocking_) {
+                  cond_.notify_all();
+                  exiting_blocking_ = true;
+               }
+               return exit || exiting_blocking_; // same as calling should_exit(), but faster
+            }
+            return true;
+         });
+         --num_waiting_;
+         if (exiting_blocking_ || should_exit_())
+            return false;
+      }
+      if( handlers_.empty() )
+         return false;
+      auto t = pop();
+      g.unlock();
+      t->execute();
+      return true;
+   }
+
+   // Only call when locking disabled
+   size_t size() const { return handlers_.size(); }
+
+   // Only call when locking disabled
+   bool empty() const { return handlers_.empty(); }
+
+   // Only call when locking disabled
+   const auto& top() const { return handlers_.top(); }
+
+   class executor
+   {
+   public:
+      executor(exec_pri_queue& q, int p, size_t o)
+            : context_(q), priority_(p), order_(o)
+      {
+      }
+
+      exec_pri_queue& context() const noexcept
+      {
+         return context_;
+      }
+
+      template <typename Function, typename Allocator>
+      void dispatch(Function f, const Allocator&) const
+      {
+         context_.add(priority_, order_, std::move(f));
+      }
+
+      template <typename Function, typename Allocator>
+      void post(Function f, const Allocator&) const
+      {
+         context_.add(priority_, order_, std::move(f));
+      }
+
+      template <typename Function, typename Allocator>
+      void defer(Function f, const Allocator&) const
+      {
+         context_.add(priority_, order_, std::move(f));
+      }
+
+      void on_work_started() const noexcept {}
+      void on_work_finished() const noexcept {}
+
+      bool operator==(const executor& other) const noexcept
+      {
+         return order_ == other.order_ && &context_ == &other.context_ && priority_ == other.priority_;
+      }
+
+      bool operator!=(const executor& other) const noexcept
+      {
+         return !operator==(other);
+      }
+
+   private:
+      exec_pri_queue& context_;
+      int priority_;
+      size_t order_;
+   };
+
+   template <typename Function>
+   boost::asio::executor_binder<Function, executor>
+   wrap(int priority, size_t order, Function&& func)
+   {
+      return boost::asio::bind_executor( executor(*this, priority, order), std::forward<Function>(func) );
+   }
+
+private:
+   class queued_handler_base
+   {
+   public:
+      queued_handler_base( int p, size_t order )
+            : priority_( p )
+            , order_( order )
+      {
+      }
+
+      virtual ~queued_handler_base() = default;
+
+      virtual void execute() = 0;
+
+      int priority() const { return priority_; }
+      // C++20
+      // friend std::weak_ordering operator<=>(const queued_handler_base&,
+      //                                       const queued_handler_base&) noexcept = default;
+      friend bool operator<(const queued_handler_base& a,
+                            const queued_handler_base& b) noexcept
+      {
+         return std::tie( a.priority_, a.order_ ) < std::tie( b.priority_, b.order_ );
+      }
+
+   private:
+      int priority_;
+      size_t order_;
+   };
+
+   template <typename Function>
+   class queued_handler : public queued_handler_base
+   {
+   public:
+      queued_handler(int p, size_t order, Function f)
+            : queued_handler_base( p, order )
+            , function_( std::move(f) )
+      {
+      }
+
+      void execute() override
+      {
+         function_();
+      }
+
+   private:
+      Function function_;
+   };
+
+   struct deref_less
+   {
+      template<typename Pointer>
+      bool operator()(const Pointer& a, const Pointer& b) noexcept(noexcept(*a < *b))
+      {
+         return *a < *b;
+      }
+   };
+
+   bool lock_enabled_ = false;
+   std::mutex mtx_;
+   std::condition_variable cond_;
+   uint32_t num_waiting_{0};
+   uint32_t max_waiting_{0};
+   bool exiting_blocking_{false};
+   std::function<bool()> should_exit_; // called holding mtx_
+   using prio_queue = std::priority_queue<std::unique_ptr<queued_handler_base>, std::deque<std::unique_ptr<queued_handler_base>>, deref_less>;
+   prio_queue handlers_;
+};
+
+} // appbase

libraries/custom_appbase/tests/custom_appbase_tests.cpp

@@ -27,28 +27,28 @@ BOOST_AUTO_TEST_CASE( default_exec_window ) {
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::read_only_trx_safe,  [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_only,  [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[7]=seq_num; ++seq_num; } );
 
    // Stop app. Use the lowest priority to make sure this function to execute the last
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
-      // read_only_trx_safe_queue should only contain the current lambda function,
-      // and general_queue should have execute all its functions
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
-      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 0 );
+   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
+      // read_only_queue should only contain the current lambda function,
+      // and read_write_queue should have executed all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 1); // pop()s after execute
+      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 0 );
       app->quit();
       } );
    app_thread.join();
 
    // both queues are cleared after execution 
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
 
    // exactly number of both queues' functions processed
    BOOST_REQUIRE_EQUAL( rslts.size(), 8 );
@@ -64,40 +64,40 @@ BOOST_AUTO_TEST_CASE( default_exec_window ) {
    BOOST_CHECK_LT( rslts[6], rslts[7] );
 }
 
-// verify functions only from read_only_trx_safe queue are processed during read window
+// verify functions only from read_only queue are processed during read window
 BOOST_AUTO_TEST_CASE( execute_from_read_queue ) {
    appbase::scoped_app app;
    auto app_thread = start_app_thread(app);
    
-   // set to run functions from read_only_trx_safe queue only
-   app->executor().set_to_read_window();
+   // set to run functions from read_only queue only
+   app->executor().set_to_read_window(1, [](){return false;});
 
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[7]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[8]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[9]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[9]=seq_num; ++seq_num; } );
 
    // stop application. Use lowest at the end to make sure this executes the last
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
-      // read_queue should have current function and write_queue should have all its functions
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
-      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 4 );
+   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
+      // read_queue should be empty (read window pops before execute) and write_queue should have all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 0); // pop()s before execute
+      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 4 );
       app->quit();
       } );
    app_thread.join();
 
    // both queues are cleared after execution 
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
 
    // exactly number of posts processed
    BOOST_REQUIRE_EQUAL( rslts.size(), 6 );
@@ -109,40 +109,40 @@ BOOST_AUTO_TEST_CASE( execute_from_read_queue ) {
    BOOST_CHECK_LT( rslts[3], rslts[4] );
 }
 
-// verify no functions are executed during  read window if read_only_trx_safe queue is empty
+// verify no functions are executed during  read window if read_only queue is empty
 BOOST_AUTO_TEST_CASE( execute_from_empty_read_queue ) {
    appbase::scoped_app app;
    auto app_thread = start_app_thread(app);
    
-   // set to run functions from read_only_trx_safe queue only
-   app->executor().set_to_read_window();
+   // set to run functions from read_only queue only
+   app->executor().set_to_read_window(1, [](){return false;});
 
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::general, [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[8]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[9]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_write, [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_write, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[9]=seq_num; ++seq_num; } );
 
    // Stop application. Use lowest at the end to make sure this executes the last
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
-      // read_queue should have current function and write_queue should have all its functions
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
-      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 10 );
+   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
+      // read_queue should be empty (read window pops before execute) and write_queue should have all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 0); // pop()s before execute
+      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 10 );
       app->quit();
       } );
    app_thread.join();
 
    // both queues are cleared after execution 
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
 
    // no results
    BOOST_REQUIRE_EQUAL( rslts.size(), 0 );
@@ -159,32 +159,32 @@ BOOST_AUTO_TEST_CASE( execute_from_both_queues ) {
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::read_only_trx_safe,  [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[8]=seq_num; ++seq_num; } );
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[9]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[10]=seq_num; ++seq_num; } );
-   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[11]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_only,  [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only,  [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only,  [&]() { rslts[9]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[10]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[11]=seq_num; ++seq_num; } );
 
    // stop application. Use lowest at the end to make sure this executes the last
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
+   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
       // read_queue should have current function and write_queue's functions are all executed 
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
-      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 0 );
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 1); // pop()s after execute
+      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 0 );
       app->quit();
       } );
 
    app_thread.join();
 
    // queues are emptied after quit
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
 
    // exactly number of posts processed
    BOOST_REQUIRE_EQUAL( rslts.size(), 12 );

libraries/libfc/CMakeLists.txt

@@ -57,8 +57,6 @@ set( fc_sources
      src/crypto/blake2.cpp
      src/crypto/k1_recover.cpp
      src/network/ip.cpp
-     src/network/resolve.cpp
-     src/network/udp_socket.cpp
      src/network/url.cpp
      src/network/http/http_client.cpp
      src/compress/zlib.cpp

libraries/libfc/include/fc/network/resolve.hpp

@@ -1,9 +0,0 @@
-#pragma once
-#include <fc/vector.hpp>
-#include <fc/network/ip.hpp>
-
-namespace fc
-{
-  std::vector<boost::asio::ip::udp::endpoint> resolve(boost::asio::io_service& io_service,
-                                                      const std::string& host, uint16_t port);
-}

libraries/libfc/include/fc/network/udp_socket.hpp

@@ -1,39 +0,0 @@
-#pragma once
-#include <fc/utility.hpp>
-#include <memory>
-
-#include <boost/asio.hpp>
-
-namespace fc {
-  namespace ip {
-    class endpoint;
-    class address;
-  }
-
-  /**
-   *  The udp_socket class has reference semantics, all copies will
-   *  refer to the same underlying socket.
-   */
-  class udp_socket {
-    public:
-      udp_socket();
-      udp_socket( const udp_socket& s );
-      ~udp_socket();
-
-      void initialize(boost::asio::io_service &);
-      void open();
-      void send_to(const char* b, size_t l, boost::asio::ip::udp::endpoint &to);
-      void send_to(const std::shared_ptr<const char>& b, size_t l, boost::asio::ip::udp::endpoint &to);
-      void close();
-
-      void set_reuse_address(bool);
-
-      void connect(const boost::asio::ip::udp::endpoint& e);
-      const boost::asio::ip::udp::endpoint local_endpoint() const;
-
-    private:
-      class                impl;
-      std::shared_ptr<impl> my;
-  };
-
-}

libraries/libfc/src/log/gelf_appender.cpp

@@ -1,6 +1,3 @@
-#include <fc/network/udp_socket.hpp>
-#include <fc/network/ip.hpp>
-#include <fc/network/resolve.hpp>
 #include <fc/exception/exception.hpp>
 #include <fc/log/gelf_appender.hpp>
 #include <fc/reflect/variant.hpp>
@@ -10,6 +7,7 @@
 #include <fc/compress/zlib.hpp>
 #include <fc/log/logger_config.hpp>
 
+#include <boost/asio.hpp>
 #include <boost/lexical_cast.hpp>
 #include <iomanip>
 #include <iostream>
@@ -19,14 +17,6 @@
 
 namespace fc
 {
-  namespace detail
-  {
-    boost::asio::ip::udp::endpoint to_asio_ep( const fc::ip::endpoint& e )
-    {
-      return boost::asio::ip::udp::endpoint(boost::asio::ip::address_v4(e.get_address()), e.port() );
-    }
-  }
-
   const std::vector<std::string> gelf_appender::config::reserved_field_names  = {
       "_id",            // per GELF specification
       "_timestamp_ns",  // Remaining names all populated by appender
@@ -36,23 +26,21 @@ namespace fc
       "_method_name",
       "_thread_name",
       "_task_name"
-   };
+  };
 
-   const std::regex gelf_appender::config::user_field_name_pattern{"^_[\\w\\.\\-]*$"}; // per GELF specification
+  const std::regex gelf_appender::config::user_field_name_pattern{"^_[\\w\\.\\-]*$"}; // per GELF specification
 
   class gelf_appender::impl
   {
   public:
     using work_guard_t = boost::asio::executor_work_guard<boost::asio::io_context::executor_type>;
     config                                         cfg;
-    std::optional<boost::asio::ip::udp::endpoint>  gelf_endpoint;
     std::thread                                    thread;
     boost::asio::io_context                        io_context;
     work_guard_t work_guard =                      boost::asio::make_work_guard(io_context);
-    udp_socket                                     gelf_socket;
+    boost::asio::ip::udp::socket                   gelf_socket;
 
-
-    impl(const variant& c)
+    impl(const variant& c) : gelf_socket(io_context)
     {
       mutable_variant_object mvo;
       from_variant(c, mvo);
@@ -84,6 +72,35 @@ namespace fc
           thread.join();
        }
     }
+
+    static std::shared_ptr<std::vector<char>> make_new_bufer(boost::asio::const_buffer buf) {
+      const char* p = static_cast<const char*>(buf.data());
+      return std::make_shared<std::vector<char>>(p, p+buf.size());
+    }
+
+    static std::shared_ptr<std::vector<char>> make_new_bufer(const std::array<boost::asio::const_buffer, 2>& bufs) {
+      auto new_buf = std::make_shared<std::vector<char>>();
+      new_buf->reserve(bufs[0].size() + bufs[1].size());
+      for (int i = 0; i < 2; ++i) {
+          const char* p = static_cast<const char*>(bufs[i].data());
+          new_buf->insert(new_buf->end(), p, p + bufs[i].size());
+      }
+      return new_buf;
+    }
+
+    template <typename Buffers>
+    void send(Buffers&& bufs) {
+      boost::system::error_code ec;
+      gelf_socket.send(std::forward<Buffers>(bufs), 0, ec);
+      if (ec == boost::asio::error::would_block) {
+          auto new_buf = make_new_bufer(std::forward<Buffers>(bufs));
+          gelf_socket.async_send(boost::asio::buffer(*new_buf),
+                                    [new_buf](const boost::system::error_code& /*ec*/, std::size_t /*bytes_transferred*/) {
+                                        // Swallow errors.  Currently only used for GELF logging, so depend on local
+                                        // log to catch anything that doesn't make it across the network.
+                                    });
+      }
+    }
   };
 
   gelf_appender::gelf_appender(const variant& args) :
@@ -95,58 +112,40 @@ namespace fc
   {
     try
     {
-      try
-      {
-        // if it's a numeric address:port, this will parse it
-        my->gelf_endpoint = detail::to_asio_ep(ip::endpoint::from_string(my->cfg.endpoint));
+      if (my->cfg.endpoint.empty()) {
+        fprintf(stderr, "The logging destination is not specified\n");
+        return;
       }
-      catch (...)
-      {
-      }
-      if (!my->gelf_endpoint)
-      {
-        // couldn't parse as a numeric ip address, try resolving as a DNS name.
-        // This can yield, so don't do it in the catch block above
-        string::size_type colon_pos = my->cfg.endpoint.find(':');
-        try
-        {
-          FC_ASSERT(colon_pos != std::string::npos, "The logging destination port is not specified");
-          string port = my->cfg.endpoint.substr(colon_pos + 1);
 
-          string hostname = my->cfg.endpoint.substr( 0, colon_pos );
+      std::string_view endpoint = my->cfg.endpoint;
+      string::size_type colon_pos = endpoint.rfind(':');
+      FC_ASSERT(colon_pos != std::string::npos, "The logging destination port is not specified");
+      auto port = endpoint.substr(colon_pos + 1);
 
-          boost::asio::ip::udp::resolver resolver{ my->io_context };
-          auto endpoints = resolver.resolve(hostname, port);
+      auto hostname = (endpoint[0] == '[' && colon_pos >= 2)  ? endpoint.substr( 1, colon_pos-2 ) : endpoint.substr( 0, colon_pos );
 
-          if (endpoints.empty())
-              FC_THROW_EXCEPTION(unknown_host_exception, "The logging destination host name can not be resolved: ${hostname}",
-                                 ("hostname", hostname));
+      boost::asio::ip::udp::resolver resolver{ my->io_context };
+      auto endpoints = resolver.resolve(hostname, port);
 
-          my->gelf_endpoint = *endpoints.begin();
+      if (endpoints.empty())
+          FC_THROW_EXCEPTION(unknown_host_exception, "The logging destination host name can not be resolved: ${hostname}",
+                              ("hostname", std::string(hostname)));
+
+      my->gelf_socket.connect(*endpoints.begin());
+      std::cerr << "opened GELF socket to endpoint " << my->cfg.endpoint << "\n";
+      my->gelf_socket.non_blocking(true);
+
+      my->thread = std::thread([this] {
+        try {
+          fc::set_os_thread_name("gelf");
+          my->io_context.run();
+        } catch (std::exception& ex) {
+          fprintf(stderr, "GELF logger caught exception at %s:%d : %s\n", __FILE__, __LINE__, ex.what());
+        } catch (...) {
+          fprintf(stderr, "GELF logger caught exception unknown exception %s:%d\n", __FILE__, __LINE__);
         }
-        catch (const boost::bad_lexical_cast&)
-        {
-          FC_THROW("Bad port: ${port}", ("port", my->cfg.endpoint.substr(colon_pos + 1, my->cfg.endpoint.size())));
-        }
-      }
+      });
 
-      if (my->gelf_endpoint)
-      {
-        my->gelf_socket.initialize(my->io_context);
-        my->gelf_socket.open();
-        std::cerr << "opened GELF socket to endpoint " << my->cfg.endpoint << "\n";
-
-        my->thread = std::thread([this] {
-          try {
-            fc::set_os_thread_name("gelf");
-            my->io_context.run();
-          } catch (std::exception& ex) {
-            fprintf(stderr, "GELF logger caught exception at %s:%d : %s\n", __FILE__, __LINE__, ex.what());
-          } catch (...) {
-            fprintf(stderr, "GELF logger caught exception unknown exception %s:%d\n", __FILE__, __LINE__);
-          }
-        });
-      }
     }
     catch (...)
     {
@@ -221,57 +220,44 @@ namespace fc
 
     if (gelf_message_as_string.size() <= max_payload_size)
     {
-      // no need to split
-      std::shared_ptr<char> send_buffer(new char[gelf_message_as_string.size()],
-                                        [](char* p){ delete[] p; });
-      memcpy(send_buffer.get(), gelf_message_as_string.c_str(),
-             gelf_message_as_string.size());
-
-      my->gelf_socket.send_to(send_buffer, gelf_message_as_string.size(),
-                              *my->gelf_endpoint);
+      my->send(boost::asio::buffer(gelf_message_as_string));
     }
     else
     {
       // split the message
+      struct gelf_header {
+        uint8_t magic[2] = { 0x1e, 0x0f};
+        uint64_t message_id;
+        uint8_t seq = 0;
+        uint8_t count = 0;
+      } header;
+
       // we need to generate an 8-byte ID for this message.
       // city hash should do
-      uint64_t message_id = city_hash64(gelf_message_as_string.c_str(), gelf_message_as_string.size());
-      const unsigned header_length = 2 /* magic */ + 8 /* msg id */ + 1 /* seq */ + 1 /* count */;
-      const unsigned body_length = max_payload_size - header_length;
-      unsigned total_number_of_packets = (gelf_message_as_string.size() + body_length - 1) / body_length;
+      header.message_id = city_hash64(gelf_message_as_string.c_str(), gelf_message_as_string.size());
+      const unsigned body_length = max_payload_size - sizeof(header);
+      header.count = (gelf_message_as_string.size() + body_length - 1) / body_length;
       unsigned bytes_sent = 0;
-      unsigned number_of_packets_sent = 0;
+
       while (bytes_sent < gelf_message_as_string.size())
       {
         unsigned bytes_to_send = std::min((unsigned)gelf_message_as_string.size() - bytes_sent,
                                           body_length);
 
-        std::shared_ptr<char> send_buffer(new char[max_payload_size],
-                                          [](char* p){ delete[] p; });
-        char* ptr = send_buffer.get();
-        // magic number for chunked message
-        *(unsigned char*)ptr++ = 0x1e;
-        *(unsigned char*)ptr++ = 0x0f;
-
-        // message id
-        memcpy(ptr, (char*)&message_id, sizeof(message_id));
-        ptr += sizeof(message_id);
-
-        *(unsigned char*)(ptr++) = number_of_packets_sent;
-        *(unsigned char*)(ptr++) = total_number_of_packets;
-        memcpy(ptr, gelf_message_as_string.c_str() + bytes_sent,
-               bytes_to_send);
-        my->gelf_socket.send_to(send_buffer, header_length + bytes_to_send,
-                                *my->gelf_endpoint);
-        ++number_of_packets_sent;
+        std::array<boost::asio::const_buffer,2> bufs = {
+          boost::asio::const_buffer(&header, sizeof(header)),
+          boost::asio::const_buffer(gelf_message_as_string.c_str() + bytes_sent, bytes_to_send)
+        };
+        my->send(bufs);
+        ++header.seq;
         bytes_sent += bytes_to_send;
       }
-      FC_ASSERT(number_of_packets_sent == total_number_of_packets);
+      FC_ASSERT(header.seq == header.count);
     }
   }
 
   void gelf_appender::log(const log_message& message) {
-    if (!my->gelf_endpoint)
+    if (!my->thread.joinable())
       return;
 
     // use now() instead of context.get_timestamp() because log_message construction can include user provided long running calls

libraries/libfc/src/network/resolve.cpp

@@ -1,33 +0,0 @@
-#include <boost/asio.hpp>
-#include <fc/exception/exception.hpp>
-
-namespace fc
-{
-  std::vector<boost::asio::ip::udp::endpoint> resolve(boost::asio::io_service& io_service,
-                                                     const std::string& host, uint16_t port)
-  {
-    using q = boost::asio::ip::udp::resolver::query;
-    using b = boost::asio::ip::resolver_query_base;
-    boost::asio::ip::udp::resolver res(io_service);
-    boost::system::error_code ec;
-    auto ep = res.resolve(q(host, std::to_string(uint64_t(port)),
-                            b::address_configured | b::numeric_service), ec);
-    if(!ec)
-    {
-      std::vector<boost::asio::ip::udp::endpoint> eps;
-      while(ep != boost::asio::ip::udp::resolver::iterator())
-      {
-        if(ep->endpoint().address().is_v4())
-        {
-         eps.push_back(*ep);
-        }
-        // TODO: add support for v6
-        ++ep;
-      }
-      return eps;
-    }
-    FC_THROW_EXCEPTION(unknown_host_exception,
-                       "name resolution failed: ${reason}",
-                       ("reason", ec.message()));
-  }
-}

libraries/libfc/src/network/udp_socket.cpp

@@ -1,111 +0,0 @@
-#include <fc/network/udp_socket.hpp>
-#include <fc/network/ip.hpp>
-
-
-namespace fc
-{
-  class udp_socket::impl
-  {
-    public:
-      impl(){}
-      ~impl(){}
-
-      std::shared_ptr<boost::asio::ip::udp::socket> _sock;
-  };
-
-  udp_socket::udp_socket()
-    : my(new impl())
-  {
-  }
-
-  void udp_socket::initialize(boost::asio::io_service& service)
-  {
-    my->_sock.reset(new boost::asio::ip::udp::socket(service));
-  }
-
-  udp_socket::~udp_socket() 
-  {
-    try 
-    {
-      if(my->_sock)
-        my->_sock->close(); //close boost socket to make any pending reads run their completion handler
-    }
-    catch (...) //avoid destructor throw and likely this is just happening because socket wasn't open.
-    {
-    }
-  }
-
-  void udp_socket::send_to(const char* buffer, size_t length, boost::asio::ip::udp::endpoint& to)
-  {
-    try 
-    {
-      my->_sock->send_to(boost::asio::buffer(buffer, length), to);
-      return;
-    } 
-    catch(const boost::system::system_error& e)
-    {
-      if(e.code() == boost::asio::error::would_block)
-      {
-          auto send_buffer_ptr = std::make_shared<std::vector<char>>(buffer, buffer+length);
-          my->_sock->async_send_to(boost::asio::buffer(send_buffer_ptr.get(), length), to,
-                                   [send_buffer_ptr](const boost::system::error_code& /*ec*/, std::size_t /*bytes_transferred*/)
-          {
-            // Swallow errors.  Currently only used for GELF logging, so depend on local
-            // log to catch anything that doesn't make it across the network.
-          });
-       }
-       // All other exceptions ignored.
-    }
-  }
-
-  void udp_socket::send_to(const std::shared_ptr<const char>& buffer, size_t length,
-                           boost::asio::ip::udp::endpoint& to)
-  {
-    try
-    {
-      my->_sock->send_to(boost::asio::buffer(buffer.get(), length), to);
-      return;
-    } 
-    catch(const boost::system::system_error& e)
-    {
-      if(e.code() == boost::asio::error::would_block)
-      {
-          auto preserved_buffer_ptr = buffer;
-          my->_sock->async_send_to(boost::asio::buffer(preserved_buffer_ptr.get(), length), to,
-                                   [preserved_buffer_ptr](const boost::system::error_code& /*ec*/, std::size_t /*bytes_transferred*/)
-          {
-            // Swallow errors.  Currently only used for GELF logging, so depend on local
-            // log to catch anything that doesn't make it across the network.
-          });
-       }
-       // All other exceptions ignored.
-    }
-  }
-
-  void udp_socket::open()
-  {
-    my->_sock->open(boost::asio::ip::udp::v4());
-    my->_sock->non_blocking(true);
-  }
-
-  void udp_socket::close()
-  {
-    my->_sock->close();
-  }
-
-  const boost::asio::ip::udp::endpoint udp_socket::local_endpoint() const
-  {
-    return my->_sock->local_endpoint();
-  }
-
-  void udp_socket::connect(const boost::asio::ip::udp::endpoint& e)
-  {
-    my->_sock->connect(e);
-  }
-
-  void udp_socket::set_reuse_address( bool s )
-  {
-    my->_sock->set_option( boost::asio::ip::udp::socket::reuse_address(s) );
-  }
-
-}

libraries/state_history/include/eosio/state_history/log.hpp

@@ -550,6 +550,9 @@ class state_history_log {
          fc::raw::pack(log, num_blocks_in_log);
       }
 
+      log.flush();
+      index.flush();
+
       auto partition_config = std::get_if<state_history::partition_config>(&config);
       if (partition_config && block_num % partition_config->stride == 0) {
          split_log();

plugins/chain_api_plugin/chain_api_plugin.cpp

@@ -75,7 +75,7 @@ void chain_api_plugin::plugin_startup() {
    ro_api.set_shorten_abi_errors( !http_plugin::verbose_errors() );
 
    _http_plugin.add_api( {
-      CHAIN_RO_CALL(get_info, 200, http_params_types::no_params)}, appbase::exec_queue::read_only_trx_safe, appbase::priority::medium_high);
+      CHAIN_RO_CALL(get_info, 200, http_params_types::no_params)}, appbase::exec_queue::read_only, appbase::priority::medium_high);
    _http_plugin.add_api({
       CHAIN_RO_CALL(get_activated_protocol_features, 200, http_params_types::possible_no_params),
       CHAIN_RO_CALL(get_block_info, 200, http_params_types::params_required),
@@ -96,19 +96,19 @@ void chain_api_plugin::plugin_startup() {
       CHAIN_RO_CALL(get_scheduled_transactions, 200, http_params_types::params_required),
       CHAIN_RO_CALL(get_required_keys, 200, http_params_types::params_required),
       CHAIN_RO_CALL(get_transaction_id, 200, http_params_types::params_required),
+      // transaction related APIs will be posted to read_write queue after keys are recovered, they are safe to run in parallel until they post to the read_write queue
       CHAIN_RO_CALL_ASYNC(send_read_only_transaction, chain_apis::read_only::send_read_only_transaction_results, 200, http_params_types::params_required),
       CHAIN_RO_CALL_ASYNC(compute_transaction, chain_apis::read_only::compute_transaction_results, 200, http_params_types::params_required),
-      // transaction related APIs will be posted to general queue after keys are recovered
       CHAIN_RW_CALL_ASYNC(push_transaction, chain_apis::read_write::push_transaction_results, 202, http_params_types::params_required),
       CHAIN_RW_CALL_ASYNC(push_transactions, chain_apis::read_write::push_transactions_results, 202, http_params_types::params_required),
       CHAIN_RW_CALL_ASYNC(send_transaction, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required),
       CHAIN_RW_CALL_ASYNC(send_transaction2, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required)
-   }, appbase::exec_queue::read_only_trx_safe);
+   }, appbase::exec_queue::read_only);
 
    // Not safe to run in parallel with read-only transactions
    _http_plugin.add_api({
       CHAIN_RW_CALL_ASYNC(push_block, chain_apis::read_write::push_block_results, 202, http_params_types::params_required)
-   }, appbase::exec_queue::general, appbase::priority::medium_low );
+   }, appbase::exec_queue::read_write, appbase::priority::medium_low );
 
    if (chain.account_queries_enabled()) {
       _http_plugin.add_async_api({
@@ -124,7 +124,7 @@ void chain_api_plugin::plugin_startup() {
    if (chain.transaction_finality_status_enabled()) {
       _http_plugin.add_api({
          CHAIN_RO_CALL_WITH_400(get_transaction_status, 200, http_params_types::params_required),
-      }, appbase::exec_queue::read_only_trx_safe);
+      }, appbase::exec_queue::read_only);
    }
 
    _http_plugin.add_api({
@@ -160,7 +160,7 @@ void chain_api_plugin::plugin_startup() {
            }
         }
       }
-   }, appbase::exec_queue::read_only_trx_safe);
+   }, appbase::exec_queue::read_only);
 }
 
 void chain_api_plugin::plugin_shutdown() {}

plugins/db_size_api_plugin/db_size_api_plugin.cpp

@@ -28,7 +28,7 @@ using namespace eosio;
 void db_size_api_plugin::plugin_startup() {
    app().get_plugin<http_plugin>().add_api({
        CALL_WITH_400(db_size, this, get,  INVOKE_R_V(this, get), 200),
-   }, appbase::exec_queue::read_only_trx_safe);
+   }, appbase::exec_queue::read_only);
 }
 
 db_size_stats db_size_api_plugin::get() {

plugins/http_plugin/http_plugin.cpp

@@ -28,7 +28,7 @@ namespace eosio {
    using std::regex;
    using boost::asio::ip::tcp;
    using std::shared_ptr;
-   
+
    static http_plugin_defaults current_http_plugin_defaults;
    static bool verbose_http_errors = false;
 
@@ -36,6 +36,10 @@ namespace eosio {
       current_http_plugin_defaults = config;
    }
 
+   std::string http_plugin::get_server_header() {
+      return current_http_plugin_defaults.server_header;
+   }
+
    using http_plugin_impl_ptr = std::shared_ptr<class http_plugin_impl>;
 
    class http_plugin_impl : public std::enable_shared_from_this<http_plugin_impl> {
@@ -47,9 +51,9 @@ namespace eosio {
 
          http_plugin_impl& operator=(const http_plugin_impl&) = delete;
          http_plugin_impl& operator=(http_plugin_impl&&) = delete;
-         
+
          std::optional<tcp::endpoint>  listen_endpoint;
-         
+
          std::optional<asio::local::stream_protocol::endpoint> unix_endpoint;
 
          shared_ptr<beast_http_listener<plain_session, tcp, tcp_socket_t > >  beast_server;
@@ -272,7 +276,7 @@ namespace eosio {
 
          my->plugin_state->server_header = current_http_plugin_defaults.server_header;
 
-         
+
          //watch out for the returns above when adding new code here
       } FC_LOG_AND_RETHROW()
    }
@@ -309,7 +313,7 @@ namespace eosio {
             if(my->unix_endpoint) {
                try {
                   my->create_beast_server(true);
-                  
+
                   my->beast_unix_server->listen(*my->unix_endpoint);
                   my->beast_unix_server->start_accept();
                } catch ( const fc::exception& e ){
@@ -334,8 +338,8 @@ namespace eosio {
                      handle_exception("node", "get_supported_apis", body.empty() ? "{}" : body, cb);
                   }
                }
-            }}, appbase::exec_queue::read_only_trx_safe);
-            
+            }}, appbase::exec_queue::read_only);
+
          } catch (...) {
             fc_elog(logger(), "http_plugin startup fails, shutting down");
             app().quit();
@@ -454,9 +458,9 @@ namespace eosio {
    fc::microseconds http_plugin::get_max_response_time()const {
       return my->plugin_state->max_response_time;
    }
-   
+
    size_t http_plugin::get_max_body_size()const {
       return my->plugin_state->max_body_size;
    }
-   
+
 }

plugins/http_plugin/include/eosio/http_plugin/http_plugin.hpp

@@ -78,6 +78,7 @@ namespace eosio {
 
         //must be called before initialize
         static void set_defaults(const http_plugin_defaults& config);
+        static std::string get_server_header();
 
         APPBASE_PLUGIN_REQUIRES()
         void set_program_options(options_description&, options_description& cfg) override;
@@ -121,7 +122,7 @@ namespace eosio {
         void register_metrics_listener(chain::plugin_interface::metrics_listener listener);
 
         size_t get_max_body_size()const;
-      
+
    private:
         std::shared_ptr<class http_plugin_impl> my;
    };
@@ -175,7 +176,7 @@ namespace eosio {
    };
 
    /**
-    * @brief Used to trim whitespace from body. 
+    * @brief Used to trim whitespace from body.
     * Returned string_view valid only for lifetime of body
     */
    inline std::string_view make_trimmed_string_view(const std::string& body) {

plugins/http_plugin/tests/unit_tests.cpp

@@ -52,7 +52,7 @@ public:
                   cb(200, fc::time_point::maximum(), fc::variant(ok ? string("yes") : string("no")));
                }
             },
-         }, appbase::exec_queue::general);
+         }, appbase::exec_queue::read_write);
    }
 
 private:

plugins/net_api_plugin/net_api_plugin.cpp

@@ -53,11 +53,7 @@ void net_api_plugin::plugin_startup() {
    ilog("starting net_api_plugin");
    // lifetime of plugin is lifetime of application
    auto& net_mgr = app().get_plugin<net_plugin>();
-   app().get_plugin<http_plugin>().add_api({
-    //   CALL(net, net_mgr, set_timeout,
-    //        INVOKE_V_R(net_mgr, set_timeout, int64_t), 200),
-    //   CALL(net, net_mgr, sign_transaction,
-    //        INVOKE_R_R_R_R(net_mgr, sign_transaction, chain::signed_transaction, flat_set<public_key_type>, chain::chain_id_type), 201),
+   app().get_plugin<http_plugin>().add_async_api({
        CALL_WITH_400(net, net_mgr, connect,
             INVOKE_R_R(net_mgr, connect, std::string), 201),
        CALL_WITH_400(net, net_mgr, disconnect,
@@ -66,9 +62,7 @@ void net_api_plugin::plugin_startup() {
             INVOKE_R_R(net_mgr, status, std::string), 201),
        CALL_WITH_400(net, net_mgr, connections,
             INVOKE_R_V(net_mgr, connections), 201),
-    //   CALL(net, net_mgr, open,
-    //        INVOKE_V_R(net_mgr, open, std::string), 200),
-   }, appbase::exec_queue::read_only_trx_safe, appbase::priority::medium_high);
+   } );
 }
 
 void net_api_plugin::plugin_initialize(const variables_map& options) {

plugins/net_plugin/net_plugin.cpp

@@ -3225,22 +3225,23 @@ namespace eosio {
             return;
          }
 
-         bool valid_block_header = !!bsp;
 
-         if( valid_block_header ) {
+         uint32_t block_num = bsp ? bsp->block_num : 0;
+
+         if( block_num != 0 ) {
             fc_dlog( logger, "validated block header, broadcasting immediately, connection ${cid}, blk num = ${num}, id = ${id}",
-                     ("cid", cid)("num", bsp->block_num)("id", bsp->id) );
+                     ("cid", cid)("num", block_num)("id", bsp->id) );
             my_impl->dispatcher->add_peer_block( bsp->id, cid ); // no need to send back to sender
             my_impl->dispatcher->bcast_block( bsp->block, bsp->id );
          }
 
-         app().executor().post(priority::medium, exec_queue::general, [ptr{std::move(ptr)}, bsp{std::move(bsp)}, id, c{std::move(c)}]() mutable {
+         app().executor().post(priority::medium, exec_queue::read_write, [ptr{std::move(ptr)}, bsp{std::move(bsp)}, id, c{std::move(c)}]() mutable {
             c->process_signed_block( id, std::move(ptr), std::move(bsp) );
          });
 
-         if( valid_block_header ) {
+         if( block_num != 0 ) {
             // ready to process immediately, so signal producer to interrupt start_block
-            my_impl->producer_plug->received_block();
+            my_impl->producer_plug->received_block(block_num);
          }
       });
    }

plugins/producer_api_plugin/producer.swagger.yaml

@@ -346,7 +346,7 @@ paths:
   /producer/schedule_snapshot:
     post:
       summary: schedule_snapshot
-      description: Submits a request to generate a schedule for automated snapshot with given parameters. If request body is empty, triest to execute snapshot immediately. Returns error when unable to create snapshot.
+      description: Submits a request to automatically generate snapshots according to a schedule specified with given parameters. If request body is empty, schedules immediate snapshot generation. Returns error when unable to accept schedule.
       operationId: schedule_snapshot
       requestBody:
         content:
@@ -377,6 +377,12 @@ paths:
             application/json:
               schema:
                 type: object
+                required:
+                    - snapshot_request_id
+                    - block_spacing
+                    - start_block_num
+                    - end_block_num
+                    - snapshot_description
                 properties:
                   snapshot_request_id:
                     type: integer
@@ -405,8 +411,8 @@ paths:
   /producer/get_snapshot_requests:
     post:
       summary: get_snapshot_requests
-      description: Returns a list of scheduled snapshots
-      operationId: get_snapshot_status
+      description: Returns a list of scheduled snapshots.
+      operationId: get_snapshot_requests
       responses:
         "201":
           description: OK
@@ -417,9 +423,16 @@ paths:
                 properties:
                   snapshot_requests:
                     type: array
-                    description: An array of scheduled snapshots
+                    description: An array of scheduled snapshot requests
                     items:
                       type: object
+                      required:
+                          - snapshot_request_id
+                          - block_spacing
+                          - start_block_num
+                          - end_block_num
+                          - snapshot_description
+                          - pending_snapshots
                       properties:
                         snapshot_request_id:
                           type: integer
@@ -444,6 +457,12 @@ paths:
                           description: List of pending snapshots
                           items:
                             type: object
+                            required:
+                                - head_block_id
+                                - head_block_num
+                                - head_block_time
+                                - version
+                                - snapshot_name
                             properties:
                               head_block_id:
                                 $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
@@ -472,7 +491,7 @@ paths:
   /producer/unschedule_snapshot:
     post:
       summary: unschedule_snapshot
-      description: Submits a request to remove identified by id recurring snapshot from schedule. Returns error when unable to create unschedule.
+      description: Removes snapshot request identified by id. Returns error if referenced snapshot request does not exist.
       operationId: unschedule_snapshot
       requestBody:
         required: true
@@ -491,6 +510,12 @@ paths:
             application/json:
               schema:
                 type: object
+                required:
+                  - snapshot_request_id
+                  - block_spacing
+                  - start_block_num
+                  - end_block_num
+                  - snapshot_description
                 properties:
                   snapshot_request_id:
                     type: integer

plugins/producer_api_plugin/producer_api_plugin.cpp

@@ -118,9 +118,11 @@ void producer_api_plugin::plugin_startup() {
             INVOKE_R_R(producer, get_account_ram_corrections, producer_plugin::get_account_ram_corrections_params), 201),
        CALL_WITH_400(producer, producer, get_unapplied_transactions,
                      INVOKE_R_R_D(producer, get_unapplied_transactions, producer_plugin::get_unapplied_transactions_params), 200),
-   }, appbase::exec_queue::read_only_trx_safe, appbase::priority::medium_high);
+       CALL_WITH_400(producer, producer, get_snapshot_requests,
+                     INVOKE_R_V(producer, get_snapshot_requests), 201),
+   }, appbase::exec_queue::read_only, appbase::priority::medium_high);
 
-   // Not safe to run in parallel with read-only transactions
+   // Not safe to run in parallel
    app().get_plugin<http_plugin>().add_api({
        CALL_WITH_400(producer, producer, pause,
             INVOKE_V_V(producer, pause), 201),
@@ -137,16 +139,14 @@ void producer_api_plugin::plugin_startup() {
        CALL_ASYNC(producer, producer, create_snapshot, producer_plugin::snapshot_information,
             INVOKE_R_V_ASYNC(producer, create_snapshot), 201),
        CALL_WITH_400(producer, producer, schedule_snapshot,
-            INVOKE_V_R_II(producer, schedule_snapshot, producer_plugin::snapshot_request_information), 201),
-       CALL_WITH_400(producer, producer, get_snapshot_requests,
-            INVOKE_R_V(producer, get_snapshot_requests), 201),
+            INVOKE_R_R_II(producer, schedule_snapshot, producer_plugin::snapshot_request_information), 201),
        CALL_WITH_400(producer, producer, unschedule_snapshot,
-            INVOKE_V_R(producer, unschedule_snapshot, producer_plugin::snapshot_request_id_information), 201),
+            INVOKE_R_R(producer, unschedule_snapshot, producer_plugin::snapshot_request_id_information), 201),
        CALL_WITH_400(producer, producer, get_integrity_hash,
             INVOKE_R_V(producer, get_integrity_hash), 201),
        CALL_WITH_400(producer, producer, schedule_protocol_feature_activations,
             INVOKE_V_R(producer, schedule_protocol_feature_activations, producer_plugin::scheduled_protocol_feature_activations), 201),
-   }, appbase::exec_queue::general, appbase::priority::medium_high);
+   }, appbase::exec_queue::read_write, appbase::priority::medium_high);
 }
 
 void producer_api_plugin::plugin_initialize(const variables_map& options) {

plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp

@@ -94,8 +94,11 @@ public:
       uint32_t snapshot_request_id = 0;
    };
 
+   struct snapshot_schedule_result : public snapshot_request_id_information, public snapshot_request_information {
+   };
+
    struct snapshot_schedule_information : public snapshot_request_id_information, public snapshot_request_information {
-      std::optional<std::vector<snapshot_information>> pending_snapshots;
+      std::vector<snapshot_information> pending_snapshots;
    };
 
    struct get_snapshot_requests_result {
@@ -159,8 +162,8 @@ public:
    integrity_hash_information get_integrity_hash() const;
 
    void create_snapshot(next_function<snapshot_information> next);
-   void schedule_snapshot(const snapshot_request_information& schedule);
-   void unschedule_snapshot(const snapshot_request_id_information& schedule);
+   snapshot_schedule_result schedule_snapshot(const snapshot_request_information& schedule);
+   snapshot_schedule_result unschedule_snapshot(const snapshot_request_id_information& schedule);
    get_snapshot_requests_result get_snapshot_requests() const;
 
    scheduled_protocol_feature_activations get_scheduled_protocol_feature_activations() const;
@@ -202,11 +205,14 @@ public:
    void register_metrics_listener(metrics_listener listener);
 
    // thread-safe, called when a new block is received
-   void received_block();
+   void received_block(uint32_t block_num);
 
-const std::set<account_name>& producer_accounts() const;
+   const std::set<account_name>& producer_accounts() const;
 
+   static void set_test_mode(bool m) { test_mode_ = m; }
  private:
+   inline static bool test_mode_{false}; // to be moved into appbase (application_base)
+
    std::shared_ptr<class producer_plugin_impl> my;
 };
 
@@ -221,6 +227,7 @@ FC_REFLECT(eosio::producer_plugin::snapshot_request_information, (block_spacing)
 FC_REFLECT(eosio::producer_plugin::snapshot_request_id_information, (snapshot_request_id))
 FC_REFLECT(eosio::producer_plugin::get_snapshot_requests_result, (snapshot_requests))
 FC_REFLECT_DERIVED(eosio::producer_plugin::snapshot_schedule_information, (eosio::producer_plugin::snapshot_request_id_information)(eosio::producer_plugin::snapshot_request_information), (pending_snapshots))
+FC_REFLECT_DERIVED(eosio::producer_plugin::snapshot_schedule_result, (eosio::producer_plugin::snapshot_request_id_information)(eosio::producer_plugin::snapshot_request_information),)
 FC_REFLECT(eosio::producer_plugin::scheduled_protocol_feature_activations, (protocol_features_to_activate))
 FC_REFLECT(eosio::producer_plugin::get_supported_protocol_features_params, (exclude_disabled)(exclude_unactivatable))
 FC_REFLECT(eosio::producer_plugin::get_account_ram_corrections_params, (lower_bound)(upper_bound)(limit)(reverse))

plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp

@@ -45,7 +45,7 @@ private:
    void x_serialize() {
       auto& vec = _snapshot_requests.get<as_vector>();
       std::vector<producer_plugin::snapshot_schedule_information> sr(vec.begin(), vec.end());
-      _snapshot_db << sr;   
+      _snapshot_db << sr;
    }
 
 public:
@@ -54,7 +54,7 @@ public:
    // snapshot_scheduler_listener
    void on_start_block(uint32_t height) {
       bool serialize_needed  = false;
-      bool snapshot_executed = false; 
+      bool snapshot_executed = false;
 
       auto execute_snapshot_with_log = [this, height, &snapshot_executed](const auto & req) {
          // one snapshot per height
@@ -64,18 +64,18 @@ public:
                ("end_block_num",   req.end_block_num)
                ("block_spacing",   req.block_spacing)
                ("height",          height));
-               
+
             execute_snapshot(req.snapshot_request_id);
             snapshot_executed = true;
          }
-      };    
+      };
 
       std::vector<uint32_t> unschedule_snapshot_request_ids;
       for(const auto& req: _snapshot_requests.get<0>()) {
          // -1 since its called from start block
          bool recurring_snapshot =  req.block_spacing &&  (height >= req.start_block_num + 1) && (!((height - req.start_block_num - 1) % req.block_spacing));
          bool onetime_snapshot   = (!req.block_spacing) && (height == req.start_block_num + 1);
-        
+
          // assume "asap" for snapshot with missed/zero start, it can have spacing
          if(!req.start_block_num) {
             // update start_block_num with current height only if this is recurring
@@ -84,16 +84,16 @@ public:
                auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
                auto it = snapshot_by_id.find(req.snapshot_request_id);
                _snapshot_requests.modify(it, [&height](auto& p) { p.start_block_num = height - 1; });
-               serialize_needed = true;              
+               serialize_needed = true;
             }
             execute_snapshot_with_log(req);
          } else if(recurring_snapshot || onetime_snapshot) {
             execute_snapshot_with_log(req);
-         }       
+         }
 
          // cleanup - remove expired (or invalid) request
-         if((!req.start_block_num && !req.block_spacing) ||         
-            (!req.block_spacing && height >= (req.start_block_num + 1)) || 
+         if((!req.start_block_num && !req.block_spacing) ||
+            (!req.block_spacing && height >= (req.start_block_num + 1)) ||
             (req.end_block_num > 0 && height >= (req.end_block_num + 1))) {
             unschedule_snapshot_request_ids.push_back(req.snapshot_request_id);
          }
@@ -108,7 +108,7 @@ public:
    }
 
    // snapshot_scheduler_handler
-   void schedule_snapshot(const producer_plugin::snapshot_request_information& sri) {
+   producer_plugin::snapshot_schedule_result schedule_snapshot(const producer_plugin::snapshot_request_information& sri) {
       auto& snapshot_by_value = _snapshot_requests.get<by_snapshot_value>();
       auto existing = snapshot_by_value.find(std::make_tuple(sri.block_spacing, sri.start_block_num, sri.end_block_num));
       EOS_ASSERT(existing == snapshot_by_value.end(), chain::duplicate_snapshot_request, "Duplicate snapshot request");
@@ -122,19 +122,27 @@ public:
          }
       }
 
-      _snapshot_requests.emplace(producer_plugin::snapshot_schedule_information {{_snapshot_id++},{sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description},{}});
+      _snapshot_requests.emplace(producer_plugin::snapshot_schedule_information {{_snapshot_id++}, {sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description},{}});
       x_serialize();
+
+      // returning snapshot_schedule_result
+      return  producer_plugin::snapshot_schedule_result{{_snapshot_id - 1}, {sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description}};
    }
 
-   virtual void unschedule_snapshot(uint32_t sri) {
+   producer_plugin::snapshot_schedule_result unschedule_snapshot(uint32_t sri) {
       auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
       auto existing = snapshot_by_id.find(sri);
       EOS_ASSERT(existing != snapshot_by_id.end(), chain::snapshot_request_not_found, "Snapshot request not found");
+
+      producer_plugin::snapshot_schedule_result result{{existing->snapshot_request_id}, {existing->block_spacing, existing->start_block_num, existing->end_block_num, existing->snapshot_description}};
       _snapshot_requests.erase(existing);
       x_serialize();
+
+      // returning snapshot_schedule_result
+      return result;
    }
 
-   virtual producer_plugin::get_snapshot_requests_result get_snapshot_requests() {
+   producer_plugin::get_snapshot_requests_result get_snapshot_requests() {
       producer_plugin::get_snapshot_requests_result result;
       auto& asvector = _snapshot_requests.get<as_vector>();
       result.snapshot_requests.reserve(asvector.size());
@@ -160,11 +168,8 @@ public:
       auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
       auto  snapshot_req   = snapshot_by_id.find(_inflight_sid);
       if (snapshot_req != snapshot_by_id.end()) {
-         _snapshot_requests.modify(snapshot_req, [&si](auto& p) { 
-            if (!p.pending_snapshots) {
-               p.pending_snapshots = std::vector<producer_plugin::snapshot_information>();
-            }
-            p.pending_snapshots->emplace_back(si);       
+         _snapshot_requests.modify(snapshot_req, [&si](auto& p) {
+            p.pending_snapshots.emplace_back(si);
          });
       }
    }
@@ -193,14 +198,12 @@ public:
             auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
             auto  snapshot_req   = snapshot_by_id.find(srid);
 
-            if (snapshot_req != snapshot_by_id.end()) {               
-               if (auto pending = snapshot_req->pending_snapshots; pending) {                 
-                  auto it = std::remove_if(pending->begin(), pending->end(), [&snapshot_info](const producer_plugin::snapshot_information & s){ return s.head_block_num <=  snapshot_info.head_block_num; });
-                  pending->erase(it, pending->end());
-                  _snapshot_requests.modify(snapshot_req, [&pending](auto& p) { 
-                     p.pending_snapshots = std::move(pending);
-                  });
-               }
+            if (snapshot_req != snapshot_by_id.end()) {
+               _snapshot_requests.modify(snapshot_req, [&](auto& p) {
+                  auto & pending = p.pending_snapshots;
+                  auto it = std::remove_if(pending.begin(), pending.end(), [&snapshot_info](const producer_plugin::snapshot_information & s){ return s.head_block_num <=  snapshot_info.head_block_num; });
+                  pending.erase(it, pending.end());
+               });
             }
          }
       };

plugins/producer_plugin/producer_plugin.cpp

@@ -14,6 +14,7 @@
 #include <fc/io/json.hpp>
 #include <fc/log/logger_config.hpp>
 #include <fc/scoped_exit.hpp>
+#include <fc/time.hpp>
 
 #include <boost/asio.hpp>
 #include <boost/date_time/posix_time/posix_time.hpp>
@@ -27,7 +28,6 @@
 #include <boost/multi_index/member.hpp>
 #include <boost/multi_index/hashed_index.hpp>
 #include <boost/multi_index/ordered_index.hpp>
-#include <boost/multi_index/random_access_index.hpp>
 #include <boost/signals2/connection.hpp>
 
 namespace bmi = boost::multi_index;
@@ -256,39 +256,53 @@ struct block_time_tracker {
       block_idle_time += idle;
    }
 
-   void add_fail_time( const fc::microseconds& fail_time ) {
-      trx_fail_time += fail_time;
-      ++trx_fail_num;
+   void add_fail_time( const fc::microseconds& fail_time, bool is_transient ) {
+      if( is_transient ) {
+         // transient time includes both success and fail time
+         transient_trx_time += fail_time;
+         ++transient_trx_num;
+      } else {
+         trx_fail_time += fail_time;
+         ++trx_fail_num;
+      }
    }
 
-   void add_success_time( const fc::microseconds& time ) {
-      trx_success_time += time;
-      ++trx_success_num;
+   void add_success_time( const fc::microseconds& time, bool is_transient ) {
+      if( is_transient ) {
+         transient_trx_time += time;
+         ++transient_trx_num;
+      } else {
+         trx_success_time += time;
+         ++trx_success_num;
+      }
    }
 
    void report( const fc::time_point& idle_trx_time, uint32_t block_num ) {
       if( _log.is_enabled( fc::log_level::debug ) ) {
          auto now = fc::time_point::now();
          add_idle_time( now - idle_trx_time );
-         fc_dlog( _log, "Block #${n} trx idle: ${i}us out of ${t}us, success: ${sn}, ${s}us, fail: ${fn}, ${f}us, other: ${o}us",
+         fc_dlog( _log, "Block #${n} trx idle: ${i}us out of ${t}us, success: ${sn}, ${s}us, fail: ${fn}, ${f}us, transient: ${trans_trx_num}, ${trans_trx_time}us, other: ${o}us",
                   ("n", block_num)
                   ("i", block_idle_time)("t", now - clear_time)("sn", trx_success_num)("s", trx_success_time)
                   ("fn", trx_fail_num)("f", trx_fail_time)
-                  ("o", (now - clear_time) - block_idle_time - trx_success_time - trx_fail_time) );
+                  ("trans_trx_num", transient_trx_num)("trans_trx_time", transient_trx_time)
+                  ("o", (now - clear_time) - block_idle_time - trx_success_time - trx_fail_time - transient_trx_time) );
       }
    }
 
    void clear() {
-      block_idle_time = trx_fail_time = trx_success_time = fc::microseconds{};
-      trx_fail_num = trx_success_num = 0;
+      block_idle_time = trx_fail_time = trx_success_time = transient_trx_time = fc::microseconds{};
+      trx_fail_num = trx_success_num = transient_trx_num = 0;
       clear_time = fc::time_point::now();
    }
 
    fc::microseconds block_idle_time;
    uint32_t trx_success_num = 0;
    uint32_t trx_fail_num = 0;
+   uint32_t transient_trx_num = 0;
    fc::microseconds trx_success_time;
    fc::microseconds trx_fail_time;
+   fc::microseconds transient_trx_time;
    fc::time_point clear_time{fc::time_point::now()};
 };
 
@@ -299,8 +313,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       producer_plugin_impl(boost::asio::io_service& io)
       :_timer(io)
       ,_transaction_ack_channel(app().get_channel<compat::channels::transaction_ack>())
-      ,_ro_write_window_timer(io)
-      ,_ro_read_window_timer(io)
+      ,_ro_timer(io)
       {
       }
 
@@ -324,9 +337,9 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       push_result push_transaction( const fc::time_point& block_deadline,
                                     const transaction_metadata_ptr& trx,
                                     bool api_trx, bool return_failure_trace,
-                                    next_function<transaction_trace_ptr> next );
+                                    const next_function<transaction_trace_ptr>& next );
       push_result handle_push_result( const transaction_metadata_ptr& trx,
-                                      next_function<transaction_trace_ptr> next,
+                                      const next_function<transaction_trace_ptr>& next,
                                       const fc::time_point& start,
                                       const chain::controller& chain,
                                       const transaction_trace_ptr& trace,
@@ -356,7 +369,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       named_thread_pool<struct prod>                            _thread_pool;
 
       std::atomic<int32_t>                                      _max_transaction_time_ms; // modified by app thread, read by net_plugin thread pool
-      std::atomic<bool>                                         _received_block{false}; // modified by net_plugin thread pool and app thread
+      std::atomic<uint32_t>                                     _received_block{0}; // modified by net_plugin thread pool
       fc::microseconds                                          _max_irreversible_block_age_us;
       int32_t                                                   _produce_time_offset_us = 0;
       int32_t                                                   _last_block_time_offset_us = 0;
@@ -411,42 +424,65 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
 
       // async snapshot scheduler
       snapshot_scheduler _snapshot_scheduler;
-      
+
       // ro for read-only
       struct ro_trx_t {
-         packed_transaction_ptr  trx;
-         next_func_t             next;
+         transaction_metadata_ptr trx;
+         next_func_t              next;
       };
-      // The queue storing read-only transactions to be executed by read-only threads
-      struct ro_trx_queue_t {
-         std::mutex              mtx;
-         std::deque<ro_trx_t>    queue;
+      // The queue storing previously exhausted read-only transactions to be re-executed by read-only threads
+      // thread-safe
+      class ro_trx_queue_t {
+      public:
+         void push_front(ro_trx_t&& t) {
+            std::lock_guard g(mtx);
+            queue.push_front(std::move(t));
+         }
+
+         bool empty() const {
+            std::lock_guard g(mtx);
+            return queue.empty();
+         }
+
+         bool pop_front(ro_trx_t& t) {
+            std::unique_lock g(mtx);
+            if (queue.empty())
+               return false;
+            t = queue.front();
+            queue.pop_front();
+            return true;
+         }
+
+      private:
+         mutable std::mutex      mtx;
+         deque<ro_trx_t>         queue;  // boost deque which is faster than std::deque
       };
-      uint16_t                        _ro_thread_pool_size{ 0 };
-      static constexpr uint16_t       _ro_max_eos_vm_oc_threads_allowed{ 8 }; // Due to uncertainty to get total virtual memory size on a 5-level paging system, set a hard limit
+
+      uint32_t                        _ro_thread_pool_size{ 0 };
+       // Due to uncertainty to get total virtual memory size on a 5-level paging system for eos-vm-oc and
+       // possible memory exhuastion for large number of contract usage for non-eos-vm-oc, set a hard limit
+      static constexpr uint32_t       _ro_max_threads_allowed{ 8 };
       named_thread_pool<struct read>  _ro_thread_pool;
       fc::microseconds                _ro_write_window_time_us{ 200000 };
       fc::microseconds                _ro_read_window_time_us{ 60000 };
       static constexpr fc::microseconds _ro_read_window_minimum_time_us{ 10000 };
       fc::microseconds                _ro_read_window_effective_time_us{ 0 }; // calculated during option initialization
-      boost::asio::deadline_timer     _ro_write_window_timer;
-      boost::asio::deadline_timer     _ro_read_window_timer;
+      std::atomic<int64_t>            _ro_all_threads_exec_time_us; // total time spent by all threads executing transactions. use atomic for simplicity and performance
+      fc::time_point                  _ro_read_window_start_time;
+      fc::time_point                  _ro_window_deadline; // only modified on app thread, read-window deadline or write-window deadline
+      boost::asio::deadline_timer     _ro_timer; // only accessible from the main thread
       fc::microseconds                _ro_max_trx_time_us{ 0 }; // calculated during option initialization
-      ro_trx_queue_t                  _ro_trx_queue;
-      std::atomic<uint32_t>            _ro_num_active_trx_exec_tasks{ 0 };
-      std::vector<std::future<bool>>  _ro_trx_exec_tasks_fut;
+      ro_trx_queue_t                  _ro_exhausted_trx_queue;
+      std::atomic<uint32_t>           _ro_num_active_exec_tasks{ 0 };
+      std::vector<std::future<bool>>  _ro_exec_tasks_fut;
 
       void start_write_window();
       void switch_to_write_window();
       void switch_to_read_window();
-      bool read_only_trx_execution_task();
-      bool process_read_only_transaction(const packed_transaction_ptr& trx,
-                                         const next_function<transaction_trace_ptr>& next,
-                                         const fc::time_point& read_window_start_time);
-      bool push_read_only_transaction(const transaction_metadata_ptr& trx,
-                                      next_function<transaction_trace_ptr> next,
-                                      const fc::time_point& read_window_start_time);
-      
+      bool read_only_execution_task(uint32_t pending_block_num);
+      void repost_exhausted_transactions(const fc::time_point& deadline);
+      bool push_read_only_transaction(transaction_metadata_ptr trx, next_function<transaction_trace_ptr> next);
+
       void consider_new_watermark( account_name producer, uint32_t block_num, block_timestamp_type timestamp) {
          auto itr = _producer_watermarks.find( producer );
          if( itr != _producer_watermarks.end() ) {
@@ -480,8 +516,9 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       }
 
       void on_irreversible_block( const signed_block_ptr& lib ) {
-         _irreversible_block_time = lib->timestamp.to_time_point();
          const chain::controller& chain = chain_plug->chain();
+         EOS_ASSERT(chain.is_write_window(), producer_exception, "write window is expected for on_irreversible_block signal");
+         _irreversible_block_time = lib->timestamp.to_time_point();
 
          // promote any pending snapshots
          auto& snapshots_by_height = _pending_snapshot_index.get<by_height>();
@@ -639,12 +676,12 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
                                          transaction_metadata::trx_type trx_type,
                                          bool return_failure_traces,
                                          next_function<transaction_trace_ptr> next) {
-         if ( trx_type == transaction_metadata::trx_type::read_only && _ro_thread_pool_size > 0 ) {
-            // Parallel read-only trx execution enabled.
-            // Store the transaction in read-only-trx queue so that it is
-            // executed in read window
-            std::lock_guard<std::mutex> g( _ro_trx_queue.mtx );
-            _ro_trx_queue.queue.push_back({trx, std::move(next)});
+         if ( trx_type == transaction_metadata::trx_type::read_only ) {
+            // Post all read only trxs to read_only queue for execution.
+            auto trx_metadata = transaction_metadata::create_no_recover_keys( trx, transaction_metadata::trx_type::read_only );
+            app().executor().post(priority::low, exec_queue::read_only, [this, trx{std::move(trx_metadata)}, next{std::move(next)}]() mutable {
+               push_read_only_transaction( std::move(trx), std::move(next) );
+            } );
             return;
          }
 
@@ -677,7 +714,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
                                                           next{std::move(next)}, trx=trx]() mutable {
             if( future.valid() ) {
                future.wait();
-               app().executor().post( priority::low, exec_queue::general, [self, future{std::move(future)}, api_trx, is_transient, next{std::move( next )}, trx{std::move(trx)}, return_failure_traces]() mutable {
+               app().executor().post( priority::low, exec_queue::read_write, [self, future{std::move(future)}, api_trx, is_transient, next{std::move( next )}, trx{std::move(trx)}, return_failure_traces]() mutable {
                   auto start = fc::time_point::now();
                   auto idle_time = start - self->_idle_trx_time;
                   self->_time_tracker.add_idle_time( idle_time );
@@ -688,7 +725,8 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
                      self->log_trx_results( trx, nullptr, ex, 0, start, is_transient );
                      next( std::move(ex) );
                      self->_idle_trx_time = fc::time_point::now();
-                     self->_time_tracker.add_fail_time(self->_idle_trx_time - start);
+                     auto dur = self->_idle_trx_time - start;
+                     self->_time_tracker.add_fail_time(dur, is_transient);
                   };
                   try {
                      auto result = future.get();
@@ -709,7 +747,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       bool process_incoming_transaction_async(const transaction_metadata_ptr& trx,
                                               bool api_trx,
                                               bool return_failure_trace,
-                                              next_function<transaction_trace_ptr> next) {
+                                              const next_function<transaction_trace_ptr>& next) {
          bool exhausted = false;
          chain::controller& chain = chain_plug->chain();
          try {
@@ -789,7 +827,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
          exhausted
       };
 
-      inline bool should_interrupt_start_block( const fc::time_point& deadline ) const;
+      inline bool should_interrupt_start_block( const fc::time_point& deadline, uint32_t pending_block_num ) const;
       start_block_result start_block();
 
       fc::time_point calculate_pending_block_time() const;
@@ -892,12 +930,12 @@ void producer_plugin::set_program_options(
           "Number of worker threads in producer thread pool")
          ("snapshots-dir", bpo::value<bfs::path>()->default_value("snapshots"),
           "the location of the snapshots directory (absolute path or relative to application data dir)")
-         ("read-only-threads", bpo::value<uint16_t>()->default_value(my->_ro_thread_pool_size),
-          "Number of worker threads in read-only transaction execution thread pool")
+         ("read-only-threads", bpo::value<uint32_t>(),
+          "Number of worker threads in read-only execution thread pool. Max 8.")
          ("read-only-write-window-time-us", bpo::value<uint32_t>()->default_value(my->_ro_write_window_time_us.count()),
-          "time in microseconds the write window lasts")
+          "Time in microseconds the write window lasts.")
          ("read-only-read-window-time-us", bpo::value<uint32_t>()->default_value(my->_ro_read_window_time_us.count()),
-          "time in microseconds the read window lasts")
+          "Time in microseconds the read window lasts.")
          ;
    config_file_options.add(producer_options);
 }
@@ -1073,11 +1111,22 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
       }
    }
 
-   my->_ro_thread_pool_size = options.at( "read-only-threads" ).as<uint16_t>();
+   if ( options.count( "read-only-threads" ) ) {
+      my->_ro_thread_pool_size = options.at( "read-only-threads" ).as<uint32_t>();
+   } else if ( my->_producers.empty() ) {
+      if( options.count( "plugin" ) ) {
+         const auto& v = options.at( "plugin" ).as<std::vector<std::string>>();
+         auto i = std::find_if( v.cbegin(), v.cend(), []( const std::string& p ) { return p == "eosio::chain_api_plugin"; } );
+         if( i != v.cend() ) {
+            // default to 3 threads for non producer nodes running chain_api_plugin if not specified
+            my->_ro_thread_pool_size = 3;
+            ilog( "chain_api_plugin configured, defaulting read-only-threads to ${t}", ("t", my->_ro_thread_pool_size) );
+         }
+      }
+   }
+   EOS_ASSERT( test_mode_ || my->_ro_thread_pool_size == 0 || my->_producers.empty(), plugin_config_exception, "--read-only-threads not allowed on producer node" );
    // only initialize other read-only options when read-only thread pool is enabled
    if ( my->_ro_thread_pool_size > 0 ) {
-      EOS_ASSERT( my->_producers.empty(), plugin_config_exception, "--read-only-threads not allowed on producer node" );
-
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
       if (chain.is_eos_vm_oc_enabled()) {
          // EOS VM OC requires 4.2TB Virtual for each executing thread. Make sure the memory
@@ -1098,37 +1147,38 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
          }
 
          EOS_ASSERT( vm_total_kb > 0, plugin_config_exception, "Unable to get system virtual memory size (not a Linux?), therefore cannot determine if the system has enough virtual memory for multi-threaded read-only transactions on EOS VM OC");
+         EOS_ASSERT( vm_total_kb > vm_used_kb, plugin_config_exception, "vm total (${t}) must be greater than vm used (${u})", ("t", vm_total_kb)("u", vm_used_kb));
+         uint32_t num_threads_supported = (vm_total_kb - vm_used_kb) / 4200000000;
          // reserve 1 for the app thread, 1 for anything else which might use VM
-         int num_threads_supported = (vm_total_kb - vm_used_kb) / 4200000000 - 2;
-         ilog("vm total in kb: ${total}, vm used in kb: ${used}, number of EOS VM OC threads supported ((vm total - vm used)/4.2 TB - 2): ${supp}", ("total", vm_total_kb) ("used", vm_used_kb) ("supp", num_threads_supported));
-         EOS_ASSERT( num_threads_supported >= my->_ro_thread_pool_size, plugin_config_exception, "--read-only-threads (${th}) greater than number of threads supported for EOS VM OC (${supp}) by the system virtual memory size", ("th", my->_ro_thread_pool_size) ("supp", num_threads_supported) );
-
-         if ( my->_ro_thread_pool_size > my->_ro_max_eos_vm_oc_threads_allowed ) {
-            wlog("--read-only-threads (${th}) greater than maximum number of threads allowed (${allowed}) for EOS Vm OC. Set it to ${allowed}", ("th", my->_ro_thread_pool_size) ("allowed", my->_ro_max_eos_vm_oc_threads_allowed));
-            my->_ro_thread_pool_size = my->_ro_max_eos_vm_oc_threads_allowed;
-         }
+         EOS_ASSERT( num_threads_supported > 2, plugin_config_exception, "With the EOS VM OC configured, there is not enough system virtual memory to support the required minimum of 3 threads (1 for main thread, 1 for read-only, and 1 for anything else), vm total: ${t}, vm used: ${u}", ("t", vm_total_kb)("u", vm_used_kb));
+         num_threads_supported -= 2;
+         auto actual_threads_allowed = std::min(my->_ro_max_threads_allowed, num_threads_supported);
+         ilog("vm total in kb: ${total}, vm used in kb: ${used}, number of EOS VM OC threads supported ((vm total - vm used)/4.2 TB - 2): ${supp}, max allowed: ${max}, actual allowed: ${actual}", ("total", vm_total_kb) ("used", vm_used_kb) ("supp", num_threads_supported) ("max", my->_ro_max_threads_allowed)("actual", actual_threads_allowed));
+         EOS_ASSERT( my->_ro_thread_pool_size <= actual_threads_allowed, plugin_config_exception, "--read-only-threads (${th}) greater than number of threads allowed for EOS VM OC (${allowed})", ("th", my->_ro_thread_pool_size) ("allowed", actual_threads_allowed) );
       }
 #endif
-
-      my->_ro_write_window_time_us = fc::microseconds( options.at( "read-only-write-window-time-us" ).as<uint32_t>() );
-      my->_ro_read_window_time_us = fc::microseconds( options.at( "read-only-read-window-time-us" ).as<uint32_t>() );
-      EOS_ASSERT( my->_ro_read_window_time_us > my->_ro_read_window_minimum_time_us, plugin_config_exception, "minimum of --read-only-read-window-time-us (${read}) must be ${min} microseconds", ("read", my->_ro_read_window_time_us) ("min", my->_ro_read_window_minimum_time_us) );
-      my->_ro_read_window_effective_time_us = my->_ro_read_window_time_us - my->_ro_read_window_minimum_time_us;
-
-      // Make sure a read-only transaction can finish within the read
-      // window if scheduled at the very beginning of the window.
-      // Use _ro_read_window_effective_time_us instead of _ro_read_window_time_us
-      // for safety marging
-      if ( my->_max_transaction_time_ms.load() > 0 ) {
-         EOS_ASSERT( my->_ro_read_window_effective_time_us > fc::milliseconds(my->_max_transaction_time_ms.load()), plugin_config_exception, "--read-only-read-window-time-us (${read}) must be greater than --max-transaction-time ${trx_time} ms plus a margin of ${min} us", ("read", my->_ro_read_window_time_us) ("trx_time", my->_max_transaction_time_ms.load()) ("min", my->_ro_read_window_minimum_time_us) );
-         my->_ro_max_trx_time_us = fc::milliseconds(my->_max_transaction_time_ms.load());
-      } else {
-         // _max_transaction_time_ms can be set to negative in testing (for unlimited)
-         my->_ro_max_trx_time_us = my->_ro_read_window_effective_time_us;
-      }
-      ilog("_ro_thread_pool_size ${s},  _ro_write_window_time_us ${ww}, _ro_read_window_time_us ${rw}, _ro_max_trx_time_us ${t}, _ro_read_window_effective_time_us ${w}", ("s", my->_ro_thread_pool_size) ("ww", my->_ro_write_window_time_us) ("rw", my->_ro_read_window_time_us) ("t", my->_ro_max_trx_time_us) ("w", my->_ro_read_window_effective_time_us));
+      EOS_ASSERT( my->_ro_thread_pool_size <= my->_ro_max_threads_allowed, plugin_config_exception, "--read-only-threads (${th}) greater than number of threads allowed (${allowed})", ("th", my->_ro_thread_pool_size) ("allowed", my->_ro_max_threads_allowed) );
    }
 
+   my->_ro_write_window_time_us = fc::microseconds( options.at( "read-only-write-window-time-us" ).as<uint32_t>() );
+   my->_ro_read_window_time_us = fc::microseconds( options.at( "read-only-read-window-time-us" ).as<uint32_t>() );
+   EOS_ASSERT( my->_ro_read_window_time_us > my->_ro_read_window_minimum_time_us, plugin_config_exception, "minimum of --read-only-read-window-time-us (${read}) must be ${min} microseconds", ("read", my->_ro_read_window_time_us) ("min", my->_ro_read_window_minimum_time_us) );
+   my->_ro_read_window_effective_time_us = my->_ro_read_window_time_us - my->_ro_read_window_minimum_time_us;
+
+   // Make sure a read-only transaction can finish within the read
+   // window if scheduled at the very beginning of the window.
+   // Use _ro_read_window_effective_time_us instead of _ro_read_window_time_us
+   // for safety margin
+   if ( my->_max_transaction_time_ms.load() > 0 ) {
+      EOS_ASSERT( my->_ro_read_window_effective_time_us > fc::milliseconds(my->_max_transaction_time_ms.load()), plugin_config_exception, "--read-only-read-window-time-us (${read}) must be greater than --max-transaction-time ${trx_time} ms plus a margin of ${min} us", ("read", my->_ro_read_window_time_us) ("trx_time", my->_max_transaction_time_ms.load()) ("min", my->_ro_read_window_minimum_time_us) );
+      my->_ro_max_trx_time_us = fc::milliseconds(my->_max_transaction_time_ms.load());
+   } else {
+      // _max_transaction_time_ms can be set to negative in testing (for unlimited)
+      my->_ro_max_trx_time_us = my->_ro_read_window_effective_time_us;
+   }
+   ilog("ro_thread_pool_size ${s}, ro_write_window_time_us ${ww}, ro_read_window_time_us ${rw}, ro_max_trx_time_us ${t}, ro_read_window_effective_time_us ${w}",
+        ("s", my->_ro_thread_pool_size)("ww", my->_ro_write_window_time_us)("rw", my->_ro_read_window_time_us)("t", my->_ro_max_trx_time_us)("w", my->_ro_read_window_effective_time_us));
+
    my->_incoming_block_sync_provider = app().get_method<incoming::methods::block_sync>().register_provider(
          [this](const signed_block_ptr& block, const std::optional<block_id_type>& block_id, const block_state_ptr& bsp) {
       return my->on_incoming_block(block, block_id, bsp);
@@ -1164,6 +1214,7 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
    my->_snapshot_scheduler.set_create_snapshot_fn([this](producer_plugin::next_function<producer_plugin::snapshot_information> next){create_snapshot(next);});
 } FC_LOG_AND_RETHROW() }
 
+using namespace std::chrono_literals;
 void producer_plugin::plugin_startup()
 { try {
    try {
@@ -1209,15 +1260,27 @@ void producer_plugin::plugin_startup()
    }
 
    if ( my->_ro_thread_pool_size > 0 ) {
+      std::atomic<uint32_t> num_threads_started = 0;
       my->_ro_thread_pool.start( my->_ro_thread_pool_size,
          []( const fc::exception& e ) {
-            fc_elog( _log, "Exception in read-only transaction thread pool, exiting: ${e}", ("e", e.to_detail_string()) );
+            fc_elog( _log, "Exception in read-only thread pool, exiting: ${e}", ("e", e.to_detail_string()) );
             app().quit();
          },
          [&]() {
             chain.init_thread_local_data();
+            ++num_threads_started;
          });
 
+      // This will be changed with std::latch or std::atomic<>::wait
+      // when C++20 is used.
+      auto time_slept_ms = 0;
+      constexpr auto max_time_slept_ms = 1000;
+      while ( num_threads_started.load() < my->_ro_thread_pool_size && time_slept_ms < max_time_slept_ms ) {
+         std::this_thread::sleep_for( 1ms );
+         ++time_slept_ms;
+      }
+      EOS_ASSERT(num_threads_started.load() == my->_ro_thread_pool_size, producer_exception, "read-only threads failed to start. num_threads_started: ${n}, time_slept_ms: ${t}ms", ("n", num_threads_started.load())("t", time_slept_ms));
+
       my->start_write_window();
    }
 
@@ -1506,14 +1569,14 @@ void producer_plugin::create_snapshot(producer_plugin::next_function<producer_pl
    }
 }
 
-void producer_plugin::schedule_snapshot(const snapshot_request_information& sri)
+producer_plugin::snapshot_schedule_result producer_plugin::schedule_snapshot(const snapshot_request_information& sri)
 {
-   my->_snapshot_scheduler.schedule_snapshot(sri);
+   return my->_snapshot_scheduler.schedule_snapshot(sri);
 }
 
-void producer_plugin::unschedule_snapshot(const snapshot_request_id_information& sri)
+producer_plugin::snapshot_schedule_result producer_plugin::unschedule_snapshot(const snapshot_request_id_information& sri)
 {
-   my->_snapshot_scheduler.unschedule_snapshot(sri.snapshot_request_id);
+   return my->_snapshot_scheduler.unschedule_snapshot(sri.snapshot_request_id);
 }
 
 producer_plugin::get_snapshot_requests_result producer_plugin::get_snapshot_requests() const
@@ -1777,12 +1840,12 @@ fc::time_point producer_plugin_impl::calculate_block_deadline( const fc::time_po
    }
 }
 
-bool producer_plugin_impl::should_interrupt_start_block( const fc::time_point& deadline ) const {
+bool producer_plugin_impl::should_interrupt_start_block( const fc::time_point& deadline, uint32_t pending_block_num ) const {
    if( _pending_block_mode == pending_block_mode::producing ) {
       return deadline <= fc::time_point::now();
    }
    // if we can produce then honor deadline so production starts on time
-   return (!_producers.empty() && deadline <= fc::time_point::now()) || _received_block;
+   return (!_producers.empty() && deadline <= fc::time_point::now()) || (_received_block >= pending_block_num);
 }
 
 producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
@@ -1802,6 +1865,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
 
    const fc::time_point now = fc::time_point::now();
    const fc::time_point block_time = calculate_pending_block_time();
+   const uint32_t pending_block_num = hbs->block_num + 1;
    const fc::time_point preprocess_deadline = calculate_block_deadline(block_time);
 
    const pending_block_mode previous_pending_mode = _pending_block_mode;
@@ -1867,7 +1931,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
    if (_pending_block_mode == pending_block_mode::producing) {
       const auto start_block_time = block_time - fc::microseconds( config::block_interval_us );
       if( now < start_block_time ) {
-         fc_dlog(_log, "Not producing block waiting for production window ${n} ${bt}", ("n", hbs->block_num + 1)("bt", block_time) );
+         fc_dlog(_log, "Not producing block waiting for production window ${n} ${bt}", ("n", pending_block_num)("bt", block_time) );
          // start_block_time instead of block_time because schedule_delayed_production_loop calculates next block time from given time
          schedule_delayed_production_loop(weak_from_this(), calculate_producer_wake_up_time(start_block_time));
          return start_block_result::waiting_for_production;
@@ -1881,7 +1945,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
    }
 
    fc_dlog(_log, "Starting block #${n} at ${time} producer ${p}",
-           ("n", hbs->block_num + 1)("time", now)("p", scheduled_producer.producer_name));
+           ("n", pending_block_num)("time", now)("p", scheduled_producer.producer_name));
 
    try {
       uint16_t blocks_to_confirm = 0;
@@ -1945,7 +2009,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
             std::swap( features_to_activate, protocol_features_to_activate );
             _protocol_features_signaled = true;
             ilog( "signaling activation of the following protocol features in block ${num}: ${features_to_activate}",
-                  ("num", hbs->block_num + 1)("features_to_activate", features_to_activate) );
+                  ("num", pending_block_num)("features_to_activate", features_to_activate) );
          }
       }
 
@@ -1971,7 +2035,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
          if( !remove_expired_blacklisted_trxs( preprocess_deadline ) )
             return start_block_result::exhausted;
          if( !_subjective_billing.remove_expired( _log, chain.pending_block_time(), fc::time_point::now(),
-                                                  [&](){ return should_interrupt_start_block( preprocess_deadline ); } ) ) {
+                                                  [&](){ return should_interrupt_start_block( preprocess_deadline, pending_block_num ); } ) ) {
             return start_block_result::exhausted;
          }
 
@@ -1994,9 +2058,11 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
             process_scheduled_and_incoming_trxs( scheduled_trx_deadline, incoming_itr );
          }
 
+         repost_exhausted_transactions( preprocess_deadline );
+
          if( app().is_quiting() ) // db guard exception above in LOG_AND_DROP could have called app().quit()
             return start_block_result::failed;
-         if ( should_interrupt_start_block( preprocess_deadline ) || block_is_exhausted() ) {
+         if ( should_interrupt_start_block( preprocess_deadline, pending_block_num ) || block_is_exhausted() ) {
             return start_block_result::exhausted;
          }
 
@@ -2023,11 +2089,12 @@ bool producer_plugin_impl::remove_expired_trxs( const fc::time_point& deadline )
 {
    chain::controller& chain = chain_plug->chain();
    auto pending_block_time = chain.pending_block_time();
+   auto pending_block_num = chain.pending_block_num();
 
    // remove all expired transactions
    size_t num_expired = 0;
    size_t orig_count = _unapplied_transactions.size();
-   bool exhausted = !_unapplied_transactions.clear_expired( pending_block_time, [&](){ return should_interrupt_start_block(deadline); },
+   bool exhausted = !_unapplied_transactions.clear_expired( pending_block_time, [&](){ return should_interrupt_start_block(deadline, pending_block_num); },
          [&num_expired]( const packed_transaction_ptr& packed_trx_ptr, trx_enum_type trx_type ) {
             // expired exception is logged as part of next() call
             ++num_expired;
@@ -2051,12 +2118,13 @@ bool producer_plugin_impl::remove_expired_blacklisted_trxs( const fc::time_point
    if(!blacklist_by_expiry.empty()) {
       const chain::controller& chain = chain_plug->chain();
       const auto lib_time = chain.last_irreversible_block_time();
+      const auto pending_block_num = chain.pending_block_num();
 
       int num_expired = 0;
       int orig_count = _blacklisted_transactions.size();
 
       while (!blacklist_by_expiry.empty() && blacklist_by_expiry.begin()->expiry <= lib_time) {
-         if ( should_interrupt_start_block( deadline ) ) {
+         if ( should_interrupt_start_block( deadline, pending_block_num ) ) {
             exhausted = true;
             break;
          }
@@ -2195,9 +2263,10 @@ producer_plugin_impl::push_transaction( const fc::time_point& block_deadline,
                                         const transaction_metadata_ptr& trx,
                                         bool api_trx,
                                         bool return_failure_trace,
-                                        next_function<transaction_trace_ptr> next )
+                                        const next_function<transaction_trace_ptr>& next )
 {
    auto start = fc::time_point::now();
+   EOS_ASSERT(!trx->is_read_only(), producer_exception, "Unexpected read-only trx");
 
    bool disable_subjective_enforcement = (api_trx && _disable_subjective_api_billing)
                                          || (!api_trx && _disable_subjective_p2p_billing)
@@ -2214,7 +2283,7 @@ producer_plugin_impl::push_transaction( const fc::time_point& block_deadline,
          log_trx_results( trx, except_ptr );
          next( except_ptr );
       }
-      _time_tracker.add_fail_time(fc::time_point::now() - start);
+      _time_tracker.add_fail_time(fc::time_point::now() - start, trx->is_transient());
       return push_result{.failed = true};
    }
 
@@ -2234,14 +2303,6 @@ producer_plugin_impl::push_transaction( const fc::time_point& block_deadline,
       }
    }
 
-   // Make sure db_read_only_mode always to be unset
-   auto db_read_only_mode_guard = fc::make_scoped_exit([trx, &chain]{
-      if( trx->is_read_only() )
-         chain.unset_db_read_only_mode();
-   });
-   if( trx->is_read_only() )
-      chain.set_db_read_only_mode();
-
    auto trace = chain.push_transaction( trx, block_deadline, max_trx_time, prev_billed_cpu_time_us, false, sub_bill );
 
    return handle_push_result(trx, next, start, chain, trace, return_failure_trace, disable_subjective_enforcement, first_auth, sub_bill, prev_billed_cpu_time_us);
@@ -2249,7 +2310,7 @@ producer_plugin_impl::push_transaction( const fc::time_point& block_deadline,
 
 producer_plugin_impl::push_result
 producer_plugin_impl::handle_push_result( const transaction_metadata_ptr& trx,
-                                          next_function<transaction_trace_ptr> next,
+                                          const next_function<transaction_trace_ptr>& next,
                                           const fc::time_point& start,
                                           const chain::controller& chain,
                                           const transaction_trace_ptr& trace,
@@ -2261,8 +2322,14 @@ producer_plugin_impl::handle_push_result( const transaction_metadata_ptr& trx,
    auto end = fc::time_point::now();
    push_result pr;
    if( trace->except ) {
-      if ( !trx->is_read_only() )
-         _time_tracker.add_fail_time(end - start);
+      // Transient trxs are dry-run or read-only.
+      // Dry-run trxs only run in write window. Read-only trxs can run in
+      // both write and read windows; time spent in read window is counted
+      // by read window summary.
+      if ( chain.is_write_window() ) {
+         auto dur = end - start;
+         _time_tracker.add_fail_time(dur, trx->is_transient());
+      }
       if( exception_is_exhausted( *trace->except ) ) {
          if( _pending_block_mode == pending_block_mode::producing ) {
             fc_dlog(_trx_failed_trace_log, "[TRX_TRACE] Block ${block_num} for producer ${prod} COULD NOT FIT, tx: ${txid} RETRYING ",
@@ -2302,8 +2369,14 @@ producer_plugin_impl::handle_push_result( const transaction_metadata_ptr& trx,
    } else {
       fc_tlog( _log, "Subjective bill for success ${a}: ${b} elapsed ${t}us, time ${r}us",
                ("a",first_auth)("b",sub_bill)("t",trace->elapsed)("r", end - start));
-      if ( !trx->is_read_only() )
-         _time_tracker.add_success_time(end - start);
+      // Transient trxs are dry-run or read-only.
+      // Dry-run trxs only run in write window. Read-only trxs can run in
+      // both write and read windows; time spent in read window is counted
+      // by read window summary.
+      if ( chain.is_write_window() ) {
+         auto dur = end - start;
+         _time_tracker.add_success_time(dur, trx->is_transient());
+      }
       log_trx_results( trx, trace, start );
       // if producing then trx is in objective cpu account billing
       if (!disable_subjective_enforcement && _pending_block_mode != pending_block_mode::producing) {
@@ -2319,12 +2392,14 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin
 {
    bool exhausted = false;
    if( !_unapplied_transactions.empty() ) {
+      const chain::controller& chain = chain_plug->chain();
+      const auto pending_block_num = chain.pending_block_num();
       int num_applied = 0, num_failed = 0, num_processed = 0;
       auto unapplied_trxs_size = _unapplied_transactions.size();
       auto itr     = _unapplied_transactions.unapplied_begin();
       auto end_itr = _unapplied_transactions.unapplied_end();
       while( itr != end_itr ) {
-         if( should_interrupt_start_block( deadline ) ) {
+         if( should_interrupt_start_block( deadline, pending_block_num ) ) {
             exhausted = true;
             break;
          }
@@ -2446,7 +2521,7 @@ void producer_plugin_impl::process_scheduled_and_incoming_trxs( const fc::time_p
          auto trace = chain.push_scheduled_transaction(trx_id, deadline, max_trx_time, 0, false);
          auto end = fc::time_point::now();
          if (trace->except) {
-            _time_tracker.add_fail_time(end - start);
+            _time_tracker.add_fail_time(end - start, false); // delayed transaction cannot be transient
             if (exception_is_exhausted(*trace->except)) {
                if( block_is_exhausted() ) {
                   exhausted = true;
@@ -2466,7 +2541,7 @@ void producer_plugin_impl::process_scheduled_and_incoming_trxs( const fc::time_p
                num_failed++;
             }
          } else {
-            _time_tracker.add_success_time(end - start);
+            _time_tracker.add_success_time(end - start, false); // delayed transaction cannot be transient
             fc_dlog(_trx_successful_trace_log,
                     "[TRX_TRACE] Block ${block_num} for producer ${prod} is ACCEPTING scheduled tx: ${txid}, time: ${r}, auth: ${a}, cpu: ${cpu}",
                     ("block_num", chain.head_block_num() + 1)("prod", get_pending_block_producer())
@@ -2499,8 +2574,10 @@ bool producer_plugin_impl::process_incoming_trxs( const fc::time_point& deadline
    if( itr != end ) {
       size_t processed = 0;
       fc_dlog( _log, "Processing ${n} pending transactions", ("n", _unapplied_transactions.incoming_size()) );
+      const chain::controller& chain = chain_plug->chain();
+      const auto pending_block_num = chain.pending_block_num();
       while( itr != end ) {
-         if ( should_interrupt_start_block( deadline ) ) {
+         if ( should_interrupt_start_block( deadline, pending_block_num ) ) {
             exhausted = true;
             break;
          }
@@ -2543,7 +2620,6 @@ bool producer_plugin_impl::block_is_exhausted() const {
 // -> Idle
 // --> Start block B (block time y.000) at time x.500
 void producer_plugin_impl::schedule_production_loop() {
-   _received_block = false;
    _timer.cancel();
 
    auto result = start_block();
@@ -2555,7 +2631,7 @@ void producer_plugin_impl::schedule_production_loop() {
       _timer.expires_from_now( boost::posix_time::microseconds( config::block_interval_us  / 10 ));
 
       // we failed to start a block, so try again later?
-      _timer.async_wait( app().executor().wrap( priority::high, exec_queue::general,
+      _timer.async_wait( app().executor().wrap( priority::high, exec_queue::read_write,
           [weak_this = weak_from_this(), cid = ++_timer_corelation_id]( const boost::system::error_code& ec ) {
              auto self = weak_this.lock();
              if( self && ec != boost::asio::error::operation_aborted && cid == self->_timer_corelation_id ) {
@@ -2608,7 +2684,7 @@ void producer_plugin_impl::schedule_maybe_produce_block( bool exhausted ) {
                ("num", chain.head_block_num() + 1)("desc", block_is_exhausted() ? "Exhausted" : "Deadline exceeded") );
    }
 
-   _timer.async_wait( app().executor().wrap( priority::high, exec_queue::general,
+   _timer.async_wait( app().executor().wrap( priority::high, exec_queue::read_write,
          [&chain, weak_this = weak_from_this(), cid=++_timer_corelation_id](const boost::system::error_code& ec) {
             auto self = weak_this.lock();
             if( self && ec != boost::asio::error::operation_aborted && cid == self->_timer_corelation_id ) {
@@ -2650,7 +2726,7 @@ void producer_plugin_impl::schedule_delayed_production_loop(const std::weak_ptr<
       fc_dlog(_log, "Scheduling Speculative/Production Change at ${time}", ("time", wake_up_time));
       static const boost::posix_time::ptime epoch(boost::gregorian::date(1970, 1, 1));
       _timer.expires_at(epoch + boost::posix_time::microseconds(wake_up_time->time_since_epoch().count()));
-      _timer.async_wait( app().executor().wrap( priority::high, exec_queue::general,
+      _timer.async_wait( app().executor().wrap( priority::high, exec_queue::read_write,
          [weak_this,cid=++_timer_corelation_id](const boost::system::error_code& ec) {
             auto self = weak_this.lock();
             if( self && ec != boost::asio::error::operation_aborted && cid == self->_timer_corelation_id ) {
@@ -2751,8 +2827,8 @@ void producer_plugin_impl::produce_block() {
         ("confs", new_bs->header.confirmed));
 }
 
-void producer_plugin::received_block() {
-   my->_received_block = true;
+void producer_plugin::received_block(uint32_t block_num) {
+   my->_received_block = block_num;
 }
 
 void producer_plugin::log_failed_transaction(const transaction_id_type& trx_id, const packed_transaction_ptr& packed_trx_ptr, const char* reason) const {
@@ -2764,34 +2840,45 @@ void producer_plugin::log_failed_transaction(const transaction_id_type& trx_id,
             ("entire_trx", packed_trx_ptr ? my->chain_plug->get_log_trx(packed_trx_ptr->get_transaction()) : fc::variant{trx_id}));
 }
 
-// Called from app thread
+// Called from only one read_only thread
 void producer_plugin_impl::switch_to_write_window() {
+   if ( _log.is_enabled( fc::log_level::debug ) ) {
+      auto now = fc::time_point::now();
+      fc_dlog( _log, "Read-only threads ${n}, read window ${r}us, total all threads ${t}us",
+               ("n", _ro_thread_pool_size)
+               ("r", now - _ro_read_window_start_time)
+               ("t", _ro_all_threads_exec_time_us.load()));
+   }
+
+   chain::controller& chain = chain_plug->chain();
+
    // this method can be called from multiple places. it is possible
    // we are already in write window.
-   if ( app().executor().is_write_window() ) {
+   if ( chain.is_write_window() ) {
       return;
    }
 
-   EOS_ASSERT(_ro_num_active_trx_exec_tasks.load() == 0 && _ro_trx_exec_tasks_fut.empty(), producer_exception, "no read-only tasks should be running before switching to write window");
-   _ro_read_window_timer.cancel();
-   _ro_write_window_timer.cancel();
+   EOS_ASSERT(_ro_num_active_exec_tasks.load() == 0 && _ro_exec_tasks_fut.empty(), producer_exception, "no read-only tasks should be running before switching to write window");
 
    start_write_window();
 }
 
-// Called from app thread
+// Called from app thread on plugin_startup
+// Called from only one read_only thread & called from app thread, but not concurrently
 void producer_plugin_impl::start_write_window() {
    chain::controller& chain = chain_plug->chain();
 
    app().executor().set_to_write_window();
+   chain.set_to_write_window();
    chain.unset_db_read_only_mode();
-   _idle_trx_time = fc::time_point::now();
+   _idle_trx_time = _ro_window_deadline = fc::time_point::now();
 
+   _ro_window_deadline += _ro_write_window_time_us; // not allowed on block producers, so no need to limit to block deadline
    auto expire_time = boost::posix_time::microseconds(_ro_write_window_time_us.count());
-   _ro_write_window_timer.expires_from_now( expire_time );
-   _ro_write_window_timer.async_wait( app().executor().wrap(  // stay on app thread
+   _ro_timer.expires_from_now( expire_time );
+   _ro_timer.async_wait( app().executor().wrap(  // stay on app thread
       priority::high,
-      exec_queue::read_only_trx_safe, // placed in read_only_trx_safe queue so it is ensured to be executed in either window
+      exec_queue::read_write, // placed in read_write so only called from main thread
       [weak_this = weak_from_this()]( const boost::system::error_code& ec ) {
          auto self = weak_this.lock();
          if( self && ec != boost::asio::error::operation_aborted ) {
@@ -2800,144 +2887,154 @@ void producer_plugin_impl::start_write_window() {
       }));
 }
 
-// Called from app thread
+// Called only from app thread
 void producer_plugin_impl::switch_to_read_window() {
-   EOS_ASSERT(app().executor().is_write_window(),  producer_exception, "expected to be in write window");
-   EOS_ASSERT(_ro_num_active_trx_exec_tasks.load() == 0 && _ro_trx_exec_tasks_fut.empty(),  producer_exception, "_ro_trx_exec_tasks_fut expected to be empty" );
+   chain::controller& chain = chain_plug->chain();
+   EOS_ASSERT(chain.is_write_window(),  producer_exception, "expected to be in write window");
+   EOS_ASSERT( _ro_num_active_exec_tasks.load() == 0 && _ro_exec_tasks_fut.empty(), producer_exception, "_ro_exec_tasks_fut expected to be empty" );
 
-   _ro_write_window_timer.cancel();
-   _ro_read_window_timer.cancel();
    _time_tracker.add_idle_time( fc::time_point::now() - _idle_trx_time );
 
    // we are in write window, so no read-only trx threads are processing transactions.
-   // _ro_trx_queue is not being accessed. No need to lock.
-   if ( _ro_trx_queue.queue.empty() ) { // no read-only trxs to process. stay in write window
+   if ( app().executor().read_only_queue().empty() ) { // no read-only tasks to process. stay in write window
       start_write_window(); // restart write window timer for next round
       return;
    }
 
-   app().executor().set_to_read_window();
-   chain_plug->chain().set_db_read_only_mode();
-   _received_block = false;
+   uint32_t pending_block_num = chain.head_block_num() + 1;
+   _ro_read_window_start_time = fc::time_point::now();
+   _ro_window_deadline = _ro_read_window_start_time + _ro_read_window_effective_time_us;
+   app().executor().set_to_read_window(_ro_thread_pool_size,
+      [received_block=&_received_block, pending_block_num, ro_window_deadline=_ro_window_deadline]() {
+         return fc::time_point::now() >= ro_window_deadline || (received_block->load() >= pending_block_num); // should_exit()
+      });
+   chain.set_to_read_window();
+   chain.set_db_read_only_mode();
+   _ro_all_threads_exec_time_us = 0;
 
-   // start a read-only transaction execution task in each thread in the thread pool
-   _ro_num_active_trx_exec_tasks = _ro_thread_pool_size;
-   for (auto i = 0; i < _ro_thread_pool_size; ++i ) {
-      _ro_trx_exec_tasks_fut.emplace_back( post_async_task( _ro_thread_pool.get_executor(), [self = this] () {
-         return self->read_only_trx_execution_task();
+   // start a read-only execution task in each thread in the thread pool
+   _ro_num_active_exec_tasks = _ro_thread_pool_size;
+   _ro_exec_tasks_fut.resize(0);
+   for (uint32_t i = 0; i < _ro_thread_pool_size; ++i ) {
+      _ro_exec_tasks_fut.emplace_back( post_async_task( _ro_thread_pool.get_executor(), [self = this, pending_block_num] () {
+         return self->read_only_execution_task(pending_block_num);
       }) );
    }
 
    auto expire_time = boost::posix_time::microseconds(_ro_read_window_time_us.count());
-   _ro_read_window_timer.expires_from_now( expire_time );
-   _ro_read_window_timer.async_wait( app().executor().wrap(  // stay on app thread
+   _ro_timer.expires_from_now( expire_time );
+   // Needs to be on read_only because that is what is being processed until switch_to_write_window().
+   _ro_timer.async_wait( app().executor().wrap(
       priority::high,
-      exec_queue::read_only_trx_safe,
+      exec_queue::read_only,
       [weak_this = weak_from_this()]( const boost::system::error_code& ec ) {
          auto self = weak_this.lock();
          if( self && ec != boost::asio::error::operation_aborted ) {
             // use future to make sure all read-only tasks finished before switching to write window
-            for ( auto& task: self->_ro_trx_exec_tasks_fut ) {
+            for ( auto& task: self->_ro_exec_tasks_fut ) {
                task.get();
             }
-            self->_ro_trx_exec_tasks_fut.clear();
+            self->_ro_exec_tasks_fut.clear();
+            // will be executed from the main app thread because all read-only threads are idle now
             self->switch_to_write_window();
           } else if ( self ) {
-             self->_ro_trx_exec_tasks_fut.clear();
+             self->_ro_exec_tasks_fut.clear();
           }
        }));
 }
 
-// Called from a read only trx thread. Run in parallel with app and other read only trx threads
-bool producer_plugin_impl::read_only_trx_execution_task() {
-   auto start = fc::time_point::now();
-   auto read_window_deadline = start + _ro_read_window_effective_time_us;
-   // We have 4 ways to break out the while loop:
+// Called from a read only thread. Run in parallel with app and other read only threads
+bool producer_plugin_impl::read_only_execution_task(uint32_t pending_block_num) {
+   // We have 3 ways to break out the while loop:
    // 1. pass read window deadline
-   // 2. Net_plugin receives a block
-   // 3. No more transactions in the read-only trx queue
-   // 4. A transaction execution is exhaused
-   while ( fc::time_point::now() < read_window_deadline && !_received_block ) {
-      std::unique_lock<std::mutex> lck( _ro_trx_queue.mtx );
-      if ( _ro_trx_queue.queue.empty() ) {
-         break;
-      }
-      auto trx = _ro_trx_queue.queue.front();
-      _ro_trx_queue.queue.pop_front();
-      lck.unlock();
-      
-      auto retry = process_read_only_transaction( trx.trx, trx.next, start );
-      if ( retry ) {
-         lck.lock();
-         _ro_trx_queue.queue.push_front(trx);
-         // Do not schedule new execution
+   // 2. net_plugin receives a block
+   // 3. no read-only tasks to execute
+   while ( fc::time_point::now() < _ro_window_deadline && _received_block < pending_block_num ) {
+      bool more = app().executor().execute_highest_read_only(); // blocks until all read only threads are idle
+      if ( !more ) {
          break;
       }
    }
 
    // If all tasks are finished, do not wait until end of read window; switch to write window now.
-   if ( --_ro_num_active_trx_exec_tasks == 0 ) {
-      // Do switching on app thread to serialize
-      app().executor().post( priority::high, exec_queue::read_only_trx_safe, [self=this]() {
-         self->_ro_trx_exec_tasks_fut.clear();
+   if ( --_ro_num_active_exec_tasks == 0 ) {
+      // Needs to be on read_only because that is what is being processed until switch_to_write_window().
+      app().executor().post( priority::high, exec_queue::read_only, [self=this]() {
+         self->_ro_exec_tasks_fut.clear();
+         // will be executed from the main app thread because all read-only threads are idle now
          self->switch_to_write_window();
       } );
+      // last thread post any exhausted back into read_only queue with slightly higher priority (low+1) so they are executed first
+      ro_trx_t t;
+      while( _ro_exhausted_trx_queue.pop_front(t) ) {
+         app().executor().post(priority::low+1, exec_queue::read_only, [this, trx{std::move(t.trx)}, next{std::move(t.next)}]() mutable {
+            push_read_only_transaction( std::move(trx), std::move(next) );
+         } );
+      }
    }
 
    return true;
 }
 
-// Called from a read only trx thread. Run in parallel with app and other read only trx threads
-// Return whether the trx needs to be retried in next read window
-bool producer_plugin_impl::process_read_only_transaction(const packed_transaction_ptr& trx, const next_function<transaction_trace_ptr>& next, const fc::time_point& read_window_start_time) {
-   chain::controller& chain = chain_plug->chain();
-   auto future = transaction_metadata::start_recover_keys( trx, _thread_pool.get_executor(),
-                                                           chain.get_chain_id(), fc::microseconds::maximum(),
-                                                           transaction_metadata::trx_type::read_only,
-                                                           chain.configured_subjective_signature_length_limit() );
-   auto exception_handler = [&next](fc::exception_ptr ex) {
-      next( std::move(ex) );
-      return false;
-   };
-   future.wait();
-   try {
-      auto trx_metadata = future.get();
-      return push_read_only_transaction( trx_metadata, next, read_window_start_time );
-   } CATCH_AND_CALL(exception_handler);
-   return false;
+// Called from app thread during start block.
+// Reschedule any exhausted read-only transactions from the last block
+void producer_plugin_impl::repost_exhausted_transactions(const fc::time_point& deadline) {
+   if ( !_ro_exhausted_trx_queue.empty() ) {
+      chain::controller& chain = chain_plug->chain();
+      uint32_t pending_block_num = chain.pending_block_num();
+      // post any exhausted back into read_only queue with slightly higher priority (low+1) so they are executed first
+      ro_trx_t t;
+      while( !should_interrupt_start_block( deadline, pending_block_num ) && _ro_exhausted_trx_queue.pop_front(t) ) {
+         app().executor().post(priority::low+1, exec_queue::read_only, [this, trx{std::move(t.trx)}, next{std::move(t.next)}]() mutable {
+            push_read_only_transaction( std::move(trx), std::move(next) );
+         } );
+      }
+   }
 }
 
-// Called from a read_only_trx execution thread
+// Called from a read_only_trx execution thread, or from app thread when executing exclusively
 // Return whether the trx needs to be retried in next read window
-bool producer_plugin_impl::push_read_only_transaction(
-            const transaction_metadata_ptr& trx,
-            next_function<transaction_trace_ptr> next,
-            const fc::time_point& read_window_start_time) {
+bool producer_plugin_impl::push_read_only_transaction(transaction_metadata_ptr trx, next_function<transaction_trace_ptr> next) {
    auto retry = false;
-   chain::controller& chain = chain_plug->chain();
-
-   if( !chain.is_building_block() ) {
-      // try next round
-      return true;
-   }
 
    try {
-      const auto block_deadline = calculate_block_deadline( chain.pending_block_time() );
       auto start = fc::time_point::now();
-      auto time_used_in_read_window_us = ( start - read_window_start_time );
-      if ( time_used_in_read_window_us >= _ro_read_window_time_us ) {
-         // already passed read-window deadline, try next round
+      chain::controller& chain = chain_plug->chain();
+      if ( !chain.is_building_block() ) {
+         _ro_exhausted_trx_queue.push_front( {std::move(trx), std::move(next)} );
          return true;
       }
 
-      auto remaining_time_in_read_window_us = _ro_read_window_time_us - time_used_in_read_window_us;
-      // Ensure the trx to finish by the end of read-window.
-      auto window_deadline = std::min( start + remaining_time_in_read_window_us, block_deadline );
+      // When executing a read-only trx on the main thread while in the write window,
+      // need to switch db mode to read only.
+      auto db_read_only_mode_guard = fc::make_scoped_exit([&]{
+         if( chain.is_write_window() )
+            chain.unset_db_read_only_mode();
+      });
+
+      if ( chain.is_write_window() ) {
+         chain.set_db_read_only_mode();
+         auto idle_time = fc::time_point::now() - _idle_trx_time;
+         _time_tracker.add_idle_time( idle_time );
+      }
+
+      // use read-window/write-window deadline if there are read/write windows, otherwise use block_deadline if only the app thead
+      auto window_deadline = (_ro_thread_pool_size != 0) ? _ro_window_deadline : calculate_block_deadline( chain.pending_block_time() );
+
+      // Ensure the trx to finish by the end of read-window or write-window or block_deadline depending on
       auto trace = chain.push_transaction( trx, window_deadline, _ro_max_trx_time_us, 0, false, 0 );
+      _ro_all_threads_exec_time_us += (fc::time_point::now() - start).count();
       auto pr = handle_push_result(trx, next, start, chain, trace, true /*return_failure_trace*/, true /*disable_subjective_enforcement*/, {} /*first_auth*/, 0 /*sub_bill*/, 0 /*prev_billed_cpu_time_us*/);
       // If a transaction was exhausted, that indicates we are close to
       // the end of read window. Retry in next round.
       retry = pr.trx_exhausted;
+      if( retry ) {
+         _ro_exhausted_trx_queue.push_front( {std::move(trx), std::move(next)} );
+      }
+
+      if ( chain.is_write_window() ) {
+         _idle_trx_time = fc::time_point::now();
+      }
    } catch ( const guard_exception& e ) {
       chain_plugin::handle_guard_exception(e);
    } catch ( boost::interprocess::bad_alloc& ) {

plugins/producer_plugin/test/test_read_only_trx.cpp

@@ -79,6 +79,7 @@ void test_trxs_common(std::vector<const char*>& specific_args) {
    appbase::scoped_app app;
    fc::temp_directory temp;
    auto temp_dir_str = temp.path().string();
+   producer_plugin::set_test_mode(true);
    
    std::promise<std::tuple<producer_plugin*, chain_plugin*>> plugin_promise;
    std::future<std::tuple<producer_plugin*, chain_plugin*>> plugin_fut = plugin_promise.get_future();
@@ -96,14 +97,19 @@ void test_trxs_common(std::vector<const char*>& specific_args) {
    auto chain_id = chain_plug->get_chain_id();
 
    std::atomic<size_t> next_calls = 0;
+   std::atomic<size_t> num_get_account_calls = 0;
    std::atomic<size_t> num_posts = 0;
    std::atomic<size_t> trace_with_except = 0;
    std::atomic<bool> trx_match = true;
-   const size_t num_pushes = 2558;
+   const size_t num_pushes = 4242;
 
    for( size_t i = 1; i <= num_pushes; ++i ) {
       auto ptrx = make_unique_trx( chain_id );
-      app->executor().post( priority::low, exec_queue::general, [ptrx, &next_calls, &num_posts, &trace_with_except, &trx_match, &app]() {
+      app->executor().post( priority::low, exec_queue::read_only, [&chain_plug=chain_plug, &num_get_account_calls]() {
+         chain_plug->get_read_only_api(fc::seconds(90)).get_account(chain_apis::read_only::get_account_params{.account_name=config::system_account_name}, fc::time_point::now()+fc::seconds(90));
+         ++num_get_account_calls;
+      });
+      app->executor().post( priority::low, exec_queue::read_write, [ptrx, &next_calls, &num_posts, &trace_with_except, &trx_match, &app]() {
          ++num_posts;
          bool return_failure_traces = true;
          app->get_method<plugin_interface::incoming::methods::transaction_async>()(ptrx,
@@ -126,12 +132,15 @@ void test_trxs_common(std::vector<const char*>& specific_args) {
                ++next_calls;
         });
       });
+      app->executor().post( priority::low, exec_queue::read_only, [&chain_plug=chain_plug]() {
+         chain_plug->get_read_only_api(fc::seconds(90)).get_consensus_parameters(chain_apis::read_only::get_consensus_parameters_params{}, fc::time_point::now()+fc::seconds(90));
+      });
    }
 
    // Wait long enough such that all transactions are executed
    auto start = fc::time_point::now();
    auto hard_deadline = start + fc::seconds(10); // To protect against waiting forever
-   while ( next_calls < num_pushes && fc::time_point::now() < hard_deadline ){
+   while ( (next_calls < num_pushes || num_get_account_calls < num_pushes) && fc::time_point::now() < hard_deadline ){
       std::this_thread::sleep_for( 100ms );;
    }
 
@@ -141,13 +150,39 @@ void test_trxs_common(std::vector<const char*>& specific_args) {
    BOOST_CHECK_EQUAL( trace_with_except, 0 ); // should not have any traces with except in it
    BOOST_CHECK_EQUAL( num_pushes, num_posts );
    BOOST_CHECK_EQUAL( num_pushes, next_calls.load() );
-   BOOST_CHECK( trx_match.load() );  // trace should match the transaction  
+   BOOST_CHECK_EQUAL( num_pushes, num_get_account_calls.load() );
+   BOOST_CHECK( trx_match.load() );  // trace should match the transaction
 }
 
 // test read-only trxs on main thread (no --read-only-threads)
 BOOST_AUTO_TEST_CASE(no_read_only_threads) {
-   std::vector<const char*> specific_args = { "-p", "eosio", "-e" };
+   std::vector<const char*> specific_args = { "-p", "eosio", "-e", "--abi-serializer-max-time-ms=999" };
    test_trxs_common(specific_args);
 }
 
+// test read-only trxs on 1 threads (with --read-only-threads)
+BOOST_AUTO_TEST_CASE(with_1_read_only_threads) {
+   std::vector<const char*> specific_args = { "-p", "eosio", "-e",
+                                              "--read-only-threads=1",
+                                              "--max-transaction-time=10",
+                                              "--abi-serializer-max-time-ms=999",
+                                              "--read-only-write-window-time-us=100000",
+                                              "--read-only-read-window-time-us=40000",
+                                              "--disable-subjective-billing=true" };
+   test_trxs_common(specific_args);
+}
+
+// test read-only trxs on 8 separate threads (with --read-only-threads)
+BOOST_AUTO_TEST_CASE(with_8_read_only_threads) {
+   std::vector<const char*> specific_args = { "-p", "eosio", "-e",
+                                              "--read-only-threads=8",
+                                              "--max-transaction-time=10",
+                                              "--abi-serializer-max-time-ms=999",
+                                              "--read-only-write-window-time-us=100000",
+                                              "--read-only-read-window-time-us=40000",
+                                              "--disable-subjective-billing=true" };
+   test_trxs_common(specific_args);
+}
+
+
 BOOST_AUTO_TEST_SUITE_END()

plugins/producer_plugin/test/test_snapshot_scheduler.cpp

@@ -67,7 +67,7 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
             fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::debug);
             std::vector<const char*> argv =
                   {"test", "--data-dir", temp.c_str(), "--config-dir", temp.c_str(),
-                   "-p", "eosio", "-e", "--max-transaction-time", "475", "--disable-subjective-billing=true"};
+                   "-p", "eosio", "-e", "--disable-subjective-billing=true"};
             appbase::app().initialize<chain_plugin, producer_plugin>(argv.size(), (char**) &argv[0]);
             appbase::app().startup();
             plugin_promise.set_value(
@@ -93,18 +93,20 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
          auto bs = chain_plug->chain().block_start.connect([&pp](uint32_t bn) {
             // catching pending snapshot
             if (!pp->get_snapshot_requests().snapshot_requests.empty()) {
-               auto& pending = pp->get_snapshot_requests().snapshot_requests.begin()->pending_snapshots;
-               if (pending && pending->size()==1) {
-                  // lets check the head block num of it, it should be 8 + 1 = 9
-                  // this means we are getting a snapshot for correct block # as well
-                  BOOST_CHECK_EQUAL(9, pending->begin()->head_block_num);   
+               const auto& snapshot_requests = pp->get_snapshot_requests().snapshot_requests;
+               auto it = find_if(snapshot_requests.begin(), snapshot_requests.end(), [](const producer_plugin::snapshot_schedule_information& obj) {return obj.snapshot_request_id == 0;});
+               // we should have a pending snapshot for request id = 0
+               BOOST_REQUIRE(it != snapshot_requests.end());
+               auto& pending = it->pending_snapshots;
+               if (pending.size()==1) {
+                  BOOST_CHECK_EQUAL(9, pending.begin()->head_block_num);   
                }
             }
          });
        
-         producer_plugin::snapshot_request_information sri1 = {.block_spacing = 8, .start_block_num = 1, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 2"};
-         producer_plugin::snapshot_request_information sri2 = {.block_spacing = 5000, .start_block_num = 100000, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 2"};
-         producer_plugin::snapshot_request_information sri3 = {.block_spacing = 2, .start_block_num = 0, .end_block_num = 3, .snapshot_description = "Example of recurring snapshot 1"};
+         producer_plugin::snapshot_request_information sri1 = {.block_spacing = 8, .start_block_num = 1, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 1"};
+         producer_plugin::snapshot_request_information sri2 = {.block_spacing = 5000, .start_block_num = 100000, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 2 that will never happen"};
+         producer_plugin::snapshot_request_information sri3 = {.block_spacing = 2, .start_block_num = 0, .end_block_num = 3, .snapshot_description = "Example of recurring snapshot 3 that will expire"};
 
          pp->schedule_snapshot(sri1);
          pp->schedule_snapshot(sri2);
@@ -118,8 +120,13 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
          // one of the snapshots is done here and request, corresponding to it should be deleted
          BOOST_CHECK_EQUAL(2, pp->get_snapshot_requests().snapshot_requests.size());
 
-         // check whether no pending snapshots present
-         BOOST_CHECK_EQUAL(0, pp->get_snapshot_requests().snapshot_requests.begin()->pending_snapshots->size());
+         // check whether no pending snapshots present for a snapshot with id 0
+         const auto& snapshot_requests = pp->get_snapshot_requests().snapshot_requests;
+         auto it = find_if(snapshot_requests.begin(), snapshot_requests.end(),[](const producer_plugin::snapshot_schedule_information& obj) {return obj.snapshot_request_id == 0;});
+
+         // snapshot request with id = 0 should be found and should not have any pending snapshots
+         BOOST_REQUIRE(it != snapshot_requests.end());
+         BOOST_CHECK(!it->pending_snapshots.size());
 
          // quit app
          appbase::app().quit();

plugins/producer_plugin/test/test_trx_full.cpp

@@ -112,7 +112,7 @@ BOOST_AUTO_TEST_CASE(producer) {
          fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::debug);
          std::vector<const char*> argv =
                {"test", "--data-dir", temp_dir_str.c_str(), "--config-dir", temp_dir_str.c_str(),
-                "-p", "eosio", "-e", "--max-transaction-time", "475", "--disable-subjective-billing=true" };
+                "-p", "eosio", "-e", "--disable-subjective-billing=true" };
          app->initialize<chain_plugin, producer_plugin>( argv.size(), (char**) &argv[0] );
          app->startup();
          plugin_promise.set_value(

plugins/prometheus_plugin/include/eosio/prometheus_plugin/simple_rest_server.hpp

@@ -0,0 +1,234 @@
+#pragma once
+
+#include <boost/beast/core.hpp>
+#include <boost/beast/http.hpp>
+#include <boost/beast/version.hpp>
+
+namespace eosio { namespace rest {
+
+   // The majority of the code here are derived from boost source
+   // libs/beast/example/http/server/async/http_server_async.cpp
+   // with minimum modification and yet reusable.
+
+   namespace beast = boost::beast;         // from <boost/beast.hpp>
+   namespace http  = beast::http;          // from <boost/beast/http.hpp>
+   namespace net   = boost::asio;          // from <boost/asio.hpp>
+   using tcp       = boost::asio::ip::tcp; // from <boost/asio/ip/tcp.hpp>
+   template <typename T>
+   class simple_server {
+      T* self() { return static_cast<T*>(this); }
+
+      void fail(beast::error_code ec, char const* what) { self()->log_error(what, ec.message()); }
+      // Return a response for the given request.
+      http::response<http::string_body> handle_request(http::request<http::string_body>&& req) {
+         auto server_header = self()->server_header();
+         // Returns a bad request response
+         auto const bad_request = [&req, &server_header](std::string_view why) {
+            http::response<http::string_body> res{ http::status::bad_request, req.version() };
+            res.set(http::field::server, server_header);
+            res.set(http::field::content_type, "text/plain");
+            res.keep_alive(req.keep_alive());
+            res.body() = std::string(why);
+            res.prepare_payload();
+            return res;
+         };
+
+         // Returns a not found response
+         auto const not_found = [&req, &server_header](std::string_view target) {
+            http::response<http::string_body> res{ http::status::not_found, req.version() };
+            res.set(http::field::server, server_header);
+            res.set(http::field::content_type, "text/plain");
+            res.keep_alive(req.keep_alive());
+            res.body() = "The resource '" + std::string(target) + "' was not found.";
+            res.prepare_payload();
+            return res;
+         };
+
+         // Returns a server error response
+         auto const server_error = [&req, &server_header](std::string_view what) {
+            http::response<http::string_body> res{ http::status::internal_server_error, req.version() };
+            res.set(http::field::server, server_header);
+            res.set(http::field::content_type, "text/plain");
+            res.keep_alive(req.keep_alive());
+            res.body() = "An error occurred: '" + std::string(what) + "'";
+            res.prepare_payload();
+            return res;
+         };
+
+         // Make sure we can handle the method
+         if (!self()->allow_method(req.method()))
+            return bad_request("Unknown HTTP-method");
+
+         // Request path must be absolute and not contain "..".
+         std::string_view target{req.target().data(), req.target().size()};
+         if (target.empty() || target[0] != '/' || target.find("..") != std::string_view::npos)
+            return bad_request("Illegal request-target");
+
+         try {
+            auto res = self()->on_request(std::move(req));
+            if (!res)
+               not_found(target);
+            return *res;
+         } catch (std::exception& ex) { return server_error(ex.what()); }
+      }
+
+      class session : public std::enable_shared_from_this<session> {
+         tcp::socket                                        socket_;
+         boost::asio::io_context::strand                    strand_;
+         beast::flat_buffer                                 buffer_;
+         http::request<http::string_body>                   req_;
+         simple_server*                                     server_;
+         std::shared_ptr<http::response<http::string_body>> res_;
+
+       public:
+         // Take ownership of the stream
+         session(net::io_context& ioc, tcp::socket&& socket, simple_server* server)
+             : socket_(std::move(socket)), strand_(ioc), server_(server) {}
+
+         // Start the asynchronous operation
+         void run() { do_read(); }
+
+         void do_read() {
+            // Make the request empty before reading,
+            // otherwise the operation behavior is undefined.
+            req_ = {};
+
+            // Read a request
+            http::async_read(
+                  socket_, buffer_, req_,
+                  boost::asio::bind_executor(strand_, [self = this->shared_from_this()](beast::error_code ec,
+                                                                                        std::size_t bytes_transferred) {
+                     self->on_read(ec, bytes_transferred);
+                  }));
+         }
+
+         void on_read(beast::error_code ec, std::size_t bytes_transferred) {
+            boost::ignore_unused(bytes_transferred);
+
+            // This means they closed the connection
+            if (ec == http::error::end_of_stream)
+               return do_close();
+
+            if (ec)
+               return server_->fail(ec, "read");
+
+            // Send the response
+            send_response(server_->handle_request(std::move(req_)));
+         }
+
+         void send_response(http::response<http::string_body>&& msg) {
+            // The lifetime of the message has to extend
+            // for the duration of the async operation so
+            // we use a shared_ptr to manage it.
+            res_ = std::make_shared<http::response<http::string_body>>(std::move(msg));
+
+            // Write the response
+            http::async_write(socket_, *res_,
+                              boost::asio::bind_executor(socket_.get_executor(),
+                                                         [self = this->shared_from_this(), close = res_->need_eof()](
+                                                               beast::error_code ec, std::size_t bytes_transferred) {
+                                                            self->on_write(ec, bytes_transferred, close);
+                                                         }));
+         }
+
+         void on_write(boost::system::error_code ec, std::size_t bytes_transferred, bool close) {
+            boost::ignore_unused(bytes_transferred);
+
+            if (ec)
+               return server_->fail(ec, "write");
+
+            if (close) {
+               // This means we should close the connection, usually because
+               // the response indicated the "Connection: close" semantic.
+               return do_close();
+            }
+
+            // We're done with the response so delete it
+            res_ = nullptr;
+
+            // Read another request
+            do_read();
+         }
+
+         void do_close() {
+            // Send a TCP shutdown
+            beast::error_code ec;
+            socket_.shutdown(tcp::socket::shutdown_send, ec);
+
+            // At this point the connection is closed gracefully
+         }
+      };
+
+      //------------------------------------------------------------------------------
+
+      // Accepts incoming connections and launches the sessions
+      class listener : public std::enable_shared_from_this<listener> {
+         net::io_context& ioc_;
+         tcp::acceptor    acceptor_;
+         tcp::socket      socket_;
+         simple_server*   server_;
+
+       public:
+         listener(net::io_context& ioc, tcp::endpoint endpoint, simple_server* server)
+             : ioc_(ioc), acceptor_(ioc), socket_(ioc), server_(server) {
+            boost::system::error_code ec;
+
+            // Open the acceptor
+            acceptor_.open(endpoint.protocol(), ec);
+            if (ec) {
+               server_->fail(ec, "open");
+               return;
+            }
+
+            // Allow address reuse
+            acceptor_.set_option(net::socket_base::reuse_address(true), ec);
+            if (ec) {
+               server_->fail(ec, "set_option");
+               return;
+            }
+
+            // Bind to the server address
+            acceptor_.bind(endpoint, ec);
+            if (ec) {
+               server_->fail(ec, "bind");
+               return;
+            }
+
+            // Start listening for connections
+            acceptor_.listen(net::socket_base::max_listen_connections, ec);
+            if (ec) {
+               server_->fail(ec, "listen");
+               return;
+            }
+         }
+
+         // Start accepting incoming connections
+         void run() {
+            if (!acceptor_.is_open())
+               return;
+            do_accept();
+         }
+
+       private:
+         void do_accept() {
+            acceptor_.async_accept(
+                  socket_, [self = this->shared_from_this()](boost::system::error_code ec) { self->on_accept(ec); });
+         }
+
+         void on_accept(boost::system::error_code ec) {
+            if (ec) {
+               server_->fail(ec, "accept");
+            } else {
+               // Create the session and run it
+               std::make_shared<session>(ioc_, std::move(socket_), server_)->run();
+            }
+
+            // Accept another connection
+            do_accept();
+         }
+      };
+
+    public:
+      void run(net::io_context& ioc, tcp::endpoint endpoint) { std::make_shared<listener>(ioc, endpoint, this)->run(); }
+   };
+}} // namespace eosio::rest

plugins/prometheus_plugin/prometheus_plugin.cpp

@@ -1,8 +1,11 @@
 #include <eosio/prometheus_plugin/prometheus_plugin.hpp>
+#include <eosio/prometheus_plugin/simple_rest_server.hpp>
+
 #include <eosio/chain/plugin_interface.hpp>
 #include <eosio/chain/thread_utils.hpp>
 #include <eosio/http_plugin/macros.hpp>
 #include <eosio/net_plugin/net_plugin.hpp>
+#include <eosio/http_plugin/http_plugin.hpp>
 #include <eosio/producer_plugin/producer_plugin.hpp>
 
 #include <fc/log/logger.hpp>
@@ -14,8 +17,8 @@
 #include <prometheus/text_serializer.h>
 #include <prometheus/registry.h>
 
-
 namespace eosio {
+   static const char* prometheus_api_name = "/v1/prometheus/metrics";
    using namespace prometheus;
    using namespace chain::plugin_interface;
 
@@ -110,12 +113,41 @@ namespace eosio {
       }
    };
 
-   struct prometheus_plugin_impl {
+   namespace http = boost::beast::http;
+   struct prometheus_plugin_impl : rest::simple_server<prometheus_plugin_impl> {
+
+      std::string server_header() const {
+         return http_plugin::get_server_header();
+      }
+
+      void log_error(char const* what, const std::string& message) {
+         elog("${what}: ${message}", ("what", what)("message", message));
+      }
+
+      bool allow_method(http::verb method) const {
+         return method == http::verb::get;
+      }
+
+      std::optional<http::response<http::string_body>>
+       on_request(http::request<http::string_body>&& req) {
+         if(req.target() != prometheus_api_name)
+            return {};
+         http::response<http::string_body> res{ http::status::ok, req.version() };
+         // Respond to GET request
+         res.set(http::field::server, server_header());
+         res.set(http::field::content_type, "text/plain");
+         res.keep_alive(req.keep_alive());
+         res.body() = metrics();
+         res.prepare_payload();
+         return res;
+      }
+
       eosio::chain::named_thread_pool<struct prom> _prometheus_thread_pool;
       boost::asio::io_context::strand _prometheus_strand;
       prometheus_plugin_metrics _metrics;
 
       map<std::string, vector<runtime_metric>> _plugin_metrics;
+      boost::asio::ip::tcp::endpoint           _endpoint;
 
       prometheus_plugin_impl(): _prometheus_strand(_prometheus_thread_pool.get_executor()){ }
 
@@ -179,32 +211,11 @@ namespace eosio {
          return body;
       }
 
-      void metrics_async(chain::plugin_interface::next_function<std::string> results) {
-         _prometheus_strand.post([self=this, results=std::move(results)]() {
-            results(self->metrics());
-         });
+      void start() {
+         run(_prometheus_thread_pool.get_executor(), _endpoint);
+         _prometheus_thread_pool.start(
+               1, [](const fc::exception& e) { elog("Prometheus exception ${e}", ("e", e)); });
       }
-
-   };
-
-   using metrics_params = fc::variant_object;
-
-   struct prometheus_api {
-      prometheus_plugin_impl& _pp;
-      fc::microseconds _max_response_time_us;
-
-      fc::time_point start() const {
-         return fc::time_point::now() + _max_response_time_us;
-      }
-
-      void metrics(const metrics_params&, chain::plugin_interface::next_function<std::string> results) {
-         _pp.metrics_async(std::move(results));
-      }
-
-      prometheus_api(prometheus_plugin_impl& plugin, const fc::microseconds& max_response_time)
-      : _pp(plugin)
-      , _max_response_time_us(max_response_time){}
-
    };
 
    prometheus_plugin::prometheus_plugin()
@@ -214,24 +225,35 @@ namespace eosio {
    prometheus_plugin::~prometheus_plugin() = default;
 
    void prometheus_plugin::set_program_options(options_description&, options_description& cfg) {
+      cfg.add_options()
+         ("prometheus-exporter-address", bpo::value<string>()->default_value("127.0.0.1:9101"),
+            "The local IP and port to listen for incoming prometheus metrics http request.");
    }
 
    void prometheus_plugin::plugin_initialize(const variables_map& options) {
       my->initialize_metrics();
 
-      auto& _http_plugin = app().get_plugin<http_plugin>();
-      fc::microseconds max_response_time = _http_plugin.get_max_response_time();
+      string lipstr = options.at("prometheus-exporter-address").as<string>();
+      EOS_ASSERT(lipstr.size() > 0, chain::plugin_config_exception, "prometheus-exporter-address must have a value");
 
-      prometheus_api handle(*my, max_response_time);
-      app().get_plugin<http_plugin>().add_async_api({
-        CALL_ASYNC_WITH_400(prometheus, handle, eosio, metrics, std::string, 200, http_params_types::no_params)}, http_content_type::plaintext);
+      string host = lipstr.substr(0, lipstr.find(':'));
+      string port = lipstr.substr(host.size() + 1, lipstr.size());
+
+      boost::system::error_code ec;
+      using tcp = boost::asio::ip::tcp;
+      tcp::resolver resolver(app().get_io_service());
+
+      my->_endpoint = *resolver.resolve(tcp::v4(), host, port, ec);
+      if (!ec) {
+         fc_ilog(logger(), "configured prometheus metrics exporter to listen on ${h}", ("h", lipstr));
+      } else {
+         fc_elog(logger(), "failed to configure prometheus metrics exporter to listen on ${h} (${m})",
+                 ("h", lipstr)("m", ec.message()));
+      }
    }
 
    void prometheus_plugin::plugin_startup() {
-      my->_prometheus_thread_pool.start(1, []( const fc::exception& e ) {
-         elog("Prometheus excpetion ${e}:${l}", ("e", e));
-      } );
-
+      my->start();
       ilog("Prometheus plugin started.");
    }
 

plugins/state_history_plugin/include/eosio/state_history_plugin/session.hpp

@@ -103,7 +103,7 @@ public:
    void pop_entry(bool call_send = true) {
       send_queue.erase(send_queue.begin());
       sending = false;
-      if (call_send)
+      if (call_send || !send_queue.empty())
          send();
    }
 
@@ -550,6 +550,7 @@ private:
 
       state_history::get_blocks_result_v0 result;
       result.head = {block_state->block_num, block_state->id};
+      to_send_block_num = std::min(block_state->block_num, to_send_block_num);
       send_update(std::move(result), block_state);
    }
 

plugins/test_control_api_plugin/test_control_api_plugin.cpp

@@ -51,7 +51,7 @@ void test_control_api_plugin::plugin_startup() {
 
    app().get_plugin<http_plugin>().add_api({
       TEST_CONTROL_RW_CALL(kill_node_on_producer, 202, http_params_types::params_required)
-   }, appbase::exec_queue::general);
+   }, appbase::exec_queue::read_write);
 }
 
 void test_control_api_plugin::plugin_shutdown() {}

plugins/wallet_api_plugin/wallet_api_plugin.cpp

@@ -115,7 +115,7 @@ void wallet_api_plugin::plugin_startup() {
             INVOKE_R_R_R(wallet_mgr, list_keys, std::string, std::string), 200),
        CALL_WITH_400(wallet, wallet_mgr, get_public_keys,
             INVOKE_R_V(wallet_mgr, get_public_keys), 200)
-   }, appbase::exec_queue::general);
+   }, appbase::exec_queue::read_write);
 }
 
 void wallet_api_plugin::plugin_initialize(const variables_map& options) {

programs/keosd/main.cpp

@@ -111,7 +111,7 @@ int main(int argc, char** argv)
                        [&a=app](string, string, url_response_callback cb) {
          cb(200, fc::time_point::maximum(), fc::variant(fc::variant_object()));
          a->quit();
-      }, appbase::exec_queue::general );
+      }, appbase::exec_queue::read_write );
       app->startup();
       app->exec();
    } catch (const fc::exception& e) {

scripts/CMakeLists.txt

@@ -6,3 +6,4 @@ configure_file(eosio-tn_up.sh eosio-tn_up.sh COPYONLY)
 configure_file(abi_is_json.py abi_is_json.py COPYONLY)
 configure_file(postinst .)
 configure_file(prerm .)
+configure_file(install_testharness_symlinks.cmake .)

scripts/install_testharness_symlinks.cmake

@@ -0,0 +1,5 @@
+execute_process( COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages  ERROR_QUIET RESULT_VARIABLE ret)
+if(ret EQUAL "0")
+    execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink ../../../share/leap_testing/tests/TestHarness ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages/TestHarness)
+    execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink ../../bin ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing/bin)
+endif()

tests/CMakeLists.txt

@@ -113,8 +113,8 @@ set_property(TEST ship_test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME ship_test_unix COMMAND tests/ship_test.py -v --num-clients 10 --num-requests 5000 --clean-run ${UNSHARE} --unix-socket WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST ship_test_unix PROPERTY LABELS nonparallelizable_tests)
 
-add_test(NAME ship_streamer_test COMMAND tests/ship_streamer_test.py -v --num-clients 10 --num-blocks 50 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
-set_property(TEST ship_streamer_test PROPERTY LABELS nonparallelizable_tests)
+add_test(NAME ship_streamer_test COMMAND tests/ship_streamer_test.py -v --num-clients 10 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+set_property(TEST ship_streamer_test PROPERTY LABELS long_running_tests)
 
 add_test(NAME p2p_dawn515_test COMMAND tests/p2p_tests/dawn_515/test.sh WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST p2p_dawn515_test PROPERTY LABELS nonparallelizable_tests)
@@ -125,11 +125,11 @@ add_test(NAME nodeos_protocol_feature_test COMMAND tests/nodeos_protocol_feature
 set_property(TEST nodeos_protocol_feature_test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME compute_transaction_test COMMAND tests/compute_transaction_test.py -v -p 2 -n 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST compute_transaction_test PROPERTY LABELS nonparallelizable_tests)
-add_test(NAME read-only-trx-basic-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME read-only-trx-basic-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --read-only-threads 0 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST read-only-trx-basic-test PROPERTY LABELS nonparallelizable_tests)
-add_test(NAME read-only-trx-parallel-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --read-only-threads 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME read-only-trx-parallel-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --read-only-threads 3 --num-test-runs 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST read-only-trx-parallel-test PROPERTY LABELS nonparallelizable_tests)
-add_test(NAME read-only-trx-parallel-eos-vm-oc-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --eos-vm-oc-enable --read-only-threads 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME read-only-trx-parallel-eos-vm-oc-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --eos-vm-oc-enable --read-only-threads 3 --num-test-runs 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST read-only-trx-parallel-eos-vm-oc-test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME subjective_billing_test COMMAND tests/subjective_billing_test.py -v -p 2 -n 4 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST subjective_billing_test PROPERTY LABELS nonparallelizable_tests)

tests/TestHarness/Cluster.py

@@ -22,8 +22,11 @@ from .Node import BlockType
 from .Node import Node
 from .WalletMgr import WalletMgr
 from .launch_transaction_generators import TransactionGeneratorsLauncher, TpsTrxGensConfig
-from .libc import unshare, CLONE_NEWNET
-from .interfaces import getInterfaceFlags, setInterfaceUp, IFF_LOOPBACK
+try:
+    from .libc import unshare, CLONE_NEWNET
+    from .interfaces import getInterfaceFlags, setInterfaceUp, IFF_LOOPBACK
+except:
+    pass
 
 # Protocol Feature Setup Policy
 class PFSetupPolicy:
@@ -1447,7 +1450,7 @@ class Cluster(object):
     def killall(self, kill=True, silent=True, allInstances=False):
         """Kill cluster nodeos instances. allInstances will kill all nodeos instances running on the system."""
         signalNum=9 if kill else 15
-        cmd="%s -k %d --nogen -p 1 -n 1 --nodeos-log-path %s" % (f"python3 {str(self.launcherPath)}", signalNum, self.nodeosLogPath)
+        cmd="%s -k %d --nogen -p 1 -n 1 --nodeos-log-path %s" % (f"{sys.executable} {str(self.launcherPath)}", signalNum, self.nodeosLogPath)
         if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
         if 0 != subprocess.call(cmd.split(), stdout=Utils.FNull):
             if not silent: Utils.Print("Launcher failed to shut down eos cluster.")
@@ -1727,4 +1730,7 @@ class Cluster(object):
                 for line in f:
                     firstTrxs.append(line.rstrip('\n'))
         Utils.Print(f"first transactions: {firstTrxs}")
-        node.waitForTransactionsInBlock(firstTrxs)
+        status = node.waitForTransactionsInBlock(firstTrxs)
+        if status is None:
+            Utils.Print('ERROR: Failed to spin up transaction generators: never received first transactions')
+        return status

tests/TestHarness/Node.py

@@ -61,7 +61,7 @@ class Node(Transactions):
             self.fetchBlock = lambda blockNum: self.processUrllibRequest("trace_api", "get_block", {"block_num":blockNum}, silentErrors=False, exitOnError=True)
             self.fetchKeyCommand = lambda: "[transaction][transaction_header][ref_block_num]"
             self.fetchRefBlock = lambda trans: trans["transaction_header"]["ref_block_num"]
-            self.cleosLimit = "--time-limit 99999"
+            self.cleosLimit = "--time-limit 999"
             self.fetchHeadBlock = lambda node, headBlock: node.processUrllibRequest("chain", "get_block_info", {"block_num":headBlock}, silentErrors=False, exitOnError=True)
 
     def __str__(self):

tests/TestHarness/WalletMgr.py

@@ -12,9 +12,9 @@ from .testUtils import Utils
 Wallet=namedtuple("Wallet", "name password host port")
 # pylint: disable=too-many-instance-attributes
 class WalletMgr(object):
-    __walletLogOutFile=f"{Utils.TestLogRoot}/test_keosd_out.log"
-    __walletLogErrFile=f"{Utils.TestLogRoot}/test_keosd_err.log"
     __walletDataDir=f"{Utils.TestLogRoot}/test_wallet_0"
+    __walletLogOutFile=f"{__walletDataDir}/test_keosd_out.log"
+    __walletLogErrFile=f"{__walletDataDir}/test_keosd_err.log"
     __MaxPort=9999
 
     # pylint: disable=too-many-arguments
@@ -83,6 +83,9 @@ class WalletMgr(object):
         cmd="%s --data-dir %s --config-dir %s --unlock-timeout=999999 --http-server-address=%s:%d --http-max-response-time-ms 99999 --verbose-http-errors" % (
             Utils.EosWalletPath, WalletMgr.__walletDataDir, WalletMgr.__walletDataDir, self.host, self.port)
         if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
+        if not os.path.isdir(WalletMgr.__walletDataDir):
+            if Utils.Debug: Utils.Print(f"Creating dir {WalletMgr.__walletDataDir} in dir: {os.getcwd()}")
+            os.mkdir(WalletMgr.__walletDataDir)
         with open(WalletMgr.__walletLogOutFile, 'w') as sout, open(WalletMgr.__walletLogErrFile, 'w') as serr:
             popen=subprocess.Popen(cmd.split(), stdout=sout, stderr=serr)
             self.__walletPid=popen.pid
@@ -299,6 +302,5 @@ class WalletMgr(object):
 
     @staticmethod
     def cleanup():
-        dataDir=WalletMgr.__walletDataDir
-        if os.path.isdir(dataDir) and os.path.exists(dataDir):
+        if os.path.isdir(WalletMgr.__walletDataDir) and os.path.exists(WalletMgr.__walletDataDir):
             shutil.rmtree(WalletMgr.__walletDataDir)

tests/TestHarness/queries.py

@@ -118,10 +118,10 @@ class NodeosQueries:
         # could be a transaction response
         if cntxt.hasKey("processed"):
             cntxt.add("processed")
-            cntxt.add("action_traces")
-            cntxt.index(0)
             if not cntxt.isSectionNull("except"):
                 return "no_block"
+            cntxt.add("action_traces")
+            cntxt.index(0)
             return cntxt.add("block_num")
 
         # or what the trace api plugin returns
@@ -242,7 +242,7 @@ class NodeosQueries:
         assert(isinstance(transId, str))
         exitOnErrorForDelayed=not delayedRetry and exitOnError
         timeout=3
-        cmdDesc="get transaction_trace"
+        cmdDesc=self.fetchTransactionCommand()
         cmd="%s %s" % (cmdDesc, transId)
         msg="(transaction id=%s)" % (transId);
         for i in range(0,(int(60/timeout) - 1)):
@@ -295,8 +295,8 @@ class NodeosQueries:
         refBlockNum=None
         key=""
         try:
-            key="[transaction][transaction_header][ref_block_num]"
-            refBlockNum=trans["transaction_header"]["ref_block_num"]
+            key = self.fetchKeyCommand()
+            refBlockNum = self.fetchRefBlock(trans)
             refBlockNum=int(refBlockNum)+1
         except (TypeError, ValueError, KeyError) as _:
             Utils.Print("transaction%s not found. Transaction: %s" % (key, trans))
@@ -346,9 +346,9 @@ class NodeosQueries:
         return self.processCleosCmd(cmd, cmdDesc, silentErrors=False, exitOnError=exitOnError, exitMsg=msg, returnType=returnType)
 
     def getTable(self, contract, scope, table, exitOnError=False):
-        cmdDesc = "get table --time-limit 999"
-        cmd="%s %s %s %s" % (cmdDesc, contract, scope, table)
-        msg="contract=%s, scope=%s, table=%s" % (contract, scope, table);
+        cmdDesc = "get table"
+        cmd=f"{cmdDesc} {self.cleosLimit} {contract} {scope} {table}"
+        msg=f"contract={contract}, scope={scope}, table={table}"
         return self.processCleosCmd(cmd, cmdDesc, exitOnError=exitOnError, exitMsg=msg)
 
     def getTableAccountBalance(self, contract, scope):
@@ -529,7 +529,7 @@ class NodeosQueries:
             return m.group(1)
         except subprocess.CalledProcessError as ex:
             end=time.perf_counter()
-            msg=ex.output.decode("utf-8")
+            msg=ex.stderr.decode("utf-8")
             Utils.Print("ERROR: Exception during code hash retrieval.  cmd Duration: %.3f sec.  %s" % (end-start, msg))
             return None
 
@@ -580,8 +580,9 @@ class NodeosQueries:
         except subprocess.CalledProcessError as ex:
             if not silentErrors:
                 end=time.perf_counter()
-                msg=ex.output.decode("utf-8")
-                errorMsg="Exception during \"%s\". Exception message: %s.  cmd Duration=%.3f sec. %s" % (cmdDesc, msg, end-start, exitMsg)
+                out=ex.output.decode("utf-8")
+                msg=ex.stderr.decode("utf-8")
+                errorMsg="Exception during \"%s\". Exception message: %s.  stdout: %s.  cmd Duration=%.3f sec. %s" % (cmdDesc, msg, out, end-start, exitMsg)
                 if exitOnError:
                     Utils.cmdError(errorMsg)
                     Utils.errorExit(errorMsg)
@@ -595,15 +596,18 @@ class NodeosQueries:
 
         return trans
 
-    def processUrllibRequest(self, resource, command, payload={}, silentErrors=False, exitOnError=False, exitMsg=None, returnType=ReturnType.json, endpoint=None):
+    def processUrllibRequest(self, resource, command, payload={}, silentErrors=False, exitOnError=False, exitMsg=None, returnType=ReturnType.json, method="POST", endpoint=None):
         if not endpoint:
             endpoint = self.endpointHttp
         cmd = f"{endpoint}/v1/{resource}/{command}"
-        req = urllib.request.Request(cmd, method="POST")
-        req.add_header('Content-Type', 'application/json')
-        data = payload
-        data = json.dumps(data)
-        data = data.encode()
+        req = urllib.request.Request(cmd, method=method)
+        if len(payload):
+            req.add_header('Content-Type', 'application/json')
+            data = payload
+            data = json.dumps(data)
+            data = data.encode()
+        else:
+            data = None
         if Utils.Debug: Utils.Print("cmd: %s %s" % (cmd, payload))
         rtn=None
         start=time.perf_counter()
@@ -771,4 +775,3 @@ class NodeosQueries:
     def getActivatedProtocolFeatures(self):
         latestBlockHeaderState = self.getLatestBlockHeaderState()
         return latestBlockHeaderState["activated_protocol_features"]["protocol_features"]
-

tests/TestHarness/transactions.py

@@ -107,7 +107,7 @@ class Transactions(NodeosQueries):
                 self.trackCmdTransaction(trans, reportStatus=reportStatus)
         except subprocess.CalledProcessError as ex:
             end=time.perf_counter()
-            msg=ex.output.decode("utf-8")
+            msg=ex.stderr.decode("utf-8")
             Utils.Print("ERROR: Exception during funds transfer.  cmd Duration: %.3f sec.  %s" % (end-start, msg))
             if exitOnError:
                 Utils.cmdError("could not transfer \"%s\" from %s to %s" % (amountStr, source, destination))
@@ -131,7 +131,7 @@ class Transactions(NodeosQueries):
                 Utils.Print("cmd Duration: %.3f sec" % (end-start))
         except subprocess.CalledProcessError as ex:
             end=time.perf_counter()
-            msg=ex.output.decode("utf-8")
+            msg=ex.stderr.decode("utf-8")
             Utils.Print("ERROR: Exception during spawn of funds transfer.  cmd Duration: %.3f sec.  %s" % (end-start, msg))
             if exitOnError:
                 Utils.cmdError("could not transfer \"%s\" from %s to %s" % (amountStr, source, destination))
@@ -158,15 +158,15 @@ class Transactions(NodeosQueries):
         except subprocess.CalledProcessError as ex:
             if not shouldFail:
                 end=time.perf_counter()
-                msg=ex.output.decode("utf-8")
-                Utils.Print("ERROR: Exception during set contract.  cmd Duration: %.3f sec.  %s" % (end-start, msg))
+                out=ex.output.decode("utf-8")
+                msg=ex.stderr.decode("utf-8")
+                Utils.Print("ERROR: Exception during set contract. stderr: %s.  stdout: %s.  cmd Duration: %.3f sec." % (msg, out, end-start))
                 return None
             else:
                 retMap={}
                 retMap["returncode"]=ex.returncode
                 retMap["cmd"]=ex.cmd
                 retMap["output"]=ex.output
-                retMap["stdout"]=ex.stdout
                 retMap["stderr"]=ex.stderr
                 return retMap
 
@@ -213,7 +213,7 @@ class Transactions(NodeosQueries):
                 Utils.Print("cmd Duration: %.3f sec" % (end-start))
             return (NodeosQueries.getTransStatus(retTrans) == 'executed', retTrans)
         except subprocess.CalledProcessError as ex:
-            msg=ex.output.decode("utf-8")
+            msg=ex.stderr.decode("utf-8")
             if not silentErrors:
                 end=time.perf_counter()
                 Utils.Print("ERROR: Exception during push transaction.  cmd Duration=%.3f sec.  %s" % (end - start, msg))
@@ -245,7 +245,6 @@ class Transactions(NodeosQueries):
         except subprocess.CalledProcessError as ex:
             msg=ex.stderr.decode("utf-8")
             output=ex.output.decode("utf-8")
-            msg=ex.output.decode("utf-8")
             if not silentErrors:
                 end=time.perf_counter()
                 Utils.Print("ERROR: Exception during push message. stderr: %s. stdout: %s.  cmd Duration=%.3f sec." % (msg, output, end - start))

tests/auto_bp_peering_test.py

@@ -35,8 +35,7 @@ args = TestHelper.parse_args({
 Utils.Debug = args.v
 killAll = args.clean_run
 dumpErrorDetails = args.dump_error_details
-# dontKill = args.leave_running
-dontKill = True
+dontKill = args.leave_running
 killEosInstances = not dontKill
 killWallet = not dontKill
 keepLogs = args.keep_logs

tests/cli_test.py

@@ -11,7 +11,7 @@ import time
 import shutil
 import signal
 
-from TestHarness import Account, Cluster, Node, ReturnType, Utils, WalletMgr
+from TestHarness import Account, Node, ReturnType, Utils, WalletMgr
 
 testSuccessful=False
 
@@ -352,7 +352,7 @@ def abi_file_with_nodeos_test():
         os.makedirs(data_dir, exist_ok=True)
         walletMgr = WalletMgr(True)
         walletMgr.launch()
-        node = Node('localhost', 8888, nodeId, cmd="./programs/nodeos/nodeos -e -p eosio --plugin eosio::trace_api_plugin --trace-no-abis --plugin eosio::producer_plugin --plugin eosio::producer_api_plugin --plugin eosio::chain_api_plugin --plugin eosio::chain_plugin --plugin eosio::http_plugin --access-control-allow-origin=* --http-validate-host=false --resource-monitor-not-shutdown-on-threshold-exceeded " + "--data-dir " + data_dir + " --config-dir " + data_dir, walletMgr=walletMgr)
+        node = Node('localhost', 8888, nodeId, cmd="./programs/nodeos/nodeos -e -p eosio --plugin eosio::trace_api_plugin --trace-no-abis --plugin eosio::producer_plugin --plugin eosio::producer_api_plugin --plugin eosio::chain_api_plugin --plugin eosio::chain_plugin --plugin eosio::http_plugin --access-control-allow-origin=* --http-validate-host=false --max-transaction-time=-1 --resource-monitor-not-shutdown-on-threshold-exceeded " + "--data-dir " + data_dir + " --config-dir " + data_dir, walletMgr=walletMgr)
         node.verifyAlive() # Setting node state to not alive
         node.relaunch(newChain=True, cachePopen=True)
         node.waitForBlock(1)
@@ -403,7 +403,7 @@ def abi_file_with_nodeos_test():
                     os.kill(node.pid, signal.SIGKILL)
         if testSuccessful:
             Utils.Print("Cleanup nodeos data.")
-            shutil.rmtree(data_dir)
+            shutil.rmtree(Utils.DataPath)
 
         if malicious_token_abi_path:
             if os.path.exists(malicious_token_abi_path):

tests/launcher.py

@@ -174,7 +174,7 @@ class launcher(object):
         parser.add_argument('-i', '--timestamp', help='set the timestamp for the first block.  Use "now" to indicate the current time')
         parser.add_argument('-l', '--launch', choices=['all', 'none', 'local'], help='select a subset of nodes to launch.  If not set, the default is to launch all unless an output file is named, in which case none are started.', default='all')
         parser.add_argument('-o', '--output', help='save a copy of the generated topology in this file and exit without launching', dest='topology_filename')
-        parser.add_argument('-k', '--kill', help='retrieve the list of previously started process ids and issue a kill to each')
+        parser.add_argument('-k', '--kill', type=int, help='kill a network as specified in arguments with given signal')
         parser.add_argument('--down', type=comma_separated, help='comma-separated list of node numbers that will be shut down', default=[])
         parser.add_argument('--bounce', type=comma_separated, help='comma-separated list of node numbers that will be restarted', default=[])
         parser.add_argument('--roll', type=comma_separated, help='comma-separated list of host names where the nodes will be rolled to a new version')
@@ -613,6 +613,17 @@ plugin = eosio::chain_api_plugin
                     if relaunch:
                         self.launch(node)
 
+    def kill(self, signum):
+        errorCode = 0
+        for node in self.network.nodes.values():
+            try:
+                with open(node.data_dir_name / f'{Utils.EosServerName}.pid', 'r') as f:
+                    pid = int(f.readline())
+                    self.terminate_wait_pid(pid, signum, raise_if_missing=False)
+            except FileNotFoundError as err:
+                errorCode = 1
+        return errorCode
+
     def start_all(self):
         if self.args.launch.lower() != 'none':
             for instance in self.network.nodes.values():
@@ -628,8 +639,9 @@ plugin = eosio::chain_api_plugin
                     f.write(f'"{node.dot_label}"->"{pname}" [dir="forward"];\n')
             f.write('}')
 
-    def terminate_wait_pid(self, pid, raise_if_missing=True):
-        '''Terminate a non-child process with SIGTERM and wait for it to exit.'''
+    def terminate_wait_pid(self, pid, signum = signal.SIGTERM, raise_if_missing=True):
+        '''Terminate a non-child process with given signal number or with SIGTERM if not
+           provided and wait for it to exit.'''
         if sys.version_info >= (3, 9) and platform.system() == 'Linux': # on our supported platforms, Python 3.9 accompanies a kernel > 5.3
             try:
                 fd = os.pidfd_open(pid)
@@ -640,7 +652,7 @@ plugin = eosio::chain_api_plugin
                 po = select.poll()
                 po.register(fd, select.POLLIN)
                 try:
-                    os.kill(pid, signal.SIGTERM)
+                    os.kill(pid, signum)
                 except ProcessLookupError:
                     if raise_if_missing:
                         raise
@@ -663,7 +675,7 @@ plugin = eosio::chain_api_plugin
                     return min(delay * 2, 0.04)
                 delay = 0.0001
                 try:
-                    os.kill(pid, signal.SIGTERM)
+                    os.kill(pid, signum)
                 except ProcessLookupError:
                     if raise_if_missing:
                         raise
@@ -676,12 +688,16 @@ plugin = eosio::chain_api_plugin
                         return
 
 if __name__ == '__main__':
+    errorCode = 0
     l = launcher(sys.argv[1:])
     if len(l.args.down):
         l.down(l.args.down)
     elif len(l.args.bounce):
         l.bounce(l.args.bounce)
+    elif l.args.kill:
+        errorCode = l.kill(l.args.kill)
     elif l.args.launch == 'all' or l.args.launch == 'local':
         l.start_all()
     for f in glob.glob(Utils.DataPath):
         shutil.rmtree(f)
+    sys.exit(errorCode)

tests/nodeos_contrl_c_test.py

@@ -115,7 +115,7 @@ try:
         errorExit("Failed to kill the seed node")
 
 finally:
-    TestHelper.shutdown(cluster, walletMgr, testSuccessful=testSuccessful, killEosInstances=True, killWallet=True, keepLogs=True, cleanRun=True, dumpErrorDetails=True)
+    TestHelper.shutdown(cluster, walletMgr, testSuccessful=testSuccessful, killEosInstances=True, killWallet=True, keepLogs=False, cleanRun=True, dumpErrorDetails=True)
 
 errorCode = 0 if testSuccessful else 1
 exit(errorCode)

tests/nodeos_snapshot_diff_test.py

@@ -133,7 +133,8 @@ try:
                                 acctPrivKeysList=[account1PrivKey,account2PrivKey], nodeId=snapshotNodeId, tpsPerGenerator=targetTpsPerGenerator,
                                 numGenerators=trxGeneratorCnt, durationSec=testTrxGenDurationSec, waitToComplete=False)
 
-    cluster.waitForTrxGeneratorsSpinup(nodeId=snapshotNodeId, numGenerators=trxGeneratorCnt)
+    status = cluster.waitForTrxGeneratorsSpinup(nodeId=snapshotNodeId, numGenerators=trxGeneratorCnt)
+    assert status is not None, "ERROR: Failed to spinup Transaction Generators"
 
     blockNum=node0.getBlockNum(BlockType.head)
     timePerBlock=500

tests/nodeos_snapshot_forked_test.py

@@ -171,7 +171,7 @@ try:
         errorExit(f"Failure - (non-production) node {nonProdNode.nodeNum} should have restarted")
 
     Print("Wait for LIB to move, which indicates prodC has forked out the branch")
-    assert prodC.waitForLibToAdvance(), \
+    assert prodC.waitForLibToAdvance(60), \
         "ERROR: Network did not reach consensus after bridge node was restarted."
  
     for prodNode in prodNodes:

tests/nodeos_startup_catchup.py

@@ -118,7 +118,8 @@ try:
                                 tpsPerGenerator=targetTpsPerGenerator, numGenerators=trxGeneratorCnt, durationSec=testTrxGenDurationSec,
                                 waitToComplete=False)
 
-    cluster.waitForTrxGeneratorsSpinup(nodeId=node0.nodeId, numGenerators=trxGeneratorCnt)
+    status = cluster.waitForTrxGeneratorsSpinup(nodeId=node0.nodeId, numGenerators=trxGeneratorCnt)
+    assert status is not None, "ERROR: Failed to spinup Transaction Generators"
 
     blockNum=head(node0)
     timePerBlock=500

tests/performance_tests/performance_test_basic.py

@@ -497,15 +497,17 @@ class PerformanceTestBasic:
             traceback.print_exc()
 
         finally:
+            # Despite keepLogs being hardcoded to False, logs will still appear on test failure in TestLogs
+            # due to testSuccessful being False
             TestHelper.shutdown(
-                self.cluster,
-                self.walletMgr,
-                testSuccessful,
-                self.testHelperConfig._killEosInstances,
-                self.testHelperConfig._killWallet,
-                self.testHelperConfig.keepLogs,
-                self.testHelperConfig.killAll,
-                self.testHelperConfig.dumpErrorDetails
+                cluster=self.cluster,
+                walletMgr=self.walletMgr,
+                testSuccessful=testSuccessful,
+                killEosInstances=self.testHelperConfig._killEosInstances,
+                killWallet=self.testHelperConfig._killWallet,
+                keepLogs=False,
+                cleanRun=self.testHelperConfig.killAll,
+                dumpErrorDetails=self.testHelperConfig.dumpErrorDetails
                 )
 
             if self.ptbConfig.delPerfLogs:

tests/plugin_http_api_test.py

@@ -40,8 +40,8 @@ class PluginHttpTest(unittest.TestCase):
         self.keosd.killall(True)
         WalletMgr.cleanup()
         Node.killAllNodeos()
-        if os.path.exists(self.data_dir):
-            shutil.rmtree(self.data_dir)
+        if os.path.exists(Utils.DataPath):
+            shutil.rmtree(Utils.DataPath)
         if os.path.exists(self.config_dir):
             shutil.rmtree(self.config_dir)
         time.sleep(self.sleep_s)
@@ -56,7 +56,7 @@ class PluginHttpTest(unittest.TestCase):
                         "http_plugin", "db_size_api_plugin", "prometheus_plugin"]
         nodeos_plugins = "--plugin eosio::" +  " --plugin eosio::".join(plugin_names)
         nodeos_flags = (" --data-dir=%s --config-dir=%s --trace-dir=%s --trace-no-abis --access-control-allow-origin=%s "
-                        "--contracts-console --http-validate-host=%s --verbose-http-errors --abi-serializer-max-time-ms 30000 --http-max-response-time-ms 30000 "
+                        "--contracts-console --http-validate-host=%s --verbose-http-errors --max-transaction-time -1 --abi-serializer-max-time-ms 30000 --http-max-response-time-ms 30000 "
                         "--p2p-peer-address localhost:9011 --resource-monitor-not-shutdown-on-threshold-exceeded ") % (self.data_dir, self.config_dir, self.data_dir, "\'*\'", "false")
         start_nodeos_cmd = ("%s -e -p eosio %s %s ") % (Utils.EosServerPath, nodeos_plugins, nodeos_flags)
         self.nodeos.launchCmd(start_nodeos_cmd, self.node_id)
@@ -1341,8 +1341,8 @@ class PluginHttpTest(unittest.TestCase):
     def test_prometheusApi(self) :
         resource = "prometheus"
         command = "metrics"
-
-        ret_text = self.nodeos.processUrllibRequest(resource, command, returnType = ReturnType.raw ).decode()
+        endpointPrometheus = f'http://{self.nodeos.host}:9101'
+        ret_text = self.nodeos.processUrllibRequest(resource, command, returnType = ReturnType.raw, method="GET", endpoint=endpointPrometheus).decode()
         # filter out all empty lines or lines starting with '#'
         data_lines = filter(lambda line: len(line) > 0 and line[0]!='#', ret_text.split('\n'))
         # converting each line into a key value pair and then construct a dictionay out of all the pairs

tests/read_only_trx_test.py

@@ -21,6 +21,7 @@ errorExit=Utils.errorExit
 
 appArgs=AppArgs()
 appArgs.add(flag="--read-only-threads", type=int, help="number of read-only threads", default=0)
+appArgs.add(flag="--num-test-runs", type=int, help="number of times to run the tests", default=1)
 appArgs.add_bool(flag="--eos-vm-oc-enable", help="enable eos-vm-oc")
 appArgs.add(flag="--wasm-runtime", type=str, help="if set to eos-vm-oc, must compile with EOSIO_EOS_VM_OC_DEVELOPER", default="eos-vm-jit")
 
@@ -44,6 +45,7 @@ dontKill=args.leave_running
 dumpErrorDetails=args.dump_error_details
 killAll=args.clean_run
 keepLogs=args.keep_logs
+numTestRuns=args.num_test_runs
 
 killWallet=not dontKill
 killEosInstances=not dontKill
@@ -91,9 +93,8 @@ try:
     specificExtraNodeosArgs={}
     # producer nodes will be mapped to 0 through pnodes-1, so the number pnodes is the no-producing API node
     specificExtraNodeosArgs[pnodes]=" --plugin eosio::net_api_plugin"
-    if args.read_only_threads > 0:
-        specificExtraNodeosArgs[pnodes]+=" --read-only-threads "
-        specificExtraNodeosArgs[pnodes]+=str(args.read_only_threads)
+    specificExtraNodeosArgs[pnodes]+=" --read-only-threads "
+    specificExtraNodeosArgs[pnodes]+=str(args.read_only_threads)
     if args.eos_vm_oc_enable:
         specificExtraNodeosArgs[pnodes]+=" --eos-vm-oc-enable"
     if args.wasm_runtime:
@@ -259,7 +260,7 @@ try:
         runReadOnlyTrxAndRpcInParallel("chain", "get_raw_code_and_abi", "account_name", expectedValue=testAccountName, payload = {"account_name":testAccountName})
         runReadOnlyTrxAndRpcInParallel("chain", "get_raw_abi", "account_name", expectedValue=testAccountName, payload = {"account_name":testAccountName})
         runReadOnlyTrxAndRpcInParallel("chain", "get_producers", "rows", payload = {"json":"true","lower_bound":""})
-        runReadOnlyTrxAndRpcInParallel("chain", "get_table_rows", code=500, payload = {"json":"true","table":"noauth"})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_table_rows", "rows", payload = {"json":"true","code":"eosio","scope":"eosio","table":"global"})
         runReadOnlyTrxAndRpcInParallel("chain", "get_table_by_scope", fieldIn="rows", payload = {"json":"true","table":"noauth"})
         runReadOnlyTrxAndRpcInParallel("chain", "get_currency_balance", code=200, payload = {"code":"eosio.token", "account":testAccountName})
         runReadOnlyTrxAndRpcInParallel("chain", "get_currency_stats", fieldIn="SYS", payload = {"code":"eosio.token", "symbol":"SYS"})
@@ -276,13 +277,26 @@ try:
         runReadOnlyTrxAndRpcInParallel("net", "connect", code=201, payload = "localhost")
         runReadOnlyTrxAndRpcInParallel("net", "disconnect", code=201, payload = "localhost")
 
+    def runEverythingParallel():
+        threadList = []
+        threadList.append(threading.Thread(target = multiReadOnlyTests))
+        threadList.append(threading.Thread(target = chainApiTests))
+        threadList.append(threading.Thread(target = netApiTests))
+        threadList.append(threading.Thread(target = mixedOpsTest))
+        for thr in threadList:
+            thr.start()
+        for thr in threadList:
+            thr.join()
+
     basicTests()
 
     if args.read_only_threads > 0: # Save test time. No need to run other tests if multi-threaded is not enabled
-        multiReadOnlyTests()
-        chainApiTests()
-        netApiTests()
-        mixedOpsTest()
+        for i in range(numTestRuns):
+            multiReadOnlyTests()
+            chainApiTests()
+            netApiTests()
+            mixedOpsTest()
+            runEverythingParallel()
 
     testSuccessful = True
 finally:

tests/ship_client.cpp

@@ -125,6 +125,8 @@ int main(int argc, char* argv[]) {
             eosio::check(result_doucment[1]["head"].IsObject(),                                 "'head' is not an object");
             eosio::check(result_doucment[1]["head"].HasMember("block_num"),                     "'head' does not contain 'block_num'");
             eosio::check(result_doucment[1]["head"]["block_num"].IsUint(),                      "'head.block_num' isn't a number");
+            eosio::check(result_doucment[1]["head"].HasMember("block_id"),                      "'head' does not contain 'block_id'");
+            eosio::check(result_doucment[1]["head"]["block_id"].IsString(),                     "'head.block_id' isn't a string");
 
             uint32_t this_block_num = result_doucment[1]["head"]["block_num"].GetUint();
 

tests/ship_streamer.cpp

@@ -111,30 +111,61 @@ int main(int argc, char* argv[]) {
       stream.write(boost::asio::buffer(request_type.json_to_bin(request_sb.GetString(), [](){})));
       stream.read_message_max(0);
 
+      //      block_num, block_id
+      std::map<uint32_t, std::string> block_ids;
       bool is_first = true;
       for(;;) {
          boost::beast::flat_buffer buffer;
          stream.read(buffer);
 
          eosio::input_stream is((const char*)buffer.data().data(), buffer.data().size());
-         rapidjson::Document result_doucment;
-         result_doucment.Parse(result_type.bin_to_json(is).c_str());
+         rapidjson::Document result_document;
+         result_document.Parse(result_type.bin_to_json(is).c_str());
 
-         eosio::check(!result_doucment.HasParseError(),                                      "Failed to parse result JSON from abieos");
-         eosio::check(result_doucment.IsArray(),                                             "result should have been an array (variant) but it's not");
-         eosio::check(result_doucment.Size() == 2,                                           "result was an array but did not contain 2 items like a variant should");
-         eosio::check(std::string(result_doucment[0].GetString()) == "get_blocks_result_v0", "result type doesn't look like get_blocks_result_v0");
-         eosio::check(result_doucment[1].IsObject(),                                         "second item in result array is not an object");
-         eosio::check(result_doucment[1].HasMember("head"),                                  "cannot find 'head' in result");
-         eosio::check(result_doucment[1]["head"].IsObject(),                                 "'head' is not an object");
-         eosio::check(result_doucment[1]["head"].HasMember("block_num"),                     "'head' does not contain 'block_num'");
-         eosio::check(result_doucment[1]["head"]["block_num"].IsUint(),                      "'head.block_num' isn't a number");
+         eosio::check(!result_document.HasParseError(),                                      "Failed to parse result JSON from abieos");
+         eosio::check(result_document.IsArray(),                                             "result should have been an array (variant) but it's not");
+         eosio::check(result_document.Size() == 2,                                           "result was an array but did not contain 2 items like a variant should");
+         eosio::check(std::string(result_document[0].GetString()) == "get_blocks_result_v0", "result type doesn't look like get_blocks_result_v0");
+         eosio::check(result_document[1].IsObject(),                                         "second item in result array is not an object");
+         eosio::check(result_document[1].HasMember("head"),                                  "cannot find 'head' in result");
+         eosio::check(result_document[1]["head"].IsObject(),                                 "'head' is not an object");
+         eosio::check(result_document[1]["head"].HasMember("block_num"),                     "'head' does not contain 'block_num'");
+         eosio::check(result_document[1]["head"]["block_num"].IsUint(),                      "'head.block_num' isn't a number");
+         eosio::check(result_document[1]["head"].HasMember("block_id"),                      "'head' does not contain 'block_id'");
+         eosio::check(result_document[1]["head"]["block_id"].IsString(),                     "'head.block_id' isn't a string");
 
          uint32_t this_block_num = 0;
-         if( result_doucment[1].HasMember("this_block") && result_doucment[1]["this_block"].IsObject() ) {
-            if( result_doucment[1]["this_block"].HasMember("block_num") && result_doucment[1]["this_block"]["block_num"].IsUint() ) {
-               this_block_num = result_doucment[1]["this_block"]["block_num"].GetUint();
+         if( result_document[1].HasMember("this_block") && result_document[1]["this_block"].IsObject() ) {
+            if( result_document[1]["this_block"].HasMember("block_num") && result_document[1]["this_block"]["block_num"].IsUint() ) {
+               this_block_num = result_document[1]["this_block"]["block_num"].GetUint();
             }
+            std::string this_block_id;
+            if( result_document[1]["this_block"].HasMember("block_id") && result_document[1]["this_block"]["block_id"].IsString() ) {
+               this_block_id = result_document[1]["this_block"]["block_id"].GetString();
+            }
+            std::string prev_block_id;
+            if( result_document[1]["prev_block"].HasMember("block_id") && result_document[1]["prev_block"]["block_id"].IsString() ) {
+               prev_block_id = result_document[1]["prev_block"]["block_id"].GetString();
+            }
+            if( !irreversible_only && !this_block_id.empty() && !prev_block_id.empty() ) {
+               // verify forks were sent
+               if (block_ids.count(this_block_num-1)) {
+                  if (block_ids[this_block_num-1] != prev_block_id) {
+                     std::cerr << "Received block: << " << this_block_num << " that does not link to previous: " << block_ids[this_block_num-1] << std::endl;
+                     return 1;
+                  }
+               }
+               block_ids[this_block_num] = this_block_id;
+
+               if( result_document[1]["last_irreversible"].HasMember("block_num") && result_document[1]["last_irreversible"]["block_num"].IsUint() ) {
+                  uint32_t lib_num = result_document[1]["last_irreversible"]["block_num"].GetUint();
+                  auto i = block_ids.lower_bound(lib_num);
+                  if (i != block_ids.end()) {
+                     block_ids.erase(block_ids.begin(), i);
+                  }
+               }
+            }
+
          }
 
          if(is_first) {
@@ -147,7 +178,7 @@ int main(int argc, char* argv[]) {
 
          rapidjson::StringBuffer result_sb;
          rapidjson::PrettyWriter<rapidjson::StringBuffer> result_writer(result_sb);
-         result_doucment[1].Accept(result_writer);
+         result_document[1].Accept(result_writer);
          std::cout << result_sb.GetString() << std::endl << "}" << std::endl;
 
          if( this_block_num == end_block_num ) break;

tests/ship_streamer_test.py

@@ -7,34 +7,33 @@ import shutil
 import signal
 import sys
 
-from TestHarness import Cluster, TestHelper, Utils, WalletMgr
+from TestHarness import Cluster, TestHelper, Utils, WalletMgr, CORE_SYMBOL
 from TestHarness.TestHelper import AppArgs
 
 ###############################################################
 # ship_streamer_test
 # 
-# This test sets up <-p> producing node(s) and <-n - -p>
-#   non-producing node(s). One of the non-producing nodes
-#   is configured with the state_history_plugin.  An instance
-#   of node will be started with ship_streamer to exercise
-#   the SHiP API.
+# This test sets up 2 producing nodes and one "bridge" node using test_control_api_plugin.
+#   One producing node has 3 of the elected producers and the other has 1 of the elected producers.
+#   All the producers are named in alphabetical order, so that the 3 producers, in the one production node, are
+#       scheduled first, followed by the 1 producer in the other producer node. Each producing node is only connected
+#       to the other producing node via the "bridge" node.
+#   The bridge node has the test_control_api_plugin, that the test uses to kill
+#       the "bridge" node to generate a fork.
+#   ship_streamer is used to connect to the state_history_plugin and verify that blocks receive link to previous
+#   blocks. If the blocks do not link then ship_streamer will exit with an error causing this test to generate an
+#   error. The fork generated by nodeos should be sent to the ship_streamer so it is able to correctly observe the
+#   fork.
 #
 ###############################################################
 
 Print=Utils.Print
 
 appArgs = AppArgs()
-extraArgs = appArgs.add(flag="--num-blocks", type=int, help="How many blocsk to stream from ship_streamer", default=20)
 extraArgs = appArgs.add(flag="--num-clients", type=int, help="How many ship_streamers should be started", default=1)
-args = TestHelper.parse_args({"-p", "-n","--dump-error-details","--keep-logs","-v","--leave-running","--clean-run","--unshared"}, applicationSpecificArgs=appArgs)
+args = TestHelper.parse_args({"--dump-error-details","--keep-logs","-v","--leave-running","--clean-run","--unshared"}, applicationSpecificArgs=appArgs)
 
 Utils.Debug=args.v
-totalProducerNodes=args.p
-totalNodes=args.n
-if totalNodes<=totalProducerNodes:
-    totalNodes=totalProducerNodes+1
-totalNonProducerNodes=totalNodes-totalProducerNodes
-totalProducers=totalProducerNodes
 cluster=Cluster(walletd=True,unshared=args.unshared)
 dumpErrorDetails=args.dump_error_details
 keepLogs=args.keep_logs
@@ -42,6 +41,12 @@ dontKill=args.leave_running
 killAll=args.clean_run
 walletPort=TestHelper.DEFAULT_WALLET_PORT
 
+totalProducerNodes=2
+totalNonProducerNodes=1
+totalNodes=totalProducerNodes+totalNonProducerNodes
+maxActiveProducers=21
+totalProducers=maxActiveProducers
+
 walletMgr=WalletMgr(True, port=walletPort)
 testSuccessful=False
 killEosInstances=not dontKill
@@ -57,12 +62,20 @@ try:
     cluster.killall(allInstances=killAll)
     cluster.cleanup()
     Print("Stand up cluster")
-    specificExtraNodeosArgs={}
-    # non-producing nodes are at the end of the cluster's nodes, so reserving the last one for state_history_plugin
-    shipNodeNum = totalNodes - 1
-    specificExtraNodeosArgs[shipNodeNum]="--plugin eosio::state_history_plugin --disable-replay-opts --trace-history --sync-fetch-span 200 --plugin eosio::net_api_plugin "
 
-    if cluster.launch(pnodes=totalProducerNodes,
+
+    # ***   setup topogrophy   ***
+
+    # "bridge" shape connects defprocera through defproducerc (3 in node0) to each other and defproduceru (1 in node1)
+    # and the only connection between those 2 groups is through the bridge node
+
+    shipNodeNum = 1
+    specificExtraNodeosArgs={}
+    specificExtraNodeosArgs[shipNodeNum]="--plugin eosio::state_history_plugin --disable-replay-opts --trace-history --chain-state-history --plugin eosio::net_api_plugin "
+    # producer nodes will be mapped to 0 through totalProducerNodes-1, so the number totalProducerNodes will be the non-producing node
+    specificExtraNodeosArgs[totalProducerNodes]="--plugin eosio::test_control_api_plugin  "
+
+    if cluster.launch(topo="bridge", pnodes=totalProducerNodes,
                       totalNodes=totalNodes, totalProducers=totalProducers,
                       specificExtraNodeosArgs=specificExtraNodeosArgs) is False:
         Utils.cmdError("launcher")
@@ -70,19 +83,81 @@ try:
 
     # ***   identify each node (producers and non-producing node)   ***
 
-    shipNode = cluster.getNode(shipNodeNum)
-    prodNode = cluster.getNode(0)
-
     #verify nodes are in sync and advancing
     cluster.waitOnClusterSync(blockAdvancing=5)
     Print("Cluster in Sync")
 
+    prodNode = cluster.getNode(0)
+    prodNode0 = prodNode
+    prodNode1 = cluster.getNode(1)
+    nonProdNode = cluster.getNode(2)
+    shipNode = cluster.getNode(shipNodeNum)
+
+
+    accounts=cluster.createAccountKeys(6)
+    if accounts is None:
+        Utils.errorExit("FAILURE - create keys")
+
+    accounts[0].name="testeraaaaaa"
+    accounts[1].name="tester111111" # needed for voting
+    accounts[2].name="tester222222" # needed for voting
+    accounts[3].name="tester333333" # needed for voting
+    accounts[4].name="tester444444" # needed for voting
+    accounts[5].name="tester555555" # needed for voting
+
+    testWalletName="test"
+
+    Print(f"Creating wallet {testWalletName}.")
+    testWallet=walletMgr.create(testWalletName, [cluster.eosioAccount,accounts[0],accounts[1],accounts[2],accounts[3],accounts[4],accounts[5]])
+
+    for _, account in cluster.defProducerAccounts.items():
+        walletMgr.importKey(account, testWallet, ignoreDupKeyWarning=True)
+
+    for i in range(0, totalNodes):
+        node=cluster.getNode(i)
+        node.producers=Cluster.parseProducers(i)
+        for prod in node.producers:
+            prodName = cluster.defProducerAccounts[prod].name
+            if prodName == "defproducera" or prodName == "defproducerb" or prodName == "defproducerc" or prodName == "defproduceru":
+                Print(f"Register producer {prodName}")
+                trans=node.regproducer(cluster.defProducerAccounts[prod], "http://mysite.com", 0, waitForTransBlock=False, exitOnError=True)
+
+    # create accounts via eosio as otherwise a bid is needed
+    for account in accounts:
+        Print(f"Create new account {account.name} via {cluster.eosioAccount.name} with private key: {account.activePrivateKey}")
+        trans=nonProdNode.createInitializeAccount(account, cluster.eosioAccount, stakedDeposit=0, waitForTransBlock=True, stakeNet=10000, stakeCPU=10000, buyRAM=10000000, exitOnError=True)
+        transferAmount="100000000.0000 {0}".format(CORE_SYMBOL)
+        Print(f"Transfer funds {transferAmount} from account {cluster.eosioAccount.name} to {account.name}")
+        nonProdNode.transferFunds(cluster.eosioAccount, account, transferAmount, "test transfer", waitForTransBlock=True)
+        trans=nonProdNode.delegatebw(account, 20000000.0000, 20000000.0000, waitForTransBlock=False, exitOnError=True)
+
+    # ***   vote using accounts   ***
+
+    cluster.waitOnClusterSync(blockAdvancing=3)
     start_block_num = shipNode.getBlockNum()
-    end_block_num = start_block_num + args.num_blocks
+
+    # vote a,b,c (node0)  u (node1)
+    voteProducers=[]
+    voteProducers.append("defproducera")
+    voteProducers.append("defproducerb")
+    voteProducers.append("defproducerc")
+    voteProducers.append("defproduceru")
+    for account in accounts:
+        Print(f"Account {account.name} vote for producers={voteProducers}")
+        trans=prodNode.vote(account, voteProducers, exitOnError=True, waitForTransBlock=False)
+
+    #verify nodes are in sync and advancing
+    cluster.waitOnClusterSync(blockAdvancing=3)
+    Print("Shutdown unneeded bios node")
+    cluster.biosNode.kill(signal.SIGTERM)
+    prodNode0.waitForProducer("defproducerc")
+
+    block_range = 350
+    end_block_num = start_block_num + block_range
 
     shipClient = "tests/ship_streamer"
-    cmd = "%s --start-block-num %d --end-block-num %d --fetch-block --fetch-traces --fetch-deltas" % (shipClient, start_block_num, end_block_num)
-    if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
+    cmd = f"{shipClient} --start-block-num {start_block_num} --end-block-num {end_block_num} --fetch-block --fetch-traces --fetch-deltas"
+    if Utils.Debug: Utils.Print(f"cmd: {cmd}")
     clients = []
     files = []
     shipTempDir = os.path.join(Utils.DataDir, "ship")
@@ -92,28 +167,61 @@ try:
     starts = []
     for i in range(0, args.num_clients):
         start = time.perf_counter()
-        outFile = open("%s%d.out" % (shipClientFilePrefix, i), "w")
-        errFile = open("%s%d.err" % (shipClientFilePrefix, i), "w")
-        Print("Start client %d" % (i))
+        outFile = open(f"{shipClientFilePrefix}{i}.out", "w")
+        errFile = open(f"{shipClientFilePrefix}{i}.err", "w")
+        Print(f"Start client {i}")
         popen=Utils.delayedCheckOutput(cmd, stdout=outFile, stderr=errFile)
         starts.append(time.perf_counter())
         clients.append((popen, cmd))
         files.append((outFile, errFile))
-        Print("Client %d started, Ship node head is: %s" % (i, shipNode.getBlockNum()))
+        Print(f"Client {i} started, Ship node head is: {shipNode.getBlockNum()}")
 
-    Print("Stopping all %d clients" % (args.num_clients))
+    # Generate a fork
+    forkAtProducer="defproducera"
+    prodNode1Prod="defproduceru"
+    preKillBlockNum=nonProdNode.getBlockNum()
+    preKillBlockProducer=nonProdNode.getBlockProducerByNum(preKillBlockNum)
+    nonProdNode.killNodeOnProducer(producer=forkAtProducer, whereInSequence=1)
+    Print(f"Current block producer {preKillBlockProducer} fork will be at producer {forkAtProducer}")
+    prodNode0.waitForProducer(forkAtProducer)
+    prodNode1.waitForProducer(prodNode1Prod)
+    if nonProdNode.verifyAlive(): # if on defproducera, need to wait again
+        prodNode0.waitForProducer(forkAtProducer)
+        prodNode1.waitForProducer(prodNode1Prod)
+    if nonProdNode.verifyAlive():
+        Utils.errorExit("Bridge did not shutdown");
+    Print("Fork started")
 
+    prodNode0.waitForProducer("defproducerb") # wait for fork to progress a bit
+
+    Print("Restore fork")
+    Print("Relaunching the non-producing bridge node to connect the producing nodes again")
+    if nonProdNode.verifyAlive():
+        Utils.errorExit("Bridge is already running");
+    if not nonProdNode.relaunch():
+        Utils.errorExit(f"Failure - (non-production) node {nonProdNode.nodeNum} should have restarted")
+
+    nonProdNode.waitForProducer(forkAtProducer)
+    nonProdNode.waitForProducer(prodNode1Prod)
+    afterForkBlockNum = nonProdNode.getBlockNum()
+    if int(afterForkBlockNum) > int(end_block_num):
+        Utils.errorExit(f"Did not stream long enough {end_block_num} to cover the fork {afterForkBlockNum}, increase block_range {block_range}")
+
+    Print(f"Stopping all {args.num_clients} clients")
     for index, (popen, _), (out, err), start in zip(range(len(clients)), clients, files, starts):
         popen.wait()
-        Print("Stopped client %d.  Ran for %.3f seconds." % (index, time.perf_counter() - start))
+        Print(f"Stopped client {index}.  Ran for {time.perf_counter() - start:.3f} seconds.")
         out.close()
         err.close()
-        outFile = open("%s%d.out" % (shipClientFilePrefix, index), "r")
+        outFile = open(f"{shipClientFilePrefix}{index}.out", "r")
         data = json.load(outFile)
         block_num = start_block_num
         for i in data:
-            print(i)
-            assert block_num == i['get_blocks_result_v0']['this_block']['block_num'], f"{block_num} != {i['get_blocks_result_v0']['this_block']['block_num']}"
+            # fork can cause block numbers to be repeated
+            this_block_num = i['get_blocks_result_v0']['this_block']['block_num']
+            if this_block_num < block_num:
+                block_num = this_block_num
+            assert block_num == this_block_num, f"{block_num} != {this_block_num}"
             assert isinstance(i['get_blocks_result_v0']['block'], str) # verify block in result
             block_num += 1
         assert block_num-1 == end_block_num, f"{block_num-1} != {end_block_num}"

tests/ship_test.py

@@ -132,7 +132,7 @@ try:
             try:
                 statuses = json.loads(" ".join(lines))
             except json.decoder.JSONDecodeError as er:
-                Utils.errorExit("javascript client output was malformed in %s. Exception: %s" % (shipClientErrorFile, er))
+                Utils.errorExit("ship_client output was malformed in %s. Exception: %s" % (shipClientErrorFile, er))
 
             for status in statuses:
                 statusDesc = status["status"]
@@ -143,7 +143,7 @@ try:
                     maxFirstBN = max(maxFirstBN, firstBlockNum)
                     minLastBN = min(minLastBN, lastBlockNum)
                 if statusDesc == "error":
-                    Utils.errorExit("javascript client reporting error see: %s." % (shipClientErrorFile))
+                    Utils.errorExit("ship_client reporting error see: %s." % (shipClientErrorFile))
 
         assert done, Print("ERROR: Did not find a \"done\" status for client %d" % (i))
 

tests/trace_plugin_test.py

@@ -8,6 +8,8 @@ import os
 
 from TestHarness import Cluster, Node, TestHelper, Utils, WalletMgr, CORE_SYMBOL
 
+testSuccessful = True
+
 class TraceApiPluginTest(unittest.TestCase):
     sleep_s = 1
     cluster=Cluster(walletd=True, defproduceraPrvtKey=None)
@@ -101,6 +103,8 @@ class TraceApiPluginTest(unittest.TestCase):
                     self.assertIn('memo', prms)
                 break
         self.assertTrue(isTrxInBlockFromTraceApi)
+        global testSuccessful
+        testSuccessful = True
 
     @classmethod
     def setUpClass(self):
@@ -109,7 +113,7 @@ class TraceApiPluginTest(unittest.TestCase):
 
     @classmethod
     def tearDownClass(self):
-        self.cleanEnv(self, shouldCleanup=False)   # not cleanup to save log in case for further investigation
+        self.cleanEnv(self, shouldCleanup=testSuccessful)
 
 if __name__ == "__main__":
     unittest.main()

tests/trx_finality_status_forked_test.py

@@ -184,7 +184,7 @@ try:
         errorExit(f"Failure - (non-production) node {nonProdNode.nodeNum} should have restarted")
 
     Print("Wait for LIB to move, which indicates prodC has forked out the branch")
-    assert prodC.waitForLibToAdvance(), \
+    assert prodC.waitForLibToAdvance(60), \
         "ERROR: Network did not reach consensus after bridge node was restarted."
 
     retStatus = prodC.getTransactionStatus(transId)

unittests/restart_chain_tests.cpp

@@ -237,19 +237,4 @@ BOOST_AUTO_TEST_CASE(test_light_validation_restart_from_block_log) {
    BOOST_CHECK_EQUAL(trace->action_traces.at(1).receipt->digest(), other_trace->action_traces.at(1).receipt->digest());
 }
 
-namespace{
-   struct scoped_temp_path {
-      boost::filesystem::path path;
-      scoped_temp_path() {
-         path = boost::filesystem::unique_path();
-         if (boost::unit_test::framework::master_test_suite().argc >= 2) {
-            path += boost::unit_test::framework::master_test_suite().argv[1];
-         }
-      }
-      ~scoped_temp_path() {
-         boost::filesystem::remove_all(path);
-      }
-   };
-}
-
 BOOST_AUTO_TEST_SUITE_END()

unittests/snapshot_tests.cpp

@@ -412,8 +412,8 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(test_compatible_versions, SNAPSHOT_SUITE, snapshot
       bfs::copy_file(source_i, source_log_dir / "blocks.index", bfs::copy_option::overwrite_if_exists);
       chain.close();
    }
-
-   auto config = tester::default_config(fc::temp_directory(), legacy_default_max_inline_action_size).first;
+   fc::temp_directory temp_dir;
+   auto config = tester::default_config(temp_dir, legacy_default_max_inline_action_size).first;
    auto genesis = eosio::chain::block_log::extract_genesis_state(source_log_dir);
    bfs::create_directories(config.blocks_dir);
    bfs::copy(source_log_dir / "blocks.log", config.blocks_dir / "blocks.log");
@@ -477,7 +477,8 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(test_pending_schedule_snapshot, SNAPSHOT_SUITE, sn
    source_log_dir_str += "prod_sched";
    const auto source_log_dir = bfs::path(source_log_dir_str.c_str());
    const uint32_t legacy_default_max_inline_action_size = 4 * 1024;
-   auto config = tester::default_config(fc::temp_directory(), legacy_default_max_inline_action_size).first;
+   fc::temp_directory temp_dir;
+   auto config = tester::default_config(temp_dir, legacy_default_max_inline_action_size).first;
    auto genesis = eosio::chain::block_log::extract_genesis_state(source_log_dir);
    bfs::create_directories(config.blocks_dir);
    bfs::copy(source_log_dir / "blocks.log", config.blocks_dir / "blocks.log");