Merge branch 'fix_missing_node_py_changes' of https://github.com/AntelopeIO/leap into fix_missing_node_py_changes

fix noisy `ignoring packed attribute` warning on `IR::ControlStructureImm`

CMakeLists.txt

@@ -14,9 +14,9 @@ set( CMAKE_CXX_EXTENSIONS ON )
 set( CXX_STANDARD_REQUIRED ON)
 
 set(VERSION_MAJOR 4)
-set(VERSION_MINOR 0)
+set(VERSION_MINOR 1)
 set(VERSION_PATCH 0)
-set(VERSION_SUFFIX rc2)
+set(VERSION_SUFFIX dev)
 
 if(VERSION_SUFFIX)
     set(VERSION_FULL "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}-${VERSION_SUFFIX}")
@@ -268,8 +268,18 @@ if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.21)
    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/lib/python3/dist-packages/TestHarness DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages COMPONENT dev EXCLUDE_FROM_ALL)
    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/share/leap_testing/bin DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing COMPONENT dev EXCLUDE_FROM_ALL)
 else()
-   # The following install(SCRIPT ...) steps are necessary for `make dev-install` to work on cmake versions < 3.21.
-   install(SCRIPT ${CMAKE_CURRENT_BINARY_DIR}/scripts/install_testharness_symlinks.cmake COMPONENT dev EXCLUDE_FROM_ALL)
+   # The following install(CODE ...) steps are necessary for `make dev-install` to work on cmake versions < 3.21.
+   # However, it can flag as an error if using `make package` instead of `make dev-install` for installation.
+   # This is due to those directories and symbolic links being created in the postinit script during the package install instead.
+
+   # Note If/when doing `make package`, it is not a true error if you see:
+   # Error creating directory "/<leap_install_dir>/lib/python3/dist-packages".
+   # CMake Error: failed to create symbolic link '/<leap_install_dir>/lib/python3/dist-packages/TestHarness': no such file or directory
+   # CMake Error: failed to create symbolic link '/<leap_install_dir>/share/leap_testing/bin': no such file or directory
+
+   install(CODE "execute_process( COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages)" COMPONENT dev EXCLUDE_FROM_ALL)
+   install(CODE "execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink ../../../share/leap_testing/tests/TestHarness ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages/TestHarness)" COMPONENT dev EXCLUDE_FROM_ALL)
+   install(CODE "execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink ../../bin ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing/bin)" COMPONENT dev EXCLUDE_FROM_ALL)
 
    # The `make package` installation of symlinks happens via the `postinst` script installed in cmake.package via the line below
 endif()

libraries/chain/block_log.cpp

@@ -1533,7 +1533,7 @@ namespace eosio { namespace chain {
       ilog("blocks.log and blocks.index agree on number of blocks");
 
       if (interval == 0) {
-         interval = std::max((log_bundle.log_index.num_blocks() + 7) >> 3, 1);
+         interval = std::max((log_bundle.log_index.num_blocks() + 7u) >> 3, 1u);
       }
       uint32_t expected_block_num = log_bundle.log_data.first_block_num();
 

libraries/chain/controller.cpp

@@ -210,12 +210,6 @@ struct pending_state {
 };
 
 struct controller_impl {
-   enum class app_window_type {
-      write, // Only main thread is running; read-only threads are not running.
-             // All read-write and read-only tasks are sequentially executed.
-      read   // Main thread and read-only threads are running read-ony tasks in parallel.
-             // Read-write tasks are not being executed.
-   };
 
    // LLVM sets the new handler, we need to reset this to throw a bad_alloc exception so we can possibly exit cleanly
    // and not just abort.
@@ -248,13 +242,15 @@ struct controller_impl {
    bool                            okay_to_print_integrity_hash_on_stop = false;
 
    std::thread::id                 main_thread_id;
+   thread_local static platform_timer timer; // a copy for main thread and each read-only thread
 #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED)
    thread_local static vm::wasm_allocator wasm_alloc; // a copy for main thread and each read-only thread
 #endif
+   // Ideally wasmif should be thread_local which must be a static.
+   // Unittests can create multiple controller objects (testers) at the same time,
+   // which overwrites the same static wasmif, is used for eosvmoc too.
    wasm_interface  wasmif;  // used by main thread and all threads for EOSVMOC
-   std::mutex threaded_wasmifs_mtx;
-   std::unordered_map<std::thread::id, std::unique_ptr<wasm_interface>> threaded_wasmifs; // one for each read-only thread, used by eos-vm and eos-vm-jit
-   app_window_type app_window = app_window_type::write;
+   thread_local static std::unique_ptr<wasm_interface> wasmif_thread_local; // a copy for each read-only thread, used by eos-vm and eos-vm-jit
 
    typedef pair<scope_name,action_name>                   handler_key;
    map< account_name, map<handler_key, apply_handler> >   apply_handlers;
@@ -341,12 +337,7 @@ struct controller_impl {
       set_activation_handler<builtin_protocol_feature_t::crypto_primitives>();
 
       self.irreversible_block.connect([this](const block_state_ptr& bsp) {
-         // producer_plugin has already asserted irreversible_block signal is
-         // called in write window
-         wasmif.current_lib(bsp->block_num);
-         for (auto& w: threaded_wasmifs) {
-            w.second->current_lib(bsp->block_num);
-         }
+         get_wasm_interface().current_lib(bsp->block_num);
       });
 
 
@@ -506,10 +497,9 @@ struct controller_impl {
 
    void replay(std::function<bool()> check_shutdown) {
       auto blog_head = blog.head();
-      if( !fork_db.root() ) {
+      if( !blog_head && !fork_db.root() ) {
          fork_db.reset( *head );
-         if (!blog_head)
-            return;
+         return;
       }
 
       replaying = true;
@@ -1173,7 +1163,6 @@ struct controller_impl {
          etrx.set_reference_block( self.head_block_id() );
       }
 
-      platform_timer timer;
       transaction_checktime_timer trx_timer(timer);
       const packed_transaction trx( std::move( etrx ) );
       transaction_context trx_context( self, trx, std::move(trx_timer), start );
@@ -1341,7 +1330,6 @@ struct controller_impl {
 
       uint32_t cpu_time_to_bill_us = billed_cpu_time_us;
 
-      platform_timer timer;
       transaction_checktime_timer trx_timer( timer );
       transaction_context trx_context( self, *trx->packed_trx(), std::move(trx_timer) );
       trx_context.leeway =  fc::microseconds(0); // avoid stealing cpu resource
@@ -1556,7 +1544,6 @@ struct controller_impl {
          }
 
          const signed_transaction& trn = trx->packed_trx()->get_signed_transaction();
-         platform_timer timer;
          transaction_checktime_timer trx_timer(timer);
          transaction_context trx_context(self, *trx->packed_trx(), std::move(trx_timer), start, trx->get_trx_type());
          if ((bool)subjective_cpu_leeway && self.is_speculative_block()) {
@@ -2693,59 +2680,38 @@ struct controller_impl {
    // only called from non-main threads (read-only trx execution threads)
    // when producer_plugin starts them
    void init_thread_local_data() {
-      EOS_ASSERT( !is_on_main_thread(), misc_exception, "init_thread_local_data called on the main thread");
+      EOS_ASSERT( !is_main_thread(), misc_exception, "init_thread_local_data called on the main thread");
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
       if ( is_eos_vm_oc_enabled() )
          // EOSVMOC needs further initialization of its thread local data
          wasmif.init_thread_local_data();
       else
 #endif
-      {
-         std::lock_guard g(threaded_wasmifs_mtx);
          // Non-EOSVMOC needs a wasmif per thread
-         threaded_wasmifs[std::this_thread::get_id()]  = std::make_unique<wasm_interface>( conf.wasm_runtime, conf.eosvmoc_tierup, db, conf.state_dir, conf.eosvmoc_config, !conf.profile_accounts.empty());
-      }
+         wasmif_thread_local = std::make_unique<wasm_interface>( conf.wasm_runtime, conf.eosvmoc_tierup, db, conf.state_dir, conf.eosvmoc_config, !conf.profile_accounts.empty());
    }
 
-   bool is_on_main_thread() { return main_thread_id == std::this_thread::get_id(); };
-
-   void set_to_write_window() {
-      app_window = app_window_type::write;
-   }
-   void set_to_read_window() {
-      app_window = app_window_type::read;
-   }
-   bool is_write_window() const {
-      return app_window == app_window_type::write;
-   }
+   bool is_main_thread() { return main_thread_id == std::this_thread::get_id(); };
 
    wasm_interface& get_wasm_interface() {
-      if ( is_on_main_thread()
+      if ( is_main_thread()
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
           || is_eos_vm_oc_enabled()
 #endif
          )
          return wasmif;
       else
-         return *threaded_wasmifs[std::this_thread::get_id()];
-   }
-
-   void code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num) {
-      // The caller of this function apply_eosio_setcode has already asserted that
-      // the transaction is not a read-only trx, which implies we are
-      // in write window. Safe to call threaded_wasmifs's code_block_num_last_used
-      wasmif.code_block_num_last_used(code_hash, vm_type, vm_version, block_num);
-      for (auto& w: threaded_wasmifs) {
-         w.second->code_block_num_last_used(code_hash, vm_type, vm_version, block_num);
-      }
+         return *wasmif_thread_local;
    }
 
    block_state_ptr fork_db_head() const;
 }; /// controller_impl
 
+thread_local platform_timer controller_impl::timer;
 #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED)
 thread_local eosio::vm::wasm_allocator controller_impl::wasm_alloc;
 #endif
+thread_local std::unique_ptr<wasm_interface> controller_impl::wasmif_thread_local;
 
 const resource_limits_manager&   controller::get_resource_limits_manager()const
 {
@@ -3720,20 +3686,6 @@ void controller::init_thread_local_data() {
    my->init_thread_local_data();
 }
 
-void controller::set_to_write_window() {
-   my->set_to_write_window();
-}
-void controller::set_to_read_window() {
-   my->set_to_read_window();
-}
-bool controller::is_write_window() const {
-   return my->is_write_window();
-}
-
-void controller::code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num) {
-   return my->code_block_num_last_used(code_hash, vm_type, vm_version, block_num);
-}
-
 /// Protocol feature activation handlers:
 
 template<>

libraries/chain/eosio_contract.cpp

@@ -160,7 +160,7 @@ void apply_eosio_setcode(apply_context& context) {
       old_size  = (int64_t)old_code_entry.code.size() * config::setcode_ram_bytes_multiplier;
       if( old_code_entry.code_ref_count == 1 ) {
          db.remove(old_code_entry);
-         context.control.code_block_num_last_used(account.code_hash, account.vm_type, account.vm_version, context.control.head_block_num() + 1);
+         context.control.get_wasm_interface().code_block_num_last_used(account.code_hash, account.vm_type, account.vm_version, context.control.head_block_num() + 1);
       } else {
          db.modify(old_code_entry, [](code_object& o) {
             --o.code_ref_count;

libraries/chain/include/eosio/chain/controller.hpp

@@ -373,10 +373,6 @@ namespace eosio { namespace chain {
       void set_db_read_only_mode();
       void unset_db_read_only_mode();
       void init_thread_local_data();
-      void set_to_write_window();
-      void set_to_read_window();
-      bool is_write_window() const;
-      void code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num);
 
       private:
          friend class apply_context;

libraries/chain/include/eosio/chain/log_index.hpp

@@ -32,7 +32,7 @@ class log_index {
    bool is_open() const { return file_.is_open(); }
 
    uint64_t back() { return nth_block_position(num_blocks()-1); }
-   int      num_blocks() const { return num_blocks_; }
+   unsigned num_blocks() const { return num_blocks_; }
    uint64_t nth_block_position(uint32_t n) {
       file_.seek(n*sizeof(uint64_t));
       uint64_t r;

libraries/chain/include/eosio/chain/wasm_interface_private.hpp

@@ -37,6 +37,7 @@ namespace eosio { namespace chain {
    struct wasm_interface_impl {
       struct wasm_cache_entry {
          digest_type                                          code_hash;
+         uint32_t                                             first_block_num_used;
          uint32_t                                             last_block_num_used;
          std::unique_ptr<wasm_instantiated_module_interface>  module;
          uint8_t                                              vm_type = 0;
@@ -108,6 +109,24 @@ namespace eosio { namespace chain {
          return it != wasm_instantiation_cache.end();
       }
 
+      std::vector<uint8_t> parse_initial_memory(const Module& module) {
+         std::vector<uint8_t> mem_image;
+
+         for(const DataSegment& data_segment : module.dataSegments) {
+            EOS_ASSERT(data_segment.baseOffset.type == InitializerExpression::Type::i32_const, wasm_exception, "");
+            EOS_ASSERT(module.memories.defs.size(), wasm_exception, "");
+            const U32 base_offset = data_segment.baseOffset.i32;
+            const Uptr memory_size = (module.memories.defs[0].type.size.min << IR::numBytesPerPageLog2);
+            if(base_offset >= memory_size || base_offset + data_segment.data.size() > memory_size)
+               FC_THROW_EXCEPTION(wasm_execution_error, "WASM data segment outside of valid memory range");
+            if(base_offset + data_segment.data.size() > mem_image.size())
+               mem_image.resize(base_offset + data_segment.data.size(), 0x00);
+            memcpy(mem_image.data() + base_offset, data_segment.data.data(), data_segment.data.size());
+         }
+
+         return mem_image;
+      }
+
       void code_block_num_last_used(const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version, const uint32_t& block_num) {
          wasm_cache_index::iterator it = wasm_instantiation_cache.find(boost::make_tuple(code_hash, vm_type, vm_version));
          if(it != wasm_instantiation_cache.end())
@@ -138,6 +157,7 @@ namespace eosio { namespace chain {
 
             it = wasm_instantiation_cache.emplace( wasm_interface_impl::wasm_cache_entry{
                                                       .code_hash = code_hash,
+                                                      .first_block_num_used = codeobject->first_block_used,
                                                       .last_block_num_used = UINT32_MAX,
                                                       .module = nullptr,
                                                       .vm_type = vm_type,
@@ -153,8 +173,24 @@ namespace eosio { namespace chain {
                trx_context.resume_billing_timer();
             });
             trx_context.pause_billing_timer();
+            IR::Module module;
+            std::vector<U8> bytes = {
+                (const U8*)codeobject->code.data(),
+                (const U8*)codeobject->code.data() + codeobject->code.size()};
+            try {
+               Serialization::MemoryInputStream stream((const U8*)bytes.data(),
+                                                       bytes.size());
+               WASM::scoped_skip_checks no_check;
+               WASM::serialize(stream, module);
+               module.userSections.clear();
+            } catch (const Serialization::FatalSerializationException& e) {
+               EOS_ASSERT(false, wasm_serialization_error, e.message.c_str());
+            } catch (const IR::ValidationException& e) {
+               EOS_ASSERT(false, wasm_serialization_error, e.message.c_str());
+            }
+
             wasm_instantiation_cache.modify(it, [&](auto& c) {
-               c.module = runtime_interface->instantiate_module(codeobject->code.data(), codeobject->code.size(), code_hash, vm_type, vm_version);
+               c.module = runtime_interface->instantiate_module((const char*)bytes.data(), bytes.size(), parse_initial_memory(module), code_hash, vm_type, vm_version);
             });
          }
          return it->module;
@@ -173,6 +209,7 @@ namespace eosio { namespace chain {
                   member<wasm_cache_entry, uint8_t,     &wasm_cache_entry::vm_version>
                >
             >,
+            ordered_non_unique<tag<by_first_block_num>, member<wasm_cache_entry, uint32_t, &wasm_cache_entry::first_block_num_used>>,
             ordered_non_unique<tag<by_last_block_num>, member<wasm_cache_entry, uint32_t, &wasm_cache_entry::last_block_num_used>>
          >
       > wasm_cache_index;

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc.hpp

@@ -30,7 +30,7 @@ class eosvmoc_runtime : public eosio::chain::wasm_runtime_interface {
    public:
       eosvmoc_runtime(const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db);
       ~eosvmoc_runtime();
-      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
+      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t> initial_memory,
                                                                              const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override;
 
       void immediately_exit_currently_running_module() override;

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/code_cache.hpp

@@ -85,6 +85,9 @@ class code_cache_base {
 
       template <typename T>
       void serialize_cache_index(fc::datastream<T>& ds);
+
+      std::thread::id _main_thread_id;
+      bool is_main_thread() const;
 };
 
 class code_cache_async : public code_cache_base {
@@ -95,7 +98,7 @@ class code_cache_async : public code_cache_base {
       //If code is in cache: returns pointer & bumps to front of MRU list
       //If code is not in cache, and not blacklisted, and not currently compiling: return nullptr and kick off compile
       //otherwise: return nullptr
-      const code_descriptor* const get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version, bool is_write_window);
+      const code_descriptor* const get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version);
 
    private:
       std::thread _monitor_reply_thread;
@@ -112,7 +115,7 @@ class code_cache_sync : public code_cache_base {
       ~code_cache_sync();
 
       //Can still fail and return nullptr if, for example, there is an expected instantiation failure
-      const code_descriptor* const get_descriptor_for_code_sync(const digest_type& code_id, const uint8_t& vm_version, bool is_write_window);
+      const code_descriptor* const get_descriptor_for_code_sync(const digest_type& code_id, const uint8_t& vm_version);
 };
 
 }}}

libraries/chain/include/eosio/chain/webassembly/eos-vm.hpp

@@ -43,7 +43,7 @@ template<typename Backend>
 class eos_vm_runtime : public eosio::chain::wasm_runtime_interface {
    public:
       eos_vm_runtime();
-      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
+      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t>,
                                                                              const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override;
 
       void immediately_exit_currently_running_module() override;
@@ -61,7 +61,7 @@ class eos_vm_runtime : public eosio::chain::wasm_runtime_interface {
 class eos_vm_profile_runtime : public eosio::chain::wasm_runtime_interface {
    public:
       eos_vm_profile_runtime();
-      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
+      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t>,
                                                                              const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override;
 
       void immediately_exit_currently_running_module() override;

libraries/chain/include/eosio/chain/webassembly/runtime_interface.hpp

@@ -20,7 +20,7 @@ class wasm_instantiated_module_interface {
 
 class wasm_runtime_interface {
    public:
-      virtual std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
+      virtual std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t> initial_memory,
                                                                                      const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) = 0;
 
       //immediately exit the currently running wasm_instantiated_module_interface. Yep, this assumes only one can possibly run at a time.

libraries/chain/wasm_interface.cpp

@@ -93,7 +93,7 @@ namespace eosio { namespace chain {
       if(my->eosvmoc) {
          const chain::eosvmoc::code_descriptor* cd = nullptr;
          try {
-            cd = my->eosvmoc->cc.get_descriptor_for_code(code_hash, vm_version, context.control.is_write_window());
+            cd = my->eosvmoc->cc.get_descriptor_for_code(code_hash, vm_version);
          }
          catch(...) {
             //swallow errors here, if EOS VM OC has gone in to the weeds we shouldn't bail: continue to try and run baseline

libraries/chain/webassembly/runtimes/eos-vm-oc.cpp

@@ -28,7 +28,7 @@ class eosvmoc_instantiated_module : public wasm_instantiated_module_interface {
       bool is_main_thread() { return _main_thread_id == std::this_thread::get_id(); };
 
       void apply(apply_context& context) override {
-         const code_descriptor* const cd = _eosvmoc_runtime.cc.get_descriptor_for_code_sync(_code_hash, _vm_version, context.control.is_write_window());
+         const code_descriptor* const cd = _eosvmoc_runtime.cc.get_descriptor_for_code_sync(_code_hash, _vm_version);
          EOS_ASSERT(cd, wasm_execution_error, "EOS VM OC instantiation failed");
 
          if ( is_main_thread() )
@@ -50,7 +50,7 @@ eosvmoc_runtime::eosvmoc_runtime(const boost::filesystem::path data_dir, const e
 eosvmoc_runtime::~eosvmoc_runtime() {
 }
 
-std::unique_ptr<wasm_instantiated_module_interface> eosvmoc_runtime::instantiate_module(const char* code_bytes, size_t code_size,
+std::unique_ptr<wasm_instantiated_module_interface> eosvmoc_runtime::instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t> initial_memory,
                                                                                         const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) {
    return std::make_unique<eosvmoc_instantiated_module>(code_hash, vm_type, *this);
 }

libraries/chain/webassembly/runtimes/eos-vm-oc/code_cache.cpp

@@ -107,11 +107,10 @@ std::tuple<size_t, size_t> code_cache_async::consume_compile_thread_queue() {
 }
 
 
-const code_descriptor* const code_cache_async::get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version, bool is_write_window) {
+const code_descriptor* const code_cache_async::get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version) {
    //if there are any outstanding compiles, process the result queue now
-   //When app is in write window, all tasks are running sequentially and read-only threads
-   //are not running. Safe to update cache entries.
-   if(is_write_window && _outstanding_compiles_and_poison.size()) {
+   //do this only on main thread (which is in single threaded write window)
+   if(is_main_thread() && _outstanding_compiles_and_poison.size()) {
       auto [count_processed, bytes_remaining] = consume_compile_thread_queue();
 
       if(count_processed)
@@ -137,11 +136,11 @@ const code_descriptor* const code_cache_async::get_descriptor_for_code(const dig
    //check for entry in cache
    code_cache_index::index<by_hash>::type::iterator it = _cache_index.get<by_hash>().find(boost::make_tuple(code_id, vm_version));
    if(it != _cache_index.get<by_hash>().end()) {
-      if (is_write_window)
+      if (is_main_thread())
          _cache_index.relocate(_cache_index.begin(), _cache_index.project<0>(it));
       return &*it;
    }
-   if(!is_write_window)
+   if(!is_main_thread()) // on read-only thread
       return nullptr;
 
    const code_tuple ct = code_tuple{code_id, vm_version};
@@ -180,15 +179,15 @@ code_cache_sync::~code_cache_sync() {
       elog("unexpected response from EOS VM OC compile monitor during shutdown");
 }
 
-const code_descriptor* const code_cache_sync::get_descriptor_for_code_sync(const digest_type& code_id, const uint8_t& vm_version, bool is_write_window) {
+const code_descriptor* const code_cache_sync::get_descriptor_for_code_sync(const digest_type& code_id, const uint8_t& vm_version) {
    //check for entry in cache
    code_cache_index::index<by_hash>::type::iterator it = _cache_index.get<by_hash>().find(boost::make_tuple(code_id, vm_version));
    if(it != _cache_index.get<by_hash>().end()) {
-      if (is_write_window)
+      if (is_main_thread())
          _cache_index.relocate(_cache_index.begin(), _cache_index.project<0>(it));
       return &*it;
    }
-   if(!is_write_window)
+   if(!is_main_thread())
       return nullptr;
 
    const code_object* const codeobject = _db.find<code_object,by_code_hash>(boost::make_tuple(code_id, 0, vm_version));
@@ -213,7 +212,8 @@ const code_descriptor* const code_cache_sync::get_descriptor_for_code_sync(const
 
 code_cache_base::code_cache_base(const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db) :
    _db(db),
-   _cache_file_path(data_dir/"code_cache.bin")
+   _cache_file_path(data_dir/"code_cache.bin"),
+   _main_thread_id(std::this_thread::get_id())
 {
    static_assert(sizeof(allocator_t) <= header_offset, "header offset intersects with allocator");
 
@@ -390,4 +390,8 @@ void code_cache_base::check_eviction_threshold(size_t free_bytes) {
    if(free_bytes < _free_bytes_eviction_threshold)
       run_eviction_round();
 }
+
+bool code_cache_base::is_main_thread() const {
+   return _main_thread_id == std::this_thread::get_id();
+}
 }}}

libraries/chain/webassembly/runtimes/eos-vm.cpp

@@ -239,7 +239,7 @@ void eos_vm_runtime<Impl>::immediately_exit_currently_running_module() {
 }
 
 template<typename Impl>
-std::unique_ptr<wasm_instantiated_module_interface> eos_vm_runtime<Impl>::instantiate_module(const char* code_bytes, size_t code_size,
+std::unique_ptr<wasm_instantiated_module_interface> eos_vm_runtime<Impl>::instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t>,
                                                                                              const digest_type&, const uint8_t&, const uint8_t&) {
 
    using backend_t = eos_vm_backend_t<Impl>;
@@ -265,7 +265,7 @@ void eos_vm_profile_runtime::immediately_exit_currently_running_module() {
    throw wasm_exit{};
 }
 
-std::unique_ptr<wasm_instantiated_module_interface> eos_vm_profile_runtime::instantiate_module(const char* code_bytes, size_t code_size,
+std::unique_ptr<wasm_instantiated_module_interface> eos_vm_profile_runtime::instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t>,
                                                                                                const digest_type&, const uint8_t&, const uint8_t&) {
 
    using backend_t = eosio::vm::backend<eos_vm_host_functions_t, eosio::vm::jit_profile, webassembly::eos_vm_runtime::apply_options, vm::profile_instr_map>;

libraries/custom_appbase/include/eosio/chain/application.hpp

@@ -1,33 +1,30 @@
 #pragma once
 
 #include <appbase/application_base.hpp>
-#include <eosio/chain/exec_pri_queue.hpp>
-#include <mutex>
+#include <appbase/execution_priority_queue.hpp>
 
 /*
- * Customize appbase to support two-queue executor.
+ * Custmomize appbase to support two-queue exector.
 */
 namespace appbase { 
 
 enum class exec_window {
-   read,  // the window during which operations from read_only queue
-          // can be executed in parallel in the read-only thread pool
-          // as well as in the app thread.
-   write, // the window during which operations from both read_write and
-          // parallel queues can be executed in app thread,
-          // while read-only operations are not executed in read-only
-          // thread pool. The read-only thread pool is not active; only
-          // the main app thread is active.
+   read,  // the window during which operations from read_only_trx_safe queue
+          // can be executed in app thread in parallel with read-only transactions
+          // in read-only transaction excuting threads.
+   write, // the window during which operations from both general and
+          // read_only_trx_safe queues can be executed in app thread,
+          // while read-only transactions are not executed in read-only
+          // transaction excuting threads.
 };
 
 enum class exec_queue {
-   read_only,          // the queue storing tasks which are safe to execute
-                       // in parallel with other read-only tasks in the read-only
-                       // thread pool as well as on the main app thread.
-                       // Multi-thread safe as long as nothing is executed from the read_write queue.
-   read_write          // the queue storing tasks which can be only executed
-                       // on the app thread while read-only tasks are
-                       // not being executed in read-only threads. Single threaded.
+   read_only_trx_safe, // the queue storing operations which are safe to execute
+                       // on app thread in parallel with read-only transactions
+                       // in read-only transaction excuting threads.
+   general             // the queue storing operations which can be only executed
+                       // on the app thread while read-only transactions are
+                       // not being executed in read-only threads
 };
 
 class two_queue_executor {
@@ -35,64 +32,56 @@ public:
 
    template <typename Func>
    auto post( int priority, exec_queue q, Func&& func ) {
-      if ( q == exec_queue::read_write )
-         return boost::asio::post(io_serv_, read_write_queue_.wrap(priority, --order_, std::forward<Func>(func)));
+      if ( q == exec_queue::general )
+         return boost::asio::post(io_serv_, general_queue_.wrap(priority, --order_, std::forward<Func>(func)));
       else
-         return boost::asio::post( io_serv_, read_only_queue_.wrap( priority, --order_, std::forward<Func>( func)));
+         return boost::asio::post(io_serv_, read_only_trx_safe_queue_.wrap(priority, --order_, std::forward<Func>(func)));
    }
 
    // Legacy and deprecated. To be removed after cleaning up its uses in base appbase
    template <typename Func>
    auto post( int priority, Func&& func ) {
-      // safer to use read_write queue for unknown type of operation since operations
-      // from read_write queue are not executed in parallel with read-only operations
-      return boost::asio::post(io_serv_, read_write_queue_.wrap(priority, --order_, std::forward<Func>(func)));
+      // safer to use general queue for unknown type of operation since operations
+      // from general queue are not executed in parallel with read-only transactions
+      return boost::asio::post(io_serv_, general_queue_.wrap(priority, --order_, std::forward<Func>(func)));
    }
 
    boost::asio::io_service& get_io_service() { return io_serv_; }
 
    bool execute_highest() {
       if ( exec_window_ == exec_window::write ) {
-         // During write window only main thread is accessing anything in two_queue_executor, no locking required
-         if( !read_write_queue_.empty() && (read_only_queue_.empty() || *read_only_queue_.top() < *read_write_queue_.top()) )  {
-            // read_write_queue_'s top function's priority greater than read_only_queue_'s top function's, or read_only_queue_ empty
-            read_write_queue_.execute_highest();
-         } else if( !read_only_queue_.empty() ) {
-            read_only_queue_.execute_highest();
+         if( !general_queue_.empty() && ( read_only_trx_safe_queue_.empty() || *read_only_trx_safe_queue_.top() < *general_queue_.top()) )  {
+            // general_queue_'s top function's priority greater than read_only_trx_safe_queue_'s top function's, or general_queue_ empty
+            general_queue_.execute_highest();
+         } else if( !read_only_trx_safe_queue_.empty() ) {
+            read_only_trx_safe_queue_.execute_highest();
          }
-         return !read_only_queue_.empty() || !read_write_queue_.empty();
+         return !read_only_trx_safe_queue_.empty() || !general_queue_.empty();
       } else {
-         // When in read window, multiple threads including main app thread are accessing two_queue_executor, locking required
-         return read_only_queue_.execute_highest_locked(false);
+         return read_only_trx_safe_queue_.execute_highest();
       }
    }
 
-   bool execute_highest_read_only() {
-      return read_only_queue_.execute_highest_locked(true);
-   }
-
    template <typename Function>
-   boost::asio::executor_binder<Function, appbase::exec_pri_queue::executor>
+   boost::asio::executor_binder<Function, appbase::execution_priority_queue::executor>
    wrap(int priority, exec_queue q, Function&& func ) {
-      if ( q == exec_queue::read_write )
-         return read_write_queue_.wrap(priority, --order_, std::forward<Function>(func));
+      if ( q == exec_queue::general )
+         return general_queue_.wrap(priority, --order_, std::forward<Function>(func));
       else
-         return read_only_queue_.wrap( priority, --order_, std::forward<Function>( func));
+         return read_only_trx_safe_queue_.wrap(priority, --order_, std::forward<Function>(func));
    }
      
    void clear() {
-      read_only_queue_.clear();
-      read_write_queue_.clear();
+      read_only_trx_safe_queue_.clear();
+      general_queue_.clear();
    }
 
-   void set_to_read_window(uint32_t num_threads, std::function<bool()> should_exit) {
+   void set_to_read_window() {
       exec_window_ = exec_window::read;
-      read_only_queue_.enable_locking(num_threads, std::move(should_exit));
    }
 
    void set_to_write_window() {
       exec_window_ = exec_window::write;
-      read_only_queue_.disable_locking();
    }
 
    bool is_read_window() const {
@@ -103,15 +92,15 @@ public:
       return exec_window_ == exec_window::write;
    }
 
-   auto& read_only_queue() { return read_only_queue_; }
+   auto& read_only_trx_safe_queue() { return read_only_trx_safe_queue_; }
 
-   auto& read_write_queue() { return read_write_queue_; }
+   auto& general_queue() { return general_queue_; }
 
    // members are ordered taking into account that the last one is destructed first
 private:
    boost::asio::io_service            io_serv_;
-   appbase::exec_pri_queue            read_only_queue_;
-   appbase::exec_pri_queue            read_write_queue_;
+   appbase::execution_priority_queue  read_only_trx_safe_queue_;
+   appbase::execution_priority_queue  general_queue_;
    std::atomic<std::size_t>           order_ { std::numeric_limits<size_t>::max() }; // to maintain FIFO ordering in both queues within priority
    exec_window                        exec_window_ { exec_window::write };
 };

libraries/custom_appbase/include/eosio/chain/exec_pri_queue.hpp

@@ -1,233 +0,0 @@
-#pragma once
-#include <boost/asio.hpp>
-
-#include <condition_variable>
-#include <mutex>
-#include <queue>
-
-namespace appbase {
-// adapted from: https://www.boost.org/doc/libs/1_69_0/doc/html/boost_asio/example/cpp11/invocation/prioritised_handlers.cpp
-
-// Locking has to be coordinated by caller, use with care.
-class exec_pri_queue : public boost::asio::execution_context
-{
-public:
-
-   void enable_locking(uint32_t num_threads, std::function<bool()> should_exit) {
-      assert(num_threads > 0 && num_waiting_ == 0);
-      lock_enabled_ = true;
-      max_waiting_ = num_threads;
-      should_exit_ = std::move(should_exit);
-      exiting_blocking_ = false;
-   }
-
-   void disable_locking() {
-      lock_enabled_ = false;
-      should_exit_ = [](){ assert(false); return true; }; // should not be called when locking is disabled
-   }
-
-   // called from appbase::application_base::exec poll_one() or run_one()
-   template <typename Function>
-   void add(int priority, size_t order, Function function)
-   {
-      std::unique_ptr<queued_handler_base> handler(new queued_handler<Function>(priority, order, std::move(function)));
-      if (lock_enabled_) {
-         std::lock_guard g( mtx_ );
-         handlers_.push( std::move( handler ) );
-         if (num_waiting_)
-            cond_.notify_one();
-      } else {
-         handlers_.push( std::move( handler ) );
-      }
-   }
-
-   // only call when no lock required
-   void clear()
-   {
-      handlers_ = prio_queue();
-   }
-
-   // only call when no lock required
-   bool execute_highest()
-   {
-      if( !handlers_.empty() ) {
-         handlers_.top()->execute();
-         handlers_.pop();
-      }
-
-      return !handlers_.empty();
-   }
-
-private:
-   // has to be defined before use, auto return type
-   auto pop() {
-      auto t = std::move(const_cast<std::unique_ptr<queued_handler_base>&>(handlers_.top()));
-      handlers_.pop();
-      return t;
-   }
-
-public:
-
-   bool execute_highest_locked(bool should_block) {
-      std::unique_lock g(mtx_);
-      if (should_block) {
-         ++num_waiting_;
-         cond_.wait(g, [this](){
-            bool exit = exiting_blocking_ || should_exit_();
-            bool empty = handlers_.empty();
-            if (empty || exit) {
-               if (((empty && num_waiting_ == max_waiting_) || exit) && !exiting_blocking_) {
-                  cond_.notify_all();
-                  exiting_blocking_ = true;
-               }
-               return exit || exiting_blocking_; // same as calling should_exit(), but faster
-            }
-            return true;
-         });
-         --num_waiting_;
-         if (exiting_blocking_ || should_exit_())
-            return false;
-      }
-      if( handlers_.empty() )
-         return false;
-      auto t = pop();
-      g.unlock();
-      t->execute();
-      return true;
-   }
-
-   // Only call when locking disabled
-   size_t size() const { return handlers_.size(); }
-
-   // Only call when locking disabled
-   bool empty() const { return handlers_.empty(); }
-
-   // Only call when locking disabled
-   const auto& top() const { return handlers_.top(); }
-
-   class executor
-   {
-   public:
-      executor(exec_pri_queue& q, int p, size_t o)
-            : context_(q), priority_(p), order_(o)
-      {
-      }
-
-      exec_pri_queue& context() const noexcept
-      {
-         return context_;
-      }
-
-      template <typename Function, typename Allocator>
-      void dispatch(Function f, const Allocator&) const
-      {
-         context_.add(priority_, order_, std::move(f));
-      }
-
-      template <typename Function, typename Allocator>
-      void post(Function f, const Allocator&) const
-      {
-         context_.add(priority_, order_, std::move(f));
-      }
-
-      template <typename Function, typename Allocator>
-      void defer(Function f, const Allocator&) const
-      {
-         context_.add(priority_, order_, std::move(f));
-      }
-
-      void on_work_started() const noexcept {}
-      void on_work_finished() const noexcept {}
-
-      bool operator==(const executor& other) const noexcept
-      {
-         return order_ == other.order_ && &context_ == &other.context_ && priority_ == other.priority_;
-      }
-
-      bool operator!=(const executor& other) const noexcept
-      {
-         return !operator==(other);
-      }
-
-   private:
-      exec_pri_queue& context_;
-      int priority_;
-      size_t order_;
-   };
-
-   template <typename Function>
-   boost::asio::executor_binder<Function, executor>
-   wrap(int priority, size_t order, Function&& func)
-   {
-      return boost::asio::bind_executor( executor(*this, priority, order), std::forward<Function>(func) );
-   }
-
-private:
-   class queued_handler_base
-   {
-   public:
-      queued_handler_base( int p, size_t order )
-            : priority_( p )
-            , order_( order )
-      {
-      }
-
-      virtual ~queued_handler_base() = default;
-
-      virtual void execute() = 0;
-
-      int priority() const { return priority_; }
-      // C++20
-      // friend std::weak_ordering operator<=>(const queued_handler_base&,
-      //                                       const queued_handler_base&) noexcept = default;
-      friend bool operator<(const queued_handler_base& a,
-                            const queued_handler_base& b) noexcept
-      {
-         return std::tie( a.priority_, a.order_ ) < std::tie( b.priority_, b.order_ );
-      }
-
-   private:
-      int priority_;
-      size_t order_;
-   };
-
-   template <typename Function>
-   class queued_handler : public queued_handler_base
-   {
-   public:
-      queued_handler(int p, size_t order, Function f)
-            : queued_handler_base( p, order )
-            , function_( std::move(f) )
-      {
-      }
-
-      void execute() override
-      {
-         function_();
-      }
-
-   private:
-      Function function_;
-   };
-
-   struct deref_less
-   {
-      template<typename Pointer>
-      bool operator()(const Pointer& a, const Pointer& b) noexcept(noexcept(*a < *b))
-      {
-         return *a < *b;
-      }
-   };
-
-   bool lock_enabled_ = false;
-   std::mutex mtx_;
-   std::condition_variable cond_;
-   uint32_t num_waiting_{0};
-   uint32_t max_waiting_{0};
-   bool exiting_blocking_{false};
-   std::function<bool()> should_exit_; // called holding mtx_
-   using prio_queue = std::priority_queue<std::unique_ptr<queued_handler_base>, std::deque<std::unique_ptr<queued_handler_base>>, deref_less>;
-   prio_queue handlers_;
-};
-
-} // appbase

libraries/custom_appbase/tests/custom_appbase_tests.cpp

@@ -27,28 +27,28 @@ BOOST_AUTO_TEST_CASE( default_exec_window ) {
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::read_only,  [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_only,  [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_only_trx_safe,  [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
 
    // Stop app. Use the lowest priority to make sure this function to execute the last
-   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
-      // read_only_queue should only contain the current lambda function,
-      // and read_write_queue should have executed all its functions
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 1); // pop()s after execute
-      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 0 );
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
+      // read_only_trx_safe_queue should only contain the current lambda function,
+      // and general_queue should have execute all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
+      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 0 );
       app->quit();
       } );
    app_thread.join();
 
    // both queues are cleared after execution 
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
 
    // exactly number of both queues' functions processed
    BOOST_REQUIRE_EQUAL( rslts.size(), 8 );
@@ -64,40 +64,40 @@ BOOST_AUTO_TEST_CASE( default_exec_window ) {
    BOOST_CHECK_LT( rslts[6], rslts[7] );
 }
 
-// verify functions only from read_only queue are processed during read window
+// verify functions only from read_only_trx_safe queue are processed during read window
 BOOST_AUTO_TEST_CASE( execute_from_read_queue ) {
    appbase::scoped_app app;
    auto app_thread = start_app_thread(app);
    
-   // set to run functions from read_only queue only
-   app->executor().set_to_read_window(1, [](){return false;});
+   // set to run functions from read_only_trx_safe queue only
+   app->executor().set_to_read_window();
 
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_only,  [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[7]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[8]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[9]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[9]=seq_num; ++seq_num; } );
 
    // stop application. Use lowest at the end to make sure this executes the last
-   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
-      // read_queue should be empty (read window pops before execute) and write_queue should have all its functions
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 0); // pop()s before execute
-      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 4 );
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
+      // read_queue should have current function and write_queue should have all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
+      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 4 );
       app->quit();
       } );
    app_thread.join();
 
    // both queues are cleared after execution 
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
 
    // exactly number of posts processed
    BOOST_REQUIRE_EQUAL( rslts.size(), 6 );
@@ -109,40 +109,40 @@ BOOST_AUTO_TEST_CASE( execute_from_read_queue ) {
    BOOST_CHECK_LT( rslts[3], rslts[4] );
 }
 
-// verify no functions are executed during  read window if read_only queue is empty
+// verify no functions are executed during  read window if read_only_trx_safe queue is empty
 BOOST_AUTO_TEST_CASE( execute_from_empty_read_queue ) {
    appbase::scoped_app app;
    auto app_thread = start_app_thread(app);
    
-   // set to run functions from read_only queue only
-   app->executor().set_to_read_window(1, [](){return false;});
+   // set to run functions from read_only_trx_safe queue only
+   app->executor().set_to_read_window();
 
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_write, [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_write, [&]() { rslts[7]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[8]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[9]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::general, [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[9]=seq_num; ++seq_num; } );
 
    // Stop application. Use lowest at the end to make sure this executes the last
-   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
-      // read_queue should be empty (read window pops before execute) and write_queue should have all its functions
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 0); // pop()s before execute
-      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 10 );
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
+      // read_queue should have current function and write_queue should have all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
+      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 10 );
       app->quit();
       } );
    app_thread.join();
 
    // both queues are cleared after execution 
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
 
    // no results
    BOOST_REQUIRE_EQUAL( rslts.size(), 0 );
@@ -159,32 +159,32 @@ BOOST_AUTO_TEST_CASE( execute_from_both_queues ) {
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::read_only,  [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::lowest, exec_queue::read_only,  [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_only,  [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[7]=seq_num; ++seq_num; } );
-   app->executor().post( priority::lowest, exec_queue::read_only,  [&]() { rslts[8]=seq_num; ++seq_num; } );
-   app->executor().post( priority::lowest, exec_queue::read_only,  [&]() { rslts[9]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[10]=seq_num; ++seq_num; } );
-   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[11]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_only_trx_safe,  [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[9]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[10]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[11]=seq_num; ++seq_num; } );
 
    // stop application. Use lowest at the end to make sure this executes the last
-   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
       // read_queue should have current function and write_queue's functions are all executed 
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 1); // pop()s after execute
-      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 0 );
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
+      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 0 );
       app->quit();
       } );
 
    app_thread.join();
 
    // queues are emptied after quit
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
 
    // exactly number of posts processed
    BOOST_REQUIRE_EQUAL( rslts.size(), 12 );

libraries/libfc/CMakeLists.txt

@@ -57,6 +57,8 @@ set( fc_sources
      src/crypto/blake2.cpp
      src/crypto/k1_recover.cpp
      src/network/ip.cpp
+     src/network/resolve.cpp
+     src/network/udp_socket.cpp
      src/network/url.cpp
      src/network/http/http_client.cpp
      src/compress/zlib.cpp

libraries/libfc/include/fc/network/resolve.hpp

@@ -0,0 +1,9 @@
+#pragma once
+#include <fc/vector.hpp>
+#include <fc/network/ip.hpp>
+
+namespace fc
+{
+  std::vector<boost::asio::ip::udp::endpoint> resolve(boost::asio::io_service& io_service,
+                                                      const std::string& host, uint16_t port);
+}

libraries/libfc/include/fc/network/udp_socket.hpp

@@ -0,0 +1,39 @@
+#pragma once
+#include <fc/utility.hpp>
+#include <memory>
+
+#include <boost/asio.hpp>
+
+namespace fc {
+  namespace ip {
+    class endpoint;
+    class address;
+  }
+
+  /**
+   *  The udp_socket class has reference semantics, all copies will
+   *  refer to the same underlying socket.
+   */
+  class udp_socket {
+    public:
+      udp_socket();
+      udp_socket( const udp_socket& s );
+      ~udp_socket();
+
+      void initialize(boost::asio::io_service &);
+      void open();
+      void send_to(const char* b, size_t l, boost::asio::ip::udp::endpoint &to);
+      void send_to(const std::shared_ptr<const char>& b, size_t l, boost::asio::ip::udp::endpoint &to);
+      void close();
+
+      void set_reuse_address(bool);
+
+      void connect(const boost::asio::ip::udp::endpoint& e);
+      const boost::asio::ip::udp::endpoint local_endpoint() const;
+
+    private:
+      class                impl;
+      std::shared_ptr<impl> my;
+  };
+
+}

libraries/libfc/src/log/gelf_appender.cpp

@@ -1,3 +1,6 @@
+#include <fc/network/udp_socket.hpp>
+#include <fc/network/ip.hpp>
+#include <fc/network/resolve.hpp>
 #include <fc/exception/exception.hpp>
 #include <fc/log/gelf_appender.hpp>
 #include <fc/reflect/variant.hpp>
@@ -7,7 +10,6 @@
 #include <fc/compress/zlib.hpp>
 #include <fc/log/logger_config.hpp>
 
-#include <boost/asio.hpp>
 #include <boost/lexical_cast.hpp>
 #include <iomanip>
 #include <iostream>
@@ -17,6 +19,14 @@
 
 namespace fc
 {
+  namespace detail
+  {
+    boost::asio::ip::udp::endpoint to_asio_ep( const fc::ip::endpoint& e )
+    {
+      return boost::asio::ip::udp::endpoint(boost::asio::ip::address_v4(e.get_address()), e.port() );
+    }
+  }
+
   const std::vector<std::string> gelf_appender::config::reserved_field_names  = {
       "_id",            // per GELF specification
       "_timestamp_ns",  // Remaining names all populated by appender
@@ -26,21 +36,23 @@ namespace fc
       "_method_name",
       "_thread_name",
       "_task_name"
-  };
+   };
 
-  const std::regex gelf_appender::config::user_field_name_pattern{"^_[\\w\\.\\-]*$"}; // per GELF specification
+   const std::regex gelf_appender::config::user_field_name_pattern{"^_[\\w\\.\\-]*$"}; // per GELF specification
 
   class gelf_appender::impl
   {
   public:
     using work_guard_t = boost::asio::executor_work_guard<boost::asio::io_context::executor_type>;
     config                                         cfg;
+    std::optional<boost::asio::ip::udp::endpoint>  gelf_endpoint;
     std::thread                                    thread;
     boost::asio::io_context                        io_context;
     work_guard_t work_guard =                      boost::asio::make_work_guard(io_context);
-    boost::asio::ip::udp::socket                   gelf_socket;
+    udp_socket                                     gelf_socket;
 
-    impl(const variant& c) : gelf_socket(io_context)
+
+    impl(const variant& c)
     {
       mutable_variant_object mvo;
       from_variant(c, mvo);
@@ -72,35 +84,6 @@ namespace fc
           thread.join();
        }
     }
-
-    static std::shared_ptr<std::vector<char>> make_new_bufer(boost::asio::const_buffer buf) {
-      const char* p = static_cast<const char*>(buf.data());
-      return std::make_shared<std::vector<char>>(p, p+buf.size());
-    }
-
-    static std::shared_ptr<std::vector<char>> make_new_bufer(const std::array<boost::asio::const_buffer, 2>& bufs) {
-      auto new_buf = std::make_shared<std::vector<char>>();
-      new_buf->reserve(bufs[0].size() + bufs[1].size());
-      for (int i = 0; i < 2; ++i) {
-          const char* p = static_cast<const char*>(bufs[i].data());
-          new_buf->insert(new_buf->end(), p, p + bufs[i].size());
-      }
-      return new_buf;
-    }
-
-    template <typename Buffers>
-    void send(Buffers&& bufs) {
-      boost::system::error_code ec;
-      gelf_socket.send(std::forward<Buffers>(bufs), 0, ec);
-      if (ec == boost::asio::error::would_block) {
-          auto new_buf = make_new_bufer(std::forward<Buffers>(bufs));
-          gelf_socket.async_send(boost::asio::buffer(*new_buf),
-                                    [new_buf](const boost::system::error_code& /*ec*/, std::size_t /*bytes_transferred*/) {
-                                        // Swallow errors.  Currently only used for GELF logging, so depend on local
-                                        // log to catch anything that doesn't make it across the network.
-                                    });
-      }
-    }
   };
 
   gelf_appender::gelf_appender(const variant& args) :
@@ -112,40 +95,58 @@ namespace fc
   {
     try
     {
-      if (my->cfg.endpoint.empty()) {
-        fprintf(stderr, "The logging destination is not specified\n");
-        return;
+      try
+      {
+        // if it's a numeric address:port, this will parse it
+        my->gelf_endpoint = detail::to_asio_ep(ip::endpoint::from_string(my->cfg.endpoint));
+      }
+      catch (...)
+      {
+      }
+      if (!my->gelf_endpoint)
+      {
+        // couldn't parse as a numeric ip address, try resolving as a DNS name.
+        // This can yield, so don't do it in the catch block above
+        string::size_type colon_pos = my->cfg.endpoint.find(':');
+        try
+        {
+          FC_ASSERT(colon_pos != std::string::npos, "The logging destination port is not specified");
+          string port = my->cfg.endpoint.substr(colon_pos + 1);
+
+          string hostname = my->cfg.endpoint.substr( 0, colon_pos );
+
+          boost::asio::ip::udp::resolver resolver{ my->io_context };
+          auto endpoints = resolver.resolve(hostname, port);
+
+          if (endpoints.empty())
+              FC_THROW_EXCEPTION(unknown_host_exception, "The logging destination host name can not be resolved: ${hostname}",
+                                 ("hostname", hostname));
+
+          my->gelf_endpoint = *endpoints.begin();
+        }
+        catch (const boost::bad_lexical_cast&)
+        {
+          FC_THROW("Bad port: ${port}", ("port", my->cfg.endpoint.substr(colon_pos + 1, my->cfg.endpoint.size())));
+        }
       }
 
-      std::string_view endpoint = my->cfg.endpoint;
-      string::size_type colon_pos = endpoint.rfind(':');
-      FC_ASSERT(colon_pos != std::string::npos, "The logging destination port is not specified");
-      auto port = endpoint.substr(colon_pos + 1);
-
-      auto hostname = (endpoint[0] == '[' && colon_pos >= 2)  ? endpoint.substr( 1, colon_pos-2 ) : endpoint.substr( 0, colon_pos );
-
-      boost::asio::ip::udp::resolver resolver{ my->io_context };
-      auto endpoints = resolver.resolve(hostname, port);
-
-      if (endpoints.empty())
-          FC_THROW_EXCEPTION(unknown_host_exception, "The logging destination host name can not be resolved: ${hostname}",
-                              ("hostname", std::string(hostname)));
-
-      my->gelf_socket.connect(*endpoints.begin());
-      std::cerr << "opened GELF socket to endpoint " << my->cfg.endpoint << "\n";
-      my->gelf_socket.non_blocking(true);
-
-      my->thread = std::thread([this] {
-        try {
-          fc::set_os_thread_name("gelf");
-          my->io_context.run();
-        } catch (std::exception& ex) {
-          fprintf(stderr, "GELF logger caught exception at %s:%d : %s\n", __FILE__, __LINE__, ex.what());
-        } catch (...) {
-          fprintf(stderr, "GELF logger caught exception unknown exception %s:%d\n", __FILE__, __LINE__);
-        }
-      });
+      if (my->gelf_endpoint)
+      {
+        my->gelf_socket.initialize(my->io_context);
+        my->gelf_socket.open();
+        std::cerr << "opened GELF socket to endpoint " << my->cfg.endpoint << "\n";
 
+        my->thread = std::thread([this] {
+          try {
+            fc::set_os_thread_name("gelf");
+            my->io_context.run();
+          } catch (std::exception& ex) {
+            fprintf(stderr, "GELF logger caught exception at %s:%d : %s\n", __FILE__, __LINE__, ex.what());
+          } catch (...) {
+            fprintf(stderr, "GELF logger caught exception unknown exception %s:%d\n", __FILE__, __LINE__);
+          }
+        });
+      }
     }
     catch (...)
     {
@@ -220,44 +221,57 @@ namespace fc
 
     if (gelf_message_as_string.size() <= max_payload_size)
     {
-      my->send(boost::asio::buffer(gelf_message_as_string));
+      // no need to split
+      std::shared_ptr<char> send_buffer(new char[gelf_message_as_string.size()],
+                                        [](char* p){ delete[] p; });
+      memcpy(send_buffer.get(), gelf_message_as_string.c_str(),
+             gelf_message_as_string.size());
+
+      my->gelf_socket.send_to(send_buffer, gelf_message_as_string.size(),
+                              *my->gelf_endpoint);
     }
     else
     {
       // split the message
-      struct gelf_header {
-        uint8_t magic[2] = { 0x1e, 0x0f};
-        uint64_t message_id;
-        uint8_t seq = 0;
-        uint8_t count = 0;
-      } header;
-
       // we need to generate an 8-byte ID for this message.
       // city hash should do
-      header.message_id = city_hash64(gelf_message_as_string.c_str(), gelf_message_as_string.size());
-      const unsigned body_length = max_payload_size - sizeof(header);
-      header.count = (gelf_message_as_string.size() + body_length - 1) / body_length;
+      uint64_t message_id = city_hash64(gelf_message_as_string.c_str(), gelf_message_as_string.size());
+      const unsigned header_length = 2 /* magic */ + 8 /* msg id */ + 1 /* seq */ + 1 /* count */;
+      const unsigned body_length = max_payload_size - header_length;
+      unsigned total_number_of_packets = (gelf_message_as_string.size() + body_length - 1) / body_length;
       unsigned bytes_sent = 0;
-
+      unsigned number_of_packets_sent = 0;
       while (bytes_sent < gelf_message_as_string.size())
       {
         unsigned bytes_to_send = std::min((unsigned)gelf_message_as_string.size() - bytes_sent,
                                           body_length);
 
-        std::array<boost::asio::const_buffer,2> bufs = {
-          boost::asio::const_buffer(&header, sizeof(header)),
-          boost::asio::const_buffer(gelf_message_as_string.c_str() + bytes_sent, bytes_to_send)
-        };
-        my->send(bufs);
-        ++header.seq;
+        std::shared_ptr<char> send_buffer(new char[max_payload_size],
+                                          [](char* p){ delete[] p; });
+        char* ptr = send_buffer.get();
+        // magic number for chunked message
+        *(unsigned char*)ptr++ = 0x1e;
+        *(unsigned char*)ptr++ = 0x0f;
+
+        // message id
+        memcpy(ptr, (char*)&message_id, sizeof(message_id));
+        ptr += sizeof(message_id);
+
+        *(unsigned char*)(ptr++) = number_of_packets_sent;
+        *(unsigned char*)(ptr++) = total_number_of_packets;
+        memcpy(ptr, gelf_message_as_string.c_str() + bytes_sent,
+               bytes_to_send);
+        my->gelf_socket.send_to(send_buffer, header_length + bytes_to_send,
+                                *my->gelf_endpoint);
+        ++number_of_packets_sent;
         bytes_sent += bytes_to_send;
       }
-      FC_ASSERT(header.seq == header.count);
+      FC_ASSERT(number_of_packets_sent == total_number_of_packets);
     }
   }
 
   void gelf_appender::log(const log_message& message) {
-    if (!my->thread.joinable())
+    if (!my->gelf_endpoint)
       return;
 
     // use now() instead of context.get_timestamp() because log_message construction can include user provided long running calls

libraries/libfc/src/network/resolve.cpp

@@ -0,0 +1,33 @@
+#include <boost/asio.hpp>
+#include <fc/exception/exception.hpp>
+
+namespace fc
+{
+  std::vector<boost::asio::ip::udp::endpoint> resolve(boost::asio::io_service& io_service,
+                                                     const std::string& host, uint16_t port)
+  {
+    using q = boost::asio::ip::udp::resolver::query;
+    using b = boost::asio::ip::resolver_query_base;
+    boost::asio::ip::udp::resolver res(io_service);
+    boost::system::error_code ec;
+    auto ep = res.resolve(q(host, std::to_string(uint64_t(port)),
+                            b::address_configured | b::numeric_service), ec);
+    if(!ec)
+    {
+      std::vector<boost::asio::ip::udp::endpoint> eps;
+      while(ep != boost::asio::ip::udp::resolver::iterator())
+      {
+        if(ep->endpoint().address().is_v4())
+        {
+         eps.push_back(*ep);
+        }
+        // TODO: add support for v6
+        ++ep;
+      }
+      return eps;
+    }
+    FC_THROW_EXCEPTION(unknown_host_exception,
+                       "name resolution failed: ${reason}",
+                       ("reason", ec.message()));
+  }
+}

libraries/libfc/src/network/udp_socket.cpp

@@ -0,0 +1,111 @@
+#include <fc/network/udp_socket.hpp>
+#include <fc/network/ip.hpp>
+
+
+namespace fc
+{
+  class udp_socket::impl
+  {
+    public:
+      impl(){}
+      ~impl(){}
+
+      std::shared_ptr<boost::asio::ip::udp::socket> _sock;
+  };
+
+  udp_socket::udp_socket()
+    : my(new impl())
+  {
+  }
+
+  void udp_socket::initialize(boost::asio::io_service& service)
+  {
+    my->_sock.reset(new boost::asio::ip::udp::socket(service));
+  }
+
+  udp_socket::~udp_socket() 
+  {
+    try 
+    {
+      if(my->_sock)
+        my->_sock->close(); //close boost socket to make any pending reads run their completion handler
+    }
+    catch (...) //avoid destructor throw and likely this is just happening because socket wasn't open.
+    {
+    }
+  }
+
+  void udp_socket::send_to(const char* buffer, size_t length, boost::asio::ip::udp::endpoint& to)
+  {
+    try 
+    {
+      my->_sock->send_to(boost::asio::buffer(buffer, length), to);
+      return;
+    } 
+    catch(const boost::system::system_error& e)
+    {
+      if(e.code() == boost::asio::error::would_block)
+      {
+          auto send_buffer_ptr = std::make_shared<std::vector<char>>(buffer, buffer+length);
+          my->_sock->async_send_to(boost::asio::buffer(send_buffer_ptr.get(), length), to,
+                                   [send_buffer_ptr](const boost::system::error_code& /*ec*/, std::size_t /*bytes_transferred*/)
+          {
+            // Swallow errors.  Currently only used for GELF logging, so depend on local
+            // log to catch anything that doesn't make it across the network.
+          });
+       }
+       // All other exceptions ignored.
+    }
+  }
+
+  void udp_socket::send_to(const std::shared_ptr<const char>& buffer, size_t length,
+                           boost::asio::ip::udp::endpoint& to)
+  {
+    try
+    {
+      my->_sock->send_to(boost::asio::buffer(buffer.get(), length), to);
+      return;
+    } 
+    catch(const boost::system::system_error& e)
+    {
+      if(e.code() == boost::asio::error::would_block)
+      {
+          auto preserved_buffer_ptr = buffer;
+          my->_sock->async_send_to(boost::asio::buffer(preserved_buffer_ptr.get(), length), to,
+                                   [preserved_buffer_ptr](const boost::system::error_code& /*ec*/, std::size_t /*bytes_transferred*/)
+          {
+            // Swallow errors.  Currently only used for GELF logging, so depend on local
+            // log to catch anything that doesn't make it across the network.
+          });
+       }
+       // All other exceptions ignored.
+    }
+  }
+
+  void udp_socket::open()
+  {
+    my->_sock->open(boost::asio::ip::udp::v4());
+    my->_sock->non_blocking(true);
+  }
+
+  void udp_socket::close()
+  {
+    my->_sock->close();
+  }
+
+  const boost::asio::ip::udp::endpoint udp_socket::local_endpoint() const
+  {
+    return my->_sock->local_endpoint();
+  }
+
+  void udp_socket::connect(const boost::asio::ip::udp::endpoint& e)
+  {
+    my->_sock->connect(e);
+  }
+
+  void udp_socket::set_reuse_address( bool s )
+  {
+    my->_sock->set_option( boost::asio::ip::udp::socket::reuse_address(s) );
+  }
+
+}

libraries/state_history/include/eosio/state_history/log.hpp

@@ -550,9 +550,6 @@ class state_history_log {
          fc::raw::pack(log, num_blocks_in_log);
       }
 
-      log.flush();
-      index.flush();
-
       auto partition_config = std::get_if<state_history::partition_config>(&config);
       if (partition_config && block_num % partition_config->stride == 0) {
          split_log();

libraries/wasm-jit/Include/IR/Operators.h

@@ -12,10 +12,11 @@ namespace IR
 	struct NoImm {};
 	struct MemoryImm {};
 
+	PACKED_STRUCT(
 	struct ControlStructureImm
 	{
 		ResultType resultType{};
-	};
+	});
 
 	struct BranchImm
 	{
@@ -675,4 +676,19 @@ namespace IR
 	};
 
 	IR_API const char* getOpcodeName(Opcode opcode);
-}
+}
+
+//paranoia for future platforms
+static_assert(sizeof(IR::OpcodeAndImm<IR::NoImm>) == 2);
+static_assert(sizeof(IR::OpcodeAndImm<IR::MemoryImm>) == 2);
+static_assert(sizeof(IR::OpcodeAndImm<IR::ControlStructureImm>) == 3);
+static_assert(sizeof(IR::OpcodeAndImm<IR::BranchImm>) == 6);
+static_assert(sizeof(IR::OpcodeAndImm<IR::BranchTableImm>) == 18);
+static_assert(sizeof(IR::OpcodeAndImm<IR::LiteralImm<I32>>) == 6);
+static_assert(sizeof(IR::OpcodeAndImm<IR::LiteralImm<I64>>) == 10);
+static_assert(sizeof(IR::OpcodeAndImm<IR::LiteralImm<F32>>) == 6);
+static_assert(sizeof(IR::OpcodeAndImm<IR::LiteralImm<F64>>) == 10);
+static_assert(sizeof(IR::OpcodeAndImm<IR::GetOrSetVariableImm<false>>) == 6);
+static_assert(sizeof(IR::OpcodeAndImm<IR::CallImm>) == 6);
+static_assert(sizeof(IR::OpcodeAndImm<IR::CallIndirectImm>) == 6);
+static_assert(sizeof(IR::OpcodeAndImm<IR::LoadOrStoreImm<0>>) == 10);

plugins/chain_api_plugin/chain_api_plugin.cpp

@@ -75,7 +75,7 @@ void chain_api_plugin::plugin_startup() {
    ro_api.set_shorten_abi_errors( !http_plugin::verbose_errors() );
 
    _http_plugin.add_api( {
-      CHAIN_RO_CALL(get_info, 200, http_params_types::no_params)}, appbase::exec_queue::read_only, appbase::priority::medium_high);
+      CHAIN_RO_CALL(get_info, 200, http_params_types::no_params)}, appbase::exec_queue::read_only_trx_safe, appbase::priority::medium_high);
    _http_plugin.add_api({
       CHAIN_RO_CALL(get_activated_protocol_features, 200, http_params_types::possible_no_params),
       CHAIN_RO_CALL(get_block_info, 200, http_params_types::params_required),
@@ -96,19 +96,19 @@ void chain_api_plugin::plugin_startup() {
       CHAIN_RO_CALL(get_scheduled_transactions, 200, http_params_types::params_required),
       CHAIN_RO_CALL(get_required_keys, 200, http_params_types::params_required),
       CHAIN_RO_CALL(get_transaction_id, 200, http_params_types::params_required),
-      // transaction related APIs will be posted to read_write queue after keys are recovered, they are safe to run in parallel until they post to the read_write queue
       CHAIN_RO_CALL_ASYNC(send_read_only_transaction, chain_apis::read_only::send_read_only_transaction_results, 200, http_params_types::params_required),
       CHAIN_RO_CALL_ASYNC(compute_transaction, chain_apis::read_only::compute_transaction_results, 200, http_params_types::params_required),
+      // transaction related APIs will be posted to general queue after keys are recovered
       CHAIN_RW_CALL_ASYNC(push_transaction, chain_apis::read_write::push_transaction_results, 202, http_params_types::params_required),
       CHAIN_RW_CALL_ASYNC(push_transactions, chain_apis::read_write::push_transactions_results, 202, http_params_types::params_required),
       CHAIN_RW_CALL_ASYNC(send_transaction, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required),
       CHAIN_RW_CALL_ASYNC(send_transaction2, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required)
-   }, appbase::exec_queue::read_only);
+   }, appbase::exec_queue::read_only_trx_safe);
 
    // Not safe to run in parallel with read-only transactions
    _http_plugin.add_api({
       CHAIN_RW_CALL_ASYNC(push_block, chain_apis::read_write::push_block_results, 202, http_params_types::params_required)
-   }, appbase::exec_queue::read_write, appbase::priority::medium_low );
+   }, appbase::exec_queue::general, appbase::priority::medium_low );
 
    if (chain.account_queries_enabled()) {
       _http_plugin.add_async_api({
@@ -124,7 +124,7 @@ void chain_api_plugin::plugin_startup() {
    if (chain.transaction_finality_status_enabled()) {
       _http_plugin.add_api({
          CHAIN_RO_CALL_WITH_400(get_transaction_status, 200, http_params_types::params_required),
-      }, appbase::exec_queue::read_only);
+      }, appbase::exec_queue::read_only_trx_safe);
    }
 
    _http_plugin.add_api({
@@ -160,7 +160,7 @@ void chain_api_plugin::plugin_startup() {
            }
         }
       }
-   }, appbase::exec_queue::read_only);
+   }, appbase::exec_queue::read_only_trx_safe);
 }
 
 void chain_api_plugin::plugin_shutdown() {}

plugins/db_size_api_plugin/db_size_api_plugin.cpp

@@ -28,7 +28,7 @@ using namespace eosio;
 void db_size_api_plugin::plugin_startup() {
    app().get_plugin<http_plugin>().add_api({
        CALL_WITH_400(db_size, this, get,  INVOKE_R_V(this, get), 200),
-   }, appbase::exec_queue::read_only);
+   }, appbase::exec_queue::read_only_trx_safe);
 }
 
 db_size_stats db_size_api_plugin::get() {

plugins/http_plugin/http_plugin.cpp

@@ -28,7 +28,7 @@ namespace eosio {
    using std::regex;
    using boost::asio::ip::tcp;
    using std::shared_ptr;
-
+   
    static http_plugin_defaults current_http_plugin_defaults;
    static bool verbose_http_errors = false;
 
@@ -36,10 +36,6 @@ namespace eosio {
       current_http_plugin_defaults = config;
    }
 
-   std::string http_plugin::get_server_header() {
-      return current_http_plugin_defaults.server_header;
-   }
-
    using http_plugin_impl_ptr = std::shared_ptr<class http_plugin_impl>;
 
    class http_plugin_impl : public std::enable_shared_from_this<http_plugin_impl> {
@@ -51,9 +47,9 @@ namespace eosio {
 
          http_plugin_impl& operator=(const http_plugin_impl&) = delete;
          http_plugin_impl& operator=(http_plugin_impl&&) = delete;
-
+         
          std::optional<tcp::endpoint>  listen_endpoint;
-
+         
          std::optional<asio::local::stream_protocol::endpoint> unix_endpoint;
 
          shared_ptr<beast_http_listener<plain_session, tcp, tcp_socket_t > >  beast_server;
@@ -276,7 +272,7 @@ namespace eosio {
 
          my->plugin_state->server_header = current_http_plugin_defaults.server_header;
 
-
+         
          //watch out for the returns above when adding new code here
       } FC_LOG_AND_RETHROW()
    }
@@ -313,7 +309,7 @@ namespace eosio {
             if(my->unix_endpoint) {
                try {
                   my->create_beast_server(true);
-
+                  
                   my->beast_unix_server->listen(*my->unix_endpoint);
                   my->beast_unix_server->start_accept();
                } catch ( const fc::exception& e ){
@@ -338,8 +334,8 @@ namespace eosio {
                      handle_exception("node", "get_supported_apis", body.empty() ? "{}" : body, cb);
                   }
                }
-            }}, appbase::exec_queue::read_only);
-
+            }}, appbase::exec_queue::read_only_trx_safe);
+            
          } catch (...) {
             fc_elog(logger(), "http_plugin startup fails, shutting down");
             app().quit();
@@ -458,9 +454,9 @@ namespace eosio {
    fc::microseconds http_plugin::get_max_response_time()const {
       return my->plugin_state->max_response_time;
    }
-
+   
    size_t http_plugin::get_max_body_size()const {
       return my->plugin_state->max_body_size;
    }
-
+   
 }

plugins/http_plugin/include/eosio/http_plugin/http_plugin.hpp

@@ -78,7 +78,6 @@ namespace eosio {
 
         //must be called before initialize
         static void set_defaults(const http_plugin_defaults& config);
-        static std::string get_server_header();
 
         APPBASE_PLUGIN_REQUIRES()
         void set_program_options(options_description&, options_description& cfg) override;
@@ -122,7 +121,7 @@ namespace eosio {
         void register_metrics_listener(chain::plugin_interface::metrics_listener listener);
 
         size_t get_max_body_size()const;
-
+      
    private:
         std::shared_ptr<class http_plugin_impl> my;
    };
@@ -176,7 +175,7 @@ namespace eosio {
    };
 
    /**
-    * @brief Used to trim whitespace from body.
+    * @brief Used to trim whitespace from body. 
     * Returned string_view valid only for lifetime of body
     */
    inline std::string_view make_trimmed_string_view(const std::string& body) {

plugins/http_plugin/tests/unit_tests.cpp

@@ -52,7 +52,7 @@ public:
                   cb(200, fc::time_point::maximum(), fc::variant(ok ? string("yes") : string("no")));
                }
             },
-         }, appbase::exec_queue::read_write);
+         }, appbase::exec_queue::general);
    }
 
 private:

plugins/net_api_plugin/net_api_plugin.cpp

@@ -53,7 +53,11 @@ void net_api_plugin::plugin_startup() {
    ilog("starting net_api_plugin");
    // lifetime of plugin is lifetime of application
    auto& net_mgr = app().get_plugin<net_plugin>();
-   app().get_plugin<http_plugin>().add_async_api({
+   app().get_plugin<http_plugin>().add_api({
+    //   CALL(net, net_mgr, set_timeout,
+    //        INVOKE_V_R(net_mgr, set_timeout, int64_t), 200),
+    //   CALL(net, net_mgr, sign_transaction,
+    //        INVOKE_R_R_R_R(net_mgr, sign_transaction, chain::signed_transaction, flat_set<public_key_type>, chain::chain_id_type), 201),
        CALL_WITH_400(net, net_mgr, connect,
             INVOKE_R_R(net_mgr, connect, std::string), 201),
        CALL_WITH_400(net, net_mgr, disconnect,
@@ -62,7 +66,9 @@ void net_api_plugin::plugin_startup() {
             INVOKE_R_R(net_mgr, status, std::string), 201),
        CALL_WITH_400(net, net_mgr, connections,
             INVOKE_R_V(net_mgr, connections), 201),
-   } );
+    //   CALL(net, net_mgr, open,
+    //        INVOKE_V_R(net_mgr, open, std::string), 200),
+   }, appbase::exec_queue::read_only_trx_safe, appbase::priority::medium_high);
 }
 
 void net_api_plugin::plugin_initialize(const variables_map& options) {

plugins/net_plugin/net_plugin.cpp

@@ -3225,23 +3225,22 @@ namespace eosio {
             return;
          }
 
+         bool valid_block_header = !!bsp;
 
-         uint32_t block_num = bsp ? bsp->block_num : 0;
-
-         if( block_num != 0 ) {
+         if( valid_block_header ) {
             fc_dlog( logger, "validated block header, broadcasting immediately, connection ${cid}, blk num = ${num}, id = ${id}",
-                     ("cid", cid)("num", block_num)("id", bsp->id) );
+                     ("cid", cid)("num", bsp->block_num)("id", bsp->id) );
             my_impl->dispatcher->add_peer_block( bsp->id, cid ); // no need to send back to sender
             my_impl->dispatcher->bcast_block( bsp->block, bsp->id );
          }
 
-         app().executor().post(priority::medium, exec_queue::read_write, [ptr{std::move(ptr)}, bsp{std::move(bsp)}, id, c{std::move(c)}]() mutable {
+         app().executor().post(priority::medium, exec_queue::general, [ptr{std::move(ptr)}, bsp{std::move(bsp)}, id, c{std::move(c)}]() mutable {
             c->process_signed_block( id, std::move(ptr), std::move(bsp) );
          });
 
-         if( block_num != 0 ) {
+         if( valid_block_header ) {
             // ready to process immediately, so signal producer to interrupt start_block
-            my_impl->producer_plug->received_block(block_num);
+            my_impl->producer_plug->received_block();
          }
       });
    }

plugins/producer_api_plugin/producer.swagger.yaml

@@ -346,7 +346,7 @@ paths:
   /producer/schedule_snapshot:
     post:
       summary: schedule_snapshot
-      description: Submits a request to automatically generate snapshots according to a schedule specified with given parameters. If request body is empty, schedules immediate snapshot generation. Returns error when unable to accept schedule.
+      description: Submits a request to generate a schedule for automated snapshot with given parameters. If request body is empty, triest to execute snapshot immediately. Returns error when unable to create snapshot.
       operationId: schedule_snapshot
       requestBody:
         content:
@@ -377,12 +377,6 @@ paths:
             application/json:
               schema:
                 type: object
-                required:
-                    - snapshot_request_id
-                    - block_spacing
-                    - start_block_num
-                    - end_block_num
-                    - snapshot_description
                 properties:
                   snapshot_request_id:
                     type: integer
@@ -411,8 +405,8 @@ paths:
   /producer/get_snapshot_requests:
     post:
       summary: get_snapshot_requests
-      description: Returns a list of scheduled snapshots.
-      operationId: get_snapshot_requests
+      description: Returns a list of scheduled snapshots
+      operationId: get_snapshot_status
       responses:
         "201":
           description: OK
@@ -423,16 +417,9 @@ paths:
                 properties:
                   snapshot_requests:
                     type: array
-                    description: An array of scheduled snapshot requests
+                    description: An array of scheduled snapshots
                     items:
                       type: object
-                      required:
-                          - snapshot_request_id
-                          - block_spacing
-                          - start_block_num
-                          - end_block_num
-                          - snapshot_description
-                          - pending_snapshots
                       properties:
                         snapshot_request_id:
                           type: integer
@@ -457,12 +444,6 @@ paths:
                           description: List of pending snapshots
                           items:
                             type: object
-                            required:
-                                - head_block_id
-                                - head_block_num
-                                - head_block_time
-                                - version
-                                - snapshot_name
                             properties:
                               head_block_id:
                                 $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
@@ -491,7 +472,7 @@ paths:
   /producer/unschedule_snapshot:
     post:
       summary: unschedule_snapshot
-      description: Removes snapshot request identified by id. Returns error if referenced snapshot request does not exist.
+      description: Submits a request to remove identified by id recurring snapshot from schedule. Returns error when unable to create unschedule.
       operationId: unschedule_snapshot
       requestBody:
         required: true
@@ -510,12 +491,6 @@ paths:
             application/json:
               schema:
                 type: object
-                required:
-                  - snapshot_request_id
-                  - block_spacing
-                  - start_block_num
-                  - end_block_num
-                  - snapshot_description
                 properties:
                   snapshot_request_id:
                     type: integer

plugins/producer_api_plugin/producer_api_plugin.cpp

@@ -118,11 +118,9 @@ void producer_api_plugin::plugin_startup() {
             INVOKE_R_R(producer, get_account_ram_corrections, producer_plugin::get_account_ram_corrections_params), 201),
        CALL_WITH_400(producer, producer, get_unapplied_transactions,
                      INVOKE_R_R_D(producer, get_unapplied_transactions, producer_plugin::get_unapplied_transactions_params), 200),
-       CALL_WITH_400(producer, producer, get_snapshot_requests,
-                     INVOKE_R_V(producer, get_snapshot_requests), 201),
-   }, appbase::exec_queue::read_only, appbase::priority::medium_high);
+   }, appbase::exec_queue::read_only_trx_safe, appbase::priority::medium_high);
 
-   // Not safe to run in parallel
+   // Not safe to run in parallel with read-only transactions
    app().get_plugin<http_plugin>().add_api({
        CALL_WITH_400(producer, producer, pause,
             INVOKE_V_V(producer, pause), 201),
@@ -139,14 +137,16 @@ void producer_api_plugin::plugin_startup() {
        CALL_ASYNC(producer, producer, create_snapshot, producer_plugin::snapshot_information,
             INVOKE_R_V_ASYNC(producer, create_snapshot), 201),
        CALL_WITH_400(producer, producer, schedule_snapshot,
-            INVOKE_R_R_II(producer, schedule_snapshot, producer_plugin::snapshot_request_information), 201),
+            INVOKE_V_R_II(producer, schedule_snapshot, producer_plugin::snapshot_request_information), 201),
+       CALL_WITH_400(producer, producer, get_snapshot_requests,
+            INVOKE_R_V(producer, get_snapshot_requests), 201),
        CALL_WITH_400(producer, producer, unschedule_snapshot,
-            INVOKE_R_R(producer, unschedule_snapshot, producer_plugin::snapshot_request_id_information), 201),
+            INVOKE_V_R(producer, unschedule_snapshot, producer_plugin::snapshot_request_id_information), 201),
        CALL_WITH_400(producer, producer, get_integrity_hash,
             INVOKE_R_V(producer, get_integrity_hash), 201),
        CALL_WITH_400(producer, producer, schedule_protocol_feature_activations,
             INVOKE_V_R(producer, schedule_protocol_feature_activations, producer_plugin::scheduled_protocol_feature_activations), 201),
-   }, appbase::exec_queue::read_write, appbase::priority::medium_high);
+   }, appbase::exec_queue::general, appbase::priority::medium_high);
 }
 
 void producer_api_plugin::plugin_initialize(const variables_map& options) {

plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp

@@ -94,11 +94,8 @@ public:
       uint32_t snapshot_request_id = 0;
    };
 
-   struct snapshot_schedule_result : public snapshot_request_id_information, public snapshot_request_information {
-   };
-
    struct snapshot_schedule_information : public snapshot_request_id_information, public snapshot_request_information {
-      std::vector<snapshot_information> pending_snapshots;
+      std::optional<std::vector<snapshot_information>> pending_snapshots;
    };
 
    struct get_snapshot_requests_result {
@@ -162,8 +159,8 @@ public:
    integrity_hash_information get_integrity_hash() const;
 
    void create_snapshot(next_function<snapshot_information> next);
-   snapshot_schedule_result schedule_snapshot(const snapshot_request_information& schedule);
-   snapshot_schedule_result unschedule_snapshot(const snapshot_request_id_information& schedule);
+   void schedule_snapshot(const snapshot_request_information& schedule);
+   void unschedule_snapshot(const snapshot_request_id_information& schedule);
    get_snapshot_requests_result get_snapshot_requests() const;
 
    scheduled_protocol_feature_activations get_scheduled_protocol_feature_activations() const;
@@ -205,14 +202,11 @@ public:
    void register_metrics_listener(metrics_listener listener);
 
    // thread-safe, called when a new block is received
-   void received_block(uint32_t block_num);
+   void received_block();
 
-   const std::set<account_name>& producer_accounts() const;
+const std::set<account_name>& producer_accounts() const;
 
-   static void set_test_mode(bool m) { test_mode_ = m; }
  private:
-   inline static bool test_mode_{false}; // to be moved into appbase (application_base)
-
    std::shared_ptr<class producer_plugin_impl> my;
 };
 
@@ -227,7 +221,6 @@ FC_REFLECT(eosio::producer_plugin::snapshot_request_information, (block_spacing)
 FC_REFLECT(eosio::producer_plugin::snapshot_request_id_information, (snapshot_request_id))
 FC_REFLECT(eosio::producer_plugin::get_snapshot_requests_result, (snapshot_requests))
 FC_REFLECT_DERIVED(eosio::producer_plugin::snapshot_schedule_information, (eosio::producer_plugin::snapshot_request_id_information)(eosio::producer_plugin::snapshot_request_information), (pending_snapshots))
-FC_REFLECT_DERIVED(eosio::producer_plugin::snapshot_schedule_result, (eosio::producer_plugin::snapshot_request_id_information)(eosio::producer_plugin::snapshot_request_information),)
 FC_REFLECT(eosio::producer_plugin::scheduled_protocol_feature_activations, (protocol_features_to_activate))
 FC_REFLECT(eosio::producer_plugin::get_supported_protocol_features_params, (exclude_disabled)(exclude_unactivatable))
 FC_REFLECT(eosio::producer_plugin::get_account_ram_corrections_params, (lower_bound)(upper_bound)(limit)(reverse))

plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp

@@ -45,7 +45,7 @@ private:
    void x_serialize() {
       auto& vec = _snapshot_requests.get<as_vector>();
       std::vector<producer_plugin::snapshot_schedule_information> sr(vec.begin(), vec.end());
-      _snapshot_db << sr;
+      _snapshot_db << sr;   
    }
 
 public:
@@ -54,7 +54,7 @@ public:
    // snapshot_scheduler_listener
    void on_start_block(uint32_t height) {
       bool serialize_needed  = false;
-      bool snapshot_executed = false;
+      bool snapshot_executed = false; 
 
       auto execute_snapshot_with_log = [this, height, &snapshot_executed](const auto & req) {
          // one snapshot per height
@@ -64,18 +64,18 @@ public:
                ("end_block_num",   req.end_block_num)
                ("block_spacing",   req.block_spacing)
                ("height",          height));
-
+               
             execute_snapshot(req.snapshot_request_id);
             snapshot_executed = true;
          }
-      };
+      };    
 
       std::vector<uint32_t> unschedule_snapshot_request_ids;
       for(const auto& req: _snapshot_requests.get<0>()) {
          // -1 since its called from start block
          bool recurring_snapshot =  req.block_spacing &&  (height >= req.start_block_num + 1) && (!((height - req.start_block_num - 1) % req.block_spacing));
          bool onetime_snapshot   = (!req.block_spacing) && (height == req.start_block_num + 1);
-
+        
          // assume "asap" for snapshot with missed/zero start, it can have spacing
          if(!req.start_block_num) {
             // update start_block_num with current height only if this is recurring
@@ -84,16 +84,16 @@ public:
                auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
                auto it = snapshot_by_id.find(req.snapshot_request_id);
                _snapshot_requests.modify(it, [&height](auto& p) { p.start_block_num = height - 1; });
-               serialize_needed = true;
+               serialize_needed = true;              
             }
             execute_snapshot_with_log(req);
          } else if(recurring_snapshot || onetime_snapshot) {
             execute_snapshot_with_log(req);
-         }
+         }       
 
          // cleanup - remove expired (or invalid) request
-         if((!req.start_block_num && !req.block_spacing) ||
-            (!req.block_spacing && height >= (req.start_block_num + 1)) ||
+         if((!req.start_block_num && !req.block_spacing) ||         
+            (!req.block_spacing && height >= (req.start_block_num + 1)) || 
             (req.end_block_num > 0 && height >= (req.end_block_num + 1))) {
             unschedule_snapshot_request_ids.push_back(req.snapshot_request_id);
          }
@@ -108,7 +108,7 @@ public:
    }
 
    // snapshot_scheduler_handler
-   producer_plugin::snapshot_schedule_result schedule_snapshot(const producer_plugin::snapshot_request_information& sri) {
+   void schedule_snapshot(const producer_plugin::snapshot_request_information& sri) {
       auto& snapshot_by_value = _snapshot_requests.get<by_snapshot_value>();
       auto existing = snapshot_by_value.find(std::make_tuple(sri.block_spacing, sri.start_block_num, sri.end_block_num));
       EOS_ASSERT(existing == snapshot_by_value.end(), chain::duplicate_snapshot_request, "Duplicate snapshot request");
@@ -122,27 +122,19 @@ public:
          }
       }
 
-      _snapshot_requests.emplace(producer_plugin::snapshot_schedule_information {{_snapshot_id++}, {sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description},{}});
+      _snapshot_requests.emplace(producer_plugin::snapshot_schedule_information {{_snapshot_id++},{sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description},{}});
       x_serialize();
-
-      // returning snapshot_schedule_result
-      return  producer_plugin::snapshot_schedule_result{{_snapshot_id - 1}, {sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description}};
    }
 
-   producer_plugin::snapshot_schedule_result unschedule_snapshot(uint32_t sri) {
+   virtual void unschedule_snapshot(uint32_t sri) {
       auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
       auto existing = snapshot_by_id.find(sri);
       EOS_ASSERT(existing != snapshot_by_id.end(), chain::snapshot_request_not_found, "Snapshot request not found");
-
-      producer_plugin::snapshot_schedule_result result{{existing->snapshot_request_id}, {existing->block_spacing, existing->start_block_num, existing->end_block_num, existing->snapshot_description}};
       _snapshot_requests.erase(existing);
       x_serialize();
-
-      // returning snapshot_schedule_result
-      return result;
    }
 
-   producer_plugin::get_snapshot_requests_result get_snapshot_requests() {
+   virtual producer_plugin::get_snapshot_requests_result get_snapshot_requests() {
       producer_plugin::get_snapshot_requests_result result;
       auto& asvector = _snapshot_requests.get<as_vector>();
       result.snapshot_requests.reserve(asvector.size());
@@ -168,8 +160,11 @@ public:
       auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
       auto  snapshot_req   = snapshot_by_id.find(_inflight_sid);
       if (snapshot_req != snapshot_by_id.end()) {
-         _snapshot_requests.modify(snapshot_req, [&si](auto& p) {
-            p.pending_snapshots.emplace_back(si);
+         _snapshot_requests.modify(snapshot_req, [&si](auto& p) { 
+            if (!p.pending_snapshots) {
+               p.pending_snapshots = std::vector<producer_plugin::snapshot_information>();
+            }
+            p.pending_snapshots->emplace_back(si);       
          });
       }
    }
@@ -198,12 +193,14 @@ public:
             auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
             auto  snapshot_req   = snapshot_by_id.find(srid);
 
-            if (snapshot_req != snapshot_by_id.end()) {
-               _snapshot_requests.modify(snapshot_req, [&](auto& p) {
-                  auto & pending = p.pending_snapshots;
-                  auto it = std::remove_if(pending.begin(), pending.end(), [&snapshot_info](const producer_plugin::snapshot_information & s){ return s.head_block_num <=  snapshot_info.head_block_num; });
-                  pending.erase(it, pending.end());
-               });
+            if (snapshot_req != snapshot_by_id.end()) {               
+               if (auto pending = snapshot_req->pending_snapshots; pending) {                 
+                  auto it = std::remove_if(pending->begin(), pending->end(), [&snapshot_info](const producer_plugin::snapshot_information & s){ return s.head_block_num <=  snapshot_info.head_block_num; });
+                  pending->erase(it, pending->end());
+                  _snapshot_requests.modify(snapshot_req, [&pending](auto& p) { 
+                     p.pending_snapshots = std::move(pending);
+                  });
+               }
             }
          }
       };

plugins/producer_plugin/producer_plugin.cpp

@@ -14,7 +14,6 @@
 #include <fc/io/json.hpp>
 #include <fc/log/logger_config.hpp>
 #include <fc/scoped_exit.hpp>
-#include <fc/time.hpp>
 
 #include <boost/asio.hpp>
 #include <boost/date_time/posix_time/posix_time.hpp>
@@ -28,6 +27,7 @@
 #include <boost/multi_index/member.hpp>
 #include <boost/multi_index/hashed_index.hpp>
 #include <boost/multi_index/ordered_index.hpp>
+#include <boost/multi_index/random_access_index.hpp>
 #include <boost/signals2/connection.hpp>
 
 namespace bmi = boost::multi_index;
@@ -256,53 +256,39 @@ struct block_time_tracker {
       block_idle_time += idle;
    }
 
-   void add_fail_time( const fc::microseconds& fail_time, bool is_transient ) {
-      if( is_transient ) {
-         // transient time includes both success and fail time
-         transient_trx_time += fail_time;
-         ++transient_trx_num;
-      } else {
-         trx_fail_time += fail_time;
-         ++trx_fail_num;
-      }
+   void add_fail_time( const fc::microseconds& fail_time ) {
+      trx_fail_time += fail_time;
+      ++trx_fail_num;
    }
 
-   void add_success_time( const fc::microseconds& time, bool is_transient ) {
-      if( is_transient ) {
-         transient_trx_time += time;
-         ++transient_trx_num;
-      } else {
-         trx_success_time += time;
-         ++trx_success_num;
-      }
+   void add_success_time( const fc::microseconds& time ) {
+      trx_success_time += time;
+      ++trx_success_num;
    }
 
    void report( const fc::time_point& idle_trx_time, uint32_t block_num ) {
       if( _log.is_enabled( fc::log_level::debug ) ) {
          auto now = fc::time_point::now();
          add_idle_time( now - idle_trx_time );
-         fc_dlog( _log, "Block #${n} trx idle: ${i}us out of ${t}us, success: ${sn}, ${s}us, fail: ${fn}, ${f}us, transient: ${trans_trx_num}, ${trans_trx_time}us, other: ${o}us",
+         fc_dlog( _log, "Block #${n} trx idle: ${i}us out of ${t}us, success: ${sn}, ${s}us, fail: ${fn}, ${f}us, other: ${o}us",
                   ("n", block_num)
                   ("i", block_idle_time)("t", now - clear_time)("sn", trx_success_num)("s", trx_success_time)
                   ("fn", trx_fail_num)("f", trx_fail_time)
-                  ("trans_trx_num", transient_trx_num)("trans_trx_time", transient_trx_time)
-                  ("o", (now - clear_time) - block_idle_time - trx_success_time - trx_fail_time - transient_trx_time) );
+                  ("o", (now - clear_time) - block_idle_time - trx_success_time - trx_fail_time) );
       }
    }
 
    void clear() {
-      block_idle_time = trx_fail_time = trx_success_time = transient_trx_time = fc::microseconds{};
-      trx_fail_num = trx_success_num = transient_trx_num = 0;
+      block_idle_time = trx_fail_time = trx_success_time = fc::microseconds{};
+      trx_fail_num = trx_success_num = 0;
       clear_time = fc::time_point::now();
    }
 
    fc::microseconds block_idle_time;
    uint32_t trx_success_num = 0;
    uint32_t trx_fail_num = 0;
-   uint32_t transient_trx_num = 0;
    fc::microseconds trx_success_time;
    fc::microseconds trx_fail_time;
-   fc::microseconds transient_trx_time;
    fc::time_point clear_time{fc::time_point::now()};
 };
 
@@ -313,7 +299,8 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       producer_plugin_impl(boost::asio::io_service& io)
       :_timer(io)
       ,_transaction_ack_channel(app().get_channel<compat::channels::transaction_ack>())
-      ,_ro_timer(io)
+      ,_ro_write_window_timer(io)
+      ,_ro_read_window_timer(io)
       {
       }
 
@@ -337,9 +324,9 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       push_result push_transaction( const fc::time_point& block_deadline,
                                     const transaction_metadata_ptr& trx,
                                     bool api_trx, bool return_failure_trace,
-                                    const next_function<transaction_trace_ptr>& next );
+                                    next_function<transaction_trace_ptr> next );
       push_result handle_push_result( const transaction_metadata_ptr& trx,
-                                      const next_function<transaction_trace_ptr>& next,
+                                      next_function<transaction_trace_ptr> next,
                                       const fc::time_point& start,
                                       const chain::controller& chain,
                                       const transaction_trace_ptr& trace,
@@ -369,7 +356,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       named_thread_pool<struct prod>                            _thread_pool;
 
       std::atomic<int32_t>                                      _max_transaction_time_ms; // modified by app thread, read by net_plugin thread pool
-      std::atomic<uint32_t>                                     _received_block{0}; // modified by net_plugin thread pool
+      std::atomic<bool>                                         _received_block{false}; // modified by net_plugin thread pool and app thread
       fc::microseconds                                          _max_irreversible_block_age_us;
       int32_t                                                   _produce_time_offset_us = 0;
       int32_t                                                   _last_block_time_offset_us = 0;
@@ -424,65 +411,42 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
 
       // async snapshot scheduler
       snapshot_scheduler _snapshot_scheduler;
-
+      
       // ro for read-only
       struct ro_trx_t {
-         transaction_metadata_ptr trx;
-         next_func_t              next;
+         packed_transaction_ptr  trx;
+         next_func_t             next;
       };
-      // The queue storing previously exhausted read-only transactions to be re-executed by read-only threads
-      // thread-safe
-      class ro_trx_queue_t {
-      public:
-         void push_front(ro_trx_t&& t) {
-            std::lock_guard g(mtx);
-            queue.push_front(std::move(t));
-         }
-
-         bool empty() const {
-            std::lock_guard g(mtx);
-            return queue.empty();
-         }
-
-         bool pop_front(ro_trx_t& t) {
-            std::unique_lock g(mtx);
-            if (queue.empty())
-               return false;
-            t = queue.front();
-            queue.pop_front();
-            return true;
-         }
-
-      private:
-         mutable std::mutex      mtx;
-         deque<ro_trx_t>         queue;  // boost deque which is faster than std::deque
+      // The queue storing read-only transactions to be executed by read-only threads
+      struct ro_trx_queue_t {
+         std::mutex              mtx;
+         std::deque<ro_trx_t>    queue;
       };
-
-      uint32_t                        _ro_thread_pool_size{ 0 };
-       // Due to uncertainty to get total virtual memory size on a 5-level paging system for eos-vm-oc and
-       // possible memory exhuastion for large number of contract usage for non-eos-vm-oc, set a hard limit
-      static constexpr uint32_t       _ro_max_threads_allowed{ 8 };
+      uint16_t                        _ro_thread_pool_size{ 0 };
+      static constexpr uint16_t       _ro_max_eos_vm_oc_threads_allowed{ 8 }; // Due to uncertainty to get total virtual memory size on a 5-level paging system, set a hard limit
       named_thread_pool<struct read>  _ro_thread_pool;
       fc::microseconds                _ro_write_window_time_us{ 200000 };
       fc::microseconds                _ro_read_window_time_us{ 60000 };
       static constexpr fc::microseconds _ro_read_window_minimum_time_us{ 10000 };
       fc::microseconds                _ro_read_window_effective_time_us{ 0 }; // calculated during option initialization
-      std::atomic<int64_t>            _ro_all_threads_exec_time_us; // total time spent by all threads executing transactions. use atomic for simplicity and performance
-      fc::time_point                  _ro_read_window_start_time;
-      fc::time_point                  _ro_window_deadline; // only modified on app thread, read-window deadline or write-window deadline
-      boost::asio::deadline_timer     _ro_timer; // only accessible from the main thread
+      boost::asio::deadline_timer     _ro_write_window_timer;
+      boost::asio::deadline_timer     _ro_read_window_timer;
       fc::microseconds                _ro_max_trx_time_us{ 0 }; // calculated during option initialization
-      ro_trx_queue_t                  _ro_exhausted_trx_queue;
-      std::atomic<uint32_t>           _ro_num_active_exec_tasks{ 0 };
-      std::vector<std::future<bool>>  _ro_exec_tasks_fut;
+      ro_trx_queue_t                  _ro_trx_queue;
+      std::atomic<uint32_t>            _ro_num_active_trx_exec_tasks{ 0 };
+      std::vector<std::future<bool>>  _ro_trx_exec_tasks_fut;
 
       void start_write_window();
       void switch_to_write_window();
       void switch_to_read_window();
-      bool read_only_execution_task(uint32_t pending_block_num);
-      void repost_exhausted_transactions(const fc::time_point& deadline);
-      bool push_read_only_transaction(transaction_metadata_ptr trx, next_function<transaction_trace_ptr> next);
-
+      bool read_only_trx_execution_task();
+      bool process_read_only_transaction(const packed_transaction_ptr& trx,
+                                         const next_function<transaction_trace_ptr>& next,
+                                         const fc::time_point& read_window_start_time);
+      bool push_read_only_transaction(const transaction_metadata_ptr& trx,
+                                      next_function<transaction_trace_ptr> next,
+                                      const fc::time_point& read_window_start_time);
+      
       void consider_new_watermark( account_name producer, uint32_t block_num, block_timestamp_type timestamp) {
          auto itr = _producer_watermarks.find( producer );
          if( itr != _producer_watermarks.end() ) {
@@ -516,9 +480,8 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       }
 
       void on_irreversible_block( const signed_block_ptr& lib ) {
-         const chain::controller& chain = chain_plug->chain();
-         EOS_ASSERT(chain.is_write_window(), producer_exception, "write window is expected for on_irreversible_block signal");
          _irreversible_block_time = lib->timestamp.to_time_point();
+         const chain::controller& chain = chain_plug->chain();
 
          // promote any pending snapshots
          auto& snapshots_by_height = _pending_snapshot_index.get<by_height>();
@@ -676,12 +639,12 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
                                          transaction_metadata::trx_type trx_type,
                                          bool return_failure_traces,
                                          next_function<transaction_trace_ptr> next) {
-         if ( trx_type == transaction_metadata::trx_type::read_only ) {
-            // Post all read only trxs to read_only queue for execution.
-            auto trx_metadata = transaction_metadata::create_no_recover_keys( trx, transaction_metadata::trx_type::read_only );
-            app().executor().post(priority::low, exec_queue::read_only, [this, trx{std::move(trx_metadata)}, next{std::move(next)}]() mutable {
-               push_read_only_transaction( std::move(trx), std::move(next) );
-            } );
+         if ( trx_type == transaction_metadata::trx_type::read_only && _ro_thread_pool_size > 0 ) {
+            // Parallel read-only trx execution enabled.
+            // Store the transaction in read-only-trx queue so that it is
+            // executed in read window
+            std::lock_guard<std::mutex> g( _ro_trx_queue.mtx );
+            _ro_trx_queue.queue.push_back({trx, std::move(next)});
             return;
          }
 
@@ -714,7 +677,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
                                                           next{std::move(next)}, trx=trx]() mutable {
             if( future.valid() ) {
                future.wait();
-               app().executor().post( priority::low, exec_queue::read_write, [self, future{std::move(future)}, api_trx, is_transient, next{std::move( next )}, trx{std::move(trx)}, return_failure_traces]() mutable {
+               app().executor().post( priority::low, exec_queue::general, [self, future{std::move(future)}, api_trx, is_transient, next{std::move( next )}, trx{std::move(trx)}, return_failure_traces]() mutable {
                   auto start = fc::time_point::now();
                   auto idle_time = start - self->_idle_trx_time;
                   self->_time_tracker.add_idle_time( idle_time );
@@ -725,8 +688,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
                      self->log_trx_results( trx, nullptr, ex, 0, start, is_transient );
                      next( std::move(ex) );
                      self->_idle_trx_time = fc::time_point::now();
-                     auto dur = self->_idle_trx_time - start;
-                     self->_time_tracker.add_fail_time(dur, is_transient);
+                     self->_time_tracker.add_fail_time(self->_idle_trx_time - start);
                   };
                   try {
                      auto result = future.get();
@@ -747,7 +709,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       bool process_incoming_transaction_async(const transaction_metadata_ptr& trx,
                                               bool api_trx,
                                               bool return_failure_trace,
-                                              const next_function<transaction_trace_ptr>& next) {
+                                              next_function<transaction_trace_ptr> next) {
          bool exhausted = false;
          chain::controller& chain = chain_plug->chain();
          try {
@@ -827,7 +789,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
          exhausted
       };
 
-      inline bool should_interrupt_start_block( const fc::time_point& deadline, uint32_t pending_block_num ) const;
+      inline bool should_interrupt_start_block( const fc::time_point& deadline ) const;
       start_block_result start_block();
 
       fc::time_point calculate_pending_block_time() const;
@@ -930,12 +892,12 @@ void producer_plugin::set_program_options(
           "Number of worker threads in producer thread pool")
          ("snapshots-dir", bpo::value<bfs::path>()->default_value("snapshots"),
           "the location of the snapshots directory (absolute path or relative to application data dir)")
-         ("read-only-threads", bpo::value<uint32_t>(),
-          "Number of worker threads in read-only execution thread pool. Max 8.")
+         ("read-only-threads", bpo::value<uint16_t>()->default_value(my->_ro_thread_pool_size),
+          "Number of worker threads in read-only transaction execution thread pool")
          ("read-only-write-window-time-us", bpo::value<uint32_t>()->default_value(my->_ro_write_window_time_us.count()),
-          "Time in microseconds the write window lasts.")
+          "time in microseconds the write window lasts")
          ("read-only-read-window-time-us", bpo::value<uint32_t>()->default_value(my->_ro_read_window_time_us.count()),
-          "Time in microseconds the read window lasts.")
+          "time in microseconds the read window lasts")
          ;
    config_file_options.add(producer_options);
 }
@@ -1111,22 +1073,11 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
       }
    }
 
-   if ( options.count( "read-only-threads" ) ) {
-      my->_ro_thread_pool_size = options.at( "read-only-threads" ).as<uint32_t>();
-   } else if ( my->_producers.empty() ) {
-      if( options.count( "plugin" ) ) {
-         const auto& v = options.at( "plugin" ).as<std::vector<std::string>>();
-         auto i = std::find_if( v.cbegin(), v.cend(), []( const std::string& p ) { return p == "eosio::chain_api_plugin"; } );
-         if( i != v.cend() ) {
-            // default to 3 threads for non producer nodes running chain_api_plugin if not specified
-            my->_ro_thread_pool_size = 3;
-            ilog( "chain_api_plugin configured, defaulting read-only-threads to ${t}", ("t", my->_ro_thread_pool_size) );
-         }
-      }
-   }
-   EOS_ASSERT( test_mode_ || my->_ro_thread_pool_size == 0 || my->_producers.empty(), plugin_config_exception, "--read-only-threads not allowed on producer node" );
+   my->_ro_thread_pool_size = options.at( "read-only-threads" ).as<uint16_t>();
    // only initialize other read-only options when read-only thread pool is enabled
    if ( my->_ro_thread_pool_size > 0 ) {
+      EOS_ASSERT( my->_producers.empty(), plugin_config_exception, "--read-only-threads not allowed on producer node" );
+
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
       if (chain.is_eos_vm_oc_enabled()) {
          // EOS VM OC requires 4.2TB Virtual for each executing thread. Make sure the memory
@@ -1147,37 +1098,36 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
          }
 
          EOS_ASSERT( vm_total_kb > 0, plugin_config_exception, "Unable to get system virtual memory size (not a Linux?), therefore cannot determine if the system has enough virtual memory for multi-threaded read-only transactions on EOS VM OC");
-         EOS_ASSERT( vm_total_kb > vm_used_kb, plugin_config_exception, "vm total (${t}) must be greater than vm used (${u})", ("t", vm_total_kb)("u", vm_used_kb));
-         uint32_t num_threads_supported = (vm_total_kb - vm_used_kb) / 4200000000;
          // reserve 1 for the app thread, 1 for anything else which might use VM
-         EOS_ASSERT( num_threads_supported > 2, plugin_config_exception, "With the EOS VM OC configured, there is not enough system virtual memory to support the required minimum of 3 threads (1 for main thread, 1 for read-only, and 1 for anything else), vm total: ${t}, vm used: ${u}", ("t", vm_total_kb)("u", vm_used_kb));
-         num_threads_supported -= 2;
-         auto actual_threads_allowed = std::min(my->_ro_max_threads_allowed, num_threads_supported);
-         ilog("vm total in kb: ${total}, vm used in kb: ${used}, number of EOS VM OC threads supported ((vm total - vm used)/4.2 TB - 2): ${supp}, max allowed: ${max}, actual allowed: ${actual}", ("total", vm_total_kb) ("used", vm_used_kb) ("supp", num_threads_supported) ("max", my->_ro_max_threads_allowed)("actual", actual_threads_allowed));
-         EOS_ASSERT( my->_ro_thread_pool_size <= actual_threads_allowed, plugin_config_exception, "--read-only-threads (${th}) greater than number of threads allowed for EOS VM OC (${allowed})", ("th", my->_ro_thread_pool_size) ("allowed", actual_threads_allowed) );
+         int num_threads_supported = (vm_total_kb - vm_used_kb) / 4200000000 - 2;
+         ilog("vm total in kb: ${total}, vm used in kb: ${used}, number of EOS VM OC threads supported ((vm total - vm used)/4.2 TB - 2): ${supp}", ("total", vm_total_kb) ("used", vm_used_kb) ("supp", num_threads_supported));
+         EOS_ASSERT( num_threads_supported >= my->_ro_thread_pool_size, plugin_config_exception, "--read-only-threads (${th}) greater than number of threads supported for EOS VM OC (${supp}) by the system virtual memory size", ("th", my->_ro_thread_pool_size) ("supp", num_threads_supported) );
+
+         if ( my->_ro_thread_pool_size > my->_ro_max_eos_vm_oc_threads_allowed ) {
+            wlog("--read-only-threads (${th}) greater than maximum number of threads allowed (${allowed}) for EOS Vm OC. Set it to ${allowed}", ("th", my->_ro_thread_pool_size) ("allowed", my->_ro_max_eos_vm_oc_threads_allowed));
+            my->_ro_thread_pool_size = my->_ro_max_eos_vm_oc_threads_allowed;
+         }
       }
 #endif
-      EOS_ASSERT( my->_ro_thread_pool_size <= my->_ro_max_threads_allowed, plugin_config_exception, "--read-only-threads (${th}) greater than number of threads allowed (${allowed})", ("th", my->_ro_thread_pool_size) ("allowed", my->_ro_max_threads_allowed) );
-   }
 
-   my->_ro_write_window_time_us = fc::microseconds( options.at( "read-only-write-window-time-us" ).as<uint32_t>() );
-   my->_ro_read_window_time_us = fc::microseconds( options.at( "read-only-read-window-time-us" ).as<uint32_t>() );
-   EOS_ASSERT( my->_ro_read_window_time_us > my->_ro_read_window_minimum_time_us, plugin_config_exception, "minimum of --read-only-read-window-time-us (${read}) must be ${min} microseconds", ("read", my->_ro_read_window_time_us) ("min", my->_ro_read_window_minimum_time_us) );
-   my->_ro_read_window_effective_time_us = my->_ro_read_window_time_us - my->_ro_read_window_minimum_time_us;
+      my->_ro_write_window_time_us = fc::microseconds( options.at( "read-only-write-window-time-us" ).as<uint32_t>() );
+      my->_ro_read_window_time_us = fc::microseconds( options.at( "read-only-read-window-time-us" ).as<uint32_t>() );
+      EOS_ASSERT( my->_ro_read_window_time_us > my->_ro_read_window_minimum_time_us, plugin_config_exception, "minimum of --read-only-read-window-time-us (${read}) must be ${min} microseconds", ("read", my->_ro_read_window_time_us) ("min", my->_ro_read_window_minimum_time_us) );
+      my->_ro_read_window_effective_time_us = my->_ro_read_window_time_us - my->_ro_read_window_minimum_time_us;
 
-   // Make sure a read-only transaction can finish within the read
-   // window if scheduled at the very beginning of the window.
-   // Use _ro_read_window_effective_time_us instead of _ro_read_window_time_us
-   // for safety margin
-   if ( my->_max_transaction_time_ms.load() > 0 ) {
-      EOS_ASSERT( my->_ro_read_window_effective_time_us > fc::milliseconds(my->_max_transaction_time_ms.load()), plugin_config_exception, "--read-only-read-window-time-us (${read}) must be greater than --max-transaction-time ${trx_time} ms plus a margin of ${min} us", ("read", my->_ro_read_window_time_us) ("trx_time", my->_max_transaction_time_ms.load()) ("min", my->_ro_read_window_minimum_time_us) );
-      my->_ro_max_trx_time_us = fc::milliseconds(my->_max_transaction_time_ms.load());
-   } else {
-      // _max_transaction_time_ms can be set to negative in testing (for unlimited)
-      my->_ro_max_trx_time_us = my->_ro_read_window_effective_time_us;
+      // Make sure a read-only transaction can finish within the read
+      // window if scheduled at the very beginning of the window.
+      // Use _ro_read_window_effective_time_us instead of _ro_read_window_time_us
+      // for safety marging
+      if ( my->_max_transaction_time_ms.load() > 0 ) {
+         EOS_ASSERT( my->_ro_read_window_effective_time_us > fc::milliseconds(my->_max_transaction_time_ms.load()), plugin_config_exception, "--read-only-read-window-time-us (${read}) must be greater than --max-transaction-time ${trx_time} ms plus a margin of ${min} us", ("read", my->_ro_read_window_time_us) ("trx_time", my->_max_transaction_time_ms.load()) ("min", my->_ro_read_window_minimum_time_us) );
+         my->_ro_max_trx_time_us = fc::milliseconds(my->_max_transaction_time_ms.load());
+      } else {
+         // _max_transaction_time_ms can be set to negative in testing (for unlimited)
+         my->_ro_max_trx_time_us = my->_ro_read_window_effective_time_us;
+      }
+      ilog("_ro_thread_pool_size ${s},  _ro_write_window_time_us ${ww}, _ro_read_window_time_us ${rw}, _ro_max_trx_time_us ${t}, _ro_read_window_effective_time_us ${w}", ("s", my->_ro_thread_pool_size) ("ww", my->_ro_write_window_time_us) ("rw", my->_ro_read_window_time_us) ("t", my->_ro_max_trx_time_us) ("w", my->_ro_read_window_effective_time_us));
    }
-   ilog("ro_thread_pool_size ${s}, ro_write_window_time_us ${ww}, ro_read_window_time_us ${rw}, ro_max_trx_time_us ${t}, ro_read_window_effective_time_us ${w}",
-        ("s", my->_ro_thread_pool_size)("ww", my->_ro_write_window_time_us)("rw", my->_ro_read_window_time_us)("t", my->_ro_max_trx_time_us)("w", my->_ro_read_window_effective_time_us));
 
    my->_incoming_block_sync_provider = app().get_method<incoming::methods::block_sync>().register_provider(
          [this](const signed_block_ptr& block, const std::optional<block_id_type>& block_id, const block_state_ptr& bsp) {
@@ -1214,7 +1164,6 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
    my->_snapshot_scheduler.set_create_snapshot_fn([this](producer_plugin::next_function<producer_plugin::snapshot_information> next){create_snapshot(next);});
 } FC_LOG_AND_RETHROW() }
 
-using namespace std::chrono_literals;
 void producer_plugin::plugin_startup()
 { try {
    try {
@@ -1260,27 +1209,15 @@ void producer_plugin::plugin_startup()
    }
 
    if ( my->_ro_thread_pool_size > 0 ) {
-      std::atomic<uint32_t> num_threads_started = 0;
       my->_ro_thread_pool.start( my->_ro_thread_pool_size,
          []( const fc::exception& e ) {
-            fc_elog( _log, "Exception in read-only thread pool, exiting: ${e}", ("e", e.to_detail_string()) );
+            fc_elog( _log, "Exception in read-only transaction thread pool, exiting: ${e}", ("e", e.to_detail_string()) );
             app().quit();
          },
          [&]() {
             chain.init_thread_local_data();
-            ++num_threads_started;
          });
 
-      // This will be changed with std::latch or std::atomic<>::wait
-      // when C++20 is used.
-      auto time_slept_ms = 0;
-      constexpr auto max_time_slept_ms = 1000;
-      while ( num_threads_started.load() < my->_ro_thread_pool_size && time_slept_ms < max_time_slept_ms ) {
-         std::this_thread::sleep_for( 1ms );
-         ++time_slept_ms;
-      }
-      EOS_ASSERT(num_threads_started.load() == my->_ro_thread_pool_size, producer_exception, "read-only threads failed to start. num_threads_started: ${n}, time_slept_ms: ${t}ms", ("n", num_threads_started.load())("t", time_slept_ms));
-
       my->start_write_window();
    }
 
@@ -1569,14 +1506,14 @@ void producer_plugin::create_snapshot(producer_plugin::next_function<producer_pl
    }
 }
 
-producer_plugin::snapshot_schedule_result producer_plugin::schedule_snapshot(const snapshot_request_information& sri)
+void producer_plugin::schedule_snapshot(const snapshot_request_information& sri)
 {
-   return my->_snapshot_scheduler.schedule_snapshot(sri);
+   my->_snapshot_scheduler.schedule_snapshot(sri);
 }
 
-producer_plugin::snapshot_schedule_result producer_plugin::unschedule_snapshot(const snapshot_request_id_information& sri)
+void producer_plugin::unschedule_snapshot(const snapshot_request_id_information& sri)
 {
-   return my->_snapshot_scheduler.unschedule_snapshot(sri.snapshot_request_id);
+   my->_snapshot_scheduler.unschedule_snapshot(sri.snapshot_request_id);
 }
 
 producer_plugin::get_snapshot_requests_result producer_plugin::get_snapshot_requests() const
@@ -1840,12 +1777,12 @@ fc::time_point producer_plugin_impl::calculate_block_deadline( const fc::time_po
    }
 }
 
-bool producer_plugin_impl::should_interrupt_start_block( const fc::time_point& deadline, uint32_t pending_block_num ) const {
+bool producer_plugin_impl::should_interrupt_start_block( const fc::time_point& deadline ) const {
    if( _pending_block_mode == pending_block_mode::producing ) {
       return deadline <= fc::time_point::now();
    }
    // if we can produce then honor deadline so production starts on time
-   return (!_producers.empty() && deadline <= fc::time_point::now()) || (_received_block >= pending_block_num);
+   return (!_producers.empty() && deadline <= fc::time_point::now()) || _received_block;
 }
 
 producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
@@ -1865,7 +1802,6 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
 
    const fc::time_point now = fc::time_point::now();
    const fc::time_point block_time = calculate_pending_block_time();
-   const uint32_t pending_block_num = hbs->block_num + 1;
    const fc::time_point preprocess_deadline = calculate_block_deadline(block_time);
 
    const pending_block_mode previous_pending_mode = _pending_block_mode;
@@ -1931,7 +1867,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
    if (_pending_block_mode == pending_block_mode::producing) {
       const auto start_block_time = block_time - fc::microseconds( config::block_interval_us );
       if( now < start_block_time ) {
-         fc_dlog(_log, "Not producing block waiting for production window ${n} ${bt}", ("n", pending_block_num)("bt", block_time) );
+         fc_dlog(_log, "Not producing block waiting for production window ${n} ${bt}", ("n", hbs->block_num + 1)("bt", block_time) );
          // start_block_time instead of block_time because schedule_delayed_production_loop calculates next block time from given time
          schedule_delayed_production_loop(weak_from_this(), calculate_producer_wake_up_time(start_block_time));
          return start_block_result::waiting_for_production;
@@ -1945,7 +1881,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
    }
 
    fc_dlog(_log, "Starting block #${n} at ${time} producer ${p}",
-           ("n", pending_block_num)("time", now)("p", scheduled_producer.producer_name));
+           ("n", hbs->block_num + 1)("time", now)("p", scheduled_producer.producer_name));
 
    try {
       uint16_t blocks_to_confirm = 0;
@@ -2009,7 +1945,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
             std::swap( features_to_activate, protocol_features_to_activate );
             _protocol_features_signaled = true;
             ilog( "signaling activation of the following protocol features in block ${num}: ${features_to_activate}",
-                  ("num", pending_block_num)("features_to_activate", features_to_activate) );
+                  ("num", hbs->block_num + 1)("features_to_activate", features_to_activate) );
          }
       }
 
@@ -2035,7 +1971,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
          if( !remove_expired_blacklisted_trxs( preprocess_deadline ) )
             return start_block_result::exhausted;
          if( !_subjective_billing.remove_expired( _log, chain.pending_block_time(), fc::time_point::now(),
-                                                  [&](){ return should_interrupt_start_block( preprocess_deadline, pending_block_num ); } ) ) {
+                                                  [&](){ return should_interrupt_start_block( preprocess_deadline ); } ) ) {
             return start_block_result::exhausted;
          }
 
@@ -2058,11 +1994,9 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
             process_scheduled_and_incoming_trxs( scheduled_trx_deadline, incoming_itr );
          }
 
-         repost_exhausted_transactions( preprocess_deadline );
-
          if( app().is_quiting() ) // db guard exception above in LOG_AND_DROP could have called app().quit()
             return start_block_result::failed;
-         if ( should_interrupt_start_block( preprocess_deadline, pending_block_num ) || block_is_exhausted() ) {
+         if ( should_interrupt_start_block( preprocess_deadline ) || block_is_exhausted() ) {
             return start_block_result::exhausted;
          }
 
@@ -2089,12 +2023,11 @@ bool producer_plugin_impl::remove_expired_trxs( const fc::time_point& deadline )
 {
    chain::controller& chain = chain_plug->chain();
    auto pending_block_time = chain.pending_block_time();
-   auto pending_block_num = chain.pending_block_num();
 
    // remove all expired transactions
    size_t num_expired = 0;
    size_t orig_count = _unapplied_transactions.size();
-   bool exhausted = !_unapplied_transactions.clear_expired( pending_block_time, [&](){ return should_interrupt_start_block(deadline, pending_block_num); },
+   bool exhausted = !_unapplied_transactions.clear_expired( pending_block_time, [&](){ return should_interrupt_start_block(deadline); },
          [&num_expired]( const packed_transaction_ptr& packed_trx_ptr, trx_enum_type trx_type ) {
             // expired exception is logged as part of next() call
             ++num_expired;
@@ -2118,13 +2051,12 @@ bool producer_plugin_impl::remove_expired_blacklisted_trxs( const fc::time_point
    if(!blacklist_by_expiry.empty()) {
       const chain::controller& chain = chain_plug->chain();
       const auto lib_time = chain.last_irreversible_block_time();
-      const auto pending_block_num = chain.pending_block_num();
 
       int num_expired = 0;
       int orig_count = _blacklisted_transactions.size();
 
       while (!blacklist_by_expiry.empty() && blacklist_by_expiry.begin()->expiry <= lib_time) {
-         if ( should_interrupt_start_block( deadline, pending_block_num ) ) {
+         if ( should_interrupt_start_block( deadline ) ) {
             exhausted = true;
             break;
          }
@@ -2263,10 +2195,9 @@ producer_plugin_impl::push_transaction( const fc::time_point& block_deadline,
                                         const transaction_metadata_ptr& trx,
                                         bool api_trx,
                                         bool return_failure_trace,
-                                        const next_function<transaction_trace_ptr>& next )
+                                        next_function<transaction_trace_ptr> next )
 {
    auto start = fc::time_point::now();
-   EOS_ASSERT(!trx->is_read_only(), producer_exception, "Unexpected read-only trx");
 
    bool disable_subjective_enforcement = (api_trx && _disable_subjective_api_billing)
                                          || (!api_trx && _disable_subjective_p2p_billing)
@@ -2283,7 +2214,7 @@ producer_plugin_impl::push_transaction( const fc::time_point& block_deadline,
          log_trx_results( trx, except_ptr );
          next( except_ptr );
       }
-      _time_tracker.add_fail_time(fc::time_point::now() - start, trx->is_transient());
+      _time_tracker.add_fail_time(fc::time_point::now() - start);
       return push_result{.failed = true};
    }
 
@@ -2303,6 +2234,14 @@ producer_plugin_impl::push_transaction( const fc::time_point& block_deadline,
       }
    }
 
+   // Make sure db_read_only_mode always to be unset
+   auto db_read_only_mode_guard = fc::make_scoped_exit([trx, &chain]{
+      if( trx->is_read_only() )
+         chain.unset_db_read_only_mode();
+   });
+   if( trx->is_read_only() )
+      chain.set_db_read_only_mode();
+
    auto trace = chain.push_transaction( trx, block_deadline, max_trx_time, prev_billed_cpu_time_us, false, sub_bill );
 
    return handle_push_result(trx, next, start, chain, trace, return_failure_trace, disable_subjective_enforcement, first_auth, sub_bill, prev_billed_cpu_time_us);
@@ -2310,7 +2249,7 @@ producer_plugin_impl::push_transaction( const fc::time_point& block_deadline,
 
 producer_plugin_impl::push_result
 producer_plugin_impl::handle_push_result( const transaction_metadata_ptr& trx,
-                                          const next_function<transaction_trace_ptr>& next,
+                                          next_function<transaction_trace_ptr> next,
                                           const fc::time_point& start,
                                           const chain::controller& chain,
                                           const transaction_trace_ptr& trace,
@@ -2322,14 +2261,8 @@ producer_plugin_impl::handle_push_result( const transaction_metadata_ptr& trx,
    auto end = fc::time_point::now();
    push_result pr;
    if( trace->except ) {
-      // Transient trxs are dry-run or read-only.
-      // Dry-run trxs only run in write window. Read-only trxs can run in
-      // both write and read windows; time spent in read window is counted
-      // by read window summary.
-      if ( chain.is_write_window() ) {
-         auto dur = end - start;
-         _time_tracker.add_fail_time(dur, trx->is_transient());
-      }
+      if ( !trx->is_read_only() )
+         _time_tracker.add_fail_time(end - start);
       if( exception_is_exhausted( *trace->except ) ) {
          if( _pending_block_mode == pending_block_mode::producing ) {
             fc_dlog(_trx_failed_trace_log, "[TRX_TRACE] Block ${block_num} for producer ${prod} COULD NOT FIT, tx: ${txid} RETRYING ",
@@ -2369,14 +2302,8 @@ producer_plugin_impl::handle_push_result( const transaction_metadata_ptr& trx,
    } else {
       fc_tlog( _log, "Subjective bill for success ${a}: ${b} elapsed ${t}us, time ${r}us",
                ("a",first_auth)("b",sub_bill)("t",trace->elapsed)("r", end - start));
-      // Transient trxs are dry-run or read-only.
-      // Dry-run trxs only run in write window. Read-only trxs can run in
-      // both write and read windows; time spent in read window is counted
-      // by read window summary.
-      if ( chain.is_write_window() ) {
-         auto dur = end - start;
-         _time_tracker.add_success_time(dur, trx->is_transient());
-      }
+      if ( !trx->is_read_only() )
+         _time_tracker.add_success_time(end - start);
       log_trx_results( trx, trace, start );
       // if producing then trx is in objective cpu account billing
       if (!disable_subjective_enforcement && _pending_block_mode != pending_block_mode::producing) {
@@ -2392,14 +2319,12 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin
 {
    bool exhausted = false;
    if( !_unapplied_transactions.empty() ) {
-      const chain::controller& chain = chain_plug->chain();
-      const auto pending_block_num = chain.pending_block_num();
       int num_applied = 0, num_failed = 0, num_processed = 0;
       auto unapplied_trxs_size = _unapplied_transactions.size();
       auto itr     = _unapplied_transactions.unapplied_begin();
       auto end_itr = _unapplied_transactions.unapplied_end();
       while( itr != end_itr ) {
-         if( should_interrupt_start_block( deadline, pending_block_num ) ) {
+         if( should_interrupt_start_block( deadline ) ) {
             exhausted = true;
             break;
          }
@@ -2521,7 +2446,7 @@ void producer_plugin_impl::process_scheduled_and_incoming_trxs( const fc::time_p
          auto trace = chain.push_scheduled_transaction(trx_id, deadline, max_trx_time, 0, false);
          auto end = fc::time_point::now();
          if (trace->except) {
-            _time_tracker.add_fail_time(end - start, false); // delayed transaction cannot be transient
+            _time_tracker.add_fail_time(end - start);
             if (exception_is_exhausted(*trace->except)) {
                if( block_is_exhausted() ) {
                   exhausted = true;
@@ -2541,7 +2466,7 @@ void producer_plugin_impl::process_scheduled_and_incoming_trxs( const fc::time_p
                num_failed++;
             }
          } else {
-            _time_tracker.add_success_time(end - start, false); // delayed transaction cannot be transient
+            _time_tracker.add_success_time(end - start);
             fc_dlog(_trx_successful_trace_log,
                     "[TRX_TRACE] Block ${block_num} for producer ${prod} is ACCEPTING scheduled tx: ${txid}, time: ${r}, auth: ${a}, cpu: ${cpu}",
                     ("block_num", chain.head_block_num() + 1)("prod", get_pending_block_producer())
@@ -2574,10 +2499,8 @@ bool producer_plugin_impl::process_incoming_trxs( const fc::time_point& deadline
    if( itr != end ) {
       size_t processed = 0;
       fc_dlog( _log, "Processing ${n} pending transactions", ("n", _unapplied_transactions.incoming_size()) );
-      const chain::controller& chain = chain_plug->chain();
-      const auto pending_block_num = chain.pending_block_num();
       while( itr != end ) {
-         if ( should_interrupt_start_block( deadline, pending_block_num ) ) {
+         if ( should_interrupt_start_block( deadline ) ) {
             exhausted = true;
             break;
          }
@@ -2620,6 +2543,7 @@ bool producer_plugin_impl::block_is_exhausted() const {
 // -> Idle
 // --> Start block B (block time y.000) at time x.500
 void producer_plugin_impl::schedule_production_loop() {
+   _received_block = false;
    _timer.cancel();
 
    auto result = start_block();
@@ -2631,7 +2555,7 @@ void producer_plugin_impl::schedule_production_loop() {
       _timer.expires_from_now( boost::posix_time::microseconds( config::block_interval_us  / 10 ));
 
       // we failed to start a block, so try again later?
-      _timer.async_wait( app().executor().wrap( priority::high, exec_queue::read_write,
+      _timer.async_wait( app().executor().wrap( priority::high, exec_queue::general,
           [weak_this = weak_from_this(), cid = ++_timer_corelation_id]( const boost::system::error_code& ec ) {
              auto self = weak_this.lock();
              if( self && ec != boost::asio::error::operation_aborted && cid == self->_timer_corelation_id ) {
@@ -2684,7 +2608,7 @@ void producer_plugin_impl::schedule_maybe_produce_block( bool exhausted ) {
                ("num", chain.head_block_num() + 1)("desc", block_is_exhausted() ? "Exhausted" : "Deadline exceeded") );
    }
 
-   _timer.async_wait( app().executor().wrap( priority::high, exec_queue::read_write,
+   _timer.async_wait( app().executor().wrap( priority::high, exec_queue::general,
          [&chain, weak_this = weak_from_this(), cid=++_timer_corelation_id](const boost::system::error_code& ec) {
             auto self = weak_this.lock();
             if( self && ec != boost::asio::error::operation_aborted && cid == self->_timer_corelation_id ) {
@@ -2726,7 +2650,7 @@ void producer_plugin_impl::schedule_delayed_production_loop(const std::weak_ptr<
       fc_dlog(_log, "Scheduling Speculative/Production Change at ${time}", ("time", wake_up_time));
       static const boost::posix_time::ptime epoch(boost::gregorian::date(1970, 1, 1));
       _timer.expires_at(epoch + boost::posix_time::microseconds(wake_up_time->time_since_epoch().count()));
-      _timer.async_wait( app().executor().wrap( priority::high, exec_queue::read_write,
+      _timer.async_wait( app().executor().wrap( priority::high, exec_queue::general,
          [weak_this,cid=++_timer_corelation_id](const boost::system::error_code& ec) {
             auto self = weak_this.lock();
             if( self && ec != boost::asio::error::operation_aborted && cid == self->_timer_corelation_id ) {
@@ -2827,8 +2751,8 @@ void producer_plugin_impl::produce_block() {
         ("confs", new_bs->header.confirmed));
 }
 
-void producer_plugin::received_block(uint32_t block_num) {
-   my->_received_block = block_num;
+void producer_plugin::received_block() {
+   my->_received_block = true;
 }
 
 void producer_plugin::log_failed_transaction(const transaction_id_type& trx_id, const packed_transaction_ptr& packed_trx_ptr, const char* reason) const {
@@ -2840,45 +2764,34 @@ void producer_plugin::log_failed_transaction(const transaction_id_type& trx_id,
             ("entire_trx", packed_trx_ptr ? my->chain_plug->get_log_trx(packed_trx_ptr->get_transaction()) : fc::variant{trx_id}));
 }
 
-// Called from only one read_only thread
+// Called from app thread
 void producer_plugin_impl::switch_to_write_window() {
-   if ( _log.is_enabled( fc::log_level::debug ) ) {
-      auto now = fc::time_point::now();
-      fc_dlog( _log, "Read-only threads ${n}, read window ${r}us, total all threads ${t}us",
-               ("n", _ro_thread_pool_size)
-               ("r", now - _ro_read_window_start_time)
-               ("t", _ro_all_threads_exec_time_us.load()));
-   }
-
-   chain::controller& chain = chain_plug->chain();
-
    // this method can be called from multiple places. it is possible
    // we are already in write window.
-   if ( chain.is_write_window() ) {
+   if ( app().executor().is_write_window() ) {
       return;
    }
 
-   EOS_ASSERT(_ro_num_active_exec_tasks.load() == 0 && _ro_exec_tasks_fut.empty(), producer_exception, "no read-only tasks should be running before switching to write window");
+   EOS_ASSERT(_ro_num_active_trx_exec_tasks.load() == 0 && _ro_trx_exec_tasks_fut.empty(), producer_exception, "no read-only tasks should be running before switching to write window");
+   _ro_read_window_timer.cancel();
+   _ro_write_window_timer.cancel();
 
    start_write_window();
 }
 
-// Called from app thread on plugin_startup
-// Called from only one read_only thread & called from app thread, but not concurrently
+// Called from app thread
 void producer_plugin_impl::start_write_window() {
    chain::controller& chain = chain_plug->chain();
 
    app().executor().set_to_write_window();
-   chain.set_to_write_window();
    chain.unset_db_read_only_mode();
-   _idle_trx_time = _ro_window_deadline = fc::time_point::now();
+   _idle_trx_time = fc::time_point::now();
 
-   _ro_window_deadline += _ro_write_window_time_us; // not allowed on block producers, so no need to limit to block deadline
    auto expire_time = boost::posix_time::microseconds(_ro_write_window_time_us.count());
-   _ro_timer.expires_from_now( expire_time );
-   _ro_timer.async_wait( app().executor().wrap(  // stay on app thread
+   _ro_write_window_timer.expires_from_now( expire_time );
+   _ro_write_window_timer.async_wait( app().executor().wrap(  // stay on app thread
       priority::high,
-      exec_queue::read_write, // placed in read_write so only called from main thread
+      exec_queue::read_only_trx_safe, // placed in read_only_trx_safe queue so it is ensured to be executed in either window
       [weak_this = weak_from_this()]( const boost::system::error_code& ec ) {
          auto self = weak_this.lock();
          if( self && ec != boost::asio::error::operation_aborted ) {
@@ -2887,154 +2800,144 @@ void producer_plugin_impl::start_write_window() {
       }));
 }
 
-// Called only from app thread
+// Called from app thread
 void producer_plugin_impl::switch_to_read_window() {
-   chain::controller& chain = chain_plug->chain();
-   EOS_ASSERT(chain.is_write_window(),  producer_exception, "expected to be in write window");
-   EOS_ASSERT( _ro_num_active_exec_tasks.load() == 0 && _ro_exec_tasks_fut.empty(), producer_exception, "_ro_exec_tasks_fut expected to be empty" );
+   EOS_ASSERT(app().executor().is_write_window(),  producer_exception, "expected to be in write window");
+   EOS_ASSERT(_ro_num_active_trx_exec_tasks.load() == 0 && _ro_trx_exec_tasks_fut.empty(),  producer_exception, "_ro_trx_exec_tasks_fut expected to be empty" );
 
+   _ro_write_window_timer.cancel();
+   _ro_read_window_timer.cancel();
    _time_tracker.add_idle_time( fc::time_point::now() - _idle_trx_time );
 
    // we are in write window, so no read-only trx threads are processing transactions.
-   if ( app().executor().read_only_queue().empty() ) { // no read-only tasks to process. stay in write window
+   // _ro_trx_queue is not being accessed. No need to lock.
+   if ( _ro_trx_queue.queue.empty() ) { // no read-only trxs to process. stay in write window
       start_write_window(); // restart write window timer for next round
       return;
    }
 
-   uint32_t pending_block_num = chain.head_block_num() + 1;
-   _ro_read_window_start_time = fc::time_point::now();
-   _ro_window_deadline = _ro_read_window_start_time + _ro_read_window_effective_time_us;
-   app().executor().set_to_read_window(_ro_thread_pool_size,
-      [received_block=&_received_block, pending_block_num, ro_window_deadline=_ro_window_deadline]() {
-         return fc::time_point::now() >= ro_window_deadline || (received_block->load() >= pending_block_num); // should_exit()
-      });
-   chain.set_to_read_window();
-   chain.set_db_read_only_mode();
-   _ro_all_threads_exec_time_us = 0;
+   app().executor().set_to_read_window();
+   chain_plug->chain().set_db_read_only_mode();
+   _received_block = false;
 
-   // start a read-only execution task in each thread in the thread pool
-   _ro_num_active_exec_tasks = _ro_thread_pool_size;
-   _ro_exec_tasks_fut.resize(0);
-   for (uint32_t i = 0; i < _ro_thread_pool_size; ++i ) {
-      _ro_exec_tasks_fut.emplace_back( post_async_task( _ro_thread_pool.get_executor(), [self = this, pending_block_num] () {
-         return self->read_only_execution_task(pending_block_num);
+   // start a read-only transaction execution task in each thread in the thread pool
+   _ro_num_active_trx_exec_tasks = _ro_thread_pool_size;
+   for (auto i = 0; i < _ro_thread_pool_size; ++i ) {
+      _ro_trx_exec_tasks_fut.emplace_back( post_async_task( _ro_thread_pool.get_executor(), [self = this] () {
+         return self->read_only_trx_execution_task();
       }) );
    }
 
    auto expire_time = boost::posix_time::microseconds(_ro_read_window_time_us.count());
-   _ro_timer.expires_from_now( expire_time );
-   // Needs to be on read_only because that is what is being processed until switch_to_write_window().
-   _ro_timer.async_wait( app().executor().wrap(
+   _ro_read_window_timer.expires_from_now( expire_time );
+   _ro_read_window_timer.async_wait( app().executor().wrap(  // stay on app thread
       priority::high,
-      exec_queue::read_only,
+      exec_queue::read_only_trx_safe,
       [weak_this = weak_from_this()]( const boost::system::error_code& ec ) {
          auto self = weak_this.lock();
          if( self && ec != boost::asio::error::operation_aborted ) {
             // use future to make sure all read-only tasks finished before switching to write window
-            for ( auto& task: self->_ro_exec_tasks_fut ) {
+            for ( auto& task: self->_ro_trx_exec_tasks_fut ) {
                task.get();
             }
-            self->_ro_exec_tasks_fut.clear();
-            // will be executed from the main app thread because all read-only threads are idle now
+            self->_ro_trx_exec_tasks_fut.clear();
             self->switch_to_write_window();
           } else if ( self ) {
-             self->_ro_exec_tasks_fut.clear();
+             self->_ro_trx_exec_tasks_fut.clear();
           }
        }));
 }
 
-// Called from a read only thread. Run in parallel with app and other read only threads
-bool producer_plugin_impl::read_only_execution_task(uint32_t pending_block_num) {
-   // We have 3 ways to break out the while loop:
+// Called from a read only trx thread. Run in parallel with app and other read only trx threads
+bool producer_plugin_impl::read_only_trx_execution_task() {
+   auto start = fc::time_point::now();
+   auto read_window_deadline = start + _ro_read_window_effective_time_us;
+   // We have 4 ways to break out the while loop:
    // 1. pass read window deadline
-   // 2. net_plugin receives a block
-   // 3. no read-only tasks to execute
-   while ( fc::time_point::now() < _ro_window_deadline && _received_block < pending_block_num ) {
-      bool more = app().executor().execute_highest_read_only(); // blocks until all read only threads are idle
-      if ( !more ) {
+   // 2. Net_plugin receives a block
+   // 3. No more transactions in the read-only trx queue
+   // 4. A transaction execution is exhaused
+   while ( fc::time_point::now() < read_window_deadline && !_received_block ) {
+      std::unique_lock<std::mutex> lck( _ro_trx_queue.mtx );
+      if ( _ro_trx_queue.queue.empty() ) {
+         break;
+      }
+      auto trx = _ro_trx_queue.queue.front();
+      _ro_trx_queue.queue.pop_front();
+      lck.unlock();
+      
+      auto retry = process_read_only_transaction( trx.trx, trx.next, start );
+      if ( retry ) {
+         lck.lock();
+         _ro_trx_queue.queue.push_front(trx);
+         // Do not schedule new execution
          break;
       }
    }
 
    // If all tasks are finished, do not wait until end of read window; switch to write window now.
-   if ( --_ro_num_active_exec_tasks == 0 ) {
-      // Needs to be on read_only because that is what is being processed until switch_to_write_window().
-      app().executor().post( priority::high, exec_queue::read_only, [self=this]() {
-         self->_ro_exec_tasks_fut.clear();
-         // will be executed from the main app thread because all read-only threads are idle now
+   if ( --_ro_num_active_trx_exec_tasks == 0 ) {
+      // Do switching on app thread to serialize
+      app().executor().post( priority::high, exec_queue::read_only_trx_safe, [self=this]() {
+         self->_ro_trx_exec_tasks_fut.clear();
          self->switch_to_write_window();
       } );
-      // last thread post any exhausted back into read_only queue with slightly higher priority (low+1) so they are executed first
-      ro_trx_t t;
-      while( _ro_exhausted_trx_queue.pop_front(t) ) {
-         app().executor().post(priority::low+1, exec_queue::read_only, [this, trx{std::move(t.trx)}, next{std::move(t.next)}]() mutable {
-            push_read_only_transaction( std::move(trx), std::move(next) );
-         } );
-      }
    }
 
    return true;
 }
 
-// Called from app thread during start block.
-// Reschedule any exhausted read-only transactions from the last block
-void producer_plugin_impl::repost_exhausted_transactions(const fc::time_point& deadline) {
-   if ( !_ro_exhausted_trx_queue.empty() ) {
-      chain::controller& chain = chain_plug->chain();
-      uint32_t pending_block_num = chain.pending_block_num();
-      // post any exhausted back into read_only queue with slightly higher priority (low+1) so they are executed first
-      ro_trx_t t;
-      while( !should_interrupt_start_block( deadline, pending_block_num ) && _ro_exhausted_trx_queue.pop_front(t) ) {
-         app().executor().post(priority::low+1, exec_queue::read_only, [this, trx{std::move(t.trx)}, next{std::move(t.next)}]() mutable {
-            push_read_only_transaction( std::move(trx), std::move(next) );
-         } );
-      }
-   }
+// Called from a read only trx thread. Run in parallel with app and other read only trx threads
+// Return whether the trx needs to be retried in next read window
+bool producer_plugin_impl::process_read_only_transaction(const packed_transaction_ptr& trx, const next_function<transaction_trace_ptr>& next, const fc::time_point& read_window_start_time) {
+   chain::controller& chain = chain_plug->chain();
+   auto future = transaction_metadata::start_recover_keys( trx, _thread_pool.get_executor(),
+                                                           chain.get_chain_id(), fc::microseconds::maximum(),
+                                                           transaction_metadata::trx_type::read_only,
+                                                           chain.configured_subjective_signature_length_limit() );
+   auto exception_handler = [&next](fc::exception_ptr ex) {
+      next( std::move(ex) );
+      return false;
+   };
+   future.wait();
+   try {
+      auto trx_metadata = future.get();
+      return push_read_only_transaction( trx_metadata, next, read_window_start_time );
+   } CATCH_AND_CALL(exception_handler);
+   return false;
 }
 
-// Called from a read_only_trx execution thread, or from app thread when executing exclusively
+// Called from a read_only_trx execution thread
 // Return whether the trx needs to be retried in next read window
-bool producer_plugin_impl::push_read_only_transaction(transaction_metadata_ptr trx, next_function<transaction_trace_ptr> next) {
+bool producer_plugin_impl::push_read_only_transaction(
+            const transaction_metadata_ptr& trx,
+            next_function<transaction_trace_ptr> next,
+            const fc::time_point& read_window_start_time) {
    auto retry = false;
+   chain::controller& chain = chain_plug->chain();
+
+   if( !chain.is_building_block() ) {
+      // try next round
+      return true;
+   }
 
    try {
+      const auto block_deadline = calculate_block_deadline( chain.pending_block_time() );
       auto start = fc::time_point::now();
-      chain::controller& chain = chain_plug->chain();
-      if ( !chain.is_building_block() ) {
-         _ro_exhausted_trx_queue.push_front( {std::move(trx), std::move(next)} );
+      auto time_used_in_read_window_us = ( start - read_window_start_time );
+      if ( time_used_in_read_window_us >= _ro_read_window_time_us ) {
+         // already passed read-window deadline, try next round
          return true;
       }
 
-      // When executing a read-only trx on the main thread while in the write window,
-      // need to switch db mode to read only.
-      auto db_read_only_mode_guard = fc::make_scoped_exit([&]{
-         if( chain.is_write_window() )
-            chain.unset_db_read_only_mode();
-      });
-
-      if ( chain.is_write_window() ) {
-         chain.set_db_read_only_mode();
-         auto idle_time = fc::time_point::now() - _idle_trx_time;
-         _time_tracker.add_idle_time( idle_time );
-      }
-
-      // use read-window/write-window deadline if there are read/write windows, otherwise use block_deadline if only the app thead
-      auto window_deadline = (_ro_thread_pool_size != 0) ? _ro_window_deadline : calculate_block_deadline( chain.pending_block_time() );
-
-      // Ensure the trx to finish by the end of read-window or write-window or block_deadline depending on
+      auto remaining_time_in_read_window_us = _ro_read_window_time_us - time_used_in_read_window_us;
+      // Ensure the trx to finish by the end of read-window.
+      auto window_deadline = std::min( start + remaining_time_in_read_window_us, block_deadline );
       auto trace = chain.push_transaction( trx, window_deadline, _ro_max_trx_time_us, 0, false, 0 );
-      _ro_all_threads_exec_time_us += (fc::time_point::now() - start).count();
       auto pr = handle_push_result(trx, next, start, chain, trace, true /*return_failure_trace*/, true /*disable_subjective_enforcement*/, {} /*first_auth*/, 0 /*sub_bill*/, 0 /*prev_billed_cpu_time_us*/);
       // If a transaction was exhausted, that indicates we are close to
       // the end of read window. Retry in next round.
       retry = pr.trx_exhausted;
-      if( retry ) {
-         _ro_exhausted_trx_queue.push_front( {std::move(trx), std::move(next)} );
-      }
-
-      if ( chain.is_write_window() ) {
-         _idle_trx_time = fc::time_point::now();
-      }
    } catch ( const guard_exception& e ) {
       chain_plugin::handle_guard_exception(e);
    } catch ( boost::interprocess::bad_alloc& ) {

plugins/producer_plugin/test/test_read_only_trx.cpp

@@ -79,7 +79,6 @@ void test_trxs_common(std::vector<const char*>& specific_args) {
    appbase::scoped_app app;
    fc::temp_directory temp;
    auto temp_dir_str = temp.path().string();
-   producer_plugin::set_test_mode(true);
    
    std::promise<std::tuple<producer_plugin*, chain_plugin*>> plugin_promise;
    std::future<std::tuple<producer_plugin*, chain_plugin*>> plugin_fut = plugin_promise.get_future();
@@ -97,19 +96,14 @@ void test_trxs_common(std::vector<const char*>& specific_args) {
    auto chain_id = chain_plug->get_chain_id();
 
    std::atomic<size_t> next_calls = 0;
-   std::atomic<size_t> num_get_account_calls = 0;
    std::atomic<size_t> num_posts = 0;
    std::atomic<size_t> trace_with_except = 0;
    std::atomic<bool> trx_match = true;
-   const size_t num_pushes = 4242;
+   const size_t num_pushes = 2558;
 
    for( size_t i = 1; i <= num_pushes; ++i ) {
       auto ptrx = make_unique_trx( chain_id );
-      app->executor().post( priority::low, exec_queue::read_only, [&chain_plug=chain_plug, &num_get_account_calls]() {
-         chain_plug->get_read_only_api(fc::seconds(90)).get_account(chain_apis::read_only::get_account_params{.account_name=config::system_account_name}, fc::time_point::now()+fc::seconds(90));
-         ++num_get_account_calls;
-      });
-      app->executor().post( priority::low, exec_queue::read_write, [ptrx, &next_calls, &num_posts, &trace_with_except, &trx_match, &app]() {
+      app->executor().post( priority::low, exec_queue::general, [ptrx, &next_calls, &num_posts, &trace_with_except, &trx_match, &app]() {
          ++num_posts;
          bool return_failure_traces = true;
          app->get_method<plugin_interface::incoming::methods::transaction_async>()(ptrx,
@@ -132,15 +126,12 @@ void test_trxs_common(std::vector<const char*>& specific_args) {
                ++next_calls;
         });
       });
-      app->executor().post( priority::low, exec_queue::read_only, [&chain_plug=chain_plug]() {
-         chain_plug->get_read_only_api(fc::seconds(90)).get_consensus_parameters(chain_apis::read_only::get_consensus_parameters_params{}, fc::time_point::now()+fc::seconds(90));
-      });
    }
 
    // Wait long enough such that all transactions are executed
    auto start = fc::time_point::now();
    auto hard_deadline = start + fc::seconds(10); // To protect against waiting forever
-   while ( (next_calls < num_pushes || num_get_account_calls < num_pushes) && fc::time_point::now() < hard_deadline ){
+   while ( next_calls < num_pushes && fc::time_point::now() < hard_deadline ){
       std::this_thread::sleep_for( 100ms );;
    }
 
@@ -150,39 +141,13 @@ void test_trxs_common(std::vector<const char*>& specific_args) {
    BOOST_CHECK_EQUAL( trace_with_except, 0 ); // should not have any traces with except in it
    BOOST_CHECK_EQUAL( num_pushes, num_posts );
    BOOST_CHECK_EQUAL( num_pushes, next_calls.load() );
-   BOOST_CHECK_EQUAL( num_pushes, num_get_account_calls.load() );
-   BOOST_CHECK( trx_match.load() );  // trace should match the transaction
+   BOOST_CHECK( trx_match.load() );  // trace should match the transaction  
 }
 
 // test read-only trxs on main thread (no --read-only-threads)
 BOOST_AUTO_TEST_CASE(no_read_only_threads) {
-   std::vector<const char*> specific_args = { "-p", "eosio", "-e", "--abi-serializer-max-time-ms=999" };
+   std::vector<const char*> specific_args = { "-p", "eosio", "-e" };
    test_trxs_common(specific_args);
 }
 
-// test read-only trxs on 1 threads (with --read-only-threads)
-BOOST_AUTO_TEST_CASE(with_1_read_only_threads) {
-   std::vector<const char*> specific_args = { "-p", "eosio", "-e",
-                                              "--read-only-threads=1",
-                                              "--max-transaction-time=10",
-                                              "--abi-serializer-max-time-ms=999",
-                                              "--read-only-write-window-time-us=100000",
-                                              "--read-only-read-window-time-us=40000",
-                                              "--disable-subjective-billing=true" };
-   test_trxs_common(specific_args);
-}
-
-// test read-only trxs on 8 separate threads (with --read-only-threads)
-BOOST_AUTO_TEST_CASE(with_8_read_only_threads) {
-   std::vector<const char*> specific_args = { "-p", "eosio", "-e",
-                                              "--read-only-threads=8",
-                                              "--max-transaction-time=10",
-                                              "--abi-serializer-max-time-ms=999",
-                                              "--read-only-write-window-time-us=100000",
-                                              "--read-only-read-window-time-us=40000",
-                                              "--disable-subjective-billing=true" };
-   test_trxs_common(specific_args);
-}
-
-
 BOOST_AUTO_TEST_SUITE_END()

plugins/producer_plugin/test/test_snapshot_scheduler.cpp

@@ -67,7 +67,7 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
             fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::debug);
             std::vector<const char*> argv =
                   {"test", "--data-dir", temp.c_str(), "--config-dir", temp.c_str(),
-                   "-p", "eosio", "-e", "--disable-subjective-billing=true"};
+                   "-p", "eosio", "-e", "--max-transaction-time", "475", "--disable-subjective-billing=true"};
             appbase::app().initialize<chain_plugin, producer_plugin>(argv.size(), (char**) &argv[0]);
             appbase::app().startup();
             plugin_promise.set_value(
@@ -93,20 +93,18 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
          auto bs = chain_plug->chain().block_start.connect([&pp](uint32_t bn) {
             // catching pending snapshot
             if (!pp->get_snapshot_requests().snapshot_requests.empty()) {
-               const auto& snapshot_requests = pp->get_snapshot_requests().snapshot_requests;
-               auto it = find_if(snapshot_requests.begin(), snapshot_requests.end(), [](const producer_plugin::snapshot_schedule_information& obj) {return obj.snapshot_request_id == 0;});
-               // we should have a pending snapshot for request id = 0
-               BOOST_REQUIRE(it != snapshot_requests.end());
-               auto& pending = it->pending_snapshots;
-               if (pending.size()==1) {
-                  BOOST_CHECK_EQUAL(9, pending.begin()->head_block_num);   
+               auto& pending = pp->get_snapshot_requests().snapshot_requests.begin()->pending_snapshots;
+               if (pending && pending->size()==1) {
+                  // lets check the head block num of it, it should be 8 + 1 = 9
+                  // this means we are getting a snapshot for correct block # as well
+                  BOOST_CHECK_EQUAL(9, pending->begin()->head_block_num);   
                }
             }
          });
        
-         producer_plugin::snapshot_request_information sri1 = {.block_spacing = 8, .start_block_num = 1, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 1"};
-         producer_plugin::snapshot_request_information sri2 = {.block_spacing = 5000, .start_block_num = 100000, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 2 that will never happen"};
-         producer_plugin::snapshot_request_information sri3 = {.block_spacing = 2, .start_block_num = 0, .end_block_num = 3, .snapshot_description = "Example of recurring snapshot 3 that will expire"};
+         producer_plugin::snapshot_request_information sri1 = {.block_spacing = 8, .start_block_num = 1, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 2"};
+         producer_plugin::snapshot_request_information sri2 = {.block_spacing = 5000, .start_block_num = 100000, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 2"};
+         producer_plugin::snapshot_request_information sri3 = {.block_spacing = 2, .start_block_num = 0, .end_block_num = 3, .snapshot_description = "Example of recurring snapshot 1"};
 
          pp->schedule_snapshot(sri1);
          pp->schedule_snapshot(sri2);
@@ -120,13 +118,8 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
          // one of the snapshots is done here and request, corresponding to it should be deleted
          BOOST_CHECK_EQUAL(2, pp->get_snapshot_requests().snapshot_requests.size());
 
-         // check whether no pending snapshots present for a snapshot with id 0
-         const auto& snapshot_requests = pp->get_snapshot_requests().snapshot_requests;
-         auto it = find_if(snapshot_requests.begin(), snapshot_requests.end(),[](const producer_plugin::snapshot_schedule_information& obj) {return obj.snapshot_request_id == 0;});
-
-         // snapshot request with id = 0 should be found and should not have any pending snapshots
-         BOOST_REQUIRE(it != snapshot_requests.end());
-         BOOST_CHECK(!it->pending_snapshots.size());
+         // check whether no pending snapshots present
+         BOOST_CHECK_EQUAL(0, pp->get_snapshot_requests().snapshot_requests.begin()->pending_snapshots->size());
 
          // quit app
          appbase::app().quit();

plugins/producer_plugin/test/test_trx_full.cpp

@@ -112,7 +112,7 @@ BOOST_AUTO_TEST_CASE(producer) {
          fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::debug);
          std::vector<const char*> argv =
                {"test", "--data-dir", temp_dir_str.c_str(), "--config-dir", temp_dir_str.c_str(),
-                "-p", "eosio", "-e", "--disable-subjective-billing=true" };
+                "-p", "eosio", "-e", "--max-transaction-time", "475", "--disable-subjective-billing=true" };
          app->initialize<chain_plugin, producer_plugin>( argv.size(), (char**) &argv[0] );
          app->startup();
          plugin_promise.set_value(

plugins/prometheus_plugin/include/eosio/prometheus_plugin/simple_rest_server.hpp

@@ -1,234 +0,0 @@
-#pragma once
-
-#include <boost/beast/core.hpp>
-#include <boost/beast/http.hpp>
-#include <boost/beast/version.hpp>
-
-namespace eosio { namespace rest {
-
-   // The majority of the code here are derived from boost source
-   // libs/beast/example/http/server/async/http_server_async.cpp
-   // with minimum modification and yet reusable.
-
-   namespace beast = boost::beast;         // from <boost/beast.hpp>
-   namespace http  = beast::http;          // from <boost/beast/http.hpp>
-   namespace net   = boost::asio;          // from <boost/asio.hpp>
-   using tcp       = boost::asio::ip::tcp; // from <boost/asio/ip/tcp.hpp>
-   template <typename T>
-   class simple_server {
-      T* self() { return static_cast<T*>(this); }
-
-      void fail(beast::error_code ec, char const* what) { self()->log_error(what, ec.message()); }
-      // Return a response for the given request.
-      http::response<http::string_body> handle_request(http::request<http::string_body>&& req) {
-         auto server_header = self()->server_header();
-         // Returns a bad request response
-         auto const bad_request = [&req, &server_header](std::string_view why) {
-            http::response<http::string_body> res{ http::status::bad_request, req.version() };
-            res.set(http::field::server, server_header);
-            res.set(http::field::content_type, "text/plain");
-            res.keep_alive(req.keep_alive());
-            res.body() = std::string(why);
-            res.prepare_payload();
-            return res;
-         };
-
-         // Returns a not found response
-         auto const not_found = [&req, &server_header](std::string_view target) {
-            http::response<http::string_body> res{ http::status::not_found, req.version() };
-            res.set(http::field::server, server_header);
-            res.set(http::field::content_type, "text/plain");
-            res.keep_alive(req.keep_alive());
-            res.body() = "The resource '" + std::string(target) + "' was not found.";
-            res.prepare_payload();
-            return res;
-         };
-
-         // Returns a server error response
-         auto const server_error = [&req, &server_header](std::string_view what) {
-            http::response<http::string_body> res{ http::status::internal_server_error, req.version() };
-            res.set(http::field::server, server_header);
-            res.set(http::field::content_type, "text/plain");
-            res.keep_alive(req.keep_alive());
-            res.body() = "An error occurred: '" + std::string(what) + "'";
-            res.prepare_payload();
-            return res;
-         };
-
-         // Make sure we can handle the method
-         if (!self()->allow_method(req.method()))
-            return bad_request("Unknown HTTP-method");
-
-         // Request path must be absolute and not contain "..".
-         std::string_view target{req.target().data(), req.target().size()};
-         if (target.empty() || target[0] != '/' || target.find("..") != std::string_view::npos)
-            return bad_request("Illegal request-target");
-
-         try {
-            auto res = self()->on_request(std::move(req));
-            if (!res)
-               not_found(target);
-            return *res;
-         } catch (std::exception& ex) { return server_error(ex.what()); }
-      }
-
-      class session : public std::enable_shared_from_this<session> {
-         tcp::socket                                        socket_;
-         boost::asio::io_context::strand                    strand_;
-         beast::flat_buffer                                 buffer_;
-         http::request<http::string_body>                   req_;
-         simple_server*                                     server_;
-         std::shared_ptr<http::response<http::string_body>> res_;
-
-       public:
-         // Take ownership of the stream
-         session(net::io_context& ioc, tcp::socket&& socket, simple_server* server)
-             : socket_(std::move(socket)), strand_(ioc), server_(server) {}
-
-         // Start the asynchronous operation
-         void run() { do_read(); }
-
-         void do_read() {
-            // Make the request empty before reading,
-            // otherwise the operation behavior is undefined.
-            req_ = {};
-
-            // Read a request
-            http::async_read(
-                  socket_, buffer_, req_,
-                  boost::asio::bind_executor(strand_, [self = this->shared_from_this()](beast::error_code ec,
-                                                                                        std::size_t bytes_transferred) {
-                     self->on_read(ec, bytes_transferred);
-                  }));
-         }
-
-         void on_read(beast::error_code ec, std::size_t bytes_transferred) {
-            boost::ignore_unused(bytes_transferred);
-
-            // This means they closed the connection
-            if (ec == http::error::end_of_stream)
-               return do_close();
-
-            if (ec)
-               return server_->fail(ec, "read");
-
-            // Send the response
-            send_response(server_->handle_request(std::move(req_)));
-         }
-
-         void send_response(http::response<http::string_body>&& msg) {
-            // The lifetime of the message has to extend
-            // for the duration of the async operation so
-            // we use a shared_ptr to manage it.
-            res_ = std::make_shared<http::response<http::string_body>>(std::move(msg));
-
-            // Write the response
-            http::async_write(socket_, *res_,
-                              boost::asio::bind_executor(socket_.get_executor(),
-                                                         [self = this->shared_from_this(), close = res_->need_eof()](
-                                                               beast::error_code ec, std::size_t bytes_transferred) {
-                                                            self->on_write(ec, bytes_transferred, close);
-                                                         }));
-         }
-
-         void on_write(boost::system::error_code ec, std::size_t bytes_transferred, bool close) {
-            boost::ignore_unused(bytes_transferred);
-
-            if (ec)
-               return server_->fail(ec, "write");
-
-            if (close) {
-               // This means we should close the connection, usually because
-               // the response indicated the "Connection: close" semantic.
-               return do_close();
-            }
-
-            // We're done with the response so delete it
-            res_ = nullptr;
-
-            // Read another request
-            do_read();
-         }
-
-         void do_close() {
-            // Send a TCP shutdown
-            beast::error_code ec;
-            socket_.shutdown(tcp::socket::shutdown_send, ec);
-
-            // At this point the connection is closed gracefully
-         }
-      };
-
-      //------------------------------------------------------------------------------
-
-      // Accepts incoming connections and launches the sessions
-      class listener : public std::enable_shared_from_this<listener> {
-         net::io_context& ioc_;
-         tcp::acceptor    acceptor_;
-         tcp::socket      socket_;
-         simple_server*   server_;
-
-       public:
-         listener(net::io_context& ioc, tcp::endpoint endpoint, simple_server* server)
-             : ioc_(ioc), acceptor_(ioc), socket_(ioc), server_(server) {
-            boost::system::error_code ec;
-
-            // Open the acceptor
-            acceptor_.open(endpoint.protocol(), ec);
-            if (ec) {
-               server_->fail(ec, "open");
-               return;
-            }
-
-            // Allow address reuse
-            acceptor_.set_option(net::socket_base::reuse_address(true), ec);
-            if (ec) {
-               server_->fail(ec, "set_option");
-               return;
-            }
-
-            // Bind to the server address
-            acceptor_.bind(endpoint, ec);
-            if (ec) {
-               server_->fail(ec, "bind");
-               return;
-            }
-
-            // Start listening for connections
-            acceptor_.listen(net::socket_base::max_listen_connections, ec);
-            if (ec) {
-               server_->fail(ec, "listen");
-               return;
-            }
-         }
-
-         // Start accepting incoming connections
-         void run() {
-            if (!acceptor_.is_open())
-               return;
-            do_accept();
-         }
-
-       private:
-         void do_accept() {
-            acceptor_.async_accept(
-                  socket_, [self = this->shared_from_this()](boost::system::error_code ec) { self->on_accept(ec); });
-         }
-
-         void on_accept(boost::system::error_code ec) {
-            if (ec) {
-               server_->fail(ec, "accept");
-            } else {
-               // Create the session and run it
-               std::make_shared<session>(ioc_, std::move(socket_), server_)->run();
-            }
-
-            // Accept another connection
-            do_accept();
-         }
-      };
-
-    public:
-      void run(net::io_context& ioc, tcp::endpoint endpoint) { std::make_shared<listener>(ioc, endpoint, this)->run(); }
-   };
-}} // namespace eosio::rest

plugins/prometheus_plugin/prometheus_plugin.cpp

@@ -1,11 +1,8 @@
 #include <eosio/prometheus_plugin/prometheus_plugin.hpp>
-#include <eosio/prometheus_plugin/simple_rest_server.hpp>
-
 #include <eosio/chain/plugin_interface.hpp>
 #include <eosio/chain/thread_utils.hpp>
 #include <eosio/http_plugin/macros.hpp>
 #include <eosio/net_plugin/net_plugin.hpp>
-#include <eosio/http_plugin/http_plugin.hpp>
 #include <eosio/producer_plugin/producer_plugin.hpp>
 
 #include <fc/log/logger.hpp>
@@ -17,8 +14,8 @@
 #include <prometheus/text_serializer.h>
 #include <prometheus/registry.h>
 
+
 namespace eosio {
-   static const char* prometheus_api_name = "/v1/prometheus/metrics";
    using namespace prometheus;
    using namespace chain::plugin_interface;
 
@@ -113,41 +110,12 @@ namespace eosio {
       }
    };
 
-   namespace http = boost::beast::http;
-   struct prometheus_plugin_impl : rest::simple_server<prometheus_plugin_impl> {
-
-      std::string server_header() const {
-         return http_plugin::get_server_header();
-      }
-
-      void log_error(char const* what, const std::string& message) {
-         elog("${what}: ${message}", ("what", what)("message", message));
-      }
-
-      bool allow_method(http::verb method) const {
-         return method == http::verb::get;
-      }
-
-      std::optional<http::response<http::string_body>>
-       on_request(http::request<http::string_body>&& req) {
-         if(req.target() != prometheus_api_name)
-            return {};
-         http::response<http::string_body> res{ http::status::ok, req.version() };
-         // Respond to GET request
-         res.set(http::field::server, server_header());
-         res.set(http::field::content_type, "text/plain");
-         res.keep_alive(req.keep_alive());
-         res.body() = metrics();
-         res.prepare_payload();
-         return res;
-      }
-
+   struct prometheus_plugin_impl {
       eosio::chain::named_thread_pool<struct prom> _prometheus_thread_pool;
       boost::asio::io_context::strand _prometheus_strand;
       prometheus_plugin_metrics _metrics;
 
       map<std::string, vector<runtime_metric>> _plugin_metrics;
-      boost::asio::ip::tcp::endpoint           _endpoint;
 
       prometheus_plugin_impl(): _prometheus_strand(_prometheus_thread_pool.get_executor()){ }
 
@@ -211,11 +179,32 @@ namespace eosio {
          return body;
       }
 
-      void start() {
-         run(_prometheus_thread_pool.get_executor(), _endpoint);
-         _prometheus_thread_pool.start(
-               1, [](const fc::exception& e) { elog("Prometheus exception ${e}", ("e", e)); });
+      void metrics_async(chain::plugin_interface::next_function<std::string> results) {
+         _prometheus_strand.post([self=this, results=std::move(results)]() {
+            results(self->metrics());
+         });
       }
+
+   };
+
+   using metrics_params = fc::variant_object;
+
+   struct prometheus_api {
+      prometheus_plugin_impl& _pp;
+      fc::microseconds _max_response_time_us;
+
+      fc::time_point start() const {
+         return fc::time_point::now() + _max_response_time_us;
+      }
+
+      void metrics(const metrics_params&, chain::plugin_interface::next_function<std::string> results) {
+         _pp.metrics_async(std::move(results));
+      }
+
+      prometheus_api(prometheus_plugin_impl& plugin, const fc::microseconds& max_response_time)
+      : _pp(plugin)
+      , _max_response_time_us(max_response_time){}
+
    };
 
    prometheus_plugin::prometheus_plugin()
@@ -225,35 +214,24 @@ namespace eosio {
    prometheus_plugin::~prometheus_plugin() = default;
 
    void prometheus_plugin::set_program_options(options_description&, options_description& cfg) {
-      cfg.add_options()
-         ("prometheus-exporter-address", bpo::value<string>()->default_value("127.0.0.1:9101"),
-            "The local IP and port to listen for incoming prometheus metrics http request.");
    }
 
    void prometheus_plugin::plugin_initialize(const variables_map& options) {
       my->initialize_metrics();
 
-      string lipstr = options.at("prometheus-exporter-address").as<string>();
-      EOS_ASSERT(lipstr.size() > 0, chain::plugin_config_exception, "prometheus-exporter-address must have a value");
+      auto& _http_plugin = app().get_plugin<http_plugin>();
+      fc::microseconds max_response_time = _http_plugin.get_max_response_time();
 
-      string host = lipstr.substr(0, lipstr.find(':'));
-      string port = lipstr.substr(host.size() + 1, lipstr.size());
-
-      boost::system::error_code ec;
-      using tcp = boost::asio::ip::tcp;
-      tcp::resolver resolver(app().get_io_service());
-
-      my->_endpoint = *resolver.resolve(tcp::v4(), host, port, ec);
-      if (!ec) {
-         fc_ilog(logger(), "configured prometheus metrics exporter to listen on ${h}", ("h", lipstr));
-      } else {
-         fc_elog(logger(), "failed to configure prometheus metrics exporter to listen on ${h} (${m})",
-                 ("h", lipstr)("m", ec.message()));
-      }
+      prometheus_api handle(*my, max_response_time);
+      app().get_plugin<http_plugin>().add_async_api({
+        CALL_ASYNC_WITH_400(prometheus, handle, eosio, metrics, std::string, 200, http_params_types::no_params)}, http_content_type::plaintext);
    }
 
    void prometheus_plugin::plugin_startup() {
-      my->start();
+      my->_prometheus_thread_pool.start(1, []( const fc::exception& e ) {
+         elog("Prometheus excpetion ${e}:${l}", ("e", e));
+      } );
+
       ilog("Prometheus plugin started.");
    }
 

plugins/state_history_plugin/include/eosio/state_history_plugin/session.hpp

@@ -103,7 +103,7 @@ public:
    void pop_entry(bool call_send = true) {
       send_queue.erase(send_queue.begin());
       sending = false;
-      if (call_send || !send_queue.empty())
+      if (call_send)
          send();
    }
 
@@ -550,7 +550,6 @@ private:
 
       state_history::get_blocks_result_v0 result;
       result.head = {block_state->block_num, block_state->id};
-      to_send_block_num = std::min(block_state->block_num, to_send_block_num);
       send_update(std::move(result), block_state);
    }
 

plugins/test_control_api_plugin/test_control_api_plugin.cpp

@@ -51,7 +51,7 @@ void test_control_api_plugin::plugin_startup() {
 
    app().get_plugin<http_plugin>().add_api({
       TEST_CONTROL_RW_CALL(kill_node_on_producer, 202, http_params_types::params_required)
-   }, appbase::exec_queue::read_write);
+   }, appbase::exec_queue::general);
 }
 
 void test_control_api_plugin::plugin_shutdown() {}

plugins/wallet_api_plugin/wallet_api_plugin.cpp

@@ -115,7 +115,7 @@ void wallet_api_plugin::plugin_startup() {
             INVOKE_R_R_R(wallet_mgr, list_keys, std::string, std::string), 200),
        CALL_WITH_400(wallet, wallet_mgr, get_public_keys,
             INVOKE_R_V(wallet_mgr, get_public_keys), 200)
-   }, appbase::exec_queue::read_write);
+   }, appbase::exec_queue::general);
 }
 
 void wallet_api_plugin::plugin_initialize(const variables_map& options) {

programs/keosd/main.cpp

@@ -111,7 +111,7 @@ int main(int argc, char** argv)
                        [&a=app](string, string, url_response_callback cb) {
          cb(200, fc::time_point::maximum(), fc::variant(fc::variant_object()));
          a->quit();
-      }, appbase::exec_queue::read_write );
+      }, appbase::exec_queue::general );
       app->startup();
       app->exec();
    } catch (const fc::exception& e) {

scripts/CMakeLists.txt

@@ -6,4 +6,3 @@ configure_file(eosio-tn_up.sh eosio-tn_up.sh COPYONLY)
 configure_file(abi_is_json.py abi_is_json.py COPYONLY)
 configure_file(postinst .)
 configure_file(prerm .)
-configure_file(install_testharness_symlinks.cmake .)

scripts/install_testharness_symlinks.cmake

@@ -1,5 +0,0 @@
-execute_process( COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages  ERROR_QUIET RESULT_VARIABLE ret)
-if(ret EQUAL "0")
-    execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink ../../../share/leap_testing/tests/TestHarness ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages/TestHarness)
-    execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink ../../bin ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing/bin)
-endif()

tests/CMakeLists.txt

@@ -113,8 +113,8 @@ set_property(TEST ship_test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME ship_test_unix COMMAND tests/ship_test.py -v --num-clients 10 --num-requests 5000 --clean-run ${UNSHARE} --unix-socket WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST ship_test_unix PROPERTY LABELS nonparallelizable_tests)
 
-add_test(NAME ship_streamer_test COMMAND tests/ship_streamer_test.py -v --num-clients 10 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
-set_property(TEST ship_streamer_test PROPERTY LABELS long_running_tests)
+add_test(NAME ship_streamer_test COMMAND tests/ship_streamer_test.py -v --num-clients 10 --num-blocks 50 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+set_property(TEST ship_streamer_test PROPERTY LABELS nonparallelizable_tests)
 
 add_test(NAME p2p_dawn515_test COMMAND tests/p2p_tests/dawn_515/test.sh WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST p2p_dawn515_test PROPERTY LABELS nonparallelizable_tests)
@@ -125,11 +125,11 @@ add_test(NAME nodeos_protocol_feature_test COMMAND tests/nodeos_protocol_feature
 set_property(TEST nodeos_protocol_feature_test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME compute_transaction_test COMMAND tests/compute_transaction_test.py -v -p 2 -n 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST compute_transaction_test PROPERTY LABELS nonparallelizable_tests)
-add_test(NAME read-only-trx-basic-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --read-only-threads 0 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME read-only-trx-basic-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST read-only-trx-basic-test PROPERTY LABELS nonparallelizable_tests)
-add_test(NAME read-only-trx-parallel-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --read-only-threads 3 --num-test-runs 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME read-only-trx-parallel-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --read-only-threads 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST read-only-trx-parallel-test PROPERTY LABELS nonparallelizable_tests)
-add_test(NAME read-only-trx-parallel-eos-vm-oc-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --eos-vm-oc-enable --read-only-threads 3 --num-test-runs 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME read-only-trx-parallel-eos-vm-oc-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --eos-vm-oc-enable --read-only-threads 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST read-only-trx-parallel-eos-vm-oc-test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME subjective_billing_test COMMAND tests/subjective_billing_test.py -v -p 2 -n 4 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST subjective_billing_test PROPERTY LABELS nonparallelizable_tests)

tests/TestHarness/Cluster.py

@@ -22,11 +22,8 @@ from .Node import BlockType
 from .Node import Node
 from .WalletMgr import WalletMgr
 from .launch_transaction_generators import TransactionGeneratorsLauncher, TpsTrxGensConfig
-try:
-    from .libc import unshare, CLONE_NEWNET
-    from .interfaces import getInterfaceFlags, setInterfaceUp, IFF_LOOPBACK
-except:
-    pass
+from .libc import unshare, CLONE_NEWNET
+from .interfaces import getInterfaceFlags, setInterfaceUp, IFF_LOOPBACK
 
 # Protocol Feature Setup Policy
 class PFSetupPolicy:
@@ -1450,7 +1447,7 @@ class Cluster(object):
     def killall(self, kill=True, silent=True, allInstances=False):
         """Kill cluster nodeos instances. allInstances will kill all nodeos instances running on the system."""
         signalNum=9 if kill else 15
-        cmd="%s -k %d --nogen -p 1 -n 1 --nodeos-log-path %s" % (f"{sys.executable} {str(self.launcherPath)}", signalNum, self.nodeosLogPath)
+        cmd="%s -k %d --nogen -p 1 -n 1 --nodeos-log-path %s" % (f"python3 {str(self.launcherPath)}", signalNum, self.nodeosLogPath)
         if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
         if 0 != subprocess.call(cmd.split(), stdout=Utils.FNull):
             if not silent: Utils.Print("Launcher failed to shut down eos cluster.")

tests/TestHarness/Node.py

@@ -61,7 +61,7 @@ class Node(Transactions):
             self.fetchBlock = lambda blockNum: self.processUrllibRequest("trace_api", "get_block", {"block_num":blockNum}, silentErrors=False, exitOnError=True)
             self.fetchKeyCommand = lambda: "[transaction][transaction_header][ref_block_num]"
             self.fetchRefBlock = lambda trans: trans["transaction_header"]["ref_block_num"]
-            self.cleosLimit = "--time-limit 999"
+            self.cleosLimit = "--time-limit 99999"
             self.fetchHeadBlock = lambda node, headBlock: node.processUrllibRequest("chain", "get_block_info", {"block_num":headBlock}, silentErrors=False, exitOnError=True)
 
     def __str__(self):

tests/TestHarness/WalletMgr.py

@@ -12,9 +12,9 @@ from .testUtils import Utils
 Wallet=namedtuple("Wallet", "name password host port")
 # pylint: disable=too-many-instance-attributes
 class WalletMgr(object):
+    __walletLogOutFile=f"{Utils.TestLogRoot}/test_keosd_out.log"
+    __walletLogErrFile=f"{Utils.TestLogRoot}/test_keosd_err.log"
     __walletDataDir=f"{Utils.TestLogRoot}/test_wallet_0"
-    __walletLogOutFile=f"{__walletDataDir}/test_keosd_out.log"
-    __walletLogErrFile=f"{__walletDataDir}/test_keosd_err.log"
     __MaxPort=9999
 
     # pylint: disable=too-many-arguments
@@ -83,9 +83,6 @@ class WalletMgr(object):
         cmd="%s --data-dir %s --config-dir %s --unlock-timeout=999999 --http-server-address=%s:%d --http-max-response-time-ms 99999 --verbose-http-errors" % (
             Utils.EosWalletPath, WalletMgr.__walletDataDir, WalletMgr.__walletDataDir, self.host, self.port)
         if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
-        if not os.path.isdir(WalletMgr.__walletDataDir):
-            if Utils.Debug: Utils.Print(f"Creating dir {WalletMgr.__walletDataDir} in dir: {os.getcwd()}")
-            os.mkdir(WalletMgr.__walletDataDir)
         with open(WalletMgr.__walletLogOutFile, 'w') as sout, open(WalletMgr.__walletLogErrFile, 'w') as serr:
             popen=subprocess.Popen(cmd.split(), stdout=sout, stderr=serr)
             self.__walletPid=popen.pid
@@ -302,5 +299,6 @@ class WalletMgr(object):
 
     @staticmethod
     def cleanup():
-        if os.path.isdir(WalletMgr.__walletDataDir) and os.path.exists(WalletMgr.__walletDataDir):
+        dataDir=WalletMgr.__walletDataDir
+        if os.path.isdir(dataDir) and os.path.exists(dataDir):
             shutil.rmtree(WalletMgr.__walletDataDir)

tests/TestHarness/queries.py

@@ -596,18 +596,15 @@ class NodeosQueries:
 
         return trans
 
-    def processUrllibRequest(self, resource, command, payload={}, silentErrors=False, exitOnError=False, exitMsg=None, returnType=ReturnType.json, method="POST", endpoint=None):
+    def processUrllibRequest(self, resource, command, payload={}, silentErrors=False, exitOnError=False, exitMsg=None, returnType=ReturnType.json, endpoint=None):
         if not endpoint:
             endpoint = self.endpointHttp
         cmd = f"{endpoint}/v1/{resource}/{command}"
-        req = urllib.request.Request(cmd, method=method)
-        if len(payload):
-            req.add_header('Content-Type', 'application/json')
-            data = payload
-            data = json.dumps(data)
-            data = data.encode()
-        else:
-            data = None
+        req = urllib.request.Request(cmd, method="POST")
+        req.add_header('Content-Type', 'application/json')
+        data = payload
+        data = json.dumps(data)
+        data = data.encode()
         if Utils.Debug: Utils.Print("cmd: %s %s" % (cmd, payload))
         rtn=None
         start=time.perf_counter()
@@ -775,3 +772,4 @@ class NodeosQueries:
     def getActivatedProtocolFeatures(self):
         latestBlockHeaderState = self.getLatestBlockHeaderState()
         return latestBlockHeaderState["activated_protocol_features"]["protocol_features"]
+

tests/auto_bp_peering_test.py

@@ -35,7 +35,8 @@ args = TestHelper.parse_args({
 Utils.Debug = args.v
 killAll = args.clean_run
 dumpErrorDetails = args.dump_error_details
-dontKill = args.leave_running
+# dontKill = args.leave_running
+dontKill = True
 killEosInstances = not dontKill
 killWallet = not dontKill
 keepLogs = args.keep_logs

tests/cli_test.py

@@ -11,7 +11,7 @@ import time
 import shutil
 import signal
 
-from TestHarness import Account, Node, ReturnType, Utils, WalletMgr
+from TestHarness import Account, Cluster, Node, ReturnType, Utils, WalletMgr
 
 testSuccessful=False
 
@@ -352,7 +352,7 @@ def abi_file_with_nodeos_test():
         os.makedirs(data_dir, exist_ok=True)
         walletMgr = WalletMgr(True)
         walletMgr.launch()
-        node = Node('localhost', 8888, nodeId, cmd="./programs/nodeos/nodeos -e -p eosio --plugin eosio::trace_api_plugin --trace-no-abis --plugin eosio::producer_plugin --plugin eosio::producer_api_plugin --plugin eosio::chain_api_plugin --plugin eosio::chain_plugin --plugin eosio::http_plugin --access-control-allow-origin=* --http-validate-host=false --max-transaction-time=-1 --resource-monitor-not-shutdown-on-threshold-exceeded " + "--data-dir " + data_dir + " --config-dir " + data_dir, walletMgr=walletMgr)
+        node = Node('localhost', 8888, nodeId, cmd="./programs/nodeos/nodeos -e -p eosio --plugin eosio::trace_api_plugin --trace-no-abis --plugin eosio::producer_plugin --plugin eosio::producer_api_plugin --plugin eosio::chain_api_plugin --plugin eosio::chain_plugin --plugin eosio::http_plugin --access-control-allow-origin=* --http-validate-host=false --resource-monitor-not-shutdown-on-threshold-exceeded " + "--data-dir " + data_dir + " --config-dir " + data_dir, walletMgr=walletMgr)
         node.verifyAlive() # Setting node state to not alive
         node.relaunch(newChain=True, cachePopen=True)
         node.waitForBlock(1)
@@ -403,7 +403,7 @@ def abi_file_with_nodeos_test():
                     os.kill(node.pid, signal.SIGKILL)
         if testSuccessful:
             Utils.Print("Cleanup nodeos data.")
-            shutil.rmtree(Utils.DataPath)
+            shutil.rmtree(data_dir)
 
         if malicious_token_abi_path:
             if os.path.exists(malicious_token_abi_path):

tests/launcher.py

@@ -174,7 +174,7 @@ class launcher(object):
         parser.add_argument('-i', '--timestamp', help='set the timestamp for the first block.  Use "now" to indicate the current time')
         parser.add_argument('-l', '--launch', choices=['all', 'none', 'local'], help='select a subset of nodes to launch.  If not set, the default is to launch all unless an output file is named, in which case none are started.', default='all')
         parser.add_argument('-o', '--output', help='save a copy of the generated topology in this file and exit without launching', dest='topology_filename')
-        parser.add_argument('-k', '--kill', type=int, help='kill a network as specified in arguments with given signal')
+        parser.add_argument('-k', '--kill', help='retrieve the list of previously started process ids and issue a kill to each')
         parser.add_argument('--down', type=comma_separated, help='comma-separated list of node numbers that will be shut down', default=[])
         parser.add_argument('--bounce', type=comma_separated, help='comma-separated list of node numbers that will be restarted', default=[])
         parser.add_argument('--roll', type=comma_separated, help='comma-separated list of host names where the nodes will be rolled to a new version')
@@ -613,17 +613,6 @@ plugin = eosio::chain_api_plugin
                     if relaunch:
                         self.launch(node)
 
-    def kill(self, signum):
-        errorCode = 0
-        for node in self.network.nodes.values():
-            try:
-                with open(node.data_dir_name / f'{Utils.EosServerName}.pid', 'r') as f:
-                    pid = int(f.readline())
-                    self.terminate_wait_pid(pid, signum, raise_if_missing=False)
-            except FileNotFoundError as err:
-                errorCode = 1
-        return errorCode
-
     def start_all(self):
         if self.args.launch.lower() != 'none':
             for instance in self.network.nodes.values():
@@ -639,9 +628,8 @@ plugin = eosio::chain_api_plugin
                     f.write(f'"{node.dot_label}"->"{pname}" [dir="forward"];\n')
             f.write('}')
 
-    def terminate_wait_pid(self, pid, signum = signal.SIGTERM, raise_if_missing=True):
-        '''Terminate a non-child process with given signal number or with SIGTERM if not
-           provided and wait for it to exit.'''
+    def terminate_wait_pid(self, pid, raise_if_missing=True):
+        '''Terminate a non-child process with SIGTERM and wait for it to exit.'''
         if sys.version_info >= (3, 9) and platform.system() == 'Linux': # on our supported platforms, Python 3.9 accompanies a kernel > 5.3
             try:
                 fd = os.pidfd_open(pid)
@@ -652,7 +640,7 @@ plugin = eosio::chain_api_plugin
                 po = select.poll()
                 po.register(fd, select.POLLIN)
                 try:
-                    os.kill(pid, signum)
+                    os.kill(pid, signal.SIGTERM)
                 except ProcessLookupError:
                     if raise_if_missing:
                         raise
@@ -675,7 +663,7 @@ plugin = eosio::chain_api_plugin
                     return min(delay * 2, 0.04)
                 delay = 0.0001
                 try:
-                    os.kill(pid, signum)
+                    os.kill(pid, signal.SIGTERM)
                 except ProcessLookupError:
                     if raise_if_missing:
                         raise
@@ -688,16 +676,12 @@ plugin = eosio::chain_api_plugin
                         return
 
 if __name__ == '__main__':
-    errorCode = 0
     l = launcher(sys.argv[1:])
     if len(l.args.down):
         l.down(l.args.down)
     elif len(l.args.bounce):
         l.bounce(l.args.bounce)
-    elif l.args.kill:
-        errorCode = l.kill(l.args.kill)
     elif l.args.launch == 'all' or l.args.launch == 'local':
         l.start_all()
     for f in glob.glob(Utils.DataPath):
         shutil.rmtree(f)
-    sys.exit(errorCode)

tests/nodeos_contrl_c_test.py

@@ -115,7 +115,7 @@ try:
         errorExit("Failed to kill the seed node")
 
 finally:
-    TestHelper.shutdown(cluster, walletMgr, testSuccessful=testSuccessful, killEosInstances=True, killWallet=True, keepLogs=False, cleanRun=True, dumpErrorDetails=True)
+    TestHelper.shutdown(cluster, walletMgr, testSuccessful=testSuccessful, killEosInstances=True, killWallet=True, keepLogs=True, cleanRun=True, dumpErrorDetails=True)
 
 errorCode = 0 if testSuccessful else 1
 exit(errorCode)

tests/nodeos_snapshot_forked_test.py

@@ -171,7 +171,7 @@ try:
         errorExit(f"Failure - (non-production) node {nonProdNode.nodeNum} should have restarted")
 
     Print("Wait for LIB to move, which indicates prodC has forked out the branch")
-    assert prodC.waitForLibToAdvance(60), \
+    assert prodC.waitForLibToAdvance(), \
         "ERROR: Network did not reach consensus after bridge node was restarted."
  
     for prodNode in prodNodes:

tests/performance_tests/performance_test_basic.py

@@ -497,17 +497,15 @@ class PerformanceTestBasic:
             traceback.print_exc()
 
         finally:
-            # Despite keepLogs being hardcoded to False, logs will still appear on test failure in TestLogs
-            # due to testSuccessful being False
             TestHelper.shutdown(
-                cluster=self.cluster,
-                walletMgr=self.walletMgr,
-                testSuccessful=testSuccessful,
-                killEosInstances=self.testHelperConfig._killEosInstances,
-                killWallet=self.testHelperConfig._killWallet,
-                keepLogs=False,
-                cleanRun=self.testHelperConfig.killAll,
-                dumpErrorDetails=self.testHelperConfig.dumpErrorDetails
+                self.cluster,
+                self.walletMgr,
+                testSuccessful,
+                self.testHelperConfig._killEosInstances,
+                self.testHelperConfig._killWallet,
+                self.testHelperConfig.keepLogs,
+                self.testHelperConfig.killAll,
+                self.testHelperConfig.dumpErrorDetails
                 )
 
             if self.ptbConfig.delPerfLogs:

tests/plugin_http_api_test.py

@@ -40,8 +40,8 @@ class PluginHttpTest(unittest.TestCase):
         self.keosd.killall(True)
         WalletMgr.cleanup()
         Node.killAllNodeos()
-        if os.path.exists(Utils.DataPath):
-            shutil.rmtree(Utils.DataPath)
+        if os.path.exists(self.data_dir):
+            shutil.rmtree(self.data_dir)
         if os.path.exists(self.config_dir):
             shutil.rmtree(self.config_dir)
         time.sleep(self.sleep_s)
@@ -56,7 +56,7 @@ class PluginHttpTest(unittest.TestCase):
                         "http_plugin", "db_size_api_plugin", "prometheus_plugin"]
         nodeos_plugins = "--plugin eosio::" +  " --plugin eosio::".join(plugin_names)
         nodeos_flags = (" --data-dir=%s --config-dir=%s --trace-dir=%s --trace-no-abis --access-control-allow-origin=%s "
-                        "--contracts-console --http-validate-host=%s --verbose-http-errors --max-transaction-time -1 --abi-serializer-max-time-ms 30000 --http-max-response-time-ms 30000 "
+                        "--contracts-console --http-validate-host=%s --verbose-http-errors --abi-serializer-max-time-ms 30000 --http-max-response-time-ms 30000 "
                         "--p2p-peer-address localhost:9011 --resource-monitor-not-shutdown-on-threshold-exceeded ") % (self.data_dir, self.config_dir, self.data_dir, "\'*\'", "false")
         start_nodeos_cmd = ("%s -e -p eosio %s %s ") % (Utils.EosServerPath, nodeos_plugins, nodeos_flags)
         self.nodeos.launchCmd(start_nodeos_cmd, self.node_id)
@@ -1341,8 +1341,8 @@ class PluginHttpTest(unittest.TestCase):
     def test_prometheusApi(self) :
         resource = "prometheus"
         command = "metrics"
-        endpointPrometheus = f'http://{self.nodeos.host}:9101'
-        ret_text = self.nodeos.processUrllibRequest(resource, command, returnType = ReturnType.raw, method="GET", endpoint=endpointPrometheus).decode()
+
+        ret_text = self.nodeos.processUrllibRequest(resource, command, returnType = ReturnType.raw ).decode()
         # filter out all empty lines or lines starting with '#'
         data_lines = filter(lambda line: len(line) > 0 and line[0]!='#', ret_text.split('\n'))
         # converting each line into a key value pair and then construct a dictionay out of all the pairs

tests/read_only_trx_test.py

@@ -21,7 +21,6 @@ errorExit=Utils.errorExit
 
 appArgs=AppArgs()
 appArgs.add(flag="--read-only-threads", type=int, help="number of read-only threads", default=0)
-appArgs.add(flag="--num-test-runs", type=int, help="number of times to run the tests", default=1)
 appArgs.add_bool(flag="--eos-vm-oc-enable", help="enable eos-vm-oc")
 appArgs.add(flag="--wasm-runtime", type=str, help="if set to eos-vm-oc, must compile with EOSIO_EOS_VM_OC_DEVELOPER", default="eos-vm-jit")
 
@@ -45,7 +44,6 @@ dontKill=args.leave_running
 dumpErrorDetails=args.dump_error_details
 killAll=args.clean_run
 keepLogs=args.keep_logs
-numTestRuns=args.num_test_runs
 
 killWallet=not dontKill
 killEosInstances=not dontKill
@@ -93,8 +91,9 @@ try:
     specificExtraNodeosArgs={}
     # producer nodes will be mapped to 0 through pnodes-1, so the number pnodes is the no-producing API node
     specificExtraNodeosArgs[pnodes]=" --plugin eosio::net_api_plugin"
-    specificExtraNodeosArgs[pnodes]+=" --read-only-threads "
-    specificExtraNodeosArgs[pnodes]+=str(args.read_only_threads)
+    if args.read_only_threads > 0:
+        specificExtraNodeosArgs[pnodes]+=" --read-only-threads "
+        specificExtraNodeosArgs[pnodes]+=str(args.read_only_threads)
     if args.eos_vm_oc_enable:
         specificExtraNodeosArgs[pnodes]+=" --eos-vm-oc-enable"
     if args.wasm_runtime:
@@ -260,7 +259,7 @@ try:
         runReadOnlyTrxAndRpcInParallel("chain", "get_raw_code_and_abi", "account_name", expectedValue=testAccountName, payload = {"account_name":testAccountName})
         runReadOnlyTrxAndRpcInParallel("chain", "get_raw_abi", "account_name", expectedValue=testAccountName, payload = {"account_name":testAccountName})
         runReadOnlyTrxAndRpcInParallel("chain", "get_producers", "rows", payload = {"json":"true","lower_bound":""})
-        runReadOnlyTrxAndRpcInParallel("chain", "get_table_rows", "rows", payload = {"json":"true","code":"eosio","scope":"eosio","table":"global"})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_table_rows", code=500, payload = {"json":"true","table":"noauth"})
         runReadOnlyTrxAndRpcInParallel("chain", "get_table_by_scope", fieldIn="rows", payload = {"json":"true","table":"noauth"})
         runReadOnlyTrxAndRpcInParallel("chain", "get_currency_balance", code=200, payload = {"code":"eosio.token", "account":testAccountName})
         runReadOnlyTrxAndRpcInParallel("chain", "get_currency_stats", fieldIn="SYS", payload = {"code":"eosio.token", "symbol":"SYS"})
@@ -277,26 +276,13 @@ try:
         runReadOnlyTrxAndRpcInParallel("net", "connect", code=201, payload = "localhost")
         runReadOnlyTrxAndRpcInParallel("net", "disconnect", code=201, payload = "localhost")
 
-    def runEverythingParallel():
-        threadList = []
-        threadList.append(threading.Thread(target = multiReadOnlyTests))
-        threadList.append(threading.Thread(target = chainApiTests))
-        threadList.append(threading.Thread(target = netApiTests))
-        threadList.append(threading.Thread(target = mixedOpsTest))
-        for thr in threadList:
-            thr.start()
-        for thr in threadList:
-            thr.join()
-
     basicTests()
 
     if args.read_only_threads > 0: # Save test time. No need to run other tests if multi-threaded is not enabled
-        for i in range(numTestRuns):
-            multiReadOnlyTests()
-            chainApiTests()
-            netApiTests()
-            mixedOpsTest()
-            runEverythingParallel()
+        multiReadOnlyTests()
+        chainApiTests()
+        netApiTests()
+        mixedOpsTest()
 
     testSuccessful = True
 finally:

tests/ship_client.cpp

@@ -125,8 +125,6 @@ int main(int argc, char* argv[]) {
             eosio::check(result_doucment[1]["head"].IsObject(),                                 "'head' is not an object");
             eosio::check(result_doucment[1]["head"].HasMember("block_num"),                     "'head' does not contain 'block_num'");
             eosio::check(result_doucment[1]["head"]["block_num"].IsUint(),                      "'head.block_num' isn't a number");
-            eosio::check(result_doucment[1]["head"].HasMember("block_id"),                      "'head' does not contain 'block_id'");
-            eosio::check(result_doucment[1]["head"]["block_id"].IsString(),                     "'head.block_id' isn't a string");
 
             uint32_t this_block_num = result_doucment[1]["head"]["block_num"].GetUint();
 

tests/ship_streamer.cpp

@@ -111,61 +111,30 @@ int main(int argc, char* argv[]) {
       stream.write(boost::asio::buffer(request_type.json_to_bin(request_sb.GetString(), [](){})));
       stream.read_message_max(0);
 
-      //      block_num, block_id
-      std::map<uint32_t, std::string> block_ids;
       bool is_first = true;
       for(;;) {
          boost::beast::flat_buffer buffer;
          stream.read(buffer);
 
          eosio::input_stream is((const char*)buffer.data().data(), buffer.data().size());
-         rapidjson::Document result_document;
-         result_document.Parse(result_type.bin_to_json(is).c_str());
+         rapidjson::Document result_doucment;
+         result_doucment.Parse(result_type.bin_to_json(is).c_str());
 
-         eosio::check(!result_document.HasParseError(),                                      "Failed to parse result JSON from abieos");
-         eosio::check(result_document.IsArray(),                                             "result should have been an array (variant) but it's not");
-         eosio::check(result_document.Size() == 2,                                           "result was an array but did not contain 2 items like a variant should");
-         eosio::check(std::string(result_document[0].GetString()) == "get_blocks_result_v0", "result type doesn't look like get_blocks_result_v0");
-         eosio::check(result_document[1].IsObject(),                                         "second item in result array is not an object");
-         eosio::check(result_document[1].HasMember("head"),                                  "cannot find 'head' in result");
-         eosio::check(result_document[1]["head"].IsObject(),                                 "'head' is not an object");
-         eosio::check(result_document[1]["head"].HasMember("block_num"),                     "'head' does not contain 'block_num'");
-         eosio::check(result_document[1]["head"]["block_num"].IsUint(),                      "'head.block_num' isn't a number");
-         eosio::check(result_document[1]["head"].HasMember("block_id"),                      "'head' does not contain 'block_id'");
-         eosio::check(result_document[1]["head"]["block_id"].IsString(),                     "'head.block_id' isn't a string");
+         eosio::check(!result_doucment.HasParseError(),                                      "Failed to parse result JSON from abieos");
+         eosio::check(result_doucment.IsArray(),                                             "result should have been an array (variant) but it's not");
+         eosio::check(result_doucment.Size() == 2,                                           "result was an array but did not contain 2 items like a variant should");
+         eosio::check(std::string(result_doucment[0].GetString()) == "get_blocks_result_v0", "result type doesn't look like get_blocks_result_v0");
+         eosio::check(result_doucment[1].IsObject(),                                         "second item in result array is not an object");
+         eosio::check(result_doucment[1].HasMember("head"),                                  "cannot find 'head' in result");
+         eosio::check(result_doucment[1]["head"].IsObject(),                                 "'head' is not an object");
+         eosio::check(result_doucment[1]["head"].HasMember("block_num"),                     "'head' does not contain 'block_num'");
+         eosio::check(result_doucment[1]["head"]["block_num"].IsUint(),                      "'head.block_num' isn't a number");
 
          uint32_t this_block_num = 0;
-         if( result_document[1].HasMember("this_block") && result_document[1]["this_block"].IsObject() ) {
-            if( result_document[1]["this_block"].HasMember("block_num") && result_document[1]["this_block"]["block_num"].IsUint() ) {
-               this_block_num = result_document[1]["this_block"]["block_num"].GetUint();
+         if( result_doucment[1].HasMember("this_block") && result_doucment[1]["this_block"].IsObject() ) {
+            if( result_doucment[1]["this_block"].HasMember("block_num") && result_doucment[1]["this_block"]["block_num"].IsUint() ) {
+               this_block_num = result_doucment[1]["this_block"]["block_num"].GetUint();
             }
-            std::string this_block_id;
-            if( result_document[1]["this_block"].HasMember("block_id") && result_document[1]["this_block"]["block_id"].IsString() ) {
-               this_block_id = result_document[1]["this_block"]["block_id"].GetString();
-            }
-            std::string prev_block_id;
-            if( result_document[1]["prev_block"].HasMember("block_id") && result_document[1]["prev_block"]["block_id"].IsString() ) {
-               prev_block_id = result_document[1]["prev_block"]["block_id"].GetString();
-            }
-            if( !irreversible_only && !this_block_id.empty() && !prev_block_id.empty() ) {
-               // verify forks were sent
-               if (block_ids.count(this_block_num-1)) {
-                  if (block_ids[this_block_num-1] != prev_block_id) {
-                     std::cerr << "Received block: << " << this_block_num << " that does not link to previous: " << block_ids[this_block_num-1] << std::endl;
-                     return 1;
-                  }
-               }
-               block_ids[this_block_num] = this_block_id;
-
-               if( result_document[1]["last_irreversible"].HasMember("block_num") && result_document[1]["last_irreversible"]["block_num"].IsUint() ) {
-                  uint32_t lib_num = result_document[1]["last_irreversible"]["block_num"].GetUint();
-                  auto i = block_ids.lower_bound(lib_num);
-                  if (i != block_ids.end()) {
-                     block_ids.erase(block_ids.begin(), i);
-                  }
-               }
-            }
-
          }
 
          if(is_first) {
@@ -178,7 +147,7 @@ int main(int argc, char* argv[]) {
 
          rapidjson::StringBuffer result_sb;
          rapidjson::PrettyWriter<rapidjson::StringBuffer> result_writer(result_sb);
-         result_document[1].Accept(result_writer);
+         result_doucment[1].Accept(result_writer);
          std::cout << result_sb.GetString() << std::endl << "}" << std::endl;
 
          if( this_block_num == end_block_num ) break;

tests/ship_streamer_test.py

@@ -7,33 +7,34 @@ import shutil
 import signal
 import sys
 
-from TestHarness import Cluster, TestHelper, Utils, WalletMgr, CORE_SYMBOL
+from TestHarness import Cluster, TestHelper, Utils, WalletMgr
 from TestHarness.TestHelper import AppArgs
 
 ###############################################################
 # ship_streamer_test
 # 
-# This test sets up 2 producing nodes and one "bridge" node using test_control_api_plugin.
-#   One producing node has 3 of the elected producers and the other has 1 of the elected producers.
-#   All the producers are named in alphabetical order, so that the 3 producers, in the one production node, are
-#       scheduled first, followed by the 1 producer in the other producer node. Each producing node is only connected
-#       to the other producing node via the "bridge" node.
-#   The bridge node has the test_control_api_plugin, that the test uses to kill
-#       the "bridge" node to generate a fork.
-#   ship_streamer is used to connect to the state_history_plugin and verify that blocks receive link to previous
-#   blocks. If the blocks do not link then ship_streamer will exit with an error causing this test to generate an
-#   error. The fork generated by nodeos should be sent to the ship_streamer so it is able to correctly observe the
-#   fork.
+# This test sets up <-p> producing node(s) and <-n - -p>
+#   non-producing node(s). One of the non-producing nodes
+#   is configured with the state_history_plugin.  An instance
+#   of node will be started with ship_streamer to exercise
+#   the SHiP API.
 #
 ###############################################################
 
 Print=Utils.Print
 
 appArgs = AppArgs()
+extraArgs = appArgs.add(flag="--num-blocks", type=int, help="How many blocsk to stream from ship_streamer", default=20)
 extraArgs = appArgs.add(flag="--num-clients", type=int, help="How many ship_streamers should be started", default=1)
-args = TestHelper.parse_args({"--dump-error-details","--keep-logs","-v","--leave-running","--clean-run","--unshared"}, applicationSpecificArgs=appArgs)
+args = TestHelper.parse_args({"-p", "-n","--dump-error-details","--keep-logs","-v","--leave-running","--clean-run","--unshared"}, applicationSpecificArgs=appArgs)
 
 Utils.Debug=args.v
+totalProducerNodes=args.p
+totalNodes=args.n
+if totalNodes<=totalProducerNodes:
+    totalNodes=totalProducerNodes+1
+totalNonProducerNodes=totalNodes-totalProducerNodes
+totalProducers=totalProducerNodes
 cluster=Cluster(walletd=True,unshared=args.unshared)
 dumpErrorDetails=args.dump_error_details
 keepLogs=args.keep_logs
@@ -41,12 +42,6 @@ dontKill=args.leave_running
 killAll=args.clean_run
 walletPort=TestHelper.DEFAULT_WALLET_PORT
 
-totalProducerNodes=2
-totalNonProducerNodes=1
-totalNodes=totalProducerNodes+totalNonProducerNodes
-maxActiveProducers=21
-totalProducers=maxActiveProducers
-
 walletMgr=WalletMgr(True, port=walletPort)
 testSuccessful=False
 killEosInstances=not dontKill
@@ -62,20 +57,12 @@ try:
     cluster.killall(allInstances=killAll)
     cluster.cleanup()
     Print("Stand up cluster")
-
-
-    # ***   setup topogrophy   ***
-
-    # "bridge" shape connects defprocera through defproducerc (3 in node0) to each other and defproduceru (1 in node1)
-    # and the only connection between those 2 groups is through the bridge node
-
-    shipNodeNum = 1
     specificExtraNodeosArgs={}
-    specificExtraNodeosArgs[shipNodeNum]="--plugin eosio::state_history_plugin --disable-replay-opts --trace-history --chain-state-history --plugin eosio::net_api_plugin "
-    # producer nodes will be mapped to 0 through totalProducerNodes-1, so the number totalProducerNodes will be the non-producing node
-    specificExtraNodeosArgs[totalProducerNodes]="--plugin eosio::test_control_api_plugin  "
+    # non-producing nodes are at the end of the cluster's nodes, so reserving the last one for state_history_plugin
+    shipNodeNum = totalNodes - 1
+    specificExtraNodeosArgs[shipNodeNum]="--plugin eosio::state_history_plugin --disable-replay-opts --trace-history --sync-fetch-span 200 --plugin eosio::net_api_plugin "
 
-    if cluster.launch(topo="bridge", pnodes=totalProducerNodes,
+    if cluster.launch(pnodes=totalProducerNodes,
                       totalNodes=totalNodes, totalProducers=totalProducers,
                       specificExtraNodeosArgs=specificExtraNodeosArgs) is False:
         Utils.cmdError("launcher")
@@ -83,81 +70,19 @@ try:
 
     # ***   identify each node (producers and non-producing node)   ***
 
+    shipNode = cluster.getNode(shipNodeNum)
+    prodNode = cluster.getNode(0)
+
     #verify nodes are in sync and advancing
     cluster.waitOnClusterSync(blockAdvancing=5)
     Print("Cluster in Sync")
 
-    prodNode = cluster.getNode(0)
-    prodNode0 = prodNode
-    prodNode1 = cluster.getNode(1)
-    nonProdNode = cluster.getNode(2)
-    shipNode = cluster.getNode(shipNodeNum)
-
-
-    accounts=cluster.createAccountKeys(6)
-    if accounts is None:
-        Utils.errorExit("FAILURE - create keys")
-
-    accounts[0].name="testeraaaaaa"
-    accounts[1].name="tester111111" # needed for voting
-    accounts[2].name="tester222222" # needed for voting
-    accounts[3].name="tester333333" # needed for voting
-    accounts[4].name="tester444444" # needed for voting
-    accounts[5].name="tester555555" # needed for voting
-
-    testWalletName="test"
-
-    Print(f"Creating wallet {testWalletName}.")
-    testWallet=walletMgr.create(testWalletName, [cluster.eosioAccount,accounts[0],accounts[1],accounts[2],accounts[3],accounts[4],accounts[5]])
-
-    for _, account in cluster.defProducerAccounts.items():
-        walletMgr.importKey(account, testWallet, ignoreDupKeyWarning=True)
-
-    for i in range(0, totalNodes):
-        node=cluster.getNode(i)
-        node.producers=Cluster.parseProducers(i)
-        for prod in node.producers:
-            prodName = cluster.defProducerAccounts[prod].name
-            if prodName == "defproducera" or prodName == "defproducerb" or prodName == "defproducerc" or prodName == "defproduceru":
-                Print(f"Register producer {prodName}")
-                trans=node.regproducer(cluster.defProducerAccounts[prod], "http://mysite.com", 0, waitForTransBlock=False, exitOnError=True)
-
-    # create accounts via eosio as otherwise a bid is needed
-    for account in accounts:
-        Print(f"Create new account {account.name} via {cluster.eosioAccount.name} with private key: {account.activePrivateKey}")
-        trans=nonProdNode.createInitializeAccount(account, cluster.eosioAccount, stakedDeposit=0, waitForTransBlock=True, stakeNet=10000, stakeCPU=10000, buyRAM=10000000, exitOnError=True)
-        transferAmount="100000000.0000 {0}".format(CORE_SYMBOL)
-        Print(f"Transfer funds {transferAmount} from account {cluster.eosioAccount.name} to {account.name}")
-        nonProdNode.transferFunds(cluster.eosioAccount, account, transferAmount, "test transfer", waitForTransBlock=True)
-        trans=nonProdNode.delegatebw(account, 20000000.0000, 20000000.0000, waitForTransBlock=False, exitOnError=True)
-
-    # ***   vote using accounts   ***
-
-    cluster.waitOnClusterSync(blockAdvancing=3)
     start_block_num = shipNode.getBlockNum()
-
-    # vote a,b,c (node0)  u (node1)
-    voteProducers=[]
-    voteProducers.append("defproducera")
-    voteProducers.append("defproducerb")
-    voteProducers.append("defproducerc")
-    voteProducers.append("defproduceru")
-    for account in accounts:
-        Print(f"Account {account.name} vote for producers={voteProducers}")
-        trans=prodNode.vote(account, voteProducers, exitOnError=True, waitForTransBlock=False)
-
-    #verify nodes are in sync and advancing
-    cluster.waitOnClusterSync(blockAdvancing=3)
-    Print("Shutdown unneeded bios node")
-    cluster.biosNode.kill(signal.SIGTERM)
-    prodNode0.waitForProducer("defproducerc")
-
-    block_range = 350
-    end_block_num = start_block_num + block_range
+    end_block_num = start_block_num + args.num_blocks
 
     shipClient = "tests/ship_streamer"
-    cmd = f"{shipClient} --start-block-num {start_block_num} --end-block-num {end_block_num} --fetch-block --fetch-traces --fetch-deltas"
-    if Utils.Debug: Utils.Print(f"cmd: {cmd}")
+    cmd = "%s --start-block-num %d --end-block-num %d --fetch-block --fetch-traces --fetch-deltas" % (shipClient, start_block_num, end_block_num)
+    if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
     clients = []
     files = []
     shipTempDir = os.path.join(Utils.DataDir, "ship")
@@ -167,61 +92,28 @@ try:
     starts = []
     for i in range(0, args.num_clients):
         start = time.perf_counter()
-        outFile = open(f"{shipClientFilePrefix}{i}.out", "w")
-        errFile = open(f"{shipClientFilePrefix}{i}.err", "w")
-        Print(f"Start client {i}")
+        outFile = open("%s%d.out" % (shipClientFilePrefix, i), "w")
+        errFile = open("%s%d.err" % (shipClientFilePrefix, i), "w")
+        Print("Start client %d" % (i))
         popen=Utils.delayedCheckOutput(cmd, stdout=outFile, stderr=errFile)
         starts.append(time.perf_counter())
         clients.append((popen, cmd))
         files.append((outFile, errFile))
-        Print(f"Client {i} started, Ship node head is: {shipNode.getBlockNum()}")
+        Print("Client %d started, Ship node head is: %s" % (i, shipNode.getBlockNum()))
 
-    # Generate a fork
-    forkAtProducer="defproducera"
-    prodNode1Prod="defproduceru"
-    preKillBlockNum=nonProdNode.getBlockNum()
-    preKillBlockProducer=nonProdNode.getBlockProducerByNum(preKillBlockNum)
-    nonProdNode.killNodeOnProducer(producer=forkAtProducer, whereInSequence=1)
-    Print(f"Current block producer {preKillBlockProducer} fork will be at producer {forkAtProducer}")
-    prodNode0.waitForProducer(forkAtProducer)
-    prodNode1.waitForProducer(prodNode1Prod)
-    if nonProdNode.verifyAlive(): # if on defproducera, need to wait again
-        prodNode0.waitForProducer(forkAtProducer)
-        prodNode1.waitForProducer(prodNode1Prod)
-    if nonProdNode.verifyAlive():
-        Utils.errorExit("Bridge did not shutdown");
-    Print("Fork started")
+    Print("Stopping all %d clients" % (args.num_clients))
 
-    prodNode0.waitForProducer("defproducerb") # wait for fork to progress a bit
-
-    Print("Restore fork")
-    Print("Relaunching the non-producing bridge node to connect the producing nodes again")
-    if nonProdNode.verifyAlive():
-        Utils.errorExit("Bridge is already running");
-    if not nonProdNode.relaunch():
-        Utils.errorExit(f"Failure - (non-production) node {nonProdNode.nodeNum} should have restarted")
-
-    nonProdNode.waitForProducer(forkAtProducer)
-    nonProdNode.waitForProducer(prodNode1Prod)
-    afterForkBlockNum = nonProdNode.getBlockNum()
-    if int(afterForkBlockNum) > int(end_block_num):
-        Utils.errorExit(f"Did not stream long enough {end_block_num} to cover the fork {afterForkBlockNum}, increase block_range {block_range}")
-
-    Print(f"Stopping all {args.num_clients} clients")
     for index, (popen, _), (out, err), start in zip(range(len(clients)), clients, files, starts):
         popen.wait()
-        Print(f"Stopped client {index}.  Ran for {time.perf_counter() - start:.3f} seconds.")
+        Print("Stopped client %d.  Ran for %.3f seconds." % (index, time.perf_counter() - start))
         out.close()
         err.close()
-        outFile = open(f"{shipClientFilePrefix}{index}.out", "r")
+        outFile = open("%s%d.out" % (shipClientFilePrefix, index), "r")
         data = json.load(outFile)
         block_num = start_block_num
         for i in data:
-            # fork can cause block numbers to be repeated
-            this_block_num = i['get_blocks_result_v0']['this_block']['block_num']
-            if this_block_num < block_num:
-                block_num = this_block_num
-            assert block_num == this_block_num, f"{block_num} != {this_block_num}"
+            print(i)
+            assert block_num == i['get_blocks_result_v0']['this_block']['block_num'], f"{block_num} != {i['get_blocks_result_v0']['this_block']['block_num']}"
             assert isinstance(i['get_blocks_result_v0']['block'], str) # verify block in result
             block_num += 1
         assert block_num-1 == end_block_num, f"{block_num-1} != {end_block_num}"

tests/ship_test.py

@@ -132,7 +132,7 @@ try:
             try:
                 statuses = json.loads(" ".join(lines))
             except json.decoder.JSONDecodeError as er:
-                Utils.errorExit("ship_client output was malformed in %s. Exception: %s" % (shipClientErrorFile, er))
+                Utils.errorExit("javascript client output was malformed in %s. Exception: %s" % (shipClientErrorFile, er))
 
             for status in statuses:
                 statusDesc = status["status"]
@@ -143,7 +143,7 @@ try:
                     maxFirstBN = max(maxFirstBN, firstBlockNum)
                     minLastBN = min(minLastBN, lastBlockNum)
                 if statusDesc == "error":
-                    Utils.errorExit("ship_client reporting error see: %s." % (shipClientErrorFile))
+                    Utils.errorExit("javascript client reporting error see: %s." % (shipClientErrorFile))
 
         assert done, Print("ERROR: Did not find a \"done\" status for client %d" % (i))
 

tests/trace_plugin_test.py

@@ -8,8 +8,6 @@ import os
 
 from TestHarness import Cluster, Node, TestHelper, Utils, WalletMgr, CORE_SYMBOL
 
-testSuccessful = True
-
 class TraceApiPluginTest(unittest.TestCase):
     sleep_s = 1
     cluster=Cluster(walletd=True, defproduceraPrvtKey=None)
@@ -103,8 +101,6 @@ class TraceApiPluginTest(unittest.TestCase):
                     self.assertIn('memo', prms)
                 break
         self.assertTrue(isTrxInBlockFromTraceApi)
-        global testSuccessful
-        testSuccessful = True
 
     @classmethod
     def setUpClass(self):
@@ -113,7 +109,7 @@ class TraceApiPluginTest(unittest.TestCase):
 
     @classmethod
     def tearDownClass(self):
-        self.cleanEnv(self, shouldCleanup=testSuccessful)
+        self.cleanEnv(self, shouldCleanup=False)   # not cleanup to save log in case for further investigation
 
 if __name__ == "__main__":
     unittest.main()

tests/trx_finality_status_forked_test.py

@@ -184,7 +184,7 @@ try:
         errorExit(f"Failure - (non-production) node {nonProdNode.nodeNum} should have restarted")
 
     Print("Wait for LIB to move, which indicates prodC has forked out the branch")
-    assert prodC.waitForLibToAdvance(60), \
+    assert prodC.waitForLibToAdvance(), \
         "ERROR: Network did not reach consensus after bridge node was restarted."
 
     retStatus = prodC.getTransactionStatus(transId)

unittests/restart_chain_tests.cpp

@@ -237,4 +237,19 @@ BOOST_AUTO_TEST_CASE(test_light_validation_restart_from_block_log) {
    BOOST_CHECK_EQUAL(trace->action_traces.at(1).receipt->digest(), other_trace->action_traces.at(1).receipt->digest());
 }
 
+namespace{
+   struct scoped_temp_path {
+      boost::filesystem::path path;
+      scoped_temp_path() {
+         path = boost::filesystem::unique_path();
+         if (boost::unit_test::framework::master_test_suite().argc >= 2) {
+            path += boost::unit_test::framework::master_test_suite().argv[1];
+         }
+      }
+      ~scoped_temp_path() {
+         boost::filesystem::remove_all(path);
+      }
+   };
+}
+
 BOOST_AUTO_TEST_SUITE_END()

unittests/snapshot_tests.cpp

@@ -412,8 +412,8 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(test_compatible_versions, SNAPSHOT_SUITE, snapshot
       bfs::copy_file(source_i, source_log_dir / "blocks.index", bfs::copy_option::overwrite_if_exists);
       chain.close();
    }
-   fc::temp_directory temp_dir;
-   auto config = tester::default_config(temp_dir, legacy_default_max_inline_action_size).first;
+
+   auto config = tester::default_config(fc::temp_directory(), legacy_default_max_inline_action_size).first;
    auto genesis = eosio::chain::block_log::extract_genesis_state(source_log_dir);
    bfs::create_directories(config.blocks_dir);
    bfs::copy(source_log_dir / "blocks.log", config.blocks_dir / "blocks.log");
@@ -477,8 +477,7 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(test_pending_schedule_snapshot, SNAPSHOT_SUITE, sn
    source_log_dir_str += "prod_sched";
    const auto source_log_dir = bfs::path(source_log_dir_str.c_str());
    const uint32_t legacy_default_max_inline_action_size = 4 * 1024;
-   fc::temp_directory temp_dir;
-   auto config = tester::default_config(temp_dir, legacy_default_max_inline_action_size).first;
+   auto config = tester::default_config(fc::temp_directory(), legacy_default_max_inline_action_size).first;
    auto genesis = eosio::chain::block_log::extract_genesis_state(source_log_dir);
    bfs::create_directories(config.blocks_dir);
    bfs::copy(source_log_dir / "blocks.log", config.blocks_dir / "blocks.log");